brianr/SM2Gen
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

check file changed and generate as .new if changed after creation

Browse Source
This commit is contained in:
Brian Read 2024-09-18 11:46:26 +01:00
parent 1811a52dec
commit b4a6be435c
10 changed files with 627 additions and 511 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -2,4 +2,5 @@ bin/
lib/
lib64
pyvenv.cfg
*.new

7
Targets/CreateStarterWebsite/CreateStarterWebsite-Custom.pm
View File

@ -1,10 +1,11 @@
#
# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 08:37:59
#
#
# Routines to be edited by the developer to provide validation for parameters
# and provison of the control data for table(s)
#
# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-13 19:20
#
##
use esmith::util;
use esmith::HostsDB;
use esmith::AccountsDB;

6
Targets/CreateStarterWebsite/CreateStarterWebsite-Custom.pm.new
View File

@ -1,10 +1,10 @@
#
# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 11:44:06
#
#
# Routines to be edited by the developer to provide validation for parameters
# and provison of the control data for table(s)
#
# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-16 12:31:58
#
use esmith::util;
use esmith::HostsDB;
use esmith::AccountsDB;

2
Targets/CreateStarterWebsite/CreateStarterWebsite.pm
View File

@ -1,6 +1,6 @@
package SrvMngr::Controller::CreateStarterWebsite;
#
# Generated by version:SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-16 12:31:58
# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 08:38:32
#
#----------------------------------------------------------------------
# heading : Miscellaneous

2
Targets/CreateStarterWebsite/_csw_PARAMS.html.ep
View File

@ -1,5 +1,5 @@
%#
%# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-16 12:31:58
%# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 09:09:54
%#
<div id="CreateStarterWebsite-PARAMS" class="partial CreateStarterWebsite-PARAMS">
<script>

6
Targets/CreateStarterWebsite/createstarterwebsite.html.ep
View File

@ -1,7 +1,7 @@
%#
%# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 08:38:32
%#
% layout 'default', title => "Sme server 2 - Create Starter Website", share_dir => './';
%#
%# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-16 12:31:58
%#
% content_for 'module' => begin
<div id="module" class="module CreateStarterWebsite-panel">

31
Targets/CreateStarterWebsite/createstarterwebsite_en.lex
View File

@ -1,18 +1,21 @@
'csw_First_header,_typically_used_for' => 'First header, Typically used for short phrases such as Leader in the field of textile manufacturing',
'csw_Text_following_first_header,_typically' => 'Text following first header, Typically used for a paragraph of marketing information. ',
'csw_Manage_CreateStarterWebsite_settings:' => 'Manage CreateStarterWebsite settings:',
'csw_Create' => 'Create',
'csw_Create_Starter_Website' => 'Create Starter Website',
#
# Generated by SM2Gen version: SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 09:28:01
#
'csw_Create_a_starter_website' => 'Create a starter website',
'csw_You_can_leave_any_field' => 'You can leave any field blank if you do not need it. ',
'csw_First_header,_typically_used_for' => 'First header, Typically used for short phrases such as Leader in the field of textile manufacturing',
'csw_Second_header,_typically_used_for' => 'Second header, Typically used for short phrases such as For more information or To order our products:',
'csw_Manage_CreateStarterWebsite_settings:' => 'Manage CreateStarterWebsite settings:',
'csw_' => '',
'csw_Text_following_second_header,_typically' => 'Text following second header, Typically used for contact or ordering information:',
'csw_To_create_a_simple_web' => 'To create a simple web page for your company, Fill in the fields below and click onCreate. ',
'csw_Do_you_wish_to_proceed?' => 'Do you wish to proceed?',
'csw_The_text_that_you_enter' => 'The text that you enter below will be line wrapped for a nicer appearance in your web page. Leave a blank line whenever you want to start a new paragraph. If you need to force a line break without starting a new paragraph (for example after each line of a mailing address), Then type the four-character sequence',
'csw_When_you_create_this_web' => 'When you create this web page, The file index. Htm will be overwritten in your web site directory. ',
'csw_Create' => 'Create',
'csw_Do_not_use_this_optionif' => 'Do not use this optionif you have already customized your web site, Since it will overwrite the index. Htm file in your web site directory. ',
'csw_Create_Starter_Website' => 'Create Starter Website',
'csw_Hello_PARAMS' => 'Hello PARAMS',
'csw_APPLY' => 'Apply',
'csw_Text_following_second_header,_typically' => 'Text following second header, Typically used for contact or ordering information:',
'csw_When_you_create_this_web' => 'When you create this web page, The file index. Htm will be overwritten in your web site directory. ',
'csw_' => '',
'csw_Do_you_wish_to_proceed?' => 'Do you wish to proceed?',
'csw_Text_following_first_header,_typically' => 'Text following first header, Typically used for a paragraph of marketing information. ',
'csw_PARAMS_panel_action_was_successful' => 'PARAMS panel action was successful',
'csw_To_create_a_simple_web' => 'To create a simple web page for your company, Fill in the fields below and click onCreate. ',
'csw_You_can_leave_any_field' => 'You can leave any field blank if you do not need it. ',
'csw_Second_header,_typically_used_for' => 'Second header, Typically used for short phrases such as For more information or To order our products:',
'csw_Do_not_use_this_optionif' => 'Do not use this optionif you have already customized your web site, Since it will overwrite the index. Htm file in your web site directory. ',
'csw_The_text_that_you_enter' => 'The text that you enter below will be line wrapped for a nicer appearance in your web page. Leave a blank line whenever you want to start a new paragraph. If you need to force a line break without starting a new paragraph (for example after each line of a mailing address), Then type the four-character sequence',

3
json5/CreateStarterWebsite.json5
View File

@ -1,3 +1,6 @@
//
// Generated by sm1-html-2-json5 version:0.5 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 11:39:12
//
{
'PackageName': 'CreateStarterWebsite',
'prefix': 'csw',

111
sm1-html-2-json5.py
View File

@ -1,10 +1,85 @@
import json
import os
import re
from bs4 import BeautifulSoup
from lxml import etree # Import lxml for HTML validation
import html
import argparse
import pkg_resources
import sys
import traceback
import os
from datetime import datetime, timedelta
sm1_html_2_json5_version = "0.5"
def assemble_version_string():
try:
chameleon_version = pkg_resources.get_distribution("Chameleon").version
except pkg_resources.DistributionNotFound:
chameleon_version = "No version information"
python_version = sys.version
version_pattern = r"(\d{1,3}\.\d{1,3}\.\d{1,3})"
version_match = re.search(version_pattern, python_version)
python_version = version_match.group(0) if version_match else "Unknown"
current_datetime = datetime.now()
formatted_datetime = current_datetime.strftime("%Y-%m-%d %H:%M:%S")
strVersion = (
"sm1-html-2-json5 version:"
+ sm1_html_2_json5_version
+ " Chameleon version:"
+ chameleon_version
+ " On Python:"
+ python_version
+ " at "
+ formatted_datetime
)
return strVersion
def check_file_version(filename, ThresholdSecs=3):
#
# Check modified versus creation date of the file and return +".new" if modified since creation + ThresholdSecs
#
try:
with open(filename, 'r') as file:
# Read the first three lines
header_lines = [file.readline().strip() for _ in range(5)]
# Extract the timestamp
timestamp_str = None
for line in header_lines:
if ' at ' in line:
# Split at 'at', expect the timestamp to be in the third part
print(line)
timestamp_str = line.split('at')[2].strip()
break
if timestamp_str is None:
print("Warning: No timestamp found. Returning original filename.")
return filename # Return the original filename if no timestamp is found
# Convert the string timestamp to a datetime object
file_timestamp = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S')
# Add the threshold seconds to the creation date
file_timestamp += timedelta(seconds=ThresholdSecs)
# Get the last modified time of the file, ignoring milliseconds
file_modified_time = datetime.fromtimestamp(os.path.getmtime(filename)).replace(microsecond=0)
print(file_modified_time,file_timestamp)
# Compare the timestamps
if file_modified_time > file_timestamp:
return f"{filename}.new"
else:
return filename
except FileNotFoundError:
print(f"Error: The file '{filename}' does not exist.")
return filename
except Exception as e:
print(f"An error occurred: {traceback.format_exc()}")
return filename
def read_html_file(filename):
@ -21,6 +96,7 @@ def validate_html(html):
except Exception as e:
raise ValueError("Invalid HTML document") from e
def convert_double_quotes_to_span(text):
"""Convert single-quoted text to <span>...</span>."""
# Use a regular expression to find single-quoted text and replace it
@ -49,26 +125,30 @@ def convert_double_quotes_to_span(text):
# print(f"++{sanitized_text}++")
# return sanitized_text
def sanitize_text(text):
# Replace newlines with spaces
print(f"--{text}--")
# Take out html entities
decoded_text = html.unescape(text)
# Take out newlines
sanitized_text = decoded_text.replace('\n', ' ').replace('\r', ' ') # Handle both Unix and Windows line endings
sanitized_text = decoded_text.replace("\n", " ").replace(
"\r", " "
) # Handle both Unix and Windows line endings
# Replace tabs with spaces
sanitized_text = sanitized_text.replace('\t', ' ')
sanitized_text = sanitized_text.replace("\t", " ")
# Replace quote characters
sanitized_text = sanitized_text.replace('"', '').replace("'", '') # Remove double and single quotes
sanitized_text = sanitized_text.replace('"', "").replace(
"'", ""
) # Remove double and single quotes
# Take out any multiple spaces - reduce to one.
sanitized_text = ' '.join(sanitized_text.split())
sanitized_text = " ".join(sanitized_text.split())
# Strip leading and trailing whitespace
sanitized_text = sanitized_text.strip()
print(f"++{sanitized_text}++")
return sanitized_text
def extract_data(html):
"""Extract paragraphs, inputs, tables, and pre blocks from HTML and organize them in order."""
soup = BeautifulSoup(html, "lxml")
@ -207,7 +287,7 @@ def insert_spaces_before_caps(text):
return re.sub(r"(?<!^)(?=[A-Z])", " ", text)
def save_to_json5(data, output_filename, package_name, header, sub_header):
def save_to_json5(data, output_filename, package_name, header, sub_header,strVersion):
"""Save extracted data to a JSON5 file with a specific structure."""
# Generate prefix from uppercase letters in PackageName made into lowercase
prefix = "".join(re.findall(r"[A-Z]", package_name)).lower()
@ -270,7 +350,8 @@ def save_to_json5(data, output_filename, package_name, header, sub_header):
"MenuNavigation": "2000 400",
"firstPanel": "PARAMS",
"signalEvent": f"smeserver-{package_name.lower()}-update",
"html": [{
"html": [
{
"Name": "params",
"route": "PARAMS",
"Header": header if header else f"{package_name} Contrib",
@ -280,16 +361,19 @@ def save_to_json5(data, output_filename, package_name, header, sub_header):
**{
k: v for item in structured_html for k, v in item.items()
}, # Flatten the structured_html into the dict
}],
}
],
}
# Save in JSON5 format (JSON with comments and unquoted keys)
with open(output_filename, "w", encoding="utf-8") as json_file:
json.dump(json5_data, json_file, ensure_ascii=False, indent=4)
# Manually format as JSON5 by adding single quotes (for simplicity)
with open(output_filename, "r+", encoding="utf-8") as json_file:
content = json_file.read()
content = f"//\n// Generated by {strVersion}\n//\n"
content = content + json_file.read()
content = content.replace(
'"', "'"
) # Replace double quotes with single quotes for JSON5
@ -299,6 +383,7 @@ def save_to_json5(data, output_filename, package_name, header, sub_header):
def main():
strVersion = assemble_version_string()
# command line parameters
parser = argparse.ArgumentParser(description="sm1--html-2-jsopn5")
parser.add_argument(
@ -329,11 +414,11 @@ def main():
directory, filename = os.path.split(input_file)
# Replace 'html' with 'json5' in the directory path
new_directory = directory.replace('/html', '/json5')
new_directory = directory.replace("/html", "/json5")
# print(new_directory)
# Construct the new path
output_file = os.path.join(new_directory, filename.replace('.html', '.json5'))
output_file = check_file_version(os.path.join(new_directory, filename.replace(".html", ".json5")))
print(output_file)
# quit(1)
@ -342,7 +427,7 @@ def main():
package_name = os.path.splitext(base_name)[0] # Use the filename without extension
# Save extracted data to JSON5
save_to_json5(data, output_file, package_name, header, sub_header)
save_to_json5(data, output_file, package_name, header, sub_header, strVersion)
print(f"Extracted data saved to '{output_file}'.")

119
sm2gen.py
View File

@ -5,12 +5,14 @@ from chameleon import PageTemplateFile, PageTemplate
import pkg_resources
import xml.etree.ElementTree as ET
import re
import os
from datetime import datetime
from openai import OpenAI
import configparser
import json
from pathlib import Path
import traceback
import os
from datetime import datetime, timedelta
#
@ -26,6 +28,29 @@ json5_html_list: list = []
ini_file_path = os.path.expanduser("~/.smegit/conf")
OPENAI_API_KEY = ""
def assemble_version_string():
try:
chameleon_version = pkg_resources.get_distribution("Chameleon").version
except pkg_resources.DistributionNotFound:
chameleon_version = "No version information"
python_version = sys.version
version_pattern = r"(\d{1,3}\.\d{1,3}\.\d{1,3})"
version_match = re.search(version_pattern, python_version)
python_version = version_match.group(0) if version_match else "Unknown"
current_datetime = datetime.now()
formatted_datetime = current_datetime.strftime("%Y-%m-%d %H:%M:%S")
strVersion = (
"SM2Gen version:"
+ SME2Gen_version
+ " Chameleon version:"
+ chameleon_version
+ " On Python:"
+ python_version
+ " at "
+ formatted_datetime
)
return strVersion
def parse_json(json_obj, prefix=""):
structured_list = []
@ -293,42 +318,54 @@ def get_translation(message="Hello", language="french"):
quit()
return translated_message
def check_file_version(filename):
import os
from datetime import datetime, timedelta
def check_file_version(filename, ThresholdSecs=3):
#
# check modified versusu creation date of file and return +".new" if modified since creation
# Check modified versus creation date of the file and return +".new" if modified since creation + ThresholdSecs
#
try:
with open(filename, 'r') as file:
# Read the first three lines
header_lines = [file.readline().strip() for _ in range(3)]
header_lines = [file.readline().strip() for _ in range(5)]
# Extract the timestamp
timestamp_str = None
for line in header_lines:
if ' at ' in line:
# Split at 'at', expect the timestamp to be in the third part
print(line)
timestamp_str = line.split('at')[2].strip()
break
else:
raise ValueError("Invalid file format: no timestamp found.")
if timestamp_str is None:
print("Warning: No timestamp found. Returning original filename.")
return filename # Return the original filename if no timestamp is found
# Convert the string timestamp to a datetime object
file_timestamp = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S')
# Add the threshold seconds to the creation date
file_timestamp += timedelta(seconds=ThresholdSecs)
# Get the last modified time of the file, ignoring milliseconds
file_modified_time = datetime.fromtimestamp(os.path.getmtime(filename)).replace(microsecond=0)
print(file_modified_time,file_timestamp)
# Compare the timestamps
if file_modified_time > (file_timestamp):
if file_modified_time > file_timestamp:
return f"{filename}.new"
else:
return filename
except FileNotFoundError:
print(f"Error: The file '{filename}' does not exist.")
return None
return filename
except Exception as e:
print(f"An error occurred: {e}")
return None
print(f"An error occurred: {traceback.format_exc()}")
return filename
def convert_lex_to_dict(pairs_string):
@ -343,27 +380,8 @@ def convert_lex_to_dict(pairs_string):
if __name__ == "__main__":
try:
chameleon_version = pkg_resources.get_distribution("Chameleon").version
except pkg_resources.DistributionNotFound:
chameleon_version = "Version information not available"
python_version = sys.version
version_pattern = r"(\d{1,3}\.\d{1,3}\.\d{1,3})"
version_match = re.search(version_pattern, python_version)
python_version = version_match.group(0) if version_match else "Unknown"
current_datetime = datetime.now()
formatted_datetime = current_datetime.strftime("%Y-%m-%d %H:%M:%S")
strVersion = (
"SM2Gen version:"
+ SME2Gen_version
+ " Chameleon version:"
+ chameleon_version
+ " On Python:"
+ python_version
+ " at "
+ formatted_datetime
)
strVersion = assemble_version_string()
json5_dict: dict = {}
json5_html_list: list = []
@ -434,7 +452,11 @@ if __name__ == "__main__":
lint_json5(json_filename)
# Get dict of it all
try:
json5_dict = json5_to_dict(json_filename)
except Exception as e:
print(f"json5 file {json_filename} failed lint test (e)")
quit(1)
# Get dict of just the html bit
json5_html_list = json5_dict["html"]
@ -454,19 +476,18 @@ if __name__ == "__main__":
directory_path.mkdir(parents=True, exist_ok=True)
target_directory_path = "Targets/" + hl("PackageName") + "/"
controller_file = target_directory_path + hl("PackageName") + ".pm"
custom_controller_file = target_directory_path + hl("PackageName") + "-Custom.pm"
# Call it .new if one is already there (and may have been editted by the developer)
if os.path.exists(custom_controller_file):
custom_controller_file = custom_controller_file + ".new"
layout_file = target_directory_path + hl("PackageName").lower() + ".html.ep"
controller_file = check_file_version(target_directory_path + hl("PackageName") + ".pm")
custom_controller_file = check_file_version(target_directory_path + hl("PackageName") + "-Custom.pm",3)
print(custom_controller_file)
layout_file = check_file_version(target_directory_path + hl("PackageName").lower() + ".html.ep")
partial_files = list()
for panel in routes:
partial_files.append(
target_directory_path + '_' + hl("prefix") + "_" + panel + ".html.ep"
partial_files.append(check_file_version(
target_directory_path + '_' + hl("prefix") + "_" + panel + ".html.ep")
)
print(f"Partial files to be created:{partial_files}")
lex_file = target_directory_path + hl("PackageName").lower() + "_en.lex"
lex_file = check_file_version(target_directory_path + hl("PackageName").lower() + "_en.lex")
print(lex_file)
tablecontrols = (
get_table_control_data()
) # arrays of hashes used to drive rows in tables
@ -657,6 +678,7 @@ if __name__ == "__main__":
lex_all += f"'{lex_str['left']}' => '{lex_str['right']}',\n"
print(f"Writing {lex_file}")
with open(lex_file, "w") as file:
file.write(f"#\n# Generated by SM2Gen version: {strVersion}\n#\n")
file.write(lex_all)
# and then play the strings back into the partials and the layout file
print("..and feed the lex string names back into other files")
@ -679,7 +701,7 @@ if __name__ == "__main__":
# Now generate all the translated lex files from a list of the languages and codes
# if specifically requested
if not args.noLang:
if args.noLang:
languages_path = "Templates/languages.json"
with open(languages_path, "r") as file:
languages_str = file.read()
@ -690,11 +712,11 @@ if __name__ == "__main__":
for lang_item in lang_dict:
print(f"Translating from english lex file to {lang_item['language']}")
code = lang_item["code"]
translated_lex_file = (
translated_lex_file = check_file_version(
f"{target_directory_path}{hl('PackageName').lower()}_{code}.lex"
)
# Only do it if the lex file is missing
if not os.path.exists(translated_lex_file):
# Only do it if the lex file is missing - Removed bjr 18Sept2024
#if not os.path.exists(translated_lex_file):
translated_dict = []
for lex_item in eng_lex_dict:
# Get it from ChatGPT
@ -706,6 +728,7 @@ if __name__ == "__main__":
)
print(f"Writing out lex file for {lang_item['code']}")
with open(translated_lex_file, "w") as file:
file.write(f"#\n# Generated by SM2Gen version: {strVersion}\n#\n")
for item in translated_dict:
# escape any nasties
translated_text = (
@ -719,8 +742,8 @@ if __name__ == "__main__":
)
file.write(line)
# print(f"{item['id']} => {item['text']}\n")
else:
print(
f"Skipping the creation of {translated_lex_file} as it exists already"
)
#else:
# print(
# f"Skipping the creation of {translated_lex_file} as it exists already"
# )
quit() # end of the program