check file changed and generate as .new if changed after creation

This commit is contained in:
Brian Read 2024-09-18 11:46:26 +01:00
parent 1811a52dec
commit b4a6be435c
10 changed files with 627 additions and 511 deletions

1
.gitignore vendored
View File

@ -2,4 +2,5 @@ bin/
lib/ lib/
lib64 lib64
pyvenv.cfg pyvenv.cfg
*.new

View File

@ -1,10 +1,11 @@
# #
# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 08:37:59
#
#
# Routines to be edited by the developer to provide validation for parameters # Routines to be edited by the developer to provide validation for parameters
# and provison of the control data for table(s) # and provison of the control data for table(s)
# #
# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-13 19:20 ##
#
use esmith::util; use esmith::util;
use esmith::HostsDB; use esmith::HostsDB;
use esmith::AccountsDB; use esmith::AccountsDB;

View File

@ -1,10 +1,10 @@
# #
# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 11:44:06
#
#
# Routines to be edited by the developer to provide validation for parameters # Routines to be edited by the developer to provide validation for parameters
# and provison of the control data for table(s) # and provison of the control data for table(s)
# #
# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-16 12:31:58
#
use esmith::util; use esmith::util;
use esmith::HostsDB; use esmith::HostsDB;
use esmith::AccountsDB; use esmith::AccountsDB;

View File

@ -1,6 +1,6 @@
package SrvMngr::Controller::CreateStarterWebsite; package SrvMngr::Controller::CreateStarterWebsite;
# #
# Generated by version:SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-16 12:31:58 # Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 08:38:32
# #
#---------------------------------------------------------------------- #----------------------------------------------------------------------
# heading : Miscellaneous # heading : Miscellaneous

View File

@ -1,5 +1,5 @@
%# %#
%# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-16 12:31:58 %# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 09:09:54
%# %#
<div id="CreateStarterWebsite-PARAMS" class="partial CreateStarterWebsite-PARAMS"> <div id="CreateStarterWebsite-PARAMS" class="partial CreateStarterWebsite-PARAMS">
<script> <script>

View File

@ -1,7 +1,7 @@
%#
%# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 08:38:32
%#
% layout 'default', title => "Sme server 2 - Create Starter Website", share_dir => './'; % layout 'default', title => "Sme server 2 - Create Starter Website", share_dir => './';
%#
%# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-16 12:31:58
%#
% content_for 'module' => begin % content_for 'module' => begin
<div id="module" class="module CreateStarterWebsite-panel"> <div id="module" class="module CreateStarterWebsite-panel">

View File

@ -1,18 +1,21 @@
'csw_First_header,_typically_used_for' => 'First header, Typically used for short phrases such as Leader in the field of textile manufacturing', #
'csw_Text_following_first_header,_typically' => 'Text following first header, Typically used for a paragraph of marketing information. ', # Generated by SM2Gen version: SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 09:28:01
'csw_Manage_CreateStarterWebsite_settings:' => 'Manage CreateStarterWebsite settings:', #
'csw_Create' => 'Create',
'csw_Create_Starter_Website' => 'Create Starter Website',
'csw_Create_a_starter_website' => 'Create a starter website', 'csw_Create_a_starter_website' => 'Create a starter website',
'csw_You_can_leave_any_field' => 'You can leave any field blank if you do not need it. ',
'csw_First_header,_typically_used_for' => 'First header, Typically used for short phrases such as Leader in the field of textile manufacturing',
'csw_Second_header,_typically_used_for' => 'Second header, Typically used for short phrases such as For more information or To order our products:',
'csw_Manage_CreateStarterWebsite_settings:' => 'Manage CreateStarterWebsite settings:',
'csw_' => '',
'csw_Text_following_second_header,_typically' => 'Text following second header, Typically used for contact or ordering information:',
'csw_To_create_a_simple_web' => 'To create a simple web page for your company, Fill in the fields below and click onCreate. ',
'csw_Do_you_wish_to_proceed?' => 'Do you wish to proceed?',
'csw_The_text_that_you_enter' => 'The text that you enter below will be line wrapped for a nicer appearance in your web page. Leave a blank line whenever you want to start a new paragraph. If you need to force a line break without starting a new paragraph (for example after each line of a mailing address), Then type the four-character sequence',
'csw_When_you_create_this_web' => 'When you create this web page, The file index. Htm will be overwritten in your web site directory. ',
'csw_Create' => 'Create',
'csw_Do_not_use_this_optionif' => 'Do not use this optionif you have already customized your web site, Since it will overwrite the index. Htm file in your web site directory. ',
'csw_Create_Starter_Website' => 'Create Starter Website',
'csw_Hello_PARAMS' => 'Hello PARAMS', 'csw_Hello_PARAMS' => 'Hello PARAMS',
'csw_APPLY' => 'Apply', 'csw_APPLY' => 'Apply',
'csw_Text_following_second_header,_typically' => 'Text following second header, Typically used for contact or ordering information:', 'csw_Text_following_first_header,_typically' => 'Text following first header, Typically used for a paragraph of marketing information. ',
'csw_When_you_create_this_web' => 'When you create this web page, The file index. Htm will be overwritten in your web site directory. ',
'csw_' => '',
'csw_Do_you_wish_to_proceed?' => 'Do you wish to proceed?',
'csw_PARAMS_panel_action_was_successful' => 'PARAMS panel action was successful', 'csw_PARAMS_panel_action_was_successful' => 'PARAMS panel action was successful',
'csw_To_create_a_simple_web' => 'To create a simple web page for your company, Fill in the fields below and click onCreate. ',
'csw_You_can_leave_any_field' => 'You can leave any field blank if you do not need it. ',
'csw_Second_header,_typically_used_for' => 'Second header, Typically used for short phrases such as For more information or To order our products:',
'csw_Do_not_use_this_optionif' => 'Do not use this optionif you have already customized your web site, Since it will overwrite the index. Htm file in your web site directory. ',
'csw_The_text_that_you_enter' => 'The text that you enter below will be line wrapped for a nicer appearance in your web page. Leave a blank line whenever you want to start a new paragraph. If you need to force a line break without starting a new paragraph (for example after each line of a mailing address), Then type the four-character sequence',

View File

@ -1,3 +1,6 @@
//
// Generated by sm1-html-2-json5 version:0.5 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 11:39:12
//
{ {
'PackageName': 'CreateStarterWebsite', 'PackageName': 'CreateStarterWebsite',
'prefix': 'csw', 'prefix': 'csw',

View File

@ -1,12 +1,87 @@
import json import json
import os
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from lxml import etree # Import lxml for HTML validation from lxml import etree # Import lxml for HTML validation
import html import html
import argparse import argparse
import pkg_resources
import sys
import traceback
import os
from datetime import datetime, timedelta
sm1_html_2_json5_version = "0.5"
def assemble_version_string():
try:
chameleon_version = pkg_resources.get_distribution("Chameleon").version
except pkg_resources.DistributionNotFound:
chameleon_version = "No version information"
python_version = sys.version
version_pattern = r"(\d{1,3}\.\d{1,3}\.\d{1,3})"
version_match = re.search(version_pattern, python_version)
python_version = version_match.group(0) if version_match else "Unknown"
current_datetime = datetime.now()
formatted_datetime = current_datetime.strftime("%Y-%m-%d %H:%M:%S")
strVersion = (
"sm1-html-2-json5 version:"
+ sm1_html_2_json5_version
+ " Chameleon version:"
+ chameleon_version
+ " On Python:"
+ python_version
+ " at "
+ formatted_datetime
)
return strVersion
def check_file_version(filename, ThresholdSecs=3):
#
# Check modified versus creation date of the file and return +".new" if modified since creation + ThresholdSecs
#
try:
with open(filename, 'r') as file:
# Read the first three lines
header_lines = [file.readline().strip() for _ in range(5)]
# Extract the timestamp
timestamp_str = None
for line in header_lines:
if ' at ' in line:
# Split at 'at', expect the timestamp to be in the third part
print(line)
timestamp_str = line.split('at')[2].strip()
break
if timestamp_str is None:
print("Warning: No timestamp found. Returning original filename.")
return filename # Return the original filename if no timestamp is found
# Convert the string timestamp to a datetime object
file_timestamp = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S')
# Add the threshold seconds to the creation date
file_timestamp += timedelta(seconds=ThresholdSecs)
# Get the last modified time of the file, ignoring milliseconds
file_modified_time = datetime.fromtimestamp(os.path.getmtime(filename)).replace(microsecond=0)
print(file_modified_time,file_timestamp)
# Compare the timestamps
if file_modified_time > file_timestamp:
return f"{filename}.new"
else:
return filename
except FileNotFoundError:
print(f"Error: The file '{filename}' does not exist.")
return filename
except Exception as e:
print(f"An error occurred: {traceback.format_exc()}")
return filename
def read_html_file(filename): def read_html_file(filename):
"""Read HTML content from a file.""" """Read HTML content from a file."""
with open(filename, "r", encoding="utf-8") as file: with open(filename, "r", encoding="utf-8") as file:
@ -21,6 +96,7 @@ def validate_html(html):
except Exception as e: except Exception as e:
raise ValueError("Invalid HTML document") from e raise ValueError("Invalid HTML document") from e
def convert_double_quotes_to_span(text): def convert_double_quotes_to_span(text):
"""Convert single-quoted text to <span>...</span>.""" """Convert single-quoted text to <span>...</span>."""
# Use a regular expression to find single-quoted text and replace it # Use a regular expression to find single-quoted text and replace it
@ -28,47 +104,51 @@ def convert_double_quotes_to_span(text):
# def sanitize_text(text): # def sanitize_text(text):
# # Replace newlines with spaces # # Replace newlines with spaces
# print(f"--{text}--") # print(f"--{text}--")
# decoded_text = html.unescape(text) # decoded_text = html.unescape(text)
# sanitized_text = decoded_text.replace("\n", "").replace(
# "\r", " "
# ) # Handle both Unix and Windows line endings
# # Replace tabs with spaces
# sanitized_text = sanitized_text.replace("\t", "")
# # map single quotes to double
# # sanitized_text = sanitized_text.replace("'", '"')
# #Map signle and double quotes to nothing
# sanitized_text.replace("'","").replace('"','')
# #Take out any multiple spaces - reduce to one.
# sanitized_text = ' '.join(sanitized_text.split())
# # Strip leading and trailing whitespace
# sanitized_text = sanitized_text.strip()
# #sanitized_text = convert_double_quotes_to_span(sanitized_text)
# print(f"++{sanitized_text}++")
# return sanitized_text
# sanitized_text = decoded_text.replace("\n", "").replace(
# "\r", " "
# ) # Handle both Unix and Windows line endings
# # Replace tabs with spaces
# sanitized_text = sanitized_text.replace("\t", "")
# # map single quotes to double
# # sanitized_text = sanitized_text.replace("'", '"')
# #Map signle and double quotes to nothing
# sanitized_text.replace("'","").replace('"','')
# #Take out any multiple spaces - reduce to one.
# sanitized_text = ' '.join(sanitized_text.split())
# # Strip leading and trailing whitespace
# sanitized_text = sanitized_text.strip()
# #sanitized_text = convert_double_quotes_to_span(sanitized_text)
# print(f"++{sanitized_text}++")
# return sanitized_text
def sanitize_text(text): def sanitize_text(text):
# Replace newlines with spaces # Replace newlines with spaces
print(f"--{text}--") print(f"--{text}--")
# Take out html entities # Take out html entities
decoded_text = html.unescape(text) decoded_text = html.unescape(text)
# Take out newlines # Take out newlines
sanitized_text = decoded_text.replace('\n', ' ').replace('\r', ' ') # Handle both Unix and Windows line endings sanitized_text = decoded_text.replace("\n", " ").replace(
"\r", " "
) # Handle both Unix and Windows line endings
# Replace tabs with spaces # Replace tabs with spaces
sanitized_text = sanitized_text.replace('\t', ' ') sanitized_text = sanitized_text.replace("\t", " ")
# Replace quote characters # Replace quote characters
sanitized_text = sanitized_text.replace('"', '').replace("'", '') # Remove double and single quotes sanitized_text = sanitized_text.replace('"', "").replace(
#Take out any multiple spaces - reduce to one. "'", ""
sanitized_text = ' '.join(sanitized_text.split()) ) # Remove double and single quotes
# Take out any multiple spaces - reduce to one.
sanitized_text = " ".join(sanitized_text.split())
# Strip leading and trailing whitespace # Strip leading and trailing whitespace
sanitized_text = sanitized_text.strip() sanitized_text = sanitized_text.strip()
print(f"++{sanitized_text}++") print(f"++{sanitized_text}++")
return sanitized_text return sanitized_text
def extract_data(html): def extract_data(html):
"""Extract paragraphs, inputs, tables, and pre blocks from HTML and organize them in order.""" """Extract paragraphs, inputs, tables, and pre blocks from HTML and organize them in order."""
soup = BeautifulSoup(html, "lxml") soup = BeautifulSoup(html, "lxml")
@ -100,7 +180,7 @@ def extract_data(html):
# Sanitise text freom newlines,tabs and escape quotes. # Sanitise text freom newlines,tabs and escape quotes.
sanitised_text = sanitize_text(text) sanitised_text = sanitize_text(text)
if sanitised_text == "": if sanitised_text == "":
continue continue
records.append({"Type": "Paragraph", "Text": sanitised_text}) records.append({"Type": "Paragraph", "Text": sanitised_text})
elif element.name == "pre": elif element.name == "pre":
@ -207,98 +287,103 @@ def insert_spaces_before_caps(text):
return re.sub(r"(?<!^)(?=[A-Z])", " ", text) return re.sub(r"(?<!^)(?=[A-Z])", " ", text)
def save_to_json5(data, output_filename, package_name, header, sub_header): def save_to_json5(data, output_filename, package_name, header, sub_header,strVersion):
"""Save extracted data to a JSON5 file with a specific structure.""" """Save extracted data to a JSON5 file with a specific structure."""
# Generate prefix from uppercase letters in PackageName made into lowercase # Generate prefix from uppercase letters in PackageName made into lowercase
prefix = "".join(re.findall(r"[A-Z]", package_name)).lower() prefix = "".join(re.findall(r"[A-Z]", package_name)).lower()
# Prepare structured html list # Prepare structured html list
structured_html = [] structured_html = []
paragraph_count = 1 paragraph_count = 1
preformatted_count = 1 preformatted_count = 1
input_count = 1 input_count = 1
table_count = 1 table_count = 1
for record in data: for record in data:
if record["Type"] == "Paragraph": if record["Type"] == "Paragraph":
structured_html.append({f"Paragraph{paragraph_count}": record["Text"]}) structured_html.append({f"Paragraph{paragraph_count}": record["Text"]})
paragraph_count += 1 paragraph_count += 1
elif record["Type"] == "Preformatted": elif record["Type"] == "Preformatted":
structured_html.append( structured_html.append(
{f"Preformatted{preformatted_count}": record["Text"]} {f"Preformatted{preformatted_count}": record["Text"]}
) )
preformatted_count += 1 preformatted_count += 1
elif record["Type"] == "Header" or record["Type"] == "SubHeader": elif record["Type"] == "Header" or record["Type"] == "SubHeader":
continue # Skip headers for input count continue # Skip headers for input count
elif record["Type"] == "Table": elif record["Type"] == "Table":
# Construct the table entry # Construct the table entry
table_structure = { table_structure = {
"Type": record["Type"], "Type": record["Type"],
"TableControl": record["TableControl"], "TableControl": record["TableControl"],
"TopHeadings": record["TopHeadings"], "TopHeadings": record["TopHeadings"],
"Columns": record["Columns"], "Columns": record["Columns"],
} }
structured_html.append({f"Table{table_count}": table_structure}) structured_html.append({f"Table{table_count}": table_structure})
table_count += 1 table_count += 1
else: # For inputs, selects, textareas, and buttons else: # For inputs, selects, textareas, and buttons
input_structure = { input_structure = {
"Type": record["Type"], "Type": record["Type"],
"Value": record.get("Value", ""), # Safely access Value "Value": record.get("Value", ""), # Safely access Value
} }
# Use .get() for the Name key to avoid KeyError # Use .get() for the Name key to avoid KeyError
input_structure["Name"] = record.get( input_structure["Name"] = record.get(
"Name", None "Name", None
) # Set to None if not present ) # Set to None if not present
input_structure["Label"] = record.get( input_structure["Label"] = record.get(
"Label", None "Label", None
) # Set to None if not present ) # Set to None if not present
# Handle specific case for Select options # Handle specific case for Select options
if "Options" in record: if "Options" in record:
input_structure["Options"] = record["Options"] input_structure["Options"] = record["Options"]
structured_html.append({f"Input{input_count}": input_structure}) structured_html.append({f"Input{input_count}": input_structure})
input_count += 1 input_count += 1
# Wrap the records with the required fields # Wrap the records with the required fields
json5_data = { json5_data = {
"PackageName": package_name, "PackageName": package_name,
"prefix": prefix, "prefix": prefix,
"MenuHeading": "Miscellaneous", "MenuHeading": "Miscellaneous",
"MenuDescription": insert_spaces_before_caps(package_name), "MenuDescription": insert_spaces_before_caps(package_name),
"MenuNavigation": "2000 400", "MenuNavigation": "2000 400",
"firstPanel": "PARAMS", "firstPanel": "PARAMS",
"signalEvent": f"smeserver-{package_name.lower()}-update", "signalEvent": f"smeserver-{package_name.lower()}-update",
"html": [{ "html": [
"Name": "params", {
"route": "PARAMS", "Name": "params",
"Header": header if header else f"{package_name} Contrib", "route": "PARAMS",
"SubHeader": sub_header "Header": header if header else f"{package_name} Contrib",
if sub_header "SubHeader": sub_header
else f"Manage {package_name} settings:", if sub_header
**{ else f"Manage {package_name} settings:",
k: v for item in structured_html for k, v in item.items() **{
}, # Flatten the structured_html into the dict k: v for item in structured_html for k, v in item.items()
}], }, # Flatten the structured_html into the dict
} }
],
}
# Save in JSON5 format (JSON with comments and unquoted keys) # Save in JSON5 format (JSON with comments and unquoted keys)
with open(output_filename, "w", encoding="utf-8") as json_file: with open(output_filename, "w", encoding="utf-8") as json_file:
json.dump(json5_data, json_file, ensure_ascii=False, indent=4)
json.dump(json5_data, json_file, ensure_ascii=False, indent=4)
# Manually format as JSON5 by adding single quotes (for simplicity) # Manually format as JSON5 by adding single quotes (for simplicity)
with open(output_filename, "r+", encoding="utf-8") as json_file: with open(output_filename, "r+", encoding="utf-8") as json_file:
content = json_file.read() content = f"//\n// Generated by {strVersion}\n//\n"
content = content.replace( content = content + json_file.read()
'"', "'" content = content.replace(
) # Replace double quotes with single quotes for JSON5 '"', "'"
json_file.seek(0) ) # Replace double quotes with single quotes for JSON5
json_file.write(content) json_file.seek(0)
json_file.truncate() # Remove any old content beyond the new content length json_file.write(content)
json_file.truncate() # Remove any old content beyond the new content length
def main(): def main():
strVersion = assemble_version_string()
# command line parameters # command line parameters
parser = argparse.ArgumentParser(description="sm1--html-2-jsopn5") parser = argparse.ArgumentParser(description="sm1--html-2-jsopn5")
parser.add_argument( parser.add_argument(
@ -325,24 +410,24 @@ def main():
# #
# Generate output JSON5 filename based on input file name # Generate output JSON5 filename based on input file name
# #
# Split the original path into directory and file name # Split the original path into directory and file name
directory, filename = os.path.split(input_file) directory, filename = os.path.split(input_file)
# Replace 'html' with 'json5' in the directory path # Replace 'html' with 'json5' in the directory path
new_directory = directory.replace('/html', '/json5') new_directory = directory.replace("/html", "/json5")
#print(new_directory) # print(new_directory)
# Construct the new path # Construct the new path
output_file = os.path.join(new_directory, filename.replace('.html', '.json5')) output_file = check_file_version(os.path.join(new_directory, filename.replace(".html", ".json5")))
print(output_file) print(output_file)
#quit(1) # quit(1)
# Generate output JSON5 filename based on input file name # Generate output JSON5 filename based on input file name
base_name = os.path.basename(input_file) # Get the file name (with extension) base_name = os.path.basename(input_file) # Get the file name (with extension)
package_name = os.path.splitext(base_name)[0] # Use the filename without extension package_name = os.path.splitext(base_name)[0] # Use the filename without extension
# Save extracted data to JSON5 # Save extracted data to JSON5
save_to_json5(data, output_file, package_name, header, sub_header) save_to_json5(data, output_file, package_name, header, sub_header, strVersion)
print(f"Extracted data saved to '{output_file}'.") print(f"Extracted data saved to '{output_file}'.")

761
sm2gen.py
View File

@ -5,12 +5,14 @@ from chameleon import PageTemplateFile, PageTemplate
import pkg_resources import pkg_resources
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
import re import re
import os
from datetime import datetime
from openai import OpenAI from openai import OpenAI
import configparser import configparser
import json import json
from pathlib import Path from pathlib import Path
import traceback
import os
from datetime import datetime, timedelta
# #
@ -26,6 +28,29 @@ json5_html_list: list = []
ini_file_path = os.path.expanduser("~/.smegit/conf") ini_file_path = os.path.expanduser("~/.smegit/conf")
OPENAI_API_KEY = "" OPENAI_API_KEY = ""
def assemble_version_string():
try:
chameleon_version = pkg_resources.get_distribution("Chameleon").version
except pkg_resources.DistributionNotFound:
chameleon_version = "No version information"
python_version = sys.version
version_pattern = r"(\d{1,3}\.\d{1,3}\.\d{1,3})"
version_match = re.search(version_pattern, python_version)
python_version = version_match.group(0) if version_match else "Unknown"
current_datetime = datetime.now()
formatted_datetime = current_datetime.strftime("%Y-%m-%d %H:%M:%S")
strVersion = (
"SM2Gen version:"
+ SME2Gen_version
+ " Chameleon version:"
+ chameleon_version
+ " On Python:"
+ python_version
+ " at "
+ formatted_datetime
)
return strVersion
def parse_json(json_obj, prefix=""): def parse_json(json_obj, prefix=""):
structured_list = [] structured_list = []
@ -293,42 +318,54 @@ def get_translation(message="Hello", language="french"):
quit() quit()
return translated_message return translated_message
def check_file_version(filename): import os
# from datetime import datetime, timedelta
# check modified versusu creation date of file and return +".new" if modified since creation
# def check_file_version(filename, ThresholdSecs=3):
#
# Check modified versus creation date of the file and return +".new" if modified since creation + ThresholdSecs
#
try: try:
with open(filename, 'r') as file: with open(filename, 'r') as file:
# Read the first three lines # Read the first three lines
header_lines = [file.readline().strip() for _ in range(3)] header_lines = [file.readline().strip() for _ in range(5)]
# Extract the timestamp # Extract the timestamp
timestamp_str = None
for line in header_lines: for line in header_lines:
if 'at' in line: if ' at ' in line:
# Split at 'at', expect the timestamp to be in the third part # Split at 'at', expect the timestamp to be in the third part
print(line)
timestamp_str = line.split('at')[2].strip() timestamp_str = line.split('at')[2].strip()
break break
else:
raise ValueError("Invalid file format: no timestamp found.") if timestamp_str is None:
print("Warning: No timestamp found. Returning original filename.")
return filename # Return the original filename if no timestamp is found
# Convert the string timestamp to a datetime object # Convert the string timestamp to a datetime object
file_timestamp = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S') file_timestamp = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S')
# Add the threshold seconds to the creation date
file_timestamp += timedelta(seconds=ThresholdSecs)
# Get the last modified time of the file, ignoring milliseconds # Get the last modified time of the file, ignoring milliseconds
file_modified_time = datetime.fromtimestamp(os.path.getmtime(filename)).replace(microsecond=0) file_modified_time = datetime.fromtimestamp(os.path.getmtime(filename)).replace(microsecond=0)
print(file_modified_time,file_timestamp)
# Compare the timestamps # Compare the timestamps
if file_modified_time > (file_timestamp): if file_modified_time > file_timestamp:
return f"{filename}.new" return f"{filename}.new"
else: else:
return filename return filename
except FileNotFoundError: except FileNotFoundError:
print(f"Error: The file '{filename}' does not exist.") print(f"Error: The file '{filename}' does not exist.")
return None return filename
except Exception as e: except Exception as e:
print(f"An error occurred: {e}") print(f"An error occurred: {traceback.format_exc()}")
return None return filename
def convert_lex_to_dict(pairs_string): def convert_lex_to_dict(pairs_string):
@ -343,384 +380,370 @@ def convert_lex_to_dict(pairs_string):
if __name__ == "__main__": if __name__ == "__main__":
try:
chameleon_version = pkg_resources.get_distribution("Chameleon").version
except pkg_resources.DistributionNotFound:
chameleon_version = "Version information not available"
python_version = sys.version
version_pattern = r"(\d{1,3}\.\d{1,3}\.\d{1,3})"
version_match = re.search(version_pattern, python_version)
python_version = version_match.group(0) if version_match else "Unknown"
current_datetime = datetime.now()
formatted_datetime = current_datetime.strftime("%Y-%m-%d %H:%M:%S")
strVersion = (
"SM2Gen version:"
+ SME2Gen_version
+ " Chameleon version:"
+ chameleon_version
+ " On Python:"
+ python_version
+ " at "
+ formatted_datetime
)
json5_dict: dict = {} strVersion = assemble_version_string()
json5_html_list: list = [] json5_dict: dict = {}
json5_html_list: list = []
print(f"SM2 code from JSON5 - {strVersion}") print(f"SM2 code from JSON5 - {strVersion}")
home_dir = "/home/brianr/clients/SM2/SM2Gen/" home_dir = "/home/brianr/clients/SM2/SM2Gen/"
json_filename = f"{home_dir}/json5/nfsshare.json5" # CreateStarterWebsite.json5" json_filename = f"{home_dir}/json5/nfsshare.json5" # CreateStarterWebsite.json5"
# read open ai key from ini file # read open ai key from ini file
# Check if the file exists # Check if the file exists
if os.path.exists(ini_file_path): if os.path.exists(ini_file_path):
# Create a configparser object and read the file # Create a configparser object and read the file
config = configparser.ConfigParser() config = configparser.ConfigParser()
config.read(ini_file_path) config.read(ini_file_path)
# Read the value of "OPENAI_API_KEY" # Read the value of "OPENAI_API_KEY"
if "OPENAI_API_KEY" in config["smegit"]: if "OPENAI_API_KEY" in config["smegit"]:
OPENAI_API_KEY = config["smegit"]["OPENAI_API_KEY"] OPENAI_API_KEY = config["smegit"]["OPENAI_API_KEY"]
# print("API Key:", OPENAI_API_KEY) # print("API Key:", OPENAI_API_KEY)
client = OpenAI(api_key=OPENAI_API_KEY) client = OpenAI(api_key=OPENAI_API_KEY)
else: else:
print("OPENAI_API_KEY not found in the configuration file.") print("OPENAI_API_KEY not found in the configuration file.")
else: else:
print("Configuration file not found at:", file_path) print("Configuration file not found at:", file_path)
# Command line parameters - not in use # Command line parameters - not in use
parser = argparse.ArgumentParser(description="SM2Gen") parser = argparse.ArgumentParser(description="SM2Gen")
parser.add_argument( parser.add_argument(
"-f", "-f",
"--filename", "--filename",
help="Specify a filename for the JSON5 file", help="Specify a filename for the JSON5 file",
default=json_filename, default=json_filename,
) )
parser.add_argument( parser.add_argument(
"-nco", "-nco",
"--noController", "--noController",
help="Stop it creating a controller file", help="Stop it creating a controller file",
default="yes", default="yes",
) )
parser.add_argument( parser.add_argument(
"-nh", "--noHtml", help="Stop it creating html files(s)", default="yes" "-nh", "--noHtml", help="Stop it creating html files(s)", default="yes"
) )
parser.add_argument( parser.add_argument(
"-nl", "-nl",
"--noLang", "--noLang",
help="Stop it creating language localise files(s)", help="Stop it creating language localise files(s)",
default="yes", default="yes",
) )
parser.add_argument( parser.add_argument(
"-ncu", "-ncu",
"--noCust", "--noCust",
help="Stop it creating Custom controller file", help="Stop it creating Custom controller file",
default="yes", default="yes",
) )
args = parser.parse_args() args = parser.parse_args()
json_filename = args.filename json_filename = args.filename
print( print(
f"JSON5 from {json_filename} with noController={args.noController}, noHtml={args.noHtml} and noLang={args.noLang}" f"JSON5 from {json_filename} with noController={args.noController}, noHtml={args.noHtml} and noLang={args.noLang}"
) # Not yet activated ) # Not yet activated
# check if json5 file exists # check if json5 file exists
json_file_path = Path(json_filename) json_file_path = Path(json_filename)
if not json_file_path.exists(): if not json_file_path.exists():
print(f"json5 file: {json_filename} not found") print(f"json5 file: {json_filename} not found")
quit(1) quit(1)
# check syntax of JSON5 # check syntax of JSON5
lint_json5(json_filename) lint_json5(json_filename)
# Get dict of it all # Get dict of it all
json5_dict = json5_to_dict(json_filename) try:
json5_dict = json5_to_dict(json_filename)
except Exception as e:
print(f"json5 file {json_filename} failed lint test (e)")
quit(1)
# Get dict of just the html bit # Get dict of just the html bit
json5_html_list = json5_dict["html"] json5_html_list = json5_dict["html"]
# Identify message # Identify message
print(f"\nGenerating mojo panels for {hl('PackageName')}") print(f"\nGenerating mojo panels for {hl('PackageName')}")
print("-----------------------------------") print("-----------------------------------")
# Routes for each panel # Routes for each panel
routes = get_all_routes() routes = get_all_routes()
lc_routes = lc_get_all_routes() lc_routes = lc_get_all_routes()
# File names # File names
# Define the path for the generated files # Define the path for the generated files
directory_path = Path("Targets/" + hl("PackageName")) directory_path = Path("Targets/" + hl("PackageName"))
# Create the directory if it doesn't exist # Create the directory if it doesn't exist
directory_path.mkdir(parents=True, exist_ok=True) directory_path.mkdir(parents=True, exist_ok=True)
target_directory_path = "Targets/" + hl("PackageName") + "/" target_directory_path = "Targets/" + hl("PackageName") + "/"
controller_file = target_directory_path + hl("PackageName") + ".pm" controller_file = check_file_version(target_directory_path + hl("PackageName") + ".pm")
custom_controller_file = target_directory_path + hl("PackageName") + "-Custom.pm" custom_controller_file = check_file_version(target_directory_path + hl("PackageName") + "-Custom.pm",3)
# Call it .new if one is already there (and may have been editted by the developer) print(custom_controller_file)
if os.path.exists(custom_controller_file): layout_file = check_file_version(target_directory_path + hl("PackageName").lower() + ".html.ep")
custom_controller_file = custom_controller_file + ".new" partial_files = list()
layout_file = target_directory_path + hl("PackageName").lower() + ".html.ep" for panel in routes:
partial_files = list() partial_files.append(check_file_version(
for panel in routes: target_directory_path + '_' + hl("prefix") + "_" + panel + ".html.ep")
partial_files.append( )
target_directory_path + '_' + hl("prefix") + "_" + panel + ".html.ep" print(f"Partial files to be created:{partial_files}")
) lex_file = check_file_version(target_directory_path + hl("PackageName").lower() + "_en.lex")
print(f"Partial files to be created:{partial_files}") print(lex_file)
lex_file = target_directory_path + hl("PackageName").lower() + "_en.lex" tablecontrols = (
tablecontrols = ( get_table_control_data()
get_table_control_data() ) # arrays of hashes used to drive rows in tables
) # arrays of hashes used to drive rows in tables
# print(strVersion,tablecontrols,routes) # print(strVersion,tablecontrols,routes)
# Generate controller file # Generate controller file
try: try:
controller_template = PageTemplateFile( controller_template = PageTemplateFile(
"Templates/controller.pm.tem", CHAMELEON_DEBUG="true" "Templates/controller.pm.tem", CHAMELEON_DEBUG="true"
) )
dbentries = get_db_fields() # Params which correspond to Db fields dbentries = get_db_fields() # Params which correspond to Db fields
try: try:
controller_perl = controller_template.render( controller_perl = controller_template.render(
version=strVersion, version=strVersion,
tablecontrols=tablecontrols, tablecontrols=tablecontrols,
dbentries=dbentries, dbentries=dbentries,
**json5_dict, **json5_dict,
panels=routes, panels=routes,
lcPackageName=json5_dict["PackageName"].lower(), lcPackageName=json5_dict["PackageName"].lower(),
) )
with open(controller_file, "w") as file: with open(controller_file, "w") as file:
file.write(controller_perl) file.write(controller_perl)
print(f"{controller_file} controller generated ok") print(f"{controller_file} controller generated ok")
except Exception as e: except Exception as e:
print(f"A Chameleon controller render error occurred: {e}") print(f"A Chameleon controller render error occurred: {e}")
except Exception as e: except Exception as e:
print(f"A Chameleon controller template error occurred: {e}") print(f"A Chameleon controller template error occurred: {e}")
# Generate Custom controller file # Generate Custom controller file
try: try:
custom_controller_template = PageTemplateFile("Templates/custom.pm.tem") custom_controller_template = PageTemplateFile("Templates/custom.pm.tem")
try: try:
custom_controller_perl = custom_controller_template.render( custom_controller_perl = custom_controller_template.render(
version=strVersion, panels=routes, tablecontrols=tablecontrols version=strVersion, panels=routes, tablecontrols=tablecontrols
) )
# We must be careful to not overwrite the custom file if the developer has already written to it - TBD # We must be careful to not overwrite the custom file if the developer has already written to it - TBD
with open(custom_controller_file, "w") as file: with open(custom_controller_file, "w") as file:
file.write(custom_controller_perl) file.write(custom_controller_perl)
print(f"{custom_controller_file} custom controller generated ok") print(f"{custom_controller_file} custom controller generated ok")
except Exception as e: except Exception as e:
print(f"A Chameleon custom controller render error occurred: {e}") print(f"A Chameleon custom controller render error occurred: {e}")
except Exception as e: except Exception as e:
print(f"A Chameleon custom controller template error occurred: {e}") print(f"A Chameleon custom controller template error occurred: {e}")
# generate Layout file # generate Layout file
layout_template = PageTemplateFile("Templates/layout.html.ep.tem") layout_template = PageTemplateFile("Templates/layout.html.ep.tem")
try: try:
try: try:
layout_mojo = layout_template.render( layout_mojo = layout_template.render(
version=strVersion, **json5_dict, conditions=routes version=strVersion, **json5_dict, conditions=routes
) )
with open(layout_file, "w") as file: with open(layout_file, "w") as file:
file.write(layout_mojo) file.write(layout_mojo)
print(f"{layout_file} mojo template layout file generated ok") print(f"{layout_file} mojo template layout file generated ok")
except Exception as e: except Exception as e:
print(f"A Chameleon render on layout file error occurred: {e}") print(f"A Chameleon render on layout file error occurred: {e}")
except Exception as e: except Exception as e:
print(f"A Chameleon template layout file error occurred: {e}") print(f"A Chameleon template layout file error occurred: {e}")
# Generate a partial file for each of the entries in the html list # Generate a partial file for each of the entries in the html list
# Pull in the template code for each of the input types # Pull in the template code for each of the input types
# html_controls = json5_to_dict('Templates/html_controls.html.ep.tem') # html_controls = json5_to_dict('Templates/html_controls.html.ep.tem')
html_controls = parse_xml_to_dict("Templates/html_controls.html.ep.xml") html_controls = parse_xml_to_dict("Templates/html_controls.html.ep.xml")
i = 0 i = 0
for html in json5_html_list: for html in json5_html_list:
# Generate a mojo template file, and then add in the controls # Generate a mojo template file, and then add in the controls
# main file first # main file first
try: try:
partial_template = PageTemplateFile("Templates/partial.html.ep.tem") partial_template = PageTemplateFile("Templates/partial.html.ep.tem")
partial_mojo_context = {**json5_dict, **html} partial_mojo_context = {**json5_dict, **html}
try: try:
partial_mojo_template = partial_template.render( partial_mojo_template = partial_template.render(
version=strVersion, **partial_mojo_context version=strVersion, **partial_mojo_context
) )
with open(partial_files[i], "w") as file: with open(partial_files[i], "w") as file:
file.write(partial_mojo_template) file.write(partial_mojo_template)
print(f"{partial_files[i]} mojo template generated ok - phase 1") print(f"{partial_files[i]} mojo template generated ok - phase 1")
except Exception as e: except Exception as e:
print( print(
f"A Chameleon render error on partial file {html['route']} occurred: {e}" f"A Chameleon render error on partial file {html['route']} occurred: {e}"
) )
except Exception as e: except Exception as e:
print(f"A Chameleon html {html['route']} error occurred: {e}") print(f"A Chameleon html {html['route']} error occurred: {e}")
# Now generate the controls from the rest of the entries in the dict. # Now generate the controls from the rest of the entries in the dict.
all_controls_html = "" all_controls_html = ""
prefix_is = hl("prefix") prefix_is = hl("prefix")
for html_control in html: for html_control in html:
inner_html = html[html_control] inner_html = html[html_control]
if isinstance(inner_html, dict): if isinstance(inner_html, dict):
try: try:
control_template = PageTemplate(html_controls[inner_html["Type"]]) control_template = PageTemplate(html_controls[inner_html["Type"]])
try: try:
control_html = control_template.render( control_html = control_template.render(
version=strVersion, **inner_html, prefix=prefix_is version=strVersion, **inner_html, prefix=prefix_is
) )
all_controls_html = all_controls_html + control_html all_controls_html = all_controls_html + control_html
except Exception as e: except Exception as e:
print( print(
f"A Chameleon render on partial file control {inner_html['Name']} error occurred: {e}" f"A Chameleon render on partial file control {inner_html['Name']} error occurred: {e}"
) )
except Exception as e: except Exception as e:
print( print(
f"A Chameleon render on partial file control {inner_html['Name']} error occurred: {e}" f"A Chameleon render on partial file control {inner_html['Name']} error occurred: {e}"
) )
else: else:
# just a simple entry - name less numerics is type # just a simple entry - name less numerics is type
html_Type = "".join(char for char in html_control if not char.isdigit()) html_Type = "".join(char for char in html_control if not char.isdigit())
try: try:
simple_control_template = PageTemplate(html_controls[html_Type]) simple_control_template = PageTemplate(html_controls[html_Type])
try: try:
simple_control_html = simple_control_template.render( simple_control_html = simple_control_template.render(
version=strVersion, Value=inner_html, prefix=prefix_is version=strVersion, Value=inner_html, prefix=prefix_is
) )
all_controls_html = all_controls_html + simple_control_html all_controls_html = all_controls_html + simple_control_html
except Exception as e: except Exception as e:
print( print(
f"A Chameleon render on partial file control {html_control} error occurred: {e}" f"A Chameleon render on partial file control {html_control} error occurred: {e}"
) )
except Exception as e: except Exception as e:
print( print(
f"A Chameleon template partial file control {html_control} error occurred: {e}" f"A Chameleon template partial file control {html_control} error occurred: {e}"
) )
# Now insert it into the partial file in the correct place. # Now insert it into the partial file in the correct place.
# Read in the text file and split at "%# Inputs etc in here." # Read in the text file and split at "%# Inputs etc in here."
with open(partial_files[i], "r") as file: with open(partial_files[i], "r") as file:
lines = file.readlines() lines = file.readlines()
index = next( index = next(
(i for i, line in enumerate(lines) if "%# Inputs etc in here." in line), (i for i, line in enumerate(lines) if "%# Inputs etc in here." in line),
len(lines), len(lines),
) )
# Insert the string at the specified index # Insert the string at the specified index
lines.insert(index + 1, all_controls_html + "\n") lines.insert(index + 1, all_controls_html + "\n")
# Write the modified content back to the file # Write the modified content back to the file
with open(partial_files[i], "w") as file: with open(partial_files[i], "w") as file:
file.writelines(lines) file.writelines(lines)
print(f"Content modified and saved to {partial_files[i]}") print(f"Content modified and saved to {partial_files[i]}")
i += 1 i += 1
# Now generate the <name>.en file # Now generate the <name>.en file
# Look through the generated files for the /l[\s|(]['|"](.*)['|"]\)/ strings. # Look through the generated files for the /l[\s|(]['|"](.*)['|"]\)/ strings.
# create a combined list of all the files # create a combined list of all the files
all_files = [controller_file, layout_file] + partial_files all_files = [controller_file, layout_file] + partial_files
all_strings = [] all_strings = []
for filename in all_files: for filename in all_files:
with open(filename, "r") as file: with open(filename, "r") as file:
file_content = file.read() file_content = file.read()
# Define the regular expression pattern to match the strings you want to extract # Define the regular expression pattern to match the strings you want to extract
pattern = r"l[\s|(][\'|\"](.*)[\'|\"]\)" pattern = r"l[\s|(][\'|\"](.*)[\'|\"]\)"
# Use re.findall to extract all occurrences of the pattern from the file content # Use re.findall to extract all occurrences of the pattern from the file content
extracted_strings = re.findall(pattern, file_content) extracted_strings = re.findall(pattern, file_content)
all_strings = all_strings + extracted_strings all_strings = all_strings + extracted_strings
# Take out any duplicates # Take out any duplicates
all_strings = deduplicate_array(all_strings) all_strings = deduplicate_array(all_strings)
# '<prefix>_english-message' => 'English Message', # '<prefix>_english-message' => 'English Message',
string_lib = [] # Array of dicts string_lib = [] # Array of dicts
for lex_message in all_strings: for lex_message in all_strings:
# If has a prefix - leave it for left hand side but delete it for the right # If has a prefix - leave it for left hand side but delete it for the right
# If has no prefix - add one for left hand side but and leave it for the right # If has no prefix - add one for left hand side but and leave it for the right
# Map all spaces to "_" on left hand side # Map all spaces to "_" on left hand side
# amd truncate it to max five words # amd truncate it to max five words
original_str = lex_message original_str = lex_message
# Checkif it starts with the prefix (any case|) # Checkif it starts with the prefix (any case|)
if lex_message.lower().startswith(hl("prefix").lower()): if lex_message.lower().startswith(hl("prefix").lower()):
left_str = lex_message left_str = lex_message
right_str = lex_message[len(hl("prefix")) + 1 :] right_str = lex_message[len(hl("prefix")) + 1 :]
# And take out any "_", map to " " # And take out any "_", map to " "
else: else:
left_str = hl("prefix") + "_" + lex_message left_str = hl("prefix") + "_" + lex_message
right_str = lex_message right_str = lex_message
right_str = right_str.replace("_", " ") right_str = right_str.replace("_", " ")
# print(f"Right:{right_str}") # print(f"Right:{right_str}")
right_str = format_text(right_str) right_str = format_text(right_str)
left_str = left_str.replace(" ", "_") left_str = left_str.replace(" ", "_")
words = left_str.split("_")[:6] words = left_str.split("_")[:6]
left_str = "_".join(words) left_str = "_".join(words)
next_lex_str = {"orig": original_str, "left": left_str, "right": right_str} next_lex_str = {"orig": original_str, "left": left_str, "right": right_str}
string_lib.append(next_lex_str) string_lib.append(next_lex_str)
# And write it to lex file # And write it to lex file
# Now process them one by one into the lexical file # Now process them one by one into the lexical file
lex_all = "" lex_all = ""
for lex_str in string_lib: for lex_str in string_lib:
lex_all += f"'{lex_str['left']}' => '{lex_str['right']}',\n" lex_all += f"'{lex_str['left']}' => '{lex_str['right']}',\n"
print(f"Writing {lex_file}") print(f"Writing {lex_file}")
with open(lex_file, "w") as file: with open(lex_file, "w") as file:
file.write(lex_all) file.write(f"#\n# Generated by SM2Gen version: {strVersion}\n#\n")
# and then play the strings back into the partials and the layout file file.write(lex_all)
print("..and feed the lex string names back into other files") # and then play the strings back into the partials and the layout file
for filename in all_files: print("..and feed the lex string names back into other files")
with open(filename, "r") as file: for filename in all_files:
file_content = file.read() with open(filename, "r") as file:
# Scan through file_content = file.read()
for item in string_lib: # Scan through
original_str = item["orig"] for item in string_lib:
left_str = item["left"] original_str = item["orig"]
right_str = item["right"] left_str = item["left"]
# Replace all occurrences of original string with left string in 'contents' right_str = item["right"]
file_content = file_content.replace( # Replace all occurrences of original string with left string in 'contents'
"l('" + original_str + "')", "l('" + left_str + "')" file_content = file_content.replace(
) "l('" + original_str + "')", "l('" + left_str + "')"
# and write it back )
with open(filename, "w") as file: # and write it back
file.write(file_content) with open(filename, "w") as file:
print(f"Write out modified:{filename}") file.write(file_content)
print(f"Write out modified:{filename}")
# Now generate all the translated lex files from a list of the languages and codes # Now generate all the translated lex files from a list of the languages and codes
# if specifically requested # if specifically requested
if not args.noLang: if args.noLang:
languages_path = "Templates/languages.json" languages_path = "Templates/languages.json"
with open(languages_path, "r") as file: with open(languages_path, "r") as file:
languages_str = file.read() languages_str = file.read()
lang_dict = json.loads(languages_str) lang_dict = json.loads(languages_str)
with open(lex_file, "r") as file: with open(lex_file, "r") as file:
lex_str = file.read() lex_str = file.read()
eng_lex_dict = convert_lex_to_dict(lex_str) eng_lex_dict = convert_lex_to_dict(lex_str)
for lang_item in lang_dict: for lang_item in lang_dict:
print(f"Translating from english lex file to {lang_item['language']}") print(f"Translating from english lex file to {lang_item['language']}")
code = lang_item["code"] code = lang_item["code"]
translated_lex_file = ( translated_lex_file = check_file_version(
f"{target_directory_path}{hl('PackageName').lower()}_{code}.lex" f"{target_directory_path}{hl('PackageName').lower()}_{code}.lex"
) )
# Only do it if the lex file is missing # Only do it if the lex file is missing - Removed bjr 18Sept2024
if not os.path.exists(translated_lex_file): #if not os.path.exists(translated_lex_file):
translated_dict = [] translated_dict = []
for lex_item in eng_lex_dict: for lex_item in eng_lex_dict:
# Get it from ChatGPT # Get it from ChatGPT
translated_text = get_translation( translated_text = get_translation(
lex_item["text"], lang_item["language"] lex_item["text"], lang_item["language"]
) )
translated_dict.append( translated_dict.append(
{"id": lex_item["id"], "text": translated_text} {"id": lex_item["id"], "text": translated_text}
) )
print(f"Writing out lex file for {lang_item['code']}") print(f"Writing out lex file for {lang_item['code']}")
with open(translated_lex_file, "w") as file: with open(translated_lex_file, "w") as file:
for item in translated_dict: file.write(f"#\n# Generated by SM2Gen version: {strVersion}\n#\n")
# escape any nasties for item in translated_dict:
translated_text = ( # escape any nasties
item["text"] translated_text = (
.replace("\\", r"\\") item["text"]
.replace('"', r"\"") .replace("\\", r"\\")
.replace("'", r"\'") .replace('"', r"\"")
) .replace("'", r"\'")
line = ( )
"'" + item["id"] + "' => " + '"' + translated_text + '",\n' line = (
) "'" + item["id"] + "' => " + '"' + translated_text + '",\n'
file.write(line) )
# print(f"{item['id']} => {item['text']}\n") file.write(line)
else: # print(f"{item['id']} => {item['text']}\n")
print( #else:
f"Skipping the creation of {translated_lex_file} as it exists already" # print(
) # f"Skipping the creation of {translated_lex_file} as it exists already"
quit() # end of the program # )
quit() # end of the program