check file changed and generate as .new if changed after creation

.gitignore

@@ -2,4 +2,5 @@ bin/
 lib/
 lib64
 pyvenv.cfg
+*.new
 

Targets/CreateStarterWebsite/CreateStarterWebsite-Custom.pm

@@ -1,10 +1,11 @@
 #
+# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 08:37:59 
+#
+#
 # Routines to be edited by the developer to provide validation for parameters
 # and provison of the control data for table(s)
 #
-# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-13 19:20 
+##
-#
-
 use esmith::util;
 use esmith::HostsDB;
 use esmith::AccountsDB;

Targets/CreateStarterWebsite/CreateStarterWebsite-Custom.pm.new

@@ -1,10 +1,10 @@
 #
+# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 11:44:06 
+#
+#
 # Routines to be edited by the developer to provide validation for parameters
 # and provison of the control data for table(s)
 #
-# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-16 12:31:58 
-#
-
 use esmith::util;
 use esmith::HostsDB;
 use esmith::AccountsDB;

Targets/CreateStarterWebsite/CreateStarterWebsite.pm

@@ -1,6 +1,6 @@
 package SrvMngr::Controller::CreateStarterWebsite;
 #
-# Generated by version:SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-16 12:31:58 
+# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 08:38:32 
 #
 #----------------------------------------------------------------------
 # heading     : Miscellaneous

Targets/CreateStarterWebsite/_csw_PARAMS.html.ep

@@ -1,5 +1,5 @@
 %#
-%# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-16 12:31:58 
+%# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 09:09:54 
 %#
 <div id="CreateStarterWebsite-PARAMS" class="partial CreateStarterWebsite-PARAMS">
   <script>

Targets/CreateStarterWebsite/createstarterwebsite.html.ep

@@ -1,7 +1,7 @@
+%#
+%# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 08:38:32 
+%#
 % layout 'default', title => "Sme server 2 - Create Starter Website", share_dir => './';
-%#
-%# Generated by SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-16 12:31:58 
-%#
 % content_for 'module' => begin
 <div id="module" class="module CreateStarterWebsite-panel">
 

Targets/CreateStarterWebsite/createstarterwebsite_en.lex

@@ -1,18 +1,21 @@
-'csw_First_header,_typically_used_for' => 'First header, Typically used for short phrases such as Leader in the field of textile manufacturing',
+#
-'csw_Text_following_first_header,_typically' => 'Text following first header, Typically used for a paragraph of marketing information.  ',
+# Generated by SM2Gen version: SM2Gen version:0.8 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 09:28:01
-'csw_Manage_CreateStarterWebsite_settings:' => 'Manage CreateStarterWebsite settings:',
+#
-'csw_Create' => 'Create',
-'csw_Create_Starter_Website' => 'Create Starter Website',
 'csw_Create_a_starter_website' => 'Create a starter website',
+'csw_You_can_leave_any_field' => 'You can leave any field blank if you do not need it.  ',
+'csw_First_header,_typically_used_for' => 'First header, Typically used for short phrases such as Leader in the field of textile manufacturing',
+'csw_Second_header,_typically_used_for' => 'Second header, Typically used for short phrases such as For more information or To order our products:',
+'csw_Manage_CreateStarterWebsite_settings:' => 'Manage CreateStarterWebsite settings:',
+'csw_' => '',
+'csw_Text_following_second_header,_typically' => 'Text following second header, Typically used for contact or ordering information:',
+'csw_To_create_a_simple_web' => 'To create a simple web page for your company, Fill in the fields below and click onCreate.  ',
+'csw_Do_you_wish_to_proceed?' => 'Do you wish to proceed?',
+'csw_The_text_that_you_enter' => 'The text that you enter below will be line wrapped for a nicer appearance in your web page.  Leave a blank line whenever you want to start a new paragraph.  If you need to force a line break without starting a new paragraph (for example after each line of a mailing address), Then type the four-character sequence',
+'csw_When_you_create_this_web' => 'When you create this web page, The file index.  Htm will be overwritten in your web site directory.  ',
+'csw_Create' => 'Create',
+'csw_Do_not_use_this_optionif' => 'Do not use this optionif you have already customized your web site, Since it will overwrite the index.  Htm file in your web site directory.  ',
+'csw_Create_Starter_Website' => 'Create Starter Website',
 'csw_Hello_PARAMS' => 'Hello PARAMS',
 'csw_APPLY' => 'Apply',
-'csw_Text_following_second_header,_typically' => 'Text following second header, Typically used for contact or ordering information:',
+'csw_Text_following_first_header,_typically' => 'Text following first header, Typically used for a paragraph of marketing information.  ',
-'csw_When_you_create_this_web' => 'When you create this web page, The file index.  Htm will be overwritten in your web site directory.  ',
-'csw_' => '',
-'csw_Do_you_wish_to_proceed?' => 'Do you wish to proceed?',
 'csw_PARAMS_panel_action_was_successful' => 'PARAMS panel action was successful',
-'csw_To_create_a_simple_web' => 'To create a simple web page for your company, Fill in the fields below and click onCreate.  ',
-'csw_You_can_leave_any_field' => 'You can leave any field blank if you do not need it.  ',
-'csw_Second_header,_typically_used_for' => 'Second header, Typically used for short phrases such as For more information or To order our products:',
-'csw_Do_not_use_this_optionif' => 'Do not use this optionif you have already customized your web site, Since it will overwrite the index.  Htm file in your web site directory.  ',
-'csw_The_text_that_you_enter' => 'The text that you enter below will be line wrapped for a nicer appearance in your web page.  Leave a blank line whenever you want to start a new paragraph.  If you need to force a line break without starting a new paragraph (for example after each line of a mailing address), Then type the four-character sequence',

json5/CreateStarterWebsite.json5

@@ -1,3 +1,6 @@
+//
+// Generated by sm1-html-2-json5 version:0.5 Chameleon version:4.5.4 On Python:3.12.3 at 2024-09-18 11:39:12
+//
 {
     'PackageName': 'CreateStarterWebsite',
     'prefix': 'csw',

sm1-html-2-json5.py

@@ -1,10 +1,85 @@
 import json
-import os
 import re
 from bs4 import BeautifulSoup
 from lxml import etree  # Import lxml for HTML validation
 import html
 import argparse
+import pkg_resources
+import sys
+import traceback
+import os
+from datetime import datetime, timedelta
+
+
+sm1_html_2_json5_version = "0.5"
+
+def assemble_version_string():
+	try:
+		chameleon_version = pkg_resources.get_distribution("Chameleon").version
+	except pkg_resources.DistributionNotFound:
+		chameleon_version = "No version information"
+	python_version = sys.version
+	version_pattern = r"(\d{1,3}\.\d{1,3}\.\d{1,3})"
+	version_match = re.search(version_pattern, python_version)
+	python_version = version_match.group(0) if version_match else "Unknown"
+	current_datetime = datetime.now()
+	formatted_datetime = current_datetime.strftime("%Y-%m-%d %H:%M:%S")
+	strVersion = (
+		"sm1-html-2-json5 version:"
+		+ sm1_html_2_json5_version
+		+ " Chameleon version:"
+		+ chameleon_version
+		+ " On Python:"
+		+ python_version
+		+ " at "
+		+ formatted_datetime
+	)
+	return strVersion
+
+def check_file_version(filename, ThresholdSecs=3):
+	#
+	# Check modified versus creation date of the file and return +".new" if modified since creation + ThresholdSecs
+	#
+	try:
+		with open(filename, 'r') as file:
+			# Read the first three lines
+			header_lines = [file.readline().strip() for _ in range(5)]
+
+		# Extract the timestamp
+		timestamp_str = None
+		for line in header_lines:
+			if ' at ' in line:
+				# Split at 'at', expect the timestamp to be in the third part
+				print(line)
+				timestamp_str = line.split('at')[2].strip()
+				break
+		
+		if timestamp_str is None:
+			print("Warning: No timestamp found. Returning original filename.")
+			return filename  # Return the original filename if no timestamp is found
+
+		# Convert the string timestamp to a datetime object
+		file_timestamp = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S')
+
+		# Add the threshold seconds to the creation date
+		file_timestamp += timedelta(seconds=ThresholdSecs)
+
+		# Get the last modified time of the file, ignoring milliseconds
+		file_modified_time = datetime.fromtimestamp(os.path.getmtime(filename)).replace(microsecond=0)
+		
+		print(file_modified_time,file_timestamp)
+
+		# Compare the timestamps
+		if file_modified_time > file_timestamp:
+			return f"{filename}.new"
+		else:
+			return filename
+	except FileNotFoundError:
+		print(f"Error: The file '{filename}' does not exist.")
+		return filename
+	except Exception as e:
+		print(f"An error occurred: {traceback.format_exc()}")
+		return filename
             
 
 def read_html_file(filename):
@@ -21,6 +96,7 @@ def validate_html(html):
     except Exception as e:
         raise ValueError("Invalid HTML document") from e
 
+
 def convert_double_quotes_to_span(text):
     """Convert single-quoted text to <span>...</span>."""
     # Use a regular expression to find single-quoted text and replace it
@@ -49,26 +125,30 @@ def convert_double_quotes_to_span(text):
 # print(f"++{sanitized_text}++")
 # return sanitized_text
 
+
 def sanitize_text(text):
     # Replace newlines with spaces
     print(f"--{text}--")
     # Take out html entities
     decoded_text = html.unescape(text)
     # Take out newlines
-    sanitized_text = decoded_text.replace('\n', ' ').replace('\r', ' ')  # Handle both Unix and Windows line endings
+    sanitized_text = decoded_text.replace("\n", " ").replace(
+        "\r", " "
+    )  # Handle both Unix and Windows line endings
     # Replace tabs with spaces
-    sanitized_text = sanitized_text.replace('\t', ' ')
+    sanitized_text = sanitized_text.replace("\t", " ")
     # Replace quote characters
-    sanitized_text = sanitized_text.replace('"', '').replace("'", '')  # Remove double and single quotes
+    sanitized_text = sanitized_text.replace('"', "").replace(
+        "'", ""
+    )  # Remove double and single quotes
     # Take out any multiple spaces - reduce to one.
-    sanitized_text = ' '.join(sanitized_text.split())
+    sanitized_text = " ".join(sanitized_text.split())
     # Strip leading and trailing whitespace
     sanitized_text = sanitized_text.strip()
     print(f"++{sanitized_text}++")
     return sanitized_text
 
 
-
 def extract_data(html):
     """Extract paragraphs, inputs, tables, and pre blocks from HTML and organize them in order."""
     soup = BeautifulSoup(html, "lxml")
@@ -207,7 +287,7 @@ def insert_spaces_before_caps(text):
     return re.sub(r"(?<!^)(?=[A-Z])", " ", text)
 
 
-def save_to_json5(data, output_filename, package_name, header, sub_header):
+def save_to_json5(data, output_filename, package_name, header, sub_header,strVersion):
 	"""Save extracted data to a JSON5 file with a specific structure."""
 	# Generate prefix from uppercase letters in PackageName made into lowercase
 	prefix = "".join(re.findall(r"[A-Z]", package_name)).lower()
@@ -270,7 +350,8 @@ def save_to_json5(data, output_filename, package_name, header, sub_header):
 		"MenuNavigation": "2000 400",
 		"firstPanel": "PARAMS",
 		"signalEvent": f"smeserver-{package_name.lower()}-update",
-        "html": [{
+		"html": [
+			{
 				"Name": "params",
 				"route": "PARAMS",
 				"Header": header if header else f"{package_name} Contrib",
@@ -280,16 +361,19 @@ def save_to_json5(data, output_filename, package_name, header, sub_header):
 				**{
 					k: v for item in structured_html for k, v in item.items()
 				},  # Flatten the structured_html into the dict
-        }],
+			}
+		],
 	}
 
 	# Save in JSON5 format (JSON with comments and unquoted keys)
 	with open(output_filename, "w", encoding="utf-8") as json_file:
+		
 		json.dump(json5_data, json_file, ensure_ascii=False, indent=4)
 
 	# Manually format as JSON5 by adding single quotes (for simplicity)
 	with open(output_filename, "r+", encoding="utf-8") as json_file:
-        content = json_file.read()
+		content = f"//\n// Generated by {strVersion}\n//\n"
+		content = content + json_file.read()
 		content = content.replace(
 			'"', "'"
 		)  # Replace double quotes with single quotes for JSON5
@@ -299,6 +383,7 @@ def save_to_json5(data, output_filename, package_name, header, sub_header):
 
 
 def main():
+	strVersion = assemble_version_string()
 	# command line parameters
 	parser = argparse.ArgumentParser(description="sm1--html-2-jsopn5")
 	parser.add_argument(
@@ -329,11 +414,11 @@ def main():
 	directory, filename = os.path.split(input_file)
 
 	# Replace 'html' with 'json5' in the directory path
-	new_directory = directory.replace('/html', '/json5')
+	new_directory = directory.replace("/html", "/json5")
 	# print(new_directory)
 
 	# Construct the new path
-	output_file = os.path.join(new_directory, filename.replace('.html', '.json5'))
+	output_file = check_file_version(os.path.join(new_directory, filename.replace(".html", ".json5")))
 	print(output_file)
 	# quit(1)
 
@@ -342,7 +427,7 @@ def main():
 	package_name = os.path.splitext(base_name)[0]  # Use the filename without extension
 
 	# Save extracted data to JSON5
-	save_to_json5(data, output_file, package_name, header, sub_header)
+	save_to_json5(data, output_file, package_name, header, sub_header, strVersion)
 	print(f"Extracted data saved to '{output_file}'.")
 
 

sm2gen.py

@@ -5,12 +5,14 @@ from chameleon import PageTemplateFile, PageTemplate
 import pkg_resources
 import xml.etree.ElementTree as ET
 import re
-import os
-from datetime import datetime
 from openai import OpenAI
 import configparser
 import json
 from pathlib import Path
+import traceback
+import os
+from datetime import datetime, timedelta
+
 
 
 #
@@ -26,6 +28,29 @@ json5_html_list: list = []
 ini_file_path = os.path.expanduser("~/.smegit/conf")
 OPENAI_API_KEY = ""
 
+def assemble_version_string():
+	try:
+		chameleon_version = pkg_resources.get_distribution("Chameleon").version
+	except pkg_resources.DistributionNotFound:
+		chameleon_version = "No version information"
+	python_version = sys.version
+	version_pattern = r"(\d{1,3}\.\d{1,3}\.\d{1,3})"
+	version_match = re.search(version_pattern, python_version)
+	python_version = version_match.group(0) if version_match else "Unknown"
+	current_datetime = datetime.now()
+	formatted_datetime = current_datetime.strftime("%Y-%m-%d %H:%M:%S")
+	strVersion = (
+		"SM2Gen version:"
+		+ SME2Gen_version
+		+ " Chameleon version:"
+		+ chameleon_version
+		+ " On Python:"
+		+ python_version
+		+ " at "
+		+ formatted_datetime
+	)
+	return strVersion
+
 
 def parse_json(json_obj, prefix=""):
     structured_list = []
@@ -293,42 +318,54 @@ def get_translation(message="Hello", language="french"):
         quit()
     return translated_message
 
-def check_file_version(filename):
+import os
+from datetime import datetime, timedelta
+
+def check_file_version(filename, ThresholdSecs=3):
     #
-	# check modified versusu creation date of file and return +".new" if modified since creation
+    # Check modified versus creation date of the file and return +".new" if modified since creation + ThresholdSecs
     #
     try:
         with open(filename, 'r') as file:
             # Read the first three lines
-            header_lines = [file.readline().strip() for _ in range(3)]
+            header_lines = [file.readline().strip() for _ in range(5)]
 
         # Extract the timestamp
+        timestamp_str = None
         for line in header_lines:
             if ' at ' in line:
                 # Split at 'at', expect the timestamp to be in the third part
+                print(line)
                 timestamp_str = line.split('at')[2].strip()
                 break
-        else:
+        
-            raise ValueError("Invalid file format: no timestamp found.")
+        if timestamp_str is None:
+            print("Warning: No timestamp found. Returning original filename.")
+            return filename  # Return the original filename if no timestamp is found
 
         # Convert the string timestamp to a datetime object
         file_timestamp = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S')
 
+        # Add the threshold seconds to the creation date
+        file_timestamp += timedelta(seconds=ThresholdSecs)
+
         # Get the last modified time of the file, ignoring milliseconds
         file_modified_time = datetime.fromtimestamp(os.path.getmtime(filename)).replace(microsecond=0)
         
+        print(file_modified_time,file_timestamp)
+
         # Compare the timestamps
-        if file_modified_time > (file_timestamp):
+        if file_modified_time > file_timestamp:
             return f"{filename}.new"
         else:
             return filename
 
     except FileNotFoundError:
         print(f"Error: The file '{filename}' does not exist.")
-        return None
+        return filename
     except Exception as e:
-        print(f"An error occurred: {e}")
+        print(f"An error occurred: {traceback.format_exc()}")
-        return None
+        return filename
 
 
 def convert_lex_to_dict(pairs_string):
@@ -343,27 +380,8 @@ def convert_lex_to_dict(pairs_string):
 
 
 if __name__ == "__main__":
-    try:
-        chameleon_version = pkg_resources.get_distribution("Chameleon").version
-    except pkg_resources.DistributionNotFound:
-        chameleon_version = "Version information not available"
-    python_version = sys.version
-    version_pattern = r"(\d{1,3}\.\d{1,3}\.\d{1,3})"
-    version_match = re.search(version_pattern, python_version)
-    python_version = version_match.group(0) if version_match else "Unknown"
-    current_datetime = datetime.now()
-    formatted_datetime = current_datetime.strftime("%Y-%m-%d %H:%M:%S")
-    strVersion = (
-        "SM2Gen version:"
-        + SME2Gen_version
-        + " Chameleon version:"
-        + chameleon_version
-        + " On Python:"
-        + python_version
-        + " at "
-        + formatted_datetime
-    )
 
+	strVersion = assemble_version_string()
 	json5_dict: dict = {}
 	json5_html_list: list = []
 
@@ -434,7 +452,11 @@ if __name__ == "__main__":
 	lint_json5(json_filename)
 
 	# Get dict of it all
+	try:
 		json5_dict = json5_to_dict(json_filename)
+	except Exception as e:
+		print(f"json5 file {json_filename} failed lint test (e)")
+		quit(1)
 
 	# Get dict of just the html bit
 	json5_html_list = json5_dict["html"]
@@ -454,19 +476,18 @@ if __name__ == "__main__":
 	directory_path.mkdir(parents=True, exist_ok=True)
 	target_directory_path = "Targets/" + hl("PackageName") + "/"
 
-    controller_file = target_directory_path + hl("PackageName") + ".pm"
+	controller_file = check_file_version(target_directory_path + hl("PackageName") + ".pm")
-    custom_controller_file = target_directory_path + hl("PackageName") + "-Custom.pm"
+	custom_controller_file = check_file_version(target_directory_path + hl("PackageName") + "-Custom.pm",3)
-    # Call it .new if one is already there (and may have been editted by the developer)
+	print(custom_controller_file)
-    if os.path.exists(custom_controller_file):
+	layout_file = check_file_version(target_directory_path + hl("PackageName").lower() + ".html.ep")
-        custom_controller_file = custom_controller_file + ".new"
-    layout_file = target_directory_path + hl("PackageName").lower() + ".html.ep"
 	partial_files = list()
 	for panel in routes:
-        partial_files.append(
+		partial_files.append(check_file_version(
-            target_directory_path + '_' + hl("prefix") + "_" + panel + ".html.ep"
+			target_directory_path + '_' + hl("prefix") + "_" + panel + ".html.ep")
 		)
 	print(f"Partial files to be created:{partial_files}")
-    lex_file = target_directory_path + hl("PackageName").lower() + "_en.lex"
+	lex_file = check_file_version(target_directory_path + hl("PackageName").lower() + "_en.lex")
+	print(lex_file)
 	tablecontrols = (
 		get_table_control_data()
 	)  # arrays of hashes used to drive rows in tables
@@ -657,6 +678,7 @@ if __name__ == "__main__":
 		lex_all += f"'{lex_str['left']}' => '{lex_str['right']}',\n"
 	print(f"Writing {lex_file}")
 	with open(lex_file, "w") as file:
+		file.write(f"#\n# Generated by SM2Gen version: {strVersion}\n#\n")
 		file.write(lex_all)
 	# and then play the strings back into the partials and the layout file
 	print("..and feed the lex string names back into other files")
@@ -679,7 +701,7 @@ if __name__ == "__main__":
 
 	#  Now generate all the translated lex files from a list of the languages and codes
 	# if specifically requested
-    if not args.noLang:
+	if args.noLang:
 		languages_path = "Templates/languages.json"
 		with open(languages_path, "r") as file:
 			languages_str = file.read()
@@ -690,11 +712,11 @@ if __name__ == "__main__":
 		for lang_item in lang_dict:
 			print(f"Translating from english lex file to {lang_item['language']}")
 			code = lang_item["code"]
-            translated_lex_file = (
+			translated_lex_file = check_file_version(
 				f"{target_directory_path}{hl('PackageName').lower()}_{code}.lex"
 			)
-            # Only do it if the lex file is missing
+			# Only do it if the lex file is missing - Removed bjr 18Sept2024
-            if not os.path.exists(translated_lex_file):
+			#if not os.path.exists(translated_lex_file):
 			translated_dict = []
 			for lex_item in eng_lex_dict:
 				# Get it from ChatGPT
@@ -706,6 +728,7 @@ if __name__ == "__main__":
 				)
 			print(f"Writing out lex file for {lang_item['code']}")
 			with open(translated_lex_file, "w") as file:
+				file.write(f"#\n# Generated by SM2Gen version: {strVersion}\n#\n")
 				for item in translated_dict:
 					# escape any nasties
 					translated_text = (
@@ -719,8 +742,8 @@ if __name__ == "__main__":
 					)
 					file.write(line)
 					# print(f"{item['id']} => {item['text']}\n")
-            else:
+			#else:
-                print(
+			#    print(
-                    f"Skipping the creation of {translated_lex_file} as it exists already"
+			#        f"Skipping the creation of {translated_lex_file} as it exists already"
-                )
+			#    )
 	quit()  # end of the program