brianr/SM2Gen
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Switch to python logging library

Browse Source
This commit is contained in:
Brian Read 2024-11-13 17:09:18 +00:00
parent 1a78388e95
commit a52983cc13
1 changed files with 60 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
sm2gen.py
View File

@ -13,6 +13,7 @@ import traceback
import os import os
from datetime import datetime, timedelta from datetime import datetime, timedelta
from spellchecker import SpellChecker from spellchecker import SpellChecker
import logging
# #
# To Do # To Do
@ -27,8 +28,14 @@ json5_html_list: list = []
ini_file_path = os.path.expanduser("~/.smegit/conf") ini_file_path = os.path.expanduser("~/.smegit/conf")
OPENAI_API_KEY = "" OPENAI_API_KEY = ""
# Configure the basic logging system
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
# Create a logger
logger = logging.getLogger(__name__)
def example_function(**kwargs): def example_function(**kwargs):
print(kwargs) logger.info(kwargs)
def spell_check_and_correct(text): def spell_check_and_correct(text):
try: try:
@ -59,12 +66,12 @@ def spell_check_and_correct(text):
# Join the corrected words back into a single string # Join the corrected words back into a single string
corrected_text = ' '.join(corrected_words) corrected_text = ' '.join(corrected_words)
except: except:
print(f"Spelling checker exception ({text})") logger.warning(f"Spelling checker exception ({text})")
return text return text
return corrected_text return corrected_text
def python_to_perl_structure(data): def python_to_perl_structure(data):
#print(data) #logger.info(data)
if data: if data:
if isinstance(data, list): if isinstance(data, list):
# Convert each dictionary in the list to Perl format: ['text' => 'value'] # Convert each dictionary in the list to Perl format: ['text' => 'value']
@ -126,7 +133,7 @@ def json5_to_list(filename):
def json5_to_pandas(filename): def json5_to_pandas(filename):
with open(filename, "r") as file: with open(filename, "r") as file:
data = json5.load(file) data = json5.load(file)
print(data) logger.debug(data)
return data.json_normalize(data) return data.json_normalize(data)
@ -147,7 +154,7 @@ def rec_print(data, prefix=""):
rec_print(val, f"{prefix}[{idx}]") rec_print(val, f"{prefix}[{idx}]")
# If neither, it's a basic type. # If neither, it's a basic type.
else: else:
print(f"{prefix}: {data}") logger.info(f"{prefix}: {data}")
def find_item(nested_dict, target_key): def find_item(nested_dict, target_key):
@ -195,10 +202,10 @@ def lint_json5(filename):
with open(filename, "r") as file: with open(filename, "r") as file:
data = file.read() data = file.read()
json5.loads(data) json5.loads(data)
print(f"{filename} as JSON5 data is valid") logger.info(f"{filename} as JSON5 data is valid")
except Exception as e: except Exception as e:
print(f"{filename} as JSON5 data is invalid") logger.warning(f"{filename} as JSON5 data is invalid")
print("Error:", str(e)) logger.warning("Error:", str(e))
sys.exit() sys.exit()
@ -221,7 +228,7 @@ def hl(keyname):
if keyname in json5_dict: if keyname in json5_dict:
return json5_dict[keyname] return json5_dict[keyname]
else: else:
print(f"{keyname} not found in JSON5 - top level") logger.warning(f"{keyname} not found in JSON5 - top level")
return "None" return "None"
@ -242,7 +249,7 @@ def has_file_been_modified(file_path):
# Get the file's creation time and last modification time in Unix timestamp # Get the file's creation time and last modification time in Unix timestamp
creation_time = os.path.getctime(file_path) creation_time = os.path.getctime(file_path)
last_modification_time = os.path.getmtime(file_path) last_modification_time = os.path.getmtime(file_path)
print(f"{creation_time}*{last_modification_time}") logger.debug(f"{creation_time}*{last_modification_time}")
quit() quit()
# Compare the creation time and last modification time # Compare the creation time and last modification time
@ -298,25 +305,21 @@ def format_text(text):
# Split the text into sentences # Split the text into sentences
sentences = text.split(".") sentences = text.split(".")
words = sentences[0].split(" ") words = sentences[0].split(" ")
# print(len(sentences))
# Deal with one capitalised word # Deal with one capitalised word
if sentences[0].isupper() and len(sentences) == 1 and len(words) == 1: if sentences[0].isupper() and len(sentences) == 1 and len(words) == 1:
return sentences[0].capitalize() return sentences[0].capitalize()
else: else:
for sentence in sentences: for sentence in sentences:
# print(sentence)
# and splt into sub phrases, based on comma # and splt into sub phrases, based on comma
formatted_phrases = [] formatted_phrases = []
phrases = sentence.split(",") phrases = sentence.split(",")
for phrase in phrases: for phrase in phrases:
# print(phrase)
phrase = phrase.lstrip() phrase = phrase.lstrip()
formatted_words = [] formatted_words = []
words = phrase.split(" ") words = phrase.split(" ")
for i, word in enumerate(words): for i, word in enumerate(words):
# print(i,word)
# Check if the word is fully uppercase or not the first # Check if the word is fully uppercase or not the first
word = word.lstrip() word = word.lstrip()
if word.isupper() or i != 0: if word.isupper() or i != 0:
@ -364,7 +367,7 @@ def get_translation(message="Hello", language="french"):
translated_message = get_completion(prompt) translated_message = get_completion(prompt)
# Look for over long messages # Look for over long messages
if len(message) / len(translated_message) < 0.2: if len(message) / len(translated_message) < 0.2:
print(f"{message} translated to {translated_message}") logger.error(f"{message} translated to {translated_message}")
quit() quit()
return translated_message return translated_message
@ -388,12 +391,11 @@ def check_file_version(filename,force_Files=False):
for line in header_lines: for line in header_lines:
if ' at ' in line: if ' at ' in line:
# Split at 'at', expect the timestamp to be in the third part # Split at 'at', expect the timestamp to be in the third part
#print(line)
timestamp_str = line.split('at')[2].strip() timestamp_str = line.split('at')[2].strip()
break break
if timestamp_str is None: if timestamp_str is None:
print("Warning: No timestamp found. Returning original filename.") logger.warning("Warning: No timestamp found. Returning original filename.")
return filename # Return the original filename if no timestamp is found return filename # Return the original filename if no timestamp is found
# Convert the string timestamp to a datetime object # Convert the string timestamp to a datetime object
@ -405,8 +407,6 @@ def check_file_version(filename,force_Files=False):
# Get the last modified time of the file, ignoring milliseconds # Get the last modified time of the file, ignoring milliseconds
file_modified_time = datetime.fromtimestamp(os.path.getmtime(filename)).replace(microsecond=0) file_modified_time = datetime.fromtimestamp(os.path.getmtime(filename)).replace(microsecond=0)
#print(file_modified_time,file_timestamp)
# Compare the timestamps # Compare the timestamps
if file_modified_time > file_timestamp: if file_modified_time > file_timestamp:
return f"{filename}.new" return f"{filename}.new"
@ -414,10 +414,10 @@ def check_file_version(filename,force_Files=False):
return filename return filename
except FileNotFoundError: except FileNotFoundError:
#print(f"Error: The file '{filename}' does not exist.") logger.warning(f"Error: The file '{filename}' does not exist.")
return filename return filename
except Exception as e: except Exception as e:
print(f"An error occurred: {traceback.format_exc()}") logger.warning(f"An error occurred: {traceback.format_exc()}")
return filename return filename
@ -456,7 +456,7 @@ def extract_input_fields(json_data, value_type):
if value.get('Type').lower() in ['readonlytext', 'text', 'select','checkbox','textarea','email']: if value.get('Type').lower() in ['readonlytext', 'text', 'select','checkbox','textarea','email']:
# input_value = value.get('Value', '') # input_value = value.get('Value', '')
input_name = value.get('Name', '') input_name = value.get('Name', '')
print(input_name) logger.debug(input_name)
# # Match and extract the value without the value_type and parentheses # # Match and extract the value without the value_type and parentheses
# match = pattern.search(input_value) # match = pattern.search(input_value)
# if match: # if match:
@ -476,7 +476,7 @@ if __name__ == "__main__":
json5_dict: dict = {} json5_dict: dict = {}
json5_html_list: list = [] json5_html_list: list = []
print(f"SM2 code from JSON5 - {strVersion}") logger.info(f"SM2 code from JSON5 - {strVersion}")
home_dir = "/home/brianr/clients/SM2/SM2Gen/" home_dir = "/home/brianr/clients/SM2/SM2Gen/"
json_filename = f"{home_dir}/json5/nfsshare.json5" # CreateStarterWebsite.json5" json_filename = f"{home_dir}/json5/nfsshare.json5" # CreateStarterWebsite.json5"
@ -491,12 +491,12 @@ if __name__ == "__main__":
# Read the value of "OPENAI_API_KEY" # Read the value of "OPENAI_API_KEY"
if "OPENAI_API_KEY" in config["smegit"]: if "OPENAI_API_KEY" in config["smegit"]:
OPENAI_API_KEY = config["smegit"]["OPENAI_API_KEY"] OPENAI_API_KEY = config["smegit"]["OPENAI_API_KEY"]
# print("API Key:", OPENAI_API_KEY) # logger.info("API Key:", OPENAI_API_KEY)
client = OpenAI(api_key=OPENAI_API_KEY) client = OpenAI(api_key=OPENAI_API_KEY)
else: else:
print("OPENAI_API_KEY not found in the configuration file.") logger.info("OPENAI_API_KEY not found in the configuration file.")
else: else:
print("Configuration file not found at:", file_path) logger.info("Configuration file not found at:", file_path)
# Command line parameters - not in use # Command line parameters - not in use
parser = argparse.ArgumentParser(description="SM2Gen") parser = argparse.ArgumentParser(description="SM2Gen")
@ -534,14 +534,14 @@ if __name__ == "__main__":
) )
args = parser.parse_args() args = parser.parse_args()
json_filename = args.filename json_filename = args.filename
print( logger.info(
f"JSON5 from {json_filename} with noController={args.noController}, noHtml={args.noHtml} and noLang={args.noLang}" f"JSON5 from {json_filename} with noController={args.noController}, noHtml={args.noHtml} and noLang={args.noLang}"
) # Not yet activated ) # Not yet activated
# check if json5 file exists # check if json5 file exists
json_file_path = Path(json_filename) json_file_path = Path(json_filename)
if not json_file_path.exists(): if not json_file_path.exists():
print(f"json5 file: {json_filename} not found") logger.info(f"json5 file: {json_filename} not found")
quit(1) quit(1)
# check syntax of JSON5 # check syntax of JSON5
@ -551,15 +551,15 @@ if __name__ == "__main__":
try: try:
json5_dict = json5_to_dict(json_filename) json5_dict = json5_to_dict(json_filename)
except Exception as e: except Exception as e:
print(f"json5 file {json_filename} failed lint test (e)") logger.info(f"json5 file {json_filename} failed lint test (e)")
quit(1) quit(1)
# Get dict of just the html bit # Get dict of just the html bit
json5_html_list = json5_dict["html"] json5_html_list = json5_dict["html"]
# Identify message # Identify message
print(f"\nGenerating mojo panels for {hl('PackageName')}") logger.info(f"\nGenerating mojo panels for {hl('PackageName')}")
print("-----------------------------------") logger.info("-----------------------------------")
# Routes for each panel # Routes for each panel
routes = get_all_routes() routes = get_all_routes()
@ -578,7 +578,7 @@ if __name__ == "__main__":
controller_file = check_file_version(target_directory_path + hl("PackageName") + ".pm",force_Files) controller_file = check_file_version(target_directory_path + hl("PackageName") + ".pm",force_Files)
custom_controller_file = check_file_version(target_directory_path + hl("PackageName") + "-Custom.pm",force_Files) custom_controller_file = check_file_version(target_directory_path + hl("PackageName") + "-Custom.pm",force_Files)
#print(custom_controller_file) #logger.info(custom_controller_file)
layout_file = check_file_version(target_directory_path + hl("PackageName").lower() + ".html.ep",force_Files) layout_file = check_file_version(target_directory_path + hl("PackageName").lower() + ".html.ep",force_Files)
css_file = check_file_version(target_directory_path + hl("PackageName").lower() + ".css",force_Files) css_file = check_file_version(target_directory_path + hl("PackageName").lower() + ".css",force_Files)
partial_files = list() partial_files = list()
@ -586,14 +586,14 @@ if __name__ == "__main__":
partial_files.append(check_file_version( partial_files.append(check_file_version(
target_directory_path + '_' + hl("prefix") + "_" + panel + ".html.ep",force_Files) target_directory_path + '_' + hl("prefix") + "_" + panel + ".html.ep",force_Files)
) )
print(f"Partial files to be created:{partial_files}") logger.info(f"Partial files to be created:{partial_files}")
lex_file = check_file_version(target_directory_path + hl("PackageName").lower() + "_en.lex",force_Files) lex_file = check_file_version(target_directory_path + hl("PackageName").lower() + "_en.lex",force_Files)
print(lex_file) logger.info(lex_file)
tablecontrols = ( tablecontrols = (
get_table_control_data() get_table_control_data()
) # arrays of hashes used to drive rows in tables ) # arrays of hashes used to drive rows in tables
# print(strVersion,tablecontrols,routes) # logger.info(strVersion,tablecontrols,routes)
# Generate controller file # Generate controller file
dbfields = [] #extract_input_fields(json5_dict, 'db') # Params which correspond to Db fields - TBD dbfields = [] #extract_input_fields(json5_dict, 'db') # Params which correspond to Db fields - TBD
@ -612,18 +612,18 @@ if __name__ == "__main__":
) )
with open(controller_file, "w") as file: with open(controller_file, "w") as file:
file.write(controller_perl) file.write(controller_perl)
print(f"{controller_file} controller generated ok") logger.info(f"{controller_file} controller generated ok")
except Exception as e: except Exception as e:
print(f"A Chameleon controller render error occurred: {e} {traceback.format_exc()}") logger.info(f"A Chameleon controller render error occurred: {e} {traceback.format_exc()}")
except Exception as e: except Exception as e:
print(f"A Chameleon controller template error occurred: {e} {traceback.format_exc()}") logger.info(f"A Chameleon controller template error occurred: {e} {traceback.format_exc()}")
# Generate Custom controller file # Generate Custom controller file
try: try:
custom_controller_template = PageTemplateFile("Templates/custom.pm.tem") custom_controller_template = PageTemplateFile("Templates/custom.pm.tem")
fields = extract_input_fields(json5_dict, 'stash') # Params which correspond to singleton values fields = extract_input_fields(json5_dict, 'stash') # Params which correspond to singleton values
#flatfields = flatten_hash_of_lists(fields) #flatfields = flatten_hash_of_lists(fields)
#print(fields) #logger.info(fields)
#quit(0) #quit(0)
try: try:
custom_controller_perl = custom_controller_template.render( custom_controller_perl = custom_controller_template.render(
@ -632,11 +632,11 @@ if __name__ == "__main__":
# We must be careful to not overwrite the custom file if the developer has already written to it - TBD # We must be careful to not overwrite the custom file if the developer has already written to it - TBD
with open(custom_controller_file, "w") as file: with open(custom_controller_file, "w") as file:
file.write(custom_controller_perl) file.write(custom_controller_perl)
print(f"{custom_controller_file} custom controller generated ok") logger.info(f"{custom_controller_file} custom controller generated ok")
except Exception as e: except Exception as e:
print(f"A Chameleon custom controller render error occurred: {e} {traceback.format_exc()}") logger.info(f"A Chameleon custom controller render error occurred: {e} {traceback.format_exc()}")
except Exception as e: except Exception as e:
print(f"A Chameleon custom controller template error occurred: {e} {traceback.format_exc()}") logger.info(f"A Chameleon custom controller template error occurred: {e} {traceback.format_exc()}")
# generate Layout file # generate Layout file
layout_template = PageTemplateFile("Templates/layout.html.ep.tem") layout_template = PageTemplateFile("Templates/layout.html.ep.tem")
@ -648,11 +648,11 @@ if __name__ == "__main__":
) )
with open(layout_file, "w") as file: with open(layout_file, "w") as file:
file.write(layout_mojo) file.write(layout_mojo)
print(f"{layout_file} mojo template layout file generated ok") logger.info(f"{layout_file} mojo template layout file generated ok")
except Exception as e: except Exception as e:
print(f"A Chameleon render on layout file error occurred: {e}") logger.info(f"A Chameleon render on layout file error occurred: {e}")
except Exception as e: except Exception as e:
print(f"A Chameleon template layout file error occurred: {e}") logger.info(f"A Chameleon template layout file error occurred: {e}")
# Generate a partial file for each of the entries in the html list # Generate a partial file for each of the entries in the html list
@ -675,13 +675,13 @@ if __name__ == "__main__":
) )
with open(partial_files[i], "w") as file: with open(partial_files[i], "w") as file:
file.write(partial_mojo_template) file.write(partial_mojo_template)
print(f"{partial_files[i]} mojo template generated ok - phase 1") logger.info(f"{partial_files[i]} mojo template generated ok - phase 1")
except Exception as e: except Exception as e:
print( logger.info(
f"A Chameleon render error on partial file {html['route']} occurred: {e}" f"A Chameleon render error on partial file {html['route']} occurred: {e}"
) )
except Exception as e: except Exception as e:
print(f"A Chameleon html {html['route']} error occurred: {e}") logger.info(f"A Chameleon html {html['route']} error occurred: {e}")
# Now generate the controls from the rest of the entries in the dict. # Now generate the controls from the rest of the entries in the dict.
all_controls_html = "" all_controls_html = ""
@ -712,11 +712,11 @@ if __name__ == "__main__":
) )
all_controls_html = all_controls_html + control_html all_controls_html = all_controls_html + control_html
except Exception as e: except Exception as e:
print( logger.info(
f"A Chameleon render on partial file control {html_control} error occurred: {e}" f"A Chameleon render on partial file control {html_control} error occurred: {e}"
) )
except Exception as e: except Exception as e:
print( logger.info(
f"A Chameleon render on partial file control {html_control} error occurred: {e}" f"A Chameleon render on partial file control {html_control} error occurred: {e}"
) )
else: else:
@ -736,11 +736,11 @@ if __name__ == "__main__":
) )
all_controls_html = all_controls_html + simple_control_html all_controls_html = all_controls_html + simple_control_html
except Exception as e: except Exception as e:
print( logger.warning(
f"A Chameleon render on partial file control {html_control} error occurred: {e}" f"A Chameleon render on partial file control {html_control} error occurred: {e}"
) )
except Exception as e: except Exception as e:
print( logger.warning(
f"A Chameleon template partial file control {html_control} error occurred: {e}" f"A Chameleon template partial file control {html_control} error occurred: {e}"
) )
@ -759,7 +759,7 @@ if __name__ == "__main__":
# Write the modified content back to the file # Write the modified content back to the file
with open(partial_files[i], "w") as file: with open(partial_files[i], "w") as file:
file.writelines(lines) file.writelines(lines)
print(f"Content modified and saved to {partial_files[i]}") logger.info(f"Content modified and saved to {partial_files[i]}")
i += 1 i += 1
# Create the css file (the header, followed by a dumy entry for each class created/used above) # Create the css file (the header, followed by a dumy entry for each class created/used above)
@ -801,7 +801,6 @@ if __name__ == "__main__":
left_str = hl("prefix") + "_" + lex_message left_str = hl("prefix") + "_" + lex_message
right_str = lex_message right_str = lex_message
right_str = right_str.replace("_", " ") right_str = right_str.replace("_", " ")
# print(f"Right:{right_str}")
right_str = format_text(right_str) right_str = format_text(right_str)
left_str = left_str.replace(" ", "_") left_str = left_str.replace(" ", "_")
words = left_str.split("_")[:6] words = left_str.split("_")[:6]
@ -813,12 +812,12 @@ if __name__ == "__main__":
lex_all = "" lex_all = ""
for lex_str in string_lib: for lex_str in string_lib:
lex_all += f"'{lex_str['left']}' => '{lex_str['right']}',\n" lex_all += f"'{lex_str['left']}' => '{lex_str['right']}',\n"
print(f"Writing {lex_file}") logger.info(f"Writing {lex_file}")
with open(lex_file, "w") as file: with open(lex_file, "w") as file:
file.write(f"#\n# Generated by SM2Gen version: {strVersion}\n#\n") file.write(f"#\n# Generated by SM2Gen version: {strVersion}\n#\n")
file.write(lex_all) file.write(lex_all)
# and then play the strings back into the partials and the layout file # and then play the strings back into the partials and the layout file
print("..and feed the lex string names back into other files") logger.debug("..and feed the lex string names back into other files")
for filename in all_files: for filename in all_files:
with open(filename, "r") as file: with open(filename, "r") as file:
file_content = file.read() file_content = file.read()
@ -834,7 +833,7 @@ if __name__ == "__main__":
# and write it back # and write it back
with open(filename, "w") as file: with open(filename, "w") as file:
file.write(file_content) file.write(file_content)
print(f"Write out modified:{filename}") logger.info(f"Write out modified:{filename}")
# Now generate all the translated lex files from a list of the languages and codes # Now generate all the translated lex files from a list of the languages and codes
# if specifically requested # if specifically requested
@ -847,7 +846,7 @@ if __name__ == "__main__":
lex_str = file.read() lex_str = file.read()
eng_lex_dict = convert_lex_to_dict(lex_str) eng_lex_dict = convert_lex_to_dict(lex_str)
for lang_item in lang_dict: for lang_item in lang_dict:
print(f"Translating from english lex file to {lang_item['language']}") logger.info(f"Translating from english lex file to {lang_item['language']}")
code = lang_item["code"] code = lang_item["code"]
translated_lex_file = check_file_version( translated_lex_file = check_file_version(
f"{target_directory_path}{hl('PackageName').lower()}_{code}.lex" f"{target_directory_path}{hl('PackageName').lower()}_{code}.lex"
@ -863,7 +862,7 @@ if __name__ == "__main__":
translated_dict.append( translated_dict.append(
{"id": lex_item["id"], "text": translated_text} {"id": lex_item["id"], "text": translated_text}
) )
print(f"Writing out lex file for {lang_item['code']}") logger.info(f"Writing out lex file for {lang_item['code']}")
with open(translated_lex_file, "w") as file: with open(translated_lex_file, "w") as file:
file.write(f"#\n# Generated by SM2Gen version: {strVersion}\n#\n") file.write(f"#\n# Generated by SM2Gen version: {strVersion}\n#\n")
for item in translated_dict: for item in translated_dict:
@ -878,9 +877,9 @@ if __name__ == "__main__":
"'" + item["id"] + "' => " + '"' + translated_text + '",\n' "'" + item["id"] + "' => " + '"' + translated_text + '",\n'
) )
file.write(line) file.write(line)
# print(f"{item['id']} => {item['text']}\n") # logger.info(f"{item['id']} => {item['text']}\n")
#else: #else:
# print( # logger.info(
# f"Skipping the creation of {translated_lex_file} as it exists already" # f"Skipping the creation of {translated_lex_file} as it exists already"
# ) # )
quit() # end of the program quit() # end of the program