import argparse import logging import os import re import sys import json # Configure logging logging.basicConfig(level=logging.ERROR, format="%(asctime)s - %(levelname)s - %(message)s") missing_files = [] def validate_panel_name(panel_name): if not panel_name[0].isupper(): logging.error(f"Error: Panel name \'{panel_name}\' must start with a capital letter.") sys.exit(1) def get_full_base_path(system): return os.path.expanduser(os.path.join("~", system, "usr", "share", "smanager")) def check_controller_file_exists(system, panel): full_base_path = get_full_base_path(system) controller_path = os.path.join(full_base_path, "lib/SrvMngr/Controller", f"{panel}.pm") if not os.path.exists(controller_path): logging.error(f"Error: Controller file \'{controller_path}\' does not exist.") sys.exit(1) return controller_path def extract_title_prefix(controller_path): prefix = None with open(controller_path, 'r') as f: content = f.read() # Regex: my $title = $c->l('_'); match = re.search( r"my\s*\$title\s*=\s*\$c->l\(\s*['\"]([A-Za-z]{2,10})_[^'\"]+['\"]\s*\)", content ) if match: prefix = match.group(1) logging.info(f"Extracted prefix: {prefix}") else: logging.error( f"Error: Could not find title prefix in '{controller_path}'.\n" "Expected format: my $title = $c->l('_something')" ) sys.exit(1) return prefix def scan_application_files(system, panel, prefix, scan_general=False): extracted_strings = {} full_base_path = get_full_base_path(system) # Controller file controller_path = os.path.join(full_base_path, "lib/SrvMngr/Controller", f"{panel}.pm") logging.info(f"Scanning controller file: {controller_path}") scan_file_for_lexical_strings(controller_path, prefix, extracted_strings, scan_general) # Template files themes = ["default", "AdminLTE"] for theme in themes: template_base_path = os.path.join(full_base_path, "themes", theme, "templates") panel_template_path = os.path.join(template_base_path, f"{panel.lower()}.html.ep") logging.info(f"Scanning panel template file: {panel_template_path}") scan_file_for_lexical_strings(panel_template_path, prefix, extracted_strings, scan_general) # Scan partials partials_dir = os.path.join(template_base_path, "partials") if os.path.exists(partials_dir): for filename in os.listdir(partials_dir): # Only scan partial files that match the pattern __.html.ep if filename.startswith(f"_{prefix.lower()}_") and filename.endswith(".html.ep"): partial_path = os.path.join(partials_dir, filename) logging.info(f"Scanning partial template file: {partial_path}") scan_file_for_lexical_strings(partial_path, prefix, extracted_strings, scan_general) # Deduplicate lists of dicts in extracted_strings for key, value in extracted_strings.items(): if isinstance(value, list) and value and isinstance(value[0], dict): # Deduplicate list of dicts using JSON serialization seen = set() deduped = [] for d in value: ser = json.dumps(d, sort_keys=True) if ser not in seen: seen.add(ser) deduped.append(d) extracted_strings[key] = deduped return extracted_strings def scan_file_for_lexical_strings(filepath, prefix, extracted_strings_dict, scan_general): if not os.path.exists(filepath): print(f"Missing file: {filepath}") return with open(filepath, 'r', encoding='utf-8') as f: content = f.read() # Always scan for l '...' or l "..." # pattern_l_call = re.compile(r"\bl\s*(['\"])(.+?)\1") # found_l_calls = pattern_l_call.findall(content) # for quote, string in found_l_calls: # if string not in extracted_strings_dict: # extracted_strings_dict[string] = [] # if filepath not in extracted_strings_dict[string]: # extracted_strings_dict[string].append(filepath) # Either scan for strings with a prefix or ones without if scan_general: pattern = re.compile(r"l[\s|(][\"|\"](.*?)[\"|\"]\)") found_strings1 = pattern.findall(content) pattern_l_call = re.compile(r"\bl\s*(['\"])(.+?)\1") found_l_calls = [match[1] for match in pattern_l_call.findall(content)] found_strings = found_strings1 + found_l_calls for s in found_strings: if not s.startswith(f"{prefix}_"): if s not in extracted_strings_dict: extracted_strings_dict[s] = [] if filepath not in extracted_strings_dict[s]: extracted_strings_dict[s].append(filepath) else: pattern = re.compile(rf"{prefix}_[a-zA-Z0-9_]+") found_strings = pattern.findall(content) for s in found_strings: if s not in extracted_strings_dict: extracted_strings_dict[s] = [] if filepath not in extracted_strings_dict[s]: extracted_strings_dict[s].append(filepath) def read_lex_file(filepath): logging.info(f"Reading file: {filepath}") lex_data = {} if not os.path.exists(filepath): logging.warning(f"File not found: {filepath}") missing_files.append(filepath) return lex_data with open(filepath, 'r', encoding='utf-8') as f: content = f.read() # This regex finds all 'key' => 'value' pairs, even if separated by commas or newlines pattern = r"'(.*?)'\s*=>\s*'(.*?)(? '{value}',{os.linesep}") def read_languages_json(filepath): if not os.path.exists(filepath): missing_files.append(filepath) return [] # Return empty list instead of exiting with open(filepath, 'r') as f: languages = json.load(f) return languages def update_file_with_new_lexical_string(filepath, old_string, new_string): try: with open(filepath, 'r') as f: content = f.read() new_content = content.replace(old_string, new_string) with open(filepath, 'w') as f: f.write(new_content) logging.info(f"Updated \'{old_string}\' to \'{new_string}\' in file: {filepath}") except Exception as e: logging.error(f"Error updating file {filepath}: {e}") def export_sorted_missing_lex(input_file1, input_file2, output_file): """ Reads two lex files, finds all entries in input_file1 missing from input_file2, sorts them alphabetically by key (case-insensitive), and writes them to output_file. """ def read_lex_file(filepath): """Reads a lex file and returns a dict of key-value pairs.""" if not os.path.exists(filepath): print(f"File not found: {filepath}") return {} with open(filepath, 'r', encoding='utf-8') as f: content = f.read() # Regex to find all 'key' => 'value' pairs pattern = r"'([^']+)'\s*=>\s*'(.*?)(? '{dict1[k]}',\n") logging.info(f"Missing lines written to {output_file}:") #for k in sorted_missing_keys: # print(f"'{k}' => '{dict1[k]}',") def main(): parser = argparse.ArgumentParser(description="Scan Mojolicious application files for lexical strings.") parser.add_argument("-p", "--panel", required=True, help="Name of the Mojolicious panel (e.g., MyPanel).") parser.add_argument("-s", "--system", default="SME11", help="System name (default: SME11).") parser.add_argument("-e", "--edit", action="store_true", help="Enable editing of original files (default: False).") parser.add_argument("-l", "--lang", action="store_true", help="Enable other language processing (default: False).") args = parser.parse_args() panel = args.panel system = args.system edit_files = args.edit do_lang = args.lang logging.info(f"Starting scan for panel: {panel}, system: {system} edit: {edit_files} lang: {do_lang}") #quit(0) validate_panel_name(panel) controller_path = check_controller_file_exists(system, panel) prefix = extract_title_prefix(controller_path) if prefix: logging.info(f"Scanning application files for strings with prefix \'{prefix}\'...") extracted_panel_strings = scan_application_files(system, panel, prefix) logging.info(f"Deduplicated extracted panel strings: {len(extracted_panel_strings)} unique strings found.") # Process panel-specific English lexical file # Output to current working directory panel_lex_output_dir = os.path.join(os.getcwd(), "output", panel.capitalize()) os.makedirs(panel_lex_output_dir, exist_ok=True) full_base_path = get_full_base_path(system) # Corrected capitalization for panel in path en_lex_path = os.path.join(full_base_path, "lib/SrvMngr/I18N/Modules", panel, f"{panel.lower()}_en.lex") en_lex_new_path = os.path.join(panel_lex_output_dir, f"{panel.lower()}_en.lex.new") en_lex_data = read_lex_file(en_lex_path) logging.info(f"Original English lex file lines: {len(en_lex_data)}") new_en_lex_data = {} for lex_string in extracted_panel_strings.keys(): if lex_string in en_lex_data: new_en_lex_data[lex_string] = en_lex_data[lex_string] else: #Replace rhs by the lhs less the prefix and no underlines, in lowercase (but capitalised) # this may make a reasonable message, derived from the lex string id. sometext = lex_string.replace(f"{prefix}_", "").replace("_", " ") # Split into words words = sometext.split() # Lowercase all words, capitalize the first if words: words = [words[0].capitalize()] + [w.lower() for w in words[1:]] sometext = ' '.join(words) new_en_lex_data[lex_string] = sometext write_lex_file(en_lex_new_path, new_en_lex_data) logging.info(f"Generated {en_lex_new_path}. Lines in new file: {len(new_en_lex_data)}, Lines in original file: {len(en_lex_data)}") #Create file of the ones not in the new lex file output_diff_file = os.path.join(panel_lex_output_dir, f"{panel.lower()}_en.lex.diff") export_sorted_missing_lex(en_lex_path, en_lex_new_path, output_diff_file) if do_lang: languages_json_path = os.path.join(".", "Templates", "languages.json") # Corrected path languages = read_languages_json(languages_json_path) for lang_entry in languages: lang_code = lang_entry["code"] if lang_code == "en": continue lang_lex_path = os.path.join(full_base_path, "lib/SrvMngr/I18N/Modules", panel, f"{panel.lower()}_{lang_code}.lex") lang_lex_new_path = os.path.join(panel_lex_output_dir, f"{panel.lower()}_{lang_code}.lex.new") lang_lex_data = read_lex_file(lang_lex_path) logging.info(f"Original {lang_code} lex file lines: {len(lang_lex_data)}") new_lang_lex_data = {} for lex_string in extracted_panel_strings.keys(): if lex_string in lang_lex_data: new_lang_lex_data[lex_string] = lang_lex_data[lex_string] else: sometext_from_en = new_en_lex_data.get(lex_string, "") new_en_lex_data[lex_string] = sometext new_lang_lex_data[lex_string] = sometext_from_en write_lex_file(lang_lex_new_path, new_lang_lex_data) logging.info(f"Generated {lang_lex_new_path}. Lines in new file: {len(new_lang_lex_data)}, Lines in original file: {len(lang_lex_data)}") logging.info("") logging.info("Scanning application files for general lexical strings...") extracted_general_strings = scan_application_files(system, panel, prefix, scan_general=True) logging.info(f"Deduplicated extracted general strings: {len(extracted_general_strings)} unique strings found.") general_lex_output_dir = os.path.join(os.getcwd(), "output", "General") os.makedirs(general_lex_output_dir, exist_ok=True) general_en_lex_path = os.path.join(full_base_path, "lib/SrvMngr/I18N/Modules", "General", "general_en.lex") general_en_lex_new_path = os.path.join(general_lex_output_dir, "general_en.lex.new") general_en_lex_data = read_lex_file(general_en_lex_path) logging.info(f"Original general English lex file lines: {len(general_en_lex_data)}") new_general_en_lex_data = read_lex_file(general_en_lex_new_path) for lex_string in extracted_general_strings.keys(): if lex_string in general_en_lex_data: new_general_en_lex_data[lex_string] = general_en_lex_data[lex_string] else: sometext = lex_string.replace("_", " ") # Split into words words = sometext.split() # Lowercase all words, capitalize the first if words: words = [words[0].capitalize()] + [w.lower() for w in words[1:]] sometext = ' '.join(words) new_general_en_lex_data[lex_string] = sometext write_lex_file(general_en_lex_new_path, new_general_en_lex_data) logging.info(f"Generated {general_en_lex_new_path}. Lines in new file: {len(new_general_en_lex_data)}, Lines in original file: {len(general_en_lex_data)}") logging.info("") if do_lang: for lang_entry in languages: lang_code = lang_entry["code"] if lang_code == "en": continue general_lang_lex_path = os.path.join(full_base_path, "lib/SrvMngr/I18N/Modules", "General", f"general_{lang_code}.lex") general_lang_lex_new_path = os.path.join(general_lex_output_dir, f"general_{lang_code}.lex.new") general_lang_lex_data = read_lex_file(general_lang_lex_path) logging.info(f"Original general {lang_code} lex file lines: {len(general_lang_lex_data)}") new_general_lang_lex_data = {} for lex_string in extracted_general_strings.keys(): if lex_string in general_lang_lex_data: new_general_lang_lex_data[lex_string] = general_lang_lex_data[lex_string] else: sometext_from_en = new_general_en_lex_data.get(lex_string, "") new_general_lang_lex_data[lex_string] = sometext_from_en write_lex_file(general_lang_lex_new_path, new_general_lang_lex_data) logging.info(f"Generated {general_lang_lex_new_path}. Lines in new file: {len(new_general_lang_lex_data)}, Lines in original file: {len(general_lang_lex_data)}") logging.info("") if edit_files: logging.info("Handling single-word lexical strings...") for lex_string, filepaths in extracted_panel_strings.items(): if lex_string.startswith(f"{prefix}_"): sometext_part = lex_string[len(prefix) + 1:] if "_" not in sometext_part: just_one_word = sometext_part if just_one_word not in new_general_en_lex_data: new_general_en_lex_data[just_one_word] = just_one_word logging.info(f"Added \'{just_one_word}\' to {general_en_lex_new_path}") write_lex_file(general_en_lex_new_path, new_general_en_lex_data) for lang_entry in languages: lang_code = lang_entry["code"] if lang_code == "en": continue general_lang_lex_path = os.path.join(full_base_path, "lib/SrvMngr/I18N/Modules", "General", f"general_{lang_code}.lex") general_lang_lex_new_path = os.path.join(general_lex_output_dir, f"general_{lang_code}.lex.new") current_general_lang_lex_data = read_lex_file(general_lang_lex_new_path) if just_one_word not in current_general_lang_lex_data: current_general_lang_lex_data[just_one_word] = just_one_word write_lex_file(general_lang_lex_new_path, current_general_lang_lex_data) logging.info(f"Added \'{just_one_word}\' to {general_lang_lex_new_path}") for filepath in filepaths: update_file_with_new_lexical_string(filepath, lex_string, just_one_word) else: logging.error("Could not determine prefix, exiting.") sys.exit(1) if missing_files: logging.warning("The following files were not found:") for f in missing_files: logging.warning(f"- {f}") if __name__ == "__main__": main()