SM2Gen/sm1-html-2-json5.py

import json
import re
from bs4 import BeautifulSoup
from lxml import etree  # Import lxml for HTML validation
import html
import argparse
import pkg_resources
import sys
import traceback
import os
from datetime import datetime, timedelta


sm1_html_2_json5_version = "0.5"

def assemble_version_string():
	try:
		chameleon_version = pkg_resources.get_distribution("Chameleon").version
	except pkg_resources.DistributionNotFound:
		chameleon_version = "No version information"
	python_version = sys.version
	version_pattern = r"(\d{1,3}\.\d{1,3}\.\d{1,3})"
	version_match = re.search(version_pattern, python_version)
	python_version = version_match.group(0) if version_match else "Unknown"
	current_datetime = datetime.now()
	formatted_datetime = current_datetime.strftime("%Y-%m-%d %H:%M:%S")
	strVersion = (
		"sm1-html-2-json5 version:"
		+ sm1_html_2_json5_version
		+ " Chameleon version:"
		+ chameleon_version
		+ " On Python:"
		+ python_version
		+ " at "
		+ formatted_datetime
	)
	return strVersion

def check_file_version(filename, ThresholdSecs=3):
	#
	# Check modified versus creation date of the file and return +".new" if modified since creation + ThresholdSecs
	#
	try:
		with open(filename, 'r') as file:
			# Read the first three lines
			header_lines = [file.readline().strip() for _ in range(5)]

		# Extract the timestamp
		timestamp_str = None
		for line in header_lines:
			if ' at ' in line:
				# Split at 'at', expect the timestamp to be in the third part
				print(line)
				timestamp_str = line.split('at')[2].strip()
				break
		
		if timestamp_str is None:
			print("Warning: No timestamp found. Returning original filename.")
			return filename  # Return the original filename if no timestamp is found

		# Convert the string timestamp to a datetime object
		file_timestamp = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S')

		# Add the threshold seconds to the creation date
		file_timestamp += timedelta(seconds=ThresholdSecs)

		# Get the last modified time of the file, ignoring milliseconds
		file_modified_time = datetime.fromtimestamp(os.path.getmtime(filename)).replace(microsecond=0)
		
		print(file_modified_time,file_timestamp)

		# Compare the timestamps
		if file_modified_time > file_timestamp:
			return f"{filename}.new"
		else:
			return filename
	except FileNotFoundError:
		print(f"Error: The file '{filename}' does not exist.")
		return filename
	except Exception as e:
		print(f"An error occurred: {traceback.format_exc()}")
		return filename
            

def read_html_file(filename):
    """Read HTML content from a file."""
    with open(filename, "r", encoding="utf-8") as file:
        return file.read()


def validate_html(html):
    """Validate the HTML content."""
    try:
        parser = etree.HTMLParser()
        etree.fromstring(html, parser)  # Attempt to parse the HTML
    except Exception as e:
        raise ValueError("Invalid HTML document") from e


def convert_double_quotes_to_span(text):
    """Convert single-quoted text to <span>...</span>."""
    # Use a regular expression to find single-quoted text and replace it
    return re.sub(r'"(.*?)"', r"<span class=emphasis-para>\1</span>", text)


# def sanitize_text(text):
# # Replace newlines with spaces
# print(f"--{text}--")
# decoded_text = html.unescape(text)

# sanitized_text = decoded_text.replace("\n", "").replace(
# "\r", " "
# )    # Handle both Unix and Windows line endings
# # Replace tabs with spaces
# sanitized_text = sanitized_text.replace("\t", "")
# # map single quotes to double
# # sanitized_text = sanitized_text.replace("'", '"')
# #Map signle and double quotes to nothing
# sanitized_text.replace("'","").replace('"','')
# #Take out any multiple spaces - reduce to one.
# sanitized_text = ' '.join(sanitized_text.split())
# # Strip leading and trailing whitespace
# sanitized_text = sanitized_text.strip()
# #sanitized_text = convert_double_quotes_to_span(sanitized_text)
# print(f"++{sanitized_text}++")
# return sanitized_text


def sanitize_text(text):
    # Replace newlines with spaces
    print(f"--{text}--")
    # Take out html entities
    decoded_text = html.unescape(text)
    # Take out newlines
    sanitized_text = decoded_text.replace("\n", " ").replace(
        "\r", " "
    )  # Handle both Unix and Windows line endings
    # Replace tabs with spaces
    sanitized_text = sanitized_text.replace("\t", " ")
    # Replace quote characters
    sanitized_text = sanitized_text.replace('"', "").replace(
        "'", ""
    )  # Remove double and single quotes
    # Take out any multiple spaces - reduce to one.
    sanitized_text = " ".join(sanitized_text.split())
    # Strip leading and trailing whitespace
    sanitized_text = sanitized_text.strip()
    print(f"++{sanitized_text}++")
    return sanitized_text


def extract_data(html):
    """Extract paragraphs, inputs, tables, and pre blocks from HTML and organize them in order."""
    soup = BeautifulSoup(html, "lxml")
    records = []

    hidden_input_names = ["page", "page_stack", ".id", "csrf_token"]

    header_text = None
    sub_header_text = None

    # Counter for tables
    table_counter = 0

    # Extract elements while preserving order
    for element in soup.find_all(
        ["h1", "h2", "p", "pre", "input", "select", "textarea", "button", "table"]
    ):
        if element.name == "h1":
            header_text = element.get_text(strip=True)
            records.append({"Type": "Header", "Text": header_text})

        elif element.name == "h2":
            sub_header_text = element.get_text(strip=True)
            records.append({"Type": "SubHeader", "Text": sub_header_text})

        elif element.name == "p":
            text = element.get_text(strip=True)
            if text:  # Ignore empty paragraphs
                # Sanitise text freom newlines,tabs and escape quotes.
                sanitised_text = sanitize_text(text)
                if sanitised_text == "":
                    continue
                records.append({"Type": "Paragraph", "Text": sanitised_text})

        elif element.name == "pre":
            text = element.get_text(strip=True)
            if text:  # Ensure non-empty before adding
                records.append({"Type": "Preformatted", "Text": text})

        elif element.name == "input":
            if (
                element.get("type") == "hidden"
                or element.get("name") in hidden_input_names
            ):
                continue

            input_info = {
                "Type": element.get("type", "text").capitalize(),
                "Name": element.get("name"),
                "Value": element.get("value", ""),
            }
            label = element.find_next("label")
            input_info["Label"] = label.get_text(strip=True) if label else None
            records.append(input_info)

        elif element.name == "select":
            options = [
                {"Value": option.get("value"), "Text": option.get_text(strip=True)}
                for option in element.find_all("option")
            ]
            select_info = {
                "Type": "Select",
                "Name": element.get("name"),
                "Options": options,
                "Label": element.find_previous("label").get_text(strip=True)
                if element.find_previous("label")
                else None,
            }
            records.append(select_info)

        elif element.name == "textarea":
            textarea_info = {
                "Type": "Textarea",
                "Name": element.get("name"),
                "Value": element.get_text(strip=True),
            }
            label = element.find_previous("label")
            textarea_info["Label"] = label.get_text(strip=True) if label else None
            records.append(textarea_info)

        elif element.name == "button":
            button_info = {
                "Type": "Button",
                "Name": element.get("name"),
                "Value": element.get_text(strip=True),
                "Label": element.find_previous("label").get_text(strip=True)
                if label
                else None,
            }
            records.append(button_info)

        elif element.name == "table" and "sme-border" in element.get("class", []):
            # Increment the table counter
            table_counter += 1

            # Prepare the TableControl format
            table_control = f"Table{table_counter}"  # e.g., "Table1", "Table2"
            top_headings = []
            columns = []

            # Extract headings from the first row
            first_row = element.find("tr")
            if first_row:
                for th in first_row.find_all("th"):
                    top_headings.append(th.get_text(strip=True))

            # Extract only the first data row's cell values for Columns
            data_rows = element.find_all("tr")[1:]  # Skip the heading row
            if data_rows:
                first_data_row = data_rows[0]  # Take the first row of data
                for idx, th in enumerate(first_row.find_all("th")):
                    td = (
                        first_data_row.find_all("td")[idx]
                        if idx < len(first_data_row.find_all("td"))
                        else None
                    )
                    if td:
                        columns.append(
                            f"{table_control}-{th.get_text(strip=True)}"
                        )  # Format as desired

            records.append(
                {
                    "Type": "Table",
                    "TableControl": table_control,
                    "TopHeadings": top_headings,
                    "Columns": columns,
                }
            )

    return records, header_text, sub_header_text


def insert_spaces_before_caps(text):
    """Insert spaces before each capital letter in a given string."""
    return re.sub(r"(?<!^)(?=[A-Z])", " ", text)


def save_to_json5(data, output_filename, package_name, header, sub_header,strVersion):
	"""Save extracted data to a JSON5 file with a specific structure."""
	# Generate prefix from uppercase letters in PackageName made into lowercase
	prefix = "".join(re.findall(r"[A-Z]", package_name)).lower()

	# Prepare structured html list
	structured_html = []
	paragraph_count = 1
	preformatted_count = 1
	input_count = 1
	table_count = 1

	for record in data:
		if record["Type"] == "Paragraph":
			structured_html.append({f"Paragraph{paragraph_count}": record["Text"]})
			paragraph_count += 1
		elif record["Type"] == "Preformatted":
			structured_html.append(
				{f"Preformatted{preformatted_count}": record["Text"]}
			)
			preformatted_count += 1
		elif record["Type"] == "Header" or record["Type"] == "SubHeader":
			continue  # Skip headers for input count
		elif record["Type"] == "Table":
			# Construct the table entry
			table_structure = {
				"Type": record["Type"],
				"TableControl": record["TableControl"],
				"TopHeadings": record["TopHeadings"],
				"Columns": record["Columns"],
			}
			structured_html.append({f"Table{table_count}": table_structure})
			table_count += 1
		else:  # For inputs, selects, textareas, and buttons
			input_structure = {
				"Type": record["Type"],
				"Value": record.get("Value", ""),  # Safely access Value
			}

			# Use .get() for the Name key to avoid KeyError
			input_structure["Name"] = record.get(
				"Name", None
			)  # Set to None if not present
			input_structure["Label"] = record.get(
				"Label", None
			)  # Set to None if not present

			# Handle specific case for Select options
			if "Options" in record:
				input_structure["Options"] = record["Options"]

			structured_html.append({f"Input{input_count}": input_structure})
			input_count += 1

	# Wrap the records with the required fields
	json5_data = {
		"PackageName": package_name,
		"prefix": prefix,
		"MenuHeading": "Miscellaneous",
		"MenuDescription": insert_spaces_before_caps(package_name),
		"MenuNavigation": "2000 400",
		"firstPanel": "PARAMS",
		"signalEvent": f"smeserver-{package_name.lower()}-update",
		"html": [
			{
				"Name": "params",
				"route": "PARAMS",
				"Header": header if header else f"{package_name} Contrib",
				"SubHeader": sub_header
				if sub_header
				else f"Manage {package_name} settings:",
				**{
					k: v for item in structured_html for k, v in item.items()
				},  # Flatten the structured_html into the dict
			}
		],
	}

	# Save in JSON5 format (JSON with comments and unquoted keys)
	with open(output_filename, "w", encoding="utf-8") as json_file:
		
		json.dump(json5_data, json_file, ensure_ascii=False, indent=4)

	# Manually format as JSON5 by adding single quotes (for simplicity)
	with open(output_filename, "r+", encoding="utf-8") as json_file:
		content = f"//\n// Generated by {strVersion}\n//\n"
		content = content + json_file.read()
		content = content.replace(
			'"', "'"
		)  # Replace double quotes with single quotes for JSON5
		json_file.seek(0)
		json_file.write(content)
		json_file.truncate()  # Remove any old content beyond the new content length


def main():
	strVersion = assemble_version_string()
	# command line parameters
	parser = argparse.ArgumentParser(description="sm1--html-2-jsopn5")
	parser.add_argument(
		"-f",
		"--filename",
		help="Specify a filename for the html file",
		default="CreateStarterWebsite.html",
	)
	args = parser.parse_args()
	input_file = "/home/brianr/clients/SM2/SM2Gen/venv/html/" + args.filename
	if not input_file.lower().endswith(".html"):
		# Add .html extension
		input_file += ".html"
	print(input_file)

	# Read HTML content
	html_content = read_html_file(input_file)

	# Validate the HTML before extracting data
	validate_html(html_content)

	# Extract data from HTML
	data, header, sub_header = extract_data(html_content)
	#
	# Generate output JSON5 filename based on input file name
	#
	# Split the original path into directory and file name
	directory, filename = os.path.split(input_file)

	# Replace 'html' with 'json5' in the directory path
	new_directory = directory.replace("/html", "/json5")
	# print(new_directory)

	# Construct the new path
	output_file = check_file_version(os.path.join(new_directory, filename.replace(".html", ".json5")))
	print(output_file)
	# quit(1)

	# Generate output JSON5 filename based on input file name
	base_name = os.path.basename(input_file)  # Get the file name (with extension)
	package_name = os.path.splitext(base_name)[0]  # Use the filename without extension

	# Save extracted data to JSON5
	save_to_json5(data, output_file, package_name, header, sub_header, strVersion)
	print(f"Extracted data saved to '{output_file}'.")


if __name__ == "__main__":
    main()