351 lines
12 KiB
Python
351 lines
12 KiB
Python
import json
|
|
import os
|
|
import re
|
|
from bs4 import BeautifulSoup
|
|
from lxml import etree # Import lxml for HTML validation
|
|
import html
|
|
import argparse
|
|
|
|
|
|
def read_html_file(filename):
|
|
"""Read HTML content from a file."""
|
|
with open(filename, "r", encoding="utf-8") as file:
|
|
return file.read()
|
|
|
|
|
|
def validate_html(html):
|
|
"""Validate the HTML content."""
|
|
try:
|
|
parser = etree.HTMLParser()
|
|
etree.fromstring(html, parser) # Attempt to parse the HTML
|
|
except Exception as e:
|
|
raise ValueError("Invalid HTML document") from e
|
|
|
|
def convert_double_quotes_to_span(text):
|
|
"""Convert single-quoted text to <span>...</span>."""
|
|
# Use a regular expression to find single-quoted text and replace it
|
|
return re.sub(r'"(.*?)"', r"<span class=emphasis-para>\1</span>", text)
|
|
|
|
|
|
# def sanitize_text(text):
|
|
# # Replace newlines with spaces
|
|
# print(f"--{text}--")
|
|
# decoded_text = html.unescape(text)
|
|
|
|
# sanitized_text = decoded_text.replace("\n", "").replace(
|
|
# "\r", " "
|
|
# ) # Handle both Unix and Windows line endings
|
|
# # Replace tabs with spaces
|
|
# sanitized_text = sanitized_text.replace("\t", "")
|
|
# # map single quotes to double
|
|
# # sanitized_text = sanitized_text.replace("'", '"')
|
|
# #Map signle and double quotes to nothing
|
|
# sanitized_text.replace("'","").replace('"','')
|
|
# #Take out any multiple spaces - reduce to one.
|
|
# sanitized_text = ' '.join(sanitized_text.split())
|
|
# # Strip leading and trailing whitespace
|
|
# sanitized_text = sanitized_text.strip()
|
|
# #sanitized_text = convert_double_quotes_to_span(sanitized_text)
|
|
# print(f"++{sanitized_text}++")
|
|
# return sanitized_text
|
|
|
|
def sanitize_text(text):
|
|
# Replace newlines with spaces
|
|
print(f"--{text}--")
|
|
# Take out html entities
|
|
decoded_text = html.unescape(text)
|
|
# Take out newlines
|
|
sanitized_text = decoded_text.replace('\n', ' ').replace('\r', ' ') # Handle both Unix and Windows line endings
|
|
# Replace tabs with spaces
|
|
sanitized_text = sanitized_text.replace('\t', ' ')
|
|
# Replace quote characters
|
|
sanitized_text = sanitized_text.replace('"', '').replace("'", '') # Remove double and single quotes
|
|
#Take out any multiple spaces - reduce to one.
|
|
sanitized_text = ' '.join(sanitized_text.split())
|
|
# Strip leading and trailing whitespace
|
|
sanitized_text = sanitized_text.strip()
|
|
print(f"++{sanitized_text}++")
|
|
return sanitized_text
|
|
|
|
|
|
|
|
def extract_data(html):
|
|
"""Extract paragraphs, inputs, tables, and pre blocks from HTML and organize them in order."""
|
|
soup = BeautifulSoup(html, "lxml")
|
|
records = []
|
|
|
|
hidden_input_names = ["page", "page_stack", ".id", "csrf_token"]
|
|
|
|
header_text = None
|
|
sub_header_text = None
|
|
|
|
# Counter for tables
|
|
table_counter = 0
|
|
|
|
# Extract elements while preserving order
|
|
for element in soup.find_all(
|
|
["h1", "h2", "p", "pre", "input", "select", "textarea", "button", "table"]
|
|
):
|
|
if element.name == "h1":
|
|
header_text = element.get_text(strip=True)
|
|
records.append({"Type": "Header", "Text": header_text})
|
|
|
|
elif element.name == "h2":
|
|
sub_header_text = element.get_text(strip=True)
|
|
records.append({"Type": "SubHeader", "Text": sub_header_text})
|
|
|
|
elif element.name == "p":
|
|
text = element.get_text(strip=True)
|
|
if text: # Ignore empty paragraphs
|
|
# Sanitise text freom newlines,tabs and escape quotes.
|
|
sanitised_text = sanitize_text(text)
|
|
if sanitised_text == "":
|
|
continue
|
|
records.append({"Type": "Paragraph", "Text": sanitised_text})
|
|
|
|
elif element.name == "pre":
|
|
text = element.get_text(strip=True)
|
|
if text: # Ensure non-empty before adding
|
|
records.append({"Type": "Preformatted", "Text": text})
|
|
|
|
elif element.name == "input":
|
|
if (
|
|
element.get("type") == "hidden"
|
|
or element.get("name") in hidden_input_names
|
|
):
|
|
continue
|
|
|
|
input_info = {
|
|
"Type": element.get("type", "text").capitalize(),
|
|
"Name": element.get("name"),
|
|
"Value": element.get("value", ""),
|
|
}
|
|
label = element.find_next("label")
|
|
input_info["Label"] = label.get_text(strip=True) if label else None
|
|
records.append(input_info)
|
|
|
|
elif element.name == "select":
|
|
options = [
|
|
{"Value": option.get("value"), "Text": option.get_text(strip=True)}
|
|
for option in element.find_all("option")
|
|
]
|
|
select_info = {
|
|
"Type": "Select",
|
|
"Name": element.get("name"),
|
|
"Options": options,
|
|
"Label": element.find_previous("label").get_text(strip=True)
|
|
if element.find_previous("label")
|
|
else None,
|
|
}
|
|
records.append(select_info)
|
|
|
|
elif element.name == "textarea":
|
|
textarea_info = {
|
|
"Type": "Textarea",
|
|
"Name": element.get("name"),
|
|
"Value": element.get_text(strip=True),
|
|
}
|
|
label = element.find_previous("label")
|
|
textarea_info["Label"] = label.get_text(strip=True) if label else None
|
|
records.append(textarea_info)
|
|
|
|
elif element.name == "button":
|
|
button_info = {
|
|
"Type": "Button",
|
|
"Name": element.get("name"),
|
|
"Value": element.get_text(strip=True),
|
|
"Label": element.find_previous("label").get_text(strip=True)
|
|
if label
|
|
else None,
|
|
}
|
|
records.append(button_info)
|
|
|
|
elif element.name == "table" and "sme-border" in element.get("class", []):
|
|
# Increment the table counter
|
|
table_counter += 1
|
|
|
|
# Prepare the TableControl format
|
|
table_control = f"Table{table_counter}" # e.g., "Table1", "Table2"
|
|
top_headings = []
|
|
columns = []
|
|
|
|
# Extract headings from the first row
|
|
first_row = element.find("tr")
|
|
if first_row:
|
|
for th in first_row.find_all("th"):
|
|
top_headings.append(th.get_text(strip=True))
|
|
|
|
# Extract only the first data row's cell values for Columns
|
|
data_rows = element.find_all("tr")[1:] # Skip the heading row
|
|
if data_rows:
|
|
first_data_row = data_rows[0] # Take the first row of data
|
|
for idx, th in enumerate(first_row.find_all("th")):
|
|
td = (
|
|
first_data_row.find_all("td")[idx]
|
|
if idx < len(first_data_row.find_all("td"))
|
|
else None
|
|
)
|
|
if td:
|
|
columns.append(
|
|
f"{table_control}-{th.get_text(strip=True)}"
|
|
) # Format as desired
|
|
|
|
records.append(
|
|
{
|
|
"Type": "Table",
|
|
"TableControl": table_control,
|
|
"TopHeadings": top_headings,
|
|
"Columns": columns,
|
|
}
|
|
)
|
|
|
|
return records, header_text, sub_header_text
|
|
|
|
|
|
def insert_spaces_before_caps(text):
|
|
"""Insert spaces before each capital letter in a given string."""
|
|
return re.sub(r"(?<!^)(?=[A-Z])", " ", text)
|
|
|
|
|
|
def save_to_json5(data, output_filename, package_name, header, sub_header):
|
|
"""Save extracted data to a JSON5 file with a specific structure."""
|
|
# Generate prefix from uppercase letters in PackageName made into lowercase
|
|
prefix = "".join(re.findall(r"[A-Z]", package_name)).lower()
|
|
|
|
# Prepare structured html list
|
|
structured_html = []
|
|
paragraph_count = 1
|
|
preformatted_count = 1
|
|
input_count = 1
|
|
table_count = 1
|
|
|
|
for record in data:
|
|
if record["Type"] == "Paragraph":
|
|
structured_html.append({f"Paragraph{paragraph_count}": record["Text"]})
|
|
paragraph_count += 1
|
|
elif record["Type"] == "Preformatted":
|
|
structured_html.append(
|
|
{f"Preformatted{preformatted_count}": record["Text"]}
|
|
)
|
|
preformatted_count += 1
|
|
elif record["Type"] == "Header" or record["Type"] == "SubHeader":
|
|
continue # Skip headers for input count
|
|
elif record["Type"] == "Table":
|
|
# Construct the table entry
|
|
table_structure = {
|
|
"Type": record["Type"],
|
|
"TableControl": record["TableControl"],
|
|
"TopHeadings": record["TopHeadings"],
|
|
"Columns": record["Columns"],
|
|
}
|
|
structured_html.append({f"Table{table_count}": table_structure})
|
|
table_count += 1
|
|
else: # For inputs, selects, textareas, and buttons
|
|
input_structure = {
|
|
"Type": record["Type"],
|
|
"Value": record.get("Value", ""), # Safely access Value
|
|
}
|
|
|
|
# Use .get() for the Name key to avoid KeyError
|
|
input_structure["Name"] = record.get(
|
|
"Name", None
|
|
) # Set to None if not present
|
|
input_structure["Label"] = record.get(
|
|
"Label", None
|
|
) # Set to None if not present
|
|
|
|
# Handle specific case for Select options
|
|
if "Options" in record:
|
|
input_structure["Options"] = record["Options"]
|
|
|
|
structured_html.append({f"Input{input_count}": input_structure})
|
|
input_count += 1
|
|
|
|
# Wrap the records with the required fields
|
|
json5_data = {
|
|
"PackageName": package_name,
|
|
"prefix": prefix,
|
|
"MenuHeading": "Miscellaneous",
|
|
"MenuDescription": insert_spaces_before_caps(package_name),
|
|
"MenuNavigation": "2000 400",
|
|
"firstPanel": "PARAMS",
|
|
"signalEvent": f"smeserver-{package_name.lower()}-update",
|
|
"html": [{
|
|
"Name": "params",
|
|
"route": "PARAMS",
|
|
"Header": header if header else f"{package_name} Contrib",
|
|
"SubHeader": sub_header
|
|
if sub_header
|
|
else f"Manage {package_name} settings:",
|
|
**{
|
|
k: v for item in structured_html for k, v in item.items()
|
|
}, # Flatten the structured_html into the dict
|
|
}],
|
|
}
|
|
|
|
# Save in JSON5 format (JSON with comments and unquoted keys)
|
|
with open(output_filename, "w", encoding="utf-8") as json_file:
|
|
json.dump(json5_data, json_file, ensure_ascii=False, indent=4)
|
|
|
|
# Manually format as JSON5 by adding single quotes (for simplicity)
|
|
with open(output_filename, "r+", encoding="utf-8") as json_file:
|
|
content = json_file.read()
|
|
content = content.replace(
|
|
'"', "'"
|
|
) # Replace double quotes with single quotes for JSON5
|
|
json_file.seek(0)
|
|
json_file.write(content)
|
|
json_file.truncate() # Remove any old content beyond the new content length
|
|
|
|
|
|
def main():
|
|
# command line parameters
|
|
parser = argparse.ArgumentParser(description="sm1--html-2-jsopn5")
|
|
parser.add_argument(
|
|
"-f",
|
|
"--filename",
|
|
help="Specify a filename for the html file",
|
|
default="CreateStarterWebsite.html",
|
|
)
|
|
args = parser.parse_args()
|
|
input_file = "/home/brianr/clients/SM2/SM2Gen/venv/html/" + args.filename
|
|
if not input_file.lower().endswith(".html"):
|
|
# Add .html extension
|
|
input_file += ".html"
|
|
print(input_file)
|
|
|
|
# Read HTML content
|
|
html_content = read_html_file(input_file)
|
|
|
|
# Validate the HTML before extracting data
|
|
validate_html(html_content)
|
|
|
|
# Extract data from HTML
|
|
data, header, sub_header = extract_data(html_content)
|
|
#
|
|
# Generate output JSON5 filename based on input file name
|
|
#
|
|
# Split the original path into directory and file name
|
|
directory, filename = os.path.split(input_file)
|
|
|
|
# Replace 'html' with 'json5' in the directory path
|
|
new_directory = directory.replace('/html', '/json5')
|
|
#print(new_directory)
|
|
|
|
# Construct the new path
|
|
output_file = os.path.join(new_directory, filename.replace('.html', '.json5'))
|
|
print(output_file)
|
|
#quit(1)
|
|
|
|
# Generate output JSON5 filename based on input file name
|
|
base_name = os.path.basename(input_file) # Get the file name (with extension)
|
|
package_name = os.path.splitext(base_name)[0] # Use the filename without extension
|
|
|
|
# Save extracted data to JSON5
|
|
save_to_json5(data, output_file, package_name, header, sub_header)
|
|
print(f"Extracted data saved to '{output_file}'.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|