Optimise journal access speeding up processing

root/usr/bin/mailstats.py

@@ -214,52 +214,98 @@ def get_logs_from_Journalctl(date='yesterday'):
     Retrieve and parse journalctl logs for a specific date and units,
     returning them as a sorted list of dictionaries.
     """
+
+    def to_us(ts):
+        # Convert a journal timestamp (datetime or int/string microseconds) to integer microseconds
+        if ts is None:
+            return None
+        if hasattr(ts, "timestamp"):
+            return int(ts.timestamp() * 1_000_000)
         try:
-		# Parse the input date to calculate the start and end of the day
+            return int(ts)
-		if date.lower() == "yesterday":
+        except Exception:
+            return None
+
+    try:
+        # Parse the input date to calculate start and end of the day
+        if isinstance(date, str) and date.lower() == "yesterday":
             target_date = datetime.now() - timedelta(days=1)
+        elif isinstance(date, datetime):
+            target_date = date
         else:
-			target_date = datetime.strptime(date, "%Y-%m-%d")
+            # Supports either a datetime.date-like object (has year attr) or a string YYYY-MM-DD
+            try:
+                target_date = datetime(date.year, date.month, date.day)
+            except Exception:
+                target_date = datetime.strptime(str(date), "%Y-%m-%d")
 
         # Define the time range for the specified date
-		since = target_date.strftime("%Y-%m-%d 00:00:00")
+        since_dt = datetime(target_date.year, target_date.month, target_date.day, 0, 0, 0, 0)
-		until = target_date.strftime("%Y-%m-%d 23:59:59")
+        until_dt = datetime(target_date.year, target_date.month, target_date.day, 23, 59, 59, 999999)
+        since_microseconds = int(since_dt.timestamp() * 1_000_000)
+        until_microseconds = int(until_dt.timestamp() * 1_000_000)
 
-		# Convert times to microseconds for querying
+        # Open the systemd journal (system-only if supported)
-		since_microseconds = int(datetime.strptime(since, "%Y-%m-%d %H:%M:%S").timestamp() * 1_000_000)
+        try:
-		until_microseconds = int(datetime.strptime(until, "%Y-%m-%d %H:%M:%S").timestamp() * 1_000_000)
+            j = journal.Reader(flags=journal.SYSTEM_ONLY)
-		
+        except Exception:
-		# Open the systemd journal
             j = journal.Reader()
 
-		# Set filters for units
+        # Set filters for units (multiple add_match on same field => OR)
         j.add_match(_SYSTEMD_UNIT="qpsmtpd.service")
         j.add_match(_SYSTEMD_UNIT="uqpsmtpd.service")
         j.add_match(_SYSTEMD_UNIT="sqpsmtpd.service")
 
-		# Filter by time range
+        # Filter by time range: seek to the start of the interval
-		j.seek_realtime(since_microseconds // 1_000_000)  # Convert back to seconds for seeking
+        j.seek_realtime(since_dt)
 
         # Retrieve logs within the time range
         logs = []
         log_count = 0
         error_count = 0
+
         for entry in j:
             try:
-				entry_timestamp = entry.get('__REALTIME_TIMESTAMP', None)
+                entry_timestamp = entry.get("__REALTIME_TIMESTAMP", None)
-				entry_microseconds = int(entry_timestamp.timestamp() * 1_000_000)
+                entry_microseconds = to_us(entry_timestamp)
-				if entry_timestamp and since_microseconds <= entry_microseconds <= until_microseconds:
+                if entry_microseconds is None:
+                    continue
+
+                # Early stop once we pass the end of the window
+                if entry_microseconds > until_microseconds:
+                    break
+
+                if entry_microseconds >= since_microseconds:
                     log_count += 1
-					# takeout ASCII Escape sequences from the message
+                    # Strip ANSI escape sequences in MESSAGE (if present and is text/bytes)
-					entry['MESSAGE'] = strip_ansi_codes(entry['MESSAGE'])
+                    try:
+                        msg = entry.get("MESSAGE", "")
+                        if isinstance(msg, (bytes, bytearray)):
+                            msg = msg.decode("utf-8", "replace")
+                        # Only call strip if ESC is present
+                        if "\x1b" in msg:
+                            msg = strip_ansi_codes(msg)
+                        entry["MESSAGE"] = msg
+                    except Exception as se:
+                        # Keep original message, just note the issue at debug level
+                        logging.debug(f"strip_ansi_codes failed: {se}")
+
                     logs.append(entry)
             except Exception as e:
-				logging.warning(f"Error - log line: {log_count} {entry['_PID']} {entry['SYSLOG_IDENTIFIER']} : {e}")
+                # Be defensive getting context fields to avoid raising inside logging
+                pid = entry.get("_PID", "?") if isinstance(entry, dict) else "?"
+                ident = entry.get("SYSLOG_IDENTIFIER", "?") if isinstance(entry, dict) else "?"
+                logging.warning(f"Error - log line: {log_count} {pid} {ident} : {e}")
                 error_count += 1
+
         if error_count:
             logging.info(f"Had {error_count} errors on journal import - probably non character bytes")
-		# Sort logs by __REALTIME_TIMESTAMP in ascending order
+
-		sorted_logs = sorted(logs, key=lambda x: x.get("__REALTIME_TIMESTAMP", 0))
+        # Sort logs by __REALTIME_TIMESTAMP in ascending order (keep original behavior)
+        sorted_logs = sorted(logs, key=lambda x: to_us(x.get("__REALTIME_TIMESTAMP")) or 0)
+
+        logging.debug(f"Collected {len(sorted_logs)} entries for {since_dt.date()} "
+                      f"between {since_dt} and {until_dt} (scanned {log_count} in-window)")
 
         return sorted_logs
 
@@ -267,7 +313,6 @@ def get_logs_from_Journalctl(date='yesterday'):
         logging.error(f"Unexpected error: {e}")
         return {}
 
-
 def transform_to_dict(data, keys, iso_date):
     """
     Transforms a 26x17 list of lists into a list of dictionaries with specified keys.