From d94bf8e0333f8ecb5dd236f89bf0b3c4fde49ca9 Mon Sep 17 00:00:00 2001 From: Brian Read Date: Wed, 3 Sep 2025 11:00:00 +0100 Subject: [PATCH] Get detail logs page working - WIP --- journalwrap.c | 179 +++++++++++++ root/opt/mailstats/css/mailstats.css | 99 ++++++- root/opt/mailstats/html/ShowDetailedLogs.php | 265 ++++++++++++++++--- root/opt/mailstats/html/showSummaryLogs.php | 133 +++++----- root/usr/bin/runmailstatsSME10.sh | 17 -- smeserver-mailstats.spec | 14 +- 6 files changed, 581 insertions(+), 126 deletions(-) create mode 100644 journalwrap.c delete mode 100755 root/usr/bin/runmailstatsSME10.sh diff --git a/journalwrap.c b/journalwrap.c new file mode 100644 index 0000000..99706d0 --- /dev/null +++ b/journalwrap.c @@ -0,0 +1,179 @@ +#include +#include +#include +#include +#include +#include +#include + +#ifndef MAX_OUTPUT_BYTES +#define MAX_OUTPUT_BYTES (2 * 1000 * 1000) // 2 MB +#endif + +static int append_bytes(char **buf, size_t *len, size_t *cap, const char *src, size_t n) { + if (*len + n + 1 > *cap) { + size_t newcap = (*cap == 0) ? 8192 : *cap; + while (*len + n + 1 > newcap) { + newcap *= 2; + if (newcap > (size_t)(MAX_OUTPUT_BYTES + 65536)) { + newcap = (size_t)(MAX_OUTPUT_BYTES + 65536); + break; + } + } + char *nbuf = realloc(*buf, newcap); + if (!nbuf) return -1; + *buf = nbuf; *cap = newcap; + } + memcpy(*buf + *len, src, n); + *len += n; + (*buf)[*len] = '\0'; + return 0; +} + +static int append_cstr(char **buf, size_t *len, size_t *cap, const char *s) { + return append_bytes(buf, len, cap, s, strlen(s)); +} + +static size_t min_size(size_t a, size_t b) { return a < b ? a : b; } + +static void sanitize_text(char *s, size_t n) { + for (size_t i = 0; i < n; i++) if (s[i] == '\0') s[i] = ' '; +} + +static void format_ts(char *out, size_t outsz, uint64_t usec) { + time_t sec = (time_t)(usec / 1000000ULL); + struct tm tm; + localtime_r(&sec, &tm); + strftime(out, outsz, "%Y-%m-%d %H:%M:%S", &tm); +} + +static const char* field_value(const void *data, size_t len, const char *key, size_t *vlen) { + size_t klen = strlen(key); + if (len < klen + 1) return NULL; + const char *p = (const char *)data; + if (memcmp(p, key, klen) != 0 || p[klen] != '=') return NULL; + *vlen = len - (klen + 1); + return p + klen + 1; +} + +static int append_entry_line(sd_journal *j, char **buf, size_t *len, size_t *cap) { + uint64_t usec = 0; + (void)sd_journal_get_realtime_usec(j, &usec); + char ts[32]; + format_ts(ts, sizeof(ts), usec); + + const void *data = NULL; + size_t dlen = 0; + const char *message = NULL; + size_t mlen = 0; + + int r = sd_journal_get_data(j, "MESSAGE", &data, &dlen); + if (r >= 0) message = field_value(data, dlen, "MESSAGE", &mlen); + + const char *ident = NULL; + size_t ilen = 0; + r = sd_journal_get_data(j, "SYSLOG_IDENTIFIER", &data, &dlen); + if (r >= 0) { + ident = field_value(data, dlen, "SYSLOG_IDENTIFIER", &ilen); + } else if (sd_journal_get_data(j, "_COMM", &data, &dlen) >= 0) { + ident = field_value(data, dlen, "_COMM", &ilen); + } + + if (append_cstr(buf, len, cap, "[") < 0) return -1; + if (append_cstr(buf, len, cap, ts) < 0) return -1; + if (append_cstr(buf, len, cap, "] ") < 0) return -1; + if (ident && ilen > 0) { + if (append_bytes(buf, len, cap, ident, ilen) < 0) return -1; + if (append_cstr(buf, len, cap, ": ") < 0) return -1; + } + + if (message && mlen > 0) { + char *tmp = malloc(mlen); + if (!tmp) return -1; + memcpy(tmp, message, mlen); + sanitize_text(tmp, mlen); + size_t to_copy = min_size(mlen, (size_t)(MAX_OUTPUT_BYTES > *len ? MAX_OUTPUT_BYTES - *len : 0)); + int ok = append_bytes(buf, len, cap, tmp, to_copy); + free(tmp); + if (ok < 0) return -1; + } else { + const char *keys[] = {"PRIORITY","SYSLOG_IDENTIFIER","_COMM","_EXE","_CMDLINE","MESSAGE"}; + for (size_t i = 0; i < sizeof(keys)/sizeof(keys[0]); i++) { + if (sd_journal_get_data(j, keys[i], &data, &dlen) < 0) continue; + if (append_cstr(buf, len, cap, (i == 0 ? "" : " ")) < 0) return -1; + if (append_bytes(buf, len, cap, (const char*)data, min_size(dlen, (size_t)(MAX_OUTPUT_BYTES - *len))) < 0) return -1; + } + } + + if (*len < MAX_OUTPUT_BYTES) { + if (append_cstr(buf, len, cap, "\n") < 0) return -1; + } + return 0; +} + +static char* journal_get_by_pid_impl(int pid) { + if (pid <= 0) { char *z = malloc(1); if (z) z[0] = '\0'; return z; } + + sd_journal *j = NULL; + if (sd_journal_open(&j, SD_JOURNAL_LOCAL_ONLY) < 0) { + char *z = malloc(1); if (z) z[0] = '\0'; return z; + } + + char match[64]; + snprintf(match, sizeof(match), "_PID=%d", pid); + if (sd_journal_add_match(j, match, 0) < 0) { + sd_journal_close(j); + char *z = malloc(1); if (z) z[0] = '\0'; return z; + } + + sd_journal_seek_head(j); + + char *buf = NULL; size_t len = 0, cap = 0; + int r; + while ((r = sd_journal_next(j)) > 0) { + if (len >= MAX_OUTPUT_BYTES) break; + if (append_entry_line(j, &buf, &len, &cap) < 0) { + free(buf); sd_journal_close(j); return NULL; + } + } + + if (len >= MAX_OUTPUT_BYTES) { + const char *trunc = "[output truncated]\n"; + (void)append_bytes(&buf, &len, &cap, trunc, strlen(trunc)); + } + + if (!buf) { buf = malloc(1); if (!buf) { sd_journal_close(j); return NULL; } buf[0] = '\0'; } + sd_journal_close(j); + return buf; +} + +#ifdef __GNUC__ +__attribute__((visibility("default"))) +#endif +char* journal_get_by_pid(int pid) { return journal_get_by_pid_impl(pid); } + +#ifdef __GNUC__ +__attribute__((visibility("default"))) +#endif +void journal_free(char* p) { free(p); } + +#ifdef BUILD_CLI +static int parse_pid(const char *s, int *out) { + if (!s || !*s) return -1; + char *end = NULL; + errno = 0; + long v = strtol(s, &end, 10); + if (errno != 0 || end == s || *end != '\0' || v <= 0 || v > 0x7fffffffL) return -1; + *out = (int)v; return 0; +} +int main(int argc, char **argv) { + if (argc != 2) { fprintf(stderr, "Usage: %s \n", argv[0]); return 2; } + int pid = 0; + if (parse_pid(argv[1], &pid) != 0) { fprintf(stderr, "Invalid pid\n"); return 2; } + char *out = journal_get_by_pid_impl(pid); + if (!out) { fprintf(stderr, "Out of memory or error\n"); return 1; } + fputs(out, stdout); + free(out); + return 0; +} +#endif diff --git a/root/opt/mailstats/css/mailstats.css b/root/opt/mailstats/css/mailstats.css index fec72c4..9634386 100644 --- a/root/opt/mailstats/css/mailstats.css +++ b/root/opt/mailstats/css/mailstats.css @@ -207,4 +207,101 @@ p.cssvalid,p.htmlvalid {float:left;margin-right:20px} .maindiv {width:100%;overflow-x:auto;font-size:1cqw} .traffictable {border-collapse:collapse;width:98%} .divseeinbrowser{text-align:center;} -.bordercollapse{border-collapse:collapse;} \ No newline at end of file +.bordercollapse{border-collapse:collapse;} + +/* ============================================== + Summary Logs Section (scoped under .mailstats-summary) + ============================================== */ +.mailstats-summary .summary-container { + width: 100%; + overflow-x: auto; + font-size: 0.85vw; +} + +/* Table styling */ +.mailstats-summary .summary-table { + border-collapse: collapse; + width: 98%; + font-size: inherit; +} + +.mailstats-summary .summary-table th { + text-align: left; + padding: 0.5em; + border-bottom: 2px solid #ddd; + background-color: #f8f8f8; +} + +.mailstats-summary .summary-table td { + padding: 0.5em; + border-bottom: 1px solid #ddd; + word-break: break-word; /* Allows breaking long words at arbitrary points */ + overflow-wrap: break-word; /* Modern standard for breaking long words */ + hyphens: auto; /* Optionally adds hyphenation if supported */ +} + +/* Zebra striping */ +.mailstats-summary .summary-table tbody tr:nth-child(even) { + background-color: #fafafa; +} + +/* Pagination */ +.mailstats-summary .pagination { + margin-top: 1em; +} + +.mailstats-summary .pagination a { + text-decoration: none; + color: #0066cc; + padding: 0.3em 0.6em; +} + +.mailstats-summary .pagination a:hover { + text-decoration: underline; +} + +.mailstats-summary table.stripes { + border-collapse: collapse; + width: 95%; + overflow-x: auto; + margin: 0.6% auto; +} + +/* Optional zebra striping */ +.mailstats-summary table.stripes tbody tr:nth-child(even) { + background-color: #fafafa; +} + +/* ============================================== + Log Detail Page (scoped under .mailstats-detail) + ============================================== */ +.mailstats-detail .detail-container { + width: 100%; + max-width: 1200px; + margin: 1em auto; + padding: 0 1em; +} + +/* Preformatted log box */ +.mailstats-detail .log { + white-space: pre-wrap; + word-wrap: break-word; + background: #111; + color: #eee; + padding: 1em; + border-radius: 6px; + font-family: monospace, monospace; + font-size: 0.95em; + line-height: 1.4; + overflow-x: auto; +} + +/* Back link styling */ +.mailstats-detail a { + color: #0066cc; + text-decoration: none; +} + +.mailstats-detail a:hover { + text-decoration: underline; +} \ No newline at end of file diff --git a/root/opt/mailstats/html/ShowDetailedLogs.php b/root/opt/mailstats/html/ShowDetailedLogs.php index 7db5acb..f8836b4 100644 --- a/root/opt/mailstats/html/ShowDetailedLogs.php +++ b/root/opt/mailstats/html/ShowDetailedLogs.php @@ -1,51 +1,240 @@ set_charset('utf8mb4'); +} catch (mysqli_sql_exception $e) { + error_log('DB connect failed: ' . $e->getMessage()); + http_response_code(500); + exit('Service temporarily unavailable.'); +} + +// Fetch the record and extract PID from JSON logData +try { + $stmt = $conn->prepare('SELECT id, logData FROM SummaryLogs WHERE id = ?'); + $stmt->bind_param('i', $id); + $stmt->execute(); + $res = $stmt->get_result(); + $row = $res->fetch_assoc(); + $stmt->close(); +} catch (mysqli_sql_exception $e) { + error_log('Query failed: ' . $e->getMessage()); + http_response_code(500); + exit('Service temporarily unavailable.'); +} + +if (!$row) { + http_response_code(404); + exit('Record not found'); +} + +$logData = $row['logData']; +$pid = null; +$data = json_decode($logData, true, 512, JSON_INVALID_UTF8_SUBSTITUTE); +if (is_array($data)) { + foreach (['id','pid', 'PID', 'Pid', 'process_id', 'ProcessId'] as $k) { + if (isset($data[$k]) && (is_int($data[$k]) || ctype_digit((string)$data[$k]))) { + $pid = (int)$data[$k]; + break; } } - fclose($file); } -function tai64nToDate($tai64n) { - // Check if the input TAI64N string is valid - if (preg_match('/^@([0-9a-f]{8})([0-9a-f]{8})$/', $tai64n, $matches)) { - // First part: seconds since epoch - $sec_hex = $matches[1]; - // Second part: nanoseconds in hex - $nsec_hex = $matches[2]; +if (!$pid || $pid < 1) { + http_response_code(422); + exit('PID not found in this record'); +} - // Convert hex to decimal - $seconds = hexdec($sec_hex); - $nanoseconds = hexdec($nsec_hex); - - // Calculate the full timestamp in seconds - $timestamp = $seconds + ($nanoseconds / 1e9); // Nanoseconds to seconds - - // Format timestamp to 'Y-m-d H:i:s' - return date('Y-m-d H:i:s', $timestamp); - } else { - throw new InvalidArgumentException("Invalid TAI64N format."); +// Journal retrieval using C wrapper +define('FFI_LIB', 'libjournalwrap.so'); // adjust if needed +define('WRAPPER_BIN', '/usr/local/bin/journalwrap'); // fallback executable path +define('MAX_OUTPUT_BYTES', 2_000_000); // 2MB safety cap + +function getJournalByPidViaFFI(int $pid): ?string { + if (!extension_loaded('FFI')) { + return null; + } + try { + // Adjust the function signatures to match your wrapper + $ffi = FFI::cdef(" + char* journal_get_by_pid(int pid); + void journal_free(char* p); + ", FFI_LIB); + $cstr = $ffi->journal_get_by_pid($pid); + if ($cstr === null) { + return ''; + } + $out = FFI::string($cstr); + $ffi->journal_free($cstr); + return $out; + } catch (Throwable $e) { + error_log('FFI journal wrapper failed: ' . $e->getMessage()); + return null; } } -chdir($directory); -foreach ($files as $file) { - process_file($file, $input_param); + +function getJournalByPidViaExec(int $pid): ?string { + // Fallback to an external wrapper binary (must be safe and not use shell) + $cmd = WRAPPER_BIN . ' ' . (string)$pid; + + $descriptorspec = [ + 0 => ['pipe', 'r'], + 1 => ['pipe', 'w'], + 2 => ['pipe', 'w'], + ]; + $pipes = []; + $proc = proc_open($cmd, $descriptorspec, $pipes, null, null, ['bypass_shell' => true]); + + if (!\is_resource($proc)) { + error_log('Failed to start journal wrapper binary'); + return null; + } + + fclose($pipes[0]); // no stdin + + stream_set_blocking($pipes[1], false); + stream_set_blocking($pipes[2], false); + + $stdout = ''; + $stderr = ''; + $start = microtime(true); + $timeout = 10.0; // seconds + $readChunk = 65536; + + while (true) { + $status = proc_get_status($proc); + $running = $status['running']; + + $read = [$pipes[1], $pipes[2]]; + $write = null; + $except = null; + $tv_sec = 0; + $tv_usec = 300000; // 300ms + stream_select($read, $write, $except, $tv_sec, $tv_usec); + + foreach ($read as $r) { + if ($r === $pipes[1]) { + $chunk = fread($pipes[1], $readChunk); + if ($chunk !== false && $chunk !== '') { + $stdout .= $chunk; + } + } elseif ($r === $pipes[2]) { + $chunk = fread($pipes[2], $readChunk); + if ($chunk !== false && $chunk !== '') { + $stderr .= $chunk; + } + } + } + + if (!$running) { + break; + } + + if ((microtime(true) - $start) > $timeout) { + proc_terminate($proc); + $stderr .= "\n[terminated due to timeout]"; + break; + } + + if (strlen($stdout) + strlen($stderr) > MAX_OUTPUT_BYTES) { + proc_terminate($proc); + $stderr .= "\n[terminated due to output size limit]"; + break; + } + } + + foreach ($pipes as $p) { + if (is_resource($p)) { + fclose($p); + } + } + $exitCode = proc_close($proc); + + if ($exitCode !== 0 && $stderr !== '') { + error_log('journal wrapper stderr: ' . $stderr); + } + + return $stdout; } + +$logs = getJournalByPidViaFFI($pid); +if ($logs === null) { + $logs = getJournalByPidViaExec($pid); +} +if ($logs === null) { + http_response_code(500); + exit('Unable to read journal for this PID'); +} + +// Safety cap to avoid rendering gigantic outputs +if (strlen($logs) > MAX_OUTPUT_BYTES) { + $logs = substr($logs, 0, MAX_OUTPUT_BYTES) . "\n[output truncated]"; +} + +// Done with DB +$conn->close(); ?> + + + + + Log details for PID <?= e($pid) ?> (record <?= e($id) ?>) + + + +
+
+

Log details for PID (record )

+

Back

+
+
+
+ + \ No newline at end of file diff --git a/root/opt/mailstats/html/showSummaryLogs.php b/root/opt/mailstats/html/showSummaryLogs.php index 061155f..5633440 100644 --- a/root/opt/mailstats/html/showSummaryLogs.php +++ b/root/opt/mailstats/html/showSummaryLogs.php @@ -172,7 +172,7 @@ function generateLogDataTable($logData) { $keys = array_keys($mergedData); $values = array_values($mergedData); - $output = ''; + $output = '
'; // Divide keys and values into sets of 6 $chunks = array_chunk($keys, 6); @@ -202,77 +202,78 @@ function generateLogDataTable($logData) { -
-

- Summary Logs for Date: - -

- 0 ? ($offset + 1) : 0; - $endRow = min($offset + $limit, $totalRows); - ?> -

Found records. Showing –.

- -
- - - - - - - - - num_rows > 0): ?> - fetch_assoc()): ?> - - - - - - - - +
+
+

+ Summary Logs for Date: + +

+ 0 ? ($offset + 1) : 0; + $endRow = min($offset + $limit, $totalRows); + ?> +

Found records. Showing –.

+ +
IdDetailsLog Data
View details
+ - + + + + + + num_rows > 0): ?> + fetch_assoc()): ?> + + + + + + + + + + + + + +
No records found for the specified date and hour.IdDetailsLog Data
View details
No records found for the specified date and hour.
+ + $date, + 'hour' => $hour, + 'page_size' => $pageSize + ]; + $prevPage = $page > 1 ? $page - 1 : null; + $nextPage = ($offset + $limit) < $totalRows ? $page + 1 : null; + ?> + + close(); } if (isset($conn) && $conn instanceof mysqli) { $conn->close(); } ?> diff --git a/root/usr/bin/runmailstatsSME10.sh b/root/usr/bin/runmailstatsSME10.sh deleted file mode 100755 index 48d6a8a..0000000 --- a/root/usr/bin/runmailstatsSME10.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -#exec 1> >(logger -t $(basename $0)) 2>&1 -perl /usr/bin/mailstats.pl /var/log/qpsmtpd/\@* /var/log/qpsmtpd/current /var/log/sqpsmtpd/\@* /var/log/sqpsmtpd/current -# and run new python one - start by copying and decoding log files -yesterday_date=$(date -d "yesterday" +'%mm %d') -#cd /var/log/qpsmtpd -#cat \@* current >/opt/mailstats/logs/current1 2>/dev/null -#cd /var/log/sqpsmtpd -#cat \@* current >/opt/mailstats/logs/current2 2>/dev/null -cd /opt/mailstats/logs -#cat current1 current2 2>/dev/null | /usr/local/bin/tai64nlocal | grep "$yesterday_date" > current1.log -python3 /usr/bin/mailstats-convert-log-sme10-to-sme11.py -yesterday_date=$(date -d "yesterday" +'%b %d') -cat output_log.txt | grep "$yesterday_date" | sort >current.log -ls -l -python3 /usr/bin/mailstats.py -echo "Done" \ No newline at end of file diff --git a/smeserver-mailstats.spec b/smeserver-mailstats.spec index de8bbd1..51d9474 100644 --- a/smeserver-mailstats.spec +++ b/smeserver-mailstats.spec @@ -25,19 +25,24 @@ Requires: python36 # So install as: dnf install smeserver-mailstats --enablerepo=epel,smecontribs Requires: html2text Requires: python3-chameleon -Requires: python3-mysql -Requires: python3-matplotlib +Requires: python3-mysql +Requires: python3-matplotlib Requires: python3-pip AutoReqProv: no %description A script that via cron.d e-mails mail statistics to admin on a daily basis. -See http://www.contribs.org/bugzilla/show_bug.cgi?id=819 +See https://wiki.koozali.org/mailstats %changelog * Tue Sep 02 2025 Brian Read 11.1-5.sme - Speed up Journal access [SME: 13121] - Fix missing blacklist URL [SME: 13121] +- Add extra security to php show summary page [SME: 13121] +- Fix up CSS for Summary Page [SME: 13121] +- Get Detail logs page working and prettyfy [SME: 13121] +- Add in C wrapper source code to interrogate journal [SME: 13121] + * Mon Sep 01 2025 Brian Read 11.1-4.sme @@ -133,6 +138,8 @@ perl createlinks /bin/rm -rf $RPM_BUILD_ROOT (cd root ; /usr/bin/find . -depth -print | /bin/cpio -dump $RPM_BUILD_ROOT) chmod +x $RPM_BUILD_ROOT/usr/bin/runmailstats.sh +chown root:root $RPM_BUILD_ROOT/etc/mailstats/dp.php +chmod 0600 $RPM_BUILD_ROOT/etc/mailstats/db.php # Define the placeholder and generate the current date and time now=$(date +"%Y-%m-%d %H:%M:%S") @@ -146,7 +153,6 @@ sed -i "s|__BUILD_DATE_TIME__|$now|" $RPM_BUILD_ROOT/usr/bin/mailstats.py /usr/bin/pip3 install -q pymysql /usr/bin/pip3 install -q numpy /usr/bin/pip3 install -q pandas -/usr/bin/pip3 install -q plotly %clean /bin/rm -rf $RPM_BUILD_ROOT