Latest version - 19Sept2025

As running on vps.bjsystems.co.uk accessible by: https://githubpr.bjsystems.co.uk

https://githubpr.bjsystems.co.uk/healthz
https://githubpr.bjsystems.co.uk/webhook/github

webhook_endpoint.py

@@ -7,13 +7,45 @@ import logging
 import hmac
 import hashlib
 import contextlib
-from typing import Optional, Tuple, Dict, Any
+from typing import Optional, Tuple, Dict, Any, List
 
 import httpx
 from fastapi import FastAPI, Header, HTTPException, Request
 from fastapi.responses import JSONResponse, PlainTextResponse
 from github import Github, Auth
+#
+# .env contents (with keys obscured))
+#
+# WEBHOOK_SECRET=xxxxxxxxxxxxxxxxxxx
+# LOG_LEVEL=INFO
 
+# # GitHub (set GITHUB_DRY_RUN=true during early tests to avoid PR writes)
+# GITHUB_TOKEN=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+# GITHUB_DRY_RUN=false
+
+# # Bugzilla base
+# BUGZILLA_BASE_URL=https://bugs.koozali.org
+# BUGZILLA_PRODUCT=SME Server 11.x
+# BUGZILLA_VERSION=11.beta1
+# BUGZILLA_COMPONENT_DEFAULT=Github PR
+
+# # Auth: prefer API key
+# BUGZILLA_AUTH_MODE=api_key
+# BUGZILLA_API_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+
+# # (or JSON-RPC login if no API key)
+# # BUGZILLA_AUTH_MODE=basic
+# BUGZILLA_USER=githubpr
+# # BUGZILLA_PASSWORD=your_password
+
+# # Behavior
+# BUGZILLA_ATTACH_DIFF=true
+# STATE_PATH=/home/githubpr/app/webhook/state/webhook_state.sqlite
+# GITEA_BASE_URL=https://src.koozali.org
+# GITEA_ORG=smeserver
+# BUGZILLA_CC_ENABLE=true
+# BUGZILLA_CC_ALLOW_NOREPLY=false
+#
 # ---------------- Logging (UTC) ----------------
 
 class UTCFormatter(logging.Formatter):
@@ -56,19 +88,24 @@ BUGZILLA_API_KEY = os.environ.get("BUGZILLA_API_KEY", "").strip()
 BUGZILLA_USER = os.environ.get("BUGZILLA_USER", "").strip()
 BUGZILLA_PASSWORD = os.environ.get("BUGZILLA_PASSWORD", "").strip()
 
-# REQUIRED fields for bug creation
+# REQUIRED fields
 BUGZILLA_PRODUCT = os.environ.get("BUGZILLA_PRODUCT", "").strip()
 BUGZILLA_VERSION = os.environ.get("BUGZILLA_VERSION", "").strip()
 
-# Component default (as requested)
+# Component default
 BUGZILLA_COMPONENT_DEFAULT = os.environ.get("BUGZILLA_COMPONENT_DEFAULT", "e-smith-*/smeserver-* packages").strip()
-# In case we still need a fallback name for unexpected errors
 BUGZILLA_COMPONENT_FALLBACK = os.environ.get("BUGZILLA_COMPONENT_FALLBACK", BUGZILLA_COMPONENT_DEFAULT).strip()
 
 # Custom field settings
 CF_PACKAGE_FIELD = os.environ.get("BUGZILLA_CF_PACKAGE_FIELD", "cf_package").strip()
 CF_PACKAGE_FALLBACK = os.environ.get("BUGZILLA_CF_PACKAGE_FALLBACK", "---").strip()
 
+# CC behavior
+BUGZILLA_CC_ENABLE = os.environ.get("BUGZILLA_CC_ENABLE", "true").strip().lower() == "true"
+BUGZILLA_CC_ALLOW_NOREPLY = os.environ.get("BUGZILLA_CC_ALLOW_NOREPLY", "false").strip().lower() == "true"
+BUGZILLA_CC_FALLBACK_COMMENT = os.environ.get("BUGZILLA_CC_FALLBACK_COMMENT", "true").strip().lower() == "true"
+AUTHOR_EMAIL_MARKER = os.environ.get("BUGZILLA_AUTHOR_EMAIL_MARKER", "[mirror-bot:author-email]").strip()
+
 # Behavior toggles
 BUGZILLA_ATTACH_DIFF = os.environ.get("BUGZILLA_ATTACH_DIFF", "true").strip().lower() == "true"
 BUGZILLA_FAILURE_LABEL = os.environ.get("BUGZILLA_FAILURE_LABEL", "bugzilla-needed").strip()
@@ -125,7 +162,6 @@ class BugzillaHybrid:
         self.rpc_url = f"{self.base}/jsonrpc.cgi"
         self._token: Optional[str] = None  # for basic via JSON-RPC
         self._detect_mode()
-        # Log auth status
         if BUGZILLA_AUTH_MODE in ("api_key", "") and BUGZILLA_API_KEY:
             logger.info("Bugzilla auth: API key mode")
         elif BUGZILLA_AUTH_MODE == "basic" or (BUGZILLA_USER and BUGZILLA_PASSWORD):
@@ -134,7 +170,6 @@ class BugzillaHybrid:
             logger.warning("Bugzilla auth NOT configured yet (set BUGZILLA_API_KEY or BUGZILLA_USER/BUGZILLA_PASSWORD).")
 
     def _detect_mode(self):
-        # Try REST at /rest then /rest.cgi
         try:
             r = httpx.get(f"{self.base}/rest/version", timeout=10)
             if r.status_code == 200:
@@ -153,7 +188,6 @@ class BugzillaHybrid:
                 return
         except Exception:
             pass
-        # Fallback to JSON-RPC
         self.mode = "jsonrpc"
         logger.warning("Bugzilla REST not available; falling back to JSON-RPC")
 
@@ -161,7 +195,6 @@ class BugzillaHybrid:
 
     def _rest_headers(self) -> Dict[str, str]:
         h = {"Accept": "application/json", "Content-Type": "application/json"}
-        # Prefer API key if available
         if BUGZILLA_API_KEY:
             h["X-BUGZILLA-API-KEY"] = BUGZILLA_API_KEY
         return h
@@ -184,7 +217,6 @@ class BugzillaHybrid:
         return data.get("result", {})
 
     def _rpc(self, method: str, params_obj: Dict[str, Any], req_id: int = 1) -> Dict[str, Any]:
-        # Decide auth automatically
         mode = BUGZILLA_AUTH_MODE
         if (mode in ("", "api_key") and BUGZILLA_API_KEY):
             params_obj = dict(params_obj, Bugzilla_api_key=BUGZILLA_API_KEY)
@@ -231,7 +263,6 @@ class BugzillaHybrid:
             except Exception as e:
                 logger.warning(f"Bugzilla REST create failed, switching to JSON-RPC: {e}")
                 self.mode = "jsonrpc"
-        # JSON-RPC
         params = {
             "product": BUGZILLA_PRODUCT,
             "component": component or BUGZILLA_COMPONENT_DEFAULT,
@@ -291,7 +322,6 @@ class BugzillaHybrid:
         self._rpc("Bug.add_comment", {"id": bug_id, "comment": comment}, req_id=2)
 
     def update_bug_fields(self, bug_id: int, fields: Dict[str, Any]):
-        # REST first
         if self.mode == "rest":
             try:
                 url = f"{self.rest_base}/bug/{bug_id}"
@@ -303,11 +333,73 @@ class BugzillaHybrid:
             except Exception as e:
                 logger.warning(f"Bugzilla REST update failed, switching to JSON-RPC: {e}")
                 self.mode = "jsonrpc"
-        # JSON-RPC fallback
         params = {"ids": [bug_id]}
         params.update(fields)
         self._rpc("Bug.update", params, req_id=5)
 
+    def add_cc(self, bug_id: int, emails: List[str]):
+        if not emails:
+            return
+        if self.mode == "rest":
+            try:
+                url = f"{self.rest_base}/bug/{bug_id}"
+                payload = {"cc": {"add": emails}}
+                r = httpx.put(url, headers=self._rest_headers(), auth=self._rest_auth(), json=payload, timeout=30)
+                if r.status_code in (404, 405):
+                    raise httpx.HTTPStatusError("REST update not available", request=r.request, response=r)
+                r.raise_for_status()
+                return
+            except Exception as e:
+                logger.warning(f"Bugzilla REST add_cc failed, switching to JSON-RPC: {e}")
+                self.mode = "jsonrpc"
+        params = {"ids": [bug_id], "cc": {"add": emails}}
+        self._rpc("Bug.update", params, req_id=6)
+
+    def list_comments(self, bug_id: int) -> List[str]:
+        texts: List[str] = []
+        if self.mode == "rest":
+            try:
+                url = f"{self.rest_base}/bug/{bug_id}/comment"
+                r = httpx.get(url, headers=self._rest_headers(), auth=self._rest_auth(), timeout=30)
+                r.raise_for_status()
+                data = r.json()
+                bugs = data.get("bugs") or {}
+                key = str(bug_id)
+                if key in bugs:
+                    for c in (bugs[key].get("comments") or []):
+                        txt = c.get("text")
+                        if isinstance(txt, str):
+                            texts.append(txt)
+                return texts
+            except Exception as e:
+                logger.warning(f"Bugzilla REST list_comments failed, switching to JSON-RPC: {e}")
+                self.mode = "jsonrpc"
+        # JSON-RPC
+        res = self._rpc("Bug.comments", {"ids": [bug_id]}, req_id=7)
+        bugs = res.get("bugs") or {}
+        key = str(bug_id)
+        if key in bugs:
+            for c in (bugs[key].get("comments") or []):
+                txt = c.get("text")
+                if isinstance(txt, str):
+                    texts.append(txt)
+        return texts
+
+    def add_or_update_author_email_note(self, bug_id: int, email: str):
+        """Post a marker comment with the author email if not already present."""
+        try:
+            existing = self.list_comments(bug_id)
+        except Exception as e:
+            logger.warning(f"Could not list comments for bug {bug_id}: {e}")
+            existing = []
+        marker_line = f"{AUTHOR_EMAIL_MARKER} PR author email could not be added to CC (not registered): {email}"
+        # Idempotent: if an existing marker has this exact email, do nothing
+        for text in existing:
+            if AUTHOR_EMAIL_MARKER in text and email in text:
+                return
+        # Otherwise add a new note
+        self.add_comment(bug_id, marker_line)
+
 # ---------------- GitHub client (PyGithub) ----------------
 
 gh = Github(auth=Auth.Token(GITHUB_TOKEN)) if GITHUB_TOKEN else Github()
@@ -336,7 +428,6 @@ def pr_comment_success(gitea_repo_url: str, bug_id: int) -> str:
     )
 
 def pr_comment_failure(gitea_repo_url: str, repo: str, pr_url: str, target_branch: str) -> str:
-    # Uses default component name and required product/version in guidance
     return (
         "Thanks for the contribution!\n\n"
         "This repository is a read-only mirror of the canonical repo on Gitea:\n"
@@ -358,11 +449,112 @@ def fetch_pr_patch(owner: str, repo: str, pr_num: int) -> bytes:
     r.raise_for_status()
     return r.content
 
+def derive_pr_author_email(owner: str, repo: str, pr_num: int, head_sha: str) -> Optional[str]:
+    try:
+        url = f"https://api.github.com/repos/{owner}/{repo}/commits/{head_sha}"
+        headers = {"Accept": "application/vnd.github+json"}
+        if GITHUB_TOKEN:
+            headers["Authorization"] = f"token {GITHUB_TOKEN}"
+        r = httpx.get(url, headers=headers, timeout=20)
+        if r.status_code == 200:
+            data = r.json()
+            email = (data.get("commit") or {}).get("author", {}).get("email")
+            if email:
+                return email
+    except Exception:
+        pass
+    try:
+        url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_num}/commits"
+        headers = {"Accept": "application/vnd.github+json"}
+        if GITHUB_TOKEN:
+            headers["Authorization"] = f"token {GITHUB_TOKEN}"
+        r = httpx.get(url, headers=headers, timeout=20)
+        if r.status_code == 200:
+            commits = r.json()
+            if commits:
+                email = (commits[0].get("commit") or {}).get("author", {}).get("email")
+                if email:
+                    return email
+    except Exception:
+        pass
+    try:
+        patch = fetch_pr_patch(owner, repo, pr_num).decode("utf-8", "ignore")
+        for line in patch.splitlines():
+            if line.startswith("From: ") and "<" in line and ">" in line:
+                cand = line[line.find("<")+1:line.find(">", line.find("<")+1)].strip()
+                if "@" in cand:
+                    return cand
+                break
+    except Exception:
+        pass
+    return None
+
 # ---------------- Routes ----------------
 
 @app.get("/healthz")
 async def healthz():
-    return PlainTextResponse("ok")
+    gh_status = {"ok": False}
+    bz_status = {"ok": False}
+
+    # GitHub quick check: rate limit endpoint
+    try:
+        gh_headers = {"Accept": "application/vnd.github+json"}
+        if GITHUB_TOKEN:
+            gh_headers["Authorization"] = f"token {GITHUB_TOKEN}"
+        r = httpx.get("https://api.github.com/rate_limit", headers=gh_headers, timeout=5)
+        gh_status["http_status"] = r.status_code
+        if r.status_code == 200:
+            data = r.json()
+            core = (data.get("resources") or {}).get("core") or {}
+            gh_status.update({
+                "remaining": core.get("remaining"),
+                "reset": core.get("reset"),
+            })
+            gh_status["ok"] = True
+        else:
+            gh_status["error"] = r.text[:200]
+    except Exception as e:
+        gh_status["error"] = str(e)[:200]
+
+    # Bugzilla quick check: REST /version if available, else JSON-RPC Bugzilla.version
+    try:
+        if getattr(bz, "mode", None) == "rest" and getattr(bz, "rest_base", None):
+            r2 = httpx.get(f"{bz.rest_base}/version",
+                           headers=bz._rest_headers(),
+                           auth=bz._rest_auth(),
+                           timeout=5)
+            bz_status["http_status"] = r2.status_code
+            if r2.status_code == 200:
+                bz_status["version"] = (r2.json() or {}).get("version")
+                bz_status["ok"] = True
+            else:
+                bz_status["error"] = r2.text[:200]
+        else:
+            payload = {"method": "Bugzilla.version", "params": [{}], "id": 99}
+            # Include API key if configured (not strictly required for version)
+            if BUGZILLA_API_KEY:
+                payload["params"][0]["Bugzilla_api_key"] = BUGZILLA_API_KEY
+            r2 = httpx.post(bz.rpc_url,
+                            headers={"Content-Type": "application/json"},
+                            json=payload,
+                            timeout=5)
+            bz_status["http_status"] = r2.status_code
+            if r2.status_code == 200:
+                data = r2.json()
+                if not data.get("error"):
+                    bz_status["version"] = (data.get("result") or {}).get("version")
+                    bz_status["ok"] = True
+                else:
+                    bz_status["error"] = str(data["error"])[:200]
+            else:
+                bz_status["error"] = r2.text[:200]
+    except Exception as e:
+        bz_status["error"] = str(e)[:200]
+
+    overall_ok = bool(gh_status.get("ok") and bz_status.get("ok"))
+    status_code = 200 if overall_ok else 503
+    return JSONResponse({"ok": overall_ok, "github": gh_status, "bugzilla": bz_status},
+                        status_code=status_code)
 
 @app.post("/webhook/github")
 async def github_webhook(
@@ -370,7 +562,6 @@ async def github_webhook(
     x_hub_signature_256: str = Header(None),
     x_github_event: str = Header(None),
 ):
-    # Validate Bugzilla required env
     if not BUGZILLA_PRODUCT or not BUGZILLA_VERSION:
         raise HTTPException(status_code=500, detail="BUGZILLA_PRODUCT and BUGZILLA_VERSION must be set")
 
@@ -389,7 +580,7 @@ async def github_webhook(
     if action not in ("opened", "reopened", "synchronize", "ready_for_review"):
         return JSONResponse({"status": "ignored", "action": action})
 
-    repo_full = payload["repository"]["full_name"]  # owner/repo
+    repo_full = payload["repository"]["full_name"]
     owner = payload["repository"]["owner"]["login"]
     repo = payload["repository"]["name"]
     pr = payload["pull_request"]
@@ -406,7 +597,6 @@ async def github_webhook(
     pr_key = f"{repo_full}#{pr_num}"
     bug_id = state.get_bug(pr_key)
 
-    # Acquire repo object only if we will write to GitHub
     gh_repo = None
     if not GITHUB_DRY_RUN:
         try:
@@ -415,7 +605,6 @@ async def github_webhook(
             logger.warning(f"Could not access GitHub repo {repo_full}: {ge}")
 
     if bug_id is None and action in ("opened", "reopened", "ready_for_review", "synchronize"):
-        # Create Bugzilla bug
         summary = f"[GH PR #{pr_num}] {GITEA_ORG}/{repo}: {pr_title}"
         description = (
             "Source\n"
@@ -427,11 +616,10 @@ async def github_webhook(
             f"{pr_body}\n"
         )
         try:
-            # Use the requested default component
             bug_id = bz.create_bug(summary, description, component=BUGZILLA_COMPONENT_DEFAULT)
             state.set_bug(pr_key, bug_id)
             logger.info(f"Created Bugzilla bug {bug_id} for {pr_key}")
-            # Set cf_package to repo name, fallback to "---" if that fails
+            # Set cf_package to repo, fallback to '---'
             try:
                 bz.update_bug_fields(bug_id, {CF_PACKAGE_FIELD: repo})
                 logger.info(f"Set {CF_PACKAGE_FIELD}={repo} on bug {bug_id}")
@@ -440,9 +628,26 @@ async def github_webhook(
                 with contextlib.suppress(Exception):
                     bz.update_bug_fields(bug_id, {CF_PACKAGE_FIELD: CF_PACKAGE_FALLBACK})
                     logger.info(f"Set {CF_PACKAGE_FIELD}={CF_PACKAGE_FALLBACK} on bug {bug_id}")
+            # Add PR author to CC (or comment fallback)
+            if BUGZILLA_CC_ENABLE:
+                email = derive_pr_author_email(owner, repo, pr_num, head_sha)
+                if email:
+                    is_noreply = email.endswith("@users.noreply.github.com")
+                    if not is_noreply or BUGZILLA_CC_ALLOW_NOREPLY:
+                        try:
+                            bz.add_cc(bug_id, [email])
+                            logger.info(f"Added CC {email} to bug {bug_id}")
+                        except Exception as e_cc:
+                            logger.warning(f"Failed to add CC {email} to bug {bug_id}: {e_cc}")
+                            if BUGZILLA_CC_FALLBACK_COMMENT:
+                                with contextlib.suppress(Exception):
+                                    bz.add_or_update_author_email_note(bug_id, email)
+                    else:
+                        logger.info(f"Skipping noreply CC {email} (set BUGZILLA_CC_ALLOW_NOREPLY=true to allow)")
+                else:
+                    logger.info("Could not derive PR author email; CC not added")
         except Exception as e:
             logger.error(f"Bugzilla create failed for {pr_key}: {e}")
-            # Post failure comment and keep PR open (best effort)
             if not GITHUB_DRY_RUN and gh_repo is not None:
                 try:
                     gh_repo.get_pull(pr_num).create_issue_comment(pr_comment_failure(gitea_repo_url, repo, pr_url, target_branch))
@@ -457,7 +662,6 @@ async def github_webhook(
                 logger.info(f"[GITHUB_DRY_RUN] Would comment failure on PR #{pr_num}, label={BUGZILLA_FAILURE_LABEL}")
             return JSONResponse({"status": "bugzilla_failed", "action": action})
 
-        # Attach patch if enabled
         if BUGZILLA_ATTACH_DIFF:
             try:
                 patch = fetch_pr_patch(owner, repo, pr_num)
@@ -470,7 +674,6 @@ async def github_webhook(
             except Exception as e:
                 logger.warning(f"Attach patch failed for bug {bug_id}: {e}")
 
-        # Comment and close PR
         comment = pr_comment_success(gitea_repo_url, bug_id)
         if not GITHUB_DRY_RUN and gh_repo is not None:
             try:
@@ -485,7 +688,6 @@ async def github_webhook(
         return JSONResponse({"status": "ok", "bug_id": bug_id})
 
     elif bug_id is not None and action == "synchronize":
-        # Attach updated patch and ensure PR closed
         try:
             if BUGZILLA_ATTACH_DIFF:
                 patch = fetch_pr_patch(owner, repo, pr_num)