diff --git a/webhook_endpoint.py b/webhook_endpoint.py index 0abaf48..cd1b85f 100644 --- a/webhook_endpoint.py +++ b/webhook_endpoint.py @@ -7,13 +7,45 @@ import logging import hmac import hashlib import contextlib -from typing import Optional, Tuple, Dict, Any +from typing import Optional, Tuple, Dict, Any, List import httpx from fastapi import FastAPI, Header, HTTPException, Request from fastapi.responses import JSONResponse, PlainTextResponse from github import Github, Auth +# +# .env contents (with keys obscured)) +# +# WEBHOOK_SECRET=xxxxxxxxxxxxxxxxxxx +# LOG_LEVEL=INFO +# # GitHub (set GITHUB_DRY_RUN=true during early tests to avoid PR writes) +# GITHUB_TOKEN=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +# GITHUB_DRY_RUN=false + +# # Bugzilla base +# BUGZILLA_BASE_URL=https://bugs.koozali.org +# BUGZILLA_PRODUCT=SME Server 11.x +# BUGZILLA_VERSION=11.beta1 +# BUGZILLA_COMPONENT_DEFAULT=Github PR + +# # Auth: prefer API key +# BUGZILLA_AUTH_MODE=api_key +# BUGZILLA_API_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +# # (or JSON-RPC login if no API key) +# # BUGZILLA_AUTH_MODE=basic +# BUGZILLA_USER=githubpr +# # BUGZILLA_PASSWORD=your_password + +# # Behavior +# BUGZILLA_ATTACH_DIFF=true +# STATE_PATH=/home/githubpr/app/webhook/state/webhook_state.sqlite +# GITEA_BASE_URL=https://src.koozali.org +# GITEA_ORG=smeserver +# BUGZILLA_CC_ENABLE=true +# BUGZILLA_CC_ALLOW_NOREPLY=false +# # ---------------- Logging (UTC) ---------------- class UTCFormatter(logging.Formatter): @@ -56,19 +88,24 @@ BUGZILLA_API_KEY = os.environ.get("BUGZILLA_API_KEY", "").strip() BUGZILLA_USER = os.environ.get("BUGZILLA_USER", "").strip() BUGZILLA_PASSWORD = os.environ.get("BUGZILLA_PASSWORD", "").strip() -# REQUIRED fields for bug creation +# REQUIRED fields BUGZILLA_PRODUCT = os.environ.get("BUGZILLA_PRODUCT", "").strip() BUGZILLA_VERSION = os.environ.get("BUGZILLA_VERSION", "").strip() -# Component default (as requested) +# Component default BUGZILLA_COMPONENT_DEFAULT = os.environ.get("BUGZILLA_COMPONENT_DEFAULT", "e-smith-*/smeserver-* packages").strip() -# In case we still need a fallback name for unexpected errors BUGZILLA_COMPONENT_FALLBACK = os.environ.get("BUGZILLA_COMPONENT_FALLBACK", BUGZILLA_COMPONENT_DEFAULT).strip() # Custom field settings CF_PACKAGE_FIELD = os.environ.get("BUGZILLA_CF_PACKAGE_FIELD", "cf_package").strip() CF_PACKAGE_FALLBACK = os.environ.get("BUGZILLA_CF_PACKAGE_FALLBACK", "---").strip() +# CC behavior +BUGZILLA_CC_ENABLE = os.environ.get("BUGZILLA_CC_ENABLE", "true").strip().lower() == "true" +BUGZILLA_CC_ALLOW_NOREPLY = os.environ.get("BUGZILLA_CC_ALLOW_NOREPLY", "false").strip().lower() == "true" +BUGZILLA_CC_FALLBACK_COMMENT = os.environ.get("BUGZILLA_CC_FALLBACK_COMMENT", "true").strip().lower() == "true" +AUTHOR_EMAIL_MARKER = os.environ.get("BUGZILLA_AUTHOR_EMAIL_MARKER", "[mirror-bot:author-email]").strip() + # Behavior toggles BUGZILLA_ATTACH_DIFF = os.environ.get("BUGZILLA_ATTACH_DIFF", "true").strip().lower() == "true" BUGZILLA_FAILURE_LABEL = os.environ.get("BUGZILLA_FAILURE_LABEL", "bugzilla-needed").strip() @@ -125,7 +162,6 @@ class BugzillaHybrid: self.rpc_url = f"{self.base}/jsonrpc.cgi" self._token: Optional[str] = None # for basic via JSON-RPC self._detect_mode() - # Log auth status if BUGZILLA_AUTH_MODE in ("api_key", "") and BUGZILLA_API_KEY: logger.info("Bugzilla auth: API key mode") elif BUGZILLA_AUTH_MODE == "basic" or (BUGZILLA_USER and BUGZILLA_PASSWORD): @@ -134,7 +170,6 @@ class BugzillaHybrid: logger.warning("Bugzilla auth NOT configured yet (set BUGZILLA_API_KEY or BUGZILLA_USER/BUGZILLA_PASSWORD).") def _detect_mode(self): - # Try REST at /rest then /rest.cgi try: r = httpx.get(f"{self.base}/rest/version", timeout=10) if r.status_code == 200: @@ -153,7 +188,6 @@ class BugzillaHybrid: return except Exception: pass - # Fallback to JSON-RPC self.mode = "jsonrpc" logger.warning("Bugzilla REST not available; falling back to JSON-RPC") @@ -161,7 +195,6 @@ class BugzillaHybrid: def _rest_headers(self) -> Dict[str, str]: h = {"Accept": "application/json", "Content-Type": "application/json"} - # Prefer API key if available if BUGZILLA_API_KEY: h["X-BUGZILLA-API-KEY"] = BUGZILLA_API_KEY return h @@ -184,7 +217,6 @@ class BugzillaHybrid: return data.get("result", {}) def _rpc(self, method: str, params_obj: Dict[str, Any], req_id: int = 1) -> Dict[str, Any]: - # Decide auth automatically mode = BUGZILLA_AUTH_MODE if (mode in ("", "api_key") and BUGZILLA_API_KEY): params_obj = dict(params_obj, Bugzilla_api_key=BUGZILLA_API_KEY) @@ -231,7 +263,6 @@ class BugzillaHybrid: except Exception as e: logger.warning(f"Bugzilla REST create failed, switching to JSON-RPC: {e}") self.mode = "jsonrpc" - # JSON-RPC params = { "product": BUGZILLA_PRODUCT, "component": component or BUGZILLA_COMPONENT_DEFAULT, @@ -291,7 +322,6 @@ class BugzillaHybrid: self._rpc("Bug.add_comment", {"id": bug_id, "comment": comment}, req_id=2) def update_bug_fields(self, bug_id: int, fields: Dict[str, Any]): - # REST first if self.mode == "rest": try: url = f"{self.rest_base}/bug/{bug_id}" @@ -303,11 +333,73 @@ class BugzillaHybrid: except Exception as e: logger.warning(f"Bugzilla REST update failed, switching to JSON-RPC: {e}") self.mode = "jsonrpc" - # JSON-RPC fallback params = {"ids": [bug_id]} params.update(fields) self._rpc("Bug.update", params, req_id=5) + def add_cc(self, bug_id: int, emails: List[str]): + if not emails: + return + if self.mode == "rest": + try: + url = f"{self.rest_base}/bug/{bug_id}" + payload = {"cc": {"add": emails}} + r = httpx.put(url, headers=self._rest_headers(), auth=self._rest_auth(), json=payload, timeout=30) + if r.status_code in (404, 405): + raise httpx.HTTPStatusError("REST update not available", request=r.request, response=r) + r.raise_for_status() + return + except Exception as e: + logger.warning(f"Bugzilla REST add_cc failed, switching to JSON-RPC: {e}") + self.mode = "jsonrpc" + params = {"ids": [bug_id], "cc": {"add": emails}} + self._rpc("Bug.update", params, req_id=6) + + def list_comments(self, bug_id: int) -> List[str]: + texts: List[str] = [] + if self.mode == "rest": + try: + url = f"{self.rest_base}/bug/{bug_id}/comment" + r = httpx.get(url, headers=self._rest_headers(), auth=self._rest_auth(), timeout=30) + r.raise_for_status() + data = r.json() + bugs = data.get("bugs") or {} + key = str(bug_id) + if key in bugs: + for c in (bugs[key].get("comments") or []): + txt = c.get("text") + if isinstance(txt, str): + texts.append(txt) + return texts + except Exception as e: + logger.warning(f"Bugzilla REST list_comments failed, switching to JSON-RPC: {e}") + self.mode = "jsonrpc" + # JSON-RPC + res = self._rpc("Bug.comments", {"ids": [bug_id]}, req_id=7) + bugs = res.get("bugs") or {} + key = str(bug_id) + if key in bugs: + for c in (bugs[key].get("comments") or []): + txt = c.get("text") + if isinstance(txt, str): + texts.append(txt) + return texts + + def add_or_update_author_email_note(self, bug_id: int, email: str): + """Post a marker comment with the author email if not already present.""" + try: + existing = self.list_comments(bug_id) + except Exception as e: + logger.warning(f"Could not list comments for bug {bug_id}: {e}") + existing = [] + marker_line = f"{AUTHOR_EMAIL_MARKER} PR author email could not be added to CC (not registered): {email}" + # Idempotent: if an existing marker has this exact email, do nothing + for text in existing: + if AUTHOR_EMAIL_MARKER in text and email in text: + return + # Otherwise add a new note + self.add_comment(bug_id, marker_line) + # ---------------- GitHub client (PyGithub) ---------------- gh = Github(auth=Auth.Token(GITHUB_TOKEN)) if GITHUB_TOKEN else Github() @@ -336,7 +428,6 @@ def pr_comment_success(gitea_repo_url: str, bug_id: int) -> str: ) def pr_comment_failure(gitea_repo_url: str, repo: str, pr_url: str, target_branch: str) -> str: - # Uses default component name and required product/version in guidance return ( "Thanks for the contribution!\n\n" "This repository is a read-only mirror of the canonical repo on Gitea:\n" @@ -358,11 +449,112 @@ def fetch_pr_patch(owner: str, repo: str, pr_num: int) -> bytes: r.raise_for_status() return r.content +def derive_pr_author_email(owner: str, repo: str, pr_num: int, head_sha: str) -> Optional[str]: + try: + url = f"https://api.github.com/repos/{owner}/{repo}/commits/{head_sha}" + headers = {"Accept": "application/vnd.github+json"} + if GITHUB_TOKEN: + headers["Authorization"] = f"token {GITHUB_TOKEN}" + r = httpx.get(url, headers=headers, timeout=20) + if r.status_code == 200: + data = r.json() + email = (data.get("commit") or {}).get("author", {}).get("email") + if email: + return email + except Exception: + pass + try: + url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_num}/commits" + headers = {"Accept": "application/vnd.github+json"} + if GITHUB_TOKEN: + headers["Authorization"] = f"token {GITHUB_TOKEN}" + r = httpx.get(url, headers=headers, timeout=20) + if r.status_code == 200: + commits = r.json() + if commits: + email = (commits[0].get("commit") or {}).get("author", {}).get("email") + if email: + return email + except Exception: + pass + try: + patch = fetch_pr_patch(owner, repo, pr_num).decode("utf-8", "ignore") + for line in patch.splitlines(): + if line.startswith("From: ") and "<" in line and ">" in line: + cand = line[line.find("<")+1:line.find(">", line.find("<")+1)].strip() + if "@" in cand: + return cand + break + except Exception: + pass + return None + # ---------------- Routes ---------------- @app.get("/healthz") async def healthz(): - return PlainTextResponse("ok") + gh_status = {"ok": False} + bz_status = {"ok": False} + + # GitHub quick check: rate limit endpoint + try: + gh_headers = {"Accept": "application/vnd.github+json"} + if GITHUB_TOKEN: + gh_headers["Authorization"] = f"token {GITHUB_TOKEN}" + r = httpx.get("https://api.github.com/rate_limit", headers=gh_headers, timeout=5) + gh_status["http_status"] = r.status_code + if r.status_code == 200: + data = r.json() + core = (data.get("resources") or {}).get("core") or {} + gh_status.update({ + "remaining": core.get("remaining"), + "reset": core.get("reset"), + }) + gh_status["ok"] = True + else: + gh_status["error"] = r.text[:200] + except Exception as e: + gh_status["error"] = str(e)[:200] + + # Bugzilla quick check: REST /version if available, else JSON-RPC Bugzilla.version + try: + if getattr(bz, "mode", None) == "rest" and getattr(bz, "rest_base", None): + r2 = httpx.get(f"{bz.rest_base}/version", + headers=bz._rest_headers(), + auth=bz._rest_auth(), + timeout=5) + bz_status["http_status"] = r2.status_code + if r2.status_code == 200: + bz_status["version"] = (r2.json() or {}).get("version") + bz_status["ok"] = True + else: + bz_status["error"] = r2.text[:200] + else: + payload = {"method": "Bugzilla.version", "params": [{}], "id": 99} + # Include API key if configured (not strictly required for version) + if BUGZILLA_API_KEY: + payload["params"][0]["Bugzilla_api_key"] = BUGZILLA_API_KEY + r2 = httpx.post(bz.rpc_url, + headers={"Content-Type": "application/json"}, + json=payload, + timeout=5) + bz_status["http_status"] = r2.status_code + if r2.status_code == 200: + data = r2.json() + if not data.get("error"): + bz_status["version"] = (data.get("result") or {}).get("version") + bz_status["ok"] = True + else: + bz_status["error"] = str(data["error"])[:200] + else: + bz_status["error"] = r2.text[:200] + except Exception as e: + bz_status["error"] = str(e)[:200] + + overall_ok = bool(gh_status.get("ok") and bz_status.get("ok")) + status_code = 200 if overall_ok else 503 + return JSONResponse({"ok": overall_ok, "github": gh_status, "bugzilla": bz_status}, + status_code=status_code) @app.post("/webhook/github") async def github_webhook( @@ -370,7 +562,6 @@ async def github_webhook( x_hub_signature_256: str = Header(None), x_github_event: str = Header(None), ): - # Validate Bugzilla required env if not BUGZILLA_PRODUCT or not BUGZILLA_VERSION: raise HTTPException(status_code=500, detail="BUGZILLA_PRODUCT and BUGZILLA_VERSION must be set") @@ -389,7 +580,7 @@ async def github_webhook( if action not in ("opened", "reopened", "synchronize", "ready_for_review"): return JSONResponse({"status": "ignored", "action": action}) - repo_full = payload["repository"]["full_name"] # owner/repo + repo_full = payload["repository"]["full_name"] owner = payload["repository"]["owner"]["login"] repo = payload["repository"]["name"] pr = payload["pull_request"] @@ -406,7 +597,6 @@ async def github_webhook( pr_key = f"{repo_full}#{pr_num}" bug_id = state.get_bug(pr_key) - # Acquire repo object only if we will write to GitHub gh_repo = None if not GITHUB_DRY_RUN: try: @@ -415,7 +605,6 @@ async def github_webhook( logger.warning(f"Could not access GitHub repo {repo_full}: {ge}") if bug_id is None and action in ("opened", "reopened", "ready_for_review", "synchronize"): - # Create Bugzilla bug summary = f"[GH PR #{pr_num}] {GITEA_ORG}/{repo}: {pr_title}" description = ( "Source\n" @@ -427,11 +616,10 @@ async def github_webhook( f"{pr_body}\n" ) try: - # Use the requested default component bug_id = bz.create_bug(summary, description, component=BUGZILLA_COMPONENT_DEFAULT) state.set_bug(pr_key, bug_id) logger.info(f"Created Bugzilla bug {bug_id} for {pr_key}") - # Set cf_package to repo name, fallback to "---" if that fails + # Set cf_package to repo, fallback to '---' try: bz.update_bug_fields(bug_id, {CF_PACKAGE_FIELD: repo}) logger.info(f"Set {CF_PACKAGE_FIELD}={repo} on bug {bug_id}") @@ -440,9 +628,26 @@ async def github_webhook( with contextlib.suppress(Exception): bz.update_bug_fields(bug_id, {CF_PACKAGE_FIELD: CF_PACKAGE_FALLBACK}) logger.info(f"Set {CF_PACKAGE_FIELD}={CF_PACKAGE_FALLBACK} on bug {bug_id}") + # Add PR author to CC (or comment fallback) + if BUGZILLA_CC_ENABLE: + email = derive_pr_author_email(owner, repo, pr_num, head_sha) + if email: + is_noreply = email.endswith("@users.noreply.github.com") + if not is_noreply or BUGZILLA_CC_ALLOW_NOREPLY: + try: + bz.add_cc(bug_id, [email]) + logger.info(f"Added CC {email} to bug {bug_id}") + except Exception as e_cc: + logger.warning(f"Failed to add CC {email} to bug {bug_id}: {e_cc}") + if BUGZILLA_CC_FALLBACK_COMMENT: + with contextlib.suppress(Exception): + bz.add_or_update_author_email_note(bug_id, email) + else: + logger.info(f"Skipping noreply CC {email} (set BUGZILLA_CC_ALLOW_NOREPLY=true to allow)") + else: + logger.info("Could not derive PR author email; CC not added") except Exception as e: logger.error(f"Bugzilla create failed for {pr_key}: {e}") - # Post failure comment and keep PR open (best effort) if not GITHUB_DRY_RUN and gh_repo is not None: try: gh_repo.get_pull(pr_num).create_issue_comment(pr_comment_failure(gitea_repo_url, repo, pr_url, target_branch)) @@ -457,7 +662,6 @@ async def github_webhook( logger.info(f"[GITHUB_DRY_RUN] Would comment failure on PR #{pr_num}, label={BUGZILLA_FAILURE_LABEL}") return JSONResponse({"status": "bugzilla_failed", "action": action}) - # Attach patch if enabled if BUGZILLA_ATTACH_DIFF: try: patch = fetch_pr_patch(owner, repo, pr_num) @@ -470,7 +674,6 @@ async def github_webhook( except Exception as e: logger.warning(f"Attach patch failed for bug {bug_id}: {e}") - # Comment and close PR comment = pr_comment_success(gitea_repo_url, bug_id) if not GITHUB_DRY_RUN and gh_repo is not None: try: @@ -485,7 +688,6 @@ async def github_webhook( return JSONResponse({"status": "ok", "bug_id": bug_id}) elif bug_id is not None and action == "synchronize": - # Attach updated patch and ensure PR closed try: if BUGZILLA_ATTACH_DIFF: patch = fetch_pr_patch(owner, repo, pr_num)