github2gerrit 0.1.10__py3-none-any.whl → 0.1.11__py3-none-any.whl

github2gerrit/duplicate_detection.py

@@ -10,7 +10,6 @@ submissions from automated tools like Dependabot.
 """
 
 import hashlib
-import json
 import logging
 import os
 import re
@@ -22,6 +21,7 @@ from datetime import UTC
 from datetime import datetime
 from datetime import timedelta
 from pathlib import Path
+from typing import Any
 
 from .gerrit_urls import create_gerrit_url_builder
 from .github_api import GhPullRequest
@@ -29,6 +29,7 @@ from .github_api import GhRepository
 from .github_api import build_client
 from .github_api import get_repo_from_env
 from .models import GitHubContext
+from .trailers import extract_github_metadata
 
 
 # Optional Gerrit REST API support
@@ -53,15 +54,23 @@ __all__ = [
 class DuplicateChangeError(Exception):
     """Raised when a duplicate change is detected."""
 
-    def __init__(self, message: str, existing_prs: list[int]) -> None:
+    def __init__(
+        self,
+        message: str,
+        existing_prs: list[int],
+        urls: list[str] | None = None,
+    ) -> None:
         super().__init__(message)
         self.existing_prs = existing_prs
+        self.urls = urls or []
 
 
 class ChangeFingerprint:
     """Represents a fingerprint of a change for duplicate detection."""
 
-    def __init__(self, title: str, body: str = "", files_changed: list[str] | None = None):
+    def __init__(
+        self, title: str, body: str = "", files_changed: list[str] | None = None
+    ):
         self.title = title.strip()
         self.body = (body or "").strip()
         self.files_changed = sorted(files_changed or [])
@@ -102,10 +111,15 @@ class ChangeFingerprint:
 
     def _compute_content_hash(self) -> str:
         """Compute a hash of the change content."""
-        content = f"{self._normalized_title}\n{self.body}\n{','.join(self.files_changed)}"
+        content = (
+            f"{self._normalized_title}\n{self.body}\n"
+            f"{','.join(self.files_changed)}"
+        )
         return hashlib.sha256(content.encode("utf-8")).hexdigest()[:16]
 
-    def is_similar_to(self, other: "ChangeFingerprint", similarity_threshold: float = 0.8) -> bool:
+    def is_similar_to(
+        self, other: "ChangeFingerprint", similarity_threshold: float = 0.8
+    ) -> bool:
         """Check if this fingerprint is similar to another."""
         # Exact normalized title match
         if self._normalized_title == other._normalized_title:
@@ -128,7 +142,9 @@ class ChangeFingerprint:
         # Check title similarity even without file changes
         return self._titles_similar(other, similarity_threshold)
 
-    def _titles_similar(self, other: "ChangeFingerprint", threshold: float) -> bool:
+    def _titles_similar(
+        self, other: "ChangeFingerprint", threshold: float
+    ) -> bool:
         """Check if titles are similar using simple string similarity."""
         title1 = self._normalized_title
         title2 = other._normalized_title
@@ -149,7 +165,10 @@ class ChangeFingerprint:
         return (intersection / union) >= threshold
 
     def __str__(self) -> str:
-        return f"ChangeFingerprint(title='{self.title[:50]}...', hash={self._content_hash})"
+        return (
+            f"ChangeFingerprint(title='{self.title[:50]}...', "
+            f"hash={self._content_hash})"
+        )
 
 
 class DuplicateDetector:
@@ -171,7 +190,9 @@ class DuplicateDetector:
         match = re.search(pattern, text)
         return match.group(1) if match else ""
 
-    def _resolve_gerrit_info_from_env_or_gitreview(self, gh: GitHubContext) -> tuple[str, str] | None:
+    def _resolve_gerrit_info_from_env_or_gitreview(
+        self, gh: GitHubContext
+    ) -> tuple[str, str] | None:
         """Resolve Gerrit host and project from environment or .gitreview file.
 
         Returns:
@@ -220,7 +241,10 @@ class DuplicateDetector:
                 url = f"https://raw.githubusercontent.com/{repo_full}/{branch}/.gitreview"
 
                 parsed = urllib.parse.urlparse(url)
-                if parsed.scheme != "https" or parsed.netloc != "raw.githubusercontent.com":
+                if (
+                    parsed.scheme != "https"
+                    or parsed.netloc != "raw.githubusercontent.com"
+                ):
                     continue
 
                 try:
@@ -228,8 +252,12 @@ class DuplicateDetector:
                     with urllib.request.urlopen(url, timeout=5) as resp:
                         text_remote = resp.read().decode("utf-8")
 
-                    host = self._match_first_group(r"(?m)^host=(.+)$", text_remote)
-                    proj = self._match_first_group(r"(?m)^project=(.+)$", text_remote)
+                    host = self._match_first_group(
+                        r"(?m)^host=(.+)$", text_remote
+                    )
+                    proj = self._match_first_group(
+                        r"(?m)^project=(.+)$", text_remote
+                    )
 
                     if host and proj:
                         project = proj.removesuffix(".git")
@@ -238,7 +266,9 @@ class DuplicateDetector:
                         return (host.strip(), "")
 
                 except Exception as exc:
-                    log.debug("Failed to fetch .gitreview from %s: %s", url, exc)
+                    log.debug(
+                        "Failed to fetch .gitreview from %s: %s", url, exc
+                    )
                     continue
 
         except Exception as exc:
@@ -246,31 +276,24 @@ class DuplicateDetector:
 
         return None
 
-    def _build_gerrit_rest_client(self, gerrit_host: str) -> object | None:
-        """Build a Gerrit REST API client if pygerrit2 is available."""
-        if GerritRestAPI is None:
-            log.debug("pygerrit2 not available, skipping Gerrit duplicate check")
-            return None
+    def _build_gerrit_rest_client(self, gerrit_host: str) -> Any | None:
+        """Build a Gerrit REST API client using centralized framework."""
+        from .gerrit_rest import build_client_for_host
 
-        # Create centralized URL builder
-        url_builder = create_gerrit_url_builder(gerrit_host)
-        base_url = url_builder.api_url()
-
-        http_user = os.getenv("GERRIT_HTTP_USER", "").strip() or os.getenv("GERRIT_SSH_USER_G2G", "").strip()
+        http_user = (
+            os.getenv("GERRIT_HTTP_USER", "").strip()
+            or os.getenv("GERRIT_SSH_USER_G2G", "").strip()
+        )
         http_pass = os.getenv("GERRIT_HTTP_PASSWORD", "").strip()
 
         try:
-            if http_user and http_pass:
-                if HTTPBasicAuth is None:
-                    log.debug("pygerrit2 HTTPBasicAuth not available")
-                    return None
-                # Type ignore needed for dynamic import returning Any
-                return GerritRestAPI(  # type: ignore[no-any-return]
-                    url=base_url, auth=HTTPBasicAuth(http_user, http_pass)
-                )
-            else:
-                # Type ignore needed for dynamic import returning Any
-                return GerritRestAPI(url=base_url)  # type: ignore[no-any-return]
+            return build_client_for_host(
+                gerrit_host,
+                timeout=8.0,
+                max_attempts=3,
+                http_user=http_user or None,
+                http_password=http_pass or None,
+            )
         except Exception as exc:
             log.debug("Failed to create Gerrit REST client: %s", exc)
             return None
@@ -299,7 +322,9 @@ class DuplicateDetector:
         hash_bytes = hashlib.sha256(hash_input.encode("utf-8")).digest()
         hash_hex = hash_bytes.hex()[:16]
 
-        log.debug("Generated GitHub change hash for %s: %s", hash_input, hash_hex)
+        log.debug(
+            "Generated GitHub change hash for %s: %s", hash_input, hash_hex
+        )
         return hash_hex
 
     def check_for_duplicates(
@@ -307,25 +332,41 @@ class DuplicateDetector:
         target_pr: GhPullRequest,
         allow_duplicates: bool = False,
         gh: GitHubContext | None = None,
+        expected_github_hash: str | None = None,
     ) -> None:
-        """Check if the target PR is a duplicate via subject equality against Gerrit.
+        """Check if the target PR is a duplicate via trailer-aware and subject
+        equality against Gerrit.
 
-        Implements a robust, dependency-free subject-first duplicate check:
+        Implements a robust, dependency-free duplicate check with trailer
+        awareness:
+        - First check for existing changes with matching GitHub-Hash trailer
+          (short-circuit)
         - Resolve Gerrit host/project from env or .gitreview
-        - Query Gerrit changes updated within the lookback window (excluding abandoned)
+        - Query Gerrit changes updated within the lookback window (excluding
+          abandoned)
         - Compare normalized subjects (first line) for exact equality
         - If any match, treat as duplicate and either warn or raise
+
+        Args:
+            target_pr: The GitHub PR to check for duplicates
+            allow_duplicates: If True, log warnings instead of raising errors
+            gh: GitHub context for resolving Gerrit configuration
+            expected_github_hash: The GitHub-Hash trailer value expected for
+                this PR
         """
         pr_number = getattr(target_pr, "number", 0)
         pr_title = (getattr(target_pr, "title", "") or "").strip()
 
         log.debug(
-            "Checking PR #%d for duplicates via subject equality against Gerrit",
+            "Checking PR #%d for duplicates via subject equality against "
+            "Gerrit",
             pr_number,
         )
 
         if not pr_title:
-            log.debug("PR #%d has empty title; skipping duplicate check", pr_number)
+            log.debug(
+                "PR #%d has empty title; skipping duplicate check", pr_number
+            )
             return
         if gh is None:
             log.debug("No GitHub context provided; skipping duplicate check")
@@ -334,7 +375,10 @@ class DuplicateDetector:
         # Resolve Gerrit target (host/project)
         gerrit_info = self._resolve_gerrit_info_from_env_or_gitreview(gh)
         if not gerrit_info:
-            log.debug("Unable to resolve Gerrit host/project; skipping duplicate check")
+            log.debug(
+                "Unable to resolve Gerrit host/project; skipping duplicate "
+                "check"
+            )
             return
         gerrit_host, gerrit_project = gerrit_info
 
@@ -342,16 +386,21 @@ class DuplicateDetector:
         def _normalize_subject(title: str) -> str:
             normalized = title.strip()
             normalized = re.sub(
-                r"^(feat|fix|docs|style|refactor|test|chore|ci|build|perf)(\(.+?\))?: ",
+                r"^(feat|fix|docs|style|refactor|test|chore|ci|build|perf)"
+                r"(\(.+?\))?: ",
                 "",
                 normalized,
                 flags=re.IGNORECASE,
             )
             normalized = re.sub(r"[*_`]", "", normalized)
-            normalized = re.sub(r"\bv\d+(\.\d+)*(-\w+)?\b", "vx.y.z", normalized)
+            normalized = re.sub(
+                r"\bv\d+(\.\d+)*(-\w+)?\b", "vx.y.z", normalized
+            )
             normalized = re.sub(r"\b\d+(\.\d+)+(-\w+)?\b", "x.y.z", normalized)
             normalized = re.sub(r"\b\d+\.\d+\b", "x.y.z", normalized)
-            normalized = re.sub(r"\b[a-f0-9]{7,40}\b", "commit_hash", normalized)
+            normalized = re.sub(
+                r"\b[a-f0-9]{7,40}\b", "commit_hash", normalized
+            )
             normalized = re.sub(r"\s+", " ", normalized).strip()
             return normalized.lower()
 
@@ -363,7 +412,6 @@ class DuplicateDetector:
 
         # Build Gerrit REST URL using centralized URL builder
         url_builder = create_gerrit_url_builder(gerrit_host)
-        api_base = url_builder.api_url().rstrip("/")
 
         # Track which base path actually works for constructing display URLs
         successful_base_path = url_builder.base_path
@@ -393,64 +441,99 @@ class DuplicateDetector:
         query = " ".join(q_parts)
         encoded_q = urllib.parse.quote(query, safe="")
 
-        # Request current commit metadata so we get 'subject'
-        # Use a modest page size
-        url = f"{api_base}/changes/?q={encoded_q}&n=50&o=CURRENT_COMMIT&o=CURRENT_FILES"
-
-        def _load_gerrit_json(url_: str) -> list[dict[str, object]]:
+        def _load_gerrit_json(query_path: str) -> list[dict[str, object]]:
             try:
-                log.debug("Querying Gerrit for duplicates: %s", url_)
-                # Ensure we only fetch over HTTPS to avoid unsafe schemes
-                parsed = urllib.parse.urlparse(url_)
-                if parsed.scheme != "https":
-                    log.debug("Skipping non-HTTPS URL for Gerrit query: %s", url_)
+                # Use centralized client that handles base path and auth
+                client = self._build_gerrit_rest_client(gerrit_host)
+                if client is None:
+                    log.debug(
+                        "Gerrit client not available; skipping duplicate check"
+                    )
                     return []
-                headers: dict[str, str] = {}
-                http_user = os.getenv("GERRIT_HTTP_USER", "").strip()
-                http_pass = os.getenv("GERRIT_HTTP_PASSWORD", "").strip()
-                if http_user and http_pass:
-                    import base64 as _b64  # localized import to avoid global import edit
-
-                    basic = _b64.b64encode(f"{http_user}:{http_pass}".encode()).decode("ascii")
-                    headers["Authorization"] = f"Basic {basic}"
-                req = urllib.request.Request(url_, headers=headers)
-                with urllib.request.urlopen(req, timeout=8) as resp:
-                    raw = resp.read().decode("utf-8", errors="replace")
-                # Strip Gerrit's XSSI prefix if present
-                if raw.startswith(")]}'"):
-                    raw = raw.split("\n", 1)[1] if "\n" in raw else ""
-                data = json.loads(raw or "[]")
+
+                log.debug("Querying Gerrit for duplicates: %s", query_path)
+                data = client.get(query_path)
                 if isinstance(data, list):
                     return data
                 else:
                     return []
-            except urllib.error.HTTPError as exc:
-                log.debug("Gerrit query failed for %s: %s", url_, exc)
-                return []
             except Exception as exc:
-                log.debug("Gerrit query failed for %s: %s", url_, exc)
+                log.debug("Gerrit query failed for %s: %s", query_path, exc)
                 return []
 
+        # Build query path for centralized client
+        query_path = (
+            f"/changes/?q={encoded_q}&n=50&o=CURRENT_COMMIT&o=CURRENT_FILES"
+        )
+
         log.debug(
-            "Gerrit duplicate query: host=%s project=%s filter=%s cutoff=%s url=%s",
+            "Gerrit duplicate query: host=%s project=%s filter=%s cutoff=%s "
+            "path=%s",
             gerrit_host,
             gerrit_project or "(any)",
             dup_filter,
             cutoff_date,
-            url,
+            query_path,
         )
-        changes = _load_gerrit_json(url)
+        changes = _load_gerrit_json(query_path)
         log.debug(
-            "Gerrit query returned %d change(s) for project=%s filter=%s after=%s",
+            "Gerrit query returned %d change(s) for project=%s filter=%s "
+            "after=%s",
             len(changes),
             gerrit_project or "(any)",
             dup_filter,
             cutoff_date,
         )
         if changes:
-            sample_subjects = ", ".join(str(c.get("subject") or "")[:60] for c in changes[:5])
+            sample_subjects = ", ".join(
+                str(c.get("subject") or "")[:60] for c in changes[:5]
+            )
             log.debug("Sample subjects: %s", sample_subjects)
 
+        # First pass: Check for trailer-based matches (GitHub-Hash)
+        if expected_github_hash:
+            log.debug(
+                "Checking for GitHub-Hash trailer matches: %s",
+                expected_github_hash,
+            )
+            trailer_matches: list[tuple[int, str]] = []
+
+            for c in changes:
+                # Extract commit message and check for GitHub trailers
+                rev = str(c.get("current_revision") or "")
+                revs_obj = c.get("revisions")
+                revs = revs_obj if isinstance(revs_obj, dict) else {}
+                cur_obj = revs.get(rev)
+                cur = cur_obj if isinstance(cur_obj, dict) else {}
+                commit = cur.get("commit") or {}
+                msg = str(commit.get("message") or "")
+
+                if msg:
+                    github_metadata = extract_github_metadata(msg)
+                    change_github_hash = github_metadata.get("GitHub-Hash", "")
+
+                    if change_github_hash == expected_github_hash:
+                        num = c.get("_number")
+                        proj = str(c.get("project") or gerrit_project or "")
+                        if isinstance(num, int):
+                            trailer_matches.append((num, proj))
+                            log.debug(
+                                "Found GitHub-Hash trailer match: change %d, "
+                                "hash %s",
+                                num,
+                                change_github_hash,
+                            )
+
+            if trailer_matches:
+                log.debug(
+                    "Found %d change(s) with matching GitHub-Hash trailer - "
+                    "treating as update targets",
+                    len(trailer_matches),
+                )
+                # These are update targets, not duplicates - allow them to
+                # proceed
+                return
+
         # Compare normalized subjects for exact equality
         matched: list[tuple[int, str]] = []
         for c in changes:
@@ -464,8 +547,11 @@ class DuplicateDetector:
                     matched.append((num, proj))
 
         if not matched:
-            # No exact subject match; proceed with similarity scoring across candidates
-            log.debug("No exact-subject matches found; entering similarity scoring")
+            # No exact subject match; proceed with similarity scoring across
+            # candidates
+            log.debug(
+                "No exact-subject matches found; entering similarity scoring"
+            )
             from .similarity import ScoringConfig
             from .similarity import aggregate_scores
             from .similarity import remove_commit_trailers
@@ -488,7 +574,8 @@ class DuplicateDetector:
                             if fname:
                                 src_files.append(str(fname))
             except Exception as exc:
-                # Best-effort; if files cannot be retrieved, proceed without them
+                # Best-effort; if files cannot be retrieved, proceed without
+                # them
                 log.debug("Failed to retrieve PR files for scoring: %s", exc)
 
             best_score = 0.0
@@ -499,7 +586,8 @@ class DuplicateDetector:
                 subj = str(c.get("subject") or "").strip()
                 if not subj:
                     continue
-                # Extract commit message and files from revisions (CURRENT_COMMIT, CURRENT_FILES)
+                # Extract commit message and files from revisions
+                # (CURRENT_COMMIT, CURRENT_FILES)
                 rev = str(c.get("current_revision") or "")
                 revs_obj = c.get("revisions")
                 revs = revs_obj if isinstance(revs_obj, dict) else {}
@@ -512,7 +600,11 @@ class DuplicateDetector:
                     cand_body_raw = msg.split("\n", 1)[1]
                 cand_body = remove_commit_trailers(cand_body_raw)
                 files_dict = cur.get("files") or {}
-                cand_files = [p for p in files_dict if isinstance(p, str) and not p.startswith("/")]
+                cand_files = [
+                    p
+                    for p in files_dict
+                    if isinstance(p, str) and not p.startswith("/")
+                ]
 
                 # Compute component scores
                 s_res = score_subjects(src_subjects, subj)
@@ -524,7 +616,9 @@ class DuplicateDetector:
                 b_res = score_bodies(src_body, cand_body)
 
                 # Aggregate
-                agg = aggregate_scores(s_res.score, f_res.score, b_res.score, config=config)
+                agg = aggregate_scores(
+                    s_res.score, f_res.score, b_res.score, config=config
+                )
                 log.debug(
                     "Aggregate score computed: %.2f (s=%.2f f=%.2f b=%.2f)",
                     agg,
@@ -533,20 +627,24 @@ class DuplicateDetector:
                     b_res.score,
                 )
 
-                # Build candidate reference and number using successful base path
+                # Build candidate reference and number using successful base
+                # path
                 num_obj = c.get("_number")
                 num = int(num_obj) if isinstance(num_obj, int) else None
                 proj = str(c.get("project") or gerrit_project or "")
 
                 # Use the base path that actually worked for API calls
-                display_url_builder = create_gerrit_url_builder(gerrit_host, successful_base_path)
+                display_url_builder = create_gerrit_url_builder(
+                    gerrit_host, successful_base_path
+                )
                 ref = (
                     display_url_builder.change_url(proj, num)
                     if proj and isinstance(num, int)
                     else (f"change {num}" if isinstance(num, int) else "")
                 )
                 log.debug(
-                    "Scoring candidate: ref=%s agg=%.2f (s=%.2f f=%.2f b=%.2f) subj='%s'",
+                    "Scoring candidate: ref=%s agg=%.2f (s=%.2f f=%.2f b=%.2f) "
+                    "subj='%s'",
                     ref or "(none)",
                     agg,
                     s_res.score,
@@ -559,7 +657,11 @@ class DuplicateDetector:
                 if agg > best_score:
                     best_score = agg
                     # Deduplicate reasons preserving order
-                    best_reasons = list(dict.fromkeys(s_res.reasons + f_res.reasons + b_res.reasons))
+                    best_reasons = list(
+                        dict.fromkeys(
+                            s_res.reasons + f_res.reasons + b_res.reasons
+                        )
+                    )
 
                 # Collect all candidates above threshold
                 if agg >= config.similarity_threshold and ref:
@@ -579,7 +681,10 @@ class DuplicateDetector:
                 for s, u, _ in hits_sorted:
                     if u:
                         log.info("Score: %.2f  URL: %s", s, u)
-                msg = f"Similar Gerrit change(s) detected [≥ {config.similarity_threshold:.2f}]"
+                msg = (
+                    f"Similar Gerrit change(s) detected "
+                    f"[≥ {config.similarity_threshold:.2f}]"
+                )
                 if best_reasons:
                     msg += f" (Reasons: {', '.join(best_reasons)})"
                 if allow_duplicates:
@@ -590,34 +695,45 @@ class DuplicateDetector:
         # Construct human-friendly references for logs
         matching_numbers: list[int] = []
         match_lines: list[str] = []
+        duplicate_urls: list[str] = []
         for n, proj in matched:
             if proj:
                 # Use the base path that actually worked for API calls
-                display_url_builder = create_gerrit_url_builder(gerrit_host, successful_base_path)
+                display_url_builder = create_gerrit_url_builder(
+                    gerrit_host, successful_base_path
+                )
                 url = display_url_builder.change_url(proj, n)
                 match_lines.append(f"Score: 1.0  URL: {url}")
-                log.info("Score: 1.0  URL: %s", url)
+                duplicate_urls.append(url)
+                log.debug("Score: 1.0  URL: %s", url)
             else:
                 match_lines.append(f"Score: 1.0  URL: change {n}")
-                log.info("Score: 1.0  URL: change %s", n)
+                duplicate_urls.append(f"change {n}")
+                log.debug("Score: 1.0  URL: change %s", n)
             matching_numbers.append(n)
 
         if not matched:
-            log.debug("No exact subject matches and no similarity matches; duplicate check passes")
+            log.debug(
+                "No exact subject matches and no similarity matches; "
+                "duplicate check passes"
+            )
             return
 
         # Remove PR number from message since cli.py already includes it
         full_message = "subject matches existing Gerrit change(s)"
         if allow_duplicates:
-            log.warning("GERRIT DUPLICATE DETECTED (allowed): %s", full_message)
+            log.debug("GERRIT DUPLICATE DETECTED (allowed): %s", full_message)
             return
-        raise DuplicateChangeError(full_message, matching_numbers)
+        raise DuplicateChangeError(
+            full_message, matching_numbers, duplicate_urls
+        )
 
 
 def check_for_duplicates(
     gh: GitHubContext,
     allow_duplicates: bool = False,
     lookback_days: int = 7,
+    expected_github_hash: str | None = None,
 ) -> None:
     """Convenience function to check for duplicates.
 
@@ -625,6 +741,7 @@ def check_for_duplicates(
         gh: GitHub context containing PR information
         allow_duplicates: If True, only log warnings; if False, raise exception
         lookback_days: Number of days to look back for similar PRs
+        expected_github_hash: The GitHub-Hash trailer value expected for this PR
 
     Raises:
         DuplicateChangeError: If duplicates found and allow_duplicates=False
@@ -646,13 +763,20 @@ def check_for_duplicates(
             lookback_days=lookback_days,
             duplicates_filter=os.getenv("DUPLICATE_TYPES", "open"),
         )
-        detector.check_for_duplicates(target_pr, allow_duplicates=allow_duplicates, gh=gh)
+        detector.check_for_duplicates(
+            target_pr,
+            allow_duplicates=allow_duplicates,
+            gh=gh,
+            expected_github_hash=expected_github_hash,
+        )
 
-        log.info("Duplicate check completed for PR #%d", gh.pr_number)
+        log.debug("Duplicate check completed for PR #%d", gh.pr_number)
 
     except DuplicateChangeError:
         # Re-raise duplicate errors
         raise
     except Exception as exc:
-        log.warning("Duplicate detection failed for PR #%d: %s", gh.pr_number, exc)
+        log.warning(
+            "Duplicate detection failed for PR #%d: %s", gh.pr_number, exc
+        )
         # Don't fail the entire process if duplicate detection has issues