open-code-review-toolkit 0.1.0__py3-none-any.whl

ocr_toolkit/posting/markers.py

@@ -0,0 +1,284 @@
+"""OCR bot note markers, fingerprints, and ownership checks."""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from typing import TYPE_CHECKING, Any
+
+from ocr_toolkit.posting.comments import clean_text, code_text, comment_line, line_number
+
+if TYPE_CHECKING:
+    from ocr_toolkit.posting.gitlab import GitLabConfig
+
+MARKER = "<!-- open-code-review-bot -->"
+
+
+MARKER_WITH_FINGERPRINT_RE = re.compile(
+    r"^<!-- open-code-review-bot(?:\s+fp=([0-9a-f]{8,64}))?\s*-->"
+)
+
+
+OCR_REPLY_COMMAND_RE = re.compile(r"(?i)\A[ \t]*/ocr[ \t]+(skip|keep)[ \t]*(?:\r?\n[ \t]*)*\Z")
+
+
+FINGERPRINT_LEN = 32  # hex characters (= 16 raw bytes from blake2b)
+
+
+def _digest_payload(parts: list[str], digest_size: int = FINGERPRINT_LEN // 2) -> str:
+    """Return a blake2b digest for normalized fingerprint fields."""
+
+    return hashlib.blake2b("\n".join(parts).encode("utf-8"), digest_size=digest_size).hexdigest()
+
+
+def _normalized_payload_parts(
+    comment: dict[str, Any], *, line_based: bool = False
+) -> list[str] | None:
+    """Build shared canonical fingerprint fields, or None for invalid input."""
+
+    path = clean_text(comment.get("path"))
+    if not path:
+        return None
+
+    content = clean_text(comment.get("content"))
+    suggestion = code_text(comment.get("suggestion_code"))
+    existing = code_text(comment.get("existing_code"))
+    category = clean_text(comment.get("category"))
+    rule_id = clean_text(comment.get("rule_id"))
+    line = comment_line(comment)
+    if line_based:
+        start_line = line_number(comment.get("start_line") or comment.get("line"))
+        return [
+            path.strip(),
+            str(start_line),
+            rule_id.lower(),
+            category.lower(),
+            " ".join(content.lower().split()),
+            " ".join(suggestion.split()),
+        ]
+
+    parts = [
+        path.strip(),
+        rule_id.lower(),
+        category.lower(),
+        " ".join(content.lower().split()),
+        " ".join(existing.split()),
+        " ".join(suggestion.split()),
+    ]
+    if not existing.strip():
+        parts.append(str(line))
+
+    return parts
+
+
+def _fingerprint_payload(comment: dict[str, Any]) -> bytes | None:
+    """Build the canonical payload bytes for a fingerprint, or None."""
+
+    parts = _normalized_payload_parts(comment)
+    if parts is None:
+        return None
+    return "\n".join(parts).encode("utf-8")
+
+
+def _line_based_fingerprint_payload(comment: dict[str, Any]) -> bytes | None:
+    """Build the previous line-based payload for compatibility checks."""
+
+    parts = _normalized_payload_parts(comment, line_based=True)
+    if parts is None:
+        return None
+    return "\n".join(parts).encode("utf-8")
+
+
+def comment_fingerprint(comment: dict[str, Any]) -> str | None:
+    """Return a stable short hash describing one OCR finding.
+
+    The hash combines path, normalized review text, and the commented code
+    fragment (`existing_code`) when OCR provides it. It omits the anchor line
+    only when that code anchor exists, so resolved findings and `/ocr skip`
+    survive ordinary line shifts without broadening generic no-code comments
+    across a whole file. Returns None for comments missing a usable path.
+
+    Uses 16 raw bytes (32 hex chars) to keep birthday-collision risk
+    well below MR sizes the bot is likely to see. The marker regex
+    accepts 8–64 hex chars, so the wider hash is forward-compatible.
+    Older markers used a line-based payload; filter code checks those
+    compatibility hashes alongside this value.
+    """
+
+    payload = _fingerprint_payload(comment)
+    if payload is None:
+        return None
+    return hashlib.blake2b(payload, digest_size=FINGERPRINT_LEN // 2).hexdigest()
+
+
+def line_based_comment_fingerprint(comment: dict[str, Any]) -> str | None:
+    """Return the previous 32-hex line-based fingerprint for compatibility."""
+
+    payload = _line_based_fingerprint_payload(comment)
+    if payload is None:
+        return None
+    return hashlib.blake2b(payload, digest_size=FINGERPRINT_LEN // 2).hexdigest()
+
+
+def occurrence_comment_fingerprint(comment: dict[str, Any], occurrence_index: int) -> str | None:
+    """Return a fingerprint that separates repeated identical findings."""
+
+    base = comment_fingerprint(comment)
+    if base is None or occurrence_index <= 0:
+        return base
+
+    return _digest_payload([base, f"occurrence:{occurrence_index}"])
+
+
+def annotate_comment_fingerprints(
+    comments: list[dict[str, Any]], existing_key: str = "_ocr_fingerprint"
+) -> None:
+    """Attach occurrence-aware fingerprints to OCR comments in posting order."""
+
+    grouped: dict[str, list[tuple[int, dict[str, Any]]]] = {}
+    for index, comment in enumerate(comments):
+        base = comment_fingerprint(comment)
+        if base is None:
+            comment.pop(existing_key, None)
+            continue
+        grouped.setdefault(base, []).append((index, comment))
+
+    for base, indexed_comments in grouped.items():
+        ordered_comments = sorted(
+            indexed_comments,
+            key=lambda item: (
+                comment_line(item[1]),
+                clean_text(item[1].get("path")),
+                code_text(item[1].get("existing_code")),
+                code_text(item[1].get("suggestion_code")),
+                item[0],
+            ),
+        )
+        for occurrence, (_index, comment) in enumerate(ordered_comments):
+            comment[existing_key] = occurrence_comment_fingerprint(comment, occurrence)
+
+
+def legacy_comment_fingerprint(comment: dict[str, Any]) -> str | None:
+    """Return the pre-migration 16-hex fingerprint for one OCR finding.
+
+    Why: markers produced before the 8→16 byte digest migration are still
+    stored in resolved/skip-flagged threads. Without this companion,
+    `/ocr skip` decisions made under the old length would silently stop
+    suppressing their findings after the migration. Same line-based
+    payload as the previous fingerprint implementation, narrower digest.
+    """
+
+    payload = _line_based_fingerprint_payload(comment)
+    if payload is None:
+        return None
+    return hashlib.blake2b(payload, digest_size=8).hexdigest()
+
+
+def comment_fingerprint_candidates(comment: dict[str, Any]) -> set[str]:
+    """Return current and backward-compatible fingerprints for a comment."""
+
+    annotated = comment.get("_ocr_fingerprint")
+    if isinstance(annotated, str) and annotated:
+        # For duplicate findings, the annotated marker is the current run's
+        # identity. Do not also include the base hash: a skip on the first
+        # duplicate would suppress every later duplicate in the same file.
+        candidates = {annotated}
+        if line_number(comment.get("start_line") or comment.get("line")) > 0:
+            line_fingerprint = line_based_comment_fingerprint(comment)
+            legacy_fingerprint = legacy_comment_fingerprint(comment)
+            if line_fingerprint:
+                candidates.add(line_fingerprint)
+            if legacy_fingerprint:
+                candidates.add(legacy_fingerprint)
+        return {fingerprint for fingerprint in candidates if fingerprint}
+
+    candidates = set(
+        filter(
+            None,
+            (
+                comment_fingerprint(comment),
+                line_based_comment_fingerprint(comment),
+                legacy_comment_fingerprint(comment),
+            ),
+        )
+    )
+    return {fingerprint for fingerprint in candidates if fingerprint}
+
+
+def build_marker(fingerprint: str | None) -> str:
+    """Render the OCR marker, embedding the fingerprint when available."""
+
+    if fingerprint:
+        return f"<!-- open-code-review-bot fp={fingerprint} -->"
+    return MARKER
+
+
+def note_starts_with_marker(body: str) -> bool:
+    """Return whether a note body is owned by this script format."""
+
+    return bool(MARKER_WITH_FINGERPRINT_RE.match(body))
+
+
+def fingerprint_from_marker(body: str) -> str | None:
+    """Return the fingerprint embedded in a bot marker, if any."""
+
+    match = MARKER_WITH_FINGERPRINT_RE.match(body)
+    if not match:
+        return None
+    return match.group(1)
+
+
+def author_id_from_note(note: dict[str, Any]) -> int | None:
+    """Extract author id from a published note or draft note object."""
+
+    raw_author_id = note.get("author_id")
+
+    if raw_author_id is None:
+        author = note.get("author")
+        if isinstance(author, dict):
+            raw_author_id = author.get("id")
+
+    try:
+        return int(raw_author_id)
+    except (TypeError, ValueError):
+        return None
+
+
+def is_diff_note(note: dict[str, Any]) -> bool:
+    """Return True if a note belongs to an inline (diff) discussion.
+
+    Such notes are owned by the /discussions cycle and must NOT be
+    deleted via DELETE /notes/{id}, otherwise a reviewer-preserved
+    thread (resolved, /ocr skip, /ocr keep) gets destroyed despite our
+    decision to keep it.
+    """
+
+    if note.get("type") == "DiffNote":
+        return True
+    if isinstance(note.get("position"), dict):
+        return True
+    return False
+
+
+def is_own_bot_note(config: GitLabConfig, note: dict[str, Any], body_field: str) -> bool:
+    """Return whether a note belongs to this bot and this script.
+
+    Authorship must be positively verifiable. If `current_user_id` could
+    not be resolved (see `print_user_id_failure_banner`) or the note has
+    no author id, refuse to claim ownership — otherwise this bot might
+    delete notes written by other users whose body happens to start with
+    the same MARKER.
+    """
+
+    body = str(note.get(body_field) or "")
+    if not note_starts_with_marker(body):
+        return False
+
+    if config.current_user_id is None:
+        return False
+
+    actual_author_id = author_id_from_note(note)
+    if actual_author_id is None:
+        return False
+
+    return actual_author_id == config.current_user_id

ocr_toolkit/posting/payloads.py

@@ -0,0 +1,141 @@
+"""GitLab note payload size limiting and marker wrapping."""
+
+from __future__ import annotations
+
+from ocr_toolkit.common.markdown import markdown_code_block
+from ocr_toolkit.posting.markers import build_marker
+from ocr_toolkit.posting.settings import (
+    MAX_INLINE_NOTE_CHARS,
+    MAX_NOTE_CHARS,
+    SUGGESTION_BLOCK_RE,
+    TRUNCATION_NOTICE,
+)
+
+
+def _fits_limit(text: str, max_chars: int) -> bool:
+    return len(text) <= max_chars and len(text.encode("utf-8")) <= max_chars
+
+
+def _utf8_prefix(text: str, max_bytes: int) -> str:
+    if max_bytes <= 0:
+        return ""
+    return text.encode("utf-8")[:max_bytes].decode("utf-8", errors="ignore")
+
+
+def truncate_note_body(body: str, max_chars: int = MAX_NOTE_CHARS) -> str:
+    """Truncate a GitLab note body without splitting Markdown structures."""
+
+    if _fits_limit(body, max_chars):
+        return body
+
+    notice = f"{TRUNCATION_NOTICE}\n\nRaw Markdown excerpt follows."
+    budget = max(0, max_chars - len(notice.encode("utf-8")) - 2)
+    excerpt = _utf8_prefix(body, budget).rstrip()
+
+    while excerpt:
+        result = f"{notice}\n\n{markdown_code_block('markdown', excerpt)}"
+        if _fits_limit(result, max_chars):
+            return result
+
+        overflow = max(len(result) - max_chars, len(result.encode("utf-8")) - max_chars)
+        excerpt = _utf8_prefix(
+            excerpt, max(0, len(excerpt.encode("utf-8")) - overflow - 1)
+        ).rstrip()
+
+    return notice[:max_chars]
+
+
+def truncate_plain_text(text: str, max_chars: int) -> str:
+    """Truncate plain text with a short notice."""
+
+    if _fits_limit(text, max_chars):
+        return text
+
+    notice = "\n\nText was truncated before publishing to GitLab."
+    if max_chars <= len(notice.encode("utf-8")):
+        return _utf8_prefix(notice, max_chars)
+
+    budget = max(0, max_chars - len(notice.encode("utf-8")))
+    return _utf8_prefix(text, budget).rstrip() + notice
+
+
+def truncate_code_text(text: str, max_chars: int) -> str:
+    """Truncate code text while preserving indentation as much as possible."""
+
+    if max_chars <= 0:
+        return ""
+
+    if _fits_limit(text, max_chars):
+        return text
+
+    notice = "\n# ... truncated before publishing to GitLab ..."
+    if max_chars <= len(notice.encode("utf-8")):
+        return _utf8_prefix(notice, max_chars)
+
+    budget = max(0, max_chars - len(notice.encode("utf-8")))
+    return _utf8_prefix(text, budget).rstrip("\n") + notice
+
+
+def limit_inline_body(body: str, max_chars: int = MAX_INLINE_NOTE_CHARS) -> str:
+    """Limit inline discussion body without breaking GitLab suggestion blocks.
+
+    If the inline body is too large, drop the suggestion block first. If the
+    plain body is still too large, truncate it as plain Markdown text.
+    """
+
+    if _fits_limit(body, max_chars):
+        return body
+
+    match = SUGGESTION_BLOCK_RE.search(body)
+    if match:
+        body_without_suggestion = body[: match.start()] + (
+            "\n\nSuggestion block was omitted because the inline comment exceeded "
+            "the safe GitLab note size."
+        )
+        if _fits_limit(body_without_suggestion, max_chars):
+            return body_without_suggestion
+
+        body = body_without_suggestion
+
+    return truncate_plain_text(body, max_chars)
+
+
+def limit_regular_note_body(body: str, max_chars: int = MAX_NOTE_CHARS) -> str:
+    """Limit a regular MR note body before calling the GitLab API."""
+
+    return truncate_note_body(body, max_chars=max_chars)
+
+
+def note_body_budget(max_chars: int, fingerprint: str | None = None) -> int:
+    """Return the body budget after reserving space for the bot marker."""
+
+    return max(0, max_chars - len(build_marker(fingerprint)) - 1)
+
+
+def build_marked_note_body(
+    body: str,
+    *,
+    fingerprint: str | None = None,
+    max_chars: int = MAX_NOTE_CHARS,
+    inline: bool = False,
+) -> str:
+    """Build the exact GitLab note payload, including marker and size limit."""
+
+    marker = build_marker(fingerprint)
+    if max_chars < len(marker) + 1:
+        raise ValueError("GitLab note payload budget is too small for OCR marker")
+
+    budget = note_body_budget(max_chars, fingerprint)
+    limited_body = (
+        limit_inline_body(body, max_chars=budget)
+        if inline
+        else limit_regular_note_body(body, max_chars=budget)
+    )
+    payload = f"{marker}\n{limited_body}"
+    if _fits_limit(payload, max_chars):
+        return payload
+
+    # Defensive fallback: the limiting helpers should already respect
+    # the marker-aware budget, but never send an over-limit API payload.
+    limited_body = truncate_plain_text(limited_body, budget)
+    return f"{marker}\n{limited_body}"

ocr_toolkit/posting/result.py

@@ -0,0 +1,116 @@
+"""OCR result artifact loading and provider failure classification."""
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
+
+from ocr_toolkit.common.redaction import sanitize_ocr_value
+from ocr_toolkit.posting.comments import clean_text
+from ocr_toolkit.posting.settings import max_result_bytes
+
+
+class OcrResultMissing(Exception):
+    """The OCR result artifact is missing or unreadable on disk."""
+
+
+class OcrResultMalformed(Exception):
+    """The OCR result artifact exists but is not valid JSON."""
+
+
+class OcrResultTooLarge(Exception):
+    """The OCR result artifact exceeds the configured safety limit."""
+
+
+def load_ocr_result(path: Path) -> Any:
+    """Load OCR JSON result from disk.
+
+    Raises `OcrResultMissing` when the artifact is absent or
+    unreadable, `OcrResultTooLarge` before reading a runaway artifact,
+    and `OcrResultMalformed` when the contents are not valid JSON. The
+    caller routes these failure modes differently: a missing artifact
+    looks like an OCR crash, while an oversized or malformed artifact is
+    a controlled OCR output contract violation.
+    """
+
+    try:
+        size = path.stat().st_size
+    except FileNotFoundError as exc:
+        raise OcrResultMissing(str(exc)) from exc
+    except OSError as exc:
+        raise OcrResultMissing(str(exc)) from exc
+
+    limit = max_result_bytes()
+    if size > limit:
+        raise OcrResultTooLarge(
+            f"OCR result JSON is {size} bytes, above OCR_MAX_RESULT_BYTES={limit}"
+        )
+
+    try:
+        with path.open("rb") as handle:
+            data = handle.read(limit + 1)
+    except FileNotFoundError as exc:
+        raise OcrResultMissing(str(exc)) from exc
+    except OSError as exc:
+        raise OcrResultMissing(str(exc)) from exc
+    if len(data) > limit:
+        raise OcrResultTooLarge(f"OCR result JSON grew above OCR_MAX_RESULT_BYTES={limit}")
+
+    try:
+        text = data.decode("utf-8")
+    except UnicodeDecodeError as exc:
+        raise OcrResultMalformed(str(exc)) from exc
+
+    try:
+        return sanitize_ocr_value(json.loads(text))
+    except (json.JSONDecodeError, RecursionError) as exc:
+        raise OcrResultMalformed(str(exc)) from exc
+
+
+LLM_BILLING_FAILURE_RE = re.compile(
+    r"(?i)\b("
+    r"(?:http\s*)?status(?:[_\s]*code)?[\"']?\s*[:=]\s*[\"']?402|"
+    r"code[\"']?\s*[:=]\s*[\"']?402|payment required|insufficient[_ -]?funds|insufficient user balance|"
+    r"insufficient balance|insufficient[_ -]?quota|quota[_ -]?exceeded|"
+    r"out of credits|credit balance"
+    r")\b"
+)
+
+
+def ocr_warning_text(warning: Any) -> str:
+    """Return warning text relevant for provider failure classification."""
+
+    if isinstance(warning, dict):
+        parts: list[str] = []
+        for key in ("type", "message", "code", "status", "status_code", "detail"):
+            text = clean_text(warning.get(key))
+            if text:
+                parts.append(f"{key}: {text}" if key in {"code", "status", "status_code"} else text)
+        for key in ("error", "details"):
+            nested = warning.get(key)
+            if isinstance(nested, dict):
+                text = ocr_warning_text(nested)
+                if text:
+                    parts.append(text)
+            else:
+                text = clean_text(nested)
+                if text:
+                    parts.append(text)
+        return "\n".join(parts)[:4000]
+    if isinstance(warning, list):
+        return "\n".join(text for value in warning[:40] if (text := ocr_warning_text(value)))[:4000]
+    return clean_text(warning)
+
+
+def llm_billing_failure_warnings(warnings: Sequence[Any]) -> list[str]:
+    """Return OCR warnings that indicate LLM provider billing/quota failure."""
+
+    matches: list[str] = []
+    for warning in warnings:
+        text = ocr_warning_text(warning)
+        if text and LLM_BILLING_FAILURE_RE.search(text):
+            matches.append(text)
+    return matches