docs-drift 0.1.0__py3-none-any.whl

docs_drift/fix/edits.py

@@ -0,0 +1,124 @@
+"""Exact-span edits, stale-hash checks, byte-preserving writes and unified patches."""
+
+import codecs
+import difflib
+import hashlib
+import os
+import tempfile
+from pathlib import Path
+
+from pydantic import Field
+
+from docs_drift.models import DecidedBy, PipelineModel, RepoPath
+
+
+class StaleDocumentError(ValueError):
+    """The caller must scan again and obtain fresh spans before writing."""
+
+
+class TextEdit(PipelineModel):
+    file: RepoPath
+    line: int = Field(ge=1)
+    column: int = Field(ge=1)
+    end_line: int = Field(ge=1)
+    end_column: int = Field(ge=1)
+    old_text: str
+    new_text: str
+    reason: str
+    claim_id: str
+    decided_by: DecidedBy = DecidedBy.rules
+    replayed: bool = False
+
+
+class FixPlan(PipelineModel):
+    edits: list[TextEdit] = Field(default_factory=list)
+    suggestions: list[str] = Field(default_factory=list)
+    hashes: dict[str, str] = Field(default_factory=dict)
+
+
+def safe_path(repo: Path, relative: str) -> Path:
+    target = (repo / relative).resolve()
+    if not target.is_relative_to(repo.resolve()):
+        raise ValueError(f"Refusing to edit {relative}: it resolves outside the repository.")
+    return target
+
+
+def decode(raw: bytes) -> tuple[str, str, bytes]:
+    for bom, encoding in (
+        (codecs.BOM_UTF8, "utf-8"),
+        (codecs.BOM_UTF16_LE, "utf-16-le"),
+        (codecs.BOM_UTF16_BE, "utf-16-be"),
+    ):
+        if raw.startswith(bom):
+            return raw[len(bom) :].decode(encoding), encoding, bom
+    return raw.decode("utf-8"), "utf-8", b""
+
+
+def offsets(text: str, edit: TextEdit) -> tuple[int, int]:
+    lines = text.splitlines(keepends=True)
+    if edit.line > len(lines) or edit.end_line > len(lines):
+        raise StaleDocumentError(f"Line spans changed in {edit.file}; scan again.")
+    start = sum(map(len, lines[: edit.line - 1])) + edit.column - 1
+    end = sum(map(len, lines[: edit.end_line - 1])) + edit.end_column - 1
+    if text[start:end] != edit.old_text:
+        raise StaleDocumentError(f"The exact edit span changed in {edit.file}; scan again.")
+    return start, end
+
+
+def render_file(repo: Path, plan: FixPlan, relative: str) -> tuple[bytes, bytes]:
+    path = safe_path(repo, relative)
+    original = path.read_bytes()
+    if hashlib.sha256(original).hexdigest() != plan.hashes[relative]:
+        raise StaleDocumentError(f"{relative} changed since the scan; scan again before applying edits.")
+    text, encoding, bom = decode(original)
+    selected = [edit for edit in plan.edits if edit.file == relative]
+    spans = sorted([(offsets(text, edit), edit) for edit in selected], key=lambda pair: pair[0], reverse=True)
+    last_start = len(text) + 1
+    for (start, end), edit in spans:
+        if end > last_start:
+            raise ValueError(f"Overlapping edits in {relative}; review conflicting suggestions manually.")
+        text = text[:start] + edit.new_text + text[end:]
+        last_start = start
+    return original, bom + text.encode(encoding)
+
+
+def unified_patch(repo: Path, plan: FixPlan) -> str:
+    parts: list[str] = []
+    for relative in sorted({edit.file for edit in plan.edits}):
+        original, modified = render_file(repo, plan, relative)
+        before_text, encoding, bom = decode(original)
+        after_text = decode(modified)[0]
+        if encoding != "utf-8":
+            raise ValueError("Unified patches require UTF-8 documents; use a reviewed exact-span write for UTF-16.")
+        prefix = "\ufeff" if bom == codecs.BOM_UTF8 else ""
+        before = prefix + before_text
+        after = prefix + after_text
+        for line in difflib.unified_diff(
+            before.splitlines(keepends=True),
+            after.splitlines(keepends=True),
+            fromfile=f"a/{relative}",
+            tofile=f"b/{relative}",
+        ):
+            parts.append(line)
+            if not line.endswith("\n"):
+                parts.append("\n\\n")
+    return "".join(parts)
+
+
+def apply_file(repo: Path, plan: FixPlan, relative: str) -> None:
+    original, modified = render_file(repo, plan, relative)
+    path = safe_path(repo, relative)
+    mode = path.stat().st_mode
+    temporary: Path | None = None
+    try:
+        with tempfile.NamedTemporaryFile(dir=path.parent, prefix=".docs-drift-", delete=False) as stream:
+            temporary = Path(stream.name)
+            stream.write(modified)
+        # Check again immediately before replacing; a changed hash requires another scan.
+        if path.read_bytes() != original:
+            raise StaleDocumentError(f"{relative} changed during review; scan again.")
+        os.chmod(temporary, mode)
+        temporary.replace(path)
+    finally:
+        if temporary and temporary.exists():
+            temporary.unlink()

docs_drift/fix/model.py

@@ -0,0 +1,139 @@
+"""Optional minimal sentence rewrites, accepted only after static self-checks."""
+
+import asyncio
+import hashlib
+import re
+import time
+
+from docs_drift.docs.defaults import extract_defaults
+from docs_drift.docs.extract_inline import extract_claims
+from docs_drift.docs.sections import parse_document
+from docs_drift.fix.edits import FixPlan, TextEdit, decode, safe_path
+from docs_drift.index.base import CodeIndex
+from docs_drift.llm.client import ClientError, RecordedClient
+from docs_drift.llm.pricing import DEFAULT_MODEL, estimate_request
+from docs_drift.llm.prompts import build_request
+from docs_drift.llm.retrieve import retrieve
+from docs_drift.llm.types import Candidate, ModelClient
+from docs_drift.llm.verify import Budget, validate_evidence
+from docs_drift.match import match_claims
+from docs_drift.models import DecidedBy, DocFile, DocFormat, Finding, ScanResult, Verdict
+
+
+def sentence_span(line: str, column: int) -> tuple[int, str]:
+    prefix = re.match(r"\s*(?:(?:[-*+>]|\d+[.)])\s+)?", line)
+    offset = prefix.end() if prefix else 0
+    for match in re.finditer(r".+?(?:[.!?](?=\s|$)|$)", line[offset:]):
+        start = offset + match.start()
+        end = offset + match.end()
+        if start <= column - 1 < end:
+            sentence = match.group().strip()
+            return start + len(match.group()) - len(match.group().lstrip()) + 1, sentence
+    return offset + 1, line[offset:].strip()
+
+
+def valid_rewrite(original: str, rewritten: str, finding: Finding, index: CodeIndex, result: ScanResult) -> bool:
+    if not rewritten.strip() or "\n" in rewritten or "\r" in rewritten:
+        return False
+    if len(re.split(r"(?<=[.!?])\s+", rewritten)) > 1:
+        return False
+    if finding.verdict == Verdict.renamed and finding.suggestion and finding.suggestion not in rewritten:
+        return False
+    known = {name for symbol in index.symbols for name in (symbol.name, symbol.qualified_name, *symbol.aliases)}
+    identifiers = re.compile(
+        r"\b[A-Za-z]\w*_[A-Za-z]\w*\b|\b[A-Za-z]\w*(?:\.[A-Za-z]\w*)+\b|"
+        r"\b[A-Za-z]\w*(?=\s*\()|\b[A-Z][a-z0-9]+(?:[A-Z][A-Za-z0-9]*)+\b"
+    )
+    introduced = set(identifiers.findall(rewritten)) - set(identifiers.findall(original))
+    if any(name not in known for name in introduced):
+        return False
+    doc = DocFile(
+        path=finding.claim.doc_path,
+        content_hash=hashlib.sha256(rewritten.encode()).hexdigest(),
+        format=DocFormat.markdown,
+    )
+    parsed = parse_document(doc, rewritten)
+    extracted = extract_claims(parsed).claims
+    subjects = [claim for claim in extracted if claim.kind.value != "default_value"]
+    defaults = extract_defaults(parsed, subjects).claims
+    checked = match_claims([*subjects, *defaults], index, result.repo_path)
+    # New uncertain references are rejected as well as proven drift: a rewrite must earn acceptance.
+    return all(item.verdict == Verdict.ok for item in checked)
+
+
+async def add_model_fixes(
+    result: ScanResult, index: CodeIndex, plan: FixPlan, client: ModelClient | None = None
+) -> FixPlan:
+    config = result.config_used.llm
+    selected_client = client or RecordedClient(config)
+    budget = Budget(max(0, config.budget - result.llm_usage.get("cost", 0)))
+    semaphore = asyncio.Semaphore(config.concurrency)
+    fixed = {edit.claim_id for edit in plan.edits}
+    occupied = {(edit.file, edit.line) for edit in plan.edits}
+    remaining = [
+        item
+        for item in result.findings
+        if item.is_drift and item.claim.claim_id not in fixed and (item.claim.doc_path, item.claim.line) not in occupied
+    ][: config.max_claims]
+
+    async def propose(finding: Finding) -> TextEdit | None:
+        sources = {item.file for item in finding.evidence if item.file in index.by_file}
+        if finding.matched_symbol:
+            sources.add(finding.matched_symbol.file)
+        if not sources:
+            return None
+        lines = decode(safe_path(result.repo_path, finding.claim.doc_path).read_bytes())[0].splitlines()
+        column, original = sentence_span(lines[finding.claim.line - 1], finding.claim.column)
+        candidate = Candidate(claim=finding.claim, source_files=sorted(sources))
+        snippets = retrieve(result.repo_path, candidate, result.symbols, config.redact_paths)
+        if not snippets:
+            return None
+        data = (
+            f"Sentence: {original}\nFinding: {finding.verdict.value}\n"
+            f"Evidence-based correction: {finding.suggestion or ''}"
+        )
+        request = build_request(config.model or DEFAULT_MODEL, data, snippets, rewrite=True)
+        async with semaphore:
+            estimate = estimate_request(request)
+            if not await budget.reserve(estimate):
+                plan.suggestions.append(f"{finding.claim.doc_path}:{finding.claim.line}: budget reached")
+                return None
+            started = time.perf_counter()
+            response = await selected_client.complete(request)
+            await budget.settle(estimate, response.usage.cost)
+            for key in ("input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens"):
+                result.llm_usage[key] = result.llm_usage.get(key, 0) + float(getattr(response.usage, key))
+            result.llm_usage["claims"] = result.llm_usage.get("claims", 0) + 1
+            result.llm_usage["seconds"] = result.llm_usage.get("seconds", 0) + time.perf_counter() - started
+        rewritten = response.result.suggested_doc_text
+        if (
+            not rewritten
+            or not validate_evidence(response, snippets)
+            or not valid_rewrite(original, rewritten, finding, index, result)
+        ):
+            return None
+        return TextEdit(
+            file=finding.claim.doc_path,
+            line=finding.claim.line,
+            column=column,
+            end_line=finding.claim.line,
+            end_column=column + len(original),
+            old_text=original,
+            new_text=rewritten,
+            reason="Model sentence rewrite passed static self-checks.",
+            claim_id=finding.claim.claim_id,
+            decided_by=DecidedBy.llm,
+            replayed=response.replayed,
+        )
+
+    try:
+        async with asyncio.timeout(config.timeout):
+            proposed = await asyncio.gather(*(propose(finding) for finding in remaining))
+    except TimeoutError:
+        plan.suggestions.append("Model fixes exceeded their total timeout; review remaining sentences manually.")
+        return plan
+    except ClientError:
+        raise
+    plan.edits.extend(edit for edit in proposed if edit is not None)
+    result.llm_usage["cost"] = result.llm_usage.get("cost", 0) + budget.spent
+    return plan

docs_drift/fix/rules.py

@@ -0,0 +1,232 @@
+"""Certain replacements only, within the scanner's original literal spans."""
+
+import hashlib
+import re
+from decimal import Decimal
+from pathlib import Path
+
+from docs_drift.docs.defaults import normalize_value
+from docs_drift.docs.sections import is_historical_span
+from docs_drift.fix.edits import FixPlan, StaleDocumentError, TextEdit, decode, safe_path
+from docs_drift.history import GitHistory, _run
+from docs_drift.models import Claim, ClaimKind, Finding, ScanResult, Verdict
+
+_WORD = re.compile(r"'[^']*'|\"[^\"]*\"|\S+")
+MIN_RENAME_FIX_CONFIDENCE = 0.9
+_UNITS = {
+    "ms": Decimal(".001"),
+    "s": Decimal(1),
+    "sec": Decimal(1),
+    "second": Decimal(1),
+    "seconds": Decimal(1),
+    "m": Decimal(60),
+    "min": Decimal(60),
+    "minute": Decimal(60),
+    "minutes": Decimal(60),
+    "h": Decimal(3600),
+    "hour": Decimal(3600),
+    "hours": Decimal(3600),
+    "b": Decimal(1),
+    "kb": Decimal(1000),
+    "mb": Decimal(1000000),
+    "gb": Decimal(1000000000),
+    "kib": Decimal(1024),
+    "mib": Decimal(1048576),
+    "gib": Decimal(1073741824),
+}
+_UNIT_VALUE = re.compile(
+    r"(?<!\w)(-?\d+(?:\.\d+)?)([\s`\"']*)(milliseconds?|seconds?|minutes?|hours?|MiB|GiB|KiB|MB|GB|KB|ms|sec|min|s|m|h|B)\b",
+    re.I,
+)
+
+
+def _edit(claim: Claim, line: str, old: str, new: str, reason: str) -> TextEdit | None:
+    # Never fall back to a file-wide substitution: a unique exact span on this line is required.
+    matches = list(re.finditer(rf"(?<![\w-]){re.escape(old)}(?![\w-])", line))
+    if len(matches) != 1 or old == new:
+        return None
+    match = matches[0]
+    return TextEdit(
+        file=claim.doc_path,
+        line=claim.line,
+        column=match.start() + 1,
+        end_line=claim.line,
+        end_column=match.end() + 1,
+        old_text=old,
+        new_text=new,
+        reason=reason,
+        claim_id=claim.claim_id,
+    )
+
+
+def _default_edit(finding: Finding, line: str) -> TextEdit | None:
+    expected = finding.claim.expected_default
+    actual = finding.matched_symbol.default if finding.matched_symbol else finding.suggestion
+    if expected is None or actual is None:
+        return None
+    old = expected.strip("`\"'")
+    new = actual.strip("`\"'")
+    expected_value = normalize_value(expected, subject=finding.claim.value)
+    actual_value = normalize_value(actual, subject=finding.claim.value)
+    if expected_value.kind in {"duration", "size"}:
+        if actual_value.kind != expected_value.kind:
+            return None
+        unit_matches = [
+            match
+            for match in _UNIT_VALUE.finditer(line)
+            if normalize_value(match.group(1) + match.group(3), subject=finding.claim.value).equals(expected_value)
+            is True
+        ]
+        if len(unit_matches) != 1:
+            return None
+        match = unit_matches[0]
+        factor = _UNITS.get(match.group(3).casefold())
+        if factor is None:
+            return None
+        old = match.group(1)
+        new = format(Decimal(actual_value.value) / factor, "f").rstrip("0").rstrip(".")
+        if "." not in format(Decimal(actual_value.value) / factor, "f"):
+            new = str(int(Decimal(actual_value.value) / factor))
+        return TextEdit(
+            file=finding.claim.doc_path,
+            line=finding.claim.line,
+            column=match.start(1) + 1,
+            end_line=finding.claim.line,
+            end_column=match.end(1) + 1,
+            old_text=old,
+            new_text=new,
+            reason="Update the default while preserving its documented unit.",
+            claim_id=finding.claim.claim_id,
+        )
+    # The verdict already established comparable defaults. Keep the documented unit and quote syntax.
+    number = re.fullmatch(r"(-?\d+(?:\.\d+)?)(?:s|B)?", old)
+    if number:
+        old = number.group(1)
+    return _edit(finding.claim, line, old, new, "Update the documented default.")
+
+
+def _removed_flag_shape(repo: Path, flag: str) -> tuple[bool, str] | None:
+    history = GitHistory(repo).search(flag)
+    if not history.commit:
+        return None
+    patch = _run(repo, ["show", "--no-ext-diff", "--format=", history.commit.sha])
+    if patch is None:
+        return None
+    removed = [line[1:] for line in patch.stdout.splitlines() if line.startswith("-") and flag in line]
+    declarations = [line for line in removed if re.search(r"(?:add_argument|\.option|\.Option)\(", line)]
+    if len(declarations) != 1:
+        return None
+    declaration = declarations[0]
+    boolean = "store_true" in declaration or "store_false" in declaration
+    if ".option(" in declaration and "add_argument" not in declaration:
+        flag_string = re.search(r"\.option\(\s*['\"]([^'\"]+)", declaration)
+        boolean = flag_string is not None and not re.search(r"[<\[]", flag_string.group(1))
+    return boolean, history.commit.short_sha
+
+
+def _command_edit(repo: Path, finding: Finding, line: str) -> tuple[TextEdit | None, str | None]:
+    unknown = set(re.findall(r"--[\w-]+", " ".join(finding.reasons)))
+    if len(unknown) != 1:
+        return None, None
+    flag = next(iter(unknown))
+    shape = _removed_flag_shape(repo, flag)
+    if shape is None:
+        return None, None
+    boolean, sha = shape
+    tokens = list(_WORD.finditer(line))
+    flags = [index for index, token in enumerate(tokens) if token.group().split("=", 1)[0] == flag]
+    if len(flags) != 1:
+        return None, None
+    index = flags[0]
+    start, end = tokens[index].span()
+    if not boolean and "=" not in tokens[index].group():
+        if index + 1 == len(tokens) or tokens[index + 1].group().startswith("-"):
+            return None, None
+        end = tokens[index + 1].end()
+    if end < len(line) and line[end] == " ":
+        end += 1
+    elif start and line[start - 1] == " ":
+        start -= 1
+    edit = TextEdit(
+        file=finding.claim.doc_path,
+        line=finding.claim.line,
+        column=start + 1,
+        end_line=finding.claim.line,
+        end_column=end + 1,
+        old_text=line[start:end],
+        new_text="",
+        reason=f"Remove {flag}, deleted in {sha}.",
+        claim_id=finding.claim.claim_id,
+    )
+    note = f"{flag} was removed in {sha}; check whether this example should change more than this."
+    return edit, note
+
+
+def plan_rule_fixes(result: ScanResult, *, contents: dict[str, str] | None = None) -> FixPlan:  # noqa: PLR0912
+    plan = FixPlan(hashes={doc.path: doc.content_hash for doc in result.docs})
+    texts: dict[str, list[str]] = {}
+    for doc in result.docs:
+        raw = (
+            contents[doc.path].encode("utf-8")
+            if contents and doc.path in contents
+            else safe_path(result.docs_repo_path or result.repo_path, doc.path).read_bytes()
+        )
+        if hashlib.sha256(raw).hexdigest() != doc.content_hash:
+            raise StaleDocumentError(f"{doc.path} changed since scan; scan again.")
+        try:
+            texts[doc.path] = decode(raw)[0].splitlines()
+        except UnicodeError:
+            plan.suggestions.append(f"{doc.path}: encoding is not safely editable; review manually.")
+    for finding in result.findings:
+        if not finding.is_drift or finding.claim.doc_path not in texts:
+            continue
+        claim = finding.claim
+        line = texts[claim.doc_path][claim.line - 1]
+        edit = None
+        if (
+            finding.verdict == Verdict.renamed
+            and finding.suggestion
+            and finding.confidence >= MIN_RENAME_FIX_CONFIDENCE
+        ):
+            replacement = finding.suggestion
+            old = claim.value
+            if claim.kind == ClaimKind.endpoint:
+                old = claim.raw_text.strip("`").split(maxsplit=1)[-1]
+                replacement = (
+                    finding.matched_symbol.raw_path
+                    if finding.matched_symbol and finding.matched_symbol.raw_path
+                    else replacement.split(maxsplit=1)[-1]
+                )
+            edit = _edit(claim, line, old, replacement, "Replace a confirmed renamed identifier.")
+        elif finding.verdict == Verdict.default_changed:
+            edit = _default_edit(finding, line)
+        elif finding.verdict == Verdict.invalid_command:
+            edit, note = _command_edit(result.repo_path, finding, line)
+            if note:
+                plan.suggestions.append(note)
+        if edit:
+            section = next((item for item in result.sections if item.section_id == claim.section_id), None)
+            if section and is_historical_span(section, claim.line):
+                continue
+            plan.edits.append(edit)
+            # Same-section duplicates retain exact locations; historical locations are excluded explicitly.
+            for location in claim.locations:
+                if location.line == claim.line or (section and is_historical_span(section, location.line)):
+                    continue
+                duplicate = _edit(
+                    claim.model_copy(update={"line": location.line}),
+                    texts[claim.doc_path][location.line - 1],
+                    edit.old_text,
+                    edit.new_text,
+                    edit.reason,
+                )
+                if duplicate:
+                    plan.edits.append(duplicate)
+        else:
+            plan.suggestions.append(
+                f"{claim.doc_path}:{claim.line}: no certain rule edit for {claim.value!r}; "
+                "remove the stale statement or mark it deprecated after review."
+            )
+    unique = {(edit.file, edit.line, edit.column, edit.end_column, edit.new_text): edit for edit in plan.edits}
+    plan.edits = list(unique.values())
+    return plan

docs_drift/git_safety.py

@@ -0,0 +1,35 @@
+"""Git conversion options that prevent repository-defined filter execution."""
+
+import os
+import subprocess
+from pathlib import Path
+
+from docs_drift.history import _run
+
+
+def disabled_filters(repo: Path) -> list[str]:
+    keys = _run(repo, ["config", "--name-only", "--list"])
+    if keys is None:
+        raise ValueError("Cannot inspect repository filter configuration safely.")
+    names = {key.rsplit(".", 1)[0] for key in keys.stdout.splitlines() if key.startswith("filter.")}
+    options = []
+    for name in sorted(names):
+        options.extend(
+            ["-c", name + ".clean=", "-c", name + ".smudge=", "-c", name + ".process=", "-c", name + ".required=false"]
+        )
+    return options
+
+
+def checkout_options(repo: Path) -> list[str]:
+    """Read conversion configuration without executing filters, hooks or target programs."""
+    env = {key: value for key, value in os.environ.items() if not key.upper().startswith("GIT_")}
+    value = subprocess.run(
+        ["git", "-C", str(repo), "config", "--get", "core.autocrlf"],
+        capture_output=True,
+        text=True,
+        timeout=3,
+        check=False,
+        env=env,
+    ).stdout.strip()
+    options = ["-c", "core.autocrlf=" + (value if value in {"true", "false", "input"} else "false")]
+    return [*options, *disabled_filters(repo)]

docs_drift/graph/queries/evidence_path.cypher

@@ -0,0 +1,6 @@
+// Reference query for consumers importing the exported evidence graph into Cypher.
+// The built-in viewer runs locally in Cytoscape; no graph database is required.
+MATCH path = (section:Section)-[:CONTAINS]->(claim:Claim)-[:REFERS_TO]->(symbol:Symbol)
+             -[:DEFINED_IN]->(file:File)-[:CHANGED_BY]->(commit:Commit)
+WHERE claim.id = $claim_id
+RETURN path;