docs-drift 0.1.0__py3-none-any.whl

docs_drift/web/examples/verification.json

@@ -0,0 +1,16 @@
+[
+  {
+    "repo_url": "https://github.com/textualize/rich-cli",
+    "sha": "46f4d2469097be395558768714a5f07ebccf1412",
+    "name": "rich-cli",
+    "verified_values": ["--line-number", "--p"],
+    "note": "Manually checked README.md:61 (--line-number) against src/rich_cli/__main__.py:358 (--line-numbers), and README.md:180 (--p) against the --print/-p declaration. Other static findings are not manually certified."
+  },
+  {
+    "repo_url": "https://github.com/simonw/files-to-prompt",
+    "sha": "ae06f2039f3a1dc1c9f3b4657fb51459490768f2",
+    "name": "files-to-prompt · historical",
+    "verified_values": ["c/--cxml"],
+    "note": "Historical snapshot, intentionally pinned before the upstream fix: README.md:55 says c/--cxml; files_to_prompt/cli.py declares -c/--cxml. Fixed upstream by f7cf729babc92604b9cd06c3d9cad32daff0f276. Other findings may reflect scanner limitations, not real defects."
+  }
+]

docs_drift/web/git.py

@@ -0,0 +1,274 @@
+"""Bounded partial clone with no user Git config, credentials, hooks or smudge filters."""
+
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import time
+from datetime import UTC, datetime, timedelta
+from pathlib import Path, PurePosixPath
+
+from docs_drift.limits import CONTROL_START, DELETE_CHARACTER, MAX_GIT_OUTPUT, MAX_PATH_CHARS
+from docs_drift.web.settings import Settings
+from docs_drift.web.validation import Repository
+
+SOURCE_SUFFIXES = {
+    ".md",
+    ".mdx",
+    ".markdown",
+    ".rst",
+    ".txt",
+    ".py",
+    ".pyi",
+    ".js",
+    ".mjs",
+    ".cjs",
+    ".jsx",
+    ".ts",
+    ".tsx",
+    ".mts",
+    ".cts",
+    ".json",
+    ".yaml",
+    ".yml",
+    ".toml",
+    ".ini",
+    ".cfg",
+    ".sh",
+    ".mk",
+}
+SOURCE_NAMES = {"Makefile", "GNUmakefile", "justfile", "Dockerfile", "Taskfile.yml"}
+
+
+class JobError(Exception):
+    def __init__(self, kind: str, message: str, hint: str):
+        super().__init__(message)
+        self.kind, self.hint = kind, hint
+
+
+def git_environment(home: Path) -> dict[str, str]:
+    # Inherit only execution essentials, never tokens, proxies, SSH agents or model keys.
+    env = {
+        k: os.environ[k]
+        for k in ("PATH", "SYSTEMROOT", "WINDIR", "COMSPEC", "PATHEXT", "TEMP", "TMP", "LANG")
+        if k in os.environ
+    }
+    env.update(
+        HOME=str(home),
+        USERPROFILE=str(home),
+        XDG_CONFIG_HOME=str(home),
+        GIT_CONFIG_GLOBAL=os.devnull,
+        GIT_CONFIG_SYSTEM=os.devnull,
+        GIT_CONFIG_NOSYSTEM="1",
+        GIT_TERMINAL_PROMPT="0",
+        GIT_LFS_SKIP_SMUDGE="1",
+        GIT_OPTIONAL_LOCKS="0",
+        PYTHONUTF8="1",
+    )
+    return env
+
+
+def disk_bytes(root: Path) -> int:
+    return sum(p.stat().st_size for p in root.rglob("*") if p.is_file() and not p.is_symlink())
+
+
+def kill_process(process: subprocess.Popen[bytes]) -> None:
+    if process.poll() is not None:
+        return
+    if os.name == "nt":
+        subprocess.run(["taskkill", "/PID", str(process.pid), "/T", "/F"], capture_output=True, check=False)
+    else:
+        process.kill()
+    process.wait(timeout=10)
+
+
+def git(args: list[str], root: Path, settings: Settings, deadline: float, *, stdin: bytes | None = None) -> bytes:
+    creationflags = 0
+    if sys.platform == "win32":
+        creationflags = subprocess.CREATE_NEW_PROCESS_GROUP
+    command = [
+        "git",
+        "-c",
+        f"core.hooksPath={os.devnull}",
+        "-c",
+        "core.fsmonitor=false",
+        "-c",
+        "protocol.file.allow=never",
+        "-c",
+        "protocol.ext.allow=never",
+        "-c",
+        "protocol.allow=never",
+        "-c",
+        "protocol.https.allow=always",
+        "-c",
+        "http.followRedirects=false",
+        "-c",
+        "credential.helper=",
+        "-c",
+        "filter.lfs.smudge=",
+        "-c",
+        "filter.lfs.required=false",
+        *args,
+    ]
+    if os.name == "nt":
+        command[1:1] = ["-c", "http.sslBackend=openssl"]
+    # Temporary files prevent a full pipe from deadlocking our disk/timeout monitor.
+
+    with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err:
+        process = subprocess.Popen(
+            command,
+            cwd=root,
+            env=git_environment(root),
+            stdin=subprocess.PIPE if stdin else subprocess.DEVNULL,
+            stdout=out,
+            stderr=err,
+            creationflags=creationflags,
+        )
+        try:
+            if stdin and process.stdin:
+                process.stdin.write(stdin)
+                process.stdin.close()
+            while process.poll() is None:
+                if time.monotonic() >= deadline:
+                    raise JobError(
+                        "timeout",
+                        "Git took too long to download this repository.",
+                        "Try a smaller repository or a pre-scanned example.",
+                    )
+                if disk_bytes(root) > settings.max_bytes:
+                    raise JobError(
+                        "too_large", "The repository exceeded the 200 MB download limit.", "Try a smaller repository."
+                    )
+                time.sleep(0.1)
+            if process.returncode:
+                raise JobError(
+                    "clone",
+                    "The repository or ref could not be fetched; it may be private, missing or redirected.",
+                    "Use the canonical public GitHub URL and check the branch or tag.",
+                )
+            out.seek(0)
+            output = out.read(8_000_001)
+            if len(output) > MAX_GIT_OUTPUT:
+                raise JobError("too_large", "The repository listing is too large.", "Try a smaller repository.")
+            return output
+        finally:
+            kill_process(process)
+
+
+def resolve(repository: Repository, root: Path, settings: Settings, deadline: float) -> str:
+    ref = repository.ref or "HEAD"
+    if re.fullmatch(r"[0-9a-fA-F]{40}", ref):
+        return ref.lower()
+    patterns = ["HEAD"] if ref == "HEAD" else [f"refs/heads/{ref}", f"refs/tags/{ref}", f"refs/tags/{ref}^{{}}"]
+    rows = git(["ls-remote", "--exit-code", repository.url, *patterns], root, settings, deadline).decode().splitlines()
+    matches = [row.split() for row in rows if row.strip()]
+    if not matches:
+        raise JobError(
+            "clone", "This branch or tag was not found.", "Check the ref or leave it empty for the default branch."
+        )
+    annotated = [row[0] for row in matches if row[1].endswith("^{}")]
+    sha = (annotated or [matches[0][0]])[0]
+    if not re.fullmatch(r"[0-9a-f]{40}", sha):
+        raise JobError("clone", "GitHub returned an unexpected commit identifier.", "Try again later.")
+    return sha
+
+
+def checkout(  # noqa: PLR0913 - trusted clone configuration, not request-controlled options.
+    repository: Repository,
+    sha: str,
+    root: Path,
+    settings: Settings,
+    deadline: float,
+    *,
+    blob_filter: str = "blob:none",
+    depth_only: bool = False,
+) -> Path:
+    repo = root / "repository"
+    repo.mkdir()
+    git(["init", "--quiet", "--template=", str(repo)], root, settings, deadline)
+    prefix = ["-C", str(repo)]
+    git([*prefix, "remote", "add", "origin", repository.url], root, settings, deadline)
+    for key, value in (
+        ("http.followRedirects", "false"),
+        ("protocol.file.allow", "never"),
+        ("protocol.ext.allow", "never"),
+        ("protocol.allow", "never"),
+        ("protocol.https.allow", "always"),
+        ("core.hooksPath", os.devnull),
+    ):
+        git([*prefix, "config", key, value], root, settings, deadline)
+    if os.name == "nt":
+        git([*prefix, "config", "http.sslBackend", "openssl"], root, settings, deadline)
+    git([*prefix, "config", "remote.origin.promisor", "true"], root, settings, deadline)
+    git([*prefix, "config", "remote.origin.partialclonefilter", blob_filter], root, settings, deadline)
+    since = (datetime.now(UTC) - timedelta(days=settings.history_days)).date().isoformat()
+    fetch = [
+        *prefix,
+        "fetch",
+        "--quiet",
+        "--no-tags",
+        "--no-recurse-submodules",
+        f"--filter={blob_filter}",
+        f"--depth={settings.history_commits}" if depth_only else f"--shallow-since={since}",
+        "origin",
+        sha,
+    ]
+    try:
+        git(fetch, root, settings, deadline)
+    except JobError as exc:
+        if exc.kind != "clone":
+            raise
+        # Old inactive projects may have no commits within the date window.
+        fetch = [arg for arg in fetch if not arg.startswith("--shallow-since=")]
+        fetch.insert(-2, f"--depth={settings.history_commits}")
+        git(fetch, root, settings, deadline)
+    listing = git([*prefix, "ls-tree", "-r", "-z", sha], root, settings, deadline).split(b"\0")
+    entries = [entry for entry in listing if entry]
+    if len(entries) > settings.max_files:
+        raise JobError(
+            "too_large", "The repository exceeds the 20,000 file limit.", "Try a smaller repository or an example."
+        )
+    selected = []
+    for entry in entries:
+        meta, raw = entry.split(b"\t", 1)
+        path = raw.decode("utf-8", errors="strict")
+        parts = PurePosixPath(path).parts
+        if (
+            len(path) > MAX_PATH_CHARS
+            or any(ord(c) < CONTROL_START or ord(c) == DELETE_CHARACTER for c in path)
+            or "\\" in path
+            or path.startswith(("/", "-"))
+            or ".." in parts
+            or any(p.lower() == ".git" for p in parts)
+        ):
+            raise JobError(
+                "filename",
+                "The repository contains unsupported filenames.",
+                "Use an example, or scan this repository locally after reviewing its paths.",
+            )
+        if meta.startswith(b"100") and not any(
+            p.startswith(".env") or p in {"node_modules", "vendor", "dist", "build"} for p in parts
+        ):
+            item = PurePosixPath(path)
+            if item.suffix.lower() in SOURCE_SUFFIXES or item.name in SOURCE_NAMES:
+                # Non-cone sparse patterns are anchored and escaped; never inject patterns through paths.
+                selected.append("/" + "".join("\\" + c if c in "*?[]!#\\" else c for c in path))
+    if not selected:
+        raise JobError(
+            "no_docs",
+            "No supported documentation or source files were found.",
+            "Try a repository with Markdown docs and Python or JavaScript source.",
+        )
+    git([*prefix, "sparse-checkout", "init", "--no-cone"], root, settings, deadline)
+    git(
+        [*prefix, "sparse-checkout", "set", "--no-cone", "--stdin"],
+        root,
+        settings,
+        deadline,
+        stdin=("\n".join(selected) + "\n").encode(),
+    )
+    git([*prefix, "checkout", "--quiet", "--detach", sha], root, settings, deadline)
+    if disk_bytes(root) > settings.max_bytes:
+        raise JobError("too_large", "The repository exceeded the 200 MB limit.", "Try a smaller repository.")
+    return repo

docs_drift/web/jobs.py

@@ -0,0 +1,352 @@
+"""Bounded replaceable queue, expiring LRU result store, and disposable workers."""
+
+import asyncio
+import hashlib
+import json
+import logging
+import os
+import re
+import subprocess
+import sys
+import time
+import uuid
+from collections import OrderedDict
+from contextlib import suppress
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any, Protocol
+
+from docs_drift.limits import CLONE_FAILURE_THRESHOLD, MAX_STORED_BYTES
+from docs_drift.models import ScanResult
+from docs_drift.web.cleanup import remove_workspace
+from docs_drift.web.git import JobError, disk_bytes, git_environment
+from docs_drift.web.settings import Settings
+from docs_drift.web.validation import Repository
+
+LOG = logging.getLogger("docs_drift.web")
+LOG.setLevel(logging.INFO)
+if not LOG.handlers:
+    LOG.addHandler(logging.StreamHandler())
+LOG.propagate = False
+STAGES = {"queued": 0, "cloning": 10, "indexing": 35, "matching": 60, "history": 80, "done": 100, "failed": 100}
+
+
+def _worker_process_options() -> dict[str, Any]:
+    if sys.platform == "win32":
+        return {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP}
+    return {"start_new_session": True}
+
+
+@dataclass
+class Job:
+    scan_id: str
+    repo_url: str
+    ref: str | None = None
+    status: str = "queued"
+    created: float = field(default_factory=time.time)
+    counts: dict[str, int] = field(default_factory=dict)
+    result: ScanResult | None = None
+    error: dict[str, str] | None = None
+    cached: bool = False
+    metrics: dict[str, float] = field(default_factory=dict)
+
+    def public(self) -> dict[str, Any]:
+        return {
+            "scan_id": self.scan_id,
+            "repo_url": self.repo_url,
+            "status": self.status,
+            "progress": STAGES[self.status],
+            "counts": self.counts,
+            "error": self.error,
+            "cached": self.cached,
+            "result": self.result.model_dump(mode="json") if self.result else None,
+            "metrics": self.metrics,
+        }
+
+
+class Store:
+    def __init__(self, settings: Settings):
+        self.settings = settings
+        self.jobs: OrderedDict[str, Job] = OrderedDict()
+        self.cache: dict[str, str] = {}
+        self.directory = settings.data_dir.resolve() / "results"
+        self.directory.mkdir(parents=True, exist_ok=True)
+
+    def get(self, ident: str) -> Job | None:
+        if not re.fullmatch(r"[a-zA-Z0-9-]{1,64}", ident):
+            return None
+        job = self.jobs.get(ident)
+        path = self.directory / f"{ident}.json"
+        if job is None and path.is_file() and path.stat().st_size <= MAX_STORED_BYTES:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            data["result"] = ScanResult.model_validate(data["result"]) if data.get("result") else None
+            job = Job(**data)
+        if job and time.time() - job.created < self.settings.ttl:
+            self.jobs[ident] = job
+            self.jobs.move_to_end(ident)
+            self.trim()
+            return job
+        self.jobs.pop(ident, None)
+        path.unlink(missing_ok=True)
+        return None
+
+    def trim(self) -> None:
+        while len(self.jobs) > self.settings.max_results:
+            inactive = next((k for k, v in self.jobs.items() if v.status in {"done", "failed"}), None)
+            if inactive is None:
+                break
+            self.jobs.pop(inactive)
+
+    def save(self, job: Job) -> None:
+        self.jobs[job.scan_id] = job
+        self.trim()
+        if job.status not in {"done", "failed"}:
+            return
+        data = asdict(job)
+        data["result"] = job.result.model_dump(mode="json") if job.result else None
+        path = self.directory / f"{job.scan_id}.json"
+        temporary = path.with_suffix(".tmp")
+        temporary.write_text(json.dumps(data), encoding="utf-8")
+        temporary.replace(path)
+        if job.result:
+            self.cache[self.key(job.repo_url, job.result.head_sha)] = job.scan_id
+        (self.directory / "cache.json").write_text(json.dumps(self.cache), encoding="utf-8")
+        self.expire()
+
+    @staticmethod
+    def key(url: str, sha: str) -> str:
+        return hashlib.sha256((url + "\0" + sha).encode()).hexdigest()
+
+    def expire(self) -> None:
+        files = sorted(
+            (p for p in self.directory.glob("*.json") if p.name != "cache.json"),
+            key=lambda p: p.stat().st_mtime,
+            reverse=True,
+        )
+        used = 0
+        for index, path in enumerate(files):
+            used += path.stat().st_size
+            if (
+                time.time() - path.stat().st_mtime > self.settings.ttl
+                or index >= self.settings.max_results
+                or used > self.settings.max_bytes
+            ):
+                path.unlink(missing_ok=True)
+                self.jobs.pop(path.stem, None)
+        self.cache = {key: ident for key, ident in self.cache.items() if (self.directory / f"{ident}.json").is_file()}
+        (self.directory / "cache.json").write_text(json.dumps(self.cache), encoding="utf-8")
+
+
+class QueueBackend(Protocol):
+    def submit(self, repository: Repository) -> Job: ...
+    async def close(self) -> None: ...
+
+
+class LocalQueue:
+    def __init__(self, settings: Settings, store: Store, test_runner: Any = None):
+        self.settings, self.store = settings, store
+        self.semaphore = asyncio.Semaphore(settings.max_concurrent)
+        self.tasks: set[asyncio.Task[None]] = set()
+        self.test_runner = test_runner  # Dependency injection only; no HTTP request can select it.
+        self.clone_failures = 0
+        self.circuit_until = 0.0
+        self.total, self.drift, self.failed = 0, 0, 0
+        self.work = settings.data_dir.resolve() / "work"
+        self.work.mkdir(parents=True, exist_ok=True)
+        cache = store.directory / "cache.json"
+        if cache.exists():
+            store.cache = json.loads(cache.read_text(encoding="utf-8"))
+
+    @property
+    def available(self) -> bool:
+        return self.settings.live_enabled and time.monotonic() >= self.circuit_until
+
+    def submit(self, repository: Repository) -> Job:
+        if len(self.tasks) >= self.settings.max_queue:
+            raise ValueError("The scan queue is full. Try an instant example or retry in a minute.")
+        job = Job(uuid.uuid4().hex, repository.url, repository.ref)
+        self.store.save(job)
+        task = asyncio.create_task(self.run(job))
+        self.tasks.add(task)
+        task.add_done_callback(self.tasks.discard)
+        return job
+
+    async def run(self, job: Job) -> None:  # noqa: PLR0912 - terminal states share mandatory cleanup.
+        async with self.semaphore:
+            start = time.monotonic()
+            workspace = self.work / job.scan_id
+            workspace.mkdir(mode=0o700)
+            try:
+                async with asyncio.timeout(self.settings.scan_timeout):
+                    if self.test_runner:
+                        await self.test_runner(job, workspace)
+                    else:
+                        await self.bounded_worker(job, workspace)
+                if job.result:
+                    job.status = "done"
+                    self.total += not job.cached
+                    self.drift += sum(f.is_drift for f in job.result.findings) if not job.cached else 0
+                    self.clone_failures = 0
+            except TimeoutError:
+                job.status, job.error = (
+                    "failed",
+                    {
+                        "kind": "timeout",
+                        "message": "The scan reached its 120 second time limit.",
+                        "hint": "Try a smaller repository or an instant example.",
+                    },
+                )
+            except JobError as exc:
+                job.status, job.error = "failed", {"kind": exc.kind, "message": str(exc), "hint": exc.hint}
+            except asyncio.CancelledError:
+                job.status, job.error = (
+                    "failed",
+                    {
+                        "kind": "restart",
+                        "message": "The service restarted during this scan.",
+                        "hint": "Please submit it again.",
+                    },
+                )
+                raise
+            except Exception:
+                job.status, job.error = (
+                    "failed",
+                    {
+                        "kind": "scan",
+                        "message": "This scan could not finish safely.",
+                        "hint": "Try an example or run the CLI locally.",
+                    },
+                )
+            finally:
+                if job.status == "failed":
+                    self.failed += 1
+                    if job.error and job.error["kind"] == "clone":
+                        self.clone_failures += 1
+                        if self.clone_failures >= CLONE_FAILURE_THRESHOLD:
+                            self.circuit_until = time.monotonic() + 300
+                job.metrics["duration_seconds"] = round(time.monotonic() - start, 3)
+                try:
+                    self.store.save(job)
+                finally:
+                    # Only trusted UUID workspaces are removed; child symlinks are not followed.
+                    if workspace.parent.resolve() == self.work and not workspace.is_symlink():
+                        cleanup = asyncio.create_task(asyncio.to_thread(remove_workspace, self.work, workspace))
+                        try:
+                            await asyncio.shield(cleanup)
+                        except asyncio.CancelledError:
+                            # Shutdown must wait for deletion already running in
+                            # a thread; cancelling its await does not stop it.
+                            await cleanup
+                            raise
+                LOG.info(
+                    json.dumps(
+                        {
+                            "event": "scan_finished",
+                            "repo": job.repo_url,
+                            "sha": job.result.head_sha if job.result else None,
+                            "duration": job.metrics["duration_seconds"],
+                            "counts": job.result.summary()["by_verdict"] if job.result else {},
+                            "failure": job.error["kind"] if job.error else None,
+                            "cached": job.cached,
+                        }
+                    )
+                )
+
+    async def bounded_worker(self, job: Job, workspace: Path) -> None:
+        async def monitor() -> None:
+            while True:
+                if await asyncio.to_thread(disk_bytes, workspace) > self.settings.max_bytes:
+                    raise JobError(
+                        "too_large",
+                        "The repository exceeded the 200 MB scan limit.",
+                        "Choose a smaller repository or a baked example.",
+                    )
+                await asyncio.sleep(0.5)
+
+        scan = asyncio.create_task(self.worker(job, workspace))
+        watch = asyncio.create_task(monitor())
+        try:
+            done, _ = await asyncio.wait({scan, watch}, return_when=asyncio.FIRST_COMPLETED)
+            for task in done:
+                task.result()
+        finally:
+            scan.cancel()
+            watch.cancel()
+            await asyncio.gather(scan, watch, return_exceptions=True)
+
+    async def worker(self, job: Job, workspace: Path) -> None:
+
+        options = _worker_process_options()
+        env = git_environment(workspace)
+        # Preserve trusted package import configuration, not credentials.
+        env["PYTHONPATH"] = str(Path(__file__).parents[2])
+        process = await asyncio.create_subprocess_exec(
+            sys.executable,
+            "-m",
+            "docs_drift.web.worker",
+            stdin=asyncio.subprocess.PIPE,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.DEVNULL,
+            env=env,
+            limit=1_000_000,
+            **options,
+        )
+        assert process.stdin and process.stdout
+        config = asdict(self.settings)
+        config["data_dir"] = str(self.settings.data_dir)
+        process.stdin.write(
+            (
+                json.dumps({"repo_url": job.repo_url, "ref": job.ref, "workspace": str(workspace), "settings": config})
+                + "\n"
+            ).encode()
+        )
+        await process.stdin.drain()
+        try:
+            while line := await process.stdout.readline():
+                data = json.loads(line)
+                if "resolved_sha" in data:
+                    cached_id = self.store.cache.get(self.store.key(job.repo_url, data["resolved_sha"]))
+                    cached = self.store.get(cached_id) if cached_id else None
+                    if cached and cached.result:
+                        job.result, job.cached = cached.result, True
+                        process.stdin.write(b"stop\n")
+                        await process.stdin.drain()
+                        break
+                    process.stdin.write(b"continue\n")
+                    await process.stdin.drain()
+                elif data.get("status") in STAGES:
+                    job.status = data["status"]
+                    job.counts = data.get("counts", job.counts)
+                    if job.status == "failed":
+                        job.error = data["error"]
+                    if job.status == "done":
+                        job.result = ScanResult.model_validate_json(
+                            (workspace / "result.json").read_text(encoding="utf-8")
+                        )
+                        job.metrics.update({k: data[k] for k in ("clone_seconds", "download_bytes")})
+            await process.wait()
+            if not job.result and job.status != "failed":
+                raise RuntimeError("Worker exited without a result")
+        finally:
+            if sys.platform != "win32":
+                with suppress(ProcessLookupError):
+                    os.killpg(process.pid, 9)
+            if process.returncode is None:
+                if os.name == "nt":
+                    killer = await asyncio.create_subprocess_exec(
+                        "taskkill",
+                        "/PID",
+                        str(process.pid),
+                        "/T",
+                        "/F",
+                        stdout=asyncio.subprocess.DEVNULL,
+                        stderr=asyncio.subprocess.DEVNULL,
+                    )
+                    await killer.wait()
+                await process.wait()
+
+    async def close(self) -> None:
+        tasks = list(self.tasks)
+        for task in tasks:
+            task.cancel()
+        await asyncio.gather(*tasks, return_exceptions=True)