llm-sunset 0.1.0__py3-none-any.whl

llm_sunset/__init__.py

@@ -0,0 +1,3 @@
+"""llm-sunset: find deprecated and soon-to-be-retired AI model IDs in your code."""
+
+__version__ = "0.1.0"

llm_sunset/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

llm_sunset/cli.py

@@ -0,0 +1,171 @@
+"""Command line interface for llm-sunset."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from datetime import date
+from typing import List, Optional
+
+from . import __version__, report
+from .data import load
+from .scanner import Matcher, scan
+
+EPILOG = """examples:
+  llm-sunset                         scan the current directory
+  llm-sunset src/ config/            scan specific paths
+  llm-sunset --provider openai       only check OpenAI deprecations
+  llm-sunset --format sarif > out.sarif
+  llm-sunset info gpt-4o-2024-05-13  look up a single model
+  llm-sunset upcoming --days 90      list every shutdown in the next 90 days
+
+Ignore a line with a comment containing:  llm-sunset: ignore
+Ignore a whole file with:                  llm-sunset: ignore-file
+"""
+
+
+def _today(value: Optional[str]) -> date:
+    return date.fromisoformat(value) if value else date.today()
+
+
+def _common(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--provider", action="append", metavar="NAME",
+                   help="only consider this provider (repeatable; e.g. openai, anthropic, google, azure, groq). "
+                        "Default: all direct providers; Azure/Vertex/Bedrock need an explicit "
+                        "--provider (or --provider all)")
+    p.add_argument("--offline", action="store_true", help="don't fetch live data; use cache or bundled snapshot")
+    p.add_argument("--data-file", metavar="PATH", help="use a local deprecations JSON file")
+    p.add_argument("--today", metavar="YYYY-MM-DD", help=argparse.SUPPRESS)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="llm-sunset",
+        description="Find AI model IDs in your code that are deprecated or about to be shut down.",
+        epilog=EPILOG,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    p.add_argument("--version", action="version", version=f"llm-sunset {__version__}")
+    sub = p.add_subparsers(dest="command")
+
+    s = sub.add_parser("scan", help="scan files (default command)")
+    s.add_argument("paths", nargs="*", default=["."])
+    s.add_argument("--format", choices=["text", "json", "markdown", "sarif", "github"], default="text")
+    s.add_argument("--fail-within", type=int, default=90, metavar="DAYS",
+                   help="treat models retiring within DAYS as errors (default: 90)")
+    s.add_argument("--warn-within", type=int, default=365, metavar="DAYS",
+                   help="hide models retiring more than DAYS from now (default: 365, -1 = show all)")
+    s.add_argument("--no-fail", action="store_true", help="always exit 0")
+    s.add_argument("--exclude", action="append", default=[], metavar="GLOB", help="exclude paths (repeatable)")
+    s.add_argument("--include-docs", action="store_true", help="also scan .md/.rst/.txt files")
+    _common(s)
+
+    i = sub.add_parser("info", help="show deprecation info for model IDs")
+    i.add_argument("models", nargs="+")
+    _common(i)
+
+    u = sub.add_parser("upcoming", help="list upcoming shutdowns")
+    u.add_argument("--days", type=int, default=180, help="look ahead this many days (default: 180)")
+    _common(u)
+    return p
+
+
+def cmd_scan(a: argparse.Namespace) -> int:
+    today = _today(a.today)
+    deps, source = load(offline=a.offline, data_file=a.data_file)
+    findings = scan(a.paths, Matcher(deps, a.provider), exclude=a.exclude, include_docs=a.include_docs)
+    if a.warn_within >= 0:
+        findings = [
+            f for f in findings
+            if f.deprecation.days_left(today) is None or f.deprecation.days_left(today) <= a.warn_within
+        ]
+
+    if a.format == "json":
+        print(report.render_json(findings, today, a.fail_within, source))
+    elif a.format == "markdown":
+        print(report.render_markdown(findings, today, a.fail_within, source))
+    elif a.format == "sarif":
+        print(report.render_sarif(findings, today, a.fail_within))
+    elif a.format == "github":
+        out = report.render_github(findings, today, a.fail_within)
+        if out:
+            print(out)
+        print(report.render_text(findings, today, a.fail_within, source))
+        summary = os.environ.get("GITHUB_STEP_SUMMARY")
+        if summary:
+            with open(summary, "a", encoding="utf-8") as fh:
+                fh.write(report.render_markdown(findings, today, a.fail_within, source) + "\n")
+    else:
+        print(report.render_text(findings, today, a.fail_within, source))
+
+    if a.no_fail:
+        return 0
+    return 1 if any(report.severity(f, today, a.fail_within) == "error" for f in findings) else 0
+
+
+def cmd_info(a: argparse.Namespace) -> int:
+    today = _today(a.today)
+    deps, source = load(offline=a.offline, data_file=a.data_file)
+    matcher = Matcher(deps, a.provider)
+    status = 0
+    for model in a.models:
+        hits = matcher.by_id.get(model, [])
+        if not hits:
+            print(f"{model}: no deprecation notice found")
+            continue
+        status = 1
+        for d in hits:
+            left = d.days_left(today)
+            if left is None:
+                when = "deprecated, no shutdown date announced"
+            elif left <= 0:
+                when = f"RETIRED on {d.shutdown_date}"
+            else:
+                when = f"retires {d.shutdown_date} ({left} days left)"
+            print(f"{model} [{d.provider}]: {when}")
+            if d.replacements:
+                print(f"    replace with: {', '.join(d.replacements)}")
+            if d.url:
+                print(f"    source: {d.url}")
+    print(f"(data: {source})", file=sys.stderr)
+    return status
+
+
+def cmd_upcoming(a: argparse.Namespace) -> int:
+    today = _today(a.today)
+    deps, source = load(offline=a.offline, data_file=a.data_file)
+    matcher = Matcher(deps, a.provider)
+    rows = []
+    for items in matcher.by_id.values():
+        for d in items:
+            left = d.days_left(today)
+            if left is not None and 0 < left <= a.days:
+                rows.append((d.shutdown_date, d))
+    rows.sort(key=lambda r: (r[0], r[1].provider, r[1].model_id))
+    if not rows:
+        print(f"No shutdowns in the next {a.days} days.")
+    for when, d in rows:
+        repl = f"  -> {', '.join(d.replacements)}" if d.replacements else ""
+        print(f"{when}  {d.provider:<14} {d.model_id}{repl}")
+    print(f"(data: {source})", file=sys.stderr)
+    return 0
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    argv = list(sys.argv[1:] if argv is None else argv)
+    known = {"scan", "info", "upcoming", "-h", "--help", "--version"}
+    if not argv or argv[0] not in known:
+        argv = ["scan"] + argv
+    a = build_parser().parse_args(argv)
+    handler = {"scan": cmd_scan, "info": cmd_info, "upcoming": cmd_upcoming}[a.command]
+    try:
+        return handler(a)
+    except BrokenPipeError:
+        return 0
+    except KeyboardInterrupt:
+        return 130
+
+
+if __name__ == "__main__":
+    sys.exit(main())

llm_sunset/data.py

@@ -0,0 +1,139 @@
+"""Load the AI model deprecation dataset.
+
+Data comes from https://deprecations.info (MIT licensed, refreshed daily by
+https://github.com/deprecations/deprecations-rss). We try, in order:
+
+1. a fresh local cache (< 24h old)
+2. the live feed
+3. a stale local cache
+4. the snapshot bundled with this package (works fully offline)
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+import urllib.request
+from dataclasses import dataclass, field
+from datetime import date
+from pathlib import Path
+from typing import Dict, Iterable, List, Optional, Tuple
+
+FEED_URLS = (
+    "https://deprecations.info/v1/deprecations.json",
+    "https://raw.githubusercontent.com/deprecations/deprecations-rss/main/data.json",
+)
+CACHE_TTL_SECONDS = 24 * 60 * 60
+SNAPSHOT_PATH = Path(__file__).with_name("snapshot.json")
+
+
+@dataclass(frozen=True)
+class Deprecation:
+    provider: str
+    model_id: str
+    shutdown_date: Optional[date]
+    deprecation_date: Optional[date]
+    replacements: Tuple[str, ...] = field(default_factory=tuple)
+    url: str = ""
+
+    def days_left(self, today: date) -> Optional[int]:
+        if self.shutdown_date is None:
+            return None
+        return (self.shutdown_date - today).days
+
+
+def _parse_date(value: object) -> Optional[date]:
+    if not value or not isinstance(value, str):
+        return None
+    try:
+        return date.fromisoformat(value[:10])
+    except ValueError:
+        return None
+
+
+def cache_path() -> Path:
+    base = os.environ.get("XDG_CACHE_HOME") or os.path.join(Path.home(), ".cache")
+    return Path(base) / "llm-sunset" / "deprecations.json"
+
+
+def _download(timeout: float) -> Optional[list]:
+    for url in FEED_URLS:
+        try:
+            req = urllib.request.Request(url, headers={"User-Agent": "llm-sunset"})
+            with urllib.request.urlopen(req, timeout=timeout) as resp:
+                data = json.loads(resp.read().decode("utf-8"))
+            if isinstance(data, list) and data:
+                return data
+        except Exception:
+            continue
+    return None
+
+
+def _read_json(path: Path) -> Optional[list]:
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+        return data if isinstance(data, list) else None
+    except Exception:
+        return None
+
+
+def load_raw(offline: bool = False, data_file: Optional[str] = None, timeout: float = 10.0) -> Tuple[list, str]:
+    """Return (records, source_description)."""
+    if data_file:
+        data = _read_json(Path(data_file))
+        if data is None:
+            raise SystemExit(f"llm-sunset: could not read data file {data_file}")
+        return data, data_file
+
+    cache = cache_path()
+    if not offline:
+        fresh = cache.exists() and time.time() - cache.stat().st_mtime < CACHE_TTL_SECONDS
+        if fresh:
+            data = _read_json(cache)
+            if data:
+                return data, "cache"
+        data = _download(timeout)
+        if data:
+            try:
+                cache.parent.mkdir(parents=True, exist_ok=True)
+                cache.write_text(json.dumps(data), encoding="utf-8")
+            except OSError:
+                pass
+            return data, "live"
+        data = _read_json(cache)
+        if data:
+            return data, "cache (stale)"
+
+    data = _read_json(SNAPSHOT_PATH)
+    if not data:
+        raise SystemExit("llm-sunset: bundled snapshot missing and feed unreachable")
+    return data, "bundled snapshot"
+
+
+def normalize(records: Iterable[dict]) -> List[Deprecation]:
+    """Turn raw records into Deprecation objects, one per (provider, model_id)."""
+    best: Dict[Tuple[str, str], Tuple[str, Deprecation]] = {}
+    for r in records:
+        model_id = (r.get("model_id") or "").strip()
+        provider = (r.get("provider") or "Unknown").strip()
+        if not model_id:
+            continue
+        dep = Deprecation(
+            provider=provider,
+            model_id=model_id,
+            shutdown_date=_parse_date(r.get("shutdown_date")),
+            deprecation_date=_parse_date(r.get("deprecation_date")),
+            replacements=tuple(x for x in (r.get("replacement_models") or []) if isinstance(x, str)),
+            url=r.get("url") or "",
+        )
+        key = (provider, model_id)
+        seen = r.get("last_observed") or r.get("announcement_date") or ""
+        if key not in best or seen >= best[key][0]:
+            best[key] = (seen, dep)
+    return [d for _, d in best.values()]
+
+
+def load(offline: bool = False, data_file: Optional[str] = None) -> Tuple[List[Deprecation], str]:
+    raw, source = load_raw(offline=offline, data_file=data_file)
+    return normalize(raw), source

llm_sunset/report.py

@@ -0,0 +1,191 @@
+"""Render findings as text, JSON, Markdown, SARIF or GitHub annotations."""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from datetime import date
+from typing import Dict, List
+
+from . import __version__
+from .scanner import Finding
+
+STATUS_ORDER = {"retired": 0, "retiring": 1, "deprecated": 2}
+
+
+def severity(f: Finding, today: date, fail_within: int) -> str:
+    left = f.deprecation.days_left(today)
+    if left is not None and left <= fail_within:
+        return "error"
+    return "warning"
+
+
+def describe(f: Finding, today: date) -> str:
+    d = f.deprecation
+    left = d.days_left(today)
+    if left is None:
+        when = "deprecated (no shutdown date announced)"
+    elif left <= 0:
+        when = f"RETIRED on {d.shutdown_date} ({-left} days ago)"
+    else:
+        when = f"retires {d.shutdown_date} ({left} days left)"
+    msg = f"{d.provider} model '{d.model_id}' {when}"
+    if d.replacements:
+        msg += f"; replace with: {', '.join(d.replacements)}"
+    return msg
+
+
+def _use_color(stream) -> bool:
+    return hasattr(stream, "isatty") and stream.isatty() and not os.environ.get("NO_COLOR")
+
+
+def render_text(findings: List[Finding], today: date, fail_within: int, source: str, files_note: str = "") -> str:
+    color = _use_color(sys.stdout)
+
+    def c(code: str, s: str) -> str:
+        return f"\033[{code}m{s}\033[0m" if color else s
+
+    if not findings:
+        return c("32", "✓ No deprecated AI model IDs found.") + f"  (data: {source})"
+
+    lines = []
+    for f in findings:
+        sev = severity(f, today, fail_within)
+        tag = c("31;1", "error  ") if sev == "error" else c("33", "warning")
+        lines.append(f"{c('1', f'{f.path}:{f.line}:{f.column}')}  {tag}  {describe(f, today)}")
+    errors = sum(1 for f in findings if severity(f, today, fail_within) == "error")
+    warnings = len(findings) - errors
+    lines.append("")
+    lines.append(
+        f"{len(findings)} finding(s): {c('31;1', str(errors) + ' error(s)')}, "
+        f"{c('33', str(warnings) + ' warning(s)')}  (errors = retired or retiring within {fail_within} days; data: {source})"
+    )
+    return "\n".join(lines)
+
+
+def _as_dict(f: Finding, today: date, fail_within: int) -> Dict:
+    d = f.deprecation
+    return {
+        "path": f.path,
+        "line": f.line,
+        "column": f.column,
+        "match": f.text,
+        "provider": d.provider,
+        "model_id": d.model_id,
+        "status": f.status(today),
+        "severity": severity(f, today, fail_within),
+        "shutdown_date": d.shutdown_date.isoformat() if d.shutdown_date else None,
+        "deprecation_date": d.deprecation_date.isoformat() if d.deprecation_date else None,
+        "days_left": d.days_left(today),
+        "replacements": list(d.replacements),
+        "url": d.url,
+    }
+
+
+def render_json(findings: List[Finding], today: date, fail_within: int, source: str) -> str:
+    return json.dumps(
+        {
+            "tool": "llm-sunset",
+            "version": __version__,
+            "date": today.isoformat(),
+            "data_source": source,
+            "findings": [_as_dict(f, today, fail_within) for f in findings],
+        },
+        indent=2,
+    )
+
+
+def render_markdown(findings: List[Finding], today: date, fail_within: int, source: str) -> str:
+    if not findings:
+        return "### 🌅 llm-sunset\n\n✅ No deprecated AI model IDs found.\n"
+    out = [
+        "### 🌅 llm-sunset: deprecated AI models found",
+        "",
+        "| | Location | Model | Provider | Shutdown | Replace with |",
+        "|---|---|---|---|---|---|",
+    ]
+    for f in findings:
+        d = f.deprecation
+        left = d.days_left(today)
+        if left is None:
+            when = "deprecated"
+        elif left <= 0:
+            when = f"**retired** {d.shutdown_date}"
+        else:
+            when = f"{d.shutdown_date} ({left}d)"
+        icon = "🔴" if severity(f, today, fail_within) == "error" else "🟡"
+        model = f"[`{d.model_id}`]({d.url})" if d.url else f"`{d.model_id}`"
+        repl = ", ".join(f"`{r}`" for r in d.replacements) or "—"
+        out.append(f"| {icon} | `{f.path}:{f.line}` | {model} | {d.provider} | {when} | {repl} |")
+    out += ["", f"<sub>Data: [deprecations.info](https://deprecations.info) ({source}) · llm-sunset {__version__}</sub>", ""]
+    return "\n".join(out)
+
+
+def render_github(findings: List[Finding], today: date, fail_within: int) -> str:
+    def esc(s: str) -> str:
+        return s.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A")
+
+    lines = []
+    for f in findings:
+        level = severity(f, today, fail_within)
+        title = f"Deprecated model: {f.deprecation.model_id}"
+        lines.append(
+            f"::{level} file={esc(f.path)},line={f.line},col={f.column},title={esc(title)}::{esc(describe(f, today))}"
+        )
+    return "\n".join(lines)
+
+
+def _uri(path: str) -> str:
+    p = path.replace(os.sep, "/")
+    while p.startswith("./"):
+        p = p[2:]
+    return p
+
+
+def render_sarif(findings: List[Finding], today: date, fail_within: int) -> str:
+    rules = {}
+    results = []
+    for f in findings:
+        d = f.deprecation
+        rule_id = f"{d.provider}/{d.model_id}"
+        if rule_id not in rules:
+            rules[rule_id] = {
+                "id": rule_id,
+                "name": "DeprecatedAIModel",
+                "shortDescription": {"text": f"Deprecated AI model {d.model_id} ({d.provider})"},
+                "helpUri": d.url or "https://deprecations.info",
+            }
+        results.append(
+            {
+                "ruleId": rule_id,
+                "level": severity(f, today, fail_within),
+                "message": {"text": describe(f, today)},
+                "locations": [
+                    {
+                        "physicalLocation": {
+                            "artifactLocation": {"uri": _uri(f.path)},
+                            "region": {"startLine": f.line, "startColumn": f.column},
+                        }
+                    }
+                ],
+            }
+        )
+    sarif = {
+        "$schema": "https://json.schemastore.org/sarif-2.1.0.json",
+        "version": "2.1.0",
+        "runs": [
+            {
+                "tool": {
+                    "driver": {
+                        "name": "llm-sunset",
+                        "version": __version__,
+                        "informationUri": "https://github.com/Ashveil1/llm-sunset",
+                        "rules": list(rules.values()),
+                    }
+                },
+                "results": results,
+            }
+        ],
+    }
+    return json.dumps(sarif, indent=2)

llm_sunset/scanner.py

@@ -0,0 +1,173 @@
+"""Find references to deprecated AI model IDs in source files."""
+
+from __future__ import annotations
+
+import bisect
+import fnmatch
+import os
+import re
+from dataclasses import dataclass
+from datetime import date
+from pathlib import Path
+from typing import Dict, Iterable, Iterator, List, Optional, Sequence
+
+from .data import Deprecation
+
+DEFAULT_EXCLUDE_DIRS = {
+    ".git", ".hg", ".svn", "node_modules", "bower_components", "vendor",
+    ".venv", "venv", "env", ".env.d", "__pycache__", ".mypy_cache", ".pytest_cache",
+    ".ruff_cache", ".tox", ".nox", "dist", "build", "out", "target", ".next",
+    ".nuxt", ".svelte-kit", ".turbo", ".cache", "coverage", ".idea", ".vscode",
+    "site-packages",
+}
+DEFAULT_EXCLUDE_GLOBS = [
+    "*.lock", "package-lock.json", "pnpm-lock.yaml", "yarn.lock", "poetry.lock",
+    "uv.lock", "Cargo.lock", "go.sum", "*.min.js", "*.map", "*.svg",
+    "*.png", "*.jpg", "*.jpeg", "*.gif", "*.webp", "*.ico", "*.pdf", "*.zip",
+    "*.gz", "*.tar", "*.woff", "*.woff2", "*.ttf", "*.mp3", "*.mp4", "*.wasm",
+    "*.so", "*.dylib", "*.dll", "*.exe", "*.bin", "*.pyc", "*.ipynb_checkpoints",
+]
+# Prose files mention old models all the time ("we migrated from gpt-3.5-turbo").
+DOC_GLOBS = ["*.md", "*.mdx", "*.rst", "*.txt", "*.adoc", "CHANGELOG*", "HISTORY*"]
+
+MAX_FILE_BYTES = 2 * 1024 * 1024
+IGNORE_LINE = "llm-sunset: ignore"
+IGNORE_FILE = "llm-sunset: ignore-file"
+
+# Characters that may legally sit right before / after a model ID. "/" ":" and
+# "@" are allowed so that "openai/gpt-4", "ft:gpt-4o:org::id" and
+# "claude-3-5-sonnet@20240620" still match.
+_LEFT = r"(?<![A-Za-z0-9._+-])"
+_RIGHT = r"(?![A-Za-z0-9_+-])(?!\.[A-Za-z0-9])"
+
+# Same character classes as _LEFT/_RIGHT; a token never ends with ".".
+_TOKEN_RE = re.compile(r"[A-Za-z0-9_+-](?:[A-Za-z0-9._+-]*[A-Za-z0-9_+-])?")
+
+# Platforms that resell models under their own (often very different) lifecycle
+# dates. Off by default so OpenAI/Anthropic users don't get Azure dates.
+PLATFORM_PROVIDERS = ("azure", "google vertex", "bedrock")
+
+
+@dataclass(frozen=True)
+class Finding:
+    path: str
+    line: int
+    column: int
+    text: str  # exact text matched in the file
+    deprecation: Deprecation
+
+    def status(self, today: date) -> str:
+        left = self.deprecation.days_left(today)
+        if left is None:
+            return "deprecated"
+        return "retired" if left <= 0 else "retiring"
+
+
+class Matcher:
+    def __init__(self, deprecations: Sequence[Deprecation], providers: Optional[Sequence[str]] = None):
+        wanted = [p.lower() for p in providers] if providers else []
+        everything = "all" in wanted
+        self.by_id: Dict[str, List[Deprecation]] = {}
+        for d in deprecations:
+            prov = d.provider.lower()
+            if everything:
+                pass
+            elif wanted:
+                if not any(w in prov for w in wanted):
+                    continue
+            elif any(p in prov for p in PLATFORM_PROVIDERS):
+                continue
+            self.by_id.setdefault(d.model_id, []).append(d)
+
+        self._ids = frozenset(self.by_id)
+        # IDs like "babbage", "davinci" or "command" are ordinary words; only
+        # match them when they are the entire contents of a string literal.
+        self._words = frozenset(i for i in self.by_id if not any(c.isdigit() for c in i))
+
+    def find(self, text: str) -> Iterator[tuple]:
+        """Yield (offset, matched_id, [Deprecation]) for every hit in text."""
+        # Fast path: tokenize + set intersection both run in C.
+        hits = self._ids.intersection(_TOKEN_RE.findall(text))
+        if not hits:
+            return
+        ids = sorted(hits - self._words, key=len, reverse=True)
+        words = sorted(hits & self._words, key=len, reverse=True)
+        if ids:
+            rx = re.compile(_LEFT + "(" + "|".join(map(re.escape, ids)) + ")" + _RIGHT)
+            for m in rx.finditer(text):
+                yield m.start(1), m.group(1), self.by_id[m.group(1)]
+        if words:
+            rx = re.compile(r"""(["'`])(""" + "|".join(map(re.escape, words)) + r")\1")
+            for m in rx.finditer(text):
+                yield m.start(2), m.group(2), self.by_id[m.group(2)]
+
+    def match_line(self, line: str) -> Iterator[tuple]:
+        return self.find(line)
+
+
+def _compile_globs(globs: Iterable[str]):
+    globs = list(globs)
+    if not globs:
+        return lambda rel, name: False
+    rx = re.compile("|".join(fnmatch.translate(g) for g in globs))
+    return lambda rel, name: bool(rx.match(name) or rx.match(rel.replace(os.sep, "/")))
+
+
+def iter_files(paths: Sequence[str], exclude: Sequence[str], include_docs: bool) -> Iterator[Path]:
+    globs = list(DEFAULT_EXCLUDE_GLOBS) + list(exclude)
+    if not include_docs:
+        globs += DOC_GLOBS
+    file_excluded = _compile_globs(globs)
+    dir_excluded = _compile_globs(exclude)
+    for p in paths:
+        root = Path(p)
+        if root.is_file():
+            yield root  # explicitly named files are always scanned
+            continue
+        for dirpath, dirnames, filenames in os.walk(root):
+            rel_dir = os.path.relpath(dirpath, root)
+            dirnames[:] = [
+                d for d in dirnames
+                if d not in DEFAULT_EXCLUDE_DIRS
+                and not dir_excluded(os.path.normpath(os.path.join(rel_dir, d)), d)
+            ]
+            for f in filenames:
+                if not file_excluded(os.path.normpath(os.path.join(rel_dir, f)), f):
+                    yield Path(dirpath) / f
+
+
+def scan_file(path: Path, matcher: Matcher) -> List[Finding]:
+    try:
+        with open(path, "rb") as fh:
+            raw = fh.read(MAX_FILE_BYTES + 1)
+    except OSError:
+        return []
+    if len(raw) > MAX_FILE_BYTES:
+        return []
+    if b"\0" in raw[:8192]:
+        return []
+    text = raw.decode("utf-8", errors="replace")
+    if IGNORE_FILE in text:
+        return []
+    findings = []
+    line_starts = None
+    for offset, matched, deps in matcher.find(text):
+        if line_starts is None:
+            line_starts = [0] + [m.end() for m in re.finditer(r"\n", text)]
+        idx = bisect.bisect_right(line_starts, offset) - 1
+        start = line_starts[idx]
+        end = text.find("\n", start)
+        line = text[start:] if end == -1 else text[start:end]
+        if IGNORE_LINE in line:
+            continue
+        for d in deps:
+            findings.append(Finding(str(path), idx + 1, offset - start + 1, matched, d))
+    return findings
+
+
+def scan(paths: Sequence[str], matcher: Matcher, exclude: Sequence[str] = (), include_docs: bool = False) -> List[Finding]:
+    out: List[Finding] = []
+    for f in iter_files(paths, exclude, include_docs):
+        out.extend(scan_file(f, matcher))
+    out.sort(key=lambda x: (x.path, x.line, x.column, x.deprecation.provider))
+    return out