ghostpkg 0.1.0__py3-none-any.whl

ghostpkg/__init__.py

@@ -0,0 +1,16 @@
+"""ghostpkg -- catch package names that do not exist before you install them."""
+
+__version__ = "0.1.0"
+
+from .assess import Finding, Verdict, assess
+from .registries import PackageFacts, RegistryError, fetch
+
+__all__ = [
+    "Finding",
+    "PackageFacts",
+    "RegistryError",
+    "Verdict",
+    "assess",
+    "fetch",
+    "__version__",
+]

ghostpkg/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

ghostpkg/assess.py

@@ -0,0 +1,144 @@
+"""Risk assessment for a package name.
+
+The policy here is shaped by a measurement, not by taste. Validation against
+the live PyPI feed showed that "young, one release, no repository link"
+describes a malicious slopsquat and an honest new project equally well: a
+detector that blocks on youth flags 100% of legitimate brand-new packages.
+
+So the default profile blocks on exactly one thing -- the package does not
+exist -- because that signal is precise and it is the one that actually
+corresponds to a hallucination. Everything softer is reported as a warning
+and left to a human. `--strict` promotes warnings to blocks for people who
+want that trade, but it is not the default and it is not recommended for CI
+that installs new packages.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+
+from .data import TOP_PYPI
+from .registries import PackageFacts
+
+YOUNG_DAYS = 90
+NEW_DAYS = 365
+
+
+class Verdict(str, Enum):
+    OK = "OK"
+    WARN = "WARN"
+    BLOCK = "BLOCK"
+
+
+@dataclass
+class Finding:
+    name: str
+    ecosystem: str
+    verdict: Verdict
+    reasons: list[str] = field(default_factory=list)
+    facts: PackageFacts | None = None
+
+    @property
+    def is_blocked(self) -> bool:
+        return self.verdict is Verdict.BLOCK
+
+
+def edit_distance(left: str, right: str, cutoff: int = 3) -> int:
+    """Levenshtein distance, abandoning early once it exceeds `cutoff`."""
+    if left == right:
+        return 0
+    if abs(len(left) - len(right)) > cutoff:
+        return cutoff + 1
+
+    previous = list(range(len(right) + 1))
+    for i, a in enumerate(left, 1):
+        current = [i]
+        for j, b in enumerate(right, 1):
+            current.append(
+                min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + (a != b))
+            )
+        if min(current) > cutoff:
+            return cutoff + 1
+        previous = current
+    return previous[-1]
+
+
+def _typo_budget(name: str) -> int:
+    """How many edits still count as a plausible typo of a popular name.
+
+    Short names are inherently close to each other -- 'flask', 'black' and
+    'click' sit within two edits -- so a flat budget produces false positives
+    on exactly the packages people use most.
+    """
+    return 2 if len(name) >= 10 else 1
+
+
+def nearest_popular(name: str, popular: frozenset[str] = TOP_PYPI) -> tuple[str, int] | None:
+    """Closest popular package name within the typo budget, if any."""
+    lowered = name.lower()
+    if lowered in popular:
+        return None
+
+    budget = _typo_budget(lowered)
+    best: tuple[str, int] | None = None
+    for candidate in popular:
+        if abs(len(candidate) - len(lowered)) > budget:
+            continue
+        distance = edit_distance(lowered, candidate, cutoff=budget)
+        if 0 < distance <= budget and (best is None or distance < best[1]):
+            best = (candidate, distance)
+            if distance == 1:
+                break
+    return best
+
+
+def assess(facts: PackageFacts, strict: bool = False) -> Finding:
+    if not facts.exists:
+        return Finding(
+            name=facts.name,
+            ecosystem=facts.ecosystem,
+            verdict=Verdict.BLOCK,
+            reasons=[f"does not exist on {facts.ecosystem}"],
+            facts=facts,
+        )
+
+    reasons: list[str] = []
+
+    if facts.age_days is not None:
+        if facts.age_days < YOUNG_DAYS:
+            reasons.append(f"first published {facts.age_days} days ago")
+        elif facts.age_days < NEW_DAYS:
+            reasons.append(f"first published {facts.age_days} days ago (under a year)")
+
+    is_young = facts.age_days is not None and facts.age_days < NEW_DAYS
+
+    if is_young:
+        neighbour = nearest_popular(facts.name)
+        if neighbour is not None:
+            popular_name, distance = neighbour
+            reasons.append(
+                f"{distance} character{'s' if distance > 1 else ''} away from "
+                f"'{popular_name}', and recently published"
+            )
+
+    if is_young and facts.release_count <= 1:
+        reasons.append("only one release")
+
+    if is_young and not facts.has_repo_url:
+        reasons.append("no repository or homepage link")
+
+    if not reasons:
+        verdict = Verdict.OK
+    elif strict:
+        verdict = Verdict.BLOCK
+    else:
+        verdict = Verdict.WARN
+
+    return Finding(
+        name=facts.name,
+        ecosystem=facts.ecosystem,
+        verdict=verdict,
+        reasons=reasons,
+        facts=facts,
+    )

ghostpkg/cli.py

@@ -0,0 +1,204 @@
+"""Command line interface."""
+
+from __future__ import annotations
+
+import argparse
+import concurrent.futures
+import json
+import os
+import re
+import sys
+from pathlib import Path
+
+from .assess import Finding, Verdict, assess
+from .registries import RegistryError, fetch
+
+EXIT_OK = 0
+EXIT_BLOCKED = 1
+EXIT_ERROR = 2
+
+MAX_WORKERS = 8
+
+
+def _use_colour(stream) -> bool:
+    if os.environ.get("NO_COLOR"):
+        return False
+    return hasattr(stream, "isatty") and stream.isatty()
+
+
+class Palette:
+    def __init__(self, enabled: bool) -> None:
+        self.enabled = enabled
+
+    def _wrap(self, code: str, text: str) -> str:
+        return f"\033[{code}m{text}\033[0m" if self.enabled else text
+
+    def red(self, text: str) -> str:
+        return self._wrap("31;1", text)
+
+    def yellow(self, text: str) -> str:
+        return self._wrap("33;1", text)
+
+    def green(self, text: str) -> str:
+        return self._wrap("32", text)
+
+    def dim(self, text: str) -> str:
+        return self._wrap("2", text)
+
+
+MARKS = {
+    Verdict.BLOCK: ("BLOCKED", "red"),
+    Verdict.WARN: ("WARNING", "yellow"),
+    Verdict.OK: ("ok", "green"),
+}
+
+
+def render(findings: list[Finding], palette: Palette, quiet: bool) -> None:
+    for finding in findings:
+        label, colour = MARKS[finding.verdict]
+        paint = getattr(palette, colour)
+        if finding.verdict is Verdict.OK:
+            if quiet:
+                continue
+            detail = ""
+            if finding.facts and finding.facts.age_days is not None:
+                years = finding.facts.age_days / 365.0
+                detail = palette.dim(
+                    f"  ({finding.facts.release_count} releases, {years:.1f}y old)"
+                )
+            print(f"  {paint(label):<8} {finding.name}{detail}")
+            continue
+
+        print(f"  {paint(label):<8} {finding.name}")
+        for reason in finding.reasons:
+            print(f"           {palette.dim('- ' + reason)}")
+
+
+def summarise(findings: list[Finding], palette: Palette) -> None:
+    blocked = [f for f in findings if f.verdict is Verdict.BLOCK]
+    warned = [f for f in findings if f.verdict is Verdict.WARN]
+
+    print()
+    if blocked:
+        names = ", ".join(f.name for f in blocked)
+        print(palette.red(f"  {len(blocked)} blocked: {names}"))
+    if warned:
+        print(palette.yellow(f"  {len(warned)} to review by hand"))
+    if not blocked and not warned:
+        print(palette.green(f"  all {len(findings)} packages look fine"))
+
+
+def evaluate(names: list[str], ecosystem: str, strict: bool) -> list[Finding]:
+    def one(name: str) -> Finding:
+        return assess(fetch(name, ecosystem), strict=strict)
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
+        return list(pool.map(one, names))
+
+
+REQUIREMENT_LINE = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*)")
+
+
+def parse_requirements(text: str) -> list[str]:
+    names = []
+    for raw in text.splitlines():
+        line = raw.strip()
+        if not line or line.startswith(("#", "-", "git+", "http")):
+            continue
+        match = REQUIREMENT_LINE.match(line)
+        if match:
+            names.append(match.group(1))
+    return names
+
+
+def parse_package_json(text: str) -> list[str]:
+    data = json.loads(text)
+    names: list[str] = []
+    for key in ("dependencies", "devDependencies", "optionalDependencies"):
+        names.extend((data.get(key) or {}).keys())
+    return names
+
+
+def load_manifest(path: Path) -> tuple[list[str], str]:
+    text = path.read_text(encoding="utf-8")
+    if path.name == "package.json":
+        return parse_package_json(text), "npm"
+    return parse_requirements(text), "pypi"
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="ghostpkg",
+        description="Catch package names that do not exist before you install them.",
+    )
+    parser.add_argument("--version", action="version", version="ghostpkg 0.1.0")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    check = sub.add_parser("check", help="check one or more package names")
+    check.add_argument("names", nargs="+")
+    check.add_argument("-e", "--ecosystem", choices=("pypi", "npm"), default="pypi")
+
+    scan = sub.add_parser("scan", help="check every dependency in a manifest")
+    scan.add_argument("path", type=Path, help="requirements.txt or package.json")
+
+    for command in (check, scan):
+        command.add_argument(
+            "--strict",
+            action="store_true",
+            help="treat warnings as blocking (flags legitimate new packages too)",
+        )
+        command.add_argument("--json", action="store_true", help="machine-readable output")
+        command.add_argument("-q", "--quiet", action="store_true", help="hide passing packages")
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+
+    if args.command == "scan":
+        if not args.path.exists():
+            print(f"ghostpkg: no such file: {args.path}", file=sys.stderr)
+            return EXIT_ERROR
+        try:
+            names, ecosystem = load_manifest(args.path)
+        except (json.JSONDecodeError, UnicodeDecodeError) as exc:
+            print(f"ghostpkg: could not parse {args.path}: {exc}", file=sys.stderr)
+            return EXIT_ERROR
+        if not names:
+            print(f"ghostpkg: no dependencies found in {args.path}", file=sys.stderr)
+            return EXIT_OK
+    else:
+        names, ecosystem = args.names, args.ecosystem
+
+    try:
+        findings = evaluate(names, ecosystem, args.strict)
+    except RegistryError as exc:
+        print(f"ghostpkg: {exc}", file=sys.stderr)
+        return EXIT_ERROR
+
+    if args.json:
+        print(
+            json.dumps(
+                [
+                    {
+                        "name": f.name,
+                        "ecosystem": f.ecosystem,
+                        "verdict": f.verdict.value,
+                        "reasons": f.reasons,
+                    }
+                    for f in findings
+                ],
+                indent=2,
+            )
+        )
+    else:
+        palette = Palette(_use_colour(sys.stdout))
+        render(findings, palette, args.quiet)
+        summarise(findings, palette)
+
+    return EXIT_BLOCKED if any(f.is_blocked for f in findings) else EXIT_OK
+
+
+if __name__ == "__main__":
+    sys.exit(main())