pkgwhy 1.0.0__py3-none-any.whl

pkgwhy/typosquat/__init__.py

@@ -0,0 +1,2 @@
+"""Typosquatting similarity detection."""
+

pkgwhy/typosquat/detector.py

@@ -0,0 +1,182 @@
+from __future__ import annotations
+
+import re
+
+from pkgwhy.core.models import TyposquatCandidate
+from pkgwhy.metadata.installed import normalize_package_name
+from pkgwhy.typosquat.popular_packages import KNOWN_LEGITIMATE_FAMILIES, POPULAR_PACKAGE_REFERENCES
+
+HOMOGLYPH_TRANSLATION = str.maketrans(
+    {
+        "0": "o",
+        "1": "l",
+        "3": "e",
+        "4": "a",
+        "5": "s",
+        "7": "t",
+        "@": "a",
+        "$": "s",
+        "\u0430": "a",
+        "\u0435": "e",
+        "\u043e": "o",
+        "\u0440": "p",
+        "\u0441": "c",
+        "\u0445": "x",
+        "\u0443": "y",
+        "\u0391": "a",
+        "\u0395": "e",
+        "\u039f": "o",
+        "\u03a1": "p",
+        "\u03a7": "x",
+        "\u03bf": "o",
+    }
+)
+
+
+def detect_typosquat(package_name: str) -> TyposquatCandidate | None:
+    """Return the strongest conservative typosquat signal for a package name."""
+
+    normalized = normalize_package_name(package_name)
+    if normalized in KNOWN_LEGITIMATE_FAMILIES:
+        return None
+
+    best: TyposquatCandidate | None = None
+    for target, references in POPULAR_PACKAGE_REFERENCES.items():
+        if normalized == normalize_package_name(target):
+            continue
+        if _is_known_family_package(normalized, target):
+            continue
+        for reference in references:
+            candidate = _compare_to_reference(package_name, normalized, target, reference)
+            if candidate is None:
+                continue
+            if best is None or candidate.similarity > best.similarity:
+                best = candidate
+    return best
+
+
+def detect_typosquats(package_names: list[str]) -> list[TyposquatCandidate]:
+    candidates = [detect_typosquat(name) for name in package_names]
+    return sorted(
+        [candidate for candidate in candidates if candidate is not None],
+        key=lambda item: (-item.similarity, -len(item.signals), item.package),
+    )
+
+
+def _compare_to_reference(
+    package_name: str,
+    normalized: str,
+    target: str,
+    reference: str,
+) -> TyposquatCandidate | None:
+    reference_normalized = normalize_package_name(reference)
+    candidate_compact = _compact(normalized)
+    reference_compact = _compact(reference_normalized)
+    candidate_homoglyph = _homoglyph_normalize(candidate_compact)
+    reference_homoglyph = _homoglyph_normalize(reference_compact)
+
+    signals: list[str] = []
+    evidence: list[str] = []
+    distance = _levenshtein(candidate_compact, reference_compact)
+    max_length = max(len(candidate_compact), len(reference_compact), 1)
+    similarity = 1 - (distance / max_length)
+
+    if distance <= 2 and similarity >= 0.72:
+        signals.append("edit_distance")
+        evidence.append(f"Edit distance to popular package reference '{reference}' is {distance}.")
+    if _is_adjacent_transposition(candidate_compact, reference_compact):
+        signals.append("adjacent_transposition")
+        evidence.append(f"Name appears to transpose adjacent characters from '{reference}'.")
+        # Adjacent swaps are common typos even when edit-distance similarity underweights them.
+        similarity = max(similarity, 0.92)
+    if _is_single_missing_or_extra_character(candidate_compact, reference_compact):
+        signals.append("missing_or_extra_character")
+        evidence.append(f"Name differs from '{reference}' by one missing or extra character.")
+        # Single missing/extra characters are strong typo signals for short package names.
+        similarity = max(similarity, 0.88)
+    if candidate_homoglyph == reference_homoglyph and candidate_compact != reference_compact:
+        signals.append("homoglyph_or_lookalike")
+        evidence.append(f"Name normalizes to the same lookalike form as '{reference}'.")
+        # Lookalike normalization catches visual similarity that plain edit distance can miss.
+        similarity = max(similarity, 0.95)
+
+    if len(signals) == 0:
+        return None
+
+    return TyposquatCandidate(
+        package=package_name,
+        normalized_package=normalized,
+        possible_target=target,
+        matched_reference=reference,
+        similarity=round(similarity, 3),
+        signals=sorted(set(signals)),
+        is_possible_typosquat=True,
+        recommendation="Possible typosquatting risk. Review package identity, source, maintainer, and purpose before use.",
+        evidence=evidence,
+    )
+
+
+def _is_known_family_package(normalized: str, target: str) -> bool:
+    target_normalized = normalize_package_name(target)
+    return (
+        normalized.startswith(f"{target_normalized}-")
+        or normalized.endswith(f"-{target_normalized}")
+    )
+
+
+def _compact(value: str) -> str:
+    return re.sub(r"[-_.]+", "", value.lower())
+
+
+def _homoglyph_normalize(value: str) -> str:
+    return value.translate(HOMOGLYPH_TRANSLATION)
+
+
+def _is_adjacent_transposition(candidate: str, reference: str) -> bool:
+    if len(candidate) != len(reference) or candidate == reference:
+        return False
+    differences = [index for index, (left, right) in enumerate(zip(candidate, reference, strict=True)) if left != right]
+    if len(differences) != 2:
+        return False
+    first, second = differences
+    return second == first + 1 and candidate[first] == reference[second] and candidate[second] == reference[first]
+
+
+def _is_single_missing_or_extra_character(candidate: str, reference: str) -> bool:
+    if abs(len(candidate) - len(reference)) != 1:
+        return False
+    shorter, longer = sorted((candidate, reference), key=len)
+    index = 0
+    skipped = False
+    for char in longer:
+        if index < len(shorter) and shorter[index] == char:
+            index += 1
+        elif skipped:
+            return False
+        else:
+            skipped = True
+    return True
+
+
+def _levenshtein(left: str, right: str) -> int:
+    if left == right:
+        return 0
+    if not left:
+        return len(right)
+    if not right:
+        return len(left)
+
+    previous = list(range(len(right) + 1))
+    for left_index, left_char in enumerate(left, start=1):
+        current = [left_index]
+        for right_index, right_char in enumerate(right, start=1):
+            substitution_cost = 0 if left_char == right_char else 1
+            current.append(
+                min(
+                    previous[right_index] + 1,
+                    current[right_index - 1] + 1,
+                    previous[right_index - 1] + substitution_cost,
+                )
+            )
+        previous = current
+    return previous[-1]

pkgwhy/typosquat/popular_packages.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+# Curated subset of commonly typosquatted PyPI targets, not an exhaustive index.
+POPULAR_PACKAGE_REFERENCES: dict[str, tuple[str, ...]] = {
+    "requests": ("requests",),
+    "numpy": ("numpy",),
+    "pandas": ("pandas",),
+    "django": ("django",),
+    "flask": ("flask",),
+    "fastapi": ("fastapi",),
+    "scikit-learn": ("scikit-learn", "sklearn"),
+    "scipy": ("scipy",),
+    "matplotlib": ("matplotlib",),
+    "pytest": ("pytest",),
+    "setuptools": ("setuptools",),
+    "pip": ("pip",),
+    "wheel": ("wheel",),
+    "typer": ("typer",),
+    "click": ("click",),
+    "rich": ("rich",),
+    "pydantic": ("pydantic",),
+    "httpx": ("httpx",),
+    "cryptography": ("cryptography",),
+    "sqlalchemy": ("sqlalchemy",),
+    "beautifulsoup4": ("beautifulsoup4", "bs4"),
+    "pillow": ("pillow", "pil"),
+}
+
+KNOWN_LEGITIMATE_FAMILIES: set[str] = {
+    "django-debug-toolbar",
+    "pytest-cov",
+    "pandas-stubs",
+    "types-requests",
+}

pkgwhy/vulnerabilities/__init__.py

@@ -0,0 +1,2 @@
+"""Known-vulnerability parsing and matching boundaries."""
+

pkgwhy/vulnerabilities/matching.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+from packaging.utils import canonicalize_name
+from packaging.version import InvalidVersion, Version
+
+from pkgwhy.core.models import Confidence, VulnerabilityMatch, VulnerabilityRecord, VulnerabilityRange
+
+
+def match_vulnerabilities(package: str, version: str | None, records: list[VulnerabilityRecord]) -> list[VulnerabilityMatch]:
+    """Return conservative matches for one package/version against advisory records."""
+    if version is None:
+        return []
+    matches: dict[str, VulnerabilityMatch] = {}
+    for record in records:
+        match = match_vulnerability(package, version, record)
+        if match is not None:
+            matches.setdefault(match.vulnerability_id, match)
+    return sorted(matches.values(), key=lambda item: item.vulnerability_id)
+
+
+def match_vulnerability(package: str, version: str, record: VulnerabilityRecord) -> VulnerabilityMatch | None:
+    if canonicalize_name(record.package_name) != canonicalize_name(package):
+        return None
+
+    evidence: list[str] = []
+    if _version_in_list(version, record.affected_versions):
+        evidence.append(f"Version {version} is explicitly listed as affected by {record.id}.")
+        return _build_match(record, package, version, evidence)
+
+    for affected_range in record.affected_ranges:
+        if _version_in_range(version, affected_range):
+            evidence.append(
+                f"Version {version} matched affected range "
+                f"introduced={affected_range.introduced or 'unknown'} "
+                f"fixed={affected_range.fixed or 'none'} "
+                f"last_affected={affected_range.last_affected or 'none'} "
+                f"limit={affected_range.limit or 'none'}."
+            )
+            return _build_match(record, package, version, evidence)
+
+    return None
+
+
+def _version_in_range(version: str, affected_range: VulnerabilityRange) -> bool:
+    if not any(
+        (affected_range.introduced, affected_range.fixed, affected_range.last_affected, affected_range.limit)
+    ):
+        return False
+    range_type = affected_range.range_type.upper() if affected_range.range_type is not None else None
+    if range_type not in {None, "ECOSYSTEM", "PYPI"}:
+        return False
+
+    try:
+        parsed_version = Version(version)
+    except InvalidVersion:
+        return False
+
+    if affected_range.introduced not in {None, "", "0"}:
+        try:
+            if parsed_version < Version(affected_range.introduced):
+                return False
+        except InvalidVersion:
+            return False
+
+    if affected_range.fixed:
+        try:
+            if parsed_version >= Version(affected_range.fixed):
+                return False
+        except InvalidVersion:
+            return False
+
+    if affected_range.last_affected:
+        try:
+            if parsed_version > Version(affected_range.last_affected):
+                return False
+        except InvalidVersion:
+            return False
+
+    if affected_range.limit:
+        try:
+            if parsed_version >= Version(affected_range.limit):
+                return False
+        except InvalidVersion:
+            return False
+
+    return True
+
+
+def _version_in_list(version: str, candidates: list[str]) -> bool:
+    if version in candidates:
+        return True
+    try:
+        parsed_version = Version(version)
+    except InvalidVersion:
+        return False
+    for candidate in candidates:
+        try:
+            if parsed_version == Version(candidate):
+                return True
+        except InvalidVersion:
+            continue
+    return False
+
+
+def _build_match(record: VulnerabilityRecord, package: str, version: str, evidence: list[str]) -> VulnerabilityMatch:
+    evidence.append(f"Advisory source: {record.source}.")
+    if record.source_url:
+        evidence.append(f"Advisory URL: {record.source_url}.")
+    return VulnerabilityMatch(
+        vulnerability_id=record.id,
+        package=package,
+        version=version,
+        aliases=record.aliases,
+        summary=record.summary,
+        severity=record.severity,
+        fixed_versions=record.fixed_versions,
+        references=record.references,
+        source=record.source,
+        source_url=record.source_url,
+        confidence=Confidence.MEDIUM,
+        evidence=evidence,
+    )

pkgwhy/vulnerabilities/osv.py

@@ -0,0 +1,330 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import UTC, datetime
+import hashlib
+import json
+import os
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+from typing import Any
+from urllib import error, request
+
+from pkgwhy.core.models import VulnerabilityRange, VulnerabilityRecord
+from pkgwhy.metadata.installed import normalize_package_name
+
+OSV_QUERY_URL = "https://api.osv.dev/v1/query"
+OSV_SOURCE = "OSV.dev"
+OSV_CACHE_ENV = "PKGWHY_CACHE_HOME"
+
+
+@dataclass(frozen=True)
+class OSVLookupResult:
+    """Cache-aware OSV lookup result with explicit freshness status."""
+
+    records: list[VulnerabilityRecord]
+    cache_status: str
+    cache_path: Path | None = None
+    warnings: tuple[str, ...] = ()
+
+
+class OSVClientError(RuntimeError):
+    """Raised when the optional OSV client cannot retrieve advisory data."""
+
+
+def load_osv_records(path: Path, package_name: str | None = None) -> list[VulnerabilityRecord]:
+    """Load OSV-like JSON from a local file without network access."""
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise ValueError(f"Could not read vulnerability data from {path}: {exc}") from exc
+    return parse_osv_payload(payload, package_name=package_name)
+
+
+def parse_osv_payload(payload: dict[str, Any] | list[Any], package_name: str | None = None) -> list[VulnerabilityRecord]:
+    """Parse a minimal OSV response or vulnerability list into internal records."""
+    if isinstance(payload, dict):
+        if isinstance(payload.get("vulns"), list):
+            vulnerabilities = payload["vulns"]
+        elif isinstance(payload.get("id"), str):
+            vulnerabilities = [payload]
+        else:
+            vulnerabilities = []
+    else:
+        vulnerabilities = payload
+
+    records: list[VulnerabilityRecord] = []
+    for item in vulnerabilities:
+        if not isinstance(item, dict):
+            continue
+        records.extend(_records_from_vulnerability(item, package_name=package_name))
+    return records
+
+
+def query_osv(package_name: str, version: str | None, *, timeout_seconds: float = 10.0) -> list[VulnerabilityRecord]:
+    """Query OSV.dev explicitly; callers decide when network access is allowed."""
+    payload = _fetch_osv_payload(package_name, version, timeout_seconds=timeout_seconds)
+    return parse_osv_payload(payload, package_name=package_name)
+
+
+def query_osv_cached(
+    package_name: str,
+    version: str | None,
+    *,
+    timeout_seconds: float = 10.0,
+    cache_dir: Path | None = None,
+) -> OSVLookupResult:
+    """Query OSV.dev with a stale-cache fallback for explicitly online callers."""
+    resolved_cache_dir = cache_dir or default_osv_cache_dir()
+    cache_path = _cache_path(resolved_cache_dir, package_name, version)
+    try:
+        payload = _fetch_osv_payload(package_name, version, timeout_seconds=timeout_seconds)
+    except OSVClientError as exc:
+        cached = _read_cached_payload(cache_path, package_name, version)
+        warning = (
+            f"OSV.dev lookup unavailable for {package_name} {version or 'unknown-version'}: {exc}. "
+            "Missing vulnerability matches are not proof of safety."
+        )
+        if cached is None:
+            return OSVLookupResult(records=[], cache_status="unavailable", cache_path=cache_path, warnings=(warning,))
+        records = parse_osv_payload(cached, package_name=package_name)
+        return OSVLookupResult(
+            records=records,
+            cache_status="stale_cache",
+            cache_path=cache_path,
+            warnings=(
+                warning,
+                "Using cached OSV.dev response. Cached advisory data may be stale.",
+            ),
+        )
+
+    warnings = _write_cached_payload(cache_path, package_name, version, payload)
+    return OSVLookupResult(
+        records=parse_osv_payload(payload, package_name=package_name),
+        cache_status="fresh",
+        cache_path=cache_path,
+        warnings=tuple(warnings),
+    )
+
+
+def default_osv_cache_dir() -> Path:
+    """Return the default OSV cache directory without creating it."""
+    configured = os.environ.get(OSV_CACHE_ENV)
+    root = Path(configured).expanduser() if configured else Path.home() / ".cache" / "pkgwhy"
+    return root / "vulnerabilities" / "osv"
+
+
+def _fetch_osv_payload(package_name: str, version: str | None, *, timeout_seconds: float) -> dict[str, Any]:
+    query: dict[str, Any] = {"package": {"name": package_name, "ecosystem": "PyPI"}}
+    if version is not None:
+        query["version"] = version
+    data = json.dumps(query).encode("utf-8")
+    req = request.Request(
+        OSV_QUERY_URL,
+        data=data,
+        headers={"Content-Type": "application/json", "Accept": "application/json"},
+        method="POST",
+    )
+    try:
+        with request.urlopen(req, timeout=timeout_seconds) as response:
+            return json.loads(response.read().decode("utf-8"))
+    except (OSError, error.HTTPError, UnicodeDecodeError, json.JSONDecodeError) as exc:
+        raise OSVClientError(f"OSV.dev query failed for {package_name}: {exc}") from exc
+
+
+def _cache_path(cache_dir: Path, package_name: str, version: str | None) -> Path:
+    normalized = normalize_package_name(package_name)
+    version_part = version or "no-version"
+    digest = hashlib.sha256(f"{normalized}\0{version_part}".encode("utf-8")).hexdigest()[:16]
+    return cache_dir / f"{normalized}-{digest}.json"
+
+
+def _write_cached_payload(cache_path: Path, package_name: str, version: str | None, payload: dict[str, Any]) -> list[str]:
+    document = {
+        "schema_version": "pkgwhy.osv_cache.v1",
+        "source": OSV_SOURCE,
+        "package": package_name,
+        "version": version,
+        "fetched_at": datetime.now(UTC).isoformat(),
+        "payload": payload,
+    }
+    tmp_path: Path | None = None
+    try:
+        cache_path.parent.mkdir(parents=True, exist_ok=True)
+        serialized = json.dumps(document, indent=2, sort_keys=True)
+        with NamedTemporaryFile("w", dir=cache_path.parent, encoding="utf-8", delete=False) as tmp_file:
+            tmp_file.write(serialized)
+            tmp_file.flush()
+            os.fsync(tmp_file.fileno())
+            tmp_path = Path(tmp_file.name)
+        os.replace(tmp_path, cache_path)
+    except OSError:
+        if tmp_path is not None:
+            try:
+                tmp_path.unlink(missing_ok=True)
+            except OSError:
+                pass
+        return ["Could not write OSV.dev cache."]
+    return []
+
+
+def _read_cached_payload(cache_path: Path, package_name: str, version: str | None) -> dict[str, Any] | None:
+    try:
+        document = json.loads(cache_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return None
+    if not isinstance(document, dict):
+        return None
+    if document.get("schema_version") != "pkgwhy.osv_cache.v1":
+        return None
+    if document.get("source") != OSV_SOURCE:
+        return None
+    cached_package = document.get("package")
+    if not isinstance(cached_package, str):
+        return None
+    if normalize_package_name(cached_package) != normalize_package_name(package_name):
+        return None
+    if document.get("version") != version:
+        return None
+    payload = document.get("payload")
+    return payload if isinstance(payload, dict) else None
+
+
+def _records_from_vulnerability(item: dict[str, Any], package_name: str | None) -> list[VulnerabilityRecord]:
+    vulnerability_id = item.get("id")
+    if not isinstance(vulnerability_id, str) or not vulnerability_id:
+        return []
+
+    affected_entries = item.get("affected")
+    if not isinstance(affected_entries, list):
+        affected_entries = []
+
+    records: list[VulnerabilityRecord] = []
+    for affected in affected_entries:
+        if not isinstance(affected, dict):
+            continue
+        package = affected.get("package")
+        package_info = package if isinstance(package, dict) else {}
+        affected_name = _string_or_none(package_info.get("name")) or package_name
+        if not affected_name:
+            continue
+        ecosystem = _string_or_none(package_info.get("ecosystem"))
+        if ecosystem and ecosystem.lower() not in {"pypi", "python"}:
+            continue
+        records.append(
+            VulnerabilityRecord(
+                id=vulnerability_id,
+                aliases=_string_list(item.get("aliases")),
+                package_name=normalize_package_name(affected_name),
+                ecosystem=ecosystem,
+                summary=_string_or_none(item.get("summary")),
+                details=_string_or_none(item.get("details")),
+                severity=_parse_severity(item.get("severity")),
+                affected_ranges=_parse_ranges(affected.get("ranges")),
+                affected_versions=_string_list(affected.get("versions")),
+                fixed_versions=_fixed_versions_from_ranges(affected.get("ranges")),
+                references=_parse_references(item.get("references")),
+                source=OSV_SOURCE,
+                source_url=f"https://osv.dev/vulnerability/{vulnerability_id}",
+            )
+        )
+    return records
+
+
+def _parse_ranges(value: Any) -> list[VulnerabilityRange]:
+    if not isinstance(value, list):
+        return []
+    ranges: list[VulnerabilityRange] = []
+    for item in value:
+        if not isinstance(item, dict):
+            continue
+        events = item.get("events")
+        if not isinstance(events, list):
+            continue
+        introduced: str | None = None
+        for event in events:
+            if not isinstance(event, dict):
+                continue
+            if "introduced" in event:
+                introduced = _string_or_none(event.get("introduced"))
+                continue
+            if "fixed" in event:
+                ranges.append(
+                    VulnerabilityRange(
+                        introduced=introduced,
+                        fixed=_string_or_none(event.get("fixed")),
+                        range_type=_string_or_none(item.get("type")),
+                    )
+                )
+                introduced = None
+                continue
+            if "last_affected" in event:
+                ranges.append(
+                    VulnerabilityRange(
+                        introduced=introduced,
+                        last_affected=_string_or_none(event.get("last_affected")),
+                        range_type=_string_or_none(item.get("type")),
+                    )
+                )
+                introduced = None
+            if "limit" in event:
+                ranges.append(
+                    VulnerabilityRange(
+                        introduced=introduced,
+                        limit=_string_or_none(event.get("limit")),
+                        range_type=_string_or_none(item.get("type")),
+                    )
+                )
+                introduced = None
+        if introduced is not None:
+            ranges.append(VulnerabilityRange(introduced=introduced, range_type=_string_or_none(item.get("type"))))
+    return ranges
+
+
+def _fixed_versions_from_ranges(value: Any) -> list[str]:
+    fixed: list[str] = []
+    if not isinstance(value, list):
+        return fixed
+    for item in value:
+        if not isinstance(item, dict):
+            continue
+        events = item.get("events")
+        if not isinstance(events, list):
+            continue
+        for event in events:
+            if isinstance(event, dict) and isinstance(event.get("fixed"), str):
+                fixed.append(event["fixed"])
+    return sorted(set(fixed))
+
+
+def _parse_references(value: Any) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    refs: list[str] = []
+    for item in value:
+        if isinstance(item, dict) and isinstance(item.get("url"), str):
+            refs.append(item["url"])
+    return refs
+
+
+def _parse_severity(value: Any) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    severities: list[str] = []
+    for item in value:
+        if not isinstance(item, dict):
+            continue
+        score = item.get("score")
+        if isinstance(score, str):
+            severities.append(score)
+    return severities
+
+
+def _string_list(value: Any) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    return [item for item in value if isinstance(item, str)]
+
+
+def _string_or_none(value: Any) -> str | None:
+    return value if isinstance(value, str) and value else None