pkgwhy 1.0.0__py3-none-any.whl

pkgwhy/inspection/python_static.py

@@ -0,0 +1,323 @@
+from __future__ import annotations
+
+import ast
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from pkgwhy.core.models import PythonStaticAnalysis
+from pkgwhy.risk.rules import make_rule_evidence
+
+MAX_STATIC_FILES = 200
+MAX_STATIC_FILE_BYTES = 1_000_000
+
+IMPORT_CAPABILITIES = {
+    "os": "Filesystem access signals",
+    "pathlib": "Filesystem access signals",
+    "shutil": "Filesystem access signals",
+    "glob": "Filesystem access signals",
+    "socket": "Network access signals",
+    "ssl": "Network access signals",
+    "urllib": "Network access signals",
+    "http.client": "Network access signals",
+    "requests": "Network access signals",
+    "httpx": "Network access signals",
+    "subprocess": "Subprocess or shell execution signals",
+    "pty": "Subprocess or shell execution signals",
+    "pickle": "Deserialisation risk signals",
+    "marshal": "Deserialisation risk signals",
+    "shelve": "Deserialisation risk signals",
+    "dill": "Deserialisation risk signals",
+    "cloudpickle": "Deserialisation risk signals",
+    "base64": "Encoded payload handling signals",
+    "zlib": "Encoded payload handling signals",
+    "importlib": "Dynamic import signals",
+}
+
+CALL_CAPABILITIES = {
+    "eval": "Dynamic code execution signals",
+    "exec": "Dynamic code execution signals",
+    "compile": "Dynamic code execution signals",
+    "__import__": "Dynamic import signals",
+    "importlib.import_module": "Dynamic import signals",
+    "subprocess.run": "Subprocess or shell execution signals",
+    "subprocess.Popen": "Subprocess or shell execution signals",
+    "subprocess.call": "Subprocess or shell execution signals",
+    "subprocess.check_call": "Subprocess or shell execution signals",
+    "subprocess.check_output": "Subprocess or shell execution signals",
+    "os.system": "Subprocess or shell execution signals",
+    "os.popen": "Subprocess or shell execution signals",
+    "os.execv": "Subprocess or shell execution signals",
+    "os.execve": "Subprocess or shell execution signals",
+    "os.spawnv": "Subprocess or shell execution signals",
+    "os.getenv": "Environment variable access signals",
+    "pickle.load": "Deserialisation risk signals",
+    "pickle.loads": "Deserialisation risk signals",
+    "marshal.loads": "Deserialisation risk signals",
+    "dill.loads": "Deserialisation risk signals",
+    "cloudpickle.loads": "Deserialisation risk signals",
+    "base64.b64decode": "Encoded payload handling signals",
+    "zlib.decompress": "Encoded payload handling signals",
+    "yaml.load": "Deserialisation risk signals",
+}
+
+CALL_RULE_IDS = {
+    "eval": "PKGWHY-PY-001",
+    "exec": "PKGWHY-PY-001",
+    "compile": "PKGWHY-PY-001",
+    "__import__": "PKGWHY-PY-002",
+    "importlib.import_module": "PKGWHY-PY-002",
+    "pickle.load": "PKGWHY-PY-003",
+    "pickle.loads": "PKGWHY-PY-003",
+    "marshal.loads": "PKGWHY-PY-003",
+    "dill.loads": "PKGWHY-PY-003",
+    "cloudpickle.loads": "PKGWHY-PY-003",
+    "base64.b64decode": "PKGWHY-PY-004",
+    "zlib.decompress": "PKGWHY-PY-004",
+    "subprocess.run": "PKGWHY-PY-005",
+    "subprocess.Popen": "PKGWHY-PY-005",
+    "subprocess.call": "PKGWHY-PY-005",
+    "subprocess.check_call": "PKGWHY-PY-005",
+    "subprocess.check_output": "PKGWHY-PY-005",
+    "os.system": "PKGWHY-PY-005",
+    "os.popen": "PKGWHY-PY-005",
+    "os.execv": "PKGWHY-PY-005",
+    "os.execve": "PKGWHY-PY-005",
+    "os.spawnv": "PKGWHY-PY-005",
+    "os.getenv": "PKGWHY-PY-006",
+    "yaml.load": "PKGWHY-PY-008",
+}
+
+CREDENTIAL_PATTERNS = tuple(
+    re.compile(pattern)
+    for pattern in (
+        r"(^|[^a-z0-9])api[_-]?key([^a-z0-9]|$)",
+        r"(^|[^a-z0-9])token([^a-z0-9]|$)",
+        r"(^|[^a-z0-9])secret([^a-z0-9]|$)",
+        r"(^|[^a-z0-9])password([^a-z0-9]|$)",
+        r"(^|[^a-z0-9])credential([^a-z0-9]|$)",
+    )
+)
+
+PACKAGE_MANAGER_PATTERN = re.compile(
+    r"(\bpython\s+-m\s+pip\b|\bpip3?\s+(install|uninstall|remove)|\buv\s+(add|remove|sync|pip)|\bpoetry\s+(add|remove|install))"
+)
+LARGE_ENCODED_LITERAL_PATTERN = re.compile(r"^[A-Za-z0-9+/]{120,}={0,2}$")
+PYTHON_OBFUSCATION_BOOTSTRAP_PATTERN = re.compile(
+    r"(__pyarmor__|pyarmor_runtime|pytransform|__armor_enter__|__armor_exit__)",
+    re.IGNORECASE,
+)
+
+
+@dataclass(frozen=True)
+class PythonSignal:
+    capability: str
+    detail: str
+    rule_id: str | None = None
+    line_number: int | None = None
+    symbol: str | None = None
+
+
+def analyze_python_files(paths: list[Path]) -> PythonStaticAnalysis:
+    capabilities: set[str] = set()
+    warnings: list[str] = []
+    evidence: list[str] = []
+    rule_evidence = []
+    files_scanned = 0
+
+    for path in [item for item in paths if item.suffix == ".py"][:MAX_STATIC_FILES]:
+        try:
+            if path.stat().st_size > MAX_STATIC_FILE_BYTES:
+                warnings.append(f"Skipped large Python file during static scan: {path.name}")
+                continue
+            source = path.read_text(encoding="utf-8")
+            tree = ast.parse(source, filename=str(path))
+        except (OSError, SyntaxError, UnicodeDecodeError) as exc:
+            warnings.append(f"Could not statically parse Python file {path.name}: {exc.__class__.__name__}")
+            continue
+
+        files_scanned += 1
+        file_capabilities = _capabilities_from_tree(tree)
+        for signal in file_capabilities:
+            capabilities.add(signal.capability)
+            location = f"{path.name}:{signal.line_number}" if signal.line_number else path.name
+            evidence.append(f"{signal.capability}: {location} references {signal.detail}")
+            if signal.rule_id:
+                rule_evidence.append(
+                    make_rule_evidence(
+                        signal.rule_id,
+                        message=f"{signal.capability}: {signal.detail}.",
+                        evidence=[f"{location} references {signal.detail}."],
+                        file_path=path.name,
+                        line_number=signal.line_number,
+                        symbol=signal.symbol or signal.detail,
+                    )
+                )
+
+    return PythonStaticAnalysis(
+        detected_capabilities=sorted(capabilities),
+        warnings=warnings,
+        evidence=evidence[:100],
+        rule_evidence=rule_evidence[:100],
+        files_scanned=files_scanned,
+    )
+
+
+def _capabilities_from_tree(tree: ast.AST) -> list[PythonSignal]:
+    detected: dict[tuple[str, str, int | None], PythonSignal] = {}
+    for node in ast.walk(tree):
+        if isinstance(node, (ast.Import, ast.ImportFrom)):
+            for imported_name in _imported_names(node):
+                capability = _capability_for_import(imported_name)
+                if capability:
+                    _add_signal(
+                        detected,
+                        PythonSignal(
+                            capability=capability,
+                            detail=imported_name,
+                            line_number=getattr(node, "lineno", None),
+                            symbol=imported_name,
+                        ),
+                    )
+        elif isinstance(node, ast.Call):
+            call_name = _call_name(node.func)
+            capability = CALL_CAPABILITIES.get(call_name)
+            if capability:
+                _add_signal(
+                    detected,
+                    PythonSignal(
+                        capability=capability,
+                        detail=call_name,
+                        rule_id=CALL_RULE_IDS.get(call_name),
+                        line_number=getattr(node, "lineno", None),
+                        symbol=call_name,
+                    ),
+                )
+            if _is_package_manager_call(call_name, node):
+                _add_signal(
+                    detected,
+                    PythonSignal(
+                        capability="Package manager manipulation signals",
+                        detail=call_name,
+                        rule_id="PKGWHY-PY-007",
+                        line_number=getattr(node, "lineno", None),
+                        symbol=call_name,
+                    ),
+                )
+        elif isinstance(node, ast.Attribute):
+            attr_name = _call_name(node)
+            if attr_name == "os.environ":
+                _add_signal(
+                    detected,
+                    PythonSignal(
+                        capability="Environment variable access signals",
+                        detail=attr_name,
+                        rule_id="PKGWHY-PY-006",
+                        line_number=getattr(node, "lineno", None),
+                        symbol=attr_name,
+                    ),
+                )
+        elif isinstance(node, ast.Constant) and isinstance(node.value, str):
+            lowered = node.value.lower()
+            if any(pattern.search(lowered) for pattern in CREDENTIAL_PATTERNS):
+                _add_signal(
+                    detected,
+                    PythonSignal(
+                        capability="Credential or token access patterns",
+                        detail="string literal containing credential-like token",
+                        rule_id="PKGWHY-PY-006",
+                        line_number=getattr(node, "lineno", None),
+                        symbol="credential-like string literal",
+                    ),
+                )
+            if LARGE_ENCODED_LITERAL_PATTERN.fullmatch(node.value.strip()):
+                _add_signal(
+                    detected,
+                    PythonSignal(
+                        capability="Encoded payload handling signals",
+                        detail="large encoded-looking string literal",
+                        rule_id="PKGWHY-PY-004",
+                        line_number=getattr(node, "lineno", None),
+                        symbol="large encoded-looking string literal",
+                    ),
+                )
+            if PYTHON_OBFUSCATION_BOOTSTRAP_PATTERN.search(node.value):
+                _add_signal(
+                    detected,
+                    PythonSignal(
+                        capability="Python obfuscation signals",
+                        detail="obfuscation-bootstrap string literal",
+                        rule_id="PKGWHY-PY-009",
+                        line_number=getattr(node, "lineno", None),
+                        symbol="obfuscation-bootstrap string literal",
+                    ),
+                )
+        elif isinstance(node, ast.Name) and PYTHON_OBFUSCATION_BOOTSTRAP_PATTERN.search(node.id):
+            _add_signal(
+                detected,
+                PythonSignal(
+                    capability="Python obfuscation signals",
+                    detail=node.id,
+                    rule_id="PKGWHY-PY-009",
+                    line_number=getattr(node, "lineno", None),
+                    symbol=node.id,
+                ),
+            )
+    return list(detected.values())
+
+
+def _imported_names(node: ast.Import | ast.ImportFrom) -> list[str]:
+    if isinstance(node, ast.Import):
+        return [alias.name for alias in node.names]
+    if node.module is None:
+        return []
+    return [node.module]
+
+
+def _capability_for_import(imported_name: str) -> str | None:
+    parts = imported_name.split(".")
+    candidates = [imported_name]
+    candidates.append(parts[0])
+    for candidate in candidates:
+        if candidate in IMPORT_CAPABILITIES:
+            return IMPORT_CAPABILITIES[candidate]
+    return None
+
+
+def _call_name(node: ast.AST) -> str:
+    if isinstance(node, ast.Name):
+        return node.id
+    if isinstance(node, ast.Attribute):
+        base = _call_name(node.value)
+        return f"{base}.{node.attr}" if base else node.attr
+    return ""
+
+
+def _add_signal(detected: dict[tuple[str, str, int | None], PythonSignal], signal: PythonSignal) -> None:
+    detected.setdefault((signal.capability, signal.detail, signal.line_number), signal)
+
+
+def _is_package_manager_call(call_name: str, node: ast.Call) -> bool:
+    if call_name in {"pip.main"}:
+        return True
+    if call_name not in {
+        "subprocess.run",
+        "subprocess.Popen",
+        "subprocess.call",
+        "subprocess.check_call",
+        "subprocess.check_output",
+        "os.system",
+    }:
+        return False
+    text_args = " ".join(_literal_text_args(node))
+    return bool(PACKAGE_MANAGER_PATTERN.search(text_args))
+
+
+def _literal_text_args(node: ast.Call) -> list[str]:
+    values: list[str] = []
+    for arg in node.args:
+        if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
+            values.append(arg.value)
+        elif isinstance(arg, (ast.List, ast.Tuple)):
+            values.extend(item.value for item in arg.elts if isinstance(item, ast.Constant) and isinstance(item.value, str))
+    return values

pkgwhy/inspection/size.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+import heapq
+from importlib.metadata import Distribution
+from pathlib import Path
+
+from pkgwhy.core.models import LargestFile, PackageSize
+
+NATIVE_SUFFIXES = {".so", ".pyd", ".dll", ".dylib", ".a", ".lib", ".exe", ".wasm"}
+JAVASCRIPT_SUFFIXES = {".js", ".mjs", ".cjs"}
+
+
+def measure_distribution_size(dist: Distribution | None) -> PackageSize:
+    if dist is None or dist.files is None:
+        return PackageSize()
+
+    total = 0
+    python_bytes = 0
+    native_binary_bytes = 0
+    javascript_bytes = 0
+    other_bytes = 0
+    file_count = 0
+    largest: list[LargestFile] = []
+
+    for package_file in dist.files:
+        try:
+            path = Path(dist.locate_file(package_file))
+        except (OSError, ValueError):
+            continue
+        if not path.is_file():
+            continue
+        try:
+            size = path.stat().st_size
+        except OSError:
+            continue
+        suffix = path.suffix.lower()
+        total += size
+        file_count += 1
+        if suffix == ".py":
+            python_bytes += size
+        elif suffix in NATIVE_SUFFIXES:
+            native_binary_bytes += size
+        elif suffix in JAVASCRIPT_SUFFIXES:
+            javascript_bytes += size
+        else:
+            other_bytes += size
+        largest.append(LargestFile(path=str(package_file), size_bytes=size))
+
+    largest = heapq.nlargest(5, largest, key=lambda item: item.size_bytes)
+    return PackageSize(
+        total_bytes=total,
+        python_bytes=python_bytes,
+        native_binary_bytes=native_binary_bytes,
+        javascript_bytes=javascript_bytes,
+        other_bytes=other_bytes,
+        file_count=file_count,
+        largest_files=largest,
+    )

pkgwhy/inspection/text_patterns.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from urllib.parse import urlparse
+
+from pkgwhy.core.models import FileStaticAnalysis
+from pkgwhy.risk.rules import make_rule_evidence
+
+MAX_TEXT_PATTERN_BYTES = 500_000
+TEXT_PATTERN_SUFFIXES = {
+    ".cfg",
+    ".ini",
+    ".js",
+    ".json",
+    ".md",
+    ".py",
+    ".toml",
+    ".txt",
+    ".yaml",
+    ".yml",
+}
+
+URL_PATTERN = re.compile(r"https?://[^\s'\"<>)\]}]+", re.IGNORECASE)
+CREDENTIAL_ASSIGNMENT_PATTERN = re.compile(
+    r"(?<![A-Za-z0-9_])['\"]?"
+    r"(?P<name>(?:[A-Za-z_][A-Za-z0-9]*[_-])*(?:api[_-]?key|token|secret|password|credential)"
+    r"(?:[_-](?:value|id|key|token|secret|password|credential))*)"
+    r"['\"]?(?![A-Za-z0-9_])"
+    r"\s*(?:=|:\s*(?:[A-Za-z_][A-Za-z0-9_\[\], .|]*\s*=)?)\s*"
+    r"(?P<quote>['\"])"
+    r"(?P<value>[A-Za-z0-9_\-./+=]{8,})"
+    r"(?P=quote)",
+    re.IGNORECASE,
+)
+
+
+def analyze_text_patterns(path: Path) -> FileStaticAnalysis:
+    """Extract conservative URL/domain and credential-like text signals."""
+
+    source = _read_small_text(path)
+    if source is None:
+        return FileStaticAnalysis()
+
+    url_references: list[str] = []
+    domain_references: list[str] = []
+    credential_references: list[str] = []
+    capabilities: set[str] = set()
+    evidence: list[str] = []
+    rule_evidence = []
+
+    for line_number, line in enumerate(source.splitlines(), start=1):
+        for match in URL_PATTERN.finditer(line):
+            sanitized_url = _sanitize_url(match.group(0))
+            domain = _domain_from_url(match.group(0))
+            if not sanitized_url or not domain:
+                continue
+            capabilities.add("URL or domain references")
+            _append_unique(url_references, sanitized_url)
+            _append_unique(domain_references, domain)
+            evidence.append(f"URL/domain reference in {path.name}:{line_number}: {domain}.")
+            rule_evidence.append(
+                make_rule_evidence(
+                    "PKGWHY-NET-001",
+                    message=f"Source text references URL/domain {domain}.",
+                    evidence=[f"{path.name}:{line_number} references domain {domain}."],
+                    file_path=path.name,
+                    line_number=line_number,
+                    symbol=domain,
+                )
+            )
+
+        for match in CREDENTIAL_ASSIGNMENT_PATTERN.finditer(line):
+            credential_name = match.group("name")
+            capabilities.add("Credential or token access patterns")
+            reference = f"{path.name}:{line_number}:{credential_name}=(masked)"
+            _append_unique(credential_references, reference)
+            evidence.append(f"Credential-like assignment in {path.name}:{line_number}: {credential_name}=(masked).")
+            rule_evidence.append(
+                make_rule_evidence(
+                    "PKGWHY-CRED-001",
+                    message=f"Credential-like assignment references {credential_name}; value masked.",
+                    evidence=[f"{path.name}:{line_number} contains {credential_name}=(masked)."],
+                    file_path=path.name,
+                    line_number=line_number,
+                    symbol=credential_name,
+                )
+            )
+
+    return FileStaticAnalysis(
+        detected_capabilities=sorted(capabilities),
+        evidence=evidence,
+        rule_evidence=rule_evidence,
+        url_references=url_references,
+        domain_references=domain_references,
+        credential_references=credential_references,
+    )
+
+
+def is_text_pattern_candidate(path: Path) -> bool:
+    return path.suffix.lower() in TEXT_PATTERN_SUFFIXES or path.name in {"setup.py", "setup.cfg", "pyproject.toml"}
+
+
+def _read_small_text(path: Path) -> str | None:
+    try:
+        if path.stat().st_size > MAX_TEXT_PATTERN_BYTES:
+            return None
+        return path.read_text(encoding="utf-8")
+    except (OSError, UnicodeDecodeError):
+        return None
+
+
+def _sanitize_url(raw_url: str) -> str | None:
+    cleaned = raw_url.rstrip(".,;:")
+    parsed = urlparse(cleaned)
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+        return None
+    host = parsed.hostname
+    if not host:
+        return None
+    path = "/..." if parsed.path and parsed.path != "/" else ""
+    return f"{parsed.scheme}://{host.lower()}{path}"
+
+
+def _domain_from_url(raw_url: str) -> str | None:
+    parsed = urlparse(raw_url.rstrip(".,;:"))
+    if parsed.scheme not in {"http", "https"}:
+        return None
+    host = parsed.hostname
+    return host.lower() if host else None
+
+
+def _append_unique(values: list[str], value: str) -> None:
+    if value not in values:
+        values.append(value)

pkgwhy/manifests/__init__.py

@@ -0,0 +1,2 @@
+"""Project manifest parsers."""
+

pkgwhy/manifests/lockfiles.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import logging
+import tomllib
+from pathlib import Path
+
+from pkgwhy.metadata.installed import normalize_package_name
+
+logger = logging.getLogger(__name__)
+
+
+def read_uv_lock_dependencies(path: Path) -> set[str]:
+    return _read_toml_lock_packages(path)
+
+
+def read_poetry_lock_dependencies(path: Path) -> set[str]:
+    return _read_toml_lock_packages(path)
+
+
+def read_lockfile_dependencies(project_root: Path) -> dict[str, set[str]]:
+    lockfiles: dict[str, set[str]] = {}
+    uv_lock = project_root / "uv.lock"
+    poetry_lock = project_root / "poetry.lock"
+    uv_dependencies = read_uv_lock_dependencies(uv_lock)
+    poetry_dependencies = read_poetry_lock_dependencies(poetry_lock)
+    if uv_dependencies:
+        lockfiles["uv.lock"] = uv_dependencies
+    if poetry_dependencies:
+        lockfiles["poetry.lock"] = poetry_dependencies
+    return lockfiles
+
+
+def _read_toml_lock_packages(path: Path) -> set[str]:
+    if not path.exists():
+        return set()
+    try:
+        data = tomllib.loads(path.read_text(encoding="utf-8"))
+    except (OSError, tomllib.TOMLDecodeError) as exc:
+        logger.debug("Unable to parse lockfile %s: %s", path, exc)
+        return set()
+
+    names: set[str] = set()
+    packages = data.get("package", [])
+    if isinstance(packages, list):
+        for package in packages:
+            if not isinstance(package, dict):
+                continue
+            name = package.get("name")
+            if isinstance(name, str):
+                names.add(normalize_package_name(name))
+    return names

pkgwhy/manifests/pyproject.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+import tomllib
+from pathlib import Path
+
+from packaging.requirements import InvalidRequirement, Requirement
+from packaging.utils import canonicalize_name
+
+
+def read_pyproject_dependencies(path: Path) -> set[str]:
+    if not path.exists():
+        return set()
+    try:
+        data = tomllib.loads(path.read_text(encoding="utf-8"))
+    except (OSError, tomllib.TOMLDecodeError):
+        return set()
+
+    names: set[str] = set()
+    project = data.get("project", {})
+    for dependency in project.get("dependencies", []):
+        _add_requirement_name(names, dependency)
+
+    optional = project.get("optional-dependencies", {})
+    if isinstance(optional, dict):
+        for dependencies in optional.values():
+            if not isinstance(dependencies, list):
+                continue
+            for dependency in dependencies:
+                _add_requirement_name(names, dependency)
+    return names
+
+
+def _add_requirement_name(names: set[str], value: str) -> None:
+    try:
+        names.add(canonicalize_name(Requirement(value).name))
+    except InvalidRequirement:
+        return

pkgwhy/manifests/requirements.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from packaging.requirements import InvalidRequirement, Requirement
+from packaging.utils import canonicalize_name
+
+
+def read_requirements_dependencies(path: Path) -> set[str]:
+    if not path.exists():
+        return set()
+    names: set[str] = set()
+    try:
+        lines = path.read_text(encoding="utf-8").splitlines()
+    except OSError:
+        return names
+    for line in lines:
+        cleaned = line.split("#", 1)[0].strip()
+        if not cleaned or cleaned.startswith(
+            ("-", "http:", "https:", "git+", "svn+", "hg+", "bzr+", "file://")
+        ):
+            continue
+        try:
+            names.add(canonicalize_name(Requirement(cleaned).name))
+        except InvalidRequirement:
+            continue
+    return names

pkgwhy/metadata/__init__.py

@@ -0,0 +1,2 @@
+"""Installed package metadata readers."""
+