pysentix 0.1.0__tar.gz

pysentix-0.1.0/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: pysentix
+Version: 0.1.0
+Summary: Pre-install supply-chain malware and typosquat detection for PyPI packages
+Requires-Python: >=3.9
+Requires-Dist: fastapi<1.0,>=0.110
+Requires-Dist: uvicorn<1.0,>=0.29
+Requires-Dist: pydantic<3.0,>=2.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-cov>=5.0; extra == "dev"
+Requires-Dist: httpx>=0.27; extra == "dev"
+Requires-Dist: ruff>=0.4; extra == "dev"
+Requires-Dist: mypy>=1.10; extra == "dev"
+Provides-Extra: ml
+Requires-Dist: xgboost>=2.0; extra == "ml"
+Requires-Dist: scikit-learn>=1.4; extra == "ml"
+Requires-Dist: shap>=0.45; extra == "ml"
+Requires-Dist: networkx>=3.2; extra == "ml"

pysentix-0.1.0/README.md

@@ -0,0 +1,44 @@
+# PyPI Sentinel (backend)
+
+Pre-install supply-chain checks for Python packages. Give it a package name; it downloads the release
+**without installing or executing anything**, inspects it, and returns a risk score with reasons.
+
+## What v0.1 does
+- **Static analysis (AST):** hidden payloads (decode + exec), dynamic exec/eval, network calls, system commands,
+  credential-file paths, env-var reads, install hooks, high-entropy strings. Each finding is tagged with *when it runs*
+  (install / import / runtime); only install- and import-time behaviour is scored.
+- **Metadata:** typosquat distance to ~120 popular packages (Damerau-Levenshtein), package age, linked repository.
+- **Release diff:** compares the scanned version with the previous release and reports newly added risky behaviour.
+- **Explainable score:** weighted rules (0-100) -> Likely safe / Suspicious / High risk, with evidence (file + line).
+- **Interfaces:** CLI (`sentinel`) and FastAPI (`/api/scan`) for the frontend / CI.
+
+## Run
+```bash
+python -m venv .venv && source .venv/bin/activate      # Windows: .venv\Scripts\activate
+pip install -e ".[dev]"
+pytest                                                   # unit tests
+sentinel scan requests                                   # scan a real package
+sentinel scan-dir samples/fake_malicious                 # harmless fake "malware" for demos
+sentinel serve --port 8000                               # API: http://127.0.0.1:8000/docs
+```
+
+## API
+`GET /api/scan?package=<name>&version=<optional>&diff=true`
+Returns `risk_score`, `verdict` (`low|suspicious|high`), `signals[]` (title, detail, severity, weight, evidence),
+`metadata`, `diff`, `files_scanned`, `duration_ms`. `GET /api/health` for status.
+
+## Safety
+Archives are read in memory; only `.py` files are decoded as text. Size, file-count and total-size limits guard against
+zip bombs. Never install or import samples. Keep real malware datasets inside a network-less VM/container.
+
+## Known limits (be honest in the report)
+v0.1 is heuristic. It cannot see payloads downloaded later from a remote server, and rules can flag legitimate
+build tools. Phase 2 (see roadmap) adds a trained classifier, obfuscation-robustness testing and dynamic analysis.
+
+## Roadmap
+1. Dataset pipeline (malicious vs. benign, time-based split) and feature table
+2. Train XGBoost/RandomForest, add SHAP explanations, baseline comparison
+3. `scripts/obfuscate.py` + robustness experiment (recall before/after obfuscation)
+4. Hijack test set for release-diff evaluation
+5. GitHub Action / pre-commit hook for `requirements.txt`
+6. Stretch: code embeddings; sandboxed dynamic analysis

pysentix-0.1.0/pyproject.toml

@@ -0,0 +1,58 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "pysentix"
+version = "0.1.0"
+description = "Pre-install supply-chain malware and typosquat detection for PyPI packages"
+requires-python = ">=3.9"
+dependencies = [
+    "fastapi>=0.110,<1.0",
+    "uvicorn>=0.29,<1.0",
+    "pydantic>=2.0,<3.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0",
+    "pytest-cov>=5.0",
+    "httpx>=0.27",
+    "ruff>=0.4",
+    "mypy>=1.10",
+]
+ml = [
+    "xgboost>=2.0",
+    "scikit-learn>=1.4",
+    "shap>=0.45",
+    "networkx>=3.2",
+]
+
+[project.scripts]
+sentinel = "pysentinel.cli:main"
+pip-sentinel = "pysentinel.cli:pip_sentinel_main"
+
+[tool.setuptools.packages.find]
+include = ["pysentinel*"]
+
+[tool.setuptools.package-data]
+pysentinel = ["data/*.txt"]
+
+[tool.ruff]
+target-version = "py39"
+line-length = 120
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "UP", "B", "SIM", "S"]
+ignore = ["S101", "S603", "S607"]  # assert in tests, subprocess args
+
+[tool.mypy]
+python_version = "3.9"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = false
+check_untyped_defs = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-v --tb=short"

pysentix-0.1.0/pysentinel/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

pysentix-0.1.0/pysentinel/adversarial.py

@@ -0,0 +1,119 @@
+"""Adversarial evasion generators and robustness evaluation tools.
+
+Tests scanner resilience against malware evasion techniques:
+- Variable and symbol renaming
+- String splitting and join-based reconstruction
+- Dead-code and dummy-branch insertion
+- Multi-layer encoding (base64 / zlib wrappers)
+"""
+from __future__ import annotations
+
+import base64
+import random
+import re
+from typing import Any, Dict, List, Optional
+
+
+def rename_variables(code: str) -> str:
+    """Rename user variables to randomized hex-like identifiers."""
+    # Find simple assignment variables that aren't Python keywords
+    keywords = {
+        "import", "from", "as", "def", "class", "return", "if", "else", "elif",
+        "for", "while", "try", "except", "finally", "with", "in", "is", "not",
+        "and", "or", "True", "False", "None", "self", "setup",
+    }
+    vars_found = set(re.findall(r"\b([a-z_][a-z0-9_]{2,15})\s*=", code)) - keywords
+    transformed = code
+    for i, var in enumerate(sorted(vars_found)):
+        hex_name = f"_0x{i+1:04x}"
+        transformed = re.sub(rf"\b{re.escape(var)}\b", hex_name, transformed)
+    return transformed
+
+
+def split_strings(code: str) -> str:
+    """Split string literals into concatenated chunks: 'abcde' -> 'ab' + 'cde'."""
+    def _splitter(m: re.Match) -> str:
+        quote = m.group(1)
+        content = m.group(2)
+        if len(content) < 6 or "\\" in content:
+            return m.group(0)
+        mid = len(content) // 2
+        return f"{quote}{content[:mid]}{quote} + {quote}{content[mid:]}{quote}"
+
+    return re.sub(r"(['\"])(.*?)\1", _splitter, code)
+
+
+def inject_dead_code(code: str) -> str:
+    """Inject dummy dead code and conditional branches matching line indentation."""
+    dead_templates = [
+        "if 1 == 2:\n    _dummy_unused = 42",
+        "_dummy_entropy_check = 'abc' * 3",
+        "_ = sum([1, 2, 3]) if False else None",
+    ]
+    lines = code.splitlines()
+    if len(lines) > 2:
+        idx = min(2, len(lines) - 1)
+        target_line = lines[idx]
+        indent = len(target_line) - len(target_line.lstrip())
+        prefix = " " * indent
+        template = random.choice(dead_templates)
+        indented = "\n".join(prefix + l if l.strip() else l for l in template.splitlines())
+        lines.insert(idx, indented)
+    return "\n".join(lines)
+
+
+def wrap_b64_exec(code: str) -> str:
+    """Wrap code into a dynamic base64 execution snippet."""
+    b64_str = base64.b64encode(code.encode()).decode()
+    return f"import base64\nexec(base64.b64decode('{b64_str}'))\n"
+
+
+def obfuscate_source(
+    code: str,
+    techniques: Optional[List[str]] = None,
+) -> str:
+    """Apply selected evasion techniques to source code."""
+    applied = techniques or ["rename", "split", "dead_code"]
+    res = code
+    if "rename" in applied:
+        res = rename_variables(res)
+    if "split" in applied:
+        res = split_strings(res)
+    if "dead_code" in applied:
+        res = inject_dead_code(res)
+    if "b64" in applied:
+        res = wrap_b64_exec(res)
+    return res
+
+
+def evaluate_adversarial_recall(
+    detector_fn: Any,
+    samples: List[str],
+) -> Dict[str, Any]:
+    """Test detection recall on original vs obfuscated variants of malicious samples."""
+    original_detected = 0
+    obfuscated_detected = 0
+    b64_detected = 0
+
+    for sample in samples:
+        # Original
+        if detector_fn(sample):
+            original_detected += 1
+
+        # Mild obfuscation (rename + split + dead code)
+        mild = obfuscate_source(sample, techniques=["rename", "split", "dead_code"])
+        if detector_fn(mild):
+            obfuscated_detected += 1
+
+        # Heavy obfuscation (base64 wrap)
+        heavy = obfuscate_source(sample, techniques=["b64"])
+        if detector_fn(heavy):
+            b64_detected += 1
+
+    total = max(1, len(samples))
+    return {
+        "sample_count": len(samples),
+        "clean_recall": round(original_detected / total, 3),
+        "mild_obfuscation_recall": round(obfuscated_detected / total, 3),
+        "heavy_obfuscation_recall": round(b64_detected / total, 3),
+    }

pysentix-0.1.0/pysentinel/api.py

@@ -0,0 +1,317 @@
+"""FastAPI application for PyPI Sentinel.
+
+Provides:
+- POST /v1/scan        — start a new scan (async)
+- GET  /v1/scan/{id}   — retrieve scan result by ID
+- GET  /v1/health      — health check
+- GET  /v1/model-info  — model / version info
+- GET  /api/scan       — legacy endpoint (kept for backwards compat with existing frontends)
+- GET  /api/health     — legacy health
+
+Security:
+- CORS locked to configured origins
+- Per-IP rate limiting (in-memory token-bucket)
+- Security headers middleware
+- Pydantic validation on all inputs
+- No stack traces to clients
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import time
+import uuid
+from collections import defaultdict
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+from fastapi import FastAPI, HTTPException, Query, Request, Response
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, Field, field_validator
+
+from . import __version__
+from .config import settings
+from .fetcher import FetchError, PackageNotFound, normalize_name
+from .logging import get_logger, new_request_id
+from .scanner import scan
+
+log = get_logger(__name__)
+
+# ── Pydantic models ──────────────────────────────────────────────────
+
+class ScanRequest(BaseModel):
+    """Request body for POST /v1/scan."""
+    package: str = Field(
+        ...,
+        min_length=1,
+        max_length=100,
+        pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$",
+        description="PyPI package name",
+    )
+    version: Optional[str] = Field(
+        None,
+        max_length=50,
+        pattern=r"^[A-Za-z0-9][A-Za-z0-9._+!-]{0,49}$",
+        description="Specific version to scan (default: latest)",
+    )
+    diff: bool = Field(True, description="Compare with previous release")
+
+    @field_validator("package")
+    @classmethod
+    def normalize_package(cls, v: str) -> str:
+        return normalize_name(v)
+
+
+class Verdict(str, Enum):
+    low = "low"
+    suspicious = "suspicious"
+    high = "high"
+
+
+class Evidence(BaseModel):
+    file: str
+    line: int
+    snippet: str = ""
+
+
+class Signal(BaseModel):
+    id: str
+    title: str
+    detail: str
+    weight: float
+    severity: str
+    evidence: List[Evidence] = []
+
+
+class ScanResult(BaseModel):
+    scan_id: str
+    package: str
+    version: str
+    risk_score: int
+    verdict: Verdict
+    verdict_label: str
+    signals: List[Signal]
+    metadata: Dict[str, Any]
+    diff: Optional[Dict[str, Any]] = None
+    graph: Optional[Dict[str, Any]] = None
+    ml_risk_probability: float = 0.0
+    explainability: Optional[Dict[str, Any]] = None
+    files_scanned: int = 0
+    obfuscation_score: float = 0.0
+    duration_ms: int = 0
+
+
+class ErrorResponse(BaseModel):
+    detail: str
+    request_id: str = ""
+
+
+class HealthResponse(BaseModel):
+    status: str = "ok"
+    version: str
+
+
+class ModelInfoResponse(BaseModel):
+    model_type: str = "calibrated-ensemble"
+    model_version: str = "1.0.0"
+    features: List[str] = []
+    ablation_results: Optional[Dict[str, Any]] = None
+    top_features: Optional[List[Dict[str, Any]]] = None
+
+
+# ── Rate limiter (in-memory token bucket) ────────────────────────────
+
+class _RateLimiter:
+    """Per-IP token-bucket rate limiter."""
+
+    def __init__(self, rpm: int, burst: int) -> None:
+        self.rate = rpm / 60.0  # tokens per second
+        self.burst = burst
+        self._buckets: Dict[str, tuple[float, float]] = {}  # ip → (tokens, last_time)
+
+    def allow(self, ip: str) -> bool:
+        now = time.time()
+        tokens, last = self._buckets.get(ip, (float(self.burst), now))
+        elapsed = now - last
+        tokens = min(float(self.burst), tokens + elapsed * self.rate)
+        if tokens >= 1.0:
+            self._buckets[ip] = (tokens - 1.0, now)
+            return True
+        self._buckets[ip] = (tokens, now)
+        return False
+
+
+_limiter = _RateLimiter(settings.rate_limit_rpm, settings.rate_limit_burst)
+
+# ── Scan cache ───────────────────────────────────────────────────────
+
+_cache: Dict[str, tuple[float, Dict[str, Any]]] = {}
+_id_index: Dict[str, Dict[str, Any]] = {}
+
+
+def _cache_key(package: str, version: Optional[str], diff: bool) -> str:
+    raw = f"{normalize_name(package)}:{version or ''}:{diff}"
+    return hashlib.sha256(raw.encode()).hexdigest()[:16]
+
+
+# ── FastAPI app ──────────────────────────────────────────────────────
+
+app = FastAPI(
+    title="PyPI Sentinel API",
+    version=__version__,
+    description="Pre-install supply-chain malware detection for Python packages.",
+    docs_url="/docs",
+    redoc_url="/redoc",
+)
+
+# CORS — locked to configured origins
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=settings.cors_origins,
+    allow_methods=["GET", "POST", "OPTIONS"],
+    allow_headers=["*"],
+    max_age=600,
+)
+
+
+# ── Middleware ───────────────────────────────────────────────────────
+
+@app.middleware("http")
+async def security_headers(request: Request, call_next: Any) -> Response:
+    """Add security headers and request ID to every response."""
+    rid = new_request_id()
+    response: Response = await call_next(request)
+    response.headers["X-Request-ID"] = rid
+    response.headers["X-Content-Type-Options"] = "nosniff"
+    response.headers["X-Frame-Options"] = "DENY"
+    response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
+    response.headers["Cache-Control"] = "no-store"
+    return response
+
+
+@app.middleware("http")
+async def rate_limit_middleware(request: Request, call_next: Any) -> Response:
+    """Enforce per-IP rate limits."""
+    if request.url.path.startswith(("/docs", "/redoc", "/openapi.json")):
+        return await call_next(request)
+
+    client_ip = request.client.host if request.client else "unknown"
+    if not _limiter.allow(client_ip):
+        log.warning("rate_limited", ip=client_ip, path=str(request.url.path))
+        return JSONResponse(
+            status_code=429,
+            content={"detail": "Too many requests. Please wait before trying again."},
+        )
+    return await call_next(request)
+
+
+# ── Exception handlers ──────────────────────────────────────────────
+
+@app.exception_handler(Exception)
+async def generic_exception_handler(request: Request, exc: Exception) -> JSONResponse:
+    """Never leak stack traces to clients."""
+    log.error("unhandled_exception", error=str(exc), path=str(request.url.path))
+    return JSONResponse(
+        status_code=500,
+        content={"detail": "An internal error occurred. Please try again later."},
+    )
+
+
+# ── v1 Endpoints ────────────────────────────────────────────────────
+
+@app.get("/v1/health", response_model=HealthResponse)
+def health_v1() -> HealthResponse:
+    """Health check endpoint."""
+    return HealthResponse(status="ok", version=__version__)
+
+
+@app.get("/v1/model-info", response_model=ModelInfoResponse)
+def model_info() -> ModelInfoResponse:
+    """Return info about the current scoring model, features, and ablation results."""
+    from .classifier import FEATURE_DEFINITIONS, classifier
+    metrics_path = os.path.join(os.path.dirname(__file__), "data", "model_metrics.json")
+    ablation = None
+    top_feats = None
+    if os.path.exists(metrics_path):
+        try:
+            with open(metrics_path, "r", encoding="utf-8") as f:
+                data = json.load(f)
+                ablation = data.get("ablation_results")
+                top_feats = data.get("top_features")
+        except Exception:
+            pass
+
+    return ModelInfoResponse(
+        model_type="calibrated-ensemble",
+        model_version=classifier.version,
+        features=[f[0] for f in FEATURE_DEFINITIONS],
+        ablation_results=ablation,
+        top_features=top_feats,
+    )
+
+
+@app.post("/v1/scan", response_model=ScanResult)
+def scan_v1(body: ScanRequest) -> ScanResult:
+    """Scan a package for supply-chain threats."""
+    key = _cache_key(body.package, body.version, body.diff)
+    hit = _cache.get(key)
+    if hit and time.time() - hit[0] < settings.cache_ttl:
+        return ScanResult(**hit[1])
+
+    try:
+        result = scan(body.package, body.version, body.diff)
+    except PackageNotFound:
+        raise HTTPException(404, f"'{body.package}' was not found on PyPI. Check the spelling.")
+    except FetchError as exc:
+        raise HTTPException(502, str(exc))
+
+    scan_id = uuid.uuid4().hex[:12]
+    result["scan_id"] = scan_id
+    _cache[key] = (time.time(), result)
+    _id_index[scan_id] = result
+    return ScanResult(**result)
+
+
+@app.get("/v1/scan/{scan_id}", response_model=ScanResult)
+def get_scan(scan_id: str) -> ScanResult:
+    """Retrieve a previous scan result by ID."""
+    result = _id_index.get(scan_id)
+    if not result:
+        raise HTTPException(404, "Scan not found.")
+    return ScanResult(**result)
+
+
+# ── Legacy endpoints (backwards compat with existing frontends) ─────
+
+@app.get("/api/health")
+def health_legacy() -> Dict[str, str]:
+    """Legacy health check."""
+    return {"status": "ok", "version": __version__}
+
+
+@app.get("/api/scan")
+def scan_legacy(
+    package: str = Query(..., pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$"),
+    version: Optional[str] = Query(None, pattern=r"^[A-Za-z0-9][A-Za-z0-9._+!-]{0,49}$"),
+    diff: bool = True,
+) -> Dict[str, Any]:
+    """Legacy scan endpoint (GET-based, kept for existing frontend compatibility)."""
+    key = _cache_key(package, version, diff)
+    hit = _cache.get(key)
+    if hit and time.time() - hit[0] < settings.cache_ttl:
+        return hit[1]
+
+    try:
+        result = scan(package, version, diff)
+    except PackageNotFound:
+        raise HTTPException(404, f"'{package}' was not found on PyPI. Check the spelling.")
+    except FetchError as exc:
+        raise HTTPException(502, str(exc))
+
+    scan_id = uuid.uuid4().hex[:12]
+    result["scan_id"] = scan_id
+    _cache[key] = (time.time(), result)
+    _id_index[scan_id] = result
+    return result