devguard-scanner 0.1.0__py3-none-any.whl

devguard/core/safefs.py

@@ -0,0 +1,280 @@
+"""Read-only, no-follow access to the scan target.
+
+This is the only module that touches files inside the target. Guarantees:
+- never follows symlinks (lstat-based walk; O_NOFOLLOW reads; parent components re-checked),
+- never opens FIFOs, sockets, or device files,
+- never writes, chmods, or creates anything,
+- bounded reads, plus file-count, byte, depth, and time budgets.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import os
+import stat
+import time
+from dataclasses import dataclass, field
+from pathlib import Path, PurePosixPath
+
+from devguard.core.config import Limits
+
+_O_NOFOLLOW = getattr(os, "O_NOFOLLOW", 0)
+_O_NONBLOCK = getattr(os, "O_NONBLOCK", 0)
+_O_BINARY = getattr(os, "O_BINARY", 0)
+_O_CLOEXEC = getattr(os, "O_CLOEXEC", 0)
+
+# Directories skipped by default. Their existence is still recorded.
+# Deliberately NOT excluded: .git/config, .git/hooks, .git/info, and any other dot-directory (INV-002).
+DEFAULT_EXCLUDED_DIRS = {
+    ".git/objects",
+    ".git/lfs",
+    ".git/logs",
+    ".git/refs",
+    ".git/modules",
+    "__pycache__",
+    ".mypy_cache",
+    ".pytest_cache",
+    ".ruff_cache",
+    ".tox",
+    ".nox",
+    ".gradle/caches",
+    ".next/cache",
+    ".parcel-cache",
+    ".turbo",
+}
+# Vendored dependency directories: inventoried and reported, deep-scanned only on request (INV-003).
+VENDORED_DIR_NAMES = {"node_modules", "bower_components", "vendor", ".venv", "venv", "site-packages", "Pods"}
+
+
+class Kind:
+    FILE = "file"
+    DIR = "dir"
+    SYMLINK = "symlink"
+    SPECIAL = "special"
+
+
+@dataclass(frozen=True)
+class FileRecord:
+    rel: str
+    kind: str
+    size: int
+    mode: int
+    vendored: bool = False
+    symlink_target: str | None = None
+    special_type: str | None = None
+
+    @property
+    def name(self) -> str:
+        return PurePosixPath(self.rel).name
+
+    @property
+    def hidden(self) -> bool:
+        return any(part.startswith(".") for part in PurePosixPath(self.rel).parts)
+
+    @property
+    def executable_bit(self) -> bool:
+        return bool(self.mode & 0o111)
+
+
+class ReadError(Exception):
+    pass
+
+
+@dataclass
+class Inventory:
+    files: list[FileRecord] = field(default_factory=list)
+    symlinks: list[FileRecord] = field(default_factory=list)
+    special: list[FileRecord] = field(default_factory=list)
+    dirs: list[str] = field(default_factory=list)
+    excluded_dirs: list[str] = field(default_factory=list)
+    vendored_dirs: list[str] = field(default_factory=list)
+    vendored_files: dict[str, FileRecord] = field(default_factory=dict)
+    excluded_count: int = 0
+    unreadable: list[tuple[str, str]] = field(default_factory=list)
+    limit_hits: list[str] = field(default_factory=list)
+    by_rel: dict[str, FileRecord] = field(default_factory=dict)
+
+    def exists(self, rel: str) -> bool:
+        return rel in self.by_rel or rel in self.vendored_files
+
+    def get(self, rel: str) -> FileRecord | None:
+        return self.by_rel.get(rel) or self.vendored_files.get(rel)
+
+
+def normalize_rel(rel: str) -> str | None:
+    """Normalize a repo-relative path. Returns None if it is absolute or escapes the root."""
+    rel = rel.replace("\\", "/").strip()
+    if not rel or rel.startswith("/") or (len(rel) > 1 and rel[1] == ":"):
+        return None
+    parts: list[str] = []
+    for part in rel.split("/"):
+        if part in ("", "."):
+            continue
+        if part == "..":
+            if not parts:
+                return None
+            parts.pop()
+            continue
+        parts.append(part)
+    return "/".join(parts) if parts else None
+
+
+class SafeFS:
+    def __init__(
+        self,
+        root: Path,
+        limits: Limits,
+        user_excludes: list[str] | None = None,
+        include_vendored: bool = False,
+        skip_dirs: list[Path] | None = None,
+    ) -> None:
+        self.root = root.resolve()
+        if not self.root.is_dir():
+            raise NotADirectoryError(str(root))
+        self.limits = limits
+        self.user_excludes = list(user_excludes or [])
+        self.include_vendored = include_vendored
+        self._skip_abs = {p.resolve() for p in (skip_dirs or [])}
+        self.bytes_read = 0
+        self.deadline = time.monotonic() + limits.timeout_s
+
+    # ------------------------------------------------------------------ walking
+    def time_left(self) -> float:
+        return self.deadline - time.monotonic()
+
+    def _excluded(self, rel: str, is_dir: bool) -> bool:
+        if is_dir:
+            if rel in DEFAULT_EXCLUDED_DIRS or PurePosixPath(rel).name in DEFAULT_EXCLUDED_DIRS:
+                return True
+            if any(rel.endswith("/" + d) for d in DEFAULT_EXCLUDED_DIRS if "/" in d):
+                return True
+        return any(
+            fnmatch.fnmatchcase(rel, pat) or fnmatch.fnmatchcase(PurePosixPath(rel).name, pat)
+            for pat in self.user_excludes
+        )
+
+    def walk(self) -> Inventory:
+        inv = Inventory()
+        # (absolute dir, rel dir, depth, inside-vendored)
+        stack: list[tuple[str, str, int, bool]] = [(str(self.root), "", 0, False)]
+        file_budget = self.limits.max_files
+        while stack:
+            abs_dir, rel_dir, depth, in_vendor = stack.pop()
+            if self.time_left() <= 0:
+                inv.limit_hits.append(f"time budget exhausted while walking ({self.limits.timeout_s}s)")
+                break
+            try:
+                with os.scandir(abs_dir) as it:
+                    entries = sorted(it, key=lambda e: e.name)
+            except OSError as exc:
+                inv.unreadable.append((rel_dir or ".", f"cannot list directory: {exc.strerror}"))
+                continue
+            subdirs: list[tuple[str, str, int, bool]] = []
+            for entry in entries:
+                rel = f"{rel_dir}/{entry.name}" if rel_dir else entry.name
+                try:
+                    st = entry.stat(follow_symlinks=False)
+                except OSError as exc:
+                    inv.unreadable.append((rel, f"cannot stat: {exc.strerror}"))
+                    continue
+                mode = st.st_mode
+                if stat.S_ISLNK(mode):
+                    try:
+                        target = os.readlink(entry.path)
+                    except OSError:
+                        target = "?"
+                    rec = FileRecord(rel, Kind.SYMLINK, 0, mode, in_vendor, symlink_target=target)
+                    inv.symlinks.append(rec)
+                    continue
+                if stat.S_ISDIR(mode):
+                    if Path(entry.path).resolve() in self._skip_abs:
+                        inv.excluded_dirs.append(rel)
+                        continue
+                    if self._excluded(rel, True):
+                        inv.excluded_dirs.append(rel)
+                        continue
+                    vendored_here = in_vendor or entry.name in VENDORED_DIR_NAMES
+                    if entry.name in VENDORED_DIR_NAMES and not in_vendor:
+                        inv.vendored_dirs.append(rel)
+                    if depth + 1 > self.limits.max_depth:
+                        inv.limit_hits.append(f"max depth {self.limits.max_depth} reached at {rel}")
+                        continue
+                    inv.dirs.append(rel)
+                    # Vendored trees are always walked (so triggers can reference files inside them),
+                    # but their files are only analyzed with --include-vendored.
+                    subdirs.append((entry.path, rel, depth + 1, vendored_here))
+                    continue
+                if stat.S_ISREG(mode):
+                    if self._excluded(rel, False):
+                        inv.excluded_count += 1
+                        continue
+                    rec = FileRecord(rel, Kind.FILE, st.st_size, mode, in_vendor)
+                    if in_vendor and not self.include_vendored:
+                        inv.vendored_files[rel] = rec
+                        continue
+                    if file_budget <= 0:
+                        if not any("max files" in h for h in inv.limit_hits):
+                            inv.limit_hits.append(f"max files {self.limits.max_files} reached")
+                        continue
+                    file_budget -= 1
+                    inv.files.append(rec)
+                    inv.by_rel[rel] = rec
+                    continue
+                special = (
+                    "fifo"
+                    if stat.S_ISFIFO(mode)
+                    else "socket"
+                    if stat.S_ISSOCK(mode)
+                    else "char-device"
+                    if stat.S_ISCHR(mode)
+                    else "block-device"
+                    if stat.S_ISBLK(mode)
+                    else "unknown"
+                )
+                inv.special.append(FileRecord(rel, Kind.SPECIAL, 0, mode, in_vendor, special_type=special))
+            stack.extend(reversed(subdirs))
+        inv.files.sort(key=lambda r: r.rel)
+        return inv
+
+    # ------------------------------------------------------------------ reading
+    def _check_parents(self, rel: str) -> None:
+        current = self.root
+        parts = PurePosixPath(rel).parts
+        for part in parts[:-1]:
+            current = current / part
+            st = os.lstat(current)
+            if not stat.S_ISDIR(st.st_mode):
+                raise ReadError("parent component is not a real directory")
+
+    def read(self, rel: str, max_bytes: int | None = None) -> bytes:
+        """Read up to max_bytes of a regular file inside the root, never following links."""
+        norm = normalize_rel(rel)
+        if norm is None or norm != rel:
+            raise ReadError("path is not a normalized path inside the scan root")
+        limit = self.limits.max_file_size if max_bytes is None else max_bytes
+        if self.bytes_read >= self.limits.max_total_bytes:
+            raise ReadError("total byte budget exhausted")
+        self._check_parents(rel)
+        path = self.root / rel
+        flags = os.O_RDONLY | _O_NOFOLLOW | _O_NONBLOCK | _O_BINARY | _O_CLOEXEC
+        try:
+            fd = os.open(path, flags)
+        except OSError as exc:
+            raise ReadError(exc.strerror or "open failed") from exc
+        try:
+            st = os.fstat(fd)
+            if not stat.S_ISREG(st.st_mode):
+                raise ReadError("not a regular file")
+            chunks: list[bytes] = []
+            remaining = min(limit, self.limits.max_total_bytes - self.bytes_read)
+            while remaining > 0:
+                chunk = os.read(fd, min(65536, remaining))
+                if not chunk:
+                    break
+                chunks.append(chunk)
+                remaining -= len(chunk)
+            data = b"".join(chunks)
+        finally:
+            os.close(fd)
+        self.bytes_read += len(data)
+        return data

devguard/core/scoring.py

@@ -0,0 +1,60 @@
+"""Deterministic post-processing: severity adjustments, de-duplication, stable ordering, verdict (COR-004/005)."""
+
+from __future__ import annotations
+
+from devguard.core.models import Finding, Severity, Verdict
+
+_AUTOLOADED_BUMP_CATEGORIES = {"capability", "obfuscation", "hidden"}
+
+
+def apply_autoload_bump(findings: list[Finding], autoloaded: set[str]) -> None:
+    """Capabilities inside files that tools load automatically are one tier more severe (SURF-012)."""
+    for f in findings:
+        if f.category in _AUTOLOADED_BUMP_CATEGORIES and f.primary_path in autoloaded:
+            f.severity = f.severity.bump(1)
+            if "auto-loaded-file" not in f.tags:
+                f.tags.append("auto-loaded-file")
+                f.explanation += " This file is loaded automatically by tooling, which raises the severity."
+
+
+def assign_ids(findings: list[Finding]) -> None:
+    for f in findings:
+        f.id = f.compute_id()
+
+
+def dedupe_and_sort(findings: list[Finding]) -> list[Finding]:
+    seen: dict[str, Finding] = {}
+    for f in findings:
+        key = f"{f.rule_id}|{f.primary_path}|{f.primary_line}"
+        prev = seen.get(key)
+        if prev is None or f.severity.rank > prev.severity.rank:
+            seen[key] = f
+    return sorted(
+        seen.values(),
+        key=lambda f: (-f.severity.rank, -f.confidence.rank, f.primary_path, f.primary_line, f.rule_id, f.id),
+    )
+
+
+def verdict(findings: list[Finding]) -> Verdict:
+    """block: do not open/install/run. review: read evidence first. indicators: weak signals only."""
+    if any(f.severity is Severity.CRITICAL for f in findings):
+        return "block"
+    if any(f.severity is Severity.HIGH and f.category == "chain" for f in findings):
+        return "block"
+    if any(f.severity is Severity.HIGH for f in findings):
+        return "review"
+    mediums = sum(1 for f in findings if f.severity is Severity.MEDIUM)
+    if mediums >= 3:
+        return "review"
+    if mediums or any(f.severity is Severity.LOW for f in findings):
+        return "indicators"
+    return "no-indicators"
+
+
+VERDICT_TEXT = {
+    "block": "Do not open, install, or run this project. Strong indicators of malicious behavior were found.",
+    "review": "Review the findings before opening this project in an IDE, installing dependencies, or running it.",
+    "indicators": "Only weak indicators were found. Skim the findings; they are often benign.",
+    "no-indicators": "No suspicious indicators were detected by the enabled checks. This does not prove the "
+    "project is safe.",
+}

devguard/core/service.py

@@ -0,0 +1,211 @@
+"""The scan pipeline. Framework-agnostic: the CLI, MCP server, and future API all call `scan()`."""
+
+from __future__ import annotations
+
+import time
+import uuid
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from functools import cache
+from pathlib import Path
+
+from devguard import RULESET_VERSION, SCHEMA_VERSION, __version__
+from devguard.analyzers.base import AnalysisContext, Analyzer, FileView
+from devguard.analyzers.carriers import CarrierAnalyzer
+from devguard.analyzers.content import ContentRuleEngine
+from devguard.analyzers.filesystem import filesystem_findings
+from devguard.analyzers.hidden import HiddenContentAnalyzer
+from devguard.analyzers.imports import ImportCollector
+from devguard.analyzers.surfaces.ai_assistants import AIAssistantConfigAnalyzer
+from devguard.analyzers.surfaces.autoload import AutoloadedConfigAnalyzer, DocInstructionAnalyzer
+from devguard.analyzers.surfaces.deps import LockfileAnalyzer
+from devguard.analyzers.surfaces.devcontainer import DevcontainerAnalyzer
+from devguard.analyzers.surfaces.env import EnvAnalyzer
+from devguard.analyzers.surfaces.git import GitAnalyzer
+from devguard.analyzers.surfaces.npm import NpmAnalyzer
+from devguard.analyzers.surfaces.python_pkg import PythonPackagingAnalyzer
+from devguard.analyzers.surfaces.vscode import VSCodeAnalyzer
+from devguard.core.config import ScanConfig
+from devguard.core.filetype import classify
+from devguard.core.models import Finding, ScanIssue, ScanResult, ScanStats
+from devguard.core.safefs import FileRecord, ReadError, SafeFS
+from devguard.core.scoring import apply_autoload_bump, assign_ids, dedupe_and_sort, verdict
+from devguard.correlate.chains import correlate, referenced_files
+
+
+class ScanError(Exception):
+    """The scan could not start (invalid target)."""
+
+
+@dataclass
+class ScanRequest:
+    target: Path
+    config: ScanConfig = field(default_factory=ScanConfig)
+    label: str | None = None
+    skip_dirs: list[Path] = field(default_factory=list)  # e.g. an output dir the user placed inside the target
+
+
+@cache
+def default_analyzers() -> tuple[Analyzer, ...]:
+    # Order matters only for readability of issues; analyzers are independent.
+    return (
+        VSCodeAnalyzer(),
+        AIAssistantConfigAnalyzer(),
+        DevcontainerAnalyzer(),
+        NpmAnalyzer(),
+        LockfileAnalyzer(),
+        PythonPackagingAnalyzer(),
+        GitAnalyzer(),
+        EnvAnalyzer(),
+        AutoloadedConfigAnalyzer(),
+        DocInstructionAnalyzer(),
+        ContentRuleEngine.load_builtin(),
+        HiddenContentAnalyzer(),
+        CarrierAnalyzer(),
+        ImportCollector(),
+    )
+
+
+@dataclass
+class ScanOutcome:
+    """The result plus the in-memory context that AI tools and the MCP server need (never serialized)."""
+
+    result: ScanResult
+    ctx: AnalysisContext
+    fs: SafeFS
+
+
+def scan(request: ScanRequest, analyzers: Sequence[Analyzer] | None = None) -> ScanResult:
+    return scan_detailed(request, analyzers).result
+
+
+def scan_detailed(request: ScanRequest, analyzers: Sequence[Analyzer] | None = None) -> ScanOutcome:
+    started = time.monotonic()
+    started_at = datetime.now(UTC).isoformat(timespec="seconds")
+    target = request.target.expanduser()
+    if not target.exists():
+        raise ScanError(f"Target does not exist: {request.target}")
+    if not target.is_dir():
+        raise ScanError(f"Target is not a directory: {request.target}")
+    cfg = request.config
+    fs = SafeFS(target, cfg.limits, cfg.exclude, cfg.include_vendored, request.skip_dirs)
+    inventory = fs.walk()
+    ctx = AnalysisContext(config=cfg, inventory=inventory)
+    stats = ScanStats(
+        considered=len(inventory.files) + len(inventory.vendored_files) + inventory.excluded_count,
+        excluded=inventory.excluded_count,
+        symlinks=len(inventory.symlinks),
+        special_files=len(inventory.special),
+        vendored_dirs=list(inventory.vendored_dirs),
+        excluded_dirs=list(inventory.excluded_dirs),
+    )
+    for rel, why in inventory.unreadable:
+        ctx.issues.append(ScanIssue(kind="error", stage="inventory", message=why, path=rel))
+    for hit in inventory.limit_hits:
+        ctx.issues.append(ScanIssue(kind="limit", stage="inventory", message=hit))
+
+    active = tuple(analyzers) if analyzers is not None else default_analyzers()
+    findings: list[Finding] = []
+    analyzed: set[str] = set()
+    timed_out = False
+    for rec in inventory.files:
+        if fs.time_left() <= 0:
+            timed_out = True
+            break
+        findings += _analyze_file(fs, rec, ctx, active, stats)
+        analyzed.add(rec.rel)
+
+    # Files referenced by triggers are always analyzed, even inside vendored directories (INV-003).
+    for _ in range(3):
+        if timed_out:
+            break
+        extra = [inventory.get(r) for r in sorted(referenced_files(ctx) - analyzed)]
+        extra_recs = [r for r in extra if r is not None][:500]
+        if not extra_recs:
+            break
+        for rec in extra_recs:
+            findings += _analyze_file(fs, rec, ctx, active, stats)
+            analyzed.add(rec.rel)
+
+    if timed_out:
+        ctx.issues.append(
+            ScanIssue(
+                kind="limit",
+                stage="analysis",
+                message=f"time budget of {cfg.limits.timeout_s}s exhausted; scan is partial",
+            )
+        )
+        stats.skipped_limits += len(inventory.files) - len(analyzed)
+
+    findings += filesystem_findings(inventory)
+    apply_autoload_bump(findings, ctx.autoloaded)
+    assign_ids(findings)
+    findings += correlate(ctx, findings)
+    assign_ids(findings)
+    final = dedupe_and_sort(findings)
+
+    partial = timed_out or bool(inventory.limit_hits) or any(i.kind in ("error", "limit") for i in ctx.issues)
+    resolved = target.resolve()
+    result = ScanResult(
+        schema_version=SCHEMA_VERSION,
+        devguard_version=__version__,
+        ruleset_version=RULESET_VERSION,
+        scan_id=f"{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:8]}",
+        started_at=started_at,
+        duration_s=round(time.monotonic() - started, 3),
+        target_label=request.label or resolved.name or str(resolved),
+        status="partial" if partial else "complete",
+        verdict=verdict(final),
+        stats=stats,
+        findings=final,
+        triggers=sorted(ctx.triggers, key=lambda t: (t.path, t.line or 0, t.kind, t.label)),
+        issues=ctx.issues,
+        config_summary=cfg.summary(),
+    )
+    return ScanOutcome(result, ctx, fs)
+
+
+def _analyze_file(
+    fs: SafeFS, rec: FileRecord, ctx: AnalysisContext, analyzers: Sequence[Analyzer], stats: ScanStats
+) -> list[Finding]:
+    limit = ctx.config.limits.max_file_size
+    try:
+        data = fs.read(rec.rel)
+    except ReadError as exc:
+        stats.unreadable += 1
+        ctx.issues.append(ScanIssue(kind="error", stage="read", message=str(exc), path=rec.rel))
+        return []
+    truncated = rec.size > len(data)
+    if truncated:
+        stats.oversized += 1
+        if len(data) >= limit:
+            ctx.issues.append(
+                ScanIssue(
+                    kind="limit",
+                    stage="read",
+                    path=rec.rel,
+                    message=f"file is {rec.size} bytes; only the first {limit} were analyzed",
+                )
+            )
+    stats.inspected += 1
+    stats.bytes_read += len(data)
+    c = classify(rec.rel, data)
+    if not c.is_text:
+        stats.binary += 1
+    fv = FileView(rec, data, c.text, c.lang, c.magic, c.lossy, truncated)
+    out: list[Finding] = []
+    for analyzer in analyzers:
+        try:
+            if analyzer.applies(fv):
+                out.extend(analyzer.analyze(fv, ctx))
+        except Exception as exc:
+            ctx.issues.append(
+                ScanIssue(
+                    kind="error",
+                    stage=f"analyzer:{analyzer.name}",
+                    path=rec.rel,
+                    message=f"{type(exc).__name__}: {exc}",
+                )
+            )
+    return out