devguard-scanner 0.1.0__py3-none-any.whl

devguard/core/filetype.py

@@ -0,0 +1,313 @@
+"""File classification by magic bytes, extension, and shebang. Never executes anything."""
+
+from __future__ import annotations
+
+import codecs
+import re
+from dataclasses import dataclass
+from pathlib import PurePosixPath
+
+# (magic id, offset, signature)
+_MAGIC: list[tuple[str, int, bytes]] = [
+    ("elf", 0, b"\x7fELF"),
+    ("pe", 0, b"MZ"),
+    ("macho", 0, b"\xfe\xed\xfa\xce"),
+    ("macho", 0, b"\xfe\xed\xfa\xcf"),
+    ("macho", 0, b"\xce\xfa\xed\xfe"),
+    ("macho", 0, b"\xcf\xfa\xed\xfe"),
+    ("java-class-or-fat-macho", 0, b"\xca\xfe\xba\xbe"),
+    ("dex", 0, b"dex\n"),
+    ("wasm", 0, b"\x00asm"),
+    ("ole", 0, b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"),
+    ("zip", 0, b"PK\x03\x04"),
+    ("zip", 0, b"PK\x05\x06"),
+    ("gzip", 0, b"\x1f\x8b"),
+    ("bzip2", 0, b"BZh"),
+    ("xz", 0, b"\xfd7zXZ\x00"),
+    ("7z", 0, b"7z\xbc\xaf\x27\x1c"),
+    ("rar", 0, b"Rar!\x1a\x07"),
+    ("tar", 257, b"ustar"),
+    ("png", 0, b"\x89PNG\r\n\x1a\n"),
+    ("jpeg", 0, b"\xff\xd8\xff"),
+    ("gif", 0, b"GIF87a"),
+    ("gif", 0, b"GIF89a"),
+    ("webp", 8, b"WEBP"),
+    ("ico", 0, b"\x00\x00\x01\x00"),
+    ("bmp", 0, b"BM"),
+    ("pdf", 0, b"%PDF"),
+    ("woff", 0, b"wOFF"),
+    ("woff2", 0, b"wOF2"),
+    ("ttf", 0, b"\x00\x01\x00\x00"),
+    ("otf", 0, b"OTTO"),
+    ("sqlite", 0, b"SQLite format 3\x00"),
+]
+
+EXECUTABLE_MAGIC = {"elf", "pe", "macho", "java-class-or-fat-macho", "dex", "wasm"}
+ARCHIVE_MAGIC = {"zip", "gzip", "bzip2", "xz", "7z", "rar", "tar"}
+
+# Extensions whose content must carry a specific magic.
+EXPECTED_MAGIC: dict[str, set[str]] = {
+    ".png": {"png"},
+    ".jpg": {"jpeg"},
+    ".jpeg": {"jpeg"},
+    ".gif": {"gif"},
+    ".webp": {"webp"},
+    ".ico": {"ico", "png"},
+    ".bmp": {"bmp"},
+    ".pdf": {"pdf"},
+    ".woff": {"woff"},
+    ".woff2": {"woff2"},
+    ".ttf": {"ttf"},
+    ".otf": {"otf", "ttf"},
+    ".zip": {"zip"},
+    ".jar": {"zip"},
+    ".whl": {"zip"},
+    ".gz": {"gzip"},
+    ".tgz": {"gzip"},
+    ".7z": {"7z"},
+    ".rar": {"rar"},
+}
+
+# Extensions that are expected to be plain text; executable magic in them is deceptive.
+TEXT_EXTENSIONS = {
+    ".txt",
+    ".md",
+    ".json",
+    ".js",
+    ".mjs",
+    ".cjs",
+    ".ts",
+    ".css",
+    ".html",
+    ".yml",
+    ".yaml",
+    ".toml",
+    ".xml",
+    ".csv",
+    ".py",
+    ".sh",
+    ".svg",
+    ".lock",
+    ".cfg",
+    ".ini",
+    ".env",
+}
+
+_EXT_LANG = {
+    ".js": "javascript",
+    ".mjs": "javascript",
+    ".cjs": "javascript",
+    ".jsx": "javascript",
+    ".ts": "javascript",
+    ".mts": "javascript",
+    ".cts": "javascript",
+    ".tsx": "javascript",
+    ".vue": "javascript",
+    ".svelte": "javascript",
+    ".py": "python",
+    ".pyw": "python",
+    ".pyi": "python",
+    ".sh": "shell",
+    ".bash": "shell",
+    ".zsh": "shell",
+    ".ksh": "shell",
+    ".command": "shell",
+    ".ps1": "powershell",
+    ".psm1": "powershell",
+    ".psd1": "powershell",
+    ".bat": "batch",
+    ".cmd": "batch",
+    ".json": "json",
+    ".jsonc": "json",
+    ".code-workspace": "json",
+    ".yml": "yaml",
+    ".yaml": "yaml",
+    ".toml": "toml",
+    ".xml": "xml",
+    ".md": "markdown",
+    ".mdc": "markdown",
+    ".markdown": "markdown",
+    ".rst": "markdown",
+    ".txt": "text",
+    ".rb": "ruby",
+    ".php": "php",
+    ".go": "go",
+    ".java": "java",
+    ".kt": "kotlin",
+    ".kts": "kotlin",
+    ".gradle": "groovy",
+    ".groovy": "groovy",
+    ".cs": "csharp",
+    ".rs": "rust",
+    ".lua": "lua",
+    ".pl": "perl",
+    ".pm": "perl",
+    ".vbs": "vbscript",
+    ".ipynb": "json",
+}
+
+_NAME_LANG = {
+    "makefile": "make",
+    "gnumakefile": "make",
+    "dockerfile": "dockerfile",
+    "rakefile": "ruby",
+    "gemfile": "ruby",
+    "vagrantfile": "ruby",
+    "jenkinsfile": "groovy",
+    ".envrc": "shell",
+    ".bashrc": "shell",
+    ".zshrc": "shell",
+    ".profile": "shell",
+    ".bash_profile": "shell",
+    ".npmrc": "ini",
+    ".yarnrc": "ini",
+    ".gitconfig": "ini",
+    "config": None,  # type: ignore[dict-item]
+}
+
+_SHEBANG_LANG = [
+    ("node", "javascript"),
+    ("deno", "javascript"),
+    ("bun", "javascript"),
+    ("python", "python"),
+    ("pwsh", "powershell"),
+    ("powershell", "powershell"),
+    ("bash", "shell"),
+    ("zsh", "shell"),
+    ("/sh", "shell"),
+    (" sh", "shell"),
+    ("ruby", "ruby"),
+    ("perl", "perl"),
+    ("php", "php"),
+]
+
+
+@dataclass(frozen=True)
+class Classification:
+    magic: str | None
+    is_text: bool
+    encoding: str | None
+    lossy: bool
+    lang: str | None
+    text: str | None = None
+
+
+def detect_magic(data: bytes) -> str | None:
+    for magic, offset, sig in _MAGIC:
+        if data[offset : offset + len(sig)] == sig:
+            if magic == "pe" and len(data) < 64:
+                continue
+            if magic == "bmp" and len(data) < 26:
+                continue
+            if magic == "ttf" and len(data) < 12:
+                continue
+            return magic
+    return None
+
+
+def decode_text(data: bytes) -> tuple[str | None, str | None, bool]:
+    """Return (text, encoding, lossy). (None, None, False) for binary content."""
+    if data.startswith(codecs.BOM_UTF8):
+        return data[3:].decode("utf-8", "replace"), "utf-8-sig", False
+    for bom, enc in (
+        (codecs.BOM_UTF32_LE, "utf-32"),
+        (codecs.BOM_UTF32_BE, "utf-32"),
+        (codecs.BOM_UTF16_LE, "utf-16"),
+        (codecs.BOM_UTF16_BE, "utf-16"),
+    ):
+        if data.startswith(bom):
+            try:
+                return data.decode(enc), enc, False
+            except UnicodeDecodeError:
+                return data.decode(enc, "replace"), enc, True
+    sample = data[:8192]
+    if b"\x00" in sample:
+        return None, None, False
+    try:
+        return data.decode("utf-8"), "utf-8", False
+    except UnicodeDecodeError as exc:
+        # A truncated multi-byte char at the very end (bounded read) is not a reason to call it lossy.
+        if exc.start >= len(data) - 3:
+            return data[: exc.start].decode("utf-8"), "utf-8", False
+    printable = sum(1 for b in sample if b >= 0x20 or b in (9, 10, 13))
+    if sample and printable / len(sample) > 0.95:
+        return data.decode("latin-1"), "latin-1", True
+    return None, None, False
+
+
+def language_for(rel: str, text: str | None) -> str | None:
+    p = PurePosixPath(rel)
+    name = p.name.lower()
+    if _NAME_LANG.get(name):
+        return _NAME_LANG[name]
+    if name.startswith(".env"):
+        return "dotenv"
+    if name.startswith("dockerfile") or name.endswith(".dockerfile"):
+        return "dockerfile"
+    if p.parent.name == "hooks" and p.parent.parent.name == ".git":
+        return _shebang_lang(text) or "shell"
+    lang = _EXT_LANG.get(p.suffix.lower())
+    if lang:
+        return lang
+    return _shebang_lang(text)
+
+
+def _shebang_lang(text: str | None) -> str | None:
+    if text and text.startswith("#!"):
+        first = text.split("\n", 1)[0].lower()
+        for needle, lang in _SHEBANG_LANG:
+            if needle in first:
+                return lang
+    return None
+
+
+def classify(rel: str, data: bytes) -> Classification:
+    magic = detect_magic(data)
+    if magic is not None and magic not in _WEAK_MAGIC:
+        return Classification(magic, False, None, False, None, None)
+    text, enc, lossy = decode_text(data)
+    if text is not None and magic in _WEAK_MAGIC:
+        magic = None  # "MZ…" / "BM…" that decodes as text is just text
+    lang = language_for(rel, text) if text else None
+    if text and lang is None and looks_like_code(text):
+        lang = guess_code_lang(text)  # e.g. JavaScript disguised as a font or image
+    return Classification(magic, text is not None, enc, lossy, lang, text)
+
+
+# Two-byte signatures that ordinary text can start with.
+_WEAK_MAGIC = {"pe", "bmp"}
+
+
+_JS_HINT = re.compile(r"\brequire\(|module\.exports|\bconst\s|\blet\s|=>|\bfunction\s*\w*\(")
+_PY_HINT = re.compile(r"^\s*(import \w|from [\w.]+ import|def \w+\(|exec\()", re.MULTILINE)
+
+
+def guess_code_lang(text: str) -> str | None:
+    head = text[:8192]
+    if _JS_HINT.search(head):
+        return "javascript"
+    if _PY_HINT.search(head):
+        return "python"
+    return None
+
+
+def looks_like_code(text: str) -> bool:
+    """Cheap heuristic: does this text look like JS/Python/shell source rather than data?"""
+    head = text[:4096]
+    tokens = (
+        "function",
+        "require(",
+        "import ",
+        "const ",
+        "var ",
+        "let ",
+        "=>",
+        "eval(",
+        "def ",
+        "exec(",
+        "#!/",
+        "child_process",
+        "module.exports",
+        "Buffer.from",
+    )
+    return sum(1 for t in tokens if t in head) >= 2

devguard/core/models.py

@@ -0,0 +1,195 @@
+"""Validated data model shared by the scanner, reporters, AI layer, and MCP server."""
+
+from __future__ import annotations
+
+import hashlib
+from enum import StrEnum
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+class Severity(StrEnum):
+    CRITICAL = "critical"
+    HIGH = "high"
+    MEDIUM = "medium"
+    LOW = "low"
+    INFO = "info"
+
+    @property
+    def rank(self) -> int:
+        return _SEVERITY_RANK[self]
+
+    def bump(self, tiers: int = 1) -> Severity:
+        """Raise by `tiers`, capped at critical. Info is never bumped (it carries no risk on its own)."""
+        if self is Severity.INFO:
+            return self
+        return _BY_RANK[min(self.rank + tiers, 4)]
+
+    @classmethod
+    def from_rank(cls, rank: int) -> Severity:
+        return _BY_RANK[max(0, min(rank, 4))]
+
+
+_SEVERITY_RANK = {
+    Severity.INFO: 0,
+    Severity.LOW: 1,
+    Severity.MEDIUM: 2,
+    Severity.HIGH: 3,
+    Severity.CRITICAL: 4,
+}
+_BY_RANK = {v: k for k, v in _SEVERITY_RANK.items()}
+
+
+class Confidence(StrEnum):
+    HIGH = "high"
+    MEDIUM = "medium"
+    LOW = "low"
+
+    @property
+    def rank(self) -> int:
+        return {"low": 0, "medium": 1, "high": 2}[self.value]
+
+
+class Source(StrEnum):
+    RULE = "rule"
+    ANALYZER = "analyzer"
+    CORRELATION = "correlation"
+    EXTERNAL = "external"
+    AI = "ai"
+
+
+class Evidence(BaseModel):
+    path: str
+    line_start: int | None = None
+    line_end: int | None = None
+    column: int | None = None
+    excerpt: str | None = None
+
+
+class Finding(BaseModel):
+    id: str = ""
+    rule_id: str
+    title: str
+    severity: Severity
+    confidence: Confidence
+    category: str
+    threat_refs: list[str] = Field(default_factory=list)
+    evidence: list[Evidence] = Field(default_factory=list)
+    explanation: str
+    benign_context: str = ""
+    next_step: str = ""
+    source: Source = Source.ANALYZER
+    related: list[str] = Field(default_factory=list)
+    tags: list[str] = Field(default_factory=list)
+
+    @property
+    def primary_path(self) -> str:
+        return self.evidence[0].path if self.evidence else ""
+
+    @property
+    def primary_line(self) -> int:
+        if self.evidence and self.evidence[0].line_start is not None:
+            return self.evidence[0].line_start
+        return 0
+
+    def compute_id(self) -> str:
+        first = self.evidence[0] if self.evidence else Evidence(path="")
+        key = f"{self.rule_id}|{first.path}|{first.line_start}|{first.excerpt or ''}"
+        return hashlib.sha256(key.encode("utf-8", "surrogatepass")).hexdigest()[:12]
+
+
+class Trigger(BaseModel):
+    """Something that causes code to run without the developer deliberately running it (or with one click)."""
+
+    id: str
+    kind: str
+    path: str
+    line: int | None = None
+    label: str = ""
+    command: str | None = None
+    autorun: bool = False
+    references: list[str] = Field(default_factory=list)
+    npm_scripts: list[str] = Field(default_factory=list)
+
+
+class ScanStats(BaseModel):
+    considered: int = 0
+    inspected: int = 0
+    excluded: int = 0
+    unreadable: int = 0
+    oversized: int = 0
+    skipped_limits: int = 0
+    binary: int = 0
+    symlinks: int = 0
+    special_files: int = 0
+    bytes_read: int = 0
+    vendored_dirs: list[str] = Field(default_factory=list)
+    excluded_dirs: list[str] = Field(default_factory=list)
+
+
+class ScanIssue(BaseModel):
+    kind: Literal["error", "warning", "limit"]
+    stage: str
+    message: str
+    path: str | None = None
+
+
+ScanStatus = Literal["complete", "partial", "failed"]
+Verdict = Literal["block", "review", "indicators", "no-indicators"]
+
+
+class AIAnnotation(BaseModel):
+    finding_id: str | None = None
+    status: Literal["supported", "plausible", "unverified", "likely-benign"]
+    summary: str
+    cited_evidence: list[Evidence] = Field(default_factory=list)
+    verification_steps: list[str] = Field(default_factory=list)
+
+
+class AIReport(BaseModel):
+    status: Literal["disabled", "completed", "failed", "skipped"] = "disabled"
+    profile: str | None = None
+    provider: str | None = None
+    plugin: str | None = None
+    model: str | None = None
+    overall_assessment: str | None = None
+    annotations: list[AIAnnotation] = Field(default_factory=list)
+    hypotheses: list[AIAnnotation] = Field(default_factory=list)
+    sent_summary: dict[str, Any] = Field(default_factory=dict)
+    usage: dict[str, Any] = Field(default_factory=dict)
+    audit: list[dict[str, Any]] = Field(default_factory=list)
+    error: str | None = None
+
+
+class ScanResult(BaseModel):
+    schema_version: str
+    devguard_version: str
+    ruleset_version: str
+    scan_id: str
+    started_at: str
+    duration_s: float
+    target_label: str
+    status: ScanStatus
+    verdict: Verdict
+    stats: ScanStats
+    findings: list[Finding]
+    triggers: list[Trigger] = Field(default_factory=list)
+    issues: list[ScanIssue] = Field(default_factory=list)
+    config_summary: dict[str, Any] = Field(default_factory=dict)
+    ai: AIReport = Field(default_factory=AIReport)
+
+    def severity_counts(self) -> dict[str, int]:
+        counts = {s.value: 0 for s in Severity}
+        for f in self.findings:
+            counts[f.severity.value] += 1
+        return counts
+
+    def deterministic_view(self) -> dict[str, Any]:
+        """Fields that must be byte-identical across runs on identical input (NFR-005)."""
+        return {
+            "verdict": self.verdict,
+            "status": self.status,
+            "findings": [f.model_dump(mode="json") for f in self.findings],
+            "triggers": [t.model_dump(mode="json") for t in self.triggers],
+        }

devguard/core/redact.py

@@ -0,0 +1,72 @@
+"""Secret masking and safe excerpt construction (SAFE-010, REP-006)."""
+
+from __future__ import annotations
+
+import hashlib
+import re
+import unicodedata
+
+_SECRET_PATTERNS = [
+    re.compile(r"\b(AKIA|ASIA)[0-9A-Z]{16}\b"),
+    re.compile(r"\bgh[pousr]_[A-Za-z0-9]{36,}\b"),
+    re.compile(r"\bgithub_pat_[A-Za-z0-9_]{40,}\b"),
+    re.compile(r"\bxox[abposr]-[A-Za-z0-9-]{10,}\b"),
+    re.compile(r"\bsk-(?:ant-|proj-)?[A-Za-z0-9_-]{20,}\b"),
+    re.compile(r"\bAIza[0-9A-Za-z_-]{35}\b"),
+    re.compile(r"\bnpm_[A-Za-z0-9]{36}\b"),
+    re.compile(r"\bglpat-[A-Za-z0-9_-]{20,}\b"),
+    re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)"),
+    re.compile(r"(?i)\b(?:_authToken|api[_-]?key|secret|password|passwd|token)\b\s*[=:]\s*['\"]?([^\s'\"]{8,})"),
+    re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b"),
+]
+
+MAX_EXCERPT = 300
+
+
+def _mask(value: str) -> str:
+    digest = hashlib.sha256(value.encode("utf-8", "surrogatepass")).hexdigest()[:8]
+    return f"{value[:4]}…[REDACTED sha256:{digest}]"
+
+
+def redact(text: str) -> str:
+    def repl(m: re.Match[str]) -> str:
+        if m.lastindex:
+            secret = m.group(1)
+            return m.group(0).replace(secret, _mask(secret))
+        return _mask(m.group(0))
+
+    for pat in _SECRET_PATTERNS:
+        text = pat.sub(repl, text)
+    return text
+
+
+def visible(text: str) -> str:
+    """Make control, bidi, and invisible format characters visible as <U+XXXX>."""
+    out = []
+    for ch in text:
+        cat = unicodedata.category(ch)
+        if ch in "\t":
+            out.append(ch)
+        elif (
+            cat in ("Cc", "Cf", "Co", "Cs", "Zl", "Zp")
+            or 0xFE00 <= ord(ch) <= 0xFE0F
+            or (0xE0100 <= ord(ch) <= 0xE01EF)
+        ):
+            out.append(f"<U+{ord(ch):04X}>")
+        else:
+            out.append(ch)
+    return "".join(out)
+
+
+def make_excerpt(line: str, column: int | None = None, limit: int = MAX_EXCERPT) -> str:
+    """Bounded, redacted, visible-character excerpt of one line, centred on `column` if long."""
+    line = line.rstrip("\r\n")
+    if len(line) > limit:
+        start = 0
+        if column is not None and column > limit // 2:
+            start = max(0, min(column - limit // 3, len(line) - limit))
+        chunk = line[start : start + limit]
+        prefix = "…" if start > 0 else ""
+        suffix = "…[truncated]" if start + limit < len(line) else ""
+        line = prefix + chunk + suffix
+    return visible(redact(line))