devguard-scanner 0.1.0__py3-none-any.whl

devguard/ai/tools.py

@@ -0,0 +1,389 @@
+"""Read-only investigation tools and the policy layer that executes them (AI-006, SAFE-012).
+
+The registry is a fixed allowlist. There is no shell, network, write, or code-evaluation tool. Every call is
+validated against its schema, confined to the scan root through SafeFS, capped in output size, redacted,
+and recorded in the audit log. Tool calls are proposals from an untrusted model, not commands.
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import codecs
+import fnmatch
+import re
+import urllib.parse
+import zlib
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+from devguard.ai.sdk import ToolCall, ToolResult, ToolSpec
+from devguard.analyzers.base import AnalysisContext, FileView
+from devguard.analyzers.content import ContentRuleEngine
+from devguard.analyzers.hidden import shannon_entropy
+from devguard.core.filetype import classify, guess_code_lang
+from devguard.core.redact import make_excerpt, redact, visible
+from devguard.core.safefs import FileRecord, Kind, ReadError, SafeFS, normalize_rel
+
+MAX_OUTPUT = 32 * 1024
+MAX_DECODED = 64 * 1024
+MAX_LINES = 400
+MAX_SEARCH_BYTES = 20 * 1024 * 1024
+
+
+class ToolDenied(Exception):
+    pass
+
+
+@dataclass
+class ToolEnv:
+    fs: SafeFS
+    ctx: AnalysisContext
+    engine: ContentRuleEngine
+
+
+@dataclass(frozen=True)
+class ToolDef:
+    spec: ToolSpec
+    fn: Callable[[ToolEnv, dict[str, Any]], str]
+
+
+def _schema(props: dict[str, Any], required: list[str]) -> dict[str, Any]:
+    return {"type": "object", "properties": props, "required": required, "additionalProperties": False}
+
+
+_PATH = {"type": "string", "maxLength": 1024, "description": "Repository-relative path, e.g. src/index.js"}
+
+
+# ----------------------------------------------------------------------------------------------- helpers
+def _record(env: ToolEnv, path: str) -> FileRecord:
+    norm = normalize_rel(path)
+    if norm is None:
+        raise ToolDenied("path must be relative and inside the repository")
+    rec = env.ctx.inventory.get(norm)
+    if rec is None:
+        raise ToolDenied(f"no such file in the scanned inventory: {norm}")
+    return rec
+
+
+def _text_of(env: ToolEnv, rec: FileRecord) -> tuple[str | None, bytes]:
+    try:
+        data = env.fs.read(rec.rel)
+    except ReadError as exc:
+        raise ToolDenied(f"cannot read {rec.rel}: {exc}") from exc
+    return classify(rec.rel, data).text, data
+
+
+def _safe(text: str) -> str:
+    return visible(redact(text))
+
+
+# ----------------------------------------------------------------------------------------------- tools
+def list_files(env: ToolEnv, a: dict[str, Any]) -> str:
+    pattern = a.get("glob", "*")
+    limit = min(int(a.get("limit", 200)), 500)
+    recs = [*env.ctx.inventory.files, *env.ctx.inventory.vendored_files.values()]
+    hits = [r for r in recs if fnmatch.fnmatchcase(r.rel, pattern) or fnmatch.fnmatchcase(r.name, pattern)]
+    lines = [f"{_safe(r.rel)}\t{r.size} bytes{' (vendored)' if r.vendored else ''}" for r in hits[:limit]]
+    more = f"\n… {len(hits) - limit} more" if len(hits) > limit else ""
+    return "\n".join(lines) + more if lines else "no matching files"
+
+
+def read_file(env: ToolEnv, a: dict[str, Any]) -> str:
+    rec = _record(env, a["path"])
+    text, _ = _text_of(env, rec)
+    if text is None:
+        raise ToolDenied("binary file; use file_info instead")
+    lines = text.split("\n")
+    start = max(1, int(a.get("start_line", 1)))
+    end = min(len(lines), int(a.get("end_line", start + MAX_LINES - 1)), start + MAX_LINES - 1)
+    out = [
+        f"{i:>6}| {_safe(lines[i - 1][:2000])}{' …[line truncated]' if len(lines[i - 1]) > 2000 else ''}"
+        for i in range(start, end + 1)
+    ]
+    header = f"{rec.rel} lines {start}-{end} of {len(lines)} (invisible characters shown as <U+XXXX>)"
+    return header + "\n" + "\n".join(out)
+
+
+def search(env: ToolEnv, a: dict[str, Any]) -> str:
+    needle = a["text"]
+    ci = bool(a.get("case_insensitive", False))
+    pattern = a.get("glob", "*")
+    limit = min(int(a.get("limit", 100)), 200)
+    hits: list[str] = []
+    scanned = 0
+    for rec in env.ctx.inventory.files:
+        if not (fnmatch.fnmatchcase(rec.rel, pattern) or fnmatch.fnmatchcase(rec.name, pattern)):
+            continue
+        if scanned > MAX_SEARCH_BYTES:
+            hits.append("… search byte budget reached")
+            break
+        try:
+            text, data = _text_of(env, rec)
+        except ToolDenied:
+            continue
+        scanned += len(data)
+        if text is None:
+            continue
+        hay, n = (text.lower(), needle.lower()) if ci else (text, needle)
+        idx = hay.find(n)
+        while idx != -1 and len(hits) < limit:
+            line_no = text.count("\n", 0, idx) + 1
+            line = text.split("\n")[line_no - 1]
+            col = idx - (text.rfind("\n", 0, idx) + 1)
+            hits.append(f"{_safe(rec.rel)}:{line_no}: {make_excerpt(line, col, limit=240)}")
+            idx = hay.find(n, idx + len(n))
+        if len(hits) >= limit:
+            break
+    return "\n".join(hits) if hits else "no matches"
+
+
+def _charcodes(s: str) -> bytes:
+    nums = [int(x) for x in re.findall(r"\d+", s)]
+    return "".join(chr(n) for n in nums if 0 <= n < 0x110000).encode("utf-8", "replace")
+
+
+def _b64(s: str) -> bytes:
+    compact = re.sub(r"\s+", "", s)
+    return base64.b64decode(compact + "=" * (-len(compact) % 4))
+
+
+def _inflate(data: bytes, wbits: int) -> bytes:
+    return zlib.decompressobj(wbits).decompress(data, MAX_DECODED)  # bounded: decompression-bomb safe
+
+
+_DECODERS: dict[str, Callable[[str], bytes]] = {
+    "base64": _b64,
+    "base64url": lambda s: base64.urlsafe_b64decode(s.strip() + "=" * (-len(s.strip()) % 4)),
+    "hex": lambda s: binascii.unhexlify(re.sub(r"(\\x|0x|\s|,)", "", s)),
+    "url": lambda s: urllib.parse.unquote_to_bytes(s),
+    "charcodes": _charcodes,
+    "rot13": lambda s: codecs.encode(s, "rot13").encode(),
+    "zlib_base64": lambda s: _inflate(_b64(s), zlib.MAX_WBITS),
+    "gzip_base64": lambda s: _inflate(_b64(s), 16 + zlib.MAX_WBITS),
+}
+
+
+def decode(env: ToolEnv, a: dict[str, Any]) -> str:
+    scheme = a["scheme"]
+    try:
+        raw = _DECODERS[scheme](a["data"])[:MAX_DECODED]
+    except (ValueError, binascii.Error, zlib.error, OSError, EOFError) as exc:
+        raise ToolDenied(f"decode failed: {exc}") from exc
+    c = classify("decoded", raw)
+    if c.text is None:
+        return f"binary result, {len(raw)} bytes, format={c.magic or 'unknown'}, first bytes: {raw[:48].hex()}"
+    note = f"decoded {len(raw)} bytes as text ({guess_code_lang(c.text) or 'unknown language'}):\n"
+    return note + _safe(c.text)
+
+
+def file_info(env: ToolEnv, a: dict[str, Any]) -> str:
+    rec = _record(env, a["path"])
+    _, data = _text_of(env, rec)
+    c = classify(rec.rel, data)
+    parts = [
+        f"path: {_safe(rec.rel)}",
+        f"size: {rec.size} bytes",
+        f"format (magic): {c.magic or 'none'}",
+        f"text: {c.is_text} (encoding {c.encoding}, lossy={c.lossy})",
+        f"language: {c.lang or 'unknown'}",
+        f"entropy (first 64KB, bits/byte): {shannon_entropy(data[:65536].decode('latin-1')):.2f}",
+        f"executable bit: {rec.executable_bit}",
+        f"vendored: {rec.vendored}",
+    ]
+    if not c.is_text:
+        strings = re.findall(rb"[\x20-\x7e]{6,}", data[:1_000_000])[:60]
+        parts.append("printable strings: " + " | ".join(_safe(s.decode()) for s in strings))
+    return "\n".join(parts)
+
+
+def run_rules(env: ToolEnv, a: dict[str, Any]) -> str:
+    rec = _record(env, a["path"])
+    text, data = _text_of(env, rec)
+    if text is None:
+        return "binary file: content rules apply to text only"
+    c = classify(rec.rel, data)
+    fv = FileView(rec, data, text, c.lang, c.magic, c.lossy)
+    return _format_findings(env.engine.analyze(fv, env.ctx))
+
+
+def scan_text(env: ToolEnv, a: dict[str, Any]) -> str:
+    text = a["text"][:MAX_DECODED]
+    lang = a.get("language") or guess_code_lang(text)
+    rec = FileRecord("<supplied-text>", Kind.FILE, len(text), 0o644)
+    fv = FileView(rec, text.encode(), text, lang, None, False)
+    return _format_findings(env.engine.analyze(fv, env.ctx))
+
+
+def _format_findings(findings: list[Any]) -> str:
+    if not findings:
+        return "no content rules matched"
+    return "\n".join(f"{f.rule_id} [{f.severity.value}] {f.title} (line {f.primary_line})" for f in findings)
+
+
+REGISTRY: dict[str, ToolDef] = {
+    "list_files": ToolDef(
+        ToolSpec(
+            "list_files",
+            "List repository files matching a glob (read-only).",
+            _schema(
+                {
+                    "glob": {"type": "string", "maxLength": 256},
+                    "limit": {"type": "integer", "minimum": 1, "maximum": 500},
+                },
+                [],
+            ),
+        ),
+        list_files,
+    ),
+    "read_file": ToolDef(
+        ToolSpec(
+            "read_file",
+            "Read up to 400 lines of a repository text file. Secrets are "
+            "redacted and invisible characters are shown as <U+XXXX>.",
+            _schema(
+                {
+                    "path": _PATH,
+                    "start_line": {"type": "integer", "minimum": 1},
+                    "end_line": {"type": "integer", "minimum": 1},
+                },
+                ["path"],
+            ),
+        ),
+        read_file,
+    ),
+    "search": ToolDef(
+        ToolSpec(
+            "search",
+            "Literal (non-regex) text search across repository files.",
+            _schema(
+                {
+                    "text": {"type": "string", "minLength": 2, "maxLength": 200},
+                    "glob": {"type": "string", "maxLength": 256},
+                    "case_insensitive": {"type": "boolean"},
+                    "limit": {"type": "integer", "minimum": 1, "maximum": 200},
+                },
+                ["text"],
+            ),
+        ),
+        search,
+    ),
+    "decode": ToolDef(
+        ToolSpec(
+            "decode",
+            "Decode a supplied string (one layer). Nothing is executed.",
+            _schema(
+                {
+                    "data": {"type": "string", "maxLength": 200_000},
+                    "scheme": {"type": "string", "enum": sorted(_DECODERS)},
+                },
+                ["data", "scheme"],
+            ),
+        ),
+        decode,
+    ),
+    "file_info": ToolDef(
+        ToolSpec(
+            "file_info",
+            "Metadata for a file: size, format, entropy, strings sample.",
+            _schema({"path": _PATH}, ["path"]),
+        ),
+        file_info,
+    ),
+    "run_rules": ToolDef(
+        ToolSpec(
+            "run_rules", "Run DevGuard's deterministic content rules on one file.", _schema({"path": _PATH}, ["path"])
+        ),
+        run_rules,
+    ),
+    "scan_text": ToolDef(
+        ToolSpec(
+            "scan_text",
+            "Run DevGuard's content rules on supplied text, e.g. a decoded payload.",
+            _schema(
+                {
+                    "text": {"type": "string", "maxLength": 65536},
+                    "language": {"type": "string", "enum": ["javascript", "python", "shell", "powershell", "batch"]},
+                },
+                ["text"],
+            ),
+        ),
+        scan_text,
+    ),
+}
+
+
+def validate_args(schema: dict[str, Any], args: Any) -> str | None:
+    """Minimal JSON-Schema subset validator for our own tool schemas. Returns an error or None."""
+    if not isinstance(args, dict):
+        return "arguments must be an object"
+    props = schema.get("properties", {})
+    for key in schema.get("required", []):
+        if key not in args:
+            return f"missing required argument {key!r}"
+    for key, value in args.items():
+        spec = props.get(key)
+        if spec is None:
+            return f"unknown argument {key!r}"
+        t = spec.get("type")
+        if t == "string":
+            if not isinstance(value, str):
+                return f"{key} must be a string"
+            if len(value) > spec.get("maxLength", 10**9) or len(value) < spec.get("minLength", 0):
+                return f"{key} has invalid length"
+            if "enum" in spec and value not in spec["enum"]:
+                return f"{key} must be one of {spec['enum']}"
+        elif t == "integer":
+            if not isinstance(value, int) or isinstance(value, bool):
+                return f"{key} must be an integer"
+            if value < spec.get("minimum", -(10**12)) or value > spec.get("maximum", 10**12):
+                return f"{key} out of range"
+        elif t == "boolean" and not isinstance(value, bool):
+            return f"{key} must be a boolean"
+    return None
+
+
+@dataclass
+class ToolPolicy:
+    env: ToolEnv
+    max_calls: int
+    calls: int = 0
+    audit: list[dict[str, Any]] = field(default_factory=list)
+
+    def specs(self) -> list[ToolSpec]:
+        return [d.spec for d in REGISTRY.values()]
+
+    def execute(self, call: ToolCall) -> ToolResult:
+        args_preview = make_excerpt(repr(call.arguments), limit=200)
+        tool = REGISTRY.get(call.name)
+        if tool is None:
+            return self._deny(call, args_preview, f"unknown tool {call.name!r}; only read-only tools exist")
+        if self.calls >= self.max_calls:
+            return self._deny(call, args_preview, "tool-call budget exhausted; give your final answer now")
+        error = validate_args(tool.spec.input_schema, call.arguments)
+        if error:
+            return self._deny(call, args_preview, f"invalid arguments: {error}")
+        self.calls += 1
+        try:
+            output = tool.fn(self.env, call.arguments)
+        except ToolDenied as exc:
+            return self._deny(call, args_preview, str(exc))
+        except Exception as exc:
+            return self._deny(call, args_preview, f"tool error: {type(exc).__name__}")
+        if len(output) > MAX_OUTPUT:
+            output = output[:MAX_OUTPUT] + "\n…[output truncated]"
+        self.audit.append(
+            {
+                "tool": call.name,
+                "args": args_preview,
+                "allowed": True,
+                "summary": f"{len(output)} chars",
+                "bytes": len(output),
+            }
+        )
+        return ToolResult(call.id, output)
+
+    def _deny(self, call: ToolCall, args_preview: str, reason: str) -> ToolResult:
+        self.audit.append({"tool": call.name, "args": args_preview, "allowed": False, "summary": reason})
+        return ToolResult(call.id, f"DENIED: {reason}", is_error=True)

devguard/analyzers/base.py

@@ -0,0 +1,135 @@
+"""Analyzer protocol, the per-file view handed to analyzers, and the shared analysis context.
+
+Analyzers never open files. They receive bytes/text through `FileView` and report findings and triggers.
+"""
+
+from __future__ import annotations
+
+import bisect
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+from functools import cached_property
+from pathlib import PurePosixPath
+from typing import Protocol
+
+from devguard.core.config import ScanConfig
+from devguard.core.models import Evidence, Finding, ScanIssue, Trigger
+from devguard.core.redact import make_excerpt
+from devguard.core.safefs import FileRecord, Inventory
+
+
+@dataclass
+class FileView:
+    record: FileRecord
+    data: bytes
+    text: str | None
+    lang: str | None
+    magic: str | None
+    lossy: bool
+    truncated: bool = False
+
+    @property
+    def rel(self) -> str:
+        return self.record.rel
+
+    @property
+    def name(self) -> str:
+        return PurePosixPath(self.record.rel).name
+
+    @property
+    def suffix(self) -> str:
+        return PurePosixPath(self.record.rel).suffix.lower()
+
+    @property
+    def parent(self) -> str:
+        parent = str(PurePosixPath(self.record.rel).parent)
+        return "" if parent == "." else parent
+
+    @cached_property
+    def lines(self) -> list[str]:
+        return (self.text or "").split("\n")
+
+    @cached_property
+    def _line_starts(self) -> list[int]:
+        starts = [0]
+        text = self.text or ""
+        idx = text.find("\n")
+        while idx != -1:
+            starts.append(idx + 1)
+            idx = text.find("\n", idx + 1)
+        return starts
+
+    def line_col(self, offset: int) -> tuple[int, int]:
+        """1-based line and column for a character offset."""
+        i = bisect.bisect_right(self._line_starts, offset) - 1
+        return i + 1, offset - self._line_starts[i] + 1
+
+    def evidence_at_offset(self, offset: int, end: int | None = None) -> Evidence:
+        line, col = self.line_col(offset)
+        end_line = self.line_col(end)[0] if end is not None and end > offset else line
+        return self.evidence_at_line(line, col, end_line)
+
+    def evidence_at_line(self, line: int | None, column: int | None = None, end_line: int | None = None) -> Evidence:
+        if line is None or not self.text:
+            return Evidence(path=self.rel)
+        text_line = self.lines[line - 1] if 0 < line <= len(self.lines) else ""
+        return Evidence(
+            path=self.rel,
+            line_start=line,
+            line_end=end_line if end_line and end_line != line else None,
+            column=column,
+            excerpt=make_excerpt(text_line, column),
+        )
+
+
+@dataclass
+class AnalysisContext:
+    config: ScanConfig
+    inventory: Inventory
+    triggers: list[Trigger] = field(default_factory=list)
+    issues: list[ScanIssue] = field(default_factory=list)
+    # package.json directory ("" = root) → script name → command
+    npm_scripts: dict[str, dict[str, str]] = field(default_factory=dict)
+    # files loaded automatically by tools/IDE extensions (config-as-code, SURF-012)
+    autoloaded: set[str] = field(default_factory=set)
+    # static file → referenced files (imports, sourced scripts, files read by loaders)
+    imports: dict[str, list[str]] = field(default_factory=dict)
+    _trigger_seq: int = 0
+
+    def add_trigger(
+        self,
+        kind: str,
+        path: str,
+        *,
+        line: int | None = None,
+        label: str = "",
+        command: str | None = None,
+        autorun: bool = False,
+        references: Iterable[str] = (),
+        npm_scripts: Iterable[str] = (),
+    ) -> Trigger:
+        self._trigger_seq += 1
+        trig = Trigger(
+            id=f"T{self._trigger_seq:04d}",
+            kind=kind,
+            path=path,
+            line=line,
+            label=label,
+            command=command,
+            autorun=autorun,
+            references=sorted(set(references)),
+            npm_scripts=sorted(set(npm_scripts)),
+        )
+        self.triggers.append(trig)
+        return trig
+
+    def warn(self, stage: str, message: str, path: str | None = None) -> None:
+        self.issues.append(ScanIssue(kind="warning", stage=stage, message=message, path=path))
+
+
+class Analyzer(Protocol):
+    name: str
+
+    def applies(self, fv: FileView) -> bool: ...
+
+    def analyze(self, fv: FileView, ctx: AnalysisContext) -> Iterable[Finding]: ...