ghostrules 0.1.0__py3-none-any.whl

ghostrules/markdown_utils.py

@@ -0,0 +1,59 @@
+"""Small Markdown helpers shared by rules (fenced code block detection)."""
+
+from __future__ import annotations
+
+import re
+from typing import NamedTuple
+
+_OPEN_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})(.*)$")
+
+
+class FencedBlock(NamedTuple):
+    """A fenced code block. ``start``/``end`` are character offsets (end exclusive) covering the
+    fence lines too; ``lang`` is the first word of the info string (lower-case, may be empty)."""
+
+    start: int
+    end: int
+    lang: str
+
+
+def fenced_blocks(text: str) -> list[FencedBlock]:
+    """Find fenced code blocks. An unterminated fence runs to the end of the text, as it does
+    when rendered."""
+    blocks: list[FencedBlock] = []
+    offset = 0
+    open_fence: tuple[str, int, int, str] | None = None  # (char, length, start, lang)
+    for raw in text.split("\n"):
+        line = raw.removesuffix("\r")
+        line_end = offset + len(raw) + 1
+        if open_fence is None:
+            match = _OPEN_RE.match(line)
+            if match and not (match.group(1)[0] == "`" and "`" in match.group(2)):
+                info = match.group(2).strip().split()
+                lang = info[0].lower() if info else ""
+                open_fence = (match.group(1)[0], len(match.group(1)), offset, lang)
+        else:
+            char, length, start, lang = open_fence
+            stripped = line.strip()
+            indent = len(line) - len(line.lstrip(" "))
+            if indent <= 3 and len(stripped) >= length and set(stripped) == {char}:
+                blocks.append(FencedBlock(start, min(line_end, len(text)), lang))
+                open_fence = None
+        offset = line_end
+    if open_fence is not None:
+        blocks.append(FencedBlock(open_fence[2], len(text), open_fence[3]))
+    return blocks
+
+
+_INLINE_CODE_RE = re.compile(r"(?<!`)(`+)(?!`)(.+?)(?<!`)\1(?!`)")
+
+
+def inline_code_spans(text: str) -> list[tuple[int, int]]:
+    """Spans (start, end offsets) of inline code such as ``<!-- x -->``. A span never crosses a
+    line, and an unmatched backtick does not open one."""
+    spans: list[tuple[int, int]] = []
+    offset = 0
+    for raw in text.split("\n"):
+        spans.extend((offset + m.start(), offset + m.end()) for m in _INLINE_CODE_RE.finditer(raw))
+        offset += len(raw) + 1
+    return spans

ghostrules/mcp_config.py

@@ -0,0 +1,243 @@
+"""Normalise MCP server definitions from agent config files, keeping source locations.
+
+Supported shapes: ``mcpServers`` (Claude Code/Desktop, Cursor, Gemini, Roo, Windsurf, also nested
+under ``projects.*`` in ``~/.claude.json``), ``servers`` (VS Code, optionally under ``mcp``) and
+``[mcp_servers.*]`` tables in Codex TOML. Locations are character offsets into the file text, so
+rules can point at the exact value (a secret, a URL, an argument) rather than at the file.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+from ghostrules.jsonloc import JNode, parse_located
+
+if TYPE_CHECKING:
+    from ghostrules.models import ScanTarget
+
+Span = tuple[int, int]
+_URL_KEYS = ("url", "serverUrl", "httpUrl")
+
+
+@dataclass(frozen=True)
+class ServerDef:
+    """One MCP server. ``spans`` maps ``name``, ``command``, ``url``, ``trust``, ``args.<i>``,
+    ``env.<KEY>`` and ``headers.<KEY>`` to the span of that value in the file text."""
+
+    name: str
+    command: str | None = None
+    args: tuple[str, ...] = ()
+    env: Mapping[str, str] = field(default_factory=dict)
+    url: str | None = None
+    headers: Mapping[str, str] = field(default_factory=dict)
+    trust: bool = False
+    spans: Mapping[str, Span] = field(default_factory=dict)
+
+    def span(self, key: str) -> Span:
+        """Span of a field, falling back to the server's own key."""
+        return self.spans.get(key) or self.spans.get("name") or (0, 0)
+
+    @property
+    def command_name(self) -> str:
+        """Executable name without its directory or Windows extension, lower-cased."""
+        if not self.command:
+            return ""
+        base = re.split(r"[\\/]", self.command.strip())[-1].lower()
+        return re.sub(r"\.(?:exe|cmd|bat|ps1)$", "", base)
+
+
+@dataclass(frozen=True)
+class EnvEntry:
+    """A ``KEY=value`` pair from an ``env`` table, with the span of its value."""
+
+    key: str
+    value: str
+    span: Span
+
+
+def _str_map(node: JNode | None, prefix: str, spans: dict[str, Span]) -> dict[str, str]:
+    out: dict[str, str] = {}
+    if node is None or node.members is None:
+        return out
+    for key, (_key_start, child) in node.members.items():
+        if isinstance(child.value, str):
+            out[key] = child.value
+            spans[f"{prefix}.{key}"] = child.string_span()
+    return out
+
+
+def _server_from_json(name: str, key_start: int, node: JNode) -> ServerDef | None:
+    if node.members is None:
+        return None
+    spans: dict[str, Span] = {"name": (key_start, key_start + len(name) + 2)}
+    command_node = node.get("command")
+    command = command_node.value if command_node and isinstance(command_node.value, str) else None
+    if command_node is not None and command is not None:
+        spans["command"] = command_node.string_span()
+    url: str | None = None
+    for key in _URL_KEYS:
+        url_node = node.get(key)
+        if url_node is not None and isinstance(url_node.value, str):
+            url = url_node.value
+            spans["url"] = url_node.string_span()
+            break
+    if command is None and url is None:
+        return None
+    args: list[str] = []
+    args_node = node.get("args")
+    if args_node is not None and args_node.elements is not None:
+        for element in args_node.elements:
+            if isinstance(element.value, str):
+                spans[f"args.{len(args)}"] = element.string_span()
+                args.append(element.value)
+    trust_node = node.get("trust")
+    trust = bool(trust_node and trust_node.value is True)
+    if trust_node is not None:
+        spans["trust"] = trust_node.start, trust_node.end
+    return ServerDef(
+        name=name,
+        command=command,
+        args=tuple(args),
+        env=_str_map(node.get("env"), "env", spans),
+        url=url,
+        headers=_str_map(node.get("headers"), "headers", spans),
+        trust=trust,
+        spans=spans,
+    )
+
+
+def _json_containers(root: JNode) -> list[JNode]:
+    found: list[JNode] = []
+
+    def add(parent: JNode | None, key: str) -> None:
+        child = parent.get(key) if parent else None
+        if child is not None and child.members is not None:
+            found.append(child)
+
+    add(root, "mcpServers")
+    add(root, "servers")
+    add(root.get("mcp"), "servers")
+    projects = root.get("projects")
+    if projects is not None and projects.members is not None:
+        for _key_start, project in projects.members.values():
+            add(project, "mcpServers")
+    return found
+
+
+def _servers_from_json(text: str) -> list[ServerDef]:
+    root = parse_located(text)
+    if root is None or root.members is None:
+        return []
+    servers: list[ServerDef] = []
+    for container in _json_containers(root):
+        assert container.members is not None
+        for name, (key_start, node) in container.members.items():
+            server = _server_from_json(name, key_start, node)
+            if server is not None:
+                servers.append(server)
+    return servers
+
+
+def _toml_region(text: str, name: str) -> Span:
+    """Span from the ``[mcp_servers.NAME]`` header to the next unrelated table header."""
+    quoted = re.escape(name)
+    header = re.compile(
+        rf"^[ \t]*\[\s*mcp_servers\s*\.\s*(?:\"{quoted}\"|'{quoted}'|{quoted})\s*[.\]]",
+        re.MULTILINE,
+    )
+    first = header.search(text)
+    if first is None:
+        return 0, len(text)
+    end = len(text)
+    for other in re.finditer(r"^[ \t]*\[", text[first.end() :], re.MULTILINE):
+        line_start = first.end() + other.start()
+        if not header.match(text, line_start):
+            end = line_start
+            break
+    return first.start(), end
+
+
+def _toml_find(text: str, region: Span, needle: str, fallback: Span) -> Span:
+    index = text.find(needle, region[0], region[1])
+    return (index, index + len(needle)) if index != -1 and needle else fallback
+
+
+def _servers_from_toml(text: str, parsed: dict[str, Any]) -> list[ServerDef]:
+    table = parsed.get("mcp_servers")
+    if not isinstance(table, dict):
+        return []
+    servers: list[ServerDef] = []
+    for name, node in table.items():
+        if not isinstance(node, dict):
+            continue
+        region = _toml_region(text, name)
+        header = (region[0], region[0])
+        spans: dict[str, Span] = {"name": header}
+        command = node.get("command") if isinstance(node.get("command"), str) else None
+        url = next((node[k] for k in _URL_KEYS if isinstance(node.get(k), str)), None)
+        if command is None and url is None:
+            continue
+        if command:
+            spans["command"] = _toml_find(text, region, command, header)
+        if url:
+            spans["url"] = _toml_find(text, region, url, header)
+        args = tuple(a for a in node.get("args", []) if isinstance(a, str))
+        for i, arg in enumerate(args):
+            spans[f"args.{i}"] = _toml_find(text, region, arg, header)
+        maps: dict[str, dict[str, str]] = {"env": {}, "headers": {}}
+        for prefix, keys in (("env", ("env",)), ("headers", ("http_headers", "headers"))):
+            for key in keys:
+                raw = node.get(key)
+                if isinstance(raw, dict):
+                    for k, v in raw.items():
+                        if isinstance(v, str):
+                            maps[prefix][k] = v
+                            spans[f"{prefix}.{k}"] = _toml_find(text, region, v, header)
+        servers.append(
+            ServerDef(
+                name=name,
+                command=command,
+                args=args,
+                env=maps["env"],
+                url=url,
+                headers=maps["headers"],
+                trust=node.get("trust") is True,
+                spans=spans,
+            )
+        )
+    return servers
+
+
+def extract_servers(target: ScanTarget) -> list[ServerDef]:
+    """All MCP servers defined in a config target (empty for other file kinds)."""
+    if target.kind != "config" or not isinstance(target.parsed, dict):
+        return []
+    if target.path.lower().endswith(".toml"):
+        return _servers_from_toml(target.text, target.parsed)
+    return _servers_from_json(target.text)
+
+
+_SETTINGS_RE = re.compile(r"(?:^|/)settings(?:\.local)?\.json$", re.IGNORECASE)
+
+
+def is_settings_file(path: str) -> bool:
+    """Whether the path is an agent ``settings.json`` / ``settings.local.json``."""
+    return _SETTINGS_RE.search(path.replace("\\", "/")) is not None
+
+
+def settings_env_entries(target: ScanTarget) -> list[EnvEntry]:
+    """Entries of the top-level ``env`` table of a settings file."""
+    if target.kind != "config" or not is_settings_file(target.path):
+        return []
+    root = parse_located(target.text)
+    env = root.get("env") if root else None
+    if env is None or env.members is None:
+        return []
+    return [
+        EnvEntry(key, node.value, node.string_span())
+        for key, (_start, node) in env.members.items()
+        if isinstance(node.value, str)
+    ]

ghostrules/models.py

@@ -0,0 +1,188 @@
+"""Core data models: severities, findings, scan targets and scan results."""
+
+from __future__ import annotations
+
+import enum
+from bisect import bisect_right
+from dataclasses import dataclass, field
+from functools import cached_property
+from typing import TYPE_CHECKING, Any, Literal
+
+from ghostrules.markdown_utils import FencedBlock, fenced_blocks, inline_code_spans
+from ghostrules.secrets import mask_secrets
+
+if TYPE_CHECKING:
+    from ghostrules.mcp_config import ServerDef
+
+Kind = Literal["instructions", "config", "other"]
+"""What a scanned file is: agent instructions, agent/MCP config, or any other text file."""
+
+
+class Severity(enum.IntEnum):
+    """Finding severity. Ordered: ``CRITICAL > HIGH > MEDIUM > LOW > INFO``."""
+
+    INFO = 0
+    LOW = 1
+    MEDIUM = 2
+    HIGH = 3
+    CRITICAL = 4
+
+    @classmethod
+    def parse(cls, value: str) -> Severity:
+        """Parse a case-insensitive severity name, raising ``ValueError`` if unknown."""
+        try:
+            return cls[value.strip().upper()]
+        except KeyError:
+            names = ", ".join(s.label for s in cls)
+            raise ValueError(f"unknown severity {value!r} (expected one of: {names})") from None
+
+    @property
+    def label(self) -> str:
+        """Lower-case name, as used in CLI flags, config and reports."""
+        return self.name.lower()
+
+    def bumped(self) -> Severity:
+        """One level more severe, capped at ``CRITICAL``."""
+        return Severity(min(int(self) + 1, int(Severity.CRITICAL)))
+
+
+@dataclass(frozen=True)
+class Finding:
+    """A single issue reported by a rule. Line/column are 1-based, in Unicode code points."""
+
+    rule_id: str
+    severity: Severity
+    title: str
+    message: str
+    path: str
+    line: int
+    column: int
+    end_column: int
+    snippet: str
+    decoded: str | None = None
+    fix_hint: str = ""
+
+    def __post_init__(self) -> None:
+        # Single choke point: whatever built this finding, no secret survives in its text, so no
+        # reporter can print one.
+        for name in ("title", "message", "snippet", "decoded", "fix_hint"):
+            value = getattr(self, name)
+            if value:
+                object.__setattr__(self, name, mask_secrets(value))
+
+    def to_dict(self) -> dict[str, Any]:
+        """JSON-serialisable representation."""
+        return {
+            "rule_id": self.rule_id,
+            "severity": self.severity.label,
+            "title": self.title,
+            "message": self.message,
+            "path": self.path,
+            "line": self.line,
+            "column": self.column,
+            "end_column": self.end_column,
+            "snippet": self.snippet,
+            "decoded": self.decoded,
+            "fix_hint": self.fix_hint,
+        }
+
+
+@dataclass
+class ScanTarget:
+    """A file (or derived text) handed to rules. Rules must treat it as read-only.
+
+    ``origin`` is ``None`` for real files; for text derived from a finding (e.g. decoded hidden
+    text) it holds the id of the rule that produced it. ``allow_domains`` is the configured
+    allow-list, handed to rules as read-only input so they stay pure.
+    """
+
+    path: str
+    kind: Kind
+    text: str
+    parsed: Any = None
+    parse_error: str | None = None
+    origin: str | None = None
+    allow_domains: tuple[str, ...] = ()
+    lines: list[str] = field(init=False)
+
+    def __post_init__(self) -> None:
+        self.lines = [line.removesuffix("\r") for line in self.text.split("\n")]
+
+    @cached_property
+    def _line_starts(self) -> list[int]:
+        starts = [0]
+        for i, ch in enumerate(self.text):
+            if ch == "\n":
+                starts.append(i + 1)
+        return starts
+
+    @cached_property
+    def normalized(self) -> tuple[str, list[int]]:
+        """Whitespace-normalised text plus, for each of its characters, the original offset.
+
+        Every run of whitespace becomes one space, so phrases broken across lines still match.
+        """
+        chars: list[str] = []
+        offsets: list[int] = []
+        in_space = False
+        for i, ch in enumerate(self.text):
+            if ch.isspace():
+                if not in_space:
+                    chars.append(" ")
+                    offsets.append(i)
+                in_space = True
+            else:
+                chars.append(ch)
+                offsets.append(i)
+                in_space = False
+        return "".join(chars), offsets
+
+    @cached_property
+    def servers(self) -> list[ServerDef]:
+        """MCP servers defined in this config file (computed once)."""
+        from ghostrules.mcp_config import extract_servers
+
+        return extract_servers(self)
+
+    @cached_property
+    def code_blocks(self) -> list[FencedBlock]:
+        """Fenced Markdown code blocks in the text (computed once)."""
+        return fenced_blocks(self.text)
+
+    @cached_property
+    def inline_code(self) -> list[tuple[int, int]]:
+        """Spans of inline code (backticks) in the text (computed once)."""
+        return inline_code_spans(self.text)
+
+    def in_inline_code(self, offset: int) -> bool:
+        """Whether ``offset`` is inside an inline code span."""
+        return any(start <= offset < end for start, end in self.inline_code)
+
+    def code_block_at(self, offset: int) -> FencedBlock | None:
+        """The fenced code block containing ``offset``, if any."""
+        for block in self.code_blocks:
+            if block.start <= offset < block.end:
+                return block
+        return None
+
+    def position(self, offset: int) -> tuple[int, int]:
+        """Convert a character offset into a 1-based ``(line, column)`` pair."""
+        index = bisect_right(self._line_starts, offset) - 1
+        return index + 1, offset - self._line_starts[index] + 1
+
+
+@dataclass
+class ScanResult:
+    """Everything a scan produced.
+
+    ``suppressed`` counts findings hidden by inline ``ghostrules-ignore`` comments, so reports can
+    surface it. ``warnings`` are non-fatal (unreadable file, unparsable JSON); ``errors`` are rule
+    crashes, which callers should treat as a failed scan.
+    """
+
+    findings: list[Finding] = field(default_factory=list)
+    scanned_files: int = 0
+    suppressed: int = 0
+    config_source: str | None = None
+    warnings: list[str] = field(default_factory=list)
+    errors: list[str] = field(default_factory=list)

ghostrules/reporters/__init__.py

@@ -0,0 +1 @@
+"""Output formatters (table, json, sarif, markdown)."""

ghostrules/reporters/common.py

@@ -0,0 +1,86 @@
+"""Helpers shared by the reporters (no ``render`` here: this is not a reporter)."""
+
+from __future__ import annotations
+
+from collections import Counter
+from dataclasses import dataclass
+
+from ghostrules.models import Finding, ScanResult, Severity
+
+SEVERITY_ORDER = (
+    Severity.CRITICAL,
+    Severity.HIGH,
+    Severity.MEDIUM,
+    Severity.LOW,
+    Severity.INFO,
+)
+
+
+def safe(text: str) -> str:
+    """Replace lone surrogates (from undecodable input bytes) so output can always be encoded."""
+    return text.encode("utf-8", errors="replace").decode("utf-8")
+
+
+def severity_counts(findings: list[Finding]) -> dict[str, int]:
+    """Finding count per severity label, most severe first, omitting zeros."""
+    counts = Counter(f.severity for f in findings)
+    return {s.label: counts[s] for s in SEVERITY_ORDER if counts[s]}
+
+
+def summary_line(result: ScanResult) -> str:
+    """One-line summary, e.g. ``3 critical · 2 high in 4 files (scanned 17)``."""
+    counts = severity_counts(result.findings)
+    if not counts and result.errors:
+        return f"Scan incomplete (scanned {result.scanned_files} files)"
+    if not counts:
+        return f"No findings (scanned {result.scanned_files} files)"
+    parts = " · ".join(f"{n} {label}" for label, n in counts.items())
+    files = len({f.path for f in result.findings})
+    noun = "file" if files == 1 else "files"
+    return f"{parts} in {files} {noun} (scanned {result.scanned_files})"
+
+
+@dataclass(frozen=True)
+class FileGroup:
+    """All findings of one file, ordered for display."""
+
+    path: str
+    findings: list[Finding]
+
+    @property
+    def worst(self) -> Severity:
+        return max(f.severity for f in self.findings)
+
+
+def group_by_file(findings: list[Finding]) -> list[FileGroup]:
+    """Group findings by file. Files with the worst findings come first; among equals, a file
+    hiding decoded text comes before one that does not, then files nearer the root, then path."""
+    by_path: dict[str, list[Finding]] = {}
+    for f in findings:
+        by_path.setdefault(f.path, []).append(f)
+    groups = [
+        FileGroup(
+            path, sorted(items, key=lambda f: (-int(f.severity), f.line, f.column, f.rule_id))
+        )
+        for path, items in by_path.items()
+    ]
+    return sorted(
+        groups,
+        key=lambda g: (
+            -int(g.worst),
+            not any(f.decoded for f in g.findings),
+            g.path.count("/"),
+            g.path,
+        ),
+    )
+
+
+def config_notes(result: ScanResult) -> list[str]:
+    """Things a reader should know that can hide findings: repo-supplied config, suppressions."""
+    notes = []
+    if result.config_source:
+        notes.append(f"configuration loaded from {result.config_source}")
+    if result.suppressed:
+        noun = "finding" if result.suppressed == 1 else "findings"
+        notes.append(f"{result.suppressed} {noun} hidden by ghostrules-ignore comments")
+    return notes

ghostrules/reporters/json_.py

@@ -0,0 +1,45 @@
+"""JSON report: ``{"version", "scanned_files", "findings", "summary", ...}``."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from ghostrules import __version__
+from ghostrules.models import ScanResult
+from ghostrules.reporters.common import safe, severity_counts
+
+
+def _clean(value: Any) -> Any:
+    if isinstance(value, str):
+        return safe(value)
+    if isinstance(value, dict):
+        return {k: _clean(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_clean(v) for v in value]
+    return value
+
+
+def to_dict(result: ScanResult) -> dict[str, Any]:
+    """The report as plain data."""
+    return _clean(
+        {
+            "version": __version__,
+            "scanned_files": result.scanned_files,
+            "findings": [f.to_dict() for f in result.findings],
+            "summary": {
+                "total": len(result.findings),
+                "by_severity": severity_counts(result.findings),
+                "files_with_findings": len({f.path for f in result.findings}),
+                "suppressed": result.suppressed,
+                "config_source": result.config_source,
+            },
+            "warnings": result.warnings,
+            "errors": result.errors,
+        }
+    )
+
+
+def render(result: ScanResult, **_options: Any) -> str:
+    """Render the JSON report."""
+    return json.dumps(to_dict(result), indent=2, ensure_ascii=False) + "\n"

ghostrules/reporters/markdown.py

@@ -0,0 +1,55 @@
+"""Markdown report, compact enough for a PR comment or ``$GITHUB_STEP_SUMMARY``."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ghostrules.models import Finding, ScanResult, Severity
+from ghostrules.reporters.common import config_notes, safe, summary_line
+
+MAX_ROWS = 100
+_ICON = {
+    Severity.CRITICAL: "🔴",
+    Severity.HIGH: "🟠",
+    Severity.MEDIUM: "🟡",
+    Severity.LOW: "🔵",
+    Severity.INFO: "⚪",
+}
+
+
+def _cell(text: str) -> str:
+    """Make text safe inside a table cell and inert as Markdown/HTML."""
+    text = safe(text).replace("\\", "\\\\").replace("|", "\\|").replace("`", "'")
+    text = text.replace("<", "&lt;").replace(">", "&gt;")
+    return " ".join(text.split())
+
+
+def _row(f: Finding) -> str:
+    where = f"`{_cell(f.path)}:{f.line}:{f.column}`"
+    detail = _cell(f.message)
+    if f.decoded:
+        detail += f"<br>**Decoded hidden text:** `{_cell(f.decoded)}`"
+    return (
+        f"| {_ICON[f.severity]} {f.severity.label} | `{f.rule_id}` | {where} | "
+        f"**{_cell(f.title)}**<br>{detail} |"
+    )
+
+
+def render(result: ScanResult, **_options: Any) -> str:
+    """Render the Markdown report."""
+    lines = ["## ghostrules", ""]
+    if not result.findings:
+        lines.append(f"✅ **{summary_line(result)}**")
+    else:
+        lines += [f"**{summary_line(result)}**", ""]
+        lines += ["| Severity | Rule | Location | Finding |", "| --- | --- | --- | --- |"]
+        lines += [_row(f) for f in result.findings[:MAX_ROWS]]
+        extra = len(result.findings) - MAX_ROWS
+        if extra > 0:
+            lines += ["", f"_…and {extra} more findings (see the JSON or SARIF report)._"]
+    notes = config_notes(result)
+    if notes:
+        lines += [""] + [f"> **Note:** {_cell(n)}" for n in notes]
+    lines += [f"> **Warning:** {_cell(w)}" for w in result.warnings]
+    lines += [f"> **Error:** {_cell(e)}" for e in result.errors]
+    return "\n".join(lines) + "\n"