ghostrules 0.1.0__py3-none-any.whl

ghostrules/__init__.py

@@ -0,0 +1,6 @@
+"""ghostrules: find the instructions your AI agent sees but you don't."""
+
+__version__ = "0.1.0"
+
+# Used for SARIF help links. Keep it in sync with the repository URL (scripts/set_owner.py).
+PROJECT_URL = "https://github.com/BardLaudian/ghostrules"

ghostrules/_compat.py

@@ -0,0 +1,12 @@
+"""Python-version compatibility shims."""
+
+from __future__ import annotations
+
+import sys
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:  # pragma: no cover - exercised only on Python 3.10
+    import tomli as tomllib
+
+__all__ = ["tomllib"]

ghostrules/clean.py

@@ -0,0 +1,41 @@
+"""``ghostrules clean``: remove the invisible characters the unicode rules flag."""
+
+from __future__ import annotations
+
+import difflib
+from dataclasses import dataclass
+
+from ghostrules.rules.unicode import removable
+from ghostrules.secrets import mask_secrets
+from ghostrules.unicode_utils import render_invisible
+
+
+@dataclass(frozen=True)
+class CleanResult:
+    """Outcome of cleaning a text."""
+
+    original: str
+    cleaned: str
+    removed: int
+    kept_bidi: int
+
+    @property
+    def changed(self) -> bool:
+        return self.original != self.cleaned
+
+
+def clean_text(text: str, *, keep_bom: bool = False) -> CleanResult:
+    """Remove exactly the characters GR001-GR004 flag (see ``rules.unicode.removable``)."""
+    removal = removable(text, keep_bom=keep_bom)
+    drop = set(removal.offsets)
+    cleaned = "".join(ch for i, ch in enumerate(text) if i not in drop)
+    return CleanResult(text, cleaned, len(drop), removal.kept_bidi)
+
+
+def unified_diff(result: CleanResult, name: str) -> str:
+    """Unified diff of the change. Invisible characters are shown as tokens so the removed
+    ones are actually visible, and secrets are masked."""
+    old = result.original.splitlines(keepends=True)
+    new = result.cleaned.splitlines(keepends=True)
+    diff = difflib.unified_diff(old, new, fromfile=name, tofile=f"{name} (cleaned)")
+    return mask_secrets(render_invisible("".join(diff), with_name=False))

ghostrules/cli.py

@@ -0,0 +1,319 @@
+"""Command line interface: ``scan`` (default), ``reveal``, ``clean``, ``rules``, ``version``."""
+
+from __future__ import annotations
+
+import enum
+import sys
+from dataclasses import replace
+from pathlib import Path
+from typing import Annotated, Any, NoReturn, Optional
+
+import typer
+from rich.console import Console, Group
+from rich.table import Table
+from rich.text import Text
+from typer.core import TyperGroup
+
+from ghostrules import __version__
+from ghostrules.clean import clean_text, unified_diff
+from ghostrules.config import Config, ConfigError, load_config
+from ghostrules.models import ScanResult, Severity
+from ghostrules.reporters import json_, markdown, sarif, table
+from ghostrules.reporters.common import safe
+from ghostrules.reveal import build as build_reveal
+from ghostrules.reveal import has_hidden_content
+from ghostrules.rules import all_rules
+from ghostrules.scanner import scan_paths
+
+EXIT_OK, EXIT_FINDINGS, EXIT_ERROR = 0, 1, 2
+
+
+class DefaultScanGroup(TyperGroup):
+    """Make ``scan`` the default command: ``ghostrules .`` and ``ghostrules --format json``."""
+
+    def parse_args(self, ctx: Any, args: list[str]) -> list[str]:
+        first = args[0] if args else None
+        if first is None or (first not in self.commands and first not in ctx.help_option_names):
+            args = ["scan", *args]
+        return super().parse_args(ctx, args)
+
+
+app = typer.Typer(
+    name="ghostrules",
+    cls=DefaultScanGroup,
+    help="Find the instructions your AI agent sees but you don't.",
+    add_completion=False,
+    rich_markup_mode=None,
+)
+
+
+class OutputFormat(str, enum.Enum):
+    """Report formats for ``scan --format``."""
+
+    table = "table"
+    json = "json"
+    sarif = "sarif"
+    markdown = "markdown"
+
+
+_RENDERERS = {
+    OutputFormat.json: json_.render,
+    OutputFormat.sarif: sarif.render,
+    OutputFormat.markdown: markdown.render,
+}
+
+
+def _fail(message: str) -> NoReturn:
+    typer.echo(f"error: {safe(message)}", err=True)
+    raise typer.Exit(EXIT_ERROR)
+
+
+def _utf8_streams() -> None:
+    """Reports contain box-drawing and ⟦U+XXXX⟧ tokens: never crash on a legacy code page."""
+    for stream in (sys.stdout, sys.stderr):
+        encoding = (getattr(stream, "encoding", None) or "").lower().replace("-", "")
+        reconfigure = getattr(stream, "reconfigure", None)
+        if reconfigure is not None and encoding != "utf8":
+            reconfigure(encoding="utf-8", errors="replace")
+
+
+def _read_text(path: Path) -> tuple[bytes, str]:
+    try:
+        data = path.read_bytes()
+    except OSError as exc:
+        _fail(f"cannot read {path}: {exc.strerror or exc}")
+    return data, data.decode("utf-8", errors="surrogateescape")
+
+
+def _parse_severity(value: str | None, *, allow_none: bool) -> Severity | None:
+    if value is None:
+        return None
+    if allow_none and value.strip().lower() == "none":
+        return None
+    try:
+        return Severity.parse(value)
+    except ValueError as exc:
+        raise typer.BadParameter(str(exc)) from None
+
+
+def _console(no_color: bool) -> Console:
+    return table.make_console(color=False if no_color else None)
+
+
+# --- scan ----------------------------------------------------------------------------------
+
+
+@app.command()
+def scan(
+    paths: Annotated[
+        Optional[list[Path]],  # noqa: UP045 - Typer needs Optional on 3.10
+        typer.Argument(help="Files or directories to scan (default: current directory)."),
+    ] = None,
+    format_: Annotated[
+        OutputFormat, typer.Option("--format", help="Report format.", case_sensitive=False)
+    ] = OutputFormat.table,
+    output: Annotated[
+        Optional[Path],  # noqa: UP045
+        typer.Option("--output", "-o", help="Write the report to this file instead of stdout."),
+    ] = None,
+    fail_on: Annotated[
+        Optional[str],  # noqa: UP045
+        typer.Option(help="Exit 1 if a finding has at least this severity, or 'none'. [high]"),
+    ] = None,
+    min_severity: Annotated[
+        Optional[str],  # noqa: UP045
+        typer.Option(help="Hide findings below this severity. [low]"),
+    ] = None,
+    all_files: Annotated[
+        bool, typer.Option("--all-files", help="Also check every text file for hidden Unicode.")
+    ] = False,
+    home: Annotated[
+        bool, typer.Option("--home", help="Also scan your user-level agent configs (read-only).")
+    ] = False,
+    disable: Annotated[
+        Optional[str],  # noqa: UP045
+        typer.Option(help="Comma-separated rule IDs to disable, e.g. GR010,GR012."),
+    ] = None,
+    config: Annotated[
+        Optional[Path],  # noqa: UP045
+        typer.Option(help="Path to a config file (default: .ghostrules.toml in the scan root)."),
+    ] = None,
+    no_color: Annotated[bool, typer.Option("--no-color", help="Plain output, no colours.")] = False,
+) -> None:
+    """Scan agent instruction and configuration files for hidden or dangerous content.
+
+    Exit codes: 0 clean (or below --fail-on), 1 findings at or above --fail-on, 2 error.
+    """
+    _utf8_streams()
+    targets = paths or [Path(".")]
+    base = targets[0] if targets[0].is_dir() else targets[0].parent
+    try:
+        cfg = load_config(base, config)
+    except ConfigError as exc:
+        _fail(str(exc))
+    cfg = _apply_overrides(cfg, fail_on, min_severity, disable)
+
+    result = scan_paths(targets, cfg, all_files=all_files, include_home=home)
+    label = ", ".join(str(p) for p in targets)
+    _emit(result, format_, output, no_color, label)
+
+    if result.errors:
+        raise typer.Exit(EXIT_ERROR)
+    if cfg.fail_on is not None and any(f.severity >= cfg.fail_on for f in result.findings):
+        raise typer.Exit(EXIT_FINDINGS)
+
+
+def _apply_overrides(
+    cfg: Config, fail_on: str | None, min_severity: str | None, disable: str | None
+) -> Config:
+    changes: dict[str, object] = {}
+    if fail_on is not None:
+        changes["fail_on"] = _parse_severity(fail_on, allow_none=True)
+    if min_severity is not None:
+        changes["min_severity"] = _parse_severity(min_severity, allow_none=False)
+    if disable:
+        ids = {part.strip().upper() for part in disable.split(",") if part.strip()}
+        changes["disable"] = cfg.disable | ids
+    return replace(cfg, **changes)  # type: ignore[arg-type]
+
+
+def _emit(
+    result: ScanResult, fmt: OutputFormat, output: Path | None, no_color: bool, label: str
+) -> None:
+    if fmt is OutputFormat.table:
+        if output is None:
+            _console(no_color).print(table.build(result, target=label))
+            return
+        text = table.render(result, color=False, width=100, target=label)
+    else:
+        text = _RENDERERS[fmt](result)
+        for warning in result.warnings:
+            typer.echo(f"warning: {safe(warning)}", err=True)
+        for error in result.errors:
+            typer.echo(f"error: {safe(error)}", err=True)
+    if output is None:
+        sys.stdout.write(text)
+        return
+    try:
+        output.write_text(text, encoding="utf-8")
+    except OSError as exc:
+        _fail(f"cannot write {output}: {exc.strerror or exc}")
+    typer.echo(f"report written to {output}", err=True)
+
+
+# --- reveal / clean ------------------------------------------------------------------------
+
+
+@app.command()
+def reveal(
+    file: Annotated[Path, typer.Argument(help="File to show.")],
+    no_color: Annotated[bool, typer.Option("--no-color", help="Plain output, no colours.")] = False,
+) -> None:
+    """Print a file with invisible characters made visible, plus the text they hide."""
+    _utf8_streams()
+    _data, text = _read_text(file)
+    console = _console(no_color)
+    header = Text.assemble(("◆ ghostrules reveal", "bold"), (f"  {safe(str(file))}", "dim"))
+    verdict = (
+        Text("hidden content found: see highlighted tokens and decoded text", style="bold red")
+        if has_hidden_content(text)
+        else Text("✔ nothing hidden: no invisible characters or HTML comments", style="green")
+    )
+    console.print(Group(header, Text(""), build_reveal(text), Text(""), verdict))
+
+
+@app.command()
+def clean(
+    file: Annotated[Path, typer.Argument(help="File to clean.")],
+    write: Annotated[
+        bool, typer.Option("--write", help="Modify the file (a .ghostrules.bak copy is kept).")
+    ] = False,
+    keep_bom: Annotated[
+        bool, typer.Option("--keep-bom", help="Keep a leading byte-order mark.")
+    ] = False,
+    no_color: Annotated[bool, typer.Option("--no-color", help="Plain output, no colours.")] = False,
+) -> None:
+    """Remove invisible characters (GR001-GR004). Without --write only show the diff."""
+    _utf8_streams()
+    data, text = _read_text(file)
+    console = _console(no_color)
+    result = clean_text(text, keep_bom=keep_bom)
+    if result.kept_bidi:
+        console.print(
+            Text(
+                f"note: kept {result.kept_bidi} bidirectional control(s): the file is mostly "
+                "right-to-left text, where they are legitimate.",
+                style="yellow",
+            )
+        )
+    if not result.changed:
+        console.print(Text("✔ Nothing to clean.", style="green"))
+        return
+    if not write:
+        console.print(_styled_diff(unified_diff(result, safe(str(file)))))
+        console.print(
+            Text(
+                f"{result.removed} character(s) would be removed. Run again with --write.",
+                style="bold",
+            )
+        )
+        return
+    backup = file.with_name(file.name + ".ghostrules.bak")
+    if backup.exists():
+        _fail(f"{backup} already exists; move it away first so no earlier backup is lost")
+    try:
+        backup.write_bytes(data)
+        file.write_bytes(result.cleaned.encode("utf-8", errors="surrogateescape"))
+    except OSError as exc:
+        _fail(f"cannot write {file}: {exc.strerror or exc}")
+    console.print(
+        Text(f"✔ Removed {result.removed} character(s) from {safe(str(file))}", style="bold green")
+    )
+    console.print(Text(f"  original saved as {safe(str(backup))}", style="dim"))
+
+
+def _styled_diff(diff: str) -> Text:
+    out = Text()
+    for line in diff.splitlines():
+        if line.startswith(("+++", "---")):
+            style = "bold"
+        elif line.startswith("@@"):
+            style = "cyan"
+        elif line.startswith("+"):
+            style = "green"
+        elif line.startswith("-"):
+            style = "red"
+        else:
+            style = ""
+        out.append(safe(line) + "\n", style=style)
+    return out
+
+
+# --- rules / version -----------------------------------------------------------------------
+
+
+@app.command()
+def rules(
+    no_color: Annotated[bool, typer.Option("--no-color", help="Plain output, no colours.")] = False,
+) -> None:
+    """List every detection rule."""
+    _utf8_streams()
+    grid = Table(box=None, pad_edge=False, header_style="bold")
+    grid.add_column("ID", no_wrap=True)
+    grid.add_column("Severity", no_wrap=True)
+    grid.add_column("Applies to", no_wrap=True, style="dim")
+    grid.add_column("Title")
+    for rule in all_rules():
+        grid.add_row(
+            Text(rule.id, style="bold"),
+            Text(rule.severity.label, style=table.TEXT_STYLE[rule.severity]),
+            Text(", ".join(sorted(rule.applies_to))),
+            Text(rule.title),
+        )
+    _console(no_color).print(grid)
+
+
+@app.command()
+def version() -> None:
+    """Print the ghostrules version."""
+    typer.echo(f"ghostrules {__version__}")

ghostrules/config.py

@@ -0,0 +1,93 @@
+"""Configuration loading: ``--config``, ``.ghostrules.toml`` or ``[tool.ghostrules]``."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from ghostrules._compat import tomllib
+from ghostrules.models import Severity
+
+CONFIG_FILENAME = ".ghostrules.toml"
+_LIST_KEYS = ("disable", "exclude", "extra_targets", "allow_domains")
+_KNOWN_KEYS = {*_LIST_KEYS, "fail_on", "min_severity"}
+
+
+class ConfigError(Exception):
+    """Raised for unreadable or invalid configuration."""
+
+
+@dataclass(frozen=True)
+class Config:
+    """Effective configuration. ``fail_on=None`` means never fail."""
+
+    disable: frozenset[str] = frozenset()
+    fail_on: Severity | None = Severity.HIGH
+    min_severity: Severity = Severity.LOW
+    exclude: tuple[str, ...] = ()
+    extra_targets: tuple[str, ...] = ()
+    allow_domains: tuple[str, ...] = ()
+    source: str | None = None
+
+
+def _string_list(table: dict[str, Any], key: str) -> list[str]:
+    value = table.get(key, [])
+    if not isinstance(value, list) or not all(isinstance(v, str) for v in value):
+        raise ConfigError(f"'{key}' must be a list of strings")
+    return value
+
+
+def parse_config(table: dict[str, Any], source: str | None = None) -> Config:
+    """Build a ``Config`` from an already-parsed TOML table, validating keys and values."""
+    unknown = sorted(set(table) - _KNOWN_KEYS)
+    if unknown:
+        raise ConfigError(f"unknown config key(s): {', '.join(unknown)}")
+    try:
+        fail_on_raw = table.get("fail_on", "high")
+        if not isinstance(fail_on_raw, str):
+            raise ConfigError("'fail_on' must be a string")
+        fail_on = None if fail_on_raw.strip().lower() == "none" else Severity.parse(fail_on_raw)
+        min_raw = table.get("min_severity", "low")
+        if not isinstance(min_raw, str):
+            raise ConfigError("'min_severity' must be a string")
+        min_severity = Severity.parse(min_raw)
+    except ValueError as exc:
+        raise ConfigError(str(exc)) from None
+    return Config(
+        disable=frozenset(rule.strip().upper() for rule in _string_list(table, "disable")),
+        fail_on=fail_on,
+        min_severity=min_severity,
+        exclude=tuple(_string_list(table, "exclude")),
+        extra_targets=tuple(_string_list(table, "extra_targets")),
+        allow_domains=tuple(d.lower() for d in _string_list(table, "allow_domains")),
+        source=source,
+    )
+
+
+def _read_toml(path: Path) -> dict[str, Any]:
+    try:
+        with path.open("rb") as handle:
+            return tomllib.load(handle)
+    except OSError as exc:
+        raise ConfigError(f"cannot read {path}: {exc.strerror or exc}") from None
+    except tomllib.TOMLDecodeError as exc:
+        raise ConfigError(f"invalid TOML in {path}: {exc}") from None
+
+
+def load_config(base_dir: Path, explicit: Path | None = None) -> Config:
+    """Load configuration, trying in order: ``explicit``, ``.ghostrules.toml``, pyproject.
+
+    Returns defaults when none is found. A pyproject without ``[tool.ghostrules]`` is skipped.
+    """
+    if explicit is not None:
+        return parse_config(_read_toml(explicit), source=str(explicit))
+    dotfile = base_dir / CONFIG_FILENAME
+    if dotfile.is_file():
+        return parse_config(_read_toml(dotfile), source=CONFIG_FILENAME)
+    pyproject = base_dir / "pyproject.toml"
+    if pyproject.is_file():
+        table = _read_toml(pyproject).get("tool", {}).get("ghostrules")
+        if isinstance(table, dict):
+            return parse_config(table, source="pyproject.toml [tool.ghostrules]")
+    return Config()

ghostrules/jsonloc.py

@@ -0,0 +1,194 @@
+"""JSON with comments: tolerant stripping and a parser that records source offsets."""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from typing import Any
+
+
+def strip_jsonc(text: str) -> str:
+    """Blank out ``//`` and ``/* */`` comments and trailing commas, outside of strings.
+
+    Characters are replaced by spaces (newlines kept) so offsets, lines and columns still match
+    the original text.
+    """
+    out: list[str] = []
+    pending_comma: int | None = None
+    in_string = False
+    i, n = 0, len(text)
+    while i < n:
+        ch = text[i]
+        if in_string:
+            out.append(ch)
+            if ch == "\\" and i + 1 < n:
+                out.append(text[i + 1])
+                i += 1
+            elif ch == '"':
+                in_string = False
+            i += 1
+        elif text.startswith("//", i):
+            end = text.find("\n", i)
+            end = n if end == -1 else end
+            out.append(" " * (end - i))
+            i = end
+        elif text.startswith("/*", i):
+            close = text.find("*/", i + 2)
+            end = n if close == -1 else close + 2
+            out.append("".join(c if c == "\n" else " " for c in text[i:end]))
+            i = end
+        else:
+            if ch in "}]" and pending_comma is not None:
+                out[pending_comma] = " "
+            if ch == ",":
+                pending_comma = len(out)
+            elif not ch.isspace():
+                pending_comma = None
+            in_string = ch == '"'
+            out.append(ch)
+            i += 1
+    return "".join(out)
+
+
+@dataclass
+class JNode:
+    """A parsed JSON value with its source span. ``end`` is exclusive.
+
+    ``members`` (objects) maps each key to ``(key_start, value_node)``; ``elements`` holds array
+    items. Both are ``None`` for scalars.
+    """
+
+    value: Any
+    start: int
+    end: int
+    members: dict[str, tuple[int, JNode]] | None = None
+    elements: list[JNode] | None = None
+
+    def get(self, key: str) -> JNode | None:
+        """Child node of an object by key."""
+        if self.members is None or key not in self.members:
+            return None
+        return self.members[key][1]
+
+    def string_span(self) -> tuple[int, int]:
+        """Span of a string value without its quotes (or the whole node for other scalars)."""
+        if isinstance(self.value, str):
+            return self.start + 1, max(self.start + 1, self.end - 1)
+        return self.start, self.end
+
+
+_SCALAR_RE = re.compile(r"-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?|true|false|null")
+_MAX_DEPTH = 100
+
+
+class _Parser:
+    def __init__(self, text: str) -> None:
+        self.text = text
+        self.pos = 1 if text.startswith("\ufeff") else 0
+
+    def _skip_ws(self) -> None:
+        while self.pos < len(self.text) and self.text[self.pos] in " \t\r\n":
+            self.pos += 1
+
+    def parse(self) -> JNode:
+        node = self._value(0)
+        self._skip_ws()
+        if self.pos != len(self.text):
+            raise ValueError("trailing data")
+        return node
+
+    def _value(self, depth: int) -> JNode:
+        if depth > _MAX_DEPTH:
+            raise ValueError("too deep")
+        self._skip_ws()
+        if self.pos >= len(self.text):
+            raise ValueError("unexpected end")
+        ch = self.text[self.pos]
+        if ch == "{":
+            return self._object(depth)
+        if ch == "[":
+            return self._array(depth)
+        if ch == '"':
+            start, raw = self.pos, self._raw_string()
+            return JNode(json.loads(raw), start, self.pos)
+        match = _SCALAR_RE.match(self.text, self.pos)
+        if not match:
+            raise ValueError("bad value")
+        self.pos = match.end()
+        return JNode(json.loads(match.group()), match.start(), match.end())
+
+    def _raw_string(self) -> str:
+        start = self.pos
+        self.pos += 1
+        while self.pos < len(self.text):
+            ch = self.text[self.pos]
+            if ch == "\\":
+                self.pos += 2
+                continue
+            self.pos += 1
+            if ch == '"':
+                return self.text[start : self.pos]
+        raise ValueError("unterminated string")
+
+    def _object(self, depth: int) -> JNode:
+        start = self.pos
+        self.pos += 1
+        members: dict[str, tuple[int, JNode]] = {}
+        value: dict[str, Any] = {}
+        self._skip_ws()
+        if self.text.startswith("}", self.pos):
+            self.pos += 1
+            return JNode(value, start, self.pos, members=members)
+        while True:
+            self._skip_ws()
+            if not self.text.startswith('"', self.pos):
+                raise ValueError("expected key")
+            key_start = self.pos
+            key = json.loads(self._raw_string())
+            self._skip_ws()
+            if not self.text.startswith(":", self.pos):
+                raise ValueError("expected colon")
+            self.pos += 1
+            node = self._value(depth + 1)
+            members[key] = (key_start, node)
+            value[key] = node.value
+            self._skip_ws()
+            if self.text.startswith(",", self.pos):
+                self.pos += 1
+            elif self.text.startswith("}", self.pos):
+                self.pos += 1
+                return JNode(value, start, self.pos, members=members)
+            else:
+                raise ValueError("expected comma or brace")
+
+    def _array(self, depth: int) -> JNode:
+        start = self.pos
+        self.pos += 1
+        elements: list[JNode] = []
+        self._skip_ws()
+        if self.text.startswith("]", self.pos):
+            self.pos += 1
+            return JNode([], start, self.pos, elements=elements)
+        while True:
+            elements.append(self._value(depth + 1))
+            self._skip_ws()
+            if self.text.startswith(",", self.pos):
+                self.pos += 1
+            elif self.text.startswith("]", self.pos):
+                self.pos += 1
+                return JNode([e.value for e in elements], start, self.pos, elements=elements)
+            else:
+                raise ValueError("expected comma or bracket")
+
+
+def parse_located(text: str) -> JNode | None:
+    """Parse JSON (comments and trailing commas tolerated) keeping source offsets.
+
+    Offsets refer to ``text`` itself, because comment stripping preserves length. Returns ``None``
+    if the text is not valid JSON.
+    """
+    try:
+        return _Parser(strip_jsonc(text)).parse()
+    except (ValueError, RecursionError):
+        return None