rlgate 0.0.1__py3-none-any.whl

rlgate/__init__.py

@@ -0,0 +1,7 @@
+"""rlgate — a deterministic auditor for reward-hacking vulnerabilities in RL/eval environments.
+
+Public surface is intentionally small; everything extensible lives behind the registries
+in ``rlgate.core.registry`` (probes, loaders, sandboxes, scoring schemas).
+"""
+
+__version__ = "0.0.1"

rlgate/baseline/invariant.py

@@ -0,0 +1,30 @@
+"""Phase B: oracle==1.0 / nop==0.0 at k=5, plus flaky detection. Fail-fast — if the baseline
+is not sane, probes that assume a sane baseline are skipped (they'd be noise)."""
+
+from __future__ import annotations
+
+from rlgate.core.context import Baseline
+from rlgate.core.task import Task
+from rlgate.sandbox.base import Sandbox
+
+
+def establish(task: Task, sandbox: Sandbox, k: int = 5) -> Baseline:
+    oracle_runs = [_run(task, sandbox, gold=True) for _ in range(k)]
+    nop_runs = [_run(task, sandbox, gold=False) for _ in range(k)]
+
+    flaky = set()
+    if len(set(oracle_runs)) > 1:
+        flaky.add("oracle")
+    if len(set(nop_runs)) > 1:
+        flaky.add("nop")
+
+    oracle = min(oracle_runs)   # worst-case; a single miss is a flaky/unfair signal
+    nop = max(nop_runs)
+    return Baseline(oracle=oracle, nop=nop, k=k, flaky_checks=frozenset(flaky))
+
+
+def _run(task: Task, sandbox: Sandbox, gold: bool) -> float:
+    patch = task.gold_patch.text if (gold and task.gold_patch) else None
+    with sandbox.overlay(patch) as ovl:
+        res = ovl.exec(task.grader.run)
+    return task.grader.pass_reward if res.exit_code == 0 else 0.0

rlgate/cli.py

@@ -0,0 +1,75 @@
+"""`rlgate` CLI. Stdlib argparse only — no runtime CLI deps."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from rlgate import __version__
+from rlgate.core.registry import discover
+from rlgate.core.report import to_json, to_markdown
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="rlgate", description=__doc__)
+    parser.add_argument("--version", action="version", version=f"rlgate {__version__}")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    p_list = sub.add_parser("list-probes", help="show all discovered probes")
+    p_list.add_argument("--family", help="filter by family")
+
+    p_audit = sub.add_parser("audit", help="audit an environment (path to descriptor or dir)")
+    p_audit.add_argument("target")
+    p_audit.add_argument("-k", type=int, default=5, help="baseline repetitions (default 5)")
+    p_audit.add_argument("--backend", choices=["auto", "local", "docker"], default="auto",
+                         help="sandbox backend (default auto: docker if the task has an image)")
+    p_audit.add_argument("--probe", action="append", help="restrict to these probe ids")
+    p_audit.add_argument("--format", choices=["md", "json"], default="md")
+    p_audit.add_argument("--gate", action="store_true",
+                         help="exit non-zero on any FIX/DROP (CI mode)")
+
+    args = parser.parse_args(argv)
+
+    if args.cmd == "list-probes":
+        return _list_probes(args.family)
+    if args.cmd == "audit":
+        return _audit(args)
+    return 2
+
+
+def _list_probes(family: str | None) -> int:
+    probes = discover()
+    for pid in sorted(probes):
+        meta = probes[pid].meta
+        if family and meta.family != family:
+            continue
+        reqs = ",".join(c.value for c in meta.requires) or "-"
+        print(f"{pid:42} {meta.kind.value:7} {meta.severity.name:8} needs[{reqs}]  {meta.doc}")
+    return 0
+
+
+def _audit(args) -> int:
+    from rlgate.core.report import eqs, triage
+    from rlgate.ingest import spec
+    from rlgate.pipeline import audit
+
+    task = spec.load(args.target)
+    result = audit(task, k=args.k, only=set(args.probe) if args.probe else None,
+                   backend=args.backend)
+
+    if args.format == "json":
+        print(to_json(result.task_id, result.findings, result.baseline_sane))
+    else:
+        print(to_markdown(result.task_id, result.findings, result.baseline_sane))
+
+    if args.gate:
+        t = triage(result.findings, result.baseline_sane)
+        if t.value != "KEEP":
+            print(f"\nGATE: {t.value} (eqs={eqs(result.findings, result.baseline_sane):.2f})",
+                  file=sys.stderr)
+            return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

rlgate/core/context.py

@@ -0,0 +1,47 @@
+"""Shared, expensive state built once and handed to every probe."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from rlgate.sandbox.base import Sandbox
+    from rlgate.variants.engine import VariantEngine
+
+
+@dataclass
+class Baseline:
+    """Phase B result: oracle/nop at k runs, flaky checks, and the per-test→region map."""
+
+    oracle: float                       # gold reward (want 1.0)
+    nop: float                          # empty-submission reward (want 0.0)
+    k: int
+    flaky_checks: frozenset[str] = frozenset()   # checks that disagreed across k runs
+    per_test: dict[str, bool] = field(default_factory=dict)  # test id → passed under gold
+    ok: bool = False                    # oracle==1.0 and nop==0.0 and no flaky checks
+
+    @property
+    def sane(self) -> bool:
+        return self.oracle == 1.0 and self.nop == 0.0 and not self.flaky_checks
+
+
+class ResultCache:
+    """Content-addressed cache so identical variant runs are free across a session."""
+
+    def __init__(self) -> None:
+        self._store: dict[str, Any] = {}
+
+    def get_or_run(self, key: str, fn):
+        if key not in self._store:
+            self._store[key] = fn()
+        return self._store[key]
+
+
+@dataclass
+class ProbeContext:
+    sandbox: Sandbox
+    baseline: Baseline
+    variants: VariantEngine
+    cache: ResultCache = field(default_factory=ResultCache)
+    config: dict = field(default_factory=dict)

rlgate/core/finding.py

@@ -0,0 +1,40 @@
+"""What every probe returns. A finding is a reproduced exploit or a concrete code location —
+never a threshold on a score."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum, IntEnum
+from pathlib import Path
+
+
+class Severity(IntEnum):
+    LOW = 1
+    MEDIUM = 2
+    HIGH = 3
+    CRITICAL = 4
+
+
+class VerdictTier(Enum):
+    CONFIRMED = "confirmed"   # execution-proven: a reproduced exploit or a concrete code fact
+    REVIEW = "review"         # detected deterministically; fairness needs NL/human adjudication
+    DEFERRED = "deferred"     # out of the deterministic core's scope; needs the LLM layer
+
+
+@dataclass
+class Evidence:
+    kind: str                      # "reproduced_exploit"|"code_line"|"variant"|"perms"|"diff"
+    detail: str
+    replay: str | None = None      # exact command to reproduce
+    artifacts: list[Path] = field(default_factory=list)
+
+
+@dataclass
+class Finding:
+    probe_id: str
+    title: str
+    severity: Severity
+    verdict: VerdictTier
+    evidence: Evidence
+    location: str | None = None    # file:line the finding anchors to
+    fix_hint: str | None = None    # the actionable remediation (FIX, not just DROP)

rlgate/core/probe.py

@@ -0,0 +1,44 @@
+"""The one interface every check implements. A probe is a self-contained module in
+``rlgate/probes/<family>/`` decorated with ``@register`` — discovery picks it up."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from enum import Enum
+
+from rlgate.core.context import ProbeContext
+from rlgate.core.finding import Finding, Severity
+from rlgate.core.task import Capability, Task
+
+
+class ProbeKind(Enum):
+    GENERIC = "generic"   # free with the engine (runs on any task with the capabilities)
+    PATTERN = "pattern"   # per-grader pattern library (§3.6–§3.8); provable once written
+
+
+@dataclass(frozen=True)
+class ProbeMeta:
+    id: str                                    # stable, e.g. "forgery.world_readable_tests"
+    family: str
+    kind: ProbeKind
+    severity: Severity
+    needs_live: bool = False                   # requires a live run (e.g. race probes)
+    languages: frozenset[str] | None = None    # None = language-agnostic
+    requires: frozenset[Capability] = field(default_factory=frozenset)  # task must provide ALL
+    doc: str = ""                              # one-line summary → probe-catalog.md
+
+
+class Probe(ABC):
+    meta: ProbeMeta
+
+    def applicable(self, task: Task) -> bool:
+        """Gate on CAPABILITIES, never on task.source / benchmark identity."""
+        caps_ok = self.meta.requires <= task.capabilities
+        lang_ok = self.meta.languages is None or bool(self.meta.languages & task.languages)
+        return caps_ok and lang_ok
+
+    @abstractmethod
+    def run(self, task: Task, ctx: ProbeContext) -> list[Finding]:
+        """Deterministic generation; stable verdicts. Return zero or more findings."""
+        raise NotImplementedError

rlgate/core/registry.py

@@ -0,0 +1,46 @@
+"""Discovery: a contributor drops a ``@register``-decorated probe into
+``rlgate/probes/<family>/`` and it is picked up automatically. External packages ship probes
+via the ``rlgate.probes`` entry-point group without touching this repo."""
+
+from __future__ import annotations
+
+import importlib
+import pkgutil
+from importlib import metadata
+
+from rlgate.core.probe import Probe
+
+_PROBES: dict[str, type[Probe]] = {}
+
+
+def register(cls: type[Probe]) -> type[Probe]:
+    pid = cls.meta.id
+    if pid in _PROBES and _PROBES[pid] is not cls:
+        raise ValueError(f"duplicate probe id: {pid}")
+    _PROBES[pid] = cls
+    return cls
+
+
+def _import_tree(package_name: str) -> None:
+    pkg = importlib.import_module(package_name)
+    for mod in pkgutil.walk_packages(pkg.__path__, prefix=package_name + "."):
+        if not mod.name.rsplit(".", 1)[-1].startswith("_"):
+            importlib.import_module(mod.name)
+
+
+def discover() -> dict[str, type[Probe]]:
+    """Import every in-repo probe module, then load third-party entry points. Idempotent."""
+    _import_tree("rlgate.probes")
+    try:
+        eps = metadata.entry_points(group="rlgate.probes")
+    except Exception:  # noqa: BLE001 - entry-point loading can raise many things; best-effort
+        eps = []
+    for ep in eps:
+        obj = ep.load()
+        if isinstance(obj, type) and issubclass(obj, Probe):
+            register(obj)
+    return dict(_PROBES)
+
+
+def all_probes() -> dict[str, type[Probe]]:
+    return dict(_PROBES)

rlgate/core/report.py

@@ -0,0 +1,78 @@
+"""Turn findings into an EQS + KEEP/FIX/DROP triage + JSON/Markdown."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict
+from enum import Enum
+
+from rlgate.core.finding import Finding, Severity, VerdictTier
+
+
+class Triage(Enum):
+    KEEP = "KEEP"
+    FIX = "FIX"
+    DROP = "DROP"
+
+
+def triage(findings: list[Finding], baseline_sane: bool) -> Triage:
+    if not baseline_sane:
+        return Triage.DROP
+    if any(f.severity >= Severity.CRITICAL for f in findings):
+        return Triage.DROP
+    if findings:
+        return Triage.FIX
+    return Triage.KEEP
+
+
+def eqs(findings: list[Finding], baseline_sane: bool) -> float:
+    """Environment Quality Score in [0,1]: 1.0 clean, down-weighted by severity."""
+    if not baseline_sane:
+        return 0.0
+    penalty = sum(0.1 * int(f.severity) for f in findings)
+    return max(0.0, 1.0 - penalty)
+
+
+def to_json(task_id: str, findings: list[Finding], baseline_sane: bool) -> str:
+    payload = {
+        "task": task_id,
+        "eqs": eqs(findings, baseline_sane),
+        "triage": triage(findings, baseline_sane).value,
+        "baseline_sane": baseline_sane,
+        "findings": [_finding_json(f) for f in findings],
+    }
+    return json.dumps(payload, indent=2)
+
+
+def _finding_json(f: Finding) -> dict:
+    d = asdict(f)
+    d["severity"] = f.severity.name
+    d["verdict"] = f.verdict.value
+    d["evidence"]["artifacts"] = [str(p) for p in f.evidence.artifacts]
+    return d
+
+
+def to_markdown(task_id: str, findings: list[Finding], baseline_sane: bool) -> str:
+    tier = {VerdictTier.CONFIRMED: "[CONFIRMED]", VerdictTier.REVIEW: "[REVIEW]",
+            VerdictTier.DEFERRED: "[DEFERRED]"}
+    lines = [
+        f"# rlgate audit — {task_id}",
+        (f"**EQS {eqs(findings, baseline_sane):.2f} · "
+         f"{triage(findings, baseline_sane).value}** "
+         f"· baseline {'OK' if baseline_sane else 'BROKEN'}"),
+        "",
+    ]
+    if not findings:
+        lines.append("_No findings._")
+    for f in sorted(findings, key=lambda x: x.severity, reverse=True):
+        lines += [
+            f"## {tier[f.verdict]} {f.title}  ({f.severity.name})",
+            f"- probe: `{f.probe_id}`" + (f" · at `{f.location}`" if f.location else ""),
+            f"- evidence: {f.evidence.detail}",
+        ]
+        if f.evidence.replay:
+            lines.append(f"- replay: `{f.evidence.replay}`")
+        if f.fix_hint:
+            lines.append(f"- fix: {f.fix_hint}")
+        lines.append("")
+    return "\n".join(lines)

rlgate/core/task.py

@@ -0,0 +1,102 @@
+"""The normalized environment contract. Every loader produces a ``Task``; every probe
+consumes one. Probes gate on ``Task.capabilities`` — never on a benchmark name."""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from enum import Enum
+from functools import cached_property
+from pathlib import Path
+
+try:
+    from unidiff import PatchSet
+except ImportError:  # keep import-time soft so `list-probes` works without extras
+    PatchSet = None  # type: ignore
+
+
+class Capability(Enum):
+    """What a task *provides*. Probes require these; this is the only thing they gate on."""
+
+    GOLD_PATCH = "gold_patch"      # a reference fix with addressable hunks
+    GIT_REPO = "git_repo"          # workspace is a git checkout
+    CONTAINER = "container"        # grader runs in a container we control
+    SCORING_SPEC = "scoring_spec"  # a declared reward formula (any schema plugin)
+    PER_TEST = "per_test"          # grader emits per-test results (junit/trx/go-json/…)
+    INSTRUCTION = "instruction"    # an agent-facing prompt exists
+    # extend freely — adding a capability is additive.
+
+
+@dataclass(frozen=True)
+class Patch:
+    """A unified diff. Canonicalized + hashed once so hunk decomposition is stable."""
+
+    text: str
+
+    @cached_property
+    def sha256(self) -> str:
+        return hashlib.sha256(self.text.encode()).hexdigest()
+
+    @cached_property
+    def patchset(self) -> PatchSet:
+        if PatchSet is None:
+            raise RuntimeError("unidiff is required to parse patches (pip install rlgate[dev])")
+        return PatchSet(self.text)
+
+
+@dataclass(frozen=True)
+class ReportSpec:
+    format: str          # "junit" | "trx" | "vitest_json" | "go_json" | "cpp_exit"
+    path: str            # where the grader writes it (inside the sandbox)
+
+
+@dataclass(frozen=True)
+class GraderSpec:
+    """How to score a submission. Image optional (local run); report optional (per-test)."""
+
+    run: str                        # command that scores the current workspace
+    image: str | None = None        # Dockerfile path or image ref; None → local sandbox
+    report: ReportSpec | None = None
+    pass_reward: float = 1.0        # reward when the run signals full success
+
+    @property
+    def capabilities(self) -> set[Capability]:
+        caps: set[Capability] = set()
+        if self.image:
+            caps.add(Capability.CONTAINER)
+        if self.report:
+            caps.add(Capability.PER_TEST)
+        return caps
+
+
+@dataclass(frozen=True)
+class ScoringSpec:
+    schema: str          # name of a registered ScoringSchema plugin
+    path: str            # path to the scoring file(s) this schema reads
+
+
+@dataclass(frozen=True)
+class Task:
+    id: str
+    source: str                                 # free-form provenance; NOT a probe switch
+    repo_dir: Path                              # the as-shipped workspace
+    grader: GraderSpec
+    instruction: str | None = None
+    gold_patch: Patch | None = None
+    scoring_spec: ScoringSpec | None = None
+    languages: frozenset[str] = frozenset()
+    meta: dict = field(default_factory=dict)    # source-specific; generic probes never read it
+
+    @cached_property
+    def capabilities(self) -> frozenset[Capability]:
+        caps: set[Capability] = set()
+        if self.gold_patch:
+            caps.add(Capability.GOLD_PATCH)
+        if self.scoring_spec:
+            caps.add(Capability.SCORING_SPEC)
+        if self.instruction:
+            caps.add(Capability.INSTRUCTION)
+        if (self.repo_dir / ".git").exists():
+            caps.add(Capability.GIT_REPO)
+        caps |= self.grader.capabilities
+        return frozenset(caps)

rlgate/ingest/spec.py

@@ -0,0 +1,75 @@
+"""Generic, declarative ingestion: describe ANY environment in `task.rlgate.yaml` and get a
+`Task`. Capabilities are derived from which keys are present — no benchmark-specific code.
+
+Example:
+    instruction_path: TASK.md
+    workspace: ./repo
+    gold_patch: ./solution/fix.patch        # optional → GOLD_PATCH probes
+    grader:
+      run: "bash run_tests.sh"
+      image: ./Dockerfile                   # optional → CONTAINER
+      report: {format: junit, path: /out/results.xml}   # optional → PER_TEST
+    scoring_spec:                           # optional → reward_shape probes
+      schema: harbor_scorepy
+      path: ./score.py
+    languages: [python]
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+
+from rlgate.core.task import GraderSpec, Patch, ReportSpec, ScoringSpec, Task
+
+DESCRIPTOR_NAME = "task.rlgate.yaml"
+
+
+def load(descriptor_path: str | Path) -> Task:
+    path = Path(descriptor_path)
+    if path.is_dir():
+        path = path / DESCRIPTOR_NAME
+    data = yaml.safe_load(path.read_text())
+    base = path.parent
+
+    def rel(p: str) -> Path:
+        return (base / p).resolve()
+
+    g = data["grader"]
+    report = None
+    if g.get("report"):
+        report = ReportSpec(format=g["report"]["format"], path=g["report"]["path"])
+    grader = GraderSpec(
+        run=g["run"],
+        image=str(rel(g["image"])) if g.get("image") else None,
+        report=report,
+        pass_reward=float(g.get("pass_reward", 1.0)),
+    )
+
+    instruction = None
+    if data.get("instruction_path"):
+        instruction = rel(data["instruction_path"]).read_text()
+    elif data.get("instruction"):
+        instruction = data["instruction"]
+
+    gold = None
+    if data.get("gold_patch"):
+        gold = Patch(text=rel(data["gold_patch"]).read_text())
+
+    scoring = None
+    if data.get("scoring_spec"):
+        s = data["scoring_spec"]
+        scoring = ScoringSpec(schema=s["schema"], path=str(rel(s["path"])))
+
+    return Task(
+        id=data.get("id", path.parent.name),
+        source=data.get("source", str(path)),
+        repo_dir=rel(data["workspace"]),
+        grader=grader,
+        instruction=instruction,
+        gold_patch=gold,
+        scoring_spec=scoring,
+        languages=frozenset(data.get("languages", [])),
+        meta=data.get("meta", {}),
+    )

rlgate/pipeline.py

@@ -0,0 +1,64 @@
+"""Orchestration: ingest → baseline (fail-fast) → applicable probes (capability-gated) →
+report. Probes gate on capabilities only; the pipeline never branches on benchmark identity."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from rlgate.baseline.invariant import establish
+from rlgate.core.context import ProbeContext
+from rlgate.core.finding import Finding
+from rlgate.core.registry import discover
+from rlgate.core.task import Task
+from rlgate.sandbox.base import Sandbox
+from rlgate.sandbox.select import build_sandbox
+from rlgate.variants.engine import VariantEngine
+
+
+@dataclass
+class AuditResult:
+    task_id: str
+    baseline_sane: bool
+    findings: list[Finding]
+
+
+def audit(task: Task, sandbox: Sandbox | None = None, k: int = 5,
+          only: set[str] | None = None, backend: str = "auto") -> AuditResult:
+    sandbox = sandbox or build_sandbox(task, prefer=backend)
+    sandbox.build()
+
+    baseline = establish(task, sandbox, k=k)
+    ctx = ProbeContext(
+        sandbox=sandbox,
+        baseline=baseline,
+        variants=VariantEngine(task, sandbox, baseline),
+    )
+
+    findings: list[Finding] = []
+    for pid, cls in discover().items():
+        if only and pid not in only:
+            continue
+        probe = cls()
+        if not probe.applicable(task):
+            continue
+        # Fail-fast: when the baseline is broken, skip probes that assume it's sane.
+        if not baseline.sane and probe.meta.family in {"grader_strength", "reward_shape"}:
+            continue
+        findings.extend(probe.run(task, ctx))
+
+    if not baseline.sane:
+        findings.insert(0, _baseline_finding(baseline))
+    return AuditResult(task.id, baseline.sane, findings)
+
+
+def _baseline_finding(baseline) -> Finding:
+    from rlgate.core.finding import Evidence, Severity, VerdictTier
+    detail = f"oracle={baseline.oracle} nop={baseline.nop} flaky={sorted(baseline.flaky_checks)}"
+    return Finding(
+        probe_id="baseline.invariant",
+        title="baseline not sane (oracle!=1.0 / nop!=0.0 / flaky)",
+        severity=Severity.CRITICAL,
+        verdict=VerdictTier.CONFIRMED,
+        evidence=Evidence(kind="reproduced_exploit", detail=detail),
+        fix_hint="the environment is broken before hacking is even discussed; fix this first.",
+    )

rlgate/probes/forgery/world_readable_tests.py

@@ -0,0 +1,44 @@
+"""§3.1 — held-out sources / scorer readable by the candidate UID (not chmod 700)."""
+
+from __future__ import annotations
+
+from rlgate.core.context import ProbeContext
+from rlgate.core.finding import Evidence, Finding, Severity, VerdictTier
+from rlgate.core.probe import Probe, ProbeKind, ProbeMeta
+from rlgate.core.registry import register
+from rlgate.core.task import Task
+
+
+@register
+class WorldReadableTests(Probe):
+    meta = ProbeMeta(
+        id="forgery.world_readable_tests",
+        family="forgery",
+        kind=ProbeKind.GENERIC,
+        severity=Severity.HIGH,
+        doc="/tests (held-out sources, scorer) is readable by the candidate UID.",
+    )
+
+    # The directory holding graded sources; overridable via task.meta without special-casing.
+    def _tests_dir(self, task: Task) -> str:
+        return task.meta.get("tests_dir", "tests")
+
+    def run(self, task: Task, ctx: ProbeContext) -> list[Finding]:
+        tdir = self._tests_dir(task)
+        with ctx.sandbox.overlay(None) as ovl:
+            perms = ovl.read_perms(tdir)
+        if perms.world_readable:
+            return [Finding(
+                probe_id=self.meta.id,
+                title=f"{tdir}/ is world-readable; held-out names leak at runtime",
+                severity=Severity.HIGH,
+                verdict=VerdictTier.CONFIRMED,
+                location=tdir,
+                evidence=Evidence(
+                    kind="perms",
+                    detail=f"mode={perms.mode:#o}",
+                    replay=f"rlgate audit <task> --probe {self.meta.id}",
+                ),
+                fix_hint="chmod 700 the graded dir; delete graded sources after build.",
+            )]
+        return []