rlgate 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. rlgate/__init__.py +7 -0
  2. rlgate/baseline/__init__.py +0 -0
  3. rlgate/baseline/invariant.py +30 -0
  4. rlgate/cli.py +75 -0
  5. rlgate/core/__init__.py +0 -0
  6. rlgate/core/context.py +47 -0
  7. rlgate/core/finding.py +40 -0
  8. rlgate/core/probe.py +44 -0
  9. rlgate/core/registry.py +46 -0
  10. rlgate/core/report.py +78 -0
  11. rlgate/core/task.py +102 -0
  12. rlgate/ingest/__init__.py +0 -0
  13. rlgate/ingest/presets/__init__.py +0 -0
  14. rlgate/ingest/schemas/__init__.py +0 -0
  15. rlgate/ingest/spec.py +75 -0
  16. rlgate/pipeline.py +64 -0
  17. rlgate/probes/__init__.py +0 -0
  18. rlgate/probes/forgery/__init__.py +0 -0
  19. rlgate/probes/forgery/world_readable_tests.py +44 -0
  20. rlgate/probes/grader_strength/__init__.py +0 -0
  21. rlgate/probes/grader_strength/revert_probe.py +44 -0
  22. rlgate/probes/hygiene/__init__.py +0 -0
  23. rlgate/probes/leakage/__init__.py +0 -0
  24. rlgate/probes/leakage/git_unreachable_objects.py +46 -0
  25. rlgate/probes/patterns/__init__.py +0 -0
  26. rlgate/probes/reward_shape/__init__.py +0 -0
  27. rlgate/probes/test_quality/__init__.py +0 -0
  28. rlgate/sandbox/__init__.py +0 -0
  29. rlgate/sandbox/_patch.py +30 -0
  30. rlgate/sandbox/base.py +63 -0
  31. rlgate/sandbox/docker.py +120 -0
  32. rlgate/sandbox/local.py +62 -0
  33. rlgate/sandbox/select.py +22 -0
  34. rlgate/variants/__init__.py +0 -0
  35. rlgate/variants/engine.py +76 -0
  36. rlgate/variants/revert.py +70 -0
  37. rlgate-0.0.1.dist-info/METADATA +81 -0
  38. rlgate-0.0.1.dist-info/RECORD +41 -0
  39. rlgate-0.0.1.dist-info/WHEEL +4 -0
  40. rlgate-0.0.1.dist-info/entry_points.txt +2 -0
  41. rlgate-0.0.1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,44 @@
1
+ """§3.4 — the flagship. Remove one hunk of the gold fix; if the grader still scores it pass,
2
+ that hunk is unguarded. Building revert survivors are exploits by construction."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from rlgate.core.context import ProbeContext
7
+ from rlgate.core.finding import Evidence, Finding, Severity, VerdictTier
8
+ from rlgate.core.probe import Probe, ProbeKind, ProbeMeta
9
+ from rlgate.core.registry import register
10
+ from rlgate.core.task import Capability, Task
11
+ from rlgate.variants.engine import VariantClass
12
+
13
+
14
+ @register
15
+ class RevertProbe(Probe):
16
+ meta = ProbeMeta(
17
+ id="grader_strength.revert_probe",
18
+ family="grader_strength",
19
+ kind=ProbeKind.GENERIC,
20
+ severity=Severity.HIGH,
21
+ requires=frozenset({Capability.GOLD_PATCH}),
22
+ doc="A gold fix hunk can be removed and the grader still passes (unguarded line).",
23
+ )
24
+
25
+ def run(self, task: Task, ctx: ProbeContext) -> list[Finding]:
26
+ results = ctx.variants.run_revert()
27
+ findings: list[Finding] = []
28
+ for r in results:
29
+ if r.is_exploit and r.klass is VariantClass.DISCRIMINATED:
30
+ findings.append(Finding(
31
+ probe_id=self.meta.id,
32
+ title=f"unguarded fix hunk: {r.variant.origin}",
33
+ severity=Severity.HIGH,
34
+ verdict=VerdictTier.CONFIRMED,
35
+ location=r.variant.origin,
36
+ evidence=Evidence(
37
+ kind="reproduced_exploit",
38
+ detail=f"removing this hunk still scores pass (reward={r.reward}); "
39
+ f"variant {r.variant.id}",
40
+ replay=f"rlgate replay {task.id} --variant {r.variant.id}",
41
+ ),
42
+ fix_hint="add a test that fails when this hunk is absent.",
43
+ ))
44
+ return findings
File without changes
File without changes
@@ -0,0 +1,46 @@
1
+ """§3.3 — the reference fix survives as an unreachable git object (the MiMo/SQLGlot leak).
2
+
3
+ We read the object store directly as the auditor, so an agent-side "git commands blocked"
4
+ guard is irrelevant — if the object is on disk, the answer is on disk."""
5
+
6
+ from __future__ import annotations
7
+
8
+ from rlgate.core.context import ProbeContext
9
+ from rlgate.core.finding import Evidence, Finding, Severity, VerdictTier
10
+ from rlgate.core.probe import Probe, ProbeKind, ProbeMeta
11
+ from rlgate.core.registry import register
12
+ from rlgate.core.task import Capability, Task
13
+
14
+
15
+ @register
16
+ class GitUnreachableObjects(Probe):
17
+ meta = ProbeMeta(
18
+ id="leakage.git_unreachable_objects",
19
+ family="leakage",
20
+ kind=ProbeKind.GENERIC,
21
+ severity=Severity.HIGH,
22
+ requires=frozenset({Capability.GIT_REPO}),
23
+ doc="Unreachable git objects (deleted-but-unpruned branches) may hold the fix.",
24
+ )
25
+
26
+ def run(self, task: Task, ctx: ProbeContext) -> list[Finding]:
27
+ with ctx.sandbox.overlay(None) as ovl:
28
+ res = ovl.exec("git fsck --unreachable --no-reflogs 2>/dev/null | head -n 50")
29
+ lines = [ln for ln in res.stdout.splitlines() if "unreachable commit" in ln]
30
+ if not lines:
31
+ return []
32
+ sample = lines[0].split()[-1][:12]
33
+ return [Finding(
34
+ probe_id=self.meta.id,
35
+ title=f"{len(lines)} unreachable git commit(s) on disk (e.g. {sample})",
36
+ severity=Severity.HIGH,
37
+ verdict=VerdictTier.CONFIRMED,
38
+ location=".git",
39
+ evidence=Evidence(
40
+ kind="reproduced_exploit",
41
+ detail="later history was deleted but not pruned; readable via the object store",
42
+ replay="git fsck --unreachable --no-reflogs",
43
+ ),
44
+ fix_hint="git reflog expire --expire=now --all && git gc --prune=now; "
45
+ "or vendor a clean snapshot at the task base commit.",
46
+ )]
File without changes
File without changes
File without changes
File without changes
@@ -0,0 +1,30 @@
1
+ """Shared, deterministic patch application (zero fuzz: all-or-nothing). Used by every
2
+ sandbox backend so variant generation behaves identically regardless of where it runs."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import subprocess
7
+ from pathlib import Path
8
+
9
+
10
+ class PatchConflict(RuntimeError):
11
+ """A variant patch did not apply cleanly — bucketed as could-not-generate."""
12
+
13
+
14
+ def apply_zero_fuzz(root: Path, patch_text: str) -> None:
15
+ """Apply a unified diff under `root` with no fuzz. Tries `git apply` then `patch`.
16
+
17
+ Zero fuzz makes the result deterministic: a hunk applies at its exact location or the
18
+ whole patch is rejected (raising PatchConflict)."""
19
+ proc = subprocess.run(
20
+ ["git", "apply", "--unsafe-paths", "-C0", "-p1", "-"],
21
+ cwd=root, input=patch_text, text=True, capture_output=True, check=False,
22
+ )
23
+ if proc.returncode == 0:
24
+ return
25
+ proc = subprocess.run(
26
+ ["patch", "-p1", "--no-backup-if-mismatch", "-F0", "-s"],
27
+ cwd=root, input=patch_text, text=True, capture_output=True, check=False,
28
+ )
29
+ if proc.returncode != 0:
30
+ raise PatchConflict(proc.stderr or proc.stdout)
rlgate/sandbox/base.py ADDED
@@ -0,0 +1,63 @@
1
+ """Execution substrate. Backends (local/docker/daytona) implement this ABC.
2
+
3
+ The key operations the probes need: build once, snapshot, cheap per-variant overlays, exec,
4
+ inspect permissions, and kill stray candidate processes (forgery/race probes)."""
5
+
6
+ from __future__ import annotations
7
+
8
+ from abc import ABC, abstractmethod
9
+ from dataclasses import dataclass
10
+ from pathlib import Path
11
+ from typing import Self
12
+
13
+
14
+ @dataclass
15
+ class ExecResult:
16
+ exit_code: int
17
+ stdout: str
18
+ stderr: str
19
+
20
+
21
+ @dataclass
22
+ class Perms:
23
+ mode: int # e.g. 0o755
24
+ exists: bool = True
25
+
26
+ @property
27
+ def world_readable(self) -> bool:
28
+ return self.exists and bool(self.mode & 0o004)
29
+
30
+
31
+ class Overlay(ABC):
32
+ """A cheap, disposable copy of the workspace with a patch applied."""
33
+
34
+ root: Path
35
+
36
+ @abstractmethod
37
+ def exec(self, cmd: str, timeout: int = 900) -> ExecResult: ...
38
+
39
+ @abstractmethod
40
+ def read_perms(self, path: str) -> Perms: ...
41
+
42
+ @abstractmethod
43
+ def close(self) -> None: ...
44
+
45
+ def __enter__(self) -> Self:
46
+ return self
47
+
48
+ def __exit__(self, *exc) -> None:
49
+ self.close()
50
+
51
+
52
+ class Sandbox(ABC):
53
+ @abstractmethod
54
+ def build(self) -> None:
55
+ """Build the task image / prepare the pristine workspace once."""
56
+
57
+ @abstractmethod
58
+ def overlay(self, patch_text: str | None = None) -> Overlay:
59
+ """A fresh overlay on the snapshot, optionally with a patch applied (zero fuzz)."""
60
+
61
+ @abstractmethod
62
+ def pkill_user(self, user: str) -> None:
63
+ """Kill detached candidate processes before scoring (race-probe defense)."""
@@ -0,0 +1,120 @@
1
+ """Docker sandbox: real isolation for running untrusted variants, container graders, and the
2
+ live forgery/race probes. Same ABC as the local backend.
3
+
4
+ Model: build the image once; keep a pristine host copy of the workspace as the snapshot.
5
+ Each overlay is a fresh container plus a host copy with the variant patch applied (zero fuzz,
6
+ shared logic) and copied in — so variant generation is identical to the local backend and the
7
+ image needs no git/patch tooling.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import shutil
13
+ import subprocess
14
+ import tempfile
15
+ import uuid
16
+ from pathlib import Path
17
+
18
+ from rlgate.sandbox._patch import apply_zero_fuzz
19
+ from rlgate.sandbox.base import ExecResult, Overlay, Perms, Sandbox
20
+
21
+
22
+ class DockerUnavailable(RuntimeError):
23
+ pass
24
+
25
+
26
+ def available() -> bool:
27
+ try:
28
+ return subprocess.run(["docker", "info"], capture_output=True, check=False).returncode == 0
29
+ except FileNotFoundError:
30
+ return False
31
+
32
+
33
+ def _docker(*args: str, **kw) -> subprocess.CompletedProcess:
34
+ return subprocess.run(["docker", *args], capture_output=True, text=True, check=False, **kw)
35
+
36
+
37
+ class DockerOverlay(Overlay):
38
+ def __init__(self, container_id: str, host_copy: Path, sandbox: DockerSandbox) -> None:
39
+ self.container_id = container_id
40
+ self.root = host_copy # host-side view (for parity); container path is /work
41
+ self._sandbox = sandbox
42
+
43
+ def exec(self, cmd: str, timeout: int = 900) -> ExecResult:
44
+ p = _docker("exec", "-w", "/work", self.container_id, "sh", "-c", cmd, timeout=timeout)
45
+ return ExecResult(p.returncode, p.stdout, p.stderr)
46
+
47
+ def read_perms(self, path: str) -> Perms:
48
+ p = _docker("exec", self.container_id, "stat", "-c", "%a", f"/work/{path.lstrip('/')}")
49
+ if p.returncode != 0:
50
+ return Perms(mode=0, exists=False)
51
+ return Perms(mode=int(p.stdout.strip(), 8))
52
+
53
+ def close(self) -> None:
54
+ _docker("rm", "-f", self.container_id)
55
+ self._sandbox._live.discard(self.container_id)
56
+ shutil.rmtree(self.root, ignore_errors=True)
57
+
58
+
59
+ class DockerSandbox(Sandbox):
60
+ def __init__(self, workspace: Path, image_ref: str | None = None,
61
+ dockerfile: Path | None = None, task_id: str = "task") -> None:
62
+ self.workspace = Path(workspace)
63
+ self.image_ref = image_ref
64
+ self.dockerfile = Path(dockerfile) if dockerfile else None
65
+ self.task_id = task_id
66
+ self._tag: str | None = None
67
+ self._snapshot: Path | None = None
68
+ self._live: set[str] = set()
69
+
70
+ @classmethod
71
+ def from_task(cls, task) -> DockerSandbox:
72
+ """grader.image is either a Dockerfile path (build it) or an existing image ref."""
73
+ img = task.grader.image
74
+ p = Path(img) if img else None
75
+ if p and p.exists() and p.is_file():
76
+ return cls(task.repo_dir, dockerfile=p, task_id=task.id)
77
+ return cls(task.repo_dir, image_ref=img, task_id=task.id)
78
+
79
+ def build(self) -> None:
80
+ if not available():
81
+ raise DockerUnavailable("docker is not available on this host")
82
+ if self.dockerfile:
83
+ self._tag = f"rlgate-{self.task_id}-{uuid.uuid4().hex[:8]}"
84
+ r = _docker("build", "-t", self._tag, "-f", str(self.dockerfile),
85
+ str(self.dockerfile.parent))
86
+ if r.returncode != 0:
87
+ raise DockerUnavailable(f"image build failed:\n{r.stderr}")
88
+ else:
89
+ self._tag = self.image_ref
90
+ # pristine host copy of the workspace (the snapshot we cp into each container)
91
+ snap = Path(tempfile.mkdtemp(prefix="rlgate-dsnap-"))
92
+ shutil.rmtree(snap)
93
+ shutil.copytree(self.workspace, snap, symlinks=True)
94
+ self._snapshot = snap
95
+
96
+ def overlay(self, patch_text: str | None = None) -> DockerOverlay:
97
+ assert self._snapshot is not None and self._tag is not None, "call build() first"
98
+ host_copy = Path(tempfile.mkdtemp(prefix="rlgate-dovl-"))
99
+ shutil.rmtree(host_copy)
100
+ shutil.copytree(self._snapshot, host_copy, symlinks=True)
101
+ if patch_text:
102
+ apply_zero_fuzz(host_copy, patch_text)
103
+
104
+ run = _docker("run", "-d", "--network=none", "-w", "/work",
105
+ self._tag, "sleep", "infinity")
106
+ if run.returncode != 0:
107
+ raise DockerUnavailable(f"container start failed:\n{run.stderr}")
108
+ cid = run.stdout.strip()
109
+ self._live.add(cid)
110
+ _docker("exec", cid, "mkdir", "-p", "/work")
111
+ cp = _docker("cp", f"{host_copy}/.", f"{cid}:/work")
112
+ if cp.returncode != 0:
113
+ _docker("rm", "-f", cid)
114
+ raise DockerUnavailable(f"workspace copy failed:\n{cp.stderr}")
115
+ return DockerOverlay(cid, host_copy, self)
116
+
117
+ def pkill_user(self, user: str) -> None:
118
+ """Kill detached candidate processes in every live container before scoring."""
119
+ for cid in list(self._live):
120
+ _docker("exec", cid, "pkill", "-u", user)
@@ -0,0 +1,62 @@
1
+ """Local sandbox: copy the workspace to a temp dir, apply a patch with zero fuzz, run the
2
+ grader via subprocess. No isolation guarantees — for dev and CI fixtures only. The Docker
3
+ backend implements the same ABC for real isolation (and the race/perms probes)."""
4
+
5
+ from __future__ import annotations
6
+
7
+ import os
8
+ import shutil
9
+ import subprocess
10
+ import tempfile
11
+ from pathlib import Path
12
+
13
+ from rlgate.sandbox._patch import PatchConflict, apply_zero_fuzz
14
+ from rlgate.sandbox.base import ExecResult, Overlay, Perms, Sandbox
15
+
16
+ __all__ = ["LocalOverlay", "LocalSandbox", "PatchConflict"]
17
+
18
+
19
+ class LocalOverlay(Overlay):
20
+ def __init__(self, root: Path) -> None:
21
+ self.root = root
22
+
23
+ def exec(self, cmd: str, timeout: int = 900) -> ExecResult:
24
+ p = subprocess.run(
25
+ cmd, shell=True, cwd=self.root, capture_output=True, text=True,
26
+ timeout=timeout, check=False,
27
+ env={**os.environ, "RLGATE_SANDBOX": "local"},
28
+ )
29
+ return ExecResult(p.returncode, p.stdout, p.stderr)
30
+
31
+ def read_perms(self, path: str) -> Perms:
32
+ target = self.root / path.lstrip("/")
33
+ if not target.exists():
34
+ return Perms(mode=0, exists=False)
35
+ return Perms(mode=os.stat(target).st_mode & 0o777)
36
+
37
+ def close(self) -> None:
38
+ shutil.rmtree(self.root, ignore_errors=True)
39
+
40
+
41
+ class LocalSandbox(Sandbox):
42
+ def __init__(self, workspace: Path) -> None:
43
+ self.workspace = Path(workspace)
44
+ self._snapshot: Path | None = None
45
+
46
+ def build(self) -> None:
47
+ snap = Path(tempfile.mkdtemp(prefix="rlgate-snap-"))
48
+ shutil.rmtree(snap)
49
+ shutil.copytree(self.workspace, snap, symlinks=True)
50
+ self._snapshot = snap
51
+
52
+ def overlay(self, patch_text: str | None = None) -> LocalOverlay:
53
+ assert self._snapshot is not None, "call build() first"
54
+ root = Path(tempfile.mkdtemp(prefix="rlgate-ovl-"))
55
+ shutil.rmtree(root)
56
+ shutil.copytree(self._snapshot, root, symlinks=True)
57
+ if patch_text:
58
+ apply_zero_fuzz(root, patch_text)
59
+ return LocalOverlay(root)
60
+
61
+ def pkill_user(self, user: str) -> None:
62
+ subprocess.run(["pkill", "-u", user], capture_output=True, check=False)
@@ -0,0 +1,22 @@
1
+ """Backend selection. `auto` uses Docker when the task declares a container image and Docker
2
+ is available, otherwise the local sandbox. Explicit `local`/`docker` override the choice."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from rlgate.core.task import Capability, Task
7
+ from rlgate.sandbox.base import Sandbox
8
+
9
+
10
+ def build_sandbox(task: Task, prefer: str = "auto") -> Sandbox:
11
+ from rlgate.sandbox.docker import DockerSandbox, available
12
+ from rlgate.sandbox.local import LocalSandbox
13
+
14
+ if prefer == "local":
15
+ return LocalSandbox(task.repo_dir)
16
+ if prefer == "docker":
17
+ return DockerSandbox.from_task(task)
18
+
19
+ # auto
20
+ if Capability.CONTAINER in task.capabilities and available():
21
+ return DockerSandbox.from_task(task)
22
+ return LocalSandbox(task.repo_dir)
File without changes
@@ -0,0 +1,76 @@
1
+ """Generate variants, build-gate them, run on the per-test subset, classify, and score.
2
+
3
+ Classification (ARCHITECTURE / PLAN §3.4):
4
+ - did_not_build : patch didn't apply, or the build/compile failed → NOT coverage signal
5
+ - failed_all : every subset test failed (refactor/rename hunk) → NOT coverage signal
6
+ - discriminated : built and some tests passed, some failed → the real signal
7
+ A *building* revert/stub survivor that still scores pass is a reproduced exploit.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from dataclasses import dataclass
13
+ from enum import Enum
14
+
15
+ from rlgate.core.context import Baseline
16
+ from rlgate.core.task import Task
17
+ from rlgate.sandbox.base import Sandbox
18
+ from rlgate.sandbox.local import PatchConflict
19
+ from rlgate.variants.revert import Variant, revert_variants
20
+
21
+
22
+ class VariantClass(Enum):
23
+ DID_NOT_BUILD = "did_not_build"
24
+ FAILED_ALL = "failed_all"
25
+ DISCRIMINATED = "discriminated"
26
+
27
+
28
+ @dataclass
29
+ class VariantResult:
30
+ variant: Variant
31
+ klass: VariantClass
32
+ reward: float # grader reward on this variant (only meaningful if built)
33
+ is_exploit: bool # built + scored pass despite being wrong-by-construction
34
+
35
+
36
+ class VariantEngine:
37
+ def __init__(self, task: Task, sandbox: Sandbox, baseline: Baseline) -> None:
38
+ self.task = task
39
+ self.sandbox = sandbox
40
+ self.baseline = baseline
41
+
42
+ def run_revert(self) -> list[VariantResult]:
43
+ assert self.task.gold_patch is not None
44
+ results: list[VariantResult] = []
45
+ for v in revert_variants(self.task.gold_patch):
46
+ results.append(self._score(v))
47
+ return results
48
+
49
+ def _score(self, v: Variant) -> VariantResult:
50
+ try:
51
+ overlay = self.sandbox.overlay(v.patch_text)
52
+ except PatchConflict:
53
+ return VariantResult(v, VariantClass.DID_NOT_BUILD, 0.0, False)
54
+ with overlay:
55
+ run = overlay.exec(self.task.grader.run)
56
+ if run.exit_code == 127 or "No such file" in run.stderr:
57
+ return VariantResult(v, VariantClass.DID_NOT_BUILD, 0.0, False)
58
+ reward = self._reward_from(run.exit_code)
59
+ # pass despite a missing fix hunk → exploit; full fail → good coverage.
60
+ if reward >= self.task.grader.pass_reward:
61
+ return VariantResult(v, VariantClass.DISCRIMINATED, reward, True)
62
+ if reward <= 0.0:
63
+ return VariantResult(v, VariantClass.FAILED_ALL, reward, False)
64
+ return VariantResult(v, VariantClass.DISCRIMINATED, reward, False)
65
+
66
+ def _reward_from(self, exit_code: int) -> float:
67
+ return self.task.grader.pass_reward if exit_code == 0 else 0.0
68
+
69
+ @staticmethod
70
+ def discrimination_score(results: list[VariantResult]) -> float:
71
+ """Fraction of *building* variants the grader correctly rejected (higher = stronger)."""
72
+ built = [r for r in results if r.klass is not VariantClass.DID_NOT_BUILD]
73
+ if not built:
74
+ return 1.0
75
+ caught = sum(1 for r in built if not r.is_exploit)
76
+ return caught / len(built)
@@ -0,0 +1,70 @@
1
+ """Deterministic variant GENERATION (see ARCHITECTURE §2a).
2
+
3
+ The gold patch has an explicit, ordered hunk structure. A revert variant = the gold patch
4
+ with exactly one hunk removed — a minimal, known-wrong solution (non-equivalent by
5
+ construction). Generation is a pure function of the patch bytes: no RNG, no search.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import hashlib
11
+ from dataclasses import dataclass
12
+
13
+ from rlgate.core.task import Patch
14
+
15
+ OPERATORS_VERSION = "operators@v1"
16
+
17
+
18
+ @dataclass(frozen=True)
19
+ class Variant:
20
+ id: str # content-addressed; identical inputs → identical id
21
+ operator: str # "revert" | "stub" | …
22
+ patch_text: str # the (gold-minus-hunk) patch to apply on the base tree
23
+ origin: str # human-readable location, e.g. "path/to/file.py#hunk2"
24
+
25
+
26
+ def _variant_id(gold_sha: str, origin: str, operator: str) -> str:
27
+ key = f"{gold_sha}|{origin}|{operator}|{OPERATORS_VERSION}"
28
+ return hashlib.sha256(key.encode()).hexdigest()[:16]
29
+
30
+
31
+ def revert_variants(gold: Patch) -> list[Variant]:
32
+ """One variant per hunk: the gold patch with that single hunk omitted.
33
+
34
+ Hunks are addressed by (file, hunk_index) from the parsed diff — stable and ordered.
35
+ """
36
+ ps = gold.patchset
37
+ variants: list[Variant] = []
38
+ for pf in ps:
39
+ for i, _hunk in enumerate(pf):
40
+ origin = f"{pf.path}#hunk{i}"
41
+ text = _patch_without(ps, pf.path, i)
42
+ if text is None:
43
+ continue # omitting this hunk left the file empty of changes; skip
44
+ variants.append(Variant(
45
+ id=_variant_id(gold.sha256, origin, "revert"),
46
+ operator="revert",
47
+ patch_text=text,
48
+ origin=origin,
49
+ ))
50
+ return variants
51
+
52
+
53
+ def _patch_without(ps, file_path: str, drop_index: int) -> str | None:
54
+ """Rebuild the unified diff omitting one hunk of one file. Pure string transform."""
55
+ out_files = []
56
+ for pf in ps:
57
+ kept = [h for j, h in enumerate(pf) if not (pf.path == file_path and j == drop_index)]
58
+ if not kept:
59
+ continue
60
+ header = _file_header(pf)
61
+ out_files.append(header + "".join(str(h) for h in kept))
62
+ text = "".join(out_files)
63
+ return text or None
64
+
65
+
66
+ def _file_header(pf) -> str:
67
+ # unidiff exposes the per-file header lines; reconstruct the minimal valid header.
68
+ src = pf.source_file or ("a/" + pf.path)
69
+ tgt = pf.target_file or ("b/" + pf.path)
70
+ return f"--- {src}\n+++ {tgt}\n"
@@ -0,0 +1,81 @@
1
+ Metadata-Version: 2.5
2
+ Name: rlgate
3
+ Version: 0.0.1
4
+ Summary: A deterministic, third-party auditor that finds reward-hacking vulnerabilities in RL/eval environments.
5
+ Project-URL: Homepage, https://github.com/deepak-foundryai/rlgate
6
+ Project-URL: Repository, https://github.com/deepak-foundryai/rlgate
7
+ Project-URL: Issues, https://github.com/deepak-foundryai/rlgate/issues
8
+ Author: rlgate contributors
9
+ License: MIT
10
+ License-File: LICENSE
11
+ Keywords: auditing,evals,grader,llm,reward,reward-hacking,rl
12
+ Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
17
+ Classifier: Topic :: Software Development :: Testing
18
+ Requires-Python: >=3.12
19
+ Requires-Dist: pyyaml>=6.0
20
+ Requires-Dist: unidiff>=0.7.5
21
+ Provides-Extra: dev
22
+ Requires-Dist: pyright>=1.1; extra == 'dev'
23
+ Requires-Dist: pytest>=8.0; extra == 'dev'
24
+ Requires-Dist: ruff>=0.6; extra == 'dev'
25
+ Description-Content-Type: text/markdown
26
+
27
+ # rlgate
28
+
29
+ **A deterministic, third-party auditor that finds reward-hacking vulnerabilities in RL/eval
30
+ environments** — before you train on them.
31
+
32
+ rlgate is not an environment framework and not a trajectory benchmark. It *consumes* any
33
+ environment (via a one-file declarative descriptor) and attacks its grader with deterministic
34
+ probes: it reverts hunks of the gold fix to find unguarded tests, plants forged artifacts to
35
+ find grading-channel forgery, walks the git object store to find leaked answers, and lints the
36
+ reward formula for design debt. Every finding is a **reproduced exploit or a line of code** —
37
+ never a threshold on a score.
38
+
39
+ See **[PLAN.md](PLAN.md)** for the methodology and **[ARCHITECTURE.md](ARCHITECTURE.md)** for
40
+ the design.
41
+
42
+ ## Quickstart
43
+
44
+ Run it as an isolated tool straight from the repo (no clone, no install):
45
+
46
+ ```bash
47
+ uvx --from git+https://github.com/deepak-foundryai/rlgate rlgate list-probes
48
+ uvx --from git+https://github.com/deepak-foundryai/rlgate rlgate audit path/to/env --gate
49
+ ```
50
+
51
+ Or for development:
52
+
53
+ ```bash
54
+ uv pip install -e ".[dev]" # or: pip install -e ".[dev]"
55
+ rlgate list-probes
56
+ rlgate audit path/to/env --gate # exits non-zero on FIX/DROP (CI mode)
57
+ ```
58
+
59
+ Once published to PyPI: `uvx rlgate audit ./env` or `uv tool install rlgate`.
60
+
61
+ Describe any environment once — no code:
62
+
63
+ ```yaml
64
+ # task.rlgate.yaml
65
+ workspace: ./repo
66
+ gold_patch: ./solution/fix.patch # optional → enables the revert probe
67
+ grader:
68
+ run: "bash run_tests.sh"
69
+ report: {format: junit, path: /out/results.xml}
70
+ ```
71
+
72
+ Capabilities are derived from which keys you fill in; probes that need a missing capability
73
+ skip silently. Nothing in a probe mentions a benchmark.
74
+
75
+ ## Status
76
+
77
+ Milestone 0: core contracts + local sandbox + baseline invariant + generic ingestion + three
78
+ probes (leakage, revert, forgery) + fixtures + CI. Contributions welcome — see
79
+ [CONTRIBUTING.md](CONTRIBUTING.md).
80
+
81
+ MIT licensed.