rlgate 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rlgate/__init__.py +7 -0
- rlgate/baseline/__init__.py +0 -0
- rlgate/baseline/invariant.py +30 -0
- rlgate/cli.py +75 -0
- rlgate/core/__init__.py +0 -0
- rlgate/core/context.py +47 -0
- rlgate/core/finding.py +40 -0
- rlgate/core/probe.py +44 -0
- rlgate/core/registry.py +46 -0
- rlgate/core/report.py +78 -0
- rlgate/core/task.py +102 -0
- rlgate/ingest/__init__.py +0 -0
- rlgate/ingest/presets/__init__.py +0 -0
- rlgate/ingest/schemas/__init__.py +0 -0
- rlgate/ingest/spec.py +75 -0
- rlgate/pipeline.py +64 -0
- rlgate/probes/__init__.py +0 -0
- rlgate/probes/forgery/__init__.py +0 -0
- rlgate/probes/forgery/world_readable_tests.py +44 -0
- rlgate/probes/grader_strength/__init__.py +0 -0
- rlgate/probes/grader_strength/revert_probe.py +44 -0
- rlgate/probes/hygiene/__init__.py +0 -0
- rlgate/probes/leakage/__init__.py +0 -0
- rlgate/probes/leakage/git_unreachable_objects.py +46 -0
- rlgate/probes/patterns/__init__.py +0 -0
- rlgate/probes/reward_shape/__init__.py +0 -0
- rlgate/probes/test_quality/__init__.py +0 -0
- rlgate/sandbox/__init__.py +0 -0
- rlgate/sandbox/_patch.py +30 -0
- rlgate/sandbox/base.py +63 -0
- rlgate/sandbox/docker.py +120 -0
- rlgate/sandbox/local.py +62 -0
- rlgate/sandbox/select.py +22 -0
- rlgate/variants/__init__.py +0 -0
- rlgate/variants/engine.py +76 -0
- rlgate/variants/revert.py +70 -0
- rlgate-0.0.1.dist-info/METADATA +81 -0
- rlgate-0.0.1.dist-info/RECORD +41 -0
- rlgate-0.0.1.dist-info/WHEEL +4 -0
- rlgate-0.0.1.dist-info/entry_points.txt +2 -0
- rlgate-0.0.1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""§3.4 — the flagship. Remove one hunk of the gold fix; if the grader still scores it pass,
|
|
2
|
+
that hunk is unguarded. Building revert survivors are exploits by construction."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from rlgate.core.context import ProbeContext
|
|
7
|
+
from rlgate.core.finding import Evidence, Finding, Severity, VerdictTier
|
|
8
|
+
from rlgate.core.probe import Probe, ProbeKind, ProbeMeta
|
|
9
|
+
from rlgate.core.registry import register
|
|
10
|
+
from rlgate.core.task import Capability, Task
|
|
11
|
+
from rlgate.variants.engine import VariantClass
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@register
|
|
15
|
+
class RevertProbe(Probe):
|
|
16
|
+
meta = ProbeMeta(
|
|
17
|
+
id="grader_strength.revert_probe",
|
|
18
|
+
family="grader_strength",
|
|
19
|
+
kind=ProbeKind.GENERIC,
|
|
20
|
+
severity=Severity.HIGH,
|
|
21
|
+
requires=frozenset({Capability.GOLD_PATCH}),
|
|
22
|
+
doc="A gold fix hunk can be removed and the grader still passes (unguarded line).",
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
def run(self, task: Task, ctx: ProbeContext) -> list[Finding]:
|
|
26
|
+
results = ctx.variants.run_revert()
|
|
27
|
+
findings: list[Finding] = []
|
|
28
|
+
for r in results:
|
|
29
|
+
if r.is_exploit and r.klass is VariantClass.DISCRIMINATED:
|
|
30
|
+
findings.append(Finding(
|
|
31
|
+
probe_id=self.meta.id,
|
|
32
|
+
title=f"unguarded fix hunk: {r.variant.origin}",
|
|
33
|
+
severity=Severity.HIGH,
|
|
34
|
+
verdict=VerdictTier.CONFIRMED,
|
|
35
|
+
location=r.variant.origin,
|
|
36
|
+
evidence=Evidence(
|
|
37
|
+
kind="reproduced_exploit",
|
|
38
|
+
detail=f"removing this hunk still scores pass (reward={r.reward}); "
|
|
39
|
+
f"variant {r.variant.id}",
|
|
40
|
+
replay=f"rlgate replay {task.id} --variant {r.variant.id}",
|
|
41
|
+
),
|
|
42
|
+
fix_hint="add a test that fails when this hunk is absent.",
|
|
43
|
+
))
|
|
44
|
+
return findings
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""§3.3 — the reference fix survives as an unreachable git object (the MiMo/SQLGlot leak).
|
|
2
|
+
|
|
3
|
+
We read the object store directly as the auditor, so an agent-side "git commands blocked"
|
|
4
|
+
guard is irrelevant — if the object is on disk, the answer is on disk."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from rlgate.core.context import ProbeContext
|
|
9
|
+
from rlgate.core.finding import Evidence, Finding, Severity, VerdictTier
|
|
10
|
+
from rlgate.core.probe import Probe, ProbeKind, ProbeMeta
|
|
11
|
+
from rlgate.core.registry import register
|
|
12
|
+
from rlgate.core.task import Capability, Task
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@register
|
|
16
|
+
class GitUnreachableObjects(Probe):
|
|
17
|
+
meta = ProbeMeta(
|
|
18
|
+
id="leakage.git_unreachable_objects",
|
|
19
|
+
family="leakage",
|
|
20
|
+
kind=ProbeKind.GENERIC,
|
|
21
|
+
severity=Severity.HIGH,
|
|
22
|
+
requires=frozenset({Capability.GIT_REPO}),
|
|
23
|
+
doc="Unreachable git objects (deleted-but-unpruned branches) may hold the fix.",
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
def run(self, task: Task, ctx: ProbeContext) -> list[Finding]:
|
|
27
|
+
with ctx.sandbox.overlay(None) as ovl:
|
|
28
|
+
res = ovl.exec("git fsck --unreachable --no-reflogs 2>/dev/null | head -n 50")
|
|
29
|
+
lines = [ln for ln in res.stdout.splitlines() if "unreachable commit" in ln]
|
|
30
|
+
if not lines:
|
|
31
|
+
return []
|
|
32
|
+
sample = lines[0].split()[-1][:12]
|
|
33
|
+
return [Finding(
|
|
34
|
+
probe_id=self.meta.id,
|
|
35
|
+
title=f"{len(lines)} unreachable git commit(s) on disk (e.g. {sample})",
|
|
36
|
+
severity=Severity.HIGH,
|
|
37
|
+
verdict=VerdictTier.CONFIRMED,
|
|
38
|
+
location=".git",
|
|
39
|
+
evidence=Evidence(
|
|
40
|
+
kind="reproduced_exploit",
|
|
41
|
+
detail="later history was deleted but not pruned; readable via the object store",
|
|
42
|
+
replay="git fsck --unreachable --no-reflogs",
|
|
43
|
+
),
|
|
44
|
+
fix_hint="git reflog expire --expire=now --all && git gc --prune=now; "
|
|
45
|
+
"or vendor a clean snapshot at the task base commit.",
|
|
46
|
+
)]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
rlgate/sandbox/_patch.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Shared, deterministic patch application (zero fuzz: all-or-nothing). Used by every
|
|
2
|
+
sandbox backend so variant generation behaves identically regardless of where it runs."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import subprocess
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class PatchConflict(RuntimeError):
|
|
11
|
+
"""A variant patch did not apply cleanly — bucketed as could-not-generate."""
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def apply_zero_fuzz(root: Path, patch_text: str) -> None:
|
|
15
|
+
"""Apply a unified diff under `root` with no fuzz. Tries `git apply` then `patch`.
|
|
16
|
+
|
|
17
|
+
Zero fuzz makes the result deterministic: a hunk applies at its exact location or the
|
|
18
|
+
whole patch is rejected (raising PatchConflict)."""
|
|
19
|
+
proc = subprocess.run(
|
|
20
|
+
["git", "apply", "--unsafe-paths", "-C0", "-p1", "-"],
|
|
21
|
+
cwd=root, input=patch_text, text=True, capture_output=True, check=False,
|
|
22
|
+
)
|
|
23
|
+
if proc.returncode == 0:
|
|
24
|
+
return
|
|
25
|
+
proc = subprocess.run(
|
|
26
|
+
["patch", "-p1", "--no-backup-if-mismatch", "-F0", "-s"],
|
|
27
|
+
cwd=root, input=patch_text, text=True, capture_output=True, check=False,
|
|
28
|
+
)
|
|
29
|
+
if proc.returncode != 0:
|
|
30
|
+
raise PatchConflict(proc.stderr or proc.stdout)
|
rlgate/sandbox/base.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Execution substrate. Backends (local/docker/daytona) implement this ABC.
|
|
2
|
+
|
|
3
|
+
The key operations the probes need: build once, snapshot, cheap per-variant overlays, exec,
|
|
4
|
+
inspect permissions, and kill stray candidate processes (forgery/race probes)."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from abc import ABC, abstractmethod
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Self
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class ExecResult:
|
|
16
|
+
exit_code: int
|
|
17
|
+
stdout: str
|
|
18
|
+
stderr: str
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class Perms:
|
|
23
|
+
mode: int # e.g. 0o755
|
|
24
|
+
exists: bool = True
|
|
25
|
+
|
|
26
|
+
@property
|
|
27
|
+
def world_readable(self) -> bool:
|
|
28
|
+
return self.exists and bool(self.mode & 0o004)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class Overlay(ABC):
|
|
32
|
+
"""A cheap, disposable copy of the workspace with a patch applied."""
|
|
33
|
+
|
|
34
|
+
root: Path
|
|
35
|
+
|
|
36
|
+
@abstractmethod
|
|
37
|
+
def exec(self, cmd: str, timeout: int = 900) -> ExecResult: ...
|
|
38
|
+
|
|
39
|
+
@abstractmethod
|
|
40
|
+
def read_perms(self, path: str) -> Perms: ...
|
|
41
|
+
|
|
42
|
+
@abstractmethod
|
|
43
|
+
def close(self) -> None: ...
|
|
44
|
+
|
|
45
|
+
def __enter__(self) -> Self:
|
|
46
|
+
return self
|
|
47
|
+
|
|
48
|
+
def __exit__(self, *exc) -> None:
|
|
49
|
+
self.close()
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class Sandbox(ABC):
|
|
53
|
+
@abstractmethod
|
|
54
|
+
def build(self) -> None:
|
|
55
|
+
"""Build the task image / prepare the pristine workspace once."""
|
|
56
|
+
|
|
57
|
+
@abstractmethod
|
|
58
|
+
def overlay(self, patch_text: str | None = None) -> Overlay:
|
|
59
|
+
"""A fresh overlay on the snapshot, optionally with a patch applied (zero fuzz)."""
|
|
60
|
+
|
|
61
|
+
@abstractmethod
|
|
62
|
+
def pkill_user(self, user: str) -> None:
|
|
63
|
+
"""Kill detached candidate processes before scoring (race-probe defense)."""
|
rlgate/sandbox/docker.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""Docker sandbox: real isolation for running untrusted variants, container graders, and the
|
|
2
|
+
live forgery/race probes. Same ABC as the local backend.
|
|
3
|
+
|
|
4
|
+
Model: build the image once; keep a pristine host copy of the workspace as the snapshot.
|
|
5
|
+
Each overlay is a fresh container plus a host copy with the variant patch applied (zero fuzz,
|
|
6
|
+
shared logic) and copied in — so variant generation is identical to the local backend and the
|
|
7
|
+
image needs no git/patch tooling.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import shutil
|
|
13
|
+
import subprocess
|
|
14
|
+
import tempfile
|
|
15
|
+
import uuid
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from rlgate.sandbox._patch import apply_zero_fuzz
|
|
19
|
+
from rlgate.sandbox.base import ExecResult, Overlay, Perms, Sandbox
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class DockerUnavailable(RuntimeError):
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def available() -> bool:
|
|
27
|
+
try:
|
|
28
|
+
return subprocess.run(["docker", "info"], capture_output=True, check=False).returncode == 0
|
|
29
|
+
except FileNotFoundError:
|
|
30
|
+
return False
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _docker(*args: str, **kw) -> subprocess.CompletedProcess:
|
|
34
|
+
return subprocess.run(["docker", *args], capture_output=True, text=True, check=False, **kw)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class DockerOverlay(Overlay):
|
|
38
|
+
def __init__(self, container_id: str, host_copy: Path, sandbox: DockerSandbox) -> None:
|
|
39
|
+
self.container_id = container_id
|
|
40
|
+
self.root = host_copy # host-side view (for parity); container path is /work
|
|
41
|
+
self._sandbox = sandbox
|
|
42
|
+
|
|
43
|
+
def exec(self, cmd: str, timeout: int = 900) -> ExecResult:
|
|
44
|
+
p = _docker("exec", "-w", "/work", self.container_id, "sh", "-c", cmd, timeout=timeout)
|
|
45
|
+
return ExecResult(p.returncode, p.stdout, p.stderr)
|
|
46
|
+
|
|
47
|
+
def read_perms(self, path: str) -> Perms:
|
|
48
|
+
p = _docker("exec", self.container_id, "stat", "-c", "%a", f"/work/{path.lstrip('/')}")
|
|
49
|
+
if p.returncode != 0:
|
|
50
|
+
return Perms(mode=0, exists=False)
|
|
51
|
+
return Perms(mode=int(p.stdout.strip(), 8))
|
|
52
|
+
|
|
53
|
+
def close(self) -> None:
|
|
54
|
+
_docker("rm", "-f", self.container_id)
|
|
55
|
+
self._sandbox._live.discard(self.container_id)
|
|
56
|
+
shutil.rmtree(self.root, ignore_errors=True)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class DockerSandbox(Sandbox):
|
|
60
|
+
def __init__(self, workspace: Path, image_ref: str | None = None,
|
|
61
|
+
dockerfile: Path | None = None, task_id: str = "task") -> None:
|
|
62
|
+
self.workspace = Path(workspace)
|
|
63
|
+
self.image_ref = image_ref
|
|
64
|
+
self.dockerfile = Path(dockerfile) if dockerfile else None
|
|
65
|
+
self.task_id = task_id
|
|
66
|
+
self._tag: str | None = None
|
|
67
|
+
self._snapshot: Path | None = None
|
|
68
|
+
self._live: set[str] = set()
|
|
69
|
+
|
|
70
|
+
@classmethod
|
|
71
|
+
def from_task(cls, task) -> DockerSandbox:
|
|
72
|
+
"""grader.image is either a Dockerfile path (build it) or an existing image ref."""
|
|
73
|
+
img = task.grader.image
|
|
74
|
+
p = Path(img) if img else None
|
|
75
|
+
if p and p.exists() and p.is_file():
|
|
76
|
+
return cls(task.repo_dir, dockerfile=p, task_id=task.id)
|
|
77
|
+
return cls(task.repo_dir, image_ref=img, task_id=task.id)
|
|
78
|
+
|
|
79
|
+
def build(self) -> None:
|
|
80
|
+
if not available():
|
|
81
|
+
raise DockerUnavailable("docker is not available on this host")
|
|
82
|
+
if self.dockerfile:
|
|
83
|
+
self._tag = f"rlgate-{self.task_id}-{uuid.uuid4().hex[:8]}"
|
|
84
|
+
r = _docker("build", "-t", self._tag, "-f", str(self.dockerfile),
|
|
85
|
+
str(self.dockerfile.parent))
|
|
86
|
+
if r.returncode != 0:
|
|
87
|
+
raise DockerUnavailable(f"image build failed:\n{r.stderr}")
|
|
88
|
+
else:
|
|
89
|
+
self._tag = self.image_ref
|
|
90
|
+
# pristine host copy of the workspace (the snapshot we cp into each container)
|
|
91
|
+
snap = Path(tempfile.mkdtemp(prefix="rlgate-dsnap-"))
|
|
92
|
+
shutil.rmtree(snap)
|
|
93
|
+
shutil.copytree(self.workspace, snap, symlinks=True)
|
|
94
|
+
self._snapshot = snap
|
|
95
|
+
|
|
96
|
+
def overlay(self, patch_text: str | None = None) -> DockerOverlay:
|
|
97
|
+
assert self._snapshot is not None and self._tag is not None, "call build() first"
|
|
98
|
+
host_copy = Path(tempfile.mkdtemp(prefix="rlgate-dovl-"))
|
|
99
|
+
shutil.rmtree(host_copy)
|
|
100
|
+
shutil.copytree(self._snapshot, host_copy, symlinks=True)
|
|
101
|
+
if patch_text:
|
|
102
|
+
apply_zero_fuzz(host_copy, patch_text)
|
|
103
|
+
|
|
104
|
+
run = _docker("run", "-d", "--network=none", "-w", "/work",
|
|
105
|
+
self._tag, "sleep", "infinity")
|
|
106
|
+
if run.returncode != 0:
|
|
107
|
+
raise DockerUnavailable(f"container start failed:\n{run.stderr}")
|
|
108
|
+
cid = run.stdout.strip()
|
|
109
|
+
self._live.add(cid)
|
|
110
|
+
_docker("exec", cid, "mkdir", "-p", "/work")
|
|
111
|
+
cp = _docker("cp", f"{host_copy}/.", f"{cid}:/work")
|
|
112
|
+
if cp.returncode != 0:
|
|
113
|
+
_docker("rm", "-f", cid)
|
|
114
|
+
raise DockerUnavailable(f"workspace copy failed:\n{cp.stderr}")
|
|
115
|
+
return DockerOverlay(cid, host_copy, self)
|
|
116
|
+
|
|
117
|
+
def pkill_user(self, user: str) -> None:
|
|
118
|
+
"""Kill detached candidate processes in every live container before scoring."""
|
|
119
|
+
for cid in list(self._live):
|
|
120
|
+
_docker("exec", cid, "pkill", "-u", user)
|
rlgate/sandbox/local.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Local sandbox: copy the workspace to a temp dir, apply a patch with zero fuzz, run the
|
|
2
|
+
grader via subprocess. No isolation guarantees — for dev and CI fixtures only. The Docker
|
|
3
|
+
backend implements the same ABC for real isolation (and the race/perms probes)."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import os
|
|
8
|
+
import shutil
|
|
9
|
+
import subprocess
|
|
10
|
+
import tempfile
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from rlgate.sandbox._patch import PatchConflict, apply_zero_fuzz
|
|
14
|
+
from rlgate.sandbox.base import ExecResult, Overlay, Perms, Sandbox
|
|
15
|
+
|
|
16
|
+
__all__ = ["LocalOverlay", "LocalSandbox", "PatchConflict"]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class LocalOverlay(Overlay):
|
|
20
|
+
def __init__(self, root: Path) -> None:
|
|
21
|
+
self.root = root
|
|
22
|
+
|
|
23
|
+
def exec(self, cmd: str, timeout: int = 900) -> ExecResult:
|
|
24
|
+
p = subprocess.run(
|
|
25
|
+
cmd, shell=True, cwd=self.root, capture_output=True, text=True,
|
|
26
|
+
timeout=timeout, check=False,
|
|
27
|
+
env={**os.environ, "RLGATE_SANDBOX": "local"},
|
|
28
|
+
)
|
|
29
|
+
return ExecResult(p.returncode, p.stdout, p.stderr)
|
|
30
|
+
|
|
31
|
+
def read_perms(self, path: str) -> Perms:
|
|
32
|
+
target = self.root / path.lstrip("/")
|
|
33
|
+
if not target.exists():
|
|
34
|
+
return Perms(mode=0, exists=False)
|
|
35
|
+
return Perms(mode=os.stat(target).st_mode & 0o777)
|
|
36
|
+
|
|
37
|
+
def close(self) -> None:
|
|
38
|
+
shutil.rmtree(self.root, ignore_errors=True)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class LocalSandbox(Sandbox):
|
|
42
|
+
def __init__(self, workspace: Path) -> None:
|
|
43
|
+
self.workspace = Path(workspace)
|
|
44
|
+
self._snapshot: Path | None = None
|
|
45
|
+
|
|
46
|
+
def build(self) -> None:
|
|
47
|
+
snap = Path(tempfile.mkdtemp(prefix="rlgate-snap-"))
|
|
48
|
+
shutil.rmtree(snap)
|
|
49
|
+
shutil.copytree(self.workspace, snap, symlinks=True)
|
|
50
|
+
self._snapshot = snap
|
|
51
|
+
|
|
52
|
+
def overlay(self, patch_text: str | None = None) -> LocalOverlay:
|
|
53
|
+
assert self._snapshot is not None, "call build() first"
|
|
54
|
+
root = Path(tempfile.mkdtemp(prefix="rlgate-ovl-"))
|
|
55
|
+
shutil.rmtree(root)
|
|
56
|
+
shutil.copytree(self._snapshot, root, symlinks=True)
|
|
57
|
+
if patch_text:
|
|
58
|
+
apply_zero_fuzz(root, patch_text)
|
|
59
|
+
return LocalOverlay(root)
|
|
60
|
+
|
|
61
|
+
def pkill_user(self, user: str) -> None:
|
|
62
|
+
subprocess.run(["pkill", "-u", user], capture_output=True, check=False)
|
rlgate/sandbox/select.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Backend selection. `auto` uses Docker when the task declares a container image and Docker
|
|
2
|
+
is available, otherwise the local sandbox. Explicit `local`/`docker` override the choice."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from rlgate.core.task import Capability, Task
|
|
7
|
+
from rlgate.sandbox.base import Sandbox
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def build_sandbox(task: Task, prefer: str = "auto") -> Sandbox:
|
|
11
|
+
from rlgate.sandbox.docker import DockerSandbox, available
|
|
12
|
+
from rlgate.sandbox.local import LocalSandbox
|
|
13
|
+
|
|
14
|
+
if prefer == "local":
|
|
15
|
+
return LocalSandbox(task.repo_dir)
|
|
16
|
+
if prefer == "docker":
|
|
17
|
+
return DockerSandbox.from_task(task)
|
|
18
|
+
|
|
19
|
+
# auto
|
|
20
|
+
if Capability.CONTAINER in task.capabilities and available():
|
|
21
|
+
return DockerSandbox.from_task(task)
|
|
22
|
+
return LocalSandbox(task.repo_dir)
|
|
File without changes
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Generate variants, build-gate them, run on the per-test subset, classify, and score.
|
|
2
|
+
|
|
3
|
+
Classification (ARCHITECTURE / PLAN §3.4):
|
|
4
|
+
- did_not_build : patch didn't apply, or the build/compile failed → NOT coverage signal
|
|
5
|
+
- failed_all : every subset test failed (refactor/rename hunk) → NOT coverage signal
|
|
6
|
+
- discriminated : built and some tests passed, some failed → the real signal
|
|
7
|
+
A *building* revert/stub survivor that still scores pass is a reproduced exploit.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from enum import Enum
|
|
14
|
+
|
|
15
|
+
from rlgate.core.context import Baseline
|
|
16
|
+
from rlgate.core.task import Task
|
|
17
|
+
from rlgate.sandbox.base import Sandbox
|
|
18
|
+
from rlgate.sandbox.local import PatchConflict
|
|
19
|
+
from rlgate.variants.revert import Variant, revert_variants
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class VariantClass(Enum):
|
|
23
|
+
DID_NOT_BUILD = "did_not_build"
|
|
24
|
+
FAILED_ALL = "failed_all"
|
|
25
|
+
DISCRIMINATED = "discriminated"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class VariantResult:
|
|
30
|
+
variant: Variant
|
|
31
|
+
klass: VariantClass
|
|
32
|
+
reward: float # grader reward on this variant (only meaningful if built)
|
|
33
|
+
is_exploit: bool # built + scored pass despite being wrong-by-construction
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class VariantEngine:
|
|
37
|
+
def __init__(self, task: Task, sandbox: Sandbox, baseline: Baseline) -> None:
|
|
38
|
+
self.task = task
|
|
39
|
+
self.sandbox = sandbox
|
|
40
|
+
self.baseline = baseline
|
|
41
|
+
|
|
42
|
+
def run_revert(self) -> list[VariantResult]:
|
|
43
|
+
assert self.task.gold_patch is not None
|
|
44
|
+
results: list[VariantResult] = []
|
|
45
|
+
for v in revert_variants(self.task.gold_patch):
|
|
46
|
+
results.append(self._score(v))
|
|
47
|
+
return results
|
|
48
|
+
|
|
49
|
+
def _score(self, v: Variant) -> VariantResult:
|
|
50
|
+
try:
|
|
51
|
+
overlay = self.sandbox.overlay(v.patch_text)
|
|
52
|
+
except PatchConflict:
|
|
53
|
+
return VariantResult(v, VariantClass.DID_NOT_BUILD, 0.0, False)
|
|
54
|
+
with overlay:
|
|
55
|
+
run = overlay.exec(self.task.grader.run)
|
|
56
|
+
if run.exit_code == 127 or "No such file" in run.stderr:
|
|
57
|
+
return VariantResult(v, VariantClass.DID_NOT_BUILD, 0.0, False)
|
|
58
|
+
reward = self._reward_from(run.exit_code)
|
|
59
|
+
# pass despite a missing fix hunk → exploit; full fail → good coverage.
|
|
60
|
+
if reward >= self.task.grader.pass_reward:
|
|
61
|
+
return VariantResult(v, VariantClass.DISCRIMINATED, reward, True)
|
|
62
|
+
if reward <= 0.0:
|
|
63
|
+
return VariantResult(v, VariantClass.FAILED_ALL, reward, False)
|
|
64
|
+
return VariantResult(v, VariantClass.DISCRIMINATED, reward, False)
|
|
65
|
+
|
|
66
|
+
def _reward_from(self, exit_code: int) -> float:
|
|
67
|
+
return self.task.grader.pass_reward if exit_code == 0 else 0.0
|
|
68
|
+
|
|
69
|
+
@staticmethod
|
|
70
|
+
def discrimination_score(results: list[VariantResult]) -> float:
|
|
71
|
+
"""Fraction of *building* variants the grader correctly rejected (higher = stronger)."""
|
|
72
|
+
built = [r for r in results if r.klass is not VariantClass.DID_NOT_BUILD]
|
|
73
|
+
if not built:
|
|
74
|
+
return 1.0
|
|
75
|
+
caught = sum(1 for r in built if not r.is_exploit)
|
|
76
|
+
return caught / len(built)
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Deterministic variant GENERATION (see ARCHITECTURE §2a).
|
|
2
|
+
|
|
3
|
+
The gold patch has an explicit, ordered hunk structure. A revert variant = the gold patch
|
|
4
|
+
with exactly one hunk removed — a minimal, known-wrong solution (non-equivalent by
|
|
5
|
+
construction). Generation is a pure function of the patch bytes: no RNG, no search.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
|
|
13
|
+
from rlgate.core.task import Patch
|
|
14
|
+
|
|
15
|
+
OPERATORS_VERSION = "operators@v1"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class Variant:
|
|
20
|
+
id: str # content-addressed; identical inputs → identical id
|
|
21
|
+
operator: str # "revert" | "stub" | …
|
|
22
|
+
patch_text: str # the (gold-minus-hunk) patch to apply on the base tree
|
|
23
|
+
origin: str # human-readable location, e.g. "path/to/file.py#hunk2"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _variant_id(gold_sha: str, origin: str, operator: str) -> str:
|
|
27
|
+
key = f"{gold_sha}|{origin}|{operator}|{OPERATORS_VERSION}"
|
|
28
|
+
return hashlib.sha256(key.encode()).hexdigest()[:16]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def revert_variants(gold: Patch) -> list[Variant]:
|
|
32
|
+
"""One variant per hunk: the gold patch with that single hunk omitted.
|
|
33
|
+
|
|
34
|
+
Hunks are addressed by (file, hunk_index) from the parsed diff — stable and ordered.
|
|
35
|
+
"""
|
|
36
|
+
ps = gold.patchset
|
|
37
|
+
variants: list[Variant] = []
|
|
38
|
+
for pf in ps:
|
|
39
|
+
for i, _hunk in enumerate(pf):
|
|
40
|
+
origin = f"{pf.path}#hunk{i}"
|
|
41
|
+
text = _patch_without(ps, pf.path, i)
|
|
42
|
+
if text is None:
|
|
43
|
+
continue # omitting this hunk left the file empty of changes; skip
|
|
44
|
+
variants.append(Variant(
|
|
45
|
+
id=_variant_id(gold.sha256, origin, "revert"),
|
|
46
|
+
operator="revert",
|
|
47
|
+
patch_text=text,
|
|
48
|
+
origin=origin,
|
|
49
|
+
))
|
|
50
|
+
return variants
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _patch_without(ps, file_path: str, drop_index: int) -> str | None:
|
|
54
|
+
"""Rebuild the unified diff omitting one hunk of one file. Pure string transform."""
|
|
55
|
+
out_files = []
|
|
56
|
+
for pf in ps:
|
|
57
|
+
kept = [h for j, h in enumerate(pf) if not (pf.path == file_path and j == drop_index)]
|
|
58
|
+
if not kept:
|
|
59
|
+
continue
|
|
60
|
+
header = _file_header(pf)
|
|
61
|
+
out_files.append(header + "".join(str(h) for h in kept))
|
|
62
|
+
text = "".join(out_files)
|
|
63
|
+
return text or None
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _file_header(pf) -> str:
|
|
67
|
+
# unidiff exposes the per-file header lines; reconstruct the minimal valid header.
|
|
68
|
+
src = pf.source_file or ("a/" + pf.path)
|
|
69
|
+
tgt = pf.target_file or ("b/" + pf.path)
|
|
70
|
+
return f"--- {src}\n+++ {tgt}\n"
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: rlgate
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: A deterministic, third-party auditor that finds reward-hacking vulnerabilities in RL/eval environments.
|
|
5
|
+
Project-URL: Homepage, https://github.com/deepak-foundryai/rlgate
|
|
6
|
+
Project-URL: Repository, https://github.com/deepak-foundryai/rlgate
|
|
7
|
+
Project-URL: Issues, https://github.com/deepak-foundryai/rlgate/issues
|
|
8
|
+
Author: rlgate contributors
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: auditing,evals,grader,llm,reward,reward-hacking,rl
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
+
Classifier: Topic :: Software Development :: Testing
|
|
18
|
+
Requires-Python: >=3.12
|
|
19
|
+
Requires-Dist: pyyaml>=6.0
|
|
20
|
+
Requires-Dist: unidiff>=0.7.5
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: pyright>=1.1; extra == 'dev'
|
|
23
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
24
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# rlgate
|
|
28
|
+
|
|
29
|
+
**A deterministic, third-party auditor that finds reward-hacking vulnerabilities in RL/eval
|
|
30
|
+
environments** — before you train on them.
|
|
31
|
+
|
|
32
|
+
rlgate is not an environment framework and not a trajectory benchmark. It *consumes* any
|
|
33
|
+
environment (via a one-file declarative descriptor) and attacks its grader with deterministic
|
|
34
|
+
probes: it reverts hunks of the gold fix to find unguarded tests, plants forged artifacts to
|
|
35
|
+
find grading-channel forgery, walks the git object store to find leaked answers, and lints the
|
|
36
|
+
reward formula for design debt. Every finding is a **reproduced exploit or a line of code** —
|
|
37
|
+
never a threshold on a score.
|
|
38
|
+
|
|
39
|
+
See **[PLAN.md](PLAN.md)** for the methodology and **[ARCHITECTURE.md](ARCHITECTURE.md)** for
|
|
40
|
+
the design.
|
|
41
|
+
|
|
42
|
+
## Quickstart
|
|
43
|
+
|
|
44
|
+
Run it as an isolated tool straight from the repo (no clone, no install):
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
uvx --from git+https://github.com/deepak-foundryai/rlgate rlgate list-probes
|
|
48
|
+
uvx --from git+https://github.com/deepak-foundryai/rlgate rlgate audit path/to/env --gate
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Or for development:
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
uv pip install -e ".[dev]" # or: pip install -e ".[dev]"
|
|
55
|
+
rlgate list-probes
|
|
56
|
+
rlgate audit path/to/env --gate # exits non-zero on FIX/DROP (CI mode)
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Once published to PyPI: `uvx rlgate audit ./env` or `uv tool install rlgate`.
|
|
60
|
+
|
|
61
|
+
Describe any environment once — no code:
|
|
62
|
+
|
|
63
|
+
```yaml
|
|
64
|
+
# task.rlgate.yaml
|
|
65
|
+
workspace: ./repo
|
|
66
|
+
gold_patch: ./solution/fix.patch # optional → enables the revert probe
|
|
67
|
+
grader:
|
|
68
|
+
run: "bash run_tests.sh"
|
|
69
|
+
report: {format: junit, path: /out/results.xml}
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Capabilities are derived from which keys you fill in; probes that need a missing capability
|
|
73
|
+
skip silently. Nothing in a probe mentions a benchmark.
|
|
74
|
+
|
|
75
|
+
## Status
|
|
76
|
+
|
|
77
|
+
Milestone 0: core contracts + local sandbox + baseline invariant + generic ingestion + three
|
|
78
|
+
probes (leakage, revert, forgery) + fixtures + CI. Contributions welcome — see
|
|
79
|
+
[CONTRIBUTING.md](CONTRIBUTING.md).
|
|
80
|
+
|
|
81
|
+
MIT licensed.
|