judgetap 0.2.1.dev41001__tar.gz → 0.2.1.dev42001__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/PKG-INFO +1 -1
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/pyproject.toml +1 -1
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/__init__.py +1 -1
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/engines/julia.py +29 -10
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_engine_julia.py +30 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/.github/workflows/ci.yml +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/.github/workflows/demo.yml +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/.github/workflows/release.yml +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/.gitignore +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/.python-version +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/.release-please-manifest.json +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/CHANGELOG.md +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/CONTRIBUTING.md +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/LICENSE +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/README.md +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/docs/SPEC.md +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/docs/demo.tape +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/release-please-config.json +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/_compat.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/api.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/cascade.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/cli.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/dashboard/__init__.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/dashboard/data.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/dashboard/page.html +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/dashboard/server.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/decision_log.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/engine.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/engines/__init__.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/engines/agentjev.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/engines/jev.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/engines/laya.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/engines/llm.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/errors.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/evaluate.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/guard/__init__.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/guard/core.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/guard/hook.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/guard/install.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/guard/loop.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/guard/rules.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/guard/stop.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/py.typed +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/secrets.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/testing.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/src/judgetap/types.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_api.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_call_accounting.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_calls.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_cascade.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_dashboard.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_decision_log.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_engine_jev.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_engine_llm.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_engine_local.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_evaluate.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_guard_agents.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_guard_core.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_guard_hook.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_guard_loop.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_guard_polish_76.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_guard_rules.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_guard_stop.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_guard_trust.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_llm_logprobs.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_questions.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_robustness_68.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/tests/test_secrets.py +0 -0
- {judgetap-0.2.1.dev41001 → judgetap-0.2.1.dev42001}/uv.lock +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: judgetap
|
|
3
|
-
Version: 0.2.1.
|
|
3
|
+
Version: 0.2.1.dev42001
|
|
4
4
|
Summary: Fast typed decisions (choice, score, yes/no) across Jev-style engines, plus a guard for coding agents. Early development.
|
|
5
5
|
Project-URL: Homepage, https://github.com/mergesafe-ai/judgetap
|
|
6
6
|
Author-email: Omer Bar-Ness <omer@zsquared.io>
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "judgetap"
|
|
3
|
-
version = "0.2.1.
|
|
3
|
+
version = "0.2.1.dev42001"
|
|
4
4
|
description = "Fast typed decisions (choice, score, yes/no) across Jev-style engines, plus a guard for coding agents. Early development."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "Apache-2.0"
|
|
@@ -6,16 +6,18 @@ the request is built with the Jev adapter's helpers. Its runtime ships in the
|
|
|
6
6
|
model repo rather than on PyPI:
|
|
7
7
|
|
|
8
8
|
python -c "from huggingface_hub import snapshot_download; \\
|
|
9
|
-
snapshot_download('SupersonicLabs/Julia-1',
|
|
10
|
-
|
|
9
|
+
import os; snapshot_download('SupersonicLabs/Julia-1', \\
|
|
10
|
+
local_dir=os.path.expanduser('~/.judgetap/models/Julia-1'))"
|
|
11
|
+
pip install -e ~/.judgetap/models/Julia-1
|
|
11
12
|
|
|
12
13
|
Spec `julia` loads the checkpoint at $JUDGETAP_JULIA_PATH (default
|
|
13
14
|
`~/.judgetap/models/Julia-1`); `julia:<path>` names it. A relative path is
|
|
14
|
-
taken from ~/.judgetap/models, never the working directory
|
|
15
|
-
|
|
16
|
-
judges them. $JUDGETAP_JULIA_DEVICE picks the device (default `cpu`).
|
|
15
|
+
taken from ~/.judgetap/models, never the working directory, and may not
|
|
16
|
+
escape it (`../x` is rejected): the guard runs inside untrusted repos, which
|
|
17
|
+
must not be able to supply the model that judges them. $JUDGETAP_JULIA_DEVICE picks the device (default `cpu`).
|
|
17
18
|
|
|
18
|
-
Loaded models are cached per process (path, device),
|
|
19
|
+
Loaded models are cached per process (path, device), at most
|
|
20
|
+
MAX_CACHED_MODELS of them (least recently used evicted), so a long-running
|
|
19
21
|
program loads each once. The guard hook is a new process per action, so an
|
|
20
22
|
in-process model still loads on every guarded action there: for the guard,
|
|
21
23
|
prefer a server engine (AgentJev, or a TypeSafe-compatible URL).
|
|
@@ -36,7 +38,8 @@ from judgetap.errors import JudgetapError
|
|
|
36
38
|
from judgetap.types import NO, YES, Question
|
|
37
39
|
|
|
38
40
|
DEFAULT_PATH = "Julia-1"
|
|
39
|
-
|
|
41
|
+
MAX_CACHED_MODELS = 2
|
|
42
|
+
_models: dict[tuple[str, str], Any] = {} # insertion order = LRU order
|
|
40
43
|
_models_lock = threading.Lock()
|
|
41
44
|
|
|
42
45
|
|
|
@@ -47,9 +50,21 @@ def _models_dir() -> Path:
|
|
|
47
50
|
|
|
48
51
|
|
|
49
52
|
def resolve_path(path: str) -> str:
|
|
50
|
-
"""Absolute paths as given; anything else under ~/.judgetap/models.
|
|
53
|
+
"""Absolute paths as given; anything else under ~/.judgetap/models.
|
|
54
|
+
|
|
55
|
+
A relative path that resolves outside the models directory (``../x``,
|
|
56
|
+
or a symlink out of it) is rejected.
|
|
57
|
+
"""
|
|
51
58
|
p = Path(path).expanduser()
|
|
52
|
-
|
|
59
|
+
if p.is_absolute():
|
|
60
|
+
return str(p)
|
|
61
|
+
base = _models_dir()
|
|
62
|
+
if not (base / p).resolve().is_relative_to(base.resolve()):
|
|
63
|
+
raise ValueError(
|
|
64
|
+
f"julia model path {path!r} escapes {base}; "
|
|
65
|
+
"use an absolute path for a model outside it"
|
|
66
|
+
)
|
|
67
|
+
return str(base / p)
|
|
53
68
|
|
|
54
69
|
|
|
55
70
|
# Julia's native limits: 2-20 options per question.
|
|
@@ -90,7 +105,9 @@ class JuliaEngine:
|
|
|
90
105
|
key = (self.path, self.device)
|
|
91
106
|
# One load per process and model, even with concurrent first calls.
|
|
92
107
|
with _models_lock:
|
|
93
|
-
if key
|
|
108
|
+
if key in _models:
|
|
109
|
+
_models[key] = _models.pop(key) # mark most recently used
|
|
110
|
+
else:
|
|
94
111
|
try:
|
|
95
112
|
from julia import load_model
|
|
96
113
|
except ImportError as err:
|
|
@@ -106,6 +123,8 @@ class JuliaEngine:
|
|
|
106
123
|
max_length=8192,
|
|
107
124
|
head_length=512,
|
|
108
125
|
)
|
|
126
|
+
while len(_models) > MAX_CACHED_MODELS:
|
|
127
|
+
del _models[next(iter(_models))]
|
|
109
128
|
self._runtime = _models[key]
|
|
110
129
|
return self._runtime
|
|
111
130
|
|
|
@@ -167,3 +167,33 @@ def test_model_loads_once_per_process_even_concurrently(tmp_path, monkeypatch):
|
|
|
167
167
|
t.join()
|
|
168
168
|
assert len(loads) == 1
|
|
169
169
|
assert len({id(e._runtime) for e in engines}) == 1
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
@pytest.mark.parametrize("spec", ["../Julia-1", "a/../../x", "../../etc"])
|
|
173
|
+
def test_relative_paths_escaping_models_dir_are_rejected(spec, tmp_path, monkeypatch):
|
|
174
|
+
monkeypatch.setenv("JUDGETAP_HOME", str(tmp_path / "home"))
|
|
175
|
+
with pytest.raises(ValueError, match="escapes"):
|
|
176
|
+
JuliaEngine(spec)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def test_relative_path_inside_models_dir_with_dotdot_is_allowed(tmp_path, monkeypatch):
|
|
180
|
+
monkeypatch.setenv("JUDGETAP_HOME", str(tmp_path / "home"))
|
|
181
|
+
path = JuliaEngine("a/../Julia-1").path
|
|
182
|
+
assert path == str(tmp_path / "home" / "models" / "a" / ".." / "Julia-1")
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def test_model_cache_keeps_two_most_recently_used(fake_julia, tmp_path, monkeypatch):
|
|
186
|
+
from judgetap.engines import julia as jmod
|
|
187
|
+
|
|
188
|
+
monkeypatch.setenv("JUDGETAP_HOME", str(tmp_path))
|
|
189
|
+
monkeypatch.setattr(jmod, "_models", {})
|
|
190
|
+
jmod.JuliaEngine("A")._get_runtime()
|
|
191
|
+
jmod.JuliaEngine("B")._get_runtime()
|
|
192
|
+
jmod.JuliaEngine("A")._get_runtime() # A is now most recent
|
|
193
|
+
jmod.JuliaEngine("C")._get_runtime() # evicts B
|
|
194
|
+
models = tmp_path / "models"
|
|
195
|
+
assert [k[0] for k in jmod._models] == [str(models / "A"), str(models / "C")]
|
|
196
|
+
assert len(fake_julia.loads) == 3
|
|
197
|
+
jmod.JuliaEngine("B")._get_runtime() # reloads B, evicts A
|
|
198
|
+
assert len(fake_julia.loads) == 4
|
|
199
|
+
assert [k[0] for k in jmod._models] == [str(models / "C"), str(models / "B")]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|