judgetap 0.0.2.dev27__tar.gz → 0.0.2.dev28__tar.gz

{judgetap-0.0.2.dev27 → judgetap-0.0.2.dev28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: judgetap
-Version: 0.0.2.dev27
+Version: 0.0.2.dev28
 Summary: Fast typed decisions (choice, score, yes/no) across Jev-style engines, plus a guard for coding agents. Early development.
 Project-URL: Homepage, https://github.com/mergesafe-ai/judgetap
 Author-email: Omer Bar-Ness <omer@zsquared.io>

{judgetap-0.0.2.dev27 → judgetap-0.0.2.dev28}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "judgetap"
-version = "0.0.2.dev27"
+version = "0.0.2.dev28"
 description = "Fast typed decisions (choice, score, yes/no) across Jev-style engines, plus a guard for coding agents. Early development."
 readme = "README.md"
 license = "Apache-2.0"

{judgetap-0.0.2.dev27 → judgetap-0.0.2.dev28}/src/judgetap/__init__.py

@@ -23,7 +23,7 @@ from judgetap.errors import (
 )
 from judgetap.types import Decision, Question
 
-__version__ = "0.0.2.dev27"  # x-release-please-version
+__version__ = "0.0.2.dev28"  # x-release-please-version
 
 __all__ = [
     "Cascade",

{judgetap-0.0.2.dev27 → judgetap-0.0.2.dev28}/src/judgetap/_compat.py

@@ -14,8 +14,11 @@ OLD_PREFIX, NEW_PREFIX = "SNAPJUDGE_", "JUDGETAP_"
 
 
 def env(name: str) -> str | None:
-    """$JUDGETAP_<name>, else the old $SNAPJUDGE_<name>."""
-    return os.environ.get(NEW_PREFIX + name) or os.environ.get(OLD_PREFIX + name)
+    """$JUDGETAP_<name>, else the old $SNAPJUDGE_<name>. The new name wins
+    whenever it is set, even to an empty string (which reads as unset)."""
+    if NEW_PREFIX + name in os.environ:
+        return os.environ[NEW_PREFIX + name] or None
+    return os.environ.get(OLD_PREFIX + name) or None
 
 
 def default_home() -> Path:
@@ -26,7 +29,6 @@ def default_home() -> Path:
 
 def env_source(name: str) -> str | None:
     """Which variable `env(name)` read: '$JUDGETAP_<name>' or '$SNAPJUDGE_<name>'."""
-    for prefix in (NEW_PREFIX, OLD_PREFIX):
-        if os.environ.get(prefix + name):
-            return f"${prefix}{name}"
-    return None
+    if NEW_PREFIX + name in os.environ:  # set, even empty: the old name is ignored
+        return f"${NEW_PREFIX}{name}" if os.environ[NEW_PREFIX + name] else None
+    return f"${OLD_PREFIX}{name}" if os.environ.get(OLD_PREFIX + name) else None

{judgetap-0.0.2.dev27 → judgetap-0.0.2.dev28}/src/judgetap/cli.py

@@ -190,7 +190,7 @@ def _dashboard(args) -> int:
     from judgetap.guard.hook import home
 
     server = serve(home(), args.port)
-    url = f"http://127.0.0.1:{args.port}/"
+    url = f"http://127.0.0.1:{server.server_address[1]}/"  # the bound port (--port 0)
     print(f"judgetap dashboard on {url} (Ctrl+C to stop); reading {home()}")
     if not args.no_browser:
         webbrowser.open(url)
@@ -238,8 +238,14 @@ def _offer_keychain(stdin=None) -> None:
 def _keys_set(args) -> int:
     import getpass
 
-    from judgetap.secrets import set_key
+    from judgetap.secrets import KNOWN_KEYS, set_key
 
+    if args.name not in KNOWN_KEYS:
+        print(
+            f"{args.name} isn't read by any engine; known keys: {', '.join(KNOWN_KEYS)}."
+        )
+        print("The llm engine reads its provider's key from the environment only.")
+        return 2
     value = getpass.getpass(f"{args.name}: ")  # never echoed, never in argv
     if not value:
         print("Nothing entered; not saved.")
@@ -254,9 +260,15 @@ def _keys_set(args) -> int:
 
 
 def _keys_status(args) -> int:
-    from judgetap.secrets import key_source
+    from judgetap.secrets import KNOWN_KEYS, key_source
 
-    for name in args.names or ["TYPESAFE_API_KEY"]:
+    unknown = [n for n in args.names if n not in KNOWN_KEYS]
+    if unknown:
+        print(
+            f"Not read by any engine: {', '.join(unknown)}; known keys: {', '.join(KNOWN_KEYS)}."
+        )
+        return 2
+    for name in args.names or KNOWN_KEYS:
         print(f"{name}: {key_source(name) or 'not set'}")
     return 0
 

{judgetap-0.0.2.dev27 → judgetap-0.0.2.dev28}/src/judgetap/dashboard/data.py

@@ -85,8 +85,12 @@ def _load(home: Path, today: date) -> dict[str, Any]:
     calls: Counter = Counter()  # true totals; the latency deques are capped
     recent: deque = deque(maxlen=MAX_RECENT)
     cost, total, false_holds, library, loops, stops = 0.0, 0, 0, 0, 0, 0
-    for n, r in _iter_jsonl(home / "guard.jsonl"):
-        r["id"] = record_id(r, n)
+    for n, raw in _iter_jsonl(home / "guard.jsonl"):
+        try:
+            r = _clean(raw)
+            r["id"] = record_id(r, n)
+        except Exception:  # noqa: BLE001, S112 -- one unreadable record never breaks the page
+            continue
         r["source"] = r.get("source") or "guard"
         r["false_alarm"] = r["id"] in false_alarms
         cost += r.get("cost_usd") or 0
@@ -118,7 +122,8 @@ def _load(home: Path, today: date) -> dict[str, Any]:
         "outcomes": dict(outcomes),
         "holds_per_1000": round(1000 * outcomes["hold"] / total, 1) if total else None,
         "false_alarms": false_holds,
-        "cost_usd": round(cost, 6),
+        # Individually finite costs can still sum past float range.
+        "cost_usd": round(cost, 6) if math.isfinite(cost) else None,
         "engines": {
             name: {
                 "calls": calls[name],
@@ -132,6 +137,89 @@ def _load(home: Path, today: date) -> dict[str, Any]:
     return {"summary": summary, "recent": list(recent)[::-1]}
 
 
+STR_FIELDS = (
+    "id",
+    "ts",
+    "session",
+    "source",
+    "tool",
+    "subject",
+    "outcome",
+    "layer",
+    "rule",
+    "reason",
+    "engine",
+    "error",
+    "call",
+    "batch",
+)
+NUM_FIELDS = ("cost_usd", "latency_ms")
+
+
+def _finite(value: Any) -> float | None:
+    """A finite int/float as float; anything else (str, bool, NaN, inf) is None."""
+    if isinstance(value, bool) or not isinstance(value, int | float):
+        return None
+    try:
+        number = float(value)  # huge JSON ints overflow here: an invalid field
+    except OverflowError:
+        return None
+    return number if math.isfinite(number) else None
+
+
+MAX_DEPTH = 8  # deeper than any field the page reads
+
+
+def _scrub(value: Any, depth: int = 0) -> Any:
+    """NaN/Infinity anywhere (even in fields the page doesn't use) become None.
+    Nesting past MAX_DEPTH is cut to None, so a pathological unrelated field
+    can't exhaust the stack and cost the record."""
+    if isinstance(value, float) and not math.isfinite(value):
+        return None
+    if isinstance(value, dict | list) and depth >= MAX_DEPTH:
+        return None
+    if isinstance(value, dict):
+        return {k: _scrub(v, depth + 1) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_scrub(v, depth + 1) for v in value]
+    return value
+
+
+def _clean(r: dict[str, Any]) -> dict[str, Any]:
+    """The log is ours but may be edited, truncated or written by an older
+    version: coerce every field the page uses to its expected type, so no
+    value can crash the summary or make the JSON invalid (NaN)."""
+    out = _scrub(dict(r))
+    for key in STR_FIELDS:
+        if key in out and not isinstance(out[key], str):
+            out[key] = None
+    for key in NUM_FIELDS:
+        if key in out:
+            out[key] = _finite(out[key])
+    if not isinstance(out.get("p"), dict):
+        out["p"] = {}
+    else:
+        out["p"] = {
+            k: v
+            for k, v in out["p"].items()
+            if isinstance(k, str) and _finite(v) is not None
+        }
+    if "calls" in out and not isinstance(out["calls"], list):
+        # Not a calls list at all: treat the record as having none, so the
+        # legacy fallback still counts its engine.
+        del out["calls"]
+    if "calls" in out:
+        # Calls are echoed back in "recent" too: keep only well-formed ones
+        # with a finite (or absent) latency.
+        calls = out["calls"]
+        out["calls"] = [
+            {**c, "latency_ms": _finite(c.get("latency_ms"))}
+            for c in calls
+            if isinstance(c, dict) and isinstance(c.get("engine"), str)
+        ]
+    return out
+
+
 def _count_calls(r: dict[str, Any], calls: Counter, by_engine) -> None:
     """Engine metrics come from the calls each record reports, each with its
     own latency: nothing is inferred. A batch writes its calls on one record
@@ -143,14 +231,17 @@ def _count_calls(r: dict[str, Any], calls: Counter, by_engine) -> None:
         for c in reported:
             if isinstance(c, dict) and isinstance(c.get("engine"), str):
                 calls[c["engine"]] += 1
-                if isinstance(c.get("latency_ms"), int | float):
-                    by_engine[c["engine"]].append(float(c["latency_ms"]))
+                latency = _finite(c.get("latency_ms"))
+                if latency is not None:
+                    by_engine[c["engine"]].append(latency)
         return
     if not r.get("engine") or r.get("error"):
         return
     if r.get("layer") == "judge":
         calls[r["engine"]] += 1
-        by_engine[r["engine"]].append(float(r.get("latency_ms") or 0))
+        latency = _finite(r.get("latency_ms"))
+        if latency is not None:  # no sample for a missing or invalid latency
+            by_engine[r["engine"]].append(latency)
     elif r.get("layer") == "stop" and r.get("outcome") in ("allow", "block"):
         # A Stop check from before calls were logged: it asked its engine
         # once, but recorded no latency, so only the call is counted.

{judgetap-0.0.2.dev27 → judgetap-0.0.2.dev28}/src/judgetap/dashboard/server.py

@@ -44,7 +44,10 @@ def make_handler(home: Path, token: str, port: int) -> type[BaseHTTPRequestHandl
             self.wfile.write(body)
 
         def _json(self, status: int, obj: object) -> None:
-            self._send(status, json.dumps(obj).encode(), "application/json")
+            # allow_nan=False: bare NaN isn't JSON and would break the page.
+            self._send(
+                status, json.dumps(obj, allow_nan=False).encode(), "application/json"
+            )
 
         def do_GET(self) -> None:
             if not self._host_ok():
@@ -94,4 +97,9 @@ def make_handler(home: Path, token: str, port: int) -> type[BaseHTTPRequestHandl
 def serve(home: Path, port: int = 8765) -> ThreadingHTTPServer:
     token = secrets.token_urlsafe(24)
     server = ThreadingHTTPServer(("127.0.0.1", port), make_handler(home, token, port))
+    # Port 0 means "any free port": the Host allow-list must use the port
+    # the OS actually bound, or every request is refused.
+    bound = server.server_address[1]
+    if bound != port:
+        server.RequestHandlerClass = make_handler(home, token, bound)
     return server

{judgetap-0.0.2.dev27 → judgetap-0.0.2.dev28}/src/judgetap/secrets.py

@@ -10,6 +10,9 @@ from __future__ import annotations
 
 import os
 
+# Keys an engine reads through get_key. Saving any other name to the keychain
+# would store something nothing reads.
+KNOWN_KEYS = ("TYPESAFE_API_KEY",)
 SERVICE = "judgetap"
 OLD_SERVICE = "snapjudge"  # keys saved before the rename (#38)
 

judgetap-0.0.2.dev28/tests/test_robustness_68.py

@@ -0,0 +1,177 @@
+import json
+import math
+import threading
+import urllib.request
+
+import pytest
+
+from judgetap.dashboard.data import load
+
+
+def write(tmp_path, rows):
+    (tmp_path / "guard.jsonl").write_text("\n".join(rows) + "\n")
+
+
+@pytest.mark.parametrize(
+    "line",
+    [
+        '{"outcome":"allow","cost_usd":"0.1"}',
+        '{"outcome":["hold"]}',
+        '{"outcome":"allow","cost_usd":NaN}',
+        '{"outcome":"allow","layer":"judge","calls":[{"engine":"e","latency_ms":NaN}]}',
+        '{"outcome":"allow","layer":{"x":1},"engine":7,"source":[1],"p":"nope","calls":"x"}',
+        '{"outcome":"allow","latency_ms":Infinity,"layer":"judge","engine":"e"}',
+    ],
+)
+def test_odd_log_values_never_break_load(tmp_path, line):
+    write(tmp_path, [json.dumps({"outcome": "hold", "layer": "rules"}), line])
+    data = load(tmp_path)
+    json.dumps(data, allow_nan=False)  # strict JSON: no NaN/Infinity left
+    assert data["summary"]["outcomes"].get("hold") == 1
+    assert all(isinstance(k, str) or k is None for k in data["summary"]["outcomes"])
+
+
+def test_nan_latency_is_not_sampled(tmp_path):
+    write(
+        tmp_path,
+        [
+            '{"outcome":"allow","layer":"judge","calls":[{"engine":"e","latency_ms":NaN},{"engine":"e","latency_ms":5}]}'
+        ],
+    )
+    e = load(tmp_path)["summary"]["engines"]["e"]
+    assert e["calls"] == 2 and e["p50_ms"] == 5 and not math.isnan(e["p50_ms"])
+
+
+def test_api_data_answers_with_a_bad_line(tmp_path):
+    from judgetap.dashboard.server import serve
+
+    write(tmp_path, ['{"outcome":"allow","cost_usd":"0.1"}', '{"outcome":["hold"]}'])
+    srv = serve(tmp_path, 0)
+    port = srv.server_address[1]
+    threading.Thread(target=srv.serve_forever, daemon=True).start()
+    try:
+        with urllib.request.urlopen(f"http://127.0.0.1:{port}/api/data") as res:
+            assert res.status == 200 and json.loads(res.read())["summary"]["total"] == 2
+    finally:
+        srv.shutdown()
+        srv.server_close()
+
+
+def test_port_zero_uses_the_bound_port(tmp_path):
+    from judgetap.dashboard.server import serve
+
+    srv = serve(tmp_path, 0)
+    port = srv.server_address[1]
+    assert port != 0
+    threading.Thread(target=srv.serve_forever, daemon=True).start()
+    try:
+        with urllib.request.urlopen(f"http://127.0.0.1:{port}/") as res:
+            assert res.status == 200
+    finally:
+        srv.shutdown()
+        srv.server_close()
+
+
+def test_keys_set_rejects_names_no_engine_reads(monkeypatch, capsys):
+    from judgetap.cli import main
+
+    asked = []
+    monkeypatch.setattr("getpass.getpass", lambda prompt: asked.append(prompt) or "v")
+    assert main(["keys", "set", "OPENAI_API_KEY"]) == 2
+    assert not asked and "isn't read by any engine" in capsys.readouterr().out
+    assert main(["keys", "status", "OPENAI_API_KEY"]) == 2
+
+
+def test_empty_new_env_wins_over_old(monkeypatch):
+    from judgetap._compat import env, env_source
+
+    monkeypatch.setenv("JUDGETAP_ENGINE", "")
+    monkeypatch.setenv("SNAPJUDGE_ENGINE", "llm")
+    assert env("ENGINE") is None and env_source("ENGINE") is None
+    monkeypatch.delenv("JUDGETAP_ENGINE")
+    assert env("ENGINE") == "llm" and env_source("ENGINE") == "$SNAPJUDGE_ENGINE"
+
+
+def test_nan_in_unknown_fields_is_scrubbed(tmp_path):
+    write(tmp_path, ['{"outcome":"allow","extra":{"x":[NaN, -Infinity]}}'])
+    json.dumps(load(tmp_path), allow_nan=False)
+
+
+def test_huge_int_cost_keeps_the_record(tmp_path):
+    import json
+
+    from judgetap.dashboard.data import load
+
+    (tmp_path / "guard.jsonl").write_text(
+        json.dumps({"outcome": "hold", "cost_usd": 10**400}) + "\n"
+    )
+    s = load(tmp_path)["summary"]
+    assert s["total"] == 1 and s["outcomes"] == {"hold": 1}
+
+
+def test_invalid_legacy_latency_adds_no_sample(tmp_path):
+    from judgetap.dashboard.data import load
+
+    (tmp_path / "guard.jsonl").write_text(
+        '{"outcome":"allow","layer":"judge","engine":"e","latency_ms":NaN}\n'
+    )
+    e = load(tmp_path)["summary"]["engines"]["e"]
+    assert e["calls"] == 1 and e["p50_ms"] is None
+
+
+def test_overflowing_cost_total_still_serves(tmp_path):
+    import json
+    import socket
+    import threading
+    import urllib.request
+    from http.server import ThreadingHTTPServer
+
+    from judgetap.dashboard.server import make_handler
+
+    (tmp_path / "guard.jsonl").write_text(
+        "\n".join(json.dumps({"outcome": "allow", "cost_usd": 1e308}) for _ in range(2))
+        + "\n"
+    )
+    with socket.socket() as sock:
+        sock.bind(("127.0.0.1", 0))
+        port = sock.getsockname()[1]
+    srv = ThreadingHTTPServer(("127.0.0.1", port), make_handler(tmp_path, "tok", port))
+    threading.Thread(target=srv.serve_forever, daemon=True).start()
+    try:
+        with urllib.request.urlopen(f"http://127.0.0.1:{port}/api/data") as res:
+            assert res.status == 200
+            assert json.loads(res.read())["summary"]["cost_usd"] is None
+    finally:
+        srv.shutdown()
+        srv.server_close()
+
+
+def test_malformed_calls_field_keeps_legacy_fallback(tmp_path):
+    import json
+
+    from judgetap.dashboard.data import load
+
+    (tmp_path / "guard.jsonl").write_text(
+        json.dumps(
+            {
+                "id": "a",
+                "outcome": "allow",
+                "layer": "judge",
+                "engine": "e",
+                "latency_ms": 12,
+                "calls": "x",
+            }
+        )
+        + "\n"
+    )
+    e = load(tmp_path)["summary"]["engines"]["e"]
+    assert e["calls"] == 1 and e["p50_ms"] == 12
+
+
+def test_deeply_nested_extra_field_does_not_drop_the_record(tmp_path):
+    from judgetap.dashboard.data import load
+
+    deep = "[" * 5000 + "]" * 5000
+    line = '{"id": "a", "outcome": "hold", "layer": "rules", "extra": ' + deep + "}"
+    (tmp_path / "guard.jsonl").write_text(line + "\n")
+    assert load(tmp_path)["summary"]["total"] == 1