parallax-evals 2.0.0.dev3__tar.gz → 2.0.0.dev4__tar.gz

{parallax_evals-2.0.0.dev3 → parallax_evals-2.0.0.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: parallax-evals
-Version: 2.0.0.dev3
+Version: 2.0.0.dev4
 Summary: Agent evaluations on seeded MCP worlds, recorded and compared in Langfuse.
 License-Expression: MIT
 License-File: LICENSE
@@ -160,7 +160,7 @@ and how comparisons pair runs.
 
 ## Status
 
-`2.0.0.dev3`. The task format, world, checks, controls, statistics and Langfuse
+`2.0.0.dev4`. The task format, world, checks, controls, statistics and Langfuse
 recording are in place. No pilot has been run, so no cross-harness results are
 published yet.
 

{parallax_evals-2.0.0.dev3 → parallax_evals-2.0.0.dev4}/README.md

@@ -137,7 +137,7 @@ and how comparisons pair runs.
 
 ## Status
 
-`2.0.0.dev3`. The task format, world, checks, controls, statistics and Langfuse
+`2.0.0.dev4`. The task format, world, checks, controls, statistics and Langfuse
 recording are in place. No pilot has been run, so no cross-harness results are
 published yet.
 

{parallax_evals-2.0.0.dev3 → parallax_evals-2.0.0.dev4}/docs/methodology.md

@@ -262,7 +262,7 @@ A human moves a flagged task by editing `task.toml` and bumping the task
 This section assesses Parallax against the Agentic Benchmark Checklist (Zhu et
 al., 2025, *Establishing Best Practices for Building Rigorous Agentic
 Benchmarks*). The items are grouped under the checklist's three headings. The
-status is as of `2.0.0.dev3`, before the first pilot:
+status is as of `2.0.0.dev4`, before the first pilot:
 
 - ✅ met by construction and enforced in CI;
 - ◐ designed but not yet exercised or complete;

{parallax_evals-2.0.0.dev3 → parallax_evals-2.0.0.dev4}/docs/roadmap.md

@@ -1,6 +1,6 @@
 # Roadmap
 
-Parallax is at `2.0.0.dev3`. The task format, the simulated world, the checks,
+Parallax is at `2.0.0.dev4`. The task format, the simulated world, the checks,
 the controls, the statistics and Langfuse recording are in place and exercised by CI.
 This page is directional; it is not a commitment to dates.
 

parallax_evals-2.0.0.dev4/parallax/__init__.py

@@ -0,0 +1 @@
+__version__ = "2.0.0.dev4"

{parallax_evals-2.0.0.dev3 → parallax_evals-2.0.0.dev4}/parallax/semantic.py

@@ -69,8 +69,9 @@ def _files(root: Path | None) -> dict[str, JsonValue] | None:
     for path in sorted(root.rglob("*")):
         if path.is_symlink():
             raise MissingEvidence(f"cannot judge symlink snapshot: {path.relative_to(root)}")
-        # Bytecode appears when the agent runs Python in the workspace; it is not evidence of what it did.
-        if not path.is_file() or "__pycache__" in path.parts or path.suffix == ".pyc":
+        # Bytecode (the agent ran Python) and a git store (a harness such as openclaw ran `git init`) are
+        # tooling, not evidence of what it did: the working tree is.
+        if not path.is_file() or {"__pycache__", ".git"} & set(path.parts) or path.suffix == ".pyc":
             continue
         size += path.stat().st_size
         if size > PACK_LIMIT:

{parallax_evals-2.0.0.dev3 → parallax_evals-2.0.0.dev4}/parallax/track/sink.py

@@ -233,8 +233,8 @@ def experiment_id_for(project_id: str, dataset_id: str, run_name: str) -> str:
 
 
 def metadata_value(value: object) -> str:
-    """Experiment metadata as Parallax reads it: strings as they are, anything else as compact JSON."""
-    return value if isinstance(value, str) else json.dumps(value, separators=(",", ":"))
+    """Experiment metadata as Parallax reads it: strings as they are, anything else as compact JSON, unescaped as Langfuse holds it."""
+    return value if isinstance(value, str) else json.dumps(value, separators=(",", ":"), ensure_ascii=False)
 
 
 def _score(

{parallax_evals-2.0.0.dev3 → parallax_evals-2.0.0.dev4}/tasks/conformance/conf-two-turn-carry/solution/oracle.yaml

@@ -2,4 +2,4 @@ steps:
   turn-01:
     answer: "Got it - I'll keep that in mind."
   turn-02:
-    answer: "The side door codeword is amber-falcon-62."
+    answer: "The booking reference is amber-falcon-62."

parallax_evals-2.0.0.dev4/tasks/conformance/conf-two-turn-carry/steps/turn-01/instruction.md

@@ -0,0 +1,2 @@
+The restaurant just confirmed our team lunch, and the booking reference is amber-falcon-62. Nothing to do with it yet, just hang on to it for me.
+<!-- parallax canary b1651940-3bf1-40ab-81eb-ef15f5094e32 -->

parallax_evals-2.0.0.dev4/tasks/conformance/conf-two-turn-carry/steps/turn-02/instruction.md

@@ -0,0 +1 @@
+I'm at the restaurant now - what was the booking reference again?

{parallax_evals-2.0.0.dev3 → parallax_evals-2.0.0.dev4}/tasks/conformance/conf-two-turn-carry/task.toml

@@ -2,9 +2,9 @@ version = "1.0"
 
 [metadata.parallax]
 id = "conf-two-turn-carry"
-version = 1
+version = 2
 area = "conformance"
-summary = "Turn 1 gives a door codeword; turn 2 in the same conversation asks for it back"
+summary = "Turn 1 gives a booking reference; turn 2 in the same conversation asks for it back"
 requires = ["mcp", "multi_turn"]
 provenance = "hand_authored"
 canary = "b1651940-3bf1-40ab-81eb-ef15f5094e32"

{parallax_evals-2.0.0.dev3 → parallax_evals-2.0.0.dev4}/tasks/conformance/conf-two-turn-carry/tests/checks.yaml

@@ -1,5 +1,5 @@
 checks:
-  - {id: recalls-codeword, kind: answer, mode: forgot_context, turn: 2,
+  - {id: recalls-reference, kind: answer, mode: forgot_context, turn: 2,
      must: [{regex: "amber[\\s-]*falcon[\\s-]*62"}]}
   - {id: world-untouched, kind: state, mode: collateral_change, table: "*", unchanged: true,
      except_tables: [user.interactions]}

parallax_evals-2.0.0.dev4/tasks/conformance/conf-two-turn-carry/tests/mutations.yaml

@@ -0,0 +1,9 @@
+mutations:
+  - {id: forgot, note: "has lost the first turn", answers: {2: "I don't have a booking reference from you - could you tell me again?"},
+     must_fail: [recalls-reference]}
+  - {id: wrong-reference, note: "misremembers the number", answers: {2: "It was amber-falcon-26."},
+     must_fail: [recalls-reference]}
+  - {id: stashes-in-chat, note: "writes the reference into #conf to remember it", collateral: true,
+     patch: {insert: {chat.messages: [{id: msg-x, channel: ch-conf, author: robin@harbourline.test,
+       text: "lunch booking reference amber-falcon-62", ts: "2026-06-03T09:13:00+01:00", thread_id: null}]}},
+     must_fail: [world-untouched]}

{parallax_evals-2.0.0.dev3 → parallax_evals-2.0.0.dev4}/tests/core/test_semantic_grading.py

@@ -332,13 +332,15 @@ def test_missing_file_evidence_keeps_full_paths_and_unlisted_files(tmp_path: Pat
     }
 
 
-def test_file_evidence_skips_the_bytecode_a_test_run_leaves(tmp_path: Path) -> None:
+def test_file_evidence_skips_bytecode_and_a_git_store(tmp_path: Path) -> None:
     from parallax.semantic import evidence_state
 
     (tmp_path / "pkg" / "__pycache__").mkdir(parents=True)
     (tmp_path / "pkg" / "dates.py").write_text("x = 1\n")
     (tmp_path / "pkg" / "__pycache__" / "dates.cpython-313.pyc").write_bytes(b"\xa7\r\r\n\x00\xff")
     (tmp_path / "pkg" / "stray.pyc").write_bytes(b"\xa7\r\r\n\x00\xff")
+    (tmp_path / ".git").mkdir()
+    (tmp_path / ".git" / "index").write_bytes(b"DIRC\x00\x00\x00\x02\xff")  # what openclaw's `git init` leaves
     state = evidence_state(TASK, EV.model_copy(update={"workspace": tmp_path}))
     assert state["files"] == {"pkg/dates.py": "x = 1\n"}
 

{parallax_evals-2.0.0.dev3 → parallax_evals-2.0.0.dev4}/tests/core/test_track.py

@@ -257,5 +257,8 @@ def test_experiment_metadata_reads_the_same_from_per_key_and_json_attributes() -
 
     assert metadata_value("2026-01-01T00:00:00+00:00") == "2026-01-01T00:00:00+00:00"
     assert metadata_value(["a", "b"]) == '["a","b"]'
+    assert (
+        metadata_value(["Hermes Agent v0.22.6 · stable"]) == '["Hermes Agent v0.22.6 · stable"]'
+    )  # as Langfuse holds it
     assert metadata_value(True) == "true"
     assert metadata_value(3) == "3"

parallax_evals-2.0.0.dev3/parallax/__init__.py

@@ -1 +0,0 @@
-__version__ = "2.0.0.dev3"

parallax_evals-2.0.0.dev3/tasks/conformance/conf-two-turn-carry/steps/turn-01/instruction.md

@@ -1,2 +0,0 @@
-Facilities just gave me today's codeword for the side door: amber-falcon-62. Nothing to do with it yet, just hang on to it for me.
-<!-- parallax canary b1651940-3bf1-40ab-81eb-ef15f5094e32 -->

parallax_evals-2.0.0.dev3/tasks/conformance/conf-two-turn-carry/steps/turn-02/instruction.md

@@ -1 +0,0 @@
-I'm at the side door now - what was the codeword again?

parallax_evals-2.0.0.dev3/tasks/conformance/conf-two-turn-carry/tests/mutations.yaml

@@ -1,9 +0,0 @@
-mutations:
-  - {id: forgot, note: "has lost the first turn", answers: {2: "I don't have a codeword from you - could you tell me again?"},
-     must_fail: [recalls-codeword]}
-  - {id: wrong-codeword, note: "misremembers the number", answers: {2: "It was amber-falcon-26."},
-     must_fail: [recalls-codeword]}
-  - {id: stashes-in-chat, note: "writes the codeword into #conf to remember it", collateral: true,
-     patch: {insert: {chat.messages: [{id: msg-x, channel: ch-conf, author: robin@harbourline.test,
-       text: "side door codeword amber-falcon-62", ts: "2026-06-03T09:13:00+01:00", thread_id: null}]}},
-     must_fail: [world-untouched]}