quantum-reasoning-skill 1.0.0

package/tests/test_reference.py

@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+import sys
+import unittest
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SPEC = importlib.util.spec_from_file_location(
+    "branch_controller", ROOT / "reference" / "branch_controller.py"
+)
+assert SPEC and SPEC.loader
+bc = importlib.util.module_from_spec(SPEC)
+sys.modules[SPEC.name] = bc
+SPEC.loader.exec_module(bc)
+
+
+class BranchControllerTests(unittest.TestCase):
+    def metrics(self, **overrides):
+        values = {
+            "evidence": 0.8,
+            "verification": 0.8,
+            "independence": 0.9,
+            "information_gain": 0.6,
+            "contradiction": 0.1,
+            "unresolved_assumptions": 0.1,
+            "normalized_cost": 0.2,
+            "shared_assumption_ratio": 0.0,
+            "semantic_similarity_to_leader": 0.0,
+        }
+        values.update(overrides)
+        return bc.BranchMetrics(**values)
+
+    def test_score_penalizes_shared_assumptions(self):
+        independent = self.metrics(shared_assumption_ratio=0.0)
+        correlated = self.metrics(shared_assumption_ratio=0.8)
+        self.assertGreater(bc.branch_score(independent), bc.branch_score(correlated))
+
+    def test_high_contradiction_rejects_branch(self):
+        branch = bc.Branch("bad", self.metrics(contradiction=0.95))
+        updated = bc.classify_branch(branch)
+        self.assertEqual(updated.state, bc.BranchState.REJECTED)
+
+    def test_material_evidence_change_revives_dormant_branch(self):
+        previous = self.metrics(evidence=0.30, verification=0.35, contradiction=0.50)
+        current = self.metrics(evidence=0.50, verification=0.35, contradiction=0.50)
+        branch = bc.Branch(
+            "revive",
+            current,
+            state=bc.BranchState.DORMANT,
+            previous_metrics=previous,
+        )
+        self.assertTrue(bc.should_revive(branch))
+        self.assertEqual(bc.update_branch_state(branch).state, bc.BranchState.ACTIVE)
+
+    def test_collapse_requires_clear_verified_leader(self):
+        leader = bc.Branch(
+            "leader",
+            self.metrics(
+                evidence=1.0,
+                verification=1.0,
+                independence=1.0,
+                information_gain=0.8,
+                contradiction=0.0,
+                unresolved_assumptions=0.0,
+                normalized_cost=0.0,
+            ),
+        )
+        weak = bc.Branch(
+            "weak",
+            self.metrics(
+                evidence=0.35,
+                verification=0.30,
+                independence=0.5,
+                information_gain=0.2,
+                contradiction=0.4,
+                unresolved_assumptions=0.5,
+                normalized_cost=0.4,
+            ),
+        )
+        can_collapse, reason, selected = bc.collapse_decision([leader, weak])
+        self.assertTrue(can_collapse, reason)
+        self.assertEqual(selected.branch_id, "leader")
+
+    def test_close_competitor_blocks_collapse(self):
+        first = bc.Branch(
+            "a",
+            self.metrics(
+                evidence=1.0,
+                verification=1.0,
+                independence=1.0,
+                contradiction=0.0,
+                unresolved_assumptions=0.0,
+                normalized_cost=0.0,
+            ),
+        )
+        second = bc.Branch(
+            "b",
+            self.metrics(
+                evidence=0.95,
+                verification=0.95,
+                independence=1.0,
+                contradiction=0.0,
+                unresolved_assumptions=0.0,
+                normalized_cost=0.0,
+            ),
+        )
+        can_collapse, reason, _ = bc.collapse_decision([first, second])
+        self.assertFalse(can_collapse)
+        self.assertIn("margin", reason)
+
+    def test_uncertainty_controls_width(self):
+        self.assertEqual(bc.recommended_width(0.10), (2, 3))
+        self.assertEqual(bc.recommended_width(0.40), (4, 6))
+        self.assertEqual(bc.recommended_width(0.90), (6, 10))
+
+    def test_diversity_ratio_detects_duplicates(self):
+        ratio = bc.diversity_ratio([0.2, 0.9, 0.4, 0.95], duplicate_threshold=0.85)
+        self.assertEqual(ratio, 0.5)
+
+    def test_invalid_metric_fails_fast(self):
+        with self.assertRaises(ValueError):
+            bc.branch_score(self.metrics(evidence=1.2))
+
+
+if __name__ == "__main__":
+    unittest.main()

package/tests/test_submission_validation.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+import importlib.util, json, sys, tempfile, unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+for name, path in (
+    ("evaluate", ROOT / "benchmark" / "evaluate.py"),
+    ("validate_submission", ROOT / "benchmark" / "validate_submission.py"),
+):
+    spec = importlib.util.spec_from_file_location(name, path)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[name] = module
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+
+evaluate = sys.modules["evaluate"]
+validator = sys.modules["validate_submission"]
+
+
+class SubmissionValidationTests(unittest.TestCase):
+    def make_bundle(self, root: Path) -> Path:
+        bundle = root / "provider-model-2026-09-07"
+        bundle.mkdir()
+        cases = [{"id": "c1", "domain": "math", "prompt": "1+1?", "accepted_answers": ["2"]}]
+        baseline = [{"case_id": "c1", "answer": "2", "tokens": 10, "tool_calls": 0, "latency_ms": 100}]
+        skill = [{"case_id": "c1", "answer": "2", "tokens": 12, "tool_calls": 0, "latency_ms": 110}]
+        (bundle / "cases.jsonl").write_text(json.dumps(cases[0]) + "\n", encoding="utf-8")
+        (bundle / "baseline.jsonl").write_text(json.dumps(baseline[0]) + "\n", encoding="utf-8")
+        (bundle / "skill.jsonl").write_text(json.dumps(skill[0]) + "\n", encoding="utf-8")
+        meta = {
+            "provider": "example",
+            "model": "model",
+            "run_date": "2026-09-07",
+            "skill_version": "0.3.0",
+            "repetitions_per_case": 1,
+            "failures_recorded": True,
+            "baseline": {"instruction_config": "baseline", "sampling": {}, "tool_availability": [], "context_limit": 1000, "output_limit": 100},
+            "skill": {"instruction_config": "SKILL.md", "sampling": {}, "tool_availability": [], "context_limit": 1000, "output_limit": 100},
+        }
+        (bundle / "metadata.json").write_text(json.dumps(meta), encoding="utf-8")
+        payload = {"skill": evaluate.summarize(cases, skill), "baseline": evaluate.summarize(cases, baseline)}
+        payload["comparison"] = evaluate.compare(payload["baseline"], payload["skill"])
+        (bundle / "comparison.json").write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
+        (bundle / "README.md").write_text("Reproducible test bundle.\n", encoding="utf-8")
+        return bundle
+
+    def test_valid_bundle(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            validator.validate_bundle(self.make_bundle(Path(tmp)))
+
+    def test_missing_artifact_fails(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            bundle = self.make_bundle(Path(tmp))
+            (bundle / "README.md").unlink()
+            with self.assertRaises(ValueError):
+                validator.validate_bundle(bundle)
+
+    def test_tampered_comparison_fails(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            bundle = self.make_bundle(Path(tmp))
+            payload = json.loads((bundle / "comparison.json").read_text())
+            payload["comparison"]["accuracy_delta"] = 99
+            (bundle / "comparison.json").write_text(json.dumps(payload), encoding="utf-8")
+            with self.assertRaises(ValueError):
+                validator.validate_bundle(bundle)
+
+    def test_unrecorded_failures_flag_fails(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            bundle = self.make_bundle(Path(tmp))
+            meta = json.loads((bundle / "metadata.json").read_text())
+            meta["failures_recorded"] = False
+            (bundle / "metadata.json").write_text(json.dumps(meta), encoding="utf-8")
+            with self.assertRaises(ValueError):
+                validator.validate_bundle(bundle)
+
+
+if __name__ == "__main__":
+    unittest.main()