quantum-reasoning-skill 1.0.0 → 1.1.0

package/reference/branch_controller.py

@@ -1,256 +0,0 @@
-"""Reference branch-control logic for the Quantum Reasoning Skill.
-
-This module is intentionally provider-agnostic. It does not generate hidden reasoning
-or call an LLM. It only manages branch telemetry and makes deterministic decisions
-from explicit, externally supplied measurements.
-
-The thresholds below are reference defaults, not scientifically validated constants.
-They are meant to be calibrated with benchmark evidence.
-"""
-
-from __future__ import annotations
-
-from dataclasses import dataclass, replace
-from enum import Enum
-from math import log2
-from typing import Iterable, Sequence
-
-
-class BranchState(str, Enum):
-    ACTIVE = "active"
-    DORMANT = "dormant"
-    REJECTED = "rejected"
-
-
-@dataclass(frozen=True)
-class BranchMetrics:
-    evidence: float
-    verification: float
-    independence: float
-    information_gain: float
-    contradiction: float
-    unresolved_assumptions: float
-    normalized_cost: float
-    shared_assumption_ratio: float = 0.0
-    semantic_similarity_to_leader: float = 0.0
-
-    def validate(self) -> None:
-        for name, value in self.__dict__.items():
-            if not 0.0 <= value <= 1.0:
-                raise ValueError(f"{name} must be in [0, 1], got {value!r}")
-
-
-@dataclass(frozen=True)
-class Branch:
-    branch_id: str
-    metrics: BranchMetrics
-    state: BranchState = BranchState.ACTIVE
-    previous_metrics: BranchMetrics | None = None
-
-
-@dataclass(frozen=True)
-class Thresholds:
-    collapse_score: float = 0.78
-    collapse_verification: float = 0.75
-    collapse_max_contradiction: float = 0.15
-    collapse_margin: float = 0.12
-    unresolved_near_leader_margin: float = 0.10
-    reject_contradiction: float = 0.85
-    dormant_score: float = 0.35
-    revive_evidence_delta: float = 0.15
-    revive_verification_delta: float = 0.25
-    revive_contradiction_drop: float = 0.20
-
-
-DEFAULT_THRESHOLDS = Thresholds()
-
-
-WEIGHTS = {
-    "evidence": 0.25,
-    "verification": 0.25,
-    "independence": 0.15,
-    "information_gain": 0.10,
-    "contradiction": -0.15,
-    "unresolved_assumptions": -0.05,
-    "normalized_cost": -0.05,
-}
-
-
-def _clamp01(value: float) -> float:
-    return max(0.0, min(1.0, value))
-
-
-def effective_independence(metrics: BranchMetrics) -> float:
-    """Penalize correlated branches and near-duplicates.
-
-    The effective independence is the declared independence reduced by the strongest
-    observed correlation signal: shared assumptions or semantic similarity.
-    """
-    metrics.validate()
-    correlation = max(
-        metrics.shared_assumption_ratio,
-        metrics.semantic_similarity_to_leader,
-    )
-    return _clamp01(metrics.independence * (1.0 - correlation))
-
-
-def branch_score(metrics: BranchMetrics) -> float:
-    """Return a deterministic support score in [0, 1]."""
-    metrics.validate()
-    values = {
-        "evidence": metrics.evidence,
-        "verification": metrics.verification,
-        "independence": effective_independence(metrics),
-        "information_gain": metrics.information_gain,
-        "contradiction": metrics.contradiction,
-        "unresolved_assumptions": metrics.unresolved_assumptions,
-        "normalized_cost": metrics.normalized_cost,
-    }
-    raw = 0.15 + sum(WEIGHTS[name] * value for name, value in values.items())
-    return _clamp01(raw)
-
-
-def rank_branches(branches: Iterable[Branch]) -> list[Branch]:
-    return sorted(
-        (branch for branch in branches if branch.state != BranchState.REJECTED),
-        key=lambda branch: branch_score(branch.metrics),
-        reverse=True,
-    )
-
-
-def classify_branch(
-    branch: Branch,
-    thresholds: Thresholds = DEFAULT_THRESHOLDS,
-) -> Branch:
-    """Move an active branch to dormant/rejected when evidence warrants it."""
-    score = branch_score(branch.metrics)
-    if branch.metrics.contradiction >= thresholds.reject_contradiction:
-        return replace(branch, state=BranchState.REJECTED)
-    if score < thresholds.dormant_score:
-        return replace(branch, state=BranchState.DORMANT)
-    return replace(branch, state=BranchState.ACTIVE)
-
-
-def should_revive(
-    branch: Branch,
-    thresholds: Thresholds = DEFAULT_THRESHOLDS,
-) -> bool:
-    """Return True when new evidence materially improves a dormant branch."""
-    if branch.state != BranchState.DORMANT or branch.previous_metrics is None:
-        return False
-    current = branch.metrics
-    previous = branch.previous_metrics
-    current.validate()
-    previous.validate()
-    return any(
-        (
-            current.evidence - previous.evidence >= thresholds.revive_evidence_delta,
-            current.verification - previous.verification
-            >= thresholds.revive_verification_delta,
-            previous.contradiction - current.contradiction
-            >= thresholds.revive_contradiction_drop,
-        )
-    )
-
-
-def update_branch_state(
-    branch: Branch,
-    thresholds: Thresholds = DEFAULT_THRESHOLDS,
-) -> Branch:
-    if should_revive(branch, thresholds):
-        return replace(branch, state=BranchState.ACTIVE)
-    return classify_branch(branch, thresholds)
-
-
-def normalized_entropy(probabilities: Sequence[float]) -> float:
-    """Normalized Shannon entropy in [0, 1]."""
-    if not probabilities:
-        return 0.0
-    if any(value < 0 for value in probabilities):
-        raise ValueError("probabilities cannot contain negative values")
-    total = sum(probabilities)
-    if total <= 0:
-        return 0.0
-    normalized = [value / total for value in probabilities if value > 0]
-    if len(normalized) <= 1:
-        return 0.0
-    entropy = -sum(value * log2(value) for value in normalized)
-    return _clamp01(entropy / log2(len(normalized)))
-
-
-def uncertainty_from_branches(branches: Sequence[Branch]) -> float:
-    """Estimate uncertainty from the score distribution of surviving branches."""
-    surviving = [
-        branch_score(branch.metrics)
-        for branch in branches
-        if branch.state != BranchState.REJECTED
-    ]
-    return normalized_entropy(surviving)
-
-
-def recommended_width(uncertainty: float) -> tuple[int, int]:
-    """Return a reference branch-count range for the next search step."""
-    if not 0.0 <= uncertainty <= 1.0:
-        raise ValueError("uncertainty must be in [0, 1]")
-    if uncertainty < 0.25:
-        return (2, 3)
-    if uncertainty < 0.55:
-        return (4, 6)
-    return (6, 10)
-
-
-def collapse_decision(
-    branches: Sequence[Branch],
-    thresholds: Thresholds = DEFAULT_THRESHOLDS,
-) -> tuple[bool, str, Branch | None]:
-    """Determine whether the search can collapse to a single branch.
-
-    Returns ``(can_collapse, reason, leader)``. Collapse is blocked when a strong,
-    unresolved alternative remains close enough to the leader to matter.
-    """
-    ranked = rank_branches(branches)
-    if not ranked:
-        return (False, "no surviving branch", None)
-
-    leader = ranked[0]
-    leader_score = branch_score(leader.metrics)
-    if leader_score < thresholds.collapse_score:
-        return (False, "leader score below collapse threshold", leader)
-    if leader.metrics.verification < thresholds.collapse_verification:
-        return (False, "leader verification below collapse threshold", leader)
-    if leader.metrics.contradiction > thresholds.collapse_max_contradiction:
-        return (False, "leader contradiction remains too high", leader)
-
-    if len(ranked) > 1:
-        runner_up = ranked[1]
-        runner_score = branch_score(runner_up.metrics)
-        if leader_score - runner_score < thresholds.collapse_margin:
-            return (False, "leader margin is too small", leader)
-        if (
-            runner_up.metrics.information_gain >= 0.60
-            and leader_score - runner_score
-            <= thresholds.unresolved_near_leader_margin
-        ):
-            return (False, "high-information alternative remains unresolved", leader)
-
-    return (True, "collapse criteria satisfied", leader)
-
-
-def diversity_ratio(
-    pairwise_similarities: Sequence[float],
-    duplicate_threshold: float = 0.85,
-) -> float:
-    """Fraction of pairwise comparisons that are materially distinct.
-
-    Similarity values must be in [0, 1]. A pair at or above the duplicate threshold
-    is treated as correlated/redundant for this diagnostic.
-    """
-    if not pairwise_similarities:
-        return 1.0
-    if not 0.0 <= duplicate_threshold <= 1.0:
-        raise ValueError("duplicate_threshold must be in [0, 1]")
-    for value in pairwise_similarities:
-        if not 0.0 <= value <= 1.0:
-            raise ValueError("similarity values must be in [0, 1]")
-    distinct = sum(value < duplicate_threshold for value in pairwise_similarities)
-    return distinct / len(pairwise_similarities)

package/tests/test_benchmark.py

@@ -1,120 +0,0 @@
-from __future__ import annotations
-
-import importlib.util
-from pathlib import Path
-import unittest
-
-
-ROOT = Path(__file__).resolve().parents[1]
-SPEC = importlib.util.spec_from_file_location(
-    "benchmark_evaluate", ROOT / "benchmark" / "evaluate.py"
-)
-assert SPEC and SPEC.loader
-evaluate = importlib.util.module_from_spec(SPEC)
-SPEC.loader.exec_module(evaluate)
-
-
-CASES = [
-    {
-        "id": "c1",
-        "domain": "math",
-        "prompt": "1+1?",
-        "accepted_answers": ["2"],
-    },
-    {
-        "id": "c2",
-        "domain": "logic",
-        "prompt": "yes?",
-        "accepted_answers": ["yes"],
-    },
-]
-
-
-class BenchmarkTests(unittest.TestCase):
-    def test_summary_computes_accuracy_and_telemetry(self):
-        results = [
-            {
-                "case_id": "c1",
-                "answer": "2",
-                "tokens": 100,
-                "tool_calls": 1,
-                "latency_ms": 50,
-                "branches_total": 4,
-                "branches_distinct": 3,
-                "revived_branches": 1,
-                "recovered_errors": 1,
-                "contradictions_found": 2,
-                "contradictions_resolved": 2,
-            },
-            {
-                "case_id": "c2",
-                "answer": "no",
-                "tokens": 200,
-                "tool_calls": 3,
-                "latency_ms": 150,
-                "branches_total": 2,
-                "branches_distinct": 1,
-                "revived_branches": 0,
-                "recovered_errors": 0,
-                "contradictions_found": 1,
-                "contradictions_resolved": 0,
-            },
-        ]
-        summary = evaluate.summarize(CASES, results)
-        self.assertEqual(summary["accuracy"], 0.5)
-        self.assertEqual(summary["mean_tokens"], 150.0)
-        self.assertEqual(summary["mean_tool_calls"], 2.0)
-        self.assertEqual(summary["mean_latency_ms"], 100.0)
-        self.assertEqual(summary["branch_diversity_ratio"], 4 / 6)
-        self.assertEqual(summary["contradiction_resolution_rate"], 2 / 3)
-
-    def test_compare_reports_cost_and_accuracy_changes(self):
-        baseline = {
-            "accuracy": 0.5,
-            "mean_tokens": 100.0,
-            "mean_tool_calls": 1.0,
-            "mean_latency_ms": 100.0,
-            "branch_diversity_ratio": 0.4,
-            "contradiction_resolution_rate": 0.5,
-        }
-        skill = {
-            "accuracy": 0.75,
-            "mean_tokens": 150.0,
-            "mean_tool_calls": 2.0,
-            "mean_latency_ms": 120.0,
-            "branch_diversity_ratio": 0.7,
-            "contradiction_resolution_rate": 0.8,
-        }
-        delta = evaluate.compare(baseline, skill)
-        self.assertEqual(delta["accuracy_delta"], 0.25)
-        self.assertEqual(delta["tokens_percent_change"], 50.0)
-        self.assertEqual(delta["tool_calls_percent_change"], 100.0)
-        self.assertEqual(delta["latency_percent_change"], 20.0)
-        self.assertAlmostEqual(delta["branch_diversity_delta"], 0.3)
-        self.assertAlmostEqual(delta["contradiction_resolution_delta"], 0.3)
-
-    def test_missing_result_fields_fail(self):
-        with self.assertRaises(ValueError):
-            evaluate.summarize(
-                CASES,
-                [{"case_id": "c1", "answer": "2", "tokens": 1}],
-            )
-
-    def test_unknown_case_id_fails(self):
-        with self.assertRaises(ValueError):
-            evaluate.summarize(
-                CASES,
-                [
-                    {
-                        "case_id": "missing",
-                        "answer": "2",
-                        "tokens": 1,
-                        "tool_calls": 0,
-                        "latency_ms": 1,
-                    }
-                ],
-            )
-
-
-if __name__ == "__main__":
-    unittest.main()

package/tests/test_reference.py

@@ -1,128 +0,0 @@
-from __future__ import annotations
-
-import importlib.util
-from pathlib import Path
-import sys
-import unittest
-
-
-ROOT = Path(__file__).resolve().parents[1]
-SPEC = importlib.util.spec_from_file_location(
-    "branch_controller", ROOT / "reference" / "branch_controller.py"
-)
-assert SPEC and SPEC.loader
-bc = importlib.util.module_from_spec(SPEC)
-sys.modules[SPEC.name] = bc
-SPEC.loader.exec_module(bc)
-
-
-class BranchControllerTests(unittest.TestCase):
-    def metrics(self, **overrides):
-        values = {
-            "evidence": 0.8,
-            "verification": 0.8,
-            "independence": 0.9,
-            "information_gain": 0.6,
-            "contradiction": 0.1,
-            "unresolved_assumptions": 0.1,
-            "normalized_cost": 0.2,
-            "shared_assumption_ratio": 0.0,
-            "semantic_similarity_to_leader": 0.0,
-        }
-        values.update(overrides)
-        return bc.BranchMetrics(**values)
-
-    def test_score_penalizes_shared_assumptions(self):
-        independent = self.metrics(shared_assumption_ratio=0.0)
-        correlated = self.metrics(shared_assumption_ratio=0.8)
-        self.assertGreater(bc.branch_score(independent), bc.branch_score(correlated))
-
-    def test_high_contradiction_rejects_branch(self):
-        branch = bc.Branch("bad", self.metrics(contradiction=0.95))
-        updated = bc.classify_branch(branch)
-        self.assertEqual(updated.state, bc.BranchState.REJECTED)
-
-    def test_material_evidence_change_revives_dormant_branch(self):
-        previous = self.metrics(evidence=0.30, verification=0.35, contradiction=0.50)
-        current = self.metrics(evidence=0.50, verification=0.35, contradiction=0.50)
-        branch = bc.Branch(
-            "revive",
-            current,
-            state=bc.BranchState.DORMANT,
-            previous_metrics=previous,
-        )
-        self.assertTrue(bc.should_revive(branch))
-        self.assertEqual(bc.update_branch_state(branch).state, bc.BranchState.ACTIVE)
-
-    def test_collapse_requires_clear_verified_leader(self):
-        leader = bc.Branch(
-            "leader",
-            self.metrics(
-                evidence=1.0,
-                verification=1.0,
-                independence=1.0,
-                information_gain=0.8,
-                contradiction=0.0,
-                unresolved_assumptions=0.0,
-                normalized_cost=0.0,
-            ),
-        )
-        weak = bc.Branch(
-            "weak",
-            self.metrics(
-                evidence=0.35,
-                verification=0.30,
-                independence=0.5,
-                information_gain=0.2,
-                contradiction=0.4,
-                unresolved_assumptions=0.5,
-                normalized_cost=0.4,
-            ),
-        )
-        can_collapse, reason, selected = bc.collapse_decision([leader, weak])
-        self.assertTrue(can_collapse, reason)
-        self.assertEqual(selected.branch_id, "leader")
-
-    def test_close_competitor_blocks_collapse(self):
-        first = bc.Branch(
-            "a",
-            self.metrics(
-                evidence=1.0,
-                verification=1.0,
-                independence=1.0,
-                contradiction=0.0,
-                unresolved_assumptions=0.0,
-                normalized_cost=0.0,
-            ),
-        )
-        second = bc.Branch(
-            "b",
-            self.metrics(
-                evidence=0.95,
-                verification=0.95,
-                independence=1.0,
-                contradiction=0.0,
-                unresolved_assumptions=0.0,
-                normalized_cost=0.0,
-            ),
-        )
-        can_collapse, reason, _ = bc.collapse_decision([first, second])
-        self.assertFalse(can_collapse)
-        self.assertIn("margin", reason)
-
-    def test_uncertainty_controls_width(self):
-        self.assertEqual(bc.recommended_width(0.10), (2, 3))
-        self.assertEqual(bc.recommended_width(0.40), (4, 6))
-        self.assertEqual(bc.recommended_width(0.90), (6, 10))
-
-    def test_diversity_ratio_detects_duplicates(self):
-        ratio = bc.diversity_ratio([0.2, 0.9, 0.4, 0.95], duplicate_threshold=0.85)
-        self.assertEqual(ratio, 0.5)
-
-    def test_invalid_metric_fails_fast(self):
-        with self.assertRaises(ValueError):
-            bc.branch_score(self.metrics(evidence=1.2))
-
-
-if __name__ == "__main__":
-    unittest.main()

package/tests/test_submission_validation.py

@@ -1,78 +0,0 @@
-from __future__ import annotations
-import importlib.util, json, sys, tempfile, unittest
-from pathlib import Path
-
-ROOT = Path(__file__).resolve().parents[1]
-for name, path in (
-    ("evaluate", ROOT / "benchmark" / "evaluate.py"),
-    ("validate_submission", ROOT / "benchmark" / "validate_submission.py"),
-):
-    spec = importlib.util.spec_from_file_location(name, path)
-    module = importlib.util.module_from_spec(spec)
-    sys.modules[name] = module
-    assert spec.loader is not None
-    spec.loader.exec_module(module)
-
-evaluate = sys.modules["evaluate"]
-validator = sys.modules["validate_submission"]
-
-
-class SubmissionValidationTests(unittest.TestCase):
-    def make_bundle(self, root: Path) -> Path:
-        bundle = root / "provider-model-2026-09-07"
-        bundle.mkdir()
-        cases = [{"id": "c1", "domain": "math", "prompt": "1+1?", "accepted_answers": ["2"]}]
-        baseline = [{"case_id": "c1", "answer": "2", "tokens": 10, "tool_calls": 0, "latency_ms": 100}]
-        skill = [{"case_id": "c1", "answer": "2", "tokens": 12, "tool_calls": 0, "latency_ms": 110}]
-        (bundle / "cases.jsonl").write_text(json.dumps(cases[0]) + "\n", encoding="utf-8")
-        (bundle / "baseline.jsonl").write_text(json.dumps(baseline[0]) + "\n", encoding="utf-8")
-        (bundle / "skill.jsonl").write_text(json.dumps(skill[0]) + "\n", encoding="utf-8")
-        meta = {
-            "provider": "example",
-            "model": "model",
-            "run_date": "2026-09-07",
-            "skill_version": "0.3.0",
-            "repetitions_per_case": 1,
-            "failures_recorded": True,
-            "baseline": {"instruction_config": "baseline", "sampling": {}, "tool_availability": [], "context_limit": 1000, "output_limit": 100},
-            "skill": {"instruction_config": "SKILL.md", "sampling": {}, "tool_availability": [], "context_limit": 1000, "output_limit": 100},
-        }
-        (bundle / "metadata.json").write_text(json.dumps(meta), encoding="utf-8")
-        payload = {"skill": evaluate.summarize(cases, skill), "baseline": evaluate.summarize(cases, baseline)}
-        payload["comparison"] = evaluate.compare(payload["baseline"], payload["skill"])
-        (bundle / "comparison.json").write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
-        (bundle / "README.md").write_text("Reproducible test bundle.\n", encoding="utf-8")
-        return bundle
-
-    def test_valid_bundle(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            validator.validate_bundle(self.make_bundle(Path(tmp)))
-
-    def test_missing_artifact_fails(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            bundle = self.make_bundle(Path(tmp))
-            (bundle / "README.md").unlink()
-            with self.assertRaises(ValueError):
-                validator.validate_bundle(bundle)
-
-    def test_tampered_comparison_fails(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            bundle = self.make_bundle(Path(tmp))
-            payload = json.loads((bundle / "comparison.json").read_text())
-            payload["comparison"]["accuracy_delta"] = 99
-            (bundle / "comparison.json").write_text(json.dumps(payload), encoding="utf-8")
-            with self.assertRaises(ValueError):
-                validator.validate_bundle(bundle)
-
-    def test_unrecorded_failures_flag_fails(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            bundle = self.make_bundle(Path(tmp))
-            meta = json.loads((bundle / "metadata.json").read_text())
-            meta["failures_recorded"] = False
-            (bundle / "metadata.json").write_text(json.dumps(meta), encoding="utf-8")
-            with self.assertRaises(ValueError):
-                validator.validate_bundle(bundle)
-
-
-if __name__ == "__main__":
-    unittest.main()