quantum-reasoning-skill 1.0.0

package/benchmark/evaluate.py

@@ -0,0 +1,257 @@
+"""Evaluate baseline and quantum-reasoning runs from JSONL files.
+
+The evaluator uses only the Python standard library. It does not call a model.
+External runners can write one result row per case and this script will validate,
+score, summarize, and compare those runs without inventing missing measurements.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import statistics
+from pathlib import Path
+from typing import Any, Iterable
+
+
+REQUIRED_RESULT_FIELDS = {
+    "case_id",
+    "answer",
+    "tokens",
+    "tool_calls",
+    "latency_ms",
+}
+
+OPTIONAL_TELEMETRY_FIELDS = {
+    "branches_total",
+    "branches_distinct",
+    "revived_branches",
+    "recovered_errors",
+    "contradictions_found",
+    "contradictions_resolved",
+}
+
+
+def read_jsonl(path: Path) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    with path.open("r", encoding="utf-8") as handle:
+        for line_number, raw in enumerate(handle, start=1):
+            line = raw.strip()
+            if not line:
+                continue
+            try:
+                value = json.loads(line)
+            except json.JSONDecodeError as exc:
+                raise ValueError(f"{path}:{line_number}: invalid JSON: {exc}") from exc
+            if not isinstance(value, dict):
+                raise ValueError(f"{path}:{line_number}: row must be a JSON object")
+            rows.append(value)
+    return rows
+
+
+def normalize_answer(value: Any) -> str:
+    return " ".join(str(value).strip().casefold().split())
+
+
+def validate_case(case: dict[str, Any]) -> None:
+    for field in ("id", "domain", "prompt", "accepted_answers"):
+        if field not in case:
+            raise ValueError(f"case missing required field: {field}")
+    if not isinstance(case["accepted_answers"], list) or not case["accepted_answers"]:
+        raise ValueError(f"case {case['id']!r}: accepted_answers must be a non-empty list")
+
+
+def validate_result(result: dict[str, Any]) -> None:
+    missing = REQUIRED_RESULT_FIELDS - set(result)
+    if missing:
+        raise ValueError(
+            f"result {result.get('case_id', '<unknown>')!r} missing fields: "
+            + ", ".join(sorted(missing))
+        )
+    for field in ("tokens", "tool_calls", "latency_ms"):
+        value = result[field]
+        if not isinstance(value, (int, float)) or value < 0:
+            raise ValueError(f"result {result['case_id']!r}: {field} must be non-negative")
+    for field in OPTIONAL_TELEMETRY_FIELDS:
+        if field in result:
+            value = result[field]
+            if not isinstance(value, (int, float)) or value < 0:
+                raise ValueError(
+                    f"result {result['case_id']!r}: {field} must be non-negative"
+                )
+    if "branches_distinct" in result and "branches_total" in result:
+        if result["branches_distinct"] > result["branches_total"]:
+            raise ValueError(
+                f"result {result['case_id']!r}: branches_distinct cannot exceed branches_total"
+            )
+    if "contradictions_resolved" in result and "contradictions_found" in result:
+        if result["contradictions_resolved"] > result["contradictions_found"]:
+            raise ValueError(
+                f"result {result['case_id']!r}: contradictions_resolved cannot exceed contradictions_found"
+            )
+
+
+def answer_is_correct(case: dict[str, Any], answer: Any) -> bool:
+    normalized = normalize_answer(answer)
+    accepted = {normalize_answer(value) for value in case["accepted_answers"]}
+    return normalized in accepted
+
+
+def safe_mean(values: Iterable[float]) -> float | None:
+    materialized = list(values)
+    if not materialized:
+        return None
+    return statistics.fmean(materialized)
+
+
+def safe_ratio(numerator: float, denominator: float) -> float | None:
+    if denominator == 0:
+        return None
+    return numerator / denominator
+
+
+def summarize(cases: list[dict[str, Any]], results: list[dict[str, Any]]) -> dict[str, Any]:
+    for case in cases:
+        validate_case(case)
+    for result in results:
+        validate_result(result)
+
+    case_by_id = {str(case["id"]): case for case in cases}
+    if len(case_by_id) != len(cases):
+        raise ValueError("case ids must be unique")
+
+    result_by_id: dict[str, dict[str, Any]] = {}
+    for result in results:
+        case_id = str(result["case_id"])
+        if case_id in result_by_id:
+            raise ValueError(f"duplicate result for case_id {case_id!r}")
+        if case_id not in case_by_id:
+            raise ValueError(f"result references unknown case_id {case_id!r}")
+        result_by_id[case_id] = result
+
+    missing_cases = sorted(set(case_by_id) - set(result_by_id))
+    evaluated = []
+    for case_id, result in result_by_id.items():
+        case = case_by_id[case_id]
+        evaluated.append((case, result, answer_is_correct(case, result["answer"])))
+
+    total = len(evaluated)
+    correct = sum(int(is_correct) for _, _, is_correct in evaluated)
+
+    branch_rows = [
+        result
+        for _, result, _ in evaluated
+        if "branches_total" in result and "branches_distinct" in result
+    ]
+    contradiction_rows = [
+        result
+        for _, result, _ in evaluated
+        if "contradictions_found" in result and "contradictions_resolved" in result
+    ]
+
+    total_branches = sum(row["branches_total"] for row in branch_rows)
+    distinct_branches = sum(row["branches_distinct"] for row in branch_rows)
+    contradictions_found = sum(row["contradictions_found"] for row in contradiction_rows)
+    contradictions_resolved = sum(
+        row["contradictions_resolved"] for row in contradiction_rows
+    )
+
+    return {
+        "cases_defined": len(cases),
+        "cases_evaluated": total,
+        "missing_case_ids": missing_cases,
+        "accuracy": safe_ratio(correct, total),
+        "mean_tokens": safe_mean(float(result["tokens"]) for _, result, _ in evaluated),
+        "mean_tool_calls": safe_mean(
+            float(result["tool_calls"]) for _, result, _ in evaluated
+        ),
+        "mean_latency_ms": safe_mean(
+            float(result["latency_ms"]) for _, result, _ in evaluated
+        ),
+        "branch_diversity_ratio": safe_ratio(distinct_branches, total_branches),
+        "revival_rate_per_case": safe_mean(
+            float(result.get("revived_branches", 0)) for _, result, _ in evaluated
+        ),
+        "error_recovery_rate_per_case": safe_mean(
+            float(result.get("recovered_errors", 0)) for _, result, _ in evaluated
+        ),
+        "contradiction_resolution_rate": safe_ratio(
+            contradictions_resolved, contradictions_found
+        ),
+    }
+
+
+def percent_change(new: float | None, old: float | None) -> float | None:
+    if new is None or old is None or old == 0:
+        return None
+    return ((new - old) / old) * 100.0
+
+
+def absolute_change(new: float | None, old: float | None) -> float | None:
+    if new is None or old is None:
+        return None
+    return new - old
+
+
+def compare(baseline: dict[str, Any], skill: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "accuracy_delta": absolute_change(skill["accuracy"], baseline["accuracy"]),
+        "tokens_percent_change": percent_change(
+            skill["mean_tokens"], baseline["mean_tokens"]
+        ),
+        "tool_calls_percent_change": percent_change(
+            skill["mean_tool_calls"], baseline["mean_tool_calls"]
+        ),
+        "latency_percent_change": percent_change(
+            skill["mean_latency_ms"], baseline["mean_latency_ms"]
+        ),
+        "branch_diversity_delta": absolute_change(
+            skill["branch_diversity_ratio"], baseline["branch_diversity_ratio"]
+        ),
+        "contradiction_resolution_delta": absolute_change(
+            skill["contradiction_resolution_rate"],
+            baseline["contradiction_resolution_rate"],
+        ),
+    }
+
+
+def ensure_finite(value: Any, label: str) -> None:
+    if isinstance(value, float) and not math.isfinite(value):
+        raise ValueError(f"{label} is not finite")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--cases", type=Path, required=True)
+    parser.add_argument("--skill", type=Path, required=True)
+    parser.add_argument("--baseline", type=Path)
+    parser.add_argument("--output", type=Path)
+    args = parser.parse_args()
+
+    cases = read_jsonl(args.cases)
+    skill_results = read_jsonl(args.skill)
+    skill_summary = summarize(cases, skill_results)
+
+    payload: dict[str, Any] = {"skill": skill_summary}
+    if args.baseline:
+        baseline_results = read_jsonl(args.baseline)
+        baseline_summary = summarize(cases, baseline_results)
+        payload["baseline"] = baseline_summary
+        payload["comparison"] = compare(baseline_summary, skill_summary)
+
+    for section_name, section in payload.items():
+        if isinstance(section, dict):
+            for key, value in section.items():
+                ensure_finite(value, f"{section_name}.{key}")
+
+    rendered = json.dumps(payload, indent=2, sort_keys=True)
+    print(rendered)
+    if args.output:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(rendered + "\n", encoding="utf-8")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/benchmark/schemas/case.schema.json

@@ -0,0 +1,20 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/Furox-Art/quantum-reasoning-skill/blob/main/benchmark/schemas/case.schema.json",
+  "title": "Quantum Reasoning benchmark case",
+  "type": "object",
+  "additionalProperties": true,
+  "required": ["id", "domain", "prompt", "accepted_answers"],
+  "properties": {
+    "id": {"type": "string", "minLength": 1},
+    "domain": {"type": "string", "minLength": 1},
+    "prompt": {"type": "string", "minLength": 1},
+    "accepted_answers": {
+      "type": "array",
+      "minItems": 1,
+      "items": {}
+    },
+    "source": {"type": ["string", "null"]},
+    "license": {"type": ["string", "null"]}
+  }
+}

package/benchmark/schemas/comparison.schema.json

@@ -0,0 +1,64 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/Furox-Art/quantum-reasoning-skill/blob/main/benchmark/schemas/comparison.schema.json",
+  "title": "Quantum Reasoning benchmark comparison output",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["baseline", "skill", "comparison"],
+  "properties": {
+    "baseline": {"$ref": "#/$defs/summary"},
+    "skill": {"$ref": "#/$defs/summary"},
+    "comparison": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "accuracy_delta",
+        "tokens_percent_change",
+        "tool_calls_percent_change",
+        "latency_percent_change",
+        "branch_diversity_delta",
+        "contradiction_resolution_delta"
+      ],
+      "properties": {
+        "accuracy_delta": {"type": ["number", "null"]},
+        "tokens_percent_change": {"type": ["number", "null"]},
+        "tool_calls_percent_change": {"type": ["number", "null"]},
+        "latency_percent_change": {"type": ["number", "null"]},
+        "branch_diversity_delta": {"type": ["number", "null"]},
+        "contradiction_resolution_delta": {"type": ["number", "null"]}
+      }
+    }
+  },
+  "$defs": {
+    "summary": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "cases_defined",
+        "cases_evaluated",
+        "missing_case_ids",
+        "accuracy",
+        "mean_tokens",
+        "mean_tool_calls",
+        "mean_latency_ms",
+        "branch_diversity_ratio",
+        "revival_rate_per_case",
+        "error_recovery_rate_per_case",
+        "contradiction_resolution_rate"
+      ],
+      "properties": {
+        "cases_defined": {"type": "integer", "minimum": 0},
+        "cases_evaluated": {"type": "integer", "minimum": 0},
+        "missing_case_ids": {"type": "array", "items": {"type": "string"}},
+        "accuracy": {"type": ["number", "null"]},
+        "mean_tokens": {"type": ["number", "null"]},
+        "mean_tool_calls": {"type": ["number", "null"]},
+        "mean_latency_ms": {"type": ["number", "null"]},
+        "branch_diversity_ratio": {"type": ["number", "null"]},
+        "revival_rate_per_case": {"type": ["number", "null"]},
+        "error_recovery_rate_per_case": {"type": ["number", "null"]},
+        "contradiction_resolution_rate": {"type": ["number", "null"]}
+      }
+    }
+  }
+}

package/benchmark/schemas/metadata.schema.json

@@ -0,0 +1,53 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/Furox-Art/quantum-reasoning-skill/blob/main/benchmark/schemas/metadata.schema.json",
+  "title": "Quantum Reasoning community benchmark metadata",
+  "type": "object",
+  "additionalProperties": true,
+  "required": [
+    "provider",
+    "model",
+    "run_date",
+    "skill_version",
+    "repetitions_per_case",
+    "baseline",
+    "skill",
+    "failures_recorded"
+  ],
+  "properties": {
+    "provider": {"type": "string", "minLength": 1},
+    "model": {"type": "string", "minLength": 1},
+    "model_version": {"type": ["string", "null"]},
+    "run_date": {"type": "string", "format": "date"},
+    "skill_version": {"type": "string", "minLength": 1},
+    "skill_commit": {"type": ["string", "null"]},
+    "cases_source": {"type": ["string", "null"]},
+    "repetitions_per_case": {"type": "integer", "minimum": 1},
+    "random_seed": {"type": ["integer", "string", "null"]},
+    "hardware_runtime": {"type": ["string", "null"]},
+    "failures_recorded": {"const": true},
+    "baseline": {"$ref": "#/$defs/condition"},
+    "skill": {"$ref": "#/$defs/condition"},
+    "notes": {"type": ["string", "null"]}
+  },
+  "$defs": {
+    "condition": {
+      "type": "object",
+      "additionalProperties": true,
+      "required": [
+        "instruction_config",
+        "sampling",
+        "tool_availability",
+        "context_limit",
+        "output_limit"
+      ],
+      "properties": {
+        "instruction_config": {"type": "string", "minLength": 1},
+        "sampling": {"type": "object"},
+        "tool_availability": {"type": ["array", "object", "string"]},
+        "context_limit": {"type": ["integer", "null"], "minimum": 1},
+        "output_limit": {"type": ["integer", "null"], "minimum": 1}
+      }
+    }
+  }
+}

package/benchmark/schemas/result.schema.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/Furox-Art/quantum-reasoning-skill/blob/main/benchmark/schemas/result.schema.json",
+  "title": "Quantum Reasoning benchmark result row",
+  "type": "object",
+  "additionalProperties": true,
+  "required": ["case_id", "answer", "tokens", "tool_calls", "latency_ms"],
+  "properties": {
+    "case_id": {"type": "string", "minLength": 1},
+    "answer": {},
+    "tokens": {"type": "number", "minimum": 0},
+    "tool_calls": {"type": "number", "minimum": 0},
+    "latency_ms": {"type": "number", "minimum": 0},
+    "status": {
+      "type": "string",
+      "enum": ["ok", "refusal", "timeout", "error"]
+    },
+    "branches_total": {"type": "number", "minimum": 0},
+    "branches_distinct": {"type": "number", "minimum": 0},
+    "revived_branches": {"type": "number", "minimum": 0},
+    "recovered_errors": {"type": "number", "minimum": 0},
+    "contradictions_found": {"type": "number", "minimum": 0},
+    "contradictions_resolved": {"type": "number", "minimum": 0}
+  }
+}

package/benchmark/validate_submission.py

@@ -0,0 +1,114 @@
+"""Validate reproducible community benchmark result bundles."""
+from __future__ import annotations
+import argparse, json, re
+from datetime import date
+from pathlib import Path
+import evaluate
+
+REQUIRED_FILES = {"metadata.json", "cases.jsonl", "baseline.jsonl", "skill.jsonl", "comparison.json", "README.md"}
+COND_FIELDS = {"instruction_config", "sampling", "tool_availability", "context_limit", "output_limit"}
+META_FIELDS = {"provider", "model", "run_date", "skill_version", "repetitions_per_case", "baseline", "skill", "failures_recorded"}
+
+
+def load_json(path: Path):
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise ValueError(f"{path}: invalid JSON: {exc}") from exc
+
+
+def validate_metadata(meta: dict) -> None:
+    if not isinstance(meta, dict):
+        raise ValueError("metadata.json must contain a JSON object")
+    missing = META_FIELDS - set(meta)
+    if missing:
+        raise ValueError(f"metadata.json missing fields: {sorted(missing)}")
+    for key in ("provider", "model", "skill_version"):
+        if not isinstance(meta[key], str) or not meta[key].strip():
+            raise ValueError(f"metadata.{key} must be a non-empty string")
+    try:
+        date.fromisoformat(meta["run_date"])
+    except Exception as exc:
+        raise ValueError("metadata.run_date must be YYYY-MM-DD") from exc
+    if not isinstance(meta["repetitions_per_case"], int) or meta["repetitions_per_case"] < 1:
+        raise ValueError("metadata.repetitions_per_case must be >= 1")
+    if meta["failures_recorded"] is not True:
+        raise ValueError("metadata.failures_recorded must be true")
+    for name in ("baseline", "skill"):
+        value = meta[name]
+        if not isinstance(value, dict):
+            raise ValueError(f"metadata.{name} must be an object")
+        missing_cond = COND_FIELDS - set(value)
+        if missing_cond:
+            raise ValueError(f"metadata.{name} missing fields: {sorted(missing_cond)}")
+        if not isinstance(value["instruction_config"], str) or not value["instruction_config"].strip():
+            raise ValueError(f"metadata.{name}.instruction_config must be non-empty")
+        if not isinstance(value["sampling"], dict):
+            raise ValueError(f"metadata.{name}.sampling must be an object")
+        for limit in ("context_limit", "output_limit"):
+            v = value[limit]
+            if v is not None and (not isinstance(v, int) or v < 1):
+                raise ValueError(f"metadata.{name}.{limit} must be null or >= 1")
+
+
+def _case_map(cases):
+    out = {}
+    for case in cases:
+        evaluate.validate_case(case)
+        cid = str(case["id"])
+        if cid in out:
+            raise ValueError(f"duplicate case id: {cid}")
+        out[cid] = case
+    return out
+
+
+def validate_bundle(bundle: Path) -> None:
+    missing = REQUIRED_FILES - {p.name for p in bundle.iterdir() if p.is_file()}
+    if missing:
+        raise ValueError(f"{bundle}: missing files: {sorted(missing)}")
+    validate_metadata(load_json(bundle / "metadata.json"))
+    cases = evaluate.read_jsonl(bundle / "cases.jsonl")
+    case_by_id = _case_map(cases)
+    baseline = evaluate.read_jsonl(bundle / "baseline.jsonl")
+    skill = evaluate.read_jsonl(bundle / "skill.jsonl")
+    for row in baseline + skill:
+        evaluate.validate_result(row)
+    for label, rows in (("baseline", baseline), ("skill", skill)):
+        ids = [str(r["case_id"]) for r in rows]
+        if len(ids) != len(set(ids)):
+            raise ValueError(f"{bundle}: duplicate {label} case_id")
+        if set(ids) != set(case_by_id):
+            raise ValueError(f"{bundle}: {label} must contain exactly one row for every case")
+    expected = {
+        "skill": evaluate.summarize(cases, skill),
+        "baseline": evaluate.summarize(cases, baseline),
+    }
+    expected["comparison"] = evaluate.compare(expected["baseline"], expected["skill"])
+    submitted = load_json(bundle / "comparison.json")
+    if submitted != expected:
+        raise ValueError(f"{bundle}: comparison.json does not match evaluator recomputation")
+    if not (bundle / "README.md").read_text(encoding="utf-8").strip():
+        raise ValueError(f"{bundle}: README.md must not be empty")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--root", type=Path, default=Path("benchmark/results/community"))
+    parser.add_argument("--allow-empty", action="store_true")
+    args = parser.parse_args()
+    root = args.root
+    if not root.exists():
+        if args.allow_empty:
+            return 0
+        raise SystemExit(f"community benchmark root not found: {root}")
+    bundles = sorted(p for p in root.iterdir() if p.is_dir())
+    if not bundles and not args.allow_empty:
+        raise SystemExit("no community benchmark bundles found")
+    for bundle in bundles:
+        validate_bundle(bundle)
+        print(f"validated {bundle}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/docs/COMPATIBILITY.md

@@ -0,0 +1,44 @@
+# Host compatibility
+
+Quantum Reasoning is model-agnostic, but not every host exposes the same controls. Compatibility is therefore defined by capabilities rather than by provider branding.
+
+## Compatibility levels
+
+| Level | Required capabilities | What works |
+| --- | --- | --- |
+| Protocol-only | Host can persist or inject `SKILL.md` instructions | Multi-branch protocol, falsification rules, stopping/collapse policy |
+| Measurable | Protocol-only plus aggregate run telemetry | Benchmark result schema, diversity/revival/contradiction measurements |
+| Controller-assisted | Measurable plus a host-side loop that can call `reference/branch_controller.py` or equivalent logic | Deterministic scoring, state transitions, uncertainty-based width and collapse checks |
+
+A host is incompatible only when it cannot reliably apply the skill instructions, truncates/replaces them in a way that changes the protocol, or prevents the requested task/tool access.
+
+## Required host contract
+
+A compatible integration should preserve these invariants:
+
+1. `SKILL.md` is supplied as a persistent system/developer/skill instruction, not as untrusted task content.
+2. The same task constraints and tool permissions apply to all candidate branches.
+3. Tool observations are treated as evidence and are not fabricated.
+4. Private chain-of-thought is not required or stored. Only aggregate control-plane telemetry may be emitted.
+5. If numeric branch telemetry is unavailable, the host follows the qualitative protocol and omits unavailable measurements rather than inventing them.
+6. Baseline-vs-skill benchmarks keep the model/version, tools, sampling and budget policy fixed.
+
+## Integration patterns
+
+### 1. Native skill directory
+
+If a host supports a skill/plugin directory, install the repository so that `SKILL.md` is the skill entry point and keep its YAML front matter intact.
+
+### 2. Persistent instruction wrapper
+
+If the host has no native skill format, load the contents of `SKILL.md` into the persistent system/developer instruction layer before the user task. Do not prepend it to benchmark prompts only in the skill condition unless the baseline wrapper is otherwise identical.
+
+### 3. Controller-assisted agent
+
+A host with an explicit agent loop can map observable aggregate branch measurements to `BranchMetrics` and use `reference/branch_controller.py` for ranking, state changes and collapse decisions. The reference controller does not generate hidden reasoning or call an LLM.
+
+See [`../examples/host-integration.md`](../examples/host-integration.md) for concrete generic examples.
+
+## Provider/model claims
+
+This repository intentionally does not label specific commercial models as "supported" without a reproducible third-party run. Community results may document a provider/model combination under `benchmark/results/community/`, but those results are measurements from the submitter, not a permanent compatibility guarantee.