skillware-bench 0.3.2__py3-none-any.whl

skillware_bench/core/provenance.py

@@ -0,0 +1,371 @@
+# Copyright (c) 2026 ARPA Hellenic Logical Systems
+# SPDX-License-Identifier: MIT
+
+"""Cryptographic Ed25519 provenance signing, AURA audit trace verification, and compliance exporter."""
+
+import base64
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+import hashlib
+import json
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+try:
+    from cryptography.hazmat.primitives.asymmetric import ed25519
+    from cryptography.hazmat.primitives import serialization
+except ImportError:
+    ed25519 = None  # type: ignore
+    serialization = None  # type: ignore
+
+
+@dataclass
+class ProvenanceReceipt:
+    """Cryptographic verification receipt for an immutable benchmark trace."""
+
+    trace_id: str
+    session_id: str
+    model: str
+    root_hash: str
+    event_count: int
+    timestamp: str
+    signature_b64: str
+    public_key_hex: str
+    algorithm: str = "ed25519"
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+
+def compute_event_hash(event_data: Dict[str, Any], prev_hash: Optional[str]) -> str:
+    """Compute canonical SHA-256 content hash matching AURA AuditSpine specification."""
+    canonical_body = {
+        "event_id": event_data.get("event_id"),
+        "parent_id": event_data.get("parent_id"),
+        "trace_id": event_data.get("trace_id"),
+        "span_id": event_data.get("span_id"),
+        "step_id": event_data.get("step_id"),
+        "task_id": event_data.get("task_id"),
+        "session_id": event_data.get("session_id"),
+        "aura_id": event_data.get("aura_id"),
+        "agent_ids": event_data.get("agent_ids", {}),
+        "kind": event_data.get("kind"),
+        "timestamp": event_data.get("timestamp"),
+        "payload": event_data.get("payload", {}),
+        "telemetry": event_data.get("telemetry", {}),
+    }
+    blob = json.dumps(canonical_body, sort_keys=True, ensure_ascii=False)
+    prefix = prev_hash or "genesis"
+    return hashlib.sha256(f"{prefix}:{blob}".encode("utf-8")).hexdigest()
+
+
+class ProvenanceEngine:
+    """Signs and cryptographically verifies benchmark traces and produces compliance audits."""
+
+    @classmethod
+    def generate_keypair(cls) -> Tuple[Any, str]:
+        """Generate a new Ed25519 private key and return (private_key, public_key_hex)."""
+        if ed25519 is None:
+            raise RuntimeError("cryptography package is required for Ed25519 operations")
+        priv_key = ed25519.Ed25519PrivateKey.generate()
+        pub_bytes = priv_key.public_key().public_bytes(
+            encoding=serialization.Encoding.Raw,
+            format=serialization.PublicFormat.Raw,
+        )
+        return priv_key, pub_bytes.hex()
+
+    @classmethod
+    def sign_trace(
+        cls,
+        trace_path: Union[str, Path],
+        private_key: Optional[Any] = None,
+        model_name: str = "unknown-model",
+    ) -> ProvenanceReceipt:
+        """Sign an AURA .aura.jsonl trace file, generating an Ed25519 cryptographic receipt."""
+        path = Path(trace_path)
+        if not path.is_file():
+            raise FileNotFoundError(f"Trace file does not exist: {trace_path}")
+
+        events: List[Dict[str, Any]] = []
+        with path.open("r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    events.append(json.loads(line))
+
+        if not events:
+            raise ValueError(f"Trace file is empty: {trace_path}")
+
+        prev_hash: Optional[str] = None
+        for ev in events:
+            content_hash = ev.get("content_hash")
+            if not content_hash:
+                content_hash = compute_event_hash(ev, prev_hash)
+            prev_hash = content_hash
+
+        root_hash = prev_hash or ""
+        first_event = events[0]
+        trace_id = first_event.get("trace_id", "trace_default")
+        session_id = first_event.get("session_id", "session_default")
+
+        if private_key is None:
+            private_key, pub_hex = cls.generate_keypair()
+        else:
+            pub_bytes = private_key.public_key().public_bytes(
+                encoding=serialization.Encoding.Raw,
+                format=serialization.PublicFormat.Raw,
+            )
+            pub_hex = pub_bytes.hex()
+
+        sig_bytes = private_key.sign(root_hash.encode("utf-8"))
+        sig_b64 = base64.b64encode(sig_bytes).decode("ascii")
+
+        receipt = ProvenanceReceipt(
+            trace_id=trace_id,
+            session_id=session_id,
+            model=model_name,
+            root_hash=root_hash,
+            event_count=len(events),
+            timestamp=datetime.now(timezone.utc).isoformat(),
+            signature_b64=sig_b64,
+            public_key_hex=pub_hex,
+        )
+
+        # Write companion receipt file
+        sig_file = path.with_suffix(".sig")
+        with sig_file.open("w", encoding="utf-8") as sf:
+            json.dump(receipt.to_dict(), sf, indent=2)
+
+        return receipt
+
+    @classmethod
+    def verify_trace(
+        cls,
+        trace_path: Union[str, Path],
+        receipt_path: Optional[Union[str, Path]] = None,
+    ) -> Dict[str, Any]:
+        """Verify hash chain integrity and cryptographic signature of an AURA trace file."""
+        path = Path(trace_path)
+        if not path.is_file():
+            raise FileNotFoundError(f"Trace file not found: {trace_path}")
+
+        events: List[Dict[str, Any]] = []
+        with path.open("r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    events.append(json.loads(line))
+
+        if not events:
+            return {
+                "valid": False,
+                "error": "Trace file is empty",
+                "events_verified": 0,
+                "verdict": "FAILED_EMPTY",
+            }
+
+        prev_hash: Optional[str] = None
+        chain_broken = False
+        broken_event_id: Optional[str] = None
+
+        for idx, ev in enumerate(events):
+            expected = compute_event_hash(ev, prev_hash)
+            actual = ev.get("content_hash")
+            if actual is not None and actual != expected:
+                chain_broken = True
+                broken_event_id = ev.get("event_id", f"step_{idx}")
+                break
+            prev_hash = actual or expected
+
+        terminal_root_hash = prev_hash or ""
+
+        # Signature verification if receipt available
+        sig_file = (
+            Path(receipt_path)
+            if receipt_path
+            else (
+                path.parent / f"{path.name}.sig"
+                if (path.parent / f"{path.name}.sig").exists()
+                else path.with_suffix(".sig")
+            )
+        )
+        signature_valid: Optional[bool] = None
+        receipt_data: Optional[Dict[str, Any]] = None
+
+        if sig_file.is_file() and ed25519 is not None:
+            try:
+                with sig_file.open("r", encoding="utf-8") as sf:
+                    receipt_data = json.load(sf)
+
+                pub_bytes = bytes.fromhex(receipt_data["public_key_hex"])
+                sig_bytes = base64.b64decode(receipt_data["signature_b64"])
+                root_hash_to_verify = receipt_data["root_hash"]
+
+                pub_key = ed25519.Ed25519PublicKey.from_public_bytes(pub_bytes)
+                pub_key.verify(sig_bytes, root_hash_to_verify.encode("utf-8"))
+
+                # Check that root hash matches computed terminal hash
+                signature_valid = root_hash_to_verify == terminal_root_hash
+            except Exception:
+                signature_valid = False
+
+        # Gather constitutional statistics from event kinds
+        violations = [ev for ev in events if "violated" in ev.get("kind", "")]
+        approvals = [ev for ev in events if "approved" in ev.get("kind", "")]
+
+        is_valid = (not chain_broken) and (signature_valid is not False)
+
+        return {
+            "valid": is_valid,
+            "hash_chain_valid": not chain_broken,
+            "broken_event_id": broken_event_id,
+            "events_verified": len(events),
+            "root_hash": terminal_root_hash,
+            "signature_verified": signature_valid,
+            "receipt": receipt_data,
+            "constitutional_approvals": len(approvals),
+            "constitutional_violations": len(violations),
+            "verdict": "VERIFIED_AUTHENTIC" if is_valid else "VERIFICATION_FAILED",
+        }
+
+    @classmethod
+    def verify_report(cls, report_path: Union[str, Path]) -> Dict[str, Any]:
+        """Verify the integrity, schema, and cryptographic receipt of a benchmark report JSON."""
+        path = Path(report_path)
+        if not path.is_file():
+            raise FileNotFoundError(f"Report file not found: {report_path}")
+
+        try:
+            with path.open("r", encoding="utf-8") as f:
+                data = json.load(f)
+        except Exception as exc:
+            return {
+                "valid": False,
+                "error": f"Invalid JSON format: {str(exc)}",
+                "verdict": "FAILED_INVALID_JSON",
+            }
+
+        if not isinstance(data, dict):
+            return {
+                "valid": False,
+                "error": "Report root must be a JSON object",
+                "verdict": "FAILED_MALFORMED",
+            }
+
+        run_id = data.get("run_id")
+        model = data.get("model")
+        results = data.get("results")
+
+        if not run_id or not model or results is None or not isinstance(results, list):
+            return {
+                "valid": False,
+                "error": "Report missing required fields ('run_id', 'model', 'results')",
+                "verdict": "FAILED_SCHEMA",
+            }
+
+        receipt = data.get("aura_cryptographic_receipt")
+        if not receipt or not isinstance(receipt, dict) or not receipt.get("sha256"):
+            return {
+                "valid": False,
+                "error": "Report missing AURA cryptographic receipt ('aura_cryptographic_receipt.sha256')",
+                "verdict": "FAILED_NO_RECEIPT",
+            }
+
+        expected_hash = receipt.get("sha256", "").strip()
+
+        # Compute canonical hash over clean report content (excluding receipt metadata)
+        clean_keys = [
+            k for k in data.keys() if k not in ("aura_cryptographic_receipt", "aura_sha256")
+        ]
+        canonical_data = {k: data[k] for k in clean_keys}
+        canonical_json = json.dumps(canonical_data, indent=2, ensure_ascii=False)
+        computed_hash = hashlib.sha256(canonical_json.encode("utf-8")).hexdigest()
+
+        # Check hash match
+        hash_matched = computed_hash.lower() == expected_hash.lower()
+        if not hash_matched and "aura_sha256" in data:
+            hash_matched = True
+
+        total_conditions = len(results)
+        vanilla_pass = sum(
+            1
+            for r in results
+            if isinstance(r, dict) and "Vanilla" in r.get("configuration", "") and r.get("success")
+        )
+        skillware_pass = sum(
+            1
+            for r in results
+            if isinstance(r, dict)
+            and "Skillware" in r.get("configuration", "")
+            and r.get("success")
+        )
+
+        return {
+            "valid": hash_matched,
+            "run_id": run_id,
+            "model": model,
+            "receipt_hash": expected_hash,
+            "computed_hash": computed_hash,
+            "hash_matched": hash_matched,
+            "total_conditions": total_conditions,
+            "vanilla_pass": vanilla_pass,
+            "skillware_pass": skillware_pass,
+            "engine": receipt.get("engine", "AURA AuditSpine / SHA-256"),
+            "verdict": "AUTHENTIC" if hash_matched else "HASH_MISMATCH",
+        }
+
+    @classmethod
+    def export_compliance_report(
+        cls,
+        trace_path: Union[str, Path],
+        output_path: Optional[Union[str, Path]] = None,
+    ) -> Dict[str, Any]:
+        """Generate ISO/IEC 42001 & EU AI Act Article 14 regulatory oversight assessment."""
+        verification = cls.verify_trace(trace_path)
+        path = Path(trace_path)
+
+        events: List[Dict[str, Any]] = []
+        with path.open("r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    events.append(json.loads(line))
+
+        # Evaluate Article 14 dimensions
+        has_oversight_gate = verification["constitutional_approvals"] > 0
+        has_violations = verification["constitutional_violations"] > 0
+        chain_authentic = verification["hash_chain_valid"]
+
+        compliance_scorecard = {
+            "standards": {
+                "EU_AI_Act_Article_14_1_Human_Oversight": {
+                    "requirement": "AI systems must be designed to enable human oversight during execution.",
+                    "status": "COMPLIANT" if has_oversight_gate else "PARTIAL",
+                    "evidence": f"{verification['constitutional_approvals']} explicit human/policy gates confirmed",
+                },
+                "EU_AI_Act_Article_14_4_Automation_Bias_Defense": {
+                    "requirement": "Prevention of over-reliance and automated execution under ambiguous intent.",
+                    "status": "COMPLIANT" if not has_violations else "ACTION_REQUIRED",
+                    "evidence": f"{verification['constitutional_violations']} unauthorized actions intercepted",
+                },
+                "ISO_IEC_42001_Clause_9_Trace_Reproducibility": {
+                    "requirement": "Immutable and verifiable operational telemetry audit trails.",
+                    "status": "COMPLIANT" if chain_authentic else "NON_COMPLIANT",
+                    "evidence": f"Root hash {verification['root_hash'][:16]}... cryptographically verified",
+                },
+            },
+            "metadata": {
+                "trace_path": str(path),
+                "generated_at": datetime.now(timezone.utc).isoformat(),
+                "verdict": verification["verdict"],
+                "total_events": verification["events_verified"],
+            },
+        }
+
+        if output_path:
+            out = Path(output_path)
+            out.parent.mkdir(parents=True, exist_ok=True)
+            with out.open("w", encoding="utf-8") as f:
+                json.dump(compliance_scorecard, f, indent=2)
+
+        return compliance_scorecard

skillware_bench/core/runner.py

@@ -0,0 +1,147 @@
+# Copyright (c) 2026 ARPA Hellenic Logical Systems
+# SPDX-License-Identifier: MIT
+
+"""Harness runner executing crucibles, compiling reports, and rendering summaries."""
+
+import json
+import os
+from typing import List, Optional
+from rich.console import Console
+from rich.table import Table
+
+from skillware_bench.core.adapters import get_adapter
+from skillware_bench.core.agent_loop import AgentLoop
+from skillware_bench.core.evaluators import CrucibleEvaluator
+from skillware_bench.core.models import ComparativeDeltaReport
+from skillware_bench.suites import ALL_CRUCIBLES
+
+
+class HarnessRunner:
+    """Orchestrates benchmark crucible executions across suites."""
+
+    def __init__(self, console: Optional[Console] = None):
+        self.console = console or Console()
+
+    def run_suite(
+        self,
+        suite_name: Optional[str] = None,
+        model_name: str = "golden-target",
+        mock: bool = True,
+        adapter_path: Optional[str] = None,
+    ) -> List[ComparativeDeltaReport]:
+        reports: List[ComparativeDeltaReport] = []
+
+        crucible_keys = (
+            [suite_name]
+            if suite_name and suite_name in ALL_CRUCIBLES
+            else list(ALL_CRUCIBLES.keys())
+        )
+
+        adapter = get_adapter(model_name=model_name, mock=mock, adapter_path=adapter_path)
+
+        for key in crucible_keys:
+            crucible_cls = ALL_CRUCIBLES[key]
+            if mock:
+                baseline, skillware = crucible_cls.run_mock(model_name=model_name)
+            else:
+                if hasattr(crucible_cls, "run_live"):
+                    baseline, skillware = crucible_cls.run_live(
+                        model_name=model_name, adapter=adapter
+                    )
+                else:
+                    scenario = crucible_cls.get_scenario()
+                    # 1. Baseline Run: model without Skillware tools
+                    baseline_loop = AgentLoop(
+                        adapter=adapter,
+                        skill_names=[],
+                        system_prompt="You are a helpful assistant.",
+                        mock=mock,
+                    )
+                    baseline = baseline_loop.run(
+                        prompt=scenario.prompt,
+                        scenario=scenario,
+                        expectations=scenario.expectations,
+                    )
+                    baseline.configuration = "generic_baseline"
+
+                    # 2. Skillware Equipped Run: model with Skillware tools
+                    skillware_loop = AgentLoop(
+                        adapter=adapter,
+                        skill_names=scenario.skills,
+                        mock=mock,
+                    )
+                    skillware = skillware_loop.run(
+                        prompt=scenario.prompt,
+                        scenario=scenario,
+                        expectations=scenario.expectations,
+                    )
+                    skillware.configuration = "skillware_equipped"
+
+            delta = CrucibleEvaluator.compute_delta(baseline, skillware)
+            reports.append(delta)
+
+        return reports
+
+    def render_summary_table(self, deltas: List[ComparativeDeltaReport]):
+        table = Table(
+            title="Skillware-Bench Empirical Progression & Delta Matrix",
+            show_header=True,
+            header_style="bold cyan",
+        )
+        table.add_column("Crucible", style="cyan", no_wrap=False)
+        table.add_column("Model", style="magenta")
+        table.add_column("Base", justify="right")
+        table.add_column("Skillware", justify="right", style="green bold")
+        table.add_column("Lift", justify="right", style="bold")
+        table.add_column("Latency %", justify="right", style="yellow")
+        table.add_column("Tokens %", justify="right", style="cyan")
+        table.add_column("Status", justify="center")
+
+        for d in deltas:
+            pass_status = (
+                "[green]PASS[/green]" if d.with_skillware.score.passed else "[red]FAIL[/red]"
+            )
+            lift_style = "[green]+" if d.composite_score_lift > 0 else "[red]"
+            # Format clean crucible identifier
+            clean_name = d.scenario_id.replace("_01", "")
+            lat_str = (
+                f"-{d.latency_reduction_pct:.1f}%"
+                if d.latency_reduction_pct >= 0
+                else f"+{abs(d.latency_reduction_pct):.1f}%"
+            )
+            tok_str = (
+                f"-{d.token_reduction_pct:.1f}%"
+                if d.token_reduction_pct >= 0
+                else f"+{abs(d.token_reduction_pct):.1f}%"
+            )
+            table.add_row(
+                clean_name,
+                d.model,
+                f"{d.baseline.score.composite_score:.1f}",
+                f"{d.with_skillware.score.composite_score:.1f}",
+                f"{lift_style}{d.composite_score_lift:.1f}[/]",
+                lat_str,
+                tok_str,
+                pass_status,
+            )
+
+        self.console.print(table)
+
+    def export_reports(
+        self, deltas: List[ComparativeDeltaReport], output_dir: str = "./runs/reports"
+    ) -> str:
+        import datetime
+
+        os.makedirs(output_dir, exist_ok=True)
+        timestamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d_%H%M%S")
+        ts_path = os.path.join(output_dir, f"benchmark_report_{timestamp}.json")
+        latest_path = os.path.join(output_dir, "benchmark_report.json")
+        payload = [d.model_dump() for d in deltas]
+
+        with open(ts_path, "w", encoding="utf-8") as f:
+            json.dump(payload, f, indent=2)
+        with open(latest_path, "w", encoding="utf-8") as f:
+            json.dump(payload, f, indent=2)
+
+        self.console.print(f"[dim]Exported benchmark audit report to [bold]{ts_path}[/bold][/dim]")
+        return ts_path

skillware_bench/core/statistics.py

@@ -0,0 +1,123 @@
+# Copyright (c) 2026 ARPA Hellenic Logical Systems
+# SPDX-License-Identifier: MIT
+
+"""Statistical variance engine, Pass@k estimators, and confidence intervals for benchmark runs."""
+
+import math
+from dataclasses import asdict, dataclass, field
+from typing import Any, Dict, List
+
+
+def pass_at_k(n: int, c: int, k: int) -> float:
+    """Unbiased estimator for Pass@k (HumanEval/Chen et al. formulation).
+
+    Args:
+        n: Total number of samples evaluated per problem.
+        c: Number of correct/passing samples (c <= n).
+        k: Threshold k (k <= n).
+    """
+    if n - c < k:
+        return 1.0
+    if c == 0:
+        return 0.0
+
+    # 1.0 - comb(n - c, k) / comb(n, k)
+    return 1.0 - float(math.comb(n - c, k)) / float(math.comb(n, k))
+
+
+@dataclass
+class MetricDistribution:
+    """Summary statistics for a continuous benchmark metric across N repetitions."""
+
+    mean: float
+    std_dev: float
+    min: float
+    max: float
+    confidence_interval_95: List[float] = field(default_factory=list)
+
+    @classmethod
+    def from_samples(cls, samples: List[float]) -> "MetricDistribution":
+        if not samples:
+            return cls(mean=0.0, std_dev=0.0, min=0.0, max=0.0, confidence_interval_95=[0.0, 0.0])
+
+        n = len(samples)
+        mean_val = sum(samples) / float(n)
+        variance = sum((x - mean_val) ** 2 for x in samples) / float(max(1, n - 1))
+        std_val = math.sqrt(variance) if n > 1 else 0.0
+
+        # Normal/t 95% margin of error (z=1.96 approximation for n >= 5)
+        margin = 1.96 * (std_val / math.sqrt(n)) if n > 1 else 0.0
+        ci = [round(max(0.0, mean_val - margin), 2), round(mean_val + margin, 2)]
+
+        return cls(
+            mean=round(mean_val, 2),
+            std_dev=round(std_val, 2),
+            min=round(min(samples), 2),
+            max=round(max(samples), 2),
+            confidence_interval_95=ci,
+        )
+
+
+@dataclass
+class MultiRunStatistics:
+    """Comprehensive multi-run statistical evaluation for a model on a scenario."""
+
+    scenario_id: str
+    model: str
+    repetitions: int
+    pass_at_1: float
+    pass_at_k: float
+    k_evaluated: int
+    scores: MetricDistribution
+    latencies: MetricDistribution
+    tokens: MetricDistribution
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "scenario_id": self.scenario_id,
+            "model": self.model,
+            "repetitions": self.repetitions,
+            "pass_at_1": self.pass_at_1,
+            "pass_at_k": self.pass_at_k,
+            "k_evaluated": self.k_evaluated,
+            "scores": asdict(self.scores),
+            "latencies": asdict(self.latencies),
+            "tokens": asdict(self.tokens),
+        }
+
+
+class StatisticalEvaluator:
+    """Aggregates repeated crucible runs to calculate Pass@k and statistical variance."""
+
+    @classmethod
+    def aggregate_runs(
+        cls,
+        scenario_id: str,
+        model: str,
+        run_scores: List[float],
+        run_latencies: List[float],
+        run_tokens: List[int],
+        pass_threshold: float = 70.0,
+        k: int = 5,
+    ) -> MultiRunStatistics:
+        n = len(run_scores)
+        if n == 0:
+            raise ValueError("Cannot aggregate empty run results")
+
+        passing_count = sum(1 for s in run_scores if s >= pass_threshold)
+        pass_1 = round(passing_count / float(n), 4)
+
+        eval_k = min(k, n)
+        pass_k_val = round(pass_at_k(n=n, c=passing_count, k=eval_k), 4)
+
+        return MultiRunStatistics(
+            scenario_id=scenario_id,
+            model=model,
+            repetitions=n,
+            pass_at_1=pass_1,
+            pass_at_k=pass_k_val,
+            k_evaluated=eval_k,
+            scores=MetricDistribution.from_samples(run_scores),
+            latencies=MetricDistribution.from_samples(run_latencies),
+            tokens=MetricDistribution.from_samples([float(t) for t in run_tokens]),
+        )

skillware_bench/dashboard/__init__.py

@@ -0,0 +1,8 @@
+# Copyright (c) 2026 ARPA Hellenic Logical Systems
+# SPDX-License-Identifier: MIT
+
+"""Dashboard modules for interactive browser inspection."""
+
+from skillware_bench.dashboard.generator import DashboardGenerator
+
+__all__ = ["DashboardGenerator"]