touchstone-platform 1.0.2__py3-none-any.whl

app/__init__.py

@@ -0,0 +1 @@
+# HuggingFace Model Benchmarking Gym

app/agent_tasks.py

@@ -0,0 +1,34 @@
+"""Agent tasks — spec-driven v2. All content comes from Gym Spec v1.0."""
+
+from __future__ import annotations
+
+from app.spec_loader import (
+    get_active_spec,
+    get_artifacts_info,
+    get_roles_info,
+    load_artifact_content,
+)
+
+
+def get_artifacts_list() -> list[dict]:
+    return get_artifacts_info()
+
+def get_artifact_content(name: str) -> str | None:
+    return load_artifact_content(name)
+
+def get_all_roles_info() -> list[dict]:
+    return get_roles_info()
+
+def get_tasks_for_role(agent_id: str) -> list:
+    spec = get_active_spec()
+    return [t for t in spec.tasks if t.agent_id == agent_id]
+
+def build_prompt(task) -> str:
+    prompt = task.prompt
+    for artifact_name in task.required_artifacts:
+        content = load_artifact_content(artifact_name)
+        if content:
+            prompt = prompt.replace(f"{{{artifact_name}}}", content)
+        else:
+            prompt = prompt.replace(f"{{{artifact_name}}}", f"[Artifact {artifact_name} not found]")
+    return prompt

app/audit/__init__.py

@@ -0,0 +1,4 @@
+from .ledger import AuditLedger, audit_ledger
+from .normalizer import InvocationRecord, normalize_artifact, normalize_batch
+from .orchestrator import AuditOrchestrator
+from .task_matcher import TaskMatcher

app/audit/ledger.py

@@ -0,0 +1,186 @@
+"""Immutable audit ledger for SOC2 compliance."""
+
+import hashlib
+import hmac
+import json
+import logging
+import os
+import time
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+AUDIT_LOG_FILE = os.environ.get(
+    "MODEL_GYM_AUDIT_LOG",
+    os.path.join("scratch", "audit", "audit_ledger.jsonl"),
+)
+
+# ---------------------------------------------------------------------------
+# HMAC secret key
+# The key is loaded from the AUDIT_HMAC_SECRET environment variable.
+# In production, set this to a strong random value (e.g. 32+ random bytes,
+# base64-encoded).  If the variable is absent we fall back to a well-known
+# dev-only placeholder and emit a warning — never use this in production.
+# ---------------------------------------------------------------------------
+_SECRET_FROM_ENV: str | None = os.environ.get("AUDIT_HMAC_SECRET")
+_IS_PRODUCTION: bool = os.environ.get("MODEL_GYM_ENV", "development").lower() == "production"
+
+if _SECRET_FROM_ENV:
+    AUDIT_HMAC_KEY: bytes = _SECRET_FROM_ENV.encode()
+elif _IS_PRODUCTION:
+    raise RuntimeError(
+        "AUDIT_HMAC_SECRET must be set in production. "
+        "Generate with: python -c \"import secrets,base64; print(base64.b64encode(secrets.token_bytes(32)).decode())\""
+    )
+else:
+    logger.warning(
+        "AUDIT_HMAC_SECRET env var not set — using insecure dev-only key. "
+        "Set TOUCHSTONE_ENV=production (or MODEL_GYM_ENV=production) to enforce this at startup."
+    )
+    AUDIT_HMAC_KEY = b"dev-only-secret"
+
+
+def _compute_hmac(previous_hash: str, event_json_bytes: bytes) -> str:
+    """Return HMAC-SHA256(key=AUDIT_HMAC_KEY, msg=previous_hash_bytes + event_json_bytes)."""
+    msg = previous_hash.encode() + event_json_bytes
+    return hmac.new(AUDIT_HMAC_KEY, msg, hashlib.sha256).hexdigest()
+
+
+class AuditLedger:
+    """Manages HMAC-signed hash-linked audit entries for processing integrity."""
+
+    def __init__(self, log_file: str = AUDIT_LOG_FILE):
+        self.log_file = Path(log_file)
+        self._ensure_log_exists()
+        # Cache: restored from the last line of the log on startup.
+        self._last_hash: str = "GENESIS"
+        self._last_seq: int = 0
+        self._restore_cache()
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _ensure_log_exists(self):
+        self.log_file.parent.mkdir(parents=True, exist_ok=True)
+        if not self.log_file.exists():
+            self.log_file.touch()
+
+    def _restore_cache(self):
+        """Read only the last line of the log to prime the in-memory cache — O(1)."""
+        if not self.log_file.exists() or self.log_file.stat().st_size == 0:
+            return
+
+        # Efficient last-line read without loading the whole file.
+        last_line: str = ""
+        with open(self.log_file, "rb") as fh:
+            # Seek to end; scan backwards for the previous newline.
+            fh.seek(0, 2)
+            size = fh.tell()
+            if size == 0:
+                return
+            # Step back past any trailing newline.
+            pos = size - 1
+            while pos >= 0:
+                fh.seek(pos)
+                ch = fh.read(1)
+                if ch == b"\n" and pos < size - 1:
+                    break
+                pos -= 1
+            fh.seek(pos + 1)
+            last_line = fh.read().decode("utf-8").strip()
+
+        if last_line:
+            try:
+                data = json.loads(last_line)
+                self._last_hash = data.get("hash", "GENESIS")
+                self._last_seq = data.get("seq", 0)
+            except json.JSONDecodeError:
+                pass  # Leave defaults; verify_integrity() will catch corruption.
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def log_action(self, action: str, actor: str, payload: dict, run_id: str | None = None):
+        """Appends an HMAC-signed, hash-linked entry to the ledger."""
+        previous_hash = self._last_hash
+        seq = self._last_seq + 1
+
+        entry = {
+            "seq": seq,
+            "timestamp": time.time(),
+            "action": action,
+            "actor": actor,
+            "run_id": run_id,
+            "payload": payload,
+            "previous_hash": previous_hash,
+        }
+
+        event_json_bytes = json.dumps(entry, sort_keys=True).encode()
+        entry_hash = _compute_hmac(previous_hash, event_json_bytes)
+
+        with open(self.log_file, "a", encoding="utf-8") as f:
+            f.write(json.dumps({**entry, "hash": entry_hash}) + "\n")
+
+        # Update cache.
+        self._last_hash = entry_hash
+        self._last_seq = seq
+
+        return entry_hash
+
+    # Keep the old method name as an alias for backward compatibility.
+    def log_event(self, action: str, actor: str, payload: dict, run_id: str | None = None):
+        return self.log_action(action, actor, payload, run_id=run_id)
+
+    def verify_integrity(self) -> bool:
+        """Verifies the entire HMAC chain for tampering using constant-time comparison."""
+        if not self.log_file.exists() or self.log_file.stat().st_size == 0:
+            return True
+
+        expected_prev_hash = "GENESIS"
+        with open(self.log_file, encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    data = json.loads(line)
+                except json.JSONDecodeError:
+                    return False
+
+                stored_hash = data.pop("hash", None)
+                if stored_hash is None:
+                    return False
+
+                # Verify the previous-hash link.
+                if data.get("previous_hash") != expected_prev_hash:
+                    return False
+
+                # Recompute HMAC.
+                event_json_bytes = json.dumps(data, sort_keys=True).encode()
+                recomputed_hash = _compute_hmac(expected_prev_hash, event_json_bytes)
+
+                # Constant-time comparison to prevent timing attacks.
+                if not hmac.compare_digest(stored_hash, recomputed_hash):
+                    return False
+
+                expected_prev_hash = stored_hash
+
+        return True
+
+    def get_logs(self, limit: int = 100) -> list[dict]:
+        """Returns the most recent audit logs."""
+        if not self.log_file.exists():
+            return []
+
+        logs = []
+        with open(self.log_file, encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    logs.append(json.loads(line))
+        return logs[-limit:]
+
+
+audit_ledger = AuditLedger()

app/audit/normalizer.py

@@ -0,0 +1,62 @@
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class InvocationRecord:
+    """Normalized representation of a single LLM invocation from any provider."""
+    uid: str                      # from connector artifact["uid"]
+    provider: str                 # "aws_bedrock" | "azure_openai" | "local"
+    model_id: str
+    prompt: str
+    completion: str
+    timestamp_iso: str
+    input_tokens: int = 0
+    output_tokens: int = 0
+    cost_usd: float = 0.0
+    latency_ms: int = 0
+    task_id: str = ""             # populated during matching
+    task_title: str = ""          # populated during matching
+    role: str = ""                # populated during matching
+    metadata: dict[str, Any] = field(default_factory=dict)  # raw source dict
+
+def normalize_artifact(artifact: dict[str, Any]) -> InvocationRecord | None:
+    """
+    Convert a connector artifact dict to an InvocationRecord.
+    Returns None if artifact["type"] != "invocation_log" or prompt is empty.
+    """
+    if artifact.get("type") != "invocation_log":
+        return None
+
+    prompt = artifact.get("prompt", "").strip()
+    if not prompt:
+        return None
+
+    return InvocationRecord(
+        uid=artifact["uid"],
+        provider=artifact.get("provider", "unknown"),
+        model_id=artifact["model_id"],
+        prompt=str(prompt)[:8000],
+        completion=str(artifact.get("completion", ""))[:8000],
+        timestamp_iso=artifact["timestamp_iso"],
+        input_tokens=artifact.get("input_tokens", 0),
+        output_tokens=artifact.get("output_tokens", 0),
+        cost_usd=artifact.get("cost_usd", 0.0),
+        latency_ms=artifact.get("latency_ms", 0),
+        metadata=artifact
+    )
+
+def normalize_batch(artifacts: list[dict[str, Any]]) -> list[InvocationRecord]:
+    """
+    Filter and normalize a list of connector artifacts.
+    Returns list of InvocationRecord, ordered by timestamp_iso ascending.
+    """
+    records = []
+    for art in artifacts:
+        record = normalize_artifact(art)
+        if record:
+            records.append(record)
+
+    # Sort chronologically
+    records.sort(key=lambda x: x.timestamp_iso)
+    return records

app/audit/orchestrator.py

@@ -0,0 +1,117 @@
+import datetime
+import logging
+from typing import Any
+
+from app.audit.normalizer import InvocationRecord, normalize_batch
+from app.audit.task_matcher import TaskMatcher
+from app.connectors.factory import ConnectorFactory
+from app.spec_v1 import GymSpecV1
+
+logger = logging.getLogger(__name__)
+
+class AuditOrchestrator:
+    """
+    Main orchestration engine for Audit Mode.
+    Cohesion point for Ingestion, Normalization, Matching, and Scoring.
+    """
+
+    def __init__(self, spec: GymSpecV1, scorer: Any = None):
+        self.spec = spec
+        if spec.mode != "audit":
+            raise ValueError("GymSpec is not in audit mode")
+
+        self.matcher = TaskMatcher(spec)
+        self.scorer = scorer # Should be an instance of ScoringEngine
+        self.incumbent_config = spec.audit.incumbent
+
+    def run_audit(self) -> dict[str, Any]:
+        """
+        Execute the full audit workflow.
+        """
+        logger.info(f"Starting audit for {self.spec.gym.name}")
+
+        # 1. Ingestion
+        artifacts = self._ingest_logs()
+
+        # 2. Normalization
+        records = normalize_batch(artifacts)
+
+        # 3. Task Matching
+        self.matcher.match_batch(records)
+        stats = self.matcher.match_stats(records)
+
+        # 4. Replay Scoring (Scoring existing outputs)
+        results = []
+        total_score = 0.0
+        scored_count = 0
+
+        for record in records:
+            if not record.task_id:
+                continue
+
+            score_result = self._score_record(record)
+            if score_result:
+                results.append({
+                    "record_uid": record.uid,
+                    "task_id": record.task_id,
+                    "score": score_result.get("overall_score", 0.0),
+                    "cost": record.cost_usd,
+                    "dimensions": score_result.get("dimensions", {})
+                })
+                total_score += score_result.get("overall_score", 0.0)
+                scored_count += 1
+
+        avg_score = total_score / scored_count if scored_count > 0 else 0.0
+        total_cost = sum(r.cost_usd for r in records)
+
+        report = {
+            "spec_id": self.spec.gym.id,
+            "timestamp": datetime.datetime.now(datetime.UTC).isoformat(),
+            "status": "completed",
+            "summary": {
+                "total_invocations": len(records),
+                "matched_invocations": stats["matched"],
+                "scored_invocations": scored_count,
+                "average_score": round(avg_score, 4),
+                "total_cost_usd": round(total_cost, 4),
+                "cost_per_matched_outcome": round(total_cost / scored_count, 4) if scored_count > 0 else 0.0
+            },
+            "results": results,
+            "tasks": stats["by_task"]
+        }
+
+        return report
+
+    def _ingest_logs(self) -> list[dict[str, Any]]:
+        """Pull logs from the configured source."""
+        conn_id = self.incumbent_config.connection_id or "audit_source"
+
+        # Build connector config from incumbent settings
+        config = {
+            "provider": self.incumbent_config.provider,
+            "model_id": self.incumbent_config.model_id,
+            "region_name": self.incumbent_config.region,
+            "log_group_name": self.incumbent_config.log_group,
+            "start_time_iso": self.incumbent_config.billing_start,
+            "end_time_iso": self.incumbent_config.billing_end,
+        }
+
+        connector = ConnectorFactory.get_connector(conn_id, config)
+        return connector.list_artifacts()
+
+    def _score_record(self, record: InvocationRecord) -> dict[str, Any] | None:
+        """Score a single invocation using the provided scorer."""
+        if not self.scorer:
+            # Fallback mock score if no scorer provided (for testing/demo)
+            return {"overall_score": 0.0, "dimensions": {}}
+
+        # The scorer should have a method like score_task(task_id, prompt, completion)
+        try:
+            return self.scorer.score_task(
+                task_id=record.task_id,
+                prompt=record.prompt,
+                completion=record.completion
+            )
+        except Exception as e:
+            logger.error(f"Scoring failed for record {record.uid}: {e!s}")
+            return None

app/audit/task_matcher.py

@@ -0,0 +1,85 @@
+import logging
+from typing import Any
+
+from app.audit.normalizer import InvocationRecord
+from app.spec_v1 import GymSpecV1
+from app.vector_stores import LexicalVectorStore
+
+logger = logging.getLogger(__name__)
+
+class TaskMatcher:
+    """
+    Matches raw LLM invocation prompts to GymSpec task definitions
+    using lexical similarity.
+    """
+
+    def __init__(self, spec: GymSpecV1):
+        self.spec = spec
+        self.tasks = spec.tasks
+        self.vector_store = LexicalVectorStore(chunk_size=200, overlap=20)
+
+        # Build index of task prompts
+        # Index format: "{task.name} {task.prompt}"
+        for task in self.tasks:
+            content = f"{task.name} {task.prompt}"
+            self.vector_store.add_document(
+                run_id="task_index",
+                content=content,
+                metadata={
+                    "task_id": task.id,
+                    "name": task.name,
+                    "agent_id": task.agent_id
+                }
+            )
+
+    def match(self, record: InvocationRecord, top_k: int = 1) -> str | None:
+        """
+        Match a single InvocationRecord to a task by prompt similarity.
+        """
+        results = self.vector_store.search(
+            run_id="task_index",
+            query=record.prompt,
+            top_k=top_k
+        )
+
+        if not results:
+            return None
+
+        best_match = results[0]
+        # Threshold: term overlap frequency > 0.3 (adjusted to avoid false positives)
+        if best_match.get("score", 0) < 0.3:
+            return None
+
+        task_id = best_match["metadata"]["task_id"]
+        record.task_id = task_id
+        record.task_title = best_match["metadata"]["name"]
+        record.role = best_match["metadata"]["agent_id"]
+
+        return task_id
+
+    def match_batch(self, records: list[InvocationRecord]) -> None:
+        """
+        Match all records in place.
+        """
+        for record in records:
+            if not self.match(record):
+                logger.warning(f"Unmatched prompt for record {record.uid}")
+
+    def match_stats(self, records: list[InvocationRecord]) -> dict[str, Any]:
+        """
+        Returns matching statistics.
+        """
+        total = len(records)
+        matched_records = [r for r in records if r.task_id]
+        matched_count = len(matched_records)
+
+        by_task = {}
+        for r in matched_records:
+            by_task[r.task_id] = by_task.get(r.task_id, 0) + 1
+
+        return {
+            "total": total,
+            "matched": matched_count,
+            "unmatched": total - matched_count,
+            "by_task": by_task,
+        }