deqio 0.1.0__py3-none-any.whl

deqio/__init__.py

@@ -0,0 +1,3 @@
+"""Deqio package."""
+
+__version__ = "0.1.0"

deqio/__main__.py

@@ -0,0 +1,5 @@
+from .cli import main
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

deqio/backends.py

@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+import gc
+from collections.abc import Callable
+from typing import Any
+
+from .config import Settings
+
+
+class BackendRuntime:
+    """Stable server-facing scoring interface for a loaded decision runtime."""
+
+    def __init__(
+        self,
+        *,
+        settings: Settings,
+        model: Any,
+        tokenizer: Any,
+        metadata: dict[str, Any],
+        direct_score: Callable[..., dict],
+        serial_factory: Callable[..., Any],
+        shared_score: Callable[..., tuple[list[dict], dict]],
+    ) -> None:
+        self.settings = settings
+        self.name = settings.backend
+        self.engine = "semif"
+        self.model = model
+        self.tokenizer = tokenizer
+        self.metadata = metadata
+        self._direct_score = direct_score
+        self._serial_factory = serial_factory
+        self._shared_score = shared_score
+        self.serial_scorer = self._new_serial_scorer()
+
+    @classmethod
+    def load(cls, settings: Settings) -> "BackendRuntime":
+        # Every model engine, including SemIf, is executed through an isolated
+        # runtime. This keeps the PyPI package free of engine-specific VCS
+        # dependencies and gives all engines the same lifecycle semantics.
+        from .systemone_runtime import SystemOneRuntime
+
+        return SystemOneRuntime.load(settings)  # type: ignore[return-value]
+
+    def _new_serial_scorer(self):
+        return self._serial_factory(
+            self.model,
+            self.tokenizer,
+            self.metadata,
+            self.settings.max_tokens,
+        )
+
+    def score(self, row: dict, mode: str) -> dict:
+        if mode == "serial":
+            return self.serial_scorer.score(row)
+        if mode == "direct":
+            return self._direct_score(
+                self.model,
+                self.tokenizer,
+                row,
+                self.metadata,
+                self.settings.max_tokens,
+            )
+        raise ValueError(f"Unsupported scoring mode: {mode}")
+
+    def score_noul(self, row: dict, mode: str) -> dict:
+        choice_row = {
+            **row,
+            "options": [
+                {"id": "yes", "description": "Yes. The evidence supports the criterion or question."},
+                {"id": "no", "description": "No. The evidence does not support the criterion or question."},
+            ],
+        }
+        return self.score(choice_row, mode)
+
+    def score_shared(self, rows: list[dict]) -> tuple[list[dict], dict]:
+        return self._shared_score(
+            self.model,
+            self.tokenizer,
+            rows,
+            self.metadata,
+            self.settings.max_tokens,
+        )
+
+    def clear_cache(self) -> dict[str, Any]:
+        """Drop reusable prefix state and release backend allocator caches when available."""
+        self.serial_scorer = self._new_serial_scorer()
+        gc.collect()
+
+        details: dict[str, Any] = {
+            "engine": self.engine,
+            "backend": self.name,
+            "prefix_cache": "cleared",
+            "model_loaded": True,
+        }
+
+        if self.name == "mlx":
+            import mlx.core as mx
+
+            before = int(mx.get_cache_memory())
+            mx.clear_cache()
+            after = int(mx.get_cache_memory())
+            details.update(
+                allocator_cache="cleared",
+                allocator_cache_bytes_before=before,
+                allocator_cache_bytes_after=after,
+            )
+        elif self.name == "cuda":
+            import torch
+
+            before = int(torch.cuda.memory_reserved())
+            torch.cuda.empty_cache()
+            after = int(torch.cuda.memory_reserved())
+            details.update(
+                allocator_cache="cleared",
+                allocator_cache_bytes_before=before,
+                allocator_cache_bytes_after=after,
+            )
+        elif self.name == "mps":
+            import torch
+
+            before = int(torch.mps.current_allocated_memory()) if hasattr(torch.mps, "current_allocated_memory") else None
+            torch.mps.empty_cache()
+            after = int(torch.mps.current_allocated_memory()) if hasattr(torch.mps, "current_allocated_memory") else None
+            details.update(
+                allocator_cache="cleared",
+                allocator_cache_bytes_before=before,
+                allocator_cache_bytes_after=after,
+            )
+
+        return details
+
+    def close(self) -> None:
+        """Release native model references before another runtime is loaded.
+
+        Live model switching calls ``close()`` before loading the replacement.
+        Dropping the model, tokenizer, metadata, and serial scorer here avoids
+        temporarily keeping two native models resident in accelerator/unified
+        memory during the switch. Allocator cleanup is best-effort so shutdown
+        cannot fail solely because a backend cache API is unavailable.
+        """
+        self.serial_scorer = None
+        self.model = None
+        self.tokenizer = None
+        self.metadata = {}
+        gc.collect()
+
+        try:
+            if self.name == "mlx":
+                import mlx.core as mx
+
+                mx.clear_cache()
+            elif self.name == "cuda":
+                import torch
+
+                torch.cuda.empty_cache()
+            elif self.name == "mps":
+                import torch
+
+                torch.mps.empty_cache()
+        except Exception:
+            # References are already dropped above. Cache cleanup is only an
+            # allocator hint and must not make model switching/shutdown fail.
+            pass

deqio/benchmark.py

@@ -0,0 +1,353 @@
+from __future__ import annotations
+
+import argparse
+import json
+import statistics
+import sys
+import time
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from .backends import BackendRuntime
+from .catalog import apply_selection, get_model, get_profile, load_catalog
+from .config import read_config_data, settings_from_data
+from .installations import installed_profiles
+
+
+DEFAULT_SUITE = Path("benchmarks/basic.json")
+
+
+@dataclass
+class BenchResult:
+    model_id: str
+    backend: str
+    engine: str
+    case_id: str
+    kind: str
+    passed: bool
+    expected: Any
+    actual: Any
+    latency_ms: float | None
+    assertions: int
+    correct: int
+    top_probability: float | None = None
+    error: str | None = None
+
+    def as_dict(self) -> dict[str, Any]:
+        return self.__dict__.copy()
+
+
+def load_suite(path: Path) -> dict[str, Any]:
+    data = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(data, dict) or not isinstance(data.get("cases"), list):
+        raise RuntimeError("Benchmark suite must contain a top-level cases array")
+    if int(data.get("schema_version", 0)) != 1:
+        raise RuntimeError("Unsupported benchmark schema_version")
+    seen: set[str] = set()
+    counts = {"noul": 0, "choice": 0, "shared": 0}
+    for case in data["cases"]:
+        if not isinstance(case, dict):
+            raise RuntimeError("Every benchmark case must be an object")
+        case_id = str(case.get("id", ""))
+        kind = str(case.get("type", ""))
+        if not case_id or case_id in seen:
+            raise RuntimeError(f"Benchmark case id is missing or duplicated: {case_id!r}")
+        if kind not in counts:
+            raise RuntimeError(f"Unsupported benchmark case type: {kind!r}")
+        seen.add(case_id)
+        counts[kind] += 1
+    return data
+
+
+def _percentile(values: list[float], p: float) -> float | None:
+    if not values:
+        return None
+    ordered = sorted(values)
+    index = round((len(ordered) - 1) * p)
+    return ordered[index]
+
+
+def _choice(raw: dict[str, Any]) -> tuple[str, float]:
+    ids = [str(value) for value in raw["option_ids"]]
+    probs = [float(value) for value in raw["probabilities"]]
+    index = max(range(len(probs)), key=probs.__getitem__)
+    return ids[index], probs[index]
+
+
+def _choice_row(case: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "id": str(case["id"]),
+        "state": case["state"],
+        "question": str(case["question"]),
+        "options": list(case["options"]),
+    }
+
+
+def _run_case(runtime: BackendRuntime, model: dict[str, str], case: dict[str, Any]) -> BenchResult:
+    kind = str(case["type"])
+    started = time.perf_counter()
+    try:
+        if kind == "noul":
+            row = {"id": str(case["id"]), "state": case["state"], "question": str(case["question"])}
+            raw = runtime.score_noul(row, "serial")
+            actual, top = _choice(raw)
+            expected = str(case["expected"])
+            latency_ms = float(raw.get("total_seconds", time.perf_counter() - started)) * 1000.0
+            return BenchResult(**model, case_id=str(case["id"]), kind=kind, passed=actual == expected,
+                               expected=expected, actual=actual, latency_ms=latency_ms,
+                               assertions=1, correct=int(actual == expected), top_probability=top)
+
+        if kind == "choice":
+            raw = runtime.score(_choice_row(case), "serial")
+            actual, top = _choice(raw)
+            expected = str(case["expected"])
+            latency_ms = float(raw.get("total_seconds", time.perf_counter() - started)) * 1000.0
+            return BenchResult(**model, case_id=str(case["id"]), kind=kind, passed=actual == expected,
+                               expected=expected, actual=actual, latency_ms=latency_ms,
+                               assertions=1, correct=int(actual == expected), top_probability=top)
+
+        rows = []
+        expected: list[str] = []
+        for decision in case["decisions"]:
+            rows.append({
+                "id": str(decision["id"]),
+                "state": case["state"],
+                "question": str(decision["question"]),
+                "options": list(decision["options"]),
+            })
+            expected.append(str(decision["expected"]))
+        raw_results, timing = runtime.score_shared(rows)
+        actual = [_choice(raw)[0] for raw in raw_results]
+        correct = sum(a == e for a, e in zip(actual, expected))
+        latency_ms = float(timing.get("total_seconds", time.perf_counter() - started)) * 1000.0
+        return BenchResult(**model, case_id=str(case["id"]), kind=kind, passed=correct == len(expected),
+                           expected=expected, actual=actual, latency_ms=latency_ms,
+                           assertions=len(expected), correct=correct)
+    except Exception as error:
+        return BenchResult(**model, case_id=str(case["id"]), kind=kind, passed=False,
+                           expected=case.get("expected") or [d.get("expected") for d in case.get("decisions", [])],
+                           actual=None, latency_ms=(time.perf_counter() - started) * 1000.0,
+                           assertions=max(1, len(case.get("decisions", []))), correct=0, error=str(error))
+
+
+def summarize(results: list[BenchResult]) -> dict[str, Any]:
+    summary: dict[str, Any] = {}
+    groups: dict[str, list[BenchResult]] = {}
+    for result in results:
+        groups.setdefault(result.kind, []).append(result)
+    all_latencies = [r.latency_ms for r in results if r.latency_ms is not None]
+    assertions = sum(r.assertions for r in results)
+    correct = sum(r.correct for r in results)
+    summary["overall"] = {
+        "cases": len(results),
+        "passed_cases": sum(r.passed for r in results),
+        "case_accuracy": (sum(r.passed for r in results) / len(results)) if results else 0.0,
+        "assertions": assertions,
+        "correct": correct,
+        "decision_accuracy": (correct / assertions) if assertions else 0.0,
+        "mean_ms": statistics.fmean(all_latencies) if all_latencies else None,
+        "median_ms": statistics.median(all_latencies) if all_latencies else None,
+        "p95_ms": _percentile(all_latencies, 0.95),
+    }
+    for kind, rows in groups.items():
+        latencies = [r.latency_ms for r in rows if r.latency_ms is not None]
+        kind_assertions = sum(r.assertions for r in rows)
+        kind_correct = sum(r.correct for r in rows)
+        summary[kind] = {
+            "cases": len(rows),
+            "passed_cases": sum(r.passed for r in rows),
+            "case_accuracy": sum(r.passed for r in rows) / len(rows),
+            "assertions": kind_assertions,
+            "correct": kind_correct,
+            "decision_accuracy": kind_correct / kind_assertions if kind_assertions else 0.0,
+            "mean_ms": statistics.fmean(latencies) if latencies else None,
+            "median_ms": statistics.median(latencies) if latencies else None,
+            "p95_ms": _percentile(latencies, 0.95),
+        }
+    return summary
+
+
+def _installed(config_path: Path, config_data: dict[str, Any], catalog: dict[str, Any]) -> list[dict[str, Any]]:
+    return [row for row in installed_profiles(
+        config_path=config_path,
+        config_data=config_data,
+        catalog=catalog,
+        active_model_id=str(config_data.get("model_id", "")),
+        active_backend=str(config_data.get("backend", "")),
+    ) if row["installed"] and row["host_compatible"]]
+
+
+def _select_profiles(rows: list[dict[str, Any]], args: argparse.Namespace) -> list[dict[str, Any]]:
+    if not rows:
+        raise RuntimeError("No installed model profiles are available on this host")
+    if args.all:
+        return rows
+    if args.model:
+        selected = []
+        for value in args.model:
+            if ":" not in value:
+                raise RuntimeError("--model must use MODEL_ID:BACKEND, for example decider-0.8b:mps")
+            model_id, backend = value.rsplit(":", 1)
+            match = next((row for row in rows if row["model_id"] == model_id and row["backend"] == backend), None)
+            if match is None:
+                raise RuntimeError(f"Installed profile not found: {value}")
+            selected.append(match)
+        return selected
+
+    print("Installed model profiles:")
+    for index, row in enumerate(rows, start=1):
+        print(f"  {index}. {row['label']} — {row['backend']} ({row['engine']})")
+    print("  A. all installed profiles")
+    value = input("Select models [A or comma-separated numbers]: ").strip().lower()
+    if value in {"", "a", "all"}:
+        return rows
+    indexes = []
+    for token in value.split(","):
+        index = int(token.strip()) - 1
+        if index not in range(len(rows)):
+            raise RuntimeError("Invalid benchmark model selection")
+        indexes.append(index)
+    return [rows[index] for index in dict.fromkeys(indexes)]
+
+
+def _runtime_settings(config_path: Path, config_data: dict[str, Any], catalog: dict[str, Any], row: dict[str, Any]):
+    entry = get_model(catalog, str(row["model_id"]))
+    profile = get_profile(catalog, str(row["model_id"]), str(row["backend"]))
+    selected = apply_selection(config_data, entry, profile, str(row["backend"]))
+    return settings_from_data(config_path, selected, apply_environment=False)
+
+
+def _warmup(runtime: BackendRuntime) -> None:
+    row = {
+        "id": "benchmark-warmup",
+        "state": "The service is ready and healthy.",
+        "question": "What is the service state?",
+        "options": [
+            {"id": "ready", "description": "The service is ready."},
+            {"id": "not_ready", "description": "The service is not ready."},
+        ],
+    }
+    runtime.score(row, "serial")
+    runtime.score(row, "serial")
+
+
+def _fmt_ms(value: Any) -> str:
+    return "-" if value is None else f"{float(value):.1f}"
+
+
+def run(args: argparse.Namespace) -> int:
+    config_path, config_data = read_config_data(args.config)
+    catalog_path = Path(str(config_data.get("model_catalog", "models.json")))
+    if not catalog_path.is_absolute():
+        catalog_path = config_path.parent / catalog_path
+    catalog = load_catalog(catalog_path)
+    suite_path = Path(args.suite).expanduser().resolve()
+    suite = load_suite(suite_path)
+    profiles = _select_profiles(_installed(config_path, config_data, catalog), args)
+
+    stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+    output_dir = Path(args.output).expanduser().resolve() if args.output else (config_path.parent / ".deqio" / "benchmarks" / stamp)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    results_path = output_dir / "results.jsonl"
+    summary_path = output_dir / "summary.json"
+
+    print(f"[bench] suite={suite.get('name', suite_path.name)} cases={len(suite['cases'])}")
+    print(f"[bench] models={len(profiles)} output={output_dir}")
+
+    all_summaries: list[dict[str, Any]] = []
+    with results_path.open("w", encoding="utf-8") as log:
+        for profile_row in profiles:
+            settings = _runtime_settings(config_path, config_data, catalog, profile_row)
+            label = f"{settings.model_id}/{settings.backend}"
+            print(f"\n[bench] Loading {label} ({settings.engine})")
+            runtime = None
+            model_results: list[BenchResult] = []
+            load_started = time.perf_counter()
+            try:
+                runtime = BackendRuntime.load(settings)
+                load_ms = (time.perf_counter() - load_started) * 1000.0
+                _warmup(runtime)
+                print(f"[bench] Ready {label} load={load_ms:.1f}ms; warmup complete")
+                totals = {kind: sum(c["type"] == kind for c in suite["cases"]) for kind in ("noul", "choice", "shared")}
+                seen = {"noul": 0, "choice": 0, "shared": 0}
+                for case in suite["cases"]:
+                    kind = str(case["type"])
+                    seen[kind] += 1
+                    result = _run_case(runtime, {"model_id": settings.model_id, "backend": settings.backend, "engine": settings.engine}, case)
+                    model_results.append(result)
+                    log.write(json.dumps(result.as_dict(), ensure_ascii=False) + "\n")
+                    log.flush()
+                    status = "PASS" if result.passed else "FAIL"
+                    detail = f"{result.correct}/{result.assertions}" if kind == "shared" else f"actual={result.actual} expected={result.expected}"
+                    error = f" error={result.error}" if result.error else ""
+                    print(f"[bench] {label:30} {kind:6} {seen[kind]:02}/{totals[kind]:02} {status:4} {detail} {_fmt_ms(result.latency_ms)}ms{error}")
+            except Exception as error:
+                load_ms = (time.perf_counter() - load_started) * 1000.0
+                print(f"[bench] ERROR loading {label}: {error}")
+            finally:
+                if runtime is not None:
+                    runtime.close()
+
+            model_summary = summarize(model_results)
+            all_summaries.append({
+                "model_id": settings.model_id,
+                "backend": settings.backend,
+                "engine": settings.engine,
+                "load_ms": load_ms,
+                "summary": model_summary,
+            })
+
+    document = {
+        "schema_version": 1,
+        "created_at": datetime.now(timezone.utc).isoformat(),
+        "suite": str(suite_path),
+        "suite_name": suite.get("name"),
+        "models": all_summaries,
+    }
+    summary_path.write_text(json.dumps(document, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+
+    print("\nBenchmark summary")
+    print(f"{'MODEL':30} {'TYPE':7} {'CASES':>7} {'ACC':>8} {'DEC ACC':>8} {'MEDIAN':>9} {'P95':>9}")
+    print("-" * 91)
+    for item in all_summaries:
+        model = f"{item['model_id']}:{item['backend']}"
+        for kind in ("overall", "noul", "choice", "shared"):
+            stats = item["summary"].get(kind, {})
+            cases = int(stats.get("cases", 0))
+            acc = float(stats.get("case_accuracy", 0.0)) * 100.0
+            dacc = float(stats.get("decision_accuracy", 0.0)) * 100.0
+            label = "all" if kind == "overall" else kind
+            print(
+                f"{model:30} {label:7} {cases:7d} {acc:7.1f}% {dacc:7.1f}% "
+                f"{_fmt_ms(stats.get('median_ms')):>7}ms {_fmt_ms(stats.get('p95_ms')):>7}ms"
+            )
+    print(f"\n[bench] results: {results_path}")
+    print(f"[bench] summary: {summary_path}")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="deqio benchmark", description="Benchmark installed Deqio model profiles.")
+    parser.add_argument("--config", help="Path to config.json (default: ./config.json)")
+    parser.add_argument("--suite", default=str(DEFAULT_SUITE), help="Benchmark suite JSON file")
+    parser.add_argument("--output", help="Output directory (default: .deqio/benchmarks/<timestamp>)")
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument("--all", action="store_true", help="Benchmark every installed host-compatible profile")
+    group.add_argument("--model", action="append", help="Benchmark MODEL_ID:BACKEND; repeat to select several")
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        return run(args)
+    except (RuntimeError, ValueError, OSError, json.JSONDecodeError) as error:
+        print(f"error: {error}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

deqio/benchmark_store.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+def benchmark_root(config_path: Path) -> Path:
+    return config_path.parent / ".deqio" / "benchmarks"
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as error:
+        raise RuntimeError(f"Invalid JSON in benchmark file {path}: {error}") from error
+    if not isinstance(data, dict):
+        raise RuntimeError(f"Benchmark file must contain a JSON object: {path}")
+    return data
+
+
+def _run_dir(config_path: Path, run_id: str) -> Path:
+    if not run_id or run_id in {".", ".."} or Path(run_id).name != run_id:
+        raise FileNotFoundError(f"Benchmark run not found: {run_id}")
+    root = benchmark_root(config_path)
+    candidate = root / run_id
+    if not candidate.is_dir():
+        raise FileNotFoundError(f"Benchmark run not found: {run_id}")
+    return candidate
+
+
+def _file_iso(path: Path) -> str:
+    return datetime.fromtimestamp(path.stat().st_mtime, tz=timezone.utc).isoformat()
+
+
+def _count_jsonl(path: Path) -> int:
+    if not path.is_file():
+        return 0
+    with path.open("r", encoding="utf-8") as handle:
+        return sum(1 for line in handle if line.strip())
+
+
+def list_benchmark_runs(config_path: Path) -> list[dict[str, Any]]:
+    root = benchmark_root(config_path)
+    if not root.is_dir():
+        return []
+
+    runs: list[dict[str, Any]] = []
+    for directory in root.iterdir():
+        if not directory.is_dir():
+            continue
+        summary_path = directory / "summary.json"
+        results_path = directory / "results.jsonl"
+        if not summary_path.is_file() and not results_path.is_file():
+            continue
+
+        summary: dict[str, Any] | None = None
+        summary_error: str | None = None
+        if summary_path.is_file():
+            try:
+                summary = _read_json(summary_path)
+            except RuntimeError as error:
+                summary_error = str(error)
+
+        created_at = None
+        suite_name = None
+        model_count = None
+        if summary is not None:
+            created_at = summary.get("created_at")
+            suite_name = summary.get("suite_name")
+            models = summary.get("models")
+            if isinstance(models, list):
+                model_count = len(models)
+
+        fallback_path = summary_path if summary_path.is_file() else results_path
+        runs.append({
+            "id": directory.name,
+            "created_at": str(created_at or _file_iso(fallback_path)),
+            "suite_name": suite_name,
+            "models": model_count,
+            "results": _count_jsonl(results_path),
+            "has_summary": summary_path.is_file(),
+            "has_results": results_path.is_file(),
+            "summary_error": summary_error,
+        })
+
+    runs.sort(key=lambda row: (str(row.get("created_at", "")), str(row["id"])), reverse=True)
+    return runs
+
+
+def read_benchmark_summary(config_path: Path, run_id: str) -> dict[str, Any]:
+    path = _run_dir(config_path, run_id) / "summary.json"
+    if not path.is_file():
+        raise FileNotFoundError(f"summary.json is missing for benchmark run: {run_id}")
+    return _read_json(path)
+
+
+def read_benchmark_results(config_path: Path, run_id: str) -> list[dict[str, Any]]:
+    path = _run_dir(config_path, run_id) / "results.jsonl"
+    if not path.is_file():
+        raise FileNotFoundError(f"results.jsonl is missing for benchmark run: {run_id}")
+
+    results: list[dict[str, Any]] = []
+    with path.open("r", encoding="utf-8") as handle:
+        for line_number, line in enumerate(handle, start=1):
+            if not line.strip():
+                continue
+            try:
+                row = json.loads(line)
+            except json.JSONDecodeError as error:
+                raise RuntimeError(
+                    f"Invalid JSONL in benchmark run {run_id} at line {line_number}: {error}"
+                ) from error
+            if not isinstance(row, dict):
+                raise RuntimeError(
+                    f"Benchmark result at line {line_number} must be a JSON object"
+                )
+            results.append(row)
+    return results