mic-evals 0.1.0__py3-none-any.whl

mic/__init__.py

@@ -0,0 +1,59 @@
+"""Typed micro-evaluations. Cloud SDKs and credentials are loaded only on demand."""
+
+from ._runtime.materialization import ainspect_dataset, inspect_dataset
+from .decorators import case_schema, dataset, eval, scorer
+from .errors import ConfigurationError, DatasetError, MicError, MissingExpectedError
+from .models import (
+    MISSING,
+    CaseSchema,
+    Dataset,
+    Evaluation,
+    JsonObject,
+    JsonValue,
+    Missing,
+    RawCase,
+    ReadLimits,
+    RunResult,
+    Score,
+    ScoreContext,
+    Scorer,
+    TaskContext,
+    TaskResult,
+)
+from .runner import apreflight, arun, preflight, run
+from .schema import Schema, SchemaError, schema
+
+__all__ = [
+    "MISSING",
+    "CaseSchema",
+    "ConfigurationError",
+    "Dataset",
+    "DatasetError",
+    "Evaluation",
+    "JsonObject",
+    "JsonValue",
+    "MicError",
+    "Missing",
+    "MissingExpectedError",
+    "RawCase",
+    "ReadLimits",
+    "RunResult",
+    "Schema",
+    "SchemaError",
+    "schema",
+    "Score",
+    "ScoreContext",
+    "Scorer",
+    "TaskContext",
+    "TaskResult",
+    "ainspect_dataset",
+    "apreflight",
+    "arun",
+    "case_schema",
+    "dataset",
+    "eval",
+    "inspect_dataset",
+    "preflight",
+    "run",
+    "scorer",
+]

mic/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

mic/_async.py

@@ -0,0 +1,36 @@
+"""Cancellation-safe joins for resources that must outlive their worker."""
+
+import asyncio
+from collections.abc import Callable, Coroutine
+from typing import Any
+
+from .errors import ConfigurationError
+
+
+def run_sync[T](
+    sync_name: str, async_name: str, factory: Callable[[], Coroutine[Any, Any, T]]
+) -> T:
+    """Run an async implementation from synchronous code without leaking a coroutine."""
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        pass
+    else:
+        raise ConfigurationError(
+            f"{sync_name} cannot run inside an event loop; use 'await {async_name}(...)'"
+        )
+    return asyncio.run(factory())
+
+
+async def drain[T](future: asyncio.Future[T]) -> T:
+    """Join after cancellation has been recorded, tolerating further cancellation.
+
+    Shielding prevents the caller from cancelling the worker. Cancellation of the
+    worker itself still propagates; it cannot make progress and must not be retried.
+    """
+    while True:
+        try:
+            return await asyncio.shield(future)
+        except asyncio.CancelledError:
+            if future.done():
+                return future.result()

mic/_runtime/__init__.py

@@ -0,0 +1 @@
+"""Internal evaluation lifecycle, scheduling and case execution."""

mic/_runtime/artifacts.py

@@ -0,0 +1,177 @@
+"""Local evidence is written before optional remote reporting."""
+
+import hashlib
+import importlib.metadata
+import platform
+import sys
+import time
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import cast
+
+from ..errors import ConfigurationError
+from ..models import JsonObject, JsonValue
+from .files import atomic_write
+from .validation import dumps, json_object
+
+
+def atomic_json(path: Path, value: JsonObject) -> None:
+    atomic_write(path, (dumps(value) + "\n",))
+
+
+def prepare_directory(path: Path) -> None:
+    if path.exists() and (not path.is_dir() or any(path.iterdir())):
+        raise ConfigurationError(f"Output directory must be empty: {path}")
+    path.mkdir(parents=True, exist_ok=True)
+
+
+def write_dataset(path: Path, rows: Sequence[JsonObject]) -> None:
+    atomic_write(path, (dumps(row) + "\n" for row in rows))
+
+
+def append_case(path: Path, case: JsonObject) -> None:
+    with path.open("a", encoding="utf-8") as stream:
+        stream.write(dumps(case) + "\n")
+
+
+@dataclass
+class Persistence:
+    """Successful writes and all failed attempts, without claiming disk recovery."""
+
+    written: set[str] = field(default_factory=set[str])
+    errors: list[tuple[str, Exception]] = field(default_factory=list[tuple[str, Exception]])
+
+    def annotate(self, exc: BaseException, path: Path) -> None:
+        if self.errors:
+            exc.add_note(f"Local evidence in {path} may be incomplete or stale")
+            for name, error in self.errors:
+                exc.add_note(f"Could not persist {name}: {type(error).__name__}: {error}")
+        elif "report.html" in self.written:
+            exc.add_note(f"Local evidence: {path / 'report.html'}")
+
+
+def has_artifact_failure(manifest: JsonObject) -> bool:
+    return any(
+        isinstance(error, dict) and error.get("phase") == "artifact"
+        for error in cast(list[JsonValue], manifest["failures"])
+    )
+
+
+def end_run(manifest: JsonObject, *, started: float) -> None:
+    """Stamp terminal timing even when no output directory could be acquired."""
+    manifest["ended_at"] = datetime.now(UTC).isoformat()
+    manifest["duration_ms"] = (time.perf_counter() - started) * 1000
+
+
+def finish_artifacts(
+    path: Path, manifest: JsonObject, cases: Sequence[JsonObject], *, started: float
+) -> Persistence:
+    """Save a terminal outcome and attempt one bounded recovery after write failures.
+
+    Both destinations are attempted independently. Recovery reprojects the updated
+    failure manifest; persistent faults may leave disk behind the in-memory result.
+    Setup errors (2) and cancellation (130) retain their original precedence.
+    """
+    from ..reporters.html import render_report
+
+    end_run(manifest, started=started)
+    result = Persistence()
+    for _ in range(2):
+        errors: list[tuple[str, Exception]] = []
+        for name in ("run.json", "report.html"):
+            try:
+                if name == "run.json":
+                    atomic_json(path / name, manifest)
+                else:
+                    atomic_write(path / name, (render_report(manifest, cases),))
+                result.written.add(name)
+            except Exception as exc:
+                # Rendering is an artifact boundary too. BaseException (including
+                # cancellation) deliberately remains outside this recovery path.
+                errors.append((name, exc))
+                result.written.discard(name)
+        if not errors:
+            break
+        result.errors.extend(errors)
+        for name, exc in errors:
+            record = failure("artifact", exc)
+            record["message"] = f"Could not persist {name}: {exc}"
+            cast(list[JsonValue], manifest["failures"]).append(record)
+        if manifest["exit_code"] not in (2, 130):
+            manifest.update({"status": "failed", "exit_code": 1})
+    return result
+
+
+def code_provenance(fn: object) -> JsonObject:
+    module_name = str(getattr(fn, "__module__", "unknown"))
+    symbol = str(getattr(fn, "__qualname__", getattr(fn, "__name__", "unknown")))
+    try:
+        package_version = importlib.metadata.version("mic-evals")
+    except importlib.metadata.PackageNotFoundError:
+        package_version = "uninstalled"
+    metadata: JsonObject = {
+        "python": platform.python_version(),
+        "package_version": package_version,
+        "definition": f"{module_name}:{symbol}",
+        "dependencies": {},
+    }
+    # Record optional schema/backend versions only when they were actually loaded.
+    dependencies: JsonObject = {}
+    for distribution, module_prefix in (("pydantic", "pydantic"),):
+        if module_prefix in sys.modules:
+            try:
+                dependencies[distribution] = importlib.metadata.version(distribution)
+            except importlib.metadata.PackageNotFoundError:
+                pass
+    metadata["dependencies"] = dependencies
+    module = sys.modules.get(module_name)
+    module_file = getattr(module, "__file__", None)
+    if module_file is not None:
+        source_path = Path(module_file).resolve()
+        if source_path.is_file():
+            metadata["source_path"] = str(source_path)
+            metadata["source_sha256"] = hashlib.sha256(source_path.read_bytes()).hexdigest()
+    # A framework content hash also works outside Git and is meaningful for an
+    # editable install with uncommitted changes.
+    package = Path(__file__).parents[1]
+    hasher = hashlib.sha256()
+    for source in sorted(package.rglob("*.py")):
+        hasher.update(str(source.relative_to(package)).encode())
+        hasher.update(source.read_bytes())
+    metadata["framework_sha256"] = hasher.hexdigest()
+    return metadata
+
+
+def failure(
+    phase: str,
+    exc: BaseException,
+    *,
+    row_index: int | None = None,
+    case_id: str | None = None,
+    trial: int | None = None,
+    scorer: str | None = None,
+) -> JsonObject:
+    import traceback
+
+    result: JsonObject = {
+        "phase": phase,
+        "type": type(exc).__name__,
+        "message": str(exc),
+        "row_index": row_index,
+        "case_id": case_id,
+        "trial": trial,
+        "traceback": "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)),
+    }
+    if scorer is not None:
+        result["scorer"] = scorer
+    return result
+
+
+def case_errors(cases: Sequence[JsonObject]) -> list[JsonValue]:
+    errors: list[JsonValue] = []
+    for case in cases:
+        for error in cast(list[JsonValue], case.get("errors", [])):
+            errors.append(json_object(error))
+    return errors

mic/_runtime/batch.py

@@ -0,0 +1,78 @@
+"""Admit bounded trials, journal completion and join workers before returning."""
+
+import asyncio
+from dataclasses import dataclass
+from pathlib import Path
+from typing import cast
+
+from .._async import drain
+from ..models import Evaluation, JsonObject
+from .artifacts import append_case
+from .callbacks import CallbackPool
+from .case import run_case
+from .materialization import DatasetSnapshot
+from .options import Options
+
+
+@dataclass
+class BatchResult:
+    cases: list[JsonObject]
+    interrupted: asyncio.CancelledError | None
+    error: Exception | None
+
+
+async def execute[I, O, E, M](
+    spec: Evaluation[I, O, E, M],
+    snapshot: DatasetSnapshot[I, E, M],
+    options: Options,
+    destination: Path,
+) -> BatchResult:
+    planned = len(snapshot.cases) * options.trials
+    cases: list[JsonObject] = []
+    pool = CallbackPool(options.concurrency)
+    next_job = 0
+    interrupted: asyncio.CancelledError | None = None
+    runtime_error: Exception | None = None
+
+    async def worker() -> None:
+        nonlocal next_job
+        while next_job < planned:
+            sequence = next_job
+            next_job += 1
+            row_index, trial_index = divmod(sequence, options.trials)
+            case = await run_case(
+                spec,
+                snapshot.cases[row_index],
+                snapshot.rows[row_index],
+                row_index,
+                trial_index + 1,
+                options,
+                pool,
+            )
+            cases.append(case)
+            append_case(destination / "cases.jsonl", case)
+            if case["status"] == "cancelled":
+                raise asyncio.CancelledError(f"Case {case['case_id']} was cancelled")
+
+    workers = [asyncio.create_task(worker()) for _ in range(min(options.concurrency, planned))]
+    try:
+        await asyncio.gather(*workers)
+    except asyncio.CancelledError as exc:
+        interrupted = exc
+        for worker_task in workers:
+            if not worker_task.done():
+                worker_task.cancel()
+        await drain(asyncio.gather(*workers, return_exceptions=True))
+    except Exception as exc:
+        runtime_error = exc
+        for worker_task in workers:
+            if not worker_task.done():
+                worker_task.cancel()
+        await drain(asyncio.gather(*workers, return_exceptions=True))
+    finally:
+        try:
+            await pool.close()
+        except asyncio.CancelledError as exc:
+            interrupted = exc
+    cases.sort(key=lambda value: (cast(int, value["row_index"]), cast(int, value["trial"])))
+    return BatchResult(cases, interrupted, runtime_error)

mic/_runtime/callbacks.py

@@ -0,0 +1,51 @@
+"""Bounded dispatch of synchronous and asynchronous callbacks."""
+
+import asyncio
+import inspect
+from collections.abc import Awaitable, Callable
+from concurrent.futures import ThreadPoolExecutor
+from functools import partial
+from typing import cast
+
+from .._async import drain
+
+
+class CallbackPool:
+    def __init__(self, concurrency: int) -> None:
+        self.executor = ThreadPoolExecutor(max_workers=concurrency, thread_name_prefix="mic")
+
+    async def invoke[T, **P](
+        self,
+        fn: Callable[P, T | Awaitable[T]],
+        *args: P.args,
+        **kwargs: P.kwargs,
+    ) -> T:
+        if inspect.iscoroutinefunction(fn):
+            return await cast(Awaitable[T], fn(*args, **kwargs))
+        loop = asyncio.get_running_loop()
+        future = loop.run_in_executor(self.executor, partial(fn, *args, **kwargs))
+        try:
+            result = await asyncio.shield(future)
+        except asyncio.CancelledError:
+            # A running Python thread cannot be killed. Retain this worker slot
+            # and drain it rather than admitting replacements under a false bound.
+            try:
+                abandoned = await drain(future)
+                if inspect.iscoroutine(abandoned):
+                    abandoned.close()
+                elif isinstance(abandoned, asyncio.Future):
+                    abandoned.cancel()
+            except Exception:
+                pass
+            raise
+        return await cast(Awaitable[T], result) if inspect.isawaitable(result) else cast(T, result)
+
+    async def close(self) -> None:
+        closing = asyncio.create_task(
+            asyncio.to_thread(self.executor.shutdown, wait=True, cancel_futures=True)
+        )
+        try:
+            await asyncio.shield(closing)
+        except asyncio.CancelledError:
+            await drain(closing)
+            raise

mic/_runtime/case.py

@@ -0,0 +1,111 @@
+"""Execute one isolated trial and preserve its successful scores and failures."""
+
+import asyncio
+import copy
+import time
+from typing import cast
+
+from ..models import Evaluation, JsonObject, JsonValue, ScoreContext, TaskContext, TaskResult
+from .artifacts import failure
+from .callbacks import CallbackPool
+from .contracts import Case
+from .options import Options
+from .validation import json_object, normalize_score, serialize, validate
+
+
+async def run_case[I, O, E, M](
+    spec: Evaluation[I, O, E, M],
+    row: Case[I, E, M],
+    encoded: JsonObject,
+    row_index: int,
+    trial: int,
+    options: Options,
+    pool: CallbackPool,
+) -> JsonObject:
+    result: JsonObject = copy.deepcopy(encoded)
+    result.pop("id", None)
+    scores: list[JsonValue] = []
+    errors: list[JsonValue] = []
+    latencies: JsonObject = {"task_ms": 0.0, "scoring_ms": 0.0, "total_ms": 0.0}
+    result.update(
+        {
+            "case_id": row.id,
+            "row_index": row_index,
+            "trial": trial,
+            "status": "completed",
+            "scores": scores,
+            "errors": errors,
+            "latency": latencies,
+            "provenance": copy.deepcopy(row.provenance),
+        }
+    )
+    started = time.perf_counter()
+    phase = "task"
+    scorer_name: str | None = None
+    phase_started = started
+    try:
+        async with asyncio.timeout(options.timeout):
+            cloned = copy.deepcopy(row)
+            ctx = TaskContext(cloned.id, trial, cloned.expected, cloned.metadata)
+            raw = await pool.invoke(spec.function, ctx, cloned.input)
+            latencies["task_ms"] = (time.perf_counter() - phase_started) * 1000
+            phase = "schema"
+            metadata = copy.deepcopy(row.metadata)
+            if isinstance(raw, TaskResult):
+                enriched = cast(TaskResult[O], raw)
+                output_raw = enriched.output
+                task_metadata = json_object(enriched.metadata, "$.task_metadata")
+                result["task_metadata"] = task_metadata
+                merged = (
+                    {}
+                    if metadata is None
+                    else json_object(serialize(spec.dataset.schema.metadata, metadata))
+                )
+                merged.update(task_metadata)
+                metadata = validate(spec.dataset.schema.metadata, merged)
+                json_object(serialize(spec.dataset.schema.metadata, metadata), "$.metadata")
+            else:
+                output_raw = raw
+            output = validate(spec.output, output_raw)
+            result["output"] = serialize(spec.output, output)
+            phase = "scorer"
+            phase_started = time.perf_counter()
+            for scorer in spec.scorers:
+                scorer_name = scorer.name
+                context = ScoreContext(
+                    copy.deepcopy(row.input),
+                    copy.deepcopy(output),
+                    copy.deepcopy(row.expected),
+                    copy.deepcopy(metadata),
+                    row.id,
+                    trial,
+                )
+                raw_score = await pool.invoke(scorer.function, context)
+                scores.append(normalize_score(raw_score, scorer.name))
+            latencies["scoring_ms"] = (time.perf_counter() - phase_started) * 1000
+    except asyncio.CancelledError as exc:
+        result["status"] = "cancelled"
+        errors.append(
+            failure(
+                "cancelled",
+                exc,
+                row_index=row_index,
+                case_id=row.id,
+                trial=trial,
+                scorer=scorer_name,
+            )
+        )
+    except Exception as exc:
+        result["status"] = "failed"
+        errors.append(
+            failure(
+                phase, exc, row_index=row_index, case_id=row.id, trial=trial, scorer=scorer_name
+            )
+        )
+    finally:
+        if phase == "task" and latencies["task_ms"] == 0:
+            latencies["task_ms"] = (time.perf_counter() - phase_started) * 1000
+        elif phase == "scorer" and latencies["scoring_ms"] == 0:
+            latencies["scoring_ms"] = (time.perf_counter() - phase_started) * 1000
+        latencies["total_ms"] = (time.perf_counter() - started) * 1000
+    return result

mic/_runtime/contracts.py

@@ -0,0 +1,14 @@
+"""Internal normalized values shared by materialization and execution."""
+
+from dataclasses import dataclass, field
+
+from ..models import JsonObject, JsonValue, Missing
+
+
+@dataclass(frozen=True)
+class Case[I, E, M]:
+    id: str
+    input: I
+    expected: E | Missing
+    metadata: M | None
+    provenance: JsonObject = field(default_factory=dict[str, JsonValue])

mic/_runtime/discovery.py

@@ -0,0 +1,57 @@
+"""Explicit-module discovery; descriptor inspection never invokes a dataset factory."""
+
+import importlib
+from dataclasses import dataclass
+from typing import Any
+
+import mic
+from mic.errors import ConfigurationError
+
+type Definition = (
+    mic.Dataset[Any, Any, Any] | mic.Evaluation[Any, Any, Any, Any] | mic.Scorer[Any, Any, Any, Any]
+)
+
+
+@dataclass(frozen=True)
+class DefinitionInfo:
+    kind: str
+    name: str
+    selector: str
+    definition: Definition
+
+
+def _kind(value: object) -> str | None:
+    if isinstance(value, mic.Dataset):
+        return "dataset"
+    if isinstance(value, mic.Evaluation):
+        return "eval"
+    if isinstance(value, mic.Scorer):
+        return "scorer"
+    return None
+
+
+def list_definitions(module_name: str) -> list[DefinitionInfo]:
+    """Import only the requested trusted Python module, then inspect its exports."""
+    try:
+        module = importlib.import_module(module_name)
+    except Exception as exc:
+        raise ConfigurationError(f"Cannot import {module_name!r}: {exc}") from exc
+    result: list[DefinitionInfo] = []
+    for symbol, value in sorted(vars(module).items()):
+        kind = _kind(value)
+        if not symbol.startswith("_") and kind is not None:
+            result.append(DefinitionInfo(kind, value.name, f"{module_name}:{symbol}", value))
+    return result
+
+
+def resolve_definition(selector: str) -> Definition:
+    """Exact ``module:symbol`` selection; no substring matching or global registry."""
+    if selector.count(":") != 1:
+        raise ConfigurationError("Select an exact definition with module:symbol")
+    module_name, symbol = selector.split(":")
+    if not module_name or not symbol.isidentifier() or symbol.startswith("_"):
+        raise ConfigurationError("Select an exact public definition with module:symbol")
+    for definition in list_definitions(module_name):
+        if definition.selector == selector:
+            return definition.definition
+    raise ConfigurationError(f"No mic definition found at {selector!r}")