cattykit 0.1.0__py3-none-any.whl

cattykit/__init__.py

@@ -0,0 +1,10 @@
+from .experiments import run_experiment
+from .models import available_models, install_model, load_model, model_info
+
+__all__ = [
+    "available_models",
+    "install_model",
+    "load_model",
+    "model_info",
+    "run_experiment",
+]

cattykit/experiments.py

@@ -0,0 +1,99 @@
+from collections import Counter
+from dataclasses import dataclass
+from pathlib import Path
+from datetime import datetime
+import sys
+
+
+from .logging import SQLiteLogger
+from .models import load_model
+
+
+@dataclass
+class ExperimentResult:
+    distributions: dict[str, Counter[str]]
+    logging_db: str
+
+
+def run_experiment(
+    model_name: str,
+    problems: list[str],
+    iterations: int,
+    logging_db: str | Path | None = None,
+    verbose: bool = True,
+) -> ExperimentResult:
+    """Runs the model on each problem for the required number of iterations.
+    The iteration count is used as the random seed given to the model.
+    The same random seeds are used for each problem.
+    If logging_db is not set, the model name and run date/time are used.
+    Counters of answers and the logging_db are returned in ExperimentResult.
+    If verbose, problems and solutions will print to stdout."""
+    interactive = verbose and sys.stdout.isatty()
+    if logging_db is None:
+        now = datetime.now().strftime("%Y%m%d-%H%M%S")
+        logging_db = f"{model_name}-experiments-{now}.sqlite"
+    distributions: dict[str, Counter[str]] = {}
+    logger = SQLiteLogger(logging_db)
+    try:
+        for problem in problems:
+            if interactive:
+                print(problem)
+                display = RunningAnswerDisplay()
+            answers: Counter[str] = Counter()
+            for seed in range(iterations):
+                model = load_model(
+                    model_name,
+                    config={"seed": seed},
+                    logger=logger,
+                )
+                try:
+                    answer = model.solve(problem)
+                    answers[answer] += 1
+                    if interactive:
+                        # print(f"{seed}: {answer}")
+                        display.update(answer)
+                finally:
+                    model.close()
+            distributions[problem] = answers
+    finally:
+        logger.close()
+    return ExperimentResult(
+        distributions=distributions,
+        logging_db=str(logging_db),
+    )
+
+
+def total_variation_distance(
+    observed: dict[str, int], expected: dict[str, int]
+) -> float:
+    observed_total = sum(observed.values())
+    expected_total = sum(expected.values())
+    answers = observed.keys() | expected.keys()
+    return 0.5 * sum(
+        abs(
+            observed.get(answer, 0) / observed_total
+            - expected.get(answer, 0) / expected_total
+        )
+        for answer in answers
+    )
+
+
+class RunningAnswerDisplay:
+    def __init__(self):
+        self.counts = Counter()
+        self.lines_printed = 0
+        self.total = 0
+
+    def update(self, answer: str) -> None:
+        self.counts[answer] += 1
+        self.total += 1
+
+        if self.lines_printed:
+            # Move cursor back up over the previously printed answer lines.
+            print(f"\033[{self.lines_printed}A", end="")
+
+        for answer, count in self.counts.most_common():
+            # Clear line, then rewrite it.
+            print(f"\033[2K{answer}: {count} / {self.total}")
+
+        self.lines_printed = len(self.counts)

cattykit/logging/__init__.py

@@ -0,0 +1,9 @@
+"""Structured logging for CattyKit model runs."""
+
+from .event import ModelEvent
+from .model_logger import ModelLogger
+from .null_logger import NullLogger
+from .print_logger import PrintLogger
+from .sqlite_logger import SQLiteLogger
+
+__all__ = ["ModelEvent", "ModelLogger", "NullLogger", "PrintLogger", "SQLiteLogger"]

cattykit/logging/event.py

@@ -0,0 +1,32 @@
+"""Structured events emitted by CattyKit models."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any
+
+
+@dataclass(frozen=True, slots=True)
+class ModelEvent:
+    """A structured event emitted during a model run."""
+
+    timestamp: datetime
+    model: str
+    kind: str
+    data: Mapping[str, Any]
+
+    @classmethod
+    def create(cls, model: str, kind: str, **data: Any) -> ModelEvent:
+        """Create an event timestamped in UTC."""
+        return cls(timestamp=datetime.now(UTC), model=model, kind=kind, data=data)
+
+    def as_dict(self) -> dict[str, Any]:
+        """Return a JSON-ready representation of the event."""
+        return {
+            "timestamp": self.timestamp.isoformat(),
+            "model": self.model,
+            "kind": self.kind,
+            "data": dict(self.data),
+        }

cattykit/logging/model_logger.py

@@ -0,0 +1,18 @@
+"""Protocol implemented by CattyKit loggers."""
+
+from typing import Protocol, runtime_checkable
+
+from .event import ModelEvent
+
+
+@runtime_checkable
+class ModelLogger(Protocol):
+    """Receives structured events emitted by a model."""
+
+    def log(self, event: ModelEvent) -> None:
+        """Record an event."""
+        ...
+
+    def close(self) -> None:
+        """Release logger resources."""
+        ...

cattykit/logging/null_logger.py

@@ -0,0 +1,13 @@
+"""No-op model logger."""
+
+from .event import ModelEvent
+
+
+class NullLogger:
+    """A logger that discards all events."""
+
+    def log(self, event: ModelEvent) -> None:
+        """Discard an event."""
+
+    def close(self) -> None:
+        """Release no resources."""

cattykit/logging/print_logger.py

@@ -0,0 +1,24 @@
+"""JSON-lines model logger."""
+
+import json
+import sys
+from typing import TextIO
+
+from .event import ModelEvent
+
+
+class PrintLogger:
+    """A logger that writes one JSON event per line to a text stream."""
+
+    def __init__(self, stream: TextIO | None = None) -> None:
+        self._stream = stream if stream is not None else sys.stdout
+
+    def log(self, event: ModelEvent) -> None:
+        """Print an event as a JSON line."""
+        print(
+            json.dumps(event.as_dict(), sort_keys=True), file=self._stream, flush=True
+        )
+
+    def close(self) -> None:
+        """Flush, but do not close, the caller-owned stream."""
+        self._stream.flush()