minima-cli 0.4.9__py3-none-any.whl

minima/metrics/report.py

@@ -0,0 +1,116 @@
+"""Ops-side calibration/savings report over the shared decision-log DB.
+
+Usage: ``minima-calibration-report [--days 30] [--org <org_id>]``. Reads the SQLite
+decision log at MINIMA_SQLITE_PATH directly (all orgs unless --org), so it works
+without the API and without a tenant key. With the in-memory store there is nothing
+to read across processes — run the service with MINIMA_RECOMMENDATION_STORE=sqlite.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import time
+from dataclasses import asdict
+
+from minima.config import get_settings
+from minima.metrics.calibration import calibration_by_task_type, cusum_flags, routing_health
+from minima.metrics.savings import summarize
+from minima.recommender.decisionlog import DecisionRecord, SqliteDecisionLog
+
+_SECONDS_PER_DAY = 86_400.0
+
+
+def _print_org(org_id: str, rows: list[DecisionRecord], settings) -> None:  # noqa: ANN001
+    health = routing_health(rows)
+    reports = calibration_by_task_type(
+        rows,
+        n_bins=settings.minima_calibration_bins,
+        shrinkage_k=settings.minima_calibration_shrinkage_k,
+    )
+    flags = cusum_flags(rows, k=settings.minima_cusum_k, h=settings.minima_cusum_h)
+    savings = summarize(rows)
+
+    print(f"\n=== org: {org_id} ===")
+    print(f"health:   {json.dumps(health)}")
+    est, real = savings.estimated, savings.realized
+    print(
+        f"savings:  est vs premium ${est.savings_vs_premium_usd:.4f} over {est.n} recs"
+        f" | est vs declared ${est.savings_vs_declared_usd:.4f} over {est.n_declared}"
+        f" | realized(vs est premium) ${real.savings_vs_premium_est_usd:.4f}"
+        f" over {real.n_reconciled} reconciled"
+    )
+    print("calibration (ECE by task_type, shrunk toward global):")
+    for r in reports:
+        print(
+            f"  {r.slice_key:<16} n={r.n:<6} ece={r.ece:.4f}"
+            f" shrunk={r.ece_shrunk:.4f} quality={r.ece_quality:.4f}"
+        )
+    if flags:
+        print("drift flags (CUSUM):")
+        for f in flags[:20]:
+            print(
+                f"  {f.cluster} / {f.model_id}: {f.direction}"
+                f" (S={f.statistic}, n={f.n})"
+            )
+    else:
+        print("drift flags: none")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--days", type=float, default=30.0)
+    parser.add_argument("--org", default=None, help="restrict to one org id")
+    parser.add_argument("--json", action="store_true", help="emit raw JSON instead of text")
+    args = parser.parse_args()
+
+    settings = get_settings()
+    backend = SqliteDecisionLog(
+        settings.minima_sqlite_path, settings.minima_decision_log_retention_days
+    )
+    since = time.time() - args.days * _SECONDS_PER_DAY
+    rows = backend.rows(since=since, org_id=args.org)
+    if not rows:
+        print(
+            f"no decisions in the last {args.days:g} days"
+            f" (db: {settings.minima_sqlite_path}"
+            f"{', org: ' + args.org if args.org else ''})"
+        )
+        return
+
+    by_org: dict[str, list[DecisionRecord]] = {}
+    for r in rows:
+        by_org.setdefault(r.org_id, []).append(r)
+
+    if args.json:
+        payload = {
+            org: {
+                "health": routing_health(org_rows),
+                "savings": asdict(summarize(org_rows)),
+                "calibration": [
+                    asdict(rep)
+                    for rep in calibration_by_task_type(
+                        org_rows,
+                        n_bins=settings.minima_calibration_bins,
+                        shrinkage_k=settings.minima_calibration_shrinkage_k,
+                    )
+                ],
+                "drift_flags": [
+                    asdict(f)
+                    for f in cusum_flags(
+                        org_rows, k=settings.minima_cusum_k, h=settings.minima_cusum_h
+                    )
+                ],
+            }
+            for org, org_rows in sorted(by_org.items())
+        }
+        print(json.dumps(payload, indent=2))
+        return
+
+    print(f"decision-log report — last {args.days:g} days, {len(rows)} decisions")
+    for org, org_rows in sorted(by_org.items()):
+        _print_org(org, org_rows, settings)
+
+
+if __name__ == "__main__":
+    main()

minima/metrics/savings.py

@@ -0,0 +1,98 @@
+"""Savings accounting over the decision log: estimated and realized, two baselines.
+
+Both counterfactual baselines are always reported side by side and explicitly labeled:
+``vs_premium`` (the most expensive scored candidate — generous, overstates savings for
+callers who would never have used the premium model) and ``vs_declared`` (the caller's
+stated default via RecommendRequest.baseline_model_id — honest, but only present when
+callers declare one). Realized figures use the actual cost reported at feedback and are
+restricted to the reconciled subset; estimated and realized are never mixed in one number.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from minima.recommender.decisionlog import DecisionRecord
+
+
+@dataclass(slots=True)
+class SavingsEstimated:
+    n: int = 0
+    cost_recommended_usd: float = 0.0
+    cost_premium_usd: float = 0.0
+    savings_vs_premium_usd: float = 0.0
+    n_declared: int = 0
+    cost_declared_usd: float = 0.0
+    savings_vs_declared_usd: float = 0.0
+
+
+@dataclass(slots=True)
+class SavingsRealized:
+    n_reconciled: int = 0
+    realized_cost_usd: float = 0.0
+    est_cost_recommended_usd: float = 0.0
+    est_cost_premium_usd: float = 0.0
+    savings_vs_premium_est_usd: float = 0.0
+    n_declared: int = 0
+    est_cost_declared_usd: float = 0.0
+    savings_vs_declared_est_usd: float = 0.0
+
+
+@dataclass(slots=True)
+class SavingsSummary:
+    estimated: SavingsEstimated = field(default_factory=SavingsEstimated)
+    realized: SavingsRealized = field(default_factory=SavingsRealized)
+
+
+def summarize(rows: list[DecisionRecord]) -> SavingsSummary:
+    est = SavingsEstimated()
+    real = SavingsRealized()
+    for r in rows:
+        est.n += 1
+        est.cost_recommended_usd += r.est_cost_recommended
+        est.cost_premium_usd += r.est_cost_premium
+        est.savings_vs_premium_usd += r.est_cost_premium - r.est_cost_recommended
+        if r.est_cost_baseline_declared is not None:
+            est.n_declared += 1
+            est.cost_declared_usd += r.est_cost_baseline_declared
+            est.savings_vs_declared_usd += (
+                r.est_cost_baseline_declared - r.est_cost_recommended
+            )
+        if r.reconciled and r.realized_cost_usd is not None and r.realized_cost_usd > 0:
+            real.n_reconciled += 1
+            real.realized_cost_usd += r.realized_cost_usd
+            real.est_cost_recommended_usd += r.est_cost_recommended
+            real.est_cost_premium_usd += r.est_cost_premium
+            # Realized chosen cost against the ESTIMATED premium baseline — the only
+            # counterfactual available (the premium model was never run). Labeled "est".
+            real.savings_vs_premium_est_usd += r.est_cost_premium - r.realized_cost_usd
+            if r.est_cost_baseline_declared is not None:
+                real.n_declared += 1
+                real.est_cost_declared_usd += r.est_cost_baseline_declared
+                real.savings_vs_declared_est_usd += (
+                    r.est_cost_baseline_declared - r.realized_cost_usd
+                )
+
+    for obj in (est, real):
+        for name in obj.__dataclass_fields__:
+            value = getattr(obj, name)
+            if isinstance(value, float):
+                setattr(obj, name, round(value, 8))
+    return SavingsSummary(estimated=est, realized=real)
+
+
+def group_rows(
+    rows: list[DecisionRecord], group_by: str | None
+) -> dict[str, list[DecisionRecord]]:
+    if group_by == "cluster":
+        key = lambda r: r.cluster  # noqa: E731
+    elif group_by == "task_type":
+        key = lambda r: r.task_type  # noqa: E731
+    elif group_by == "lane":
+        key = lambda r: r.lane  # noqa: E731
+    else:
+        return {}
+    grouped: dict[str, list[DecisionRecord]] = {}
+    for r in rows:
+        grouped.setdefault(key(r), []).append(r)
+    return grouped

minima/recommender/__init__.py

@@ -0,0 +1 @@
+"""The recommendation engine: classify -> recall -> aggregate -> score -> optimize."""

minima/recommender/_pg_pool.py

@@ -0,0 +1,38 @@
+"""Shared psycopg2 ThreadedConnectionPool registry keyed by database URL.
+
+All four durable store classes (RecStore, DecisionLog, Propensity, DurableRefs) call
+get_pool() with the same URL, so the process holds exactly one connection pool per
+database regardless of how many store objects are instantiated.
+"""
+
+from __future__ import annotations
+
+import contextlib
+from collections.abc import Generator
+from threading import Lock
+
+import psycopg2
+from psycopg2.pool import ThreadedConnectionPool
+
+_pools: dict[str, ThreadedConnectionPool] = {}
+_lock = Lock()
+
+
+def get_pool(url: str, minconn: int = 1, maxconn: int = 5) -> ThreadedConnectionPool:
+    with _lock:
+        if url not in _pools:
+            _pools[url] = ThreadedConnectionPool(minconn, maxconn, url)
+        return _pools[url]
+
+
+@contextlib.contextmanager
+def cursor(url: str) -> Generator[psycopg2.extensions.cursor, None, None]:
+    """Yield a cursor inside a committed transaction; return the connection to the pool."""
+    pool = get_pool(url)
+    conn = pool.getconn()
+    try:
+        with conn:  # commits on clean exit, rolls back on exception
+            with conn.cursor() as cur:
+                yield cur
+    finally:
+        pool.putconn(conn)

minima/recommender/_redis_client.py

@@ -0,0 +1,32 @@
+"""Shared Redis client registry keyed by URL.
+
+RecStore and DurableRefs both call get_client() with the same URL, so the process
+holds exactly one connection pool (redis.ConnectionPool is built into redis.Redis)
+per Redis instance.
+"""
+
+from __future__ import annotations
+
+from threading import Lock
+
+import redis as _redis
+
+_clients: dict[str, _redis.Redis] = {}
+_lock = Lock()
+
+
+def get_client(url: str) -> _redis.Redis:
+    with _lock:
+        if url not in _clients:
+            _clients[url] = _redis.from_url(url, decode_responses=True)
+        return _clients[url]
+
+
+def decode(value: bytes | str) -> str:
+    """Narrow a redis response to ``str``.
+
+    ``get_client`` always sets ``decode_responses=True``, so values are ``str`` at
+    runtime; the redis-py stubs still type them ``bytes | str``, which this helper
+    reconciles for callers (and decodes correctly if the flag ever flips).
+    """
+    return value.decode() if isinstance(value, bytes) else value

minima/recommender/aggregate.py

@@ -0,0 +1,157 @@
+"""Turn recalled outcomes into a weighted per-model summary."""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Iterable
+
+from minima.memory.records import OutcomeRecord, RecalledEvidence, clamp01
+from minima.recommender.types import ModelAggregate
+
+# Floor on the confidence multiplier so freshly-seeded (un-reinforced) but
+# topically-relevant evidence still counts — just less than reinforced evidence.
+KC_FLOOR = 0.3
+STALE_DECAY = 0.5
+
+_SECONDS_PER_DAY = 86_400.0
+
+
+def age_decay(
+    recorded_at: float | None,
+    *,
+    half_life_days: float,
+    floor: float,
+    now: float | None = None,
+) -> float | None:
+    """Exponential observation-age decay: halves every half-life, floored.
+
+    None when the record has no timestamp (legacy schema v1) — caller falls back to
+    the binary staleness penalty. Future-dated timestamps clamp to no decay.
+    """
+    if recorded_at is None or recorded_at <= 0.0 or half_life_days <= 0.0:
+        return None
+    ref_now = now if now is not None else time.time()
+    age_days = max(0.0, ref_now - recorded_at) / _SECONDS_PER_DAY
+    return max(floor, 0.5 ** (age_days / half_life_days))
+
+
+def neighbor_weight(
+    ev: RecalledEvidence,
+    *,
+    half_life_days: float = 0.0,
+    decay_floor: float = 0.1,
+    now: float | None = None,
+) -> float:
+    similarity = max(0.0, ev.score)
+    confidence_mult = KC_FLOOR + (1.0 - KC_FLOOR) * clamp01(ev.knowledge_confidence)
+    # Observation-age decay when the record carries a timestamp; supersession (is_stale)
+    # still caps the multiplier at STALE_DECAY. Records without a timestamp keep the
+    # legacy binary behavior. knowledge_confidence is left untouched on purpose: its
+    # server-side recency component tracks *reinforcement* recency, this tracks
+    # *observation* age.
+    decay = age_decay(
+        ev.record.recorded_at if ev.record else None,
+        half_life_days=half_life_days,
+        floor=decay_floor,
+        now=now,
+    )
+    if decay is None:
+        decay = STALE_DECAY if ev.is_stale else 1.0
+    elif ev.is_stale:
+        decay = min(decay, STALE_DECAY)
+    return similarity * confidence_mult * decay
+
+
+def seed_factor(n_live: int, *, seed_weight: float, crowdout_n: int) -> float:
+    """Weight multiplier for seeded evidence, crowded out linearly by live outcomes."""
+    if crowdout_n <= 0:
+        return seed_weight
+    return seed_weight * max(0.0, 1.0 - n_live / float(crowdout_n))
+
+
+def aggregate_by_model(
+    evidence: Iterable[RecalledEvidence],
+    candidate_ids: set[str] | None = None,
+    *,
+    half_life_days: float = 0.0,
+    decay_floor: float = 0.1,
+    seed_weight: float = 1.0,
+    seed_crowdout_n: int = 0,
+    now: float | None = None,
+) -> dict[str, ModelAggregate]:
+    """Group neighbors by model and accumulate weighted success statistics.
+
+    Two passes: the first counts live (non-seed) outcomes per model so seeded evidence
+    can be crowded out as real feedback accumulates; the second accumulates weights.
+    Defaults preserve legacy behavior (no age decay, seeds at full weight).
+    """
+    items: list[tuple[RecalledEvidence, OutcomeRecord]] = []
+    n_live: dict[str, int] = {}
+    for ev in evidence:
+        rec = ev.record
+        if rec is None:
+            continue
+        if candidate_ids is not None and rec.model_id not in candidate_ids:
+            continue
+        items.append((ev, rec))
+        if rec.source_dataset is None:
+            n_live[rec.model_id] = n_live.get(rec.model_id, 0) + 1
+
+    aggs: dict[str, ModelAggregate] = {}
+    kc_totals: dict[str, float] = {}
+
+    for ev, rec in items:
+        model_id = rec.model_id
+
+        weight = neighbor_weight(
+            ev, half_life_days=half_life_days, decay_floor=decay_floor, now=now
+        )
+        if rec.source_dataset is not None and seed_weight != 1.0:
+            weight *= seed_factor(
+                n_live.get(model_id, 0), seed_weight=seed_weight, crowdout_n=seed_crowdout_n
+            )
+        agg = aggs.get(model_id)
+        if agg is None:
+            agg = ModelAggregate(model_id=model_id)
+            aggs[model_id] = agg
+            kc_totals[model_id] = 0.0
+
+        y = clamp01(rec.quality_score)
+        agg.weight_sum += weight
+        agg.weighted_success += weight * y
+        agg.n += 1
+        agg.evidence.append(ev)
+        kc_totals[model_id] += clamp01(ev.knowledge_confidence)
+        # Observed cost is derived on demand from agg.evidence (robust median, similarity
+        # weighted) — see ModelAggregate.observed_cost — not accumulated here.
+
+    for model_id, agg in aggs.items():
+        agg.avg_knowledge_confidence = kc_totals[model_id] / agg.n if agg.n else 0.0
+
+    return aggs
+
+
+def is_conflicted(agg: ModelAggregate, min_n: int = 4, lo: float = 0.35, hi: float = 0.70) -> bool:
+    """A model whose neighbors show mixed success — broadened to catch degrading models."""
+    return agg.n >= min_n and lo <= agg.weighted_success_rate <= hi
+
+
+def apply_ipw(
+    aggs: dict[str, ModelAggregate],
+    propensities: dict[str, float],
+    clip_low: float,
+    clip_high: float,
+) -> None:
+    """Re-weight each model's evidence mass by clipped inverse propensity, in place.
+
+    Scaling weight_sum and weighted_success by the same factor preserves the
+    empirical success rate while up-weighting evidence from rarely-recommended
+    models (low propensity) so it isn't drowned out by selection bias.
+    """
+    for model_id, agg in aggs.items():
+        pi = propensities.get(model_id)
+        if not pi or pi <= 0:
+            continue
+        factor = min(clip_high, max(clip_low, 1.0 / pi))
+        agg.weight_sum *= factor
+        agg.weighted_success *= factor

minima/recommender/classify.py

@@ -0,0 +1,165 @@
+"""Heuristic task classification: type and difficulty from the prompt text.
+
+Cheap and deterministic. Caller-supplied ``task_type``/``difficulty`` always win.
+A cheap-LLM classifier is layered in a later phase when confidence is low.
+"""
+
+from __future__ import annotations
+
+import re
+
+from minima.schemas.common import Difficulty, TaskInput, TaskType
+
+# Ordered most-specific-first; the first matching pattern wins.
+_TYPE_PATTERNS: list[tuple[TaskType, re.Pattern[str]]] = [
+    (
+        TaskType.code,
+        re.compile(
+            r"```|\bdef \b|\bclass \b|\bfunction\b|\bimport \b|\bSELECT \b|regex|stack ?trace|compile|refactor|implement|unit test|debug",
+            re.I,
+        ),
+    ),
+    (
+        TaskType.translation,
+        re.compile(
+            r"\btranslate\b|\bin (french|spanish|german|chinese|japanese|hindi)\b|\bto (french|spanish|german)\b",
+            re.I,
+        ),
+    ),
+    (
+        TaskType.summarization,
+        re.compile(r"\bsummari[sz]e\b|\btl;?dr\b|\bsummary\b|\bcondense\b|\bin brief\b", re.I),
+    ),
+    (
+        TaskType.extraction,
+        re.compile(
+            r"\bextract\b|\bparse\b|\bpull out\b|\blist all\b|\bfind all\b|\bjson schema\b|\bfields?\b.*\bfrom\b",
+            re.I,
+        ),
+    ),
+    (
+        TaskType.classification,
+        re.compile(
+            r"\bclassif|\bcategori[sz]e\b|\blabel\b|\bsentiment\b|\bwhich (category|class|label)\b|\btrue or false\b",
+            re.I,
+        ),
+    ),
+    (
+        TaskType.tool_use,
+        re.compile(
+            r"\bcall the\b|\buse the tool\b|\bfunction call\b|\btool[_ ]?call\b|\bapi call\b", re.I
+        ),
+    ),
+    (
+        TaskType.rag,
+        re.compile(
+            r"\bbased on the (following|context|document)\b|\baccording to the\b|\bcontext:\b|\bgiven the passage\b",
+            re.I,
+        ),
+    ),
+    (
+        TaskType.creative,
+        re.compile(
+            r"\bwrite a (story|poem|song|essay)\b|\bcreative\b|\bimagine\b|\bbrainstorm\b", re.I
+        ),
+    ),
+    (
+        TaskType.reasoning,
+        re.compile(
+            r"\bprove\b|\bcalculate\b|\bsolve\b|\bequation\b|\bstep[- ]by[- ]step\b|\breason(ing)?\b|\bderive\b|\bwhy (does|is|are)\b",
+            re.I,
+        ),
+    ),
+    (
+        TaskType.qa,
+        re.compile(r"^\s*(what|who|when|where|why|how|which|is|are|does|can)\b|\?\s*$", re.I),
+    ),
+]
+
+_COMPLEXITY_MARKERS = re.compile(
+    r"\b(and then|after that|must|ensure|constraint|optimi[sz]e|edge case|step \d|\d\.\s)\b", re.I
+)
+
+# Task types that tend to need more capability for the same text length.
+_HARD_TYPES = {TaskType.code, TaskType.reasoning}
+_EASY_TYPES = {
+    TaskType.classification,
+    TaskType.extraction,
+    TaskType.summarization,
+    TaskType.translation,
+}
+
+_ORDER = [
+    Difficulty.trivial,
+    Difficulty.easy,
+    Difficulty.medium,
+    Difficulty.hard,
+    Difficulty.expert,
+]
+
+
+def infer_task_type(text: str) -> TaskType:
+    for task_type, pattern in _TYPE_PATTERNS:
+        if pattern.search(text):
+            return task_type
+    return TaskType.other
+
+
+def infer_difficulty(text: str, task_type: TaskType) -> Difficulty:
+    words = len(text.split())
+    if words < 40:
+        base = 1  # easy
+    elif words < 150:
+        base = 2  # medium
+    elif words < 400:
+        base = 3  # hard
+    else:
+        base = 4  # expert
+
+    if len(_COMPLEXITY_MARKERS.findall(text)) >= 2:
+        base += 1  # multiple multi-step / constraint markers
+
+    if task_type in _HARD_TYPES:
+        base += 1
+    elif task_type in _EASY_TYPES:
+        base -= 1
+
+    index = max(0, min(len(_ORDER) - 1, base))
+    return _ORDER[index]
+
+
+def classify(task: TaskInput) -> tuple[TaskType, Difficulty]:
+    task_type = task.task_type or infer_task_type(task.task)
+    difficulty = task.difficulty or infer_difficulty(task.task, task_type)
+    return task_type, difficulty
+
+
+def classify_from_neighbors(
+    votes: list[tuple[str, float]], *, min_neighbors: int = 2, min_share: float = 0.5
+) -> TaskType | None:
+    """Disambiguate an `other` classification from ANN-recalled semantic neighbors.
+
+    ``votes`` is ``(neighbor_task_type, similarity)`` over recalled outcomes. Returns the
+    similarity-weighted plurality type when it is non-`other`, has >= ``min_neighbors``
+    supporters, and holds >= ``min_share`` of the weighted vote; else None. This is the free,
+    semantic alternative to a paid LLM-classify call for prompts the regex can't place.
+    """
+    weighted: dict[str, float] = {}
+    counts: dict[str, int] = {}
+    total = 0.0
+    for tt, weight in votes:
+        if not tt or tt == TaskType.other.value:
+            continue
+        w = max(0.0, weight)
+        weighted[tt] = weighted.get(tt, 0.0) + w
+        counts[tt] = counts.get(tt, 0) + 1
+        total += w
+    if total <= 0.0:
+        return None
+    best = max(weighted, key=weighted.__getitem__)
+    if counts[best] < min_neighbors or (weighted[best] / total) < min_share:
+        return None
+    try:
+        return TaskType(best)
+    except ValueError:
+        return None