loopiter 0.2.0a1__py3-none-any.whl

loopiter/__init__.py

@@ -0,0 +1,16 @@
+"""Loopiter Python: zero runtime dependencies, no telemetry, no automatic deployment."""
+
+from ._validation import LoopiterError, fingerprint
+from .client import DeploymentAdapter, FeedbackLoop
+from .store import FeedbackStore, InMemoryStore, StoreTransaction
+
+__version__ = "0.2.0a1"
+__all__ = [
+    "DeploymentAdapter",
+    "FeedbackLoop",
+    "FeedbackStore",
+    "InMemoryStore",
+    "LoopiterError",
+    "StoreTransaction",
+    "fingerprint",
+]

loopiter/_validation.py

@@ -0,0 +1,335 @@
+"""Strict JSON and lifecycle validation. Not a wire-format compatibility layer."""
+
+import hashlib
+import json
+import math
+import re
+from datetime import UTC, datetime
+from typing import Any
+
+Record = dict[str, Any]
+EXECUTION_KINDS = "agent turn inference prediction tool workflow retrieval custom".split()
+SIGNAL_KINDS = "rating correction outcome reward approval tool_result custom".split()
+TARGET_KINDS = (
+    "prompt routing rule retrieval model dataset threshold tool_schema workflow "
+    "agent_topology code capacity custom"
+).split()
+RISKS = "low medium high critical".split()
+STATUSES = "proposed evaluated approved deployed rejected superseded rolled_back historical".split()
+COLLECTIONS = "executions signals candidates targets attempts events".split()
+EXECUTION_FIELDS = (
+    "id kind episode_id parent_execution_id entity_id input output artifacts metadata "
+    "started_at completed_at"
+).split()
+SIGNAL_FIELDS = (
+    "id execution_id episode_id kind name value correction source confidence metadata observed_at"
+).split()
+CANDIDATE_FIELDS = "id target proposed_change evidence risk metadata".split()
+
+
+class LoopiterError(Exception):
+    def __init__(self, code: str, message: str, *, attempt_id: str | None = None):
+        super().__init__(message)
+        self.code = code
+        self.attempt_id = attempt_id
+
+
+def fail(code: str, message: str) -> None:
+    raise LoopiterError(code, message)
+
+
+def nonempty(value: Any, name: str) -> None:
+    if type(value) is not str or not value.strip() or "\0" in value:
+        fail("invalid_input", f"{name} must be nonempty text without NUL.")
+
+
+def enum(value: Any, allowed: list[str], name: str) -> None:
+    if type(value) is not str or value not in allowed:
+        fail("invalid_input", f"Invalid {name}.")
+
+
+def finite(value: Any, name: str, minimum: float = -math.inf) -> None:
+    if type(value) not in (int, float):
+        fail("invalid_input", f"{name} must be a finite number.")
+    try:
+        valid = math.isfinite(value) and value >= minimum
+    except OverflowError:
+        valid = False
+    if not valid:
+        fail("invalid_input", f"Invalid {name}.")
+
+
+def integer(value: Any, name: str, minimum: int = 1) -> None:
+    if type(value) is not int or not minimum <= value <= 2**53 - 1:
+        fail("invalid_input", f"{name} must be an integer >= {minimum}.")
+
+
+def obj(value: Any, name: str = "input") -> None:
+    if type(value) is not dict:
+        fail("invalid_input", f"{name} must be a plain dict.")
+
+
+def fields(value: Any, allowed: list[str], required: tuple[str, ...] = ()) -> None:
+    obj(value)
+    if set(value) - set(allowed) or set(required) - set(value):
+        fail("invalid_input", "Unknown or missing required fields.")
+
+
+def timestamp(value: Any, name: str = "timestamp") -> datetime:
+    if type(value) is not str or not re.fullmatch(
+        r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,6})?Z", value
+    ):
+        fail("invalid_input", f"{name} must be an ISO UTC timestamp ending in Z.")
+    try:
+        return datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError:
+        fail("invalid_input", f"Invalid calendar date in {name}.")
+    raise AssertionError("unreachable")
+
+
+def now() -> str:
+    return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
+
+
+def window_valid(window: Any) -> None:
+    fields(window, ["from", "to"])
+    for key, value in window.items():
+        timestamp(value, key)
+    if "from" in window and "to" in window and timestamp(window["from"]) >= timestamp(window["to"]):
+        fail("invalid_input", "Time window must be increasing (from inclusive, to exclusive).")
+
+
+def in_window(value: str, window: Record) -> bool:
+    t = timestamp(value)
+    return ("from" not in window or t >= timestamp(window["from"])) and (
+        "to" not in window or t < timestamp(window["to"])
+    )
+
+
+def json_value(value: Any, maximum: int = 262144) -> None:
+    seen: set[int] = set()
+
+    def visit(v: Any, depth: int) -> None:
+        if depth > 32:
+            fail("invalid_input", "JSON nesting exceeds 32 levels.")
+        if type(v) is str:
+            if "\0" in v:
+                fail("invalid_input", "NUL is not supported in JSON strings.")
+            try:
+                v.encode("utf-8")
+            except UnicodeEncodeError:
+                fail("invalid_input", "JSON strings must be valid Unicode without lone surrogates.")
+            return
+        if v is None or type(v) is bool:
+            return
+        if type(v) in (int, float):
+            finite(v, "JSON number")
+            return
+        if type(v) not in (list, dict) or id(v) in seen:
+            fail("invalid_input", "Expected acyclic plain JSON values.")
+        if type(v) is dict and any(type(k) is not str for k in v):
+            fail("invalid_input", "JSON keys must be strings.")
+        seen.add(id(v))
+        if type(v) is dict:
+            for key in v:
+                visit(key, depth + 1)
+        for item in v.values() if type(v) is dict else v:
+            visit(item, depth + 1)
+        seen.remove(id(v))
+
+    visit(value, 0)
+    if len(canonical(value).encode("utf-8")) > maximum:
+        fail("payload_limit", f"Payload exceeds {maximum} bytes.")
+
+
+def canonical(value: Any) -> str:
+    return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False)
+
+
+def fingerprint(value: Any) -> str:
+    return hashlib.sha256(canonical(value).encode("utf-8")).hexdigest()
+
+
+def target(value: Any) -> None:
+    fields(value, ["kind", "key"], ("kind", "key"))
+    enum(value["kind"], TARGET_KINDS, "target kind")
+    nonempty(value["key"], "target key")
+
+
+def execution_input(value: Any) -> None:
+    fields(value, EXECUTION_FIELDS, ("kind",))
+    enum(value["kind"], EXECUTION_KINDS, "execution kind")
+    for key in ("id", "episode_id", "parent_execution_id", "entity_id"):
+        if key in value:
+            nonempty(value[key], key)
+    for key in ("started_at", "completed_at"):
+        if key in value:
+            timestamp(value[key], key)
+    for key in ("metadata", "artifacts"):
+        if key in value:
+            obj(value[key], key)
+    if "completed_at" in value and "started_at" in value:
+        if timestamp(value["completed_at"]) < timestamp(value["started_at"]):
+            fail("invalid_input", "Completion predates execution.")
+
+
+def signal_input(value: Any) -> None:
+    fields(value, SIGNAL_FIELDS, ("kind", "name", "source", "value"))
+    enum(value["kind"], SIGNAL_KINDS, "signal kind")
+    for key in ("name", "source", "id", "episode_id", "execution_id"):
+        if key in value:
+            nonempty(value[key], key)
+    if not (value.get("execution_id") or value.get("episode_id")):
+        fail("invalid_input", "Signal needs execution_id or episode_id.")
+    if "confidence" in value:
+        finite(value["confidence"], "confidence", 0)
+        if value["confidence"] > 1:
+            fail("invalid_input", "Confidence exceeds 1.")
+    if "observed_at" in value:
+        timestamp(value["observed_at"])
+    if "metadata" in value:
+        obj(value["metadata"], "metadata")
+
+
+def candidate_input(value: Any) -> None:
+    fields(value, CANDIDATE_FIELDS, ("target", "proposed_change", "evidence"))
+    target(value["target"])
+    if "id" in value:
+        nonempty(value["id"], "id")
+    if "risk" in value:
+        enum(value["risk"], RISKS, "risk")
+    if "metadata" in value:
+        obj(value["metadata"], "metadata")
+
+
+def evaluation(value: Any) -> None:
+    fields(value, ["passed", "metrics", "notes"], ("passed",))
+    json_value(value)
+    if type(value["passed"]) is not bool:
+        fail("invalid_input", "passed must be a boolean.")
+    if "metrics" in value:
+        obj(value["metrics"], "metrics")
+        for key, number in value["metrics"].items():
+            nonempty(key, "metric name")
+            finite(number, key)
+    if "notes" in value and type(value["notes"]) is not str:
+        fail("invalid_input", "notes must be text.")
+
+
+def receipt(value: Any) -> None:
+    fields(
+        value,
+        ["attempt_id", "artifact_version", "previous_artifact_version", "metadata"],
+        ("attempt_id", "artifact_version", "previous_artifact_version"),
+    )
+    nonempty(value["attempt_id"], "attempt_id")
+    for key in ("artifact_version", "previous_artifact_version"):
+        if value[key] is not None:
+            nonempty(value[key], key)
+    if "metadata" in value:
+        obj(value["metadata"], "metadata")
+
+
+def content_hash(row: Record) -> str:
+    return fingerprint({k: row[k] for k in ("target", "proposed_change", "risk", "metadata")})
+
+
+def stored(kind: str, row: Any, namespace: str) -> None:
+    enum(kind, COLLECTIONS, "collection")
+    obj(row, "adapter record")
+    json_value(row, 16 * 1024 * 1024)
+    nonempty(row.get("id"), "record ID")
+    integer(row.get("revision"), "revision")
+    if row.get("namespace") != namespace:
+        fail("namespace_mismatch", "Adapter returned another namespace.")
+    if timestamp(row.get("updated_at")) < timestamp(row.get("created_at")):
+        fail("integrity_error", "Update predates creation.")
+    if kind in ("executions", "signals", "candidates"):
+        allowed, validator = {
+            "executions": (EXECUTION_FIELDS, execution_input),
+            "signals": (SIGNAL_FIELDS, signal_input),
+            "candidates": (CANDIDATE_FIELDS, candidate_input),
+        }[kind]
+        validator({k: row[k] for k in allowed if k in row})
+        nonempty(row.get("input_hash"), "input_hash")
+        obj(row.get("metadata"), "metadata")
+    if kind == "executions":
+        timestamp(row.get("started_at"))
+        obj(row.get("artifacts"), "artifacts")
+    if kind == "signals":
+        timestamp(row.get("observed_at"))
+        finite(row.get("confidence"), "confidence", 0)
+    if kind == "candidates":
+        enum(row.get("status"), STATUSES, "candidate status")
+        enum(row.get("risk"), RISKS, "risk")
+        if row.get("content_hash") != content_hash(row) or row.get("evidence_hash") != fingerprint(
+            row["evidence"]
+        ):
+            fail("integrity_error", "Candidate content/evidence hash mismatch.")
+        if type(row.get("evaluations")) is not list:
+            fail("integrity_error", "Invalid evaluations.")
+        for e in row["evaluations"]:
+            obj(e, "evaluation")
+            evaluation({k: e[k] for k in ("passed", "metrics", "notes") if k in e})
+            for key in ("id", "evaluator", "version", "dataset_hash"):
+                nonempty(e.get(key), key)
+            timestamp(e.get("created_at"))
+            if (
+                e.get("candidate_hash") != row["content_hash"]
+                or e.get("evidence_hash") != row["evidence_hash"]
+            ):
+                fail("integrity_error", "Evaluation bound to different content.")
+        if "approval" in row:
+            a = row["approval"]
+            obj(a, "approval")
+            nonempty(a.get("actor"), "actor")
+            timestamp(a.get("approved_at"))
+            e = row["evaluations"][-1] if row["evaluations"] else {}
+            if (
+                not e.get("passed")
+                or e.get("id") != a.get("evaluation_id")
+                or a.get("candidate_hash") != row["content_hash"]
+            ):
+                fail("integrity_error", "Approval does not match latest passing evaluation.")
+        if (
+            row["status"] in ("approved", "deployed", "superseded", "rolled_back")
+            and "approval" not in row
+        ):
+            fail("integrity_error", "Lifecycle record missing approval.")
+        if "deployment_receipt" in row:
+            receipt(row["deployment_receipt"])
+    if kind in ("targets", "attempts"):
+        target(row.get("target"))
+        if kind == "targets" and row["id"] != fingerprint(row["target"]):
+            fail("integrity_error", "Target identity mismatch.")
+        for key in (
+            "active_candidate_id",
+            "pending_attempt_id",
+            "restore_candidate_id",
+            "previous_candidate_id",
+        ):
+            if key in row:
+                nonempty(row[key], key)
+    if kind == "targets" and row.get("artifact_version") is not None:
+        nonempty(row["artifact_version"], "artifact_version")
+    if kind == "attempts":
+        nonempty(row.get("candidate_id"), "candidate_id")
+        enum(row.get("operation"), ["apply", "rollback"], "operation")
+        enum(row.get("status"), ["pending", "succeeded", "not_applied"], "attempt status")
+        for key in ["expected_artifact_version"] + (
+            ["restore_artifact_version"] if row["operation"] == "rollback" else []
+        ):
+            if key not in row:
+                fail("integrity_error", "Missing artifact version.")
+            if row[key] is not None:
+                nonempty(row[key], key)
+        if "receipt" in row:
+            receipt(row["receipt"])
+            if row["receipt"]["attempt_id"] != row["id"]:
+                fail("integrity_error", "Attempt receipt mismatch.")
+        if row["status"] == "succeeded" and "receipt" not in row:
+            fail("integrity_error", "Succeeded attempt has no receipt.")
+    if kind == "events":
+        nonempty(row.get("type"), "event type")
+        nonempty(row.get("subject_id"), "subject_id")
+        obj(row.get("details"), "event details")

loopiter/analysis.py

@@ -0,0 +1,261 @@
+"""Descriptive structured segments, not causal inference or a probability of improvement."""
+
+from collections import defaultdict
+from copy import deepcopy
+from datetime import timedelta
+from itertools import combinations
+from typing import TYPE_CHECKING, Any
+
+from . import _validation as v
+
+if TYPE_CHECKING:
+    from .client import FeedbackLoop
+
+
+def default_signal_score(signal: v.Record) -> float | None:
+    value = signal["value"]
+    if type(value) is bool:
+        return float(value)
+    if type(value) in (int, float):
+        return value
+    if type(value) is str:
+        if value.lower() in ("yes", "good", "correct", "positive", "success"):
+            return 1
+        if value.lower() in ("no", "bad", "incorrect", "negative", "failure"):
+            return 0
+    return None
+
+
+async def _read_all(loop, tx, kind, maximum, window):
+    rows, cursor = [], None
+    while True:
+        limit = min(1000, maximum + 1 - len(rows))
+        page = loop._page(
+            kind, await tx.list(kind, limit=limit, cursor=cursor, window=window), limit, cursor
+        )
+        for row in page["items"]:
+            when = row["observed_at"] if kind == "signals" else row["started_at"]
+            if not v.in_window(when, window):
+                v.fail("integrity_error", "Adapter ignored the query window.")
+        rows.extend(page["items"])
+        if len(rows) > maximum:
+            v.fail(
+                "query_limit",
+                f"Analysis exceeds {maximum} {kind}; narrow the window or increase maximum_records.",
+            )
+        cursor = page.get("next_cursor")
+        if cursor is None:
+            return rows
+
+
+def _summary(units):
+    total, weight = 0.0, 0.0
+    for values in units.values():
+        w = sum(item[0]["confidence"] for item in values.values())
+        weighted = sum(item[0]["confidence"] * item[1] for item in values.values())
+        unit_weight = w / len(values)
+        total += weighted / w * unit_weight
+        weight += unit_weight
+    if not weight:
+        return None
+    mean = total / weight
+    v.finite(mean, "aggregated mean")
+    v.finite(weight, "effective weight")
+    return {"mean": mean, "weight": weight, "count": len(units)}
+
+
+def _bucket(time, size):
+    if size == "day":
+        return time[:10]
+    if size == "month":
+        return time[:7]
+    day = v.timestamp(time)
+    return (day - timedelta(days=day.weekday())).date().isoformat()
+
+
+def _unit(row):
+    return "episode:" + row["episode_id"] if row.get("episode_id") else "execution:" + row["id"]
+
+
+async def analyze(
+    loop: "FeedbackLoop",
+    *,
+    dimensions: list[str],
+    maximum_dimension_depth: int = 2,
+    execution_kinds: list[str] | None = None,
+    signal_kinds: list[str] | None = None,
+    signal_names: list[str] | None = None,
+    execution_window: v.Record | None = None,
+    observation_window: v.Record | None = None,
+    minimum_support: int = 5,
+    minimum_scored_count: int = 2,
+    minimum_effect_size: float = 0,
+    minimum_recurrence: int = 1,
+    minimum_distinct_entities: int = 0,
+    time_bucket: str = "week",
+    include_episode_signals: bool = True,
+    maximum_records: int = 20000,
+    score=None,
+    scoring_version: str = "default-v1",
+) -> list[v.Record]:
+    if type(dimensions) is not list or not 1 <= len(dimensions) <= 8:
+        v.fail("invalid_input", "Choose 1–8 dimensions.")
+    for d in dimensions:
+        v.nonempty(d, "dimension")
+    if len(set(dimensions)) != len(dimensions):
+        v.fail("invalid_input", "Duplicate dimensions.")
+    for name, items, allowed in (
+        ("execution_kinds", execution_kinds, v.EXECUTION_KINDS),
+        ("signal_kinds", signal_kinds, v.SIGNAL_KINDS),
+        ("signal_names", signal_names, None),
+    ):
+        if items is not None:
+            if type(items) is not list:
+                v.fail("invalid_input", f"{name} must be a list.")
+            for item in items:
+                v.nonempty(item, name)
+                if allowed is not None:
+                    v.enum(item, allowed, name)
+    execution_window = {} if execution_window is None else execution_window
+    observation_window = {} if observation_window is None else observation_window
+    v.window_valid(execution_window)
+    v.window_valid(observation_window)
+    for name, number in (
+        ("maximum_records", maximum_records),
+        ("maximum_dimension_depth", maximum_dimension_depth),
+        ("minimum_support", minimum_support),
+        ("minimum_scored_count", minimum_scored_count),
+        ("minimum_recurrence", minimum_recurrence),
+    ):
+        v.integer(number, name)
+    if maximum_dimension_depth > 2:
+        v.fail("query_limit", "Alpha supports dimension depth 1 or 2.")
+    v.integer(minimum_distinct_entities, "minimum_distinct_entities", 0)
+    v.finite(minimum_effect_size, "minimum_effect_size", 0)
+    v.enum(time_bucket, ["day", "week", "month"], "time_bucket")
+    if type(include_episode_signals) is not bool:
+        v.fail("invalid_input", "include_episode_signals must be boolean.")
+    v.nonempty(scoring_version, "scoring_version")
+    if score is not None and not callable(score):
+        v.fail("invalid_input", "score must be a synchronous callable.")
+    async with loop.store.transaction(loop.namespace) as tx:
+        executions = await _read_all(loop, tx, "executions", maximum_records, execution_window)
+        signals = await _read_all(loop, tx, "signals", maximum_records, observation_window)
+    selected = [e for e in executions if execution_kinds is None or e["kind"] in execution_kinds]
+    by_execution, by_episode = defaultdict(list), defaultdict(list)
+    for signal in signals:
+        if signal_names is not None and signal["name"] not in signal_names:
+            continue
+        if signal_kinds is not None and signal["kind"] not in signal_kinds:
+            continue
+        value = (score or default_signal_score)(deepcopy(signal))
+        if value is not None:
+            v.finite(value, "score callback result")
+        if value is None or signal["confidence"] <= 0:
+            continue
+        index = by_execution if signal.get("execution_id") else by_episode
+        index[signal.get("execution_id", signal.get("episode_id"))].append((signal, value))
+
+    def units_for(rows):
+        units = defaultdict(dict)
+        for row in rows:
+            items = [*by_execution[row["id"]]]
+            if include_episode_signals and row.get("episode_id"):
+                items.extend(by_episode[row["episode_id"]])
+            for item in items:
+                units[_unit(row)][item[0]["id"]] = item
+        return units
+
+    baseline_units = units_for(selected)
+    baseline = _summary(baseline_units)
+    if baseline is None:
+        return []
+    sets = [(d,) for d in dimensions]
+    if maximum_dimension_depth == 2:
+        sets.extend(combinations(dimensions, 2))
+    segments: dict[str, Any] = {}
+    missing = object()
+    for row in selected:
+        for subset in sets:
+            values = {}
+            for path in subset:
+                value: Any = row
+                for component in path.split("."):
+                    value = value.get(component, missing) if type(value) is dict else missing
+                if value is missing:
+                    break
+                values[path] = value
+            if len(values) == len(subset):
+                segments.setdefault(v.canonical(values), (values, []))[1].append(row)
+    baseline_fingerprint = v.fingerprint(
+        {
+            "executions": sorted([r["id"], r["revision"], r["input_hash"]] for r in selected),
+            "units": sorted(
+                [
+                    key,
+                    sorted(
+                        [s["id"], s["input_hash"], val, s["confidence"]]
+                        for s, val in items.values()
+                    ),
+                ]
+                for key, items in baseline_units.items()
+            ),
+            "scoring_version": scoring_version,
+        }
+    )
+    findings = []
+    for values, rows in segments.values():
+        units = units_for(rows)
+        stats = _summary(units)
+        if stats is None or len(units) < minimum_support or stats["count"] < minimum_scored_count:
+            continue
+        effect = stats["mean"] - baseline["mean"]
+        v.finite(effect, "effect size")
+        if abs(effect) < minimum_effect_size:
+            continue
+        unique, corrections = {}, defaultdict(int)
+        for items in units.values():
+            unit_corrections = set()
+            for key, item in items.items():
+                unique[key] = item
+                if "correction" in item[0]:
+                    unit_corrections.add(v.canonical(item[0]["correction"]))
+            for correction in unit_corrections:
+                corrections[correction] += 1
+        buckets = {_bucket(item[0]["observed_at"], time_bucket) for item in unique.values()}
+        eligible = [r for r in rows if _unit(r) in units]
+        entities = {r["entity_id"] for r in eligible if r.get("entity_id")}
+        if len(buckets) < minimum_recurrence or len(entities) < minimum_distinct_entities:
+            continue
+        manifest = {
+            "version": 1,
+            "execution_ids": sorted(r["id"] for r in eligible),
+            "signal_ids": sorted(unique),
+            "execution_window": execution_window,
+            "observation_window": observation_window,
+            "scoring_version": scoring_version,
+        }
+        manifest["fingerprint"] = v.fingerprint(
+            {"manifest": manifest, "baseline": baseline_fingerprint}
+        )
+        findings.append(
+            {
+                "id": v.fingerprint({"namespace": loop.namespace, "dimensions": values}),
+                "namespace": loop.namespace,
+                "dimensions": values,
+                "support": len(units),
+                "execution_count": len(rows),
+                "unique_signal_count": len(unique),
+                "episode_count": len({r["episode_id"] for r in eligible if r.get("episode_id")}),
+                "scored_count": stats["count"],
+                "effective_weight": stats["weight"],
+                "mean_score": stats["mean"],
+                "baseline_score": baseline["mean"],
+                "effect_size": effect,
+                "recurrence": len(buckets),
+                "distinct_entities": len(entities),
+                "correction_counts": dict(corrections),
+                "evidence": manifest,
+            }
+        )
+    return sorted(findings, key=lambda f: (f["effect_size"], f["id"]))