agent-learning-kit 0.1.0__py3-none-any.whl

fi/alk/telemetry/__init__.py

@@ -0,0 +1,149 @@
+"""Account-integrated telemetry (Phase 8): local run ledger + keyed sync.
+
+Two channels, never a third (P8-D1): (a) the always-on local run ledger —
+every kit run appends one content-addressed, hash-chained row to
+``${AGENT_LEARNING_HOME:-~/.agent-learning}/ledger/runs.jsonl``; (b) keyed
+sync to the USER'S OWN Future AGI account when their keys resolve. There is
+no anonymous analytics channel anywhere in the kit — structurally absent and
+gate-proven (``telemetry_boundary``, gate #72).
+
+Module scope imports are stdlib + the stdlib-only package internals; the
+network-capable sync lane (``_sync``) is imported lazily inside functions
+only, after the kill switch and key gates. ``AGENT_LEARNING_TELEMETRY=off``
+binds everything, including vendored ``fi/*`` (P8-D6).
+"""
+
+from __future__ import annotations
+
+from typing import Any, Mapping
+
+from ._contract import (  # noqa: F401 — package canon re-exports
+    AGENT_LEARNING_RUN_KIND,
+    EVIDENCE_CLASSES,
+    GAP_SCHEMA,
+    GENESIS,
+    LEDGER_DIR_NAME,
+    LEDGER_HOME_ENV,
+    LEDGER_PATH_ENV,
+    LEDGER_ROW_SCHEMA,
+    NON_CANONICAL_FIELDS,
+    PHASES,
+    RELEASE_ADMISSIBLE_EVIDENCE_CLASSES,
+    ROW_FIELDS,
+    SYNC_MODE_AUTO,
+    SYNC_MODE_ENV,
+    SYNC_MODE_LOCAL,
+    SYNC_MODES,
+    SYNC_STATES,
+    TELEMETRY_ENV,
+    TELEMETRY_OFF_VALUE,
+    TOMBSTONE_FIELDS,
+    TOMBSTONE_REASONS,
+    TOMBSTONE_SCHEMA,
+    UNREADABLE_LINE_SCHEMA,
+    VERDICTS,
+    kill_switch_on,
+    ledger_dir,
+    sync_mode,
+)
+from ._ledger import RunLedger  # noqa: F401
+from ._queue import TelemetryQueue, global_queue  # noqa: F401
+from ._row import (  # noqa: F401
+    build_ledger_row,
+    canonical_row_address,
+    canonical_row_bytes,
+    content_admissible,
+    declared_required_env,
+)
+from ._run import (  # noqa: F401
+    RunRecorder,
+    RunSummary,
+    emit_run,
+    run_telemetry,
+)
+from ._url import build_dashboard_url  # noqa: F401
+
+__all__ = [
+    "AGENT_LEARNING_RUN_KIND",
+    "EVIDENCE_CLASSES",
+    "GAP_SCHEMA",
+    "GENESIS",
+    "LEDGER_DIR_NAME",
+    "LEDGER_HOME_ENV",
+    "LEDGER_PATH_ENV",
+    "LEDGER_ROW_SCHEMA",
+    "NON_CANONICAL_FIELDS",
+    "PHASES",
+    "RELEASE_ADMISSIBLE_EVIDENCE_CLASSES",
+    "ROW_FIELDS",
+    "RunLedger",
+    "RunRecorder",
+    "RunSummary",
+    "SYNC_MODE_AUTO",
+    "SYNC_MODE_ENV",
+    "SYNC_MODE_LOCAL",
+    "SYNC_MODES",
+    "SYNC_STATES",
+    "TELEMETRY_ENV",
+    "TELEMETRY_OFF_VALUE",
+    "TOMBSTONE_FIELDS",
+    "TOMBSTONE_REASONS",
+    "TOMBSTONE_SCHEMA",
+    "TelemetryQueue",
+    "UNREADABLE_LINE_SCHEMA",
+    "VERDICTS",
+    "build_dashboard_url",
+    "build_ledger_row",
+    "canonical_row_address",
+    "canonical_row_bytes",
+    "content_admissible",
+    "declared_required_env",
+    "emit_run",
+    "flush",
+    "kill_switch_on",
+    "ledger_dir",
+    "record_run",
+    "run_telemetry",
+    "sync_mode",
+]
+
+
+def _handle_row(row: Mapping[str, Any], dropped: int) -> None:
+    """Drain-side handler: ledger append (+ gap marker for any drops since
+    the last successful append). Runs on the worker thread; every failure is
+    swallowed by the queue (R§3.5).
+
+    Sync is EXPLICIT in v1 — ``agent-learn runs sync [<id>|--queued]`` or the
+    SDK ``telemetry._sync.sync_run`` — never fired from the emission path.
+    Emission-time auto-sync would turn every stray key in the environment
+    (test/example dummies included) into a network attempt inside release
+    flows; rows queue locally instead and the queued-sync path is idempotent
+    by content address, so nothing is ever lost (P8-D3, R§3.5).
+    """
+
+    ledger = RunLedger()
+    if dropped > 0:
+        ledger.append_gap(dropped)
+    ledger.append(row)
+
+
+def record_run(run_payload: Mapping[str, Any]) -> None:
+    """The single emission hook target (ARCH Decision 7): called once at the
+    run-manifest boundary for every ``agent-learning.run.v1`` payload.
+
+    Out of the critical path: builds the redacted, content-addressed row and
+    does an O(1) bounded enqueue. ``AGENT_LEARNING_TELEMETRY=off`` disables
+    everything — ledger append and sync alike (P8-D6).
+    """
+
+    if kill_switch_on():
+        return
+    required_env = declared_required_env(run_payload)
+    row = build_ledger_row(run_payload, required_env=required_env)
+    global_queue(_handle_row).enqueue(row)
+
+
+def flush(timeout: float = 5.0) -> bool:
+    """Best-effort drain of the emission queue (atexit calls this too)."""
+
+    return global_queue(_handle_row).flush(timeout)

fi/alk/telemetry/_contract.py

@@ -0,0 +1,141 @@
+"""Telemetry-local contract: row schemas, ledger paths, kill switch (Phase 8).
+
+Imports: stdlib only, plus the two reused live/ seams. Evidence classes and
+the run kind are IMPORTED from ``live/_contract.py`` and never redeclared —
+one vocabulary, one redaction seam (ARCH §2.0; the VS-Code "components exempt
+from the off switch" failure mode is structurally impossible with one seam).
+The ``telemetry_boundary`` gate scans this package like any release module.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+# --- reused verbatim from live/ (the ONE vocabulary; never redeclared) ------
+from ..live._contract import (  # noqa: F401 — re-exported package canon
+    AGENT_LEARNING_RUN_KIND,
+    EVIDENCE_CLASSES,
+    RELEASE_ADMISSIBLE_EVIDENCE_CLASSES,
+    VERDICTS,
+)
+
+# --- row schema tags (ARCH §3; REVIEW-RULINGS MF3) ---------------------------
+LEDGER_ROW_SCHEMA = "agent-learning.ledger-row.v1"
+TOMBSTONE_SCHEMA = "agent-learning.ledger-tombstone.v1"
+GAP_SCHEMA = "agent-learning.ledger-gap.v1"
+UNREADABLE_LINE_SCHEMA = "agent-learning.ledger-unreadable-line.v1"
+
+# --- hash-chain genesis sentinel (ARCH §2b; MF4: self-describing string) ----
+GENESIS = "agent-learning.ledger.genesis.v1"
+
+# --- semconv pin (PRD §4.4: OTel GenAI semconv is Development mid-2026) -----
+SEMCONV_VERSION_ENV = "OTEL_SEMCONV_STABILITY_OPT_IN"
+
+# --- kill switch (P8-D6: binds EVERYTHING including vendored fi/*) ----------
+TELEMETRY_ENV = "AGENT_LEARNING_TELEMETRY"
+TELEMETRY_OFF_VALUE = "off"
+
+# --- sync mode (Phase 14, W&B `WANDB_MODE` analogue) -------------------------
+# Reconciles the user's "keys present -> dashboard" (W&B-online) intent with the
+# P8 doctrine that emission must NOT auto-sync in release/CI flows. ``auto``
+# (default) emits when keys resolve; ``local`` queues locally + explicit
+# ``runs sync`` only. The kill switch overrides both. The test/gate harness
+# pins ``local`` so no internal flow makes a surprise network call.
+SYNC_MODE_ENV = "AGENT_LEARNING_SYNC"
+SYNC_MODE_AUTO = "auto"
+SYNC_MODE_LOCAL = "local"
+SYNC_MODES = (SYNC_MODE_AUTO, SYNC_MODE_LOCAL)
+
+# --- ledger disk layout (ARCH §2a / Decision 9; MF5) -------------------------
+LEDGER_HOME_ENV = "AGENT_LEARNING_HOME"
+LEDGER_PATH_ENV = "AGENT_LEARNING_LEDGER_PATH"  # overrides the DIRECTORY
+LEDGER_DIR_NAME = "ledger"
+ROWS_FILENAME = "runs.jsonl"
+CHAIN_HEAD_FILENAME = "chain.head"
+SYNC_CURSOR_FILENAME = "sync.cursor"
+
+# --- canonical row field set (ARCH §2a table; MF3) ---------------------------
+ROW_FIELDS = (
+    "schema",
+    "kind",
+    "phase",
+    "evidence_class",
+    "verdict",
+    "scores",
+    "gate_outcomes",
+    "semconv_version",
+    "manifest_address",
+    "asset_refs",
+    "trace_ids",
+    "content_bearing",
+    "redaction",
+    "created_at",
+    "run_id",
+    "chain",
+)
+
+# fields excluded from the canonical hash preimage — they ARE the hash /
+# envelope (ARCH §2a: the addressed core excludes exactly these three):
+NON_CANONICAL_FIELDS = ("created_at", "run_id", "chain")
+
+# --- tombstone fields (ARCH §2b; never rewrite a row — append) ---------------
+TOMBSTONE_FIELDS = (
+    "schema",
+    "kind",
+    "tombstones",
+    "reason",
+    "redacted_fields",
+    "evidence_class",
+    "created_at",
+    "run_id",
+    "chain",
+)
+TOMBSTONE_REASONS = ("forget", "rollback", "redaction")
+
+# --- workflow phase enum (ARCH §3) -------------------------------------------
+PHASES = ("simulate", "evals", "optimize", "redteam", "suite", "live")
+
+# --- sync attribute namespace (frozen with platform; PLATFORM-GROUNDING §6) --
+FI_KIT_RUN_ID_ATTR = "fi.kit.run_id"
+FI_KIT_PHASE_ATTR = "fi.kit.phase"
+FI_KIT_WORLD_ATTR = "fi.kit.world"
+
+# --- sync states the viewer renders (UI-UX §1.1 SYNCED column) ---------------
+SYNC_STATES = ("local", "metadata", "metadata+content", "queued", "off")
+
+
+def sync_mode() -> str:
+    """The W&B-style telemetry mode (Phase 14). ``off`` when the kill switch is
+    set (it binds everything, P8-D6); otherwise ``AGENT_LEARNING_SYNC`` —
+    ``auto`` (default: emit when keys resolve) or ``local`` (queue only)."""
+
+    if kill_switch_on():
+        return TELEMETRY_OFF_VALUE
+    value = os.environ.get(SYNC_MODE_ENV, "").strip().lower()
+    return value if value in SYNC_MODES else SYNC_MODE_AUTO
+
+
+def kill_switch_on() -> bool:
+    """``AGENT_LEARNING_TELEMETRY=off`` disables ledger + sync, binding every
+    component including vendored ``fi/*`` (P8-D6; the gate's check 1 statically
+    verifies every emission path routes through this one guard)."""
+
+    return os.environ.get(TELEMETRY_ENV, "").strip().lower() == TELEMETRY_OFF_VALUE
+
+
+def ledger_dir(root: str | Path | None = None) -> Path:
+    """Resolve the user-owned ledger DIRECTORY (ARCH Decision 9).
+
+    Precedence: explicit ``root`` arg > ``AGENT_LEARNING_LEDGER_PATH`` (the
+    directory override for tests/CI/per-project layouts) >
+    ``${AGENT_LEARNING_HOME:-~/.agent-learning}/ledger/``.
+    """
+
+    if root is not None:
+        return Path(root)
+    override = os.environ.get(LEDGER_PATH_ENV)
+    if override:
+        return Path(override)
+    home = os.environ.get(LEDGER_HOME_ENV) or (Path.home() / ".agent-learning")
+    return Path(home) / LEDGER_DIR_NAME

fi/alk/telemetry/_emit.py

@@ -0,0 +1,182 @@
+"""Export-result-aware OTLP emit (Phase 14, ARCH §2) — the truthful span sender.
+
+THE FIX for the false-``synced`` bug (RESEARCH §1.2): the OTLP HTTP exporter
+swallows export failures (it logs a 401 to stderr and returns
+``SpanExportResult.FAILURE`` without raising), so the old ``register(...)`` path
+completed its ``try`` block and reported ``synced`` while *nothing landed*. This
+module owns the exporter via a recording wrapper, so a reported success is an
+*observed* ``SpanExportResult.SUCCESS`` — never an assumption.
+
+Vendored-engine boundary (gate #72): every ``fi_instrumentation`` import is
+in-function. This module is imported only from inside the keyed branch of
+``_run`` / ``_sync`` — never on the keyless import graph.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Mapping, Sequence
+
+from ._contract import (
+    FI_KIT_PHASE_ATTR,
+    FI_KIT_RUN_ID_ATTR,
+    FI_KIT_WORLD_ATTR,
+)
+
+# project_type/version/semconv resource attrs the collector resolves a project by
+# (verified constants, fi_instrumentation/otel.py:57-89; RESEARCH §2.3).
+_RES_PROJECT_NAME = "project_name"
+_RES_PROJECT_TYPE = "project_type"
+_RES_PROJECT_VERSION_NAME = "project_version_name"
+_RES_PROJECT_VERSION_ID = "project_version_id"
+_RES_EVAL_TAGS = "eval_tags"
+_RES_METADATA = "metadata"
+_RES_SEMCONV = "semantic_convention"
+
+DEFAULT_PROJECT_NAME = "agent-learning"
+RUN_SPAN_NAME = "agent-learning.run"
+
+
+def _recording_exporter(inner: Any) -> Any:
+    """Wrap an OTLP SpanExporter so every ``export()`` result is recorded — the
+    single source of truth for 'did the span actually land'."""
+
+    from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
+
+    class _RecordingExporter(SpanExporter):  # type: ignore[misc]
+        def __init__(self, delegate: Any) -> None:
+            self._inner = delegate
+            self.results: list[Any] = []
+
+        def export(self, spans: Any) -> Any:
+            result = self._inner.export(spans)
+            self.results.append(result)
+            return result
+
+        def shutdown(self) -> Any:
+            return self._inner.shutdown()
+
+        def force_flush(self, timeout_millis: int = 30_000) -> bool:
+            flush = getattr(self._inner, "force_flush", None)
+            return flush(timeout_millis) if flush else True
+
+        @property
+        def ok(self) -> bool:
+            # success == at least one export, and EVERY export succeeded.
+            return bool(self.results) and all(
+                r is SpanExportResult.SUCCESS for r in self.results
+            )
+
+        @property
+        def last_reason(self) -> str:
+            if not self.results:
+                return "no_export_attempted"
+            return "ok" if self.ok else "export_rejected"
+
+    return _RecordingExporter(inner)
+
+
+def _build_provider(project_name: str, headers: Mapping[str, str]) -> tuple[Any, Any]:
+    """Build a private tracer provider (NOT the global one — RESEARCH §4) with the
+    FI resource attrs the dashboard filters on, and swap in the recording exporter
+    so we can observe the real export result. Returns (provider, recorder)."""
+
+    import uuid
+
+    import fi_instrumentation.otel as fio
+    from fi_instrumentation.otel import SimpleSpanProcessor, Transport
+    from fi_instrumentation.settings import UuidIdGenerator
+    from opentelemetry.sdk.resources import Resource
+
+    # SimpleSpanProcessor(headers, transport) builds the correctly-configured OTLP
+    # exporter (endpoint resolved from FI_BASE_URL/BASE_URL). We then swap its
+    # exporter for the recording wrapper — reusing fi's construction, observing
+    # the result (the HTTPSpanExporter ctor itself does not accept transport).
+    processor = SimpleSpanProcessor(
+        headers=dict(headers), transport=Transport.HTTP
+    )
+    recorder = _recording_exporter(processor.span_exporter)
+    processor.span_exporter = recorder
+
+    resource = Resource(
+        attributes={
+            _RES_PROJECT_NAME: project_name,
+            _RES_PROJECT_TYPE: "experiment",
+            _RES_PROJECT_VERSION_NAME: DEFAULT_PROJECT_NAME,
+            _RES_PROJECT_VERSION_ID: str(uuid.uuid4()),
+            _RES_EVAL_TAGS: "[]",
+            _RES_METADATA: "{}",
+            _RES_SEMCONV: "fi",
+        }
+    )
+    provider = fio.TracerProvider(
+        resource=resource,
+        id_generator=UuidIdGenerator(),
+        transport=Transport.HTTP,
+        verbose=False,
+    )
+    provider.add_span_processor(processor)
+    return provider, recorder
+
+
+def _scalar(value: Any) -> Any:
+    """OTel span attributes accept str/bool/int/float (and homogeneous lists).
+    Coerce anything else to a compact string so a metric mapping never breaks
+    the emit."""
+
+    if isinstance(value, (str, bool, int, float)):
+        return value
+    import json
+
+    return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)[:1024]
+
+
+def keyed_emit(
+    *,
+    span_name: str,
+    root_attrs: Mapping[str, Any],
+    children: Sequence[tuple[str, Mapping[str, Any]]] = (),
+    project_name: str,
+    headers: Mapping[str, str],
+    run_id: str = "",
+    phase: str = "",
+    world: str | None = None,
+) -> dict[str, Any]:
+    """Emit one run as a trace (root span + child spans) to the user's account and
+    report the OBSERVED export result.
+
+    Returns ``{status, trace_id, reason}`` where ``status`` is ``"synced"`` only
+    when the recording exporter saw ``SpanExportResult.SUCCESS``; otherwise
+    ``"export_failed"`` with a reason. Any exception degrades to local
+    (``"deferred"``) — never propagates into the caller's run (R§3.5).
+    """
+
+    try:
+        provider, recorder = _build_provider(project_name, headers)
+        tracer = provider.get_tracer("fi.alk.telemetry")
+        trace_id_hex = ""
+        with tracer.start_as_current_span(span_name) as root:
+            trace_id_hex = format(root.get_span_context().trace_id, "032x")
+            if run_id:
+                root.set_attribute(FI_KIT_RUN_ID_ATTR, run_id)
+            if phase:
+                root.set_attribute(FI_KIT_PHASE_ATTR, phase)
+            if world:
+                root.set_attribute(FI_KIT_WORLD_ATTR, str(world))
+            for key, value in root_attrs.items():
+                root.set_attribute(str(key), _scalar(value))
+            for child_name, child_attrs in children:
+                with tracer.start_as_current_span(str(child_name)) as child:
+                    for key, value in (child_attrs or {}).items():
+                        child.set_attribute(str(key), _scalar(value))
+        provider.force_flush()
+        provider.shutdown()
+    except BaseException as exc:  # noqa: BLE001 — degrade-to-local (R§3.5)
+        return {"status": "deferred", "trace_id": None, "reason": f"{type(exc).__name__}: {exc}"}
+
+    if recorder.ok:
+        return {"status": "synced", "trace_id": trace_id_hex, "reason": None}
+    return {
+        "status": "export_failed",
+        "trace_id": None,  # nothing landed → no viewable trace
+        "reason": recorder.last_reason,
+    }