aftersight 0.1.0__py3-none-any.whl

aftersight/__init__.py

@@ -0,0 +1,21 @@
+"""Observability infrastructure for self-improving agents.
+
+    import aftersight
+    aftersight.start()
+
+Or, with no code change at all:
+
+    aftersight run python my_agent.py
+"""
+
+from aftersight.run.api import (
+    artifact_dir,
+    current,
+    log,
+    span,
+    start,
+    trace,
+)
+
+__all__ = ["start", "span", "trace", "log", "current", "artifact_dir"]
+__version__ = "0.1.0"

aftersight/assets/NAVIGATE.md

@@ -0,0 +1,129 @@
+# Navigating these runs
+
+You are a coding agent. This directory is the execution history of the agent
+system in this repo. Read this file once, then use `rg` and `jq`.
+
+## Layout
+
+```
+index.jsonl                one line per run, start here for "which run?"
+latest -> runs/<run_id>    most recent run
+sessions/<session_id>      -> the run folder that session resumed into
+runs/<run_id>/
+  outline.md               folded map of the run, read this first
+  agent.logs               full transcript, nothing truncated
+  trace.jsonl              same events, machine-readable, has parent_seq
+  analytics.json           status, timings, per-agent cost, error list
+  meta.json                framework, git sha, argv, attempts[]
+  blobs/<sha>.txt          payloads over 8 KB, plain text
+  artifacts/               whatever the app chose to keep
+```
+
+Run ids are `YYYYMMDD_HHMMSS_hash`, so they sort chronologically. Glob run
+files as `runs/*/...` and never from the root: `latest` and `sessions/` are
+symlinks into `runs/`, so a root glob counts every run twice.
+
+## The three-step loop
+
+1. **Which run?** `index.jsonl`
+2. **What happened in it?** `runs/<run_id>/outline.md`, then jump to a `#seq`
+3. **Exactly what went in and out?** `runs/<run_id>/agent.logs` at that `#seq`
+
+## Anchors
+
+Every event has a zero-padded sequence number. It is the same number in
+`outline.md`, `agent.logs` and `trace.jsonl`, so you can move between them:
+
+```sh
+rg -n '^#0016 ' latest/agent.logs          # jump to one event
+rg -n '#0016' runs/<run_id>/               # every mention of it
+```
+
+Multi-line payloads are framed. The closing line repeats the anchor, so a hit
+landing inside a block can find both edges:
+
+```
+#0004 14:22:35.512  +2.4s  llm.response  triage   stop=end_turn · 386 tok · $0.0005
+┌──────────────────────────────────
+{"decision": "work", ...}
+└─ #0004 end · 86 B ───────────────
+```
+
+Column 3 is the delta from the previous event. Scan it to find slow steps. A
+negative delta means an event whose span declared its kind late and was therefore
+placed at its end rather than its start; the timestamp is still correct.
+
+## Recipes
+
+```sh
+# failed runs, newest last
+jq -r 'select(.status!="ok") | "\(.run_id) \(.errors) errs $\(.cost_usd) \(.last_error)"' index.jsonl
+
+# the most common failure across all history
+rg -oN '"error_type": "[A-Za-z]+"' runs/*/trace.jsonl | cut -d'"' -f4 | sort | uniq -c | sort -rn
+
+# every error in one run, with its anchor
+#   filter on payload.error_type, not .status, agent.end also carries status "error"
+jq -r 'select(.payload.error_type) | "#\(.seq|tostring|("0000"+.)[-4:]) \(.name) \(.payload.error_type): \(.payload.error)"' latest/trace.jsonl
+
+# what a tool was actually called with, every time
+rg -N '^#\d+ .* tool\.call     run_python' runs/*/agent.logs
+
+# read one event and its payload block
+rg -n -A40 '^#0023 ' latest/agent.logs
+
+# where the money went
+jq -r '.cost.by_agent | to_entries[] | "\(.key) $\(.value.cost_usd)"' latest/analytics.json
+
+# slowest steps in a run, exclude run.* or the run total wins every time
+jq -r 'select(.dur_ms!=null and (.type|startswith("run.")|not)) | "\(.dur_ms)ms #\(.seq|tostring|("0000"+.)[-4:]) \(.type) \(.name)"' latest/trace.jsonl | sort -rn | head
+
+# did this session get resumed, and why did the earlier attempt end?
+jq '.attempts' latest/meta.json
+rg -n '^==== attempt|^---- attempt' latest/agent.logs
+
+# a prompt too big to inline
+rg -n 'blobs/' latest/agent.logs        # find the pointer
+rg -n 'PCA' latest/blobs/*.txt          # then grep the blob like any text file
+
+# same failure across runs, or just this one?
+rg -lN 'TimeoutError' runs/*/agent.logs
+```
+
+## Counting things correctly
+
+Three traps, each of which silently produces a wrong answer rather than an error:
+
+- `status: "error"` appears on `agent.end` too. Count failures by
+  `payload.error_type`.
+- `run.end` carries the run total in `dur_ms`. Exclude `run.*` when ranking steps.
+- `agent.end` carries a cost rollup. Sum leaf costs, or read
+  `analytics.json → cost.by_agent`.
+
+## Event types
+
+`run.start` `run.resume` `run.end` · `agent.start` `agent.end` ·
+`llm.prompt` `llm.response` · `tool.call` `tool.result` `tool.error` ·
+`error` · `artifact` · `log`
+
+Namespaced on purpose: `rg 'tool\.'` gets all tool activity, `rg '\.error'`
+gets every failure shape.
+
+## Resume
+
+A session that is resumed appends into the same run folder under a new
+`==== attempt N ====` banner. Sequence numbers continue across attempts and
+are never reused. An attempt that died without `run.end` is marked
+`---- attempt N ended ... without run.end ----`, which is how you tell a crash
+from a clean failure.
+
+## When you are looking for a systemic problem
+
+Do not read one run. Aggregate first:
+
+```sh
+rg -oN '"error_type": "[A-Za-z]+"' runs/*/trace.jsonl | cut -d'"' -f4 | sort | uniq -c | sort -rn
+jq -s 'group_by(.status)[] | {status: .[0].status, n: length}' index.jsonl
+```
+
+Then open the cheapest run that shows the pattern, not the biggest.

aftersight/assets/skill/SKILL.md

@@ -0,0 +1,39 @@
+---
+name: aftersight
+description: Use when investigating how this project's agent behaved: a failed or slow agent run, a recurring tool failure, unexpected model output, rising cost, or any question about what an agent actually sent and received. Reads the execution traces in .runs/.
+---
+
+# Reading this project's agent traces
+
+Agent runs in this repo record to `.runs/`. That directory is the evidence for
+what an agent actually did. Read it before theorising about behaviour.
+
+Start with `.runs/NAVIGATE.md`. It documents the layout, the `#seq` anchor
+scheme and a set of verified `rg`/`jq` recipes.
+
+## The loop
+
+1. `.runs/index.jsonl` holds one line per run. Pick the run.
+2. `.runs/runs/<run_id>/outline.md` is the folded map. Find the `#seq` that
+   matters.
+3. `.runs/runs/<run_id>/agent.logs` at that anchor has the exact prompt,
+   completion, tool arguments or traceback.
+
+## Rules that keep answers correct
+
+- Aggregate before you read. One run shows an incident; `runs/*/trace.jsonl`
+  shows whether it is systemic.
+- Glob as `runs/*/…`. `latest` and `sessions/` are symlinks into `runs/`, so
+  globbing the root double-counts.
+- Count failures by `payload.error_type`, not `status == "error"`, because
+  `agent.end` carries that status too.
+- Exclude `run.*` when ranking slow steps; `run.end` holds the run total.
+- A prompt over 8 KB lives in `blobs/` as plain text. Grep it like any file.
+- Sequence numbers continue across a resumed session, and an attempt that died
+  without `run.end` is marked as such. A crash and a clean failure are different
+  findings.
+
+## When asked to improve the agent
+
+Ground every claim in an anchor. "Tool `x` timed out in 4 of the last 9 runs
+(`#0044`, `#0071`, …)" is actionable; "the executor seems flaky" is not.

aftersight/autostart.py

@@ -0,0 +1,8 @@
+import os
+
+from aftersight import constants
+
+if os.environ.get(constants.ENV_AUTOSTART) == "1":
+    import aftersight
+
+    aftersight.start(session_id=os.environ.get(constants.ENV_SESSION) or None)

aftersight/cli.py

@@ -0,0 +1,91 @@
+"""Two commands. Anything else `rg` and `jq` already do better.
+
+    aftersight run python my_agent.py     # record, no code change
+    aftersight skill                      # teach your coding agent to read it
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+from aftersight import constants
+
+_SITECUSTOMIZE = '''\
+# Generated by `aftersight run`. Chains to any real sitecustomize.
+import os, sys
+
+_here = os.path.dirname(os.path.abspath(__file__))
+sys.path[:] = [p for p in sys.path if os.path.abspath(p) != _here]
+sys.modules.pop("sitecustomize", None)
+try:
+    import sitecustomize  # noqa: F401  the project's own, if it has one
+except ImportError:
+    pass
+try:
+    import aftersight.autostart  # noqa: F401
+except Exception as exc:  # noqa: BLE001
+    print(f"aftersight: autostart failed: {exc}", file=sys.stderr)
+'''
+
+
+def cmd_run(args: argparse.Namespace) -> int:
+    """Records a program without editing it.
+
+    A generated `sitecustomize` on PYTHONPATH is how coverage-style tools attach
+    to an interpreter they do not control; it also survives `uv run`, `pytest`
+    and any wrapper that eventually execs Python.
+    """
+    if not args.command:
+        print("usage: aftersight run <command> [args...]", file=sys.stderr)
+        return 2
+    staging = tempfile.mkdtemp(prefix="aftersight-")
+    try:
+        (Path(staging) / "sitecustomize.py").write_text(_SITECUSTOMIZE)
+        env = dict(os.environ)
+        env[constants.ENV_AUTOSTART] = "1"
+        env["PYTHONPATH"] = os.pathsep.join(
+            [staging] + ([env["PYTHONPATH"]] if env.get("PYTHONPATH") else []))
+        if args.session:
+            env[constants.ENV_SESSION] = args.session
+        if args.root:
+            env[constants.ENV_ROOT] = args.root
+        return subprocess.run(args.command, env=env).returncode
+    finally:
+        shutil.rmtree(staging, ignore_errors=True)
+
+
+def cmd_skill(args: argparse.Namespace) -> int:
+    source = Path(__file__).resolve().parent / "assets" / "skill" / "SKILL.md"
+    target = Path(args.into) / "aftersight" / "SKILL.md"
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text(source.read_text())
+    print(f"installed {target}")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="aftersight", description=__doc__)
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    run = sub.add_parser("run", help="run a command with recording enabled")
+    run.add_argument("--session", help="session id; a resumed session reuses its run folder")
+    run.add_argument("--root", help="telemetry root (default .runs)")
+    run.add_argument("command", nargs=argparse.REMAINDER)
+    run.set_defaults(func=cmd_run)
+
+    skill = sub.add_parser("skill", help="install the Claude Code skill")
+    skill.add_argument("--into", default=".claude/skills", help="skills directory")
+    skill.set_defaults(func=cmd_skill)
+
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

aftersight/config.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import logging
+import os
+from dataclasses import dataclass, replace
+from pathlib import Path
+
+from aftersight import constants
+
+
+@dataclass(frozen=True)
+class Config:
+    enabled: bool = True
+    root: Path = Path(constants.DEFAULT_ROOT)
+    keep_runs: int = constants.DEFAULT_KEEP_RUNS
+    blob_max_bytes: int = constants.DEFAULT_BLOB_MAX_BYTES
+    blob_preview_lines: int = constants.DEFAULT_BLOB_PREVIEW_LINES
+    capture_otel: bool = True
+    capture_logging: bool = True
+    log_level: int = logging.WARNING
+    redact: bool = True
+    quiet: bool = False
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    raw = os.environ.get(name)
+    if raw is None:
+        return default
+    lowered = raw.strip().lower()
+    if lowered in constants.TRUTHY:
+        return True
+    if lowered in constants.FALSY:
+        return False
+    return default
+
+
+def _env_int(name: str, default: int) -> int:
+    try:
+        return int(os.environ[name])
+    except (KeyError, ValueError):
+        return default
+
+
+def load(**overrides) -> Config:
+    """Defaults, then environment, then explicit `start()` arguments."""
+    cfg = Config(
+        enabled=_env_bool(constants.ENV_ENABLED, True),
+        root=Path(os.environ.get(constants.ENV_ROOT, constants.DEFAULT_ROOT)),
+        keep_runs=_env_int(constants.ENV_KEEP_RUNS, constants.DEFAULT_KEEP_RUNS),
+        blob_max_bytes=_env_int(constants.ENV_BLOB_MAX, constants.DEFAULT_BLOB_MAX_BYTES),
+        capture_otel=_env_bool(constants.ENV_OTEL, True),
+        capture_logging=_env_bool(constants.ENV_LOGGING, True),
+        redact=_env_bool(constants.ENV_REDACT, True),
+        quiet=_env_bool(constants.ENV_QUIET, False),
+    )
+    known = {f for f in Config.__dataclass_fields__}
+    clean = {k: v for k, v in overrides.items() if k in known and v is not None}
+    if "root" in clean:
+        clean["root"] = Path(clean["root"])
+    return replace(cfg, **clean) if clean else cfg

aftersight/constants.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+# Dotted on purpose: `rg 'tool\.'` finds all tool activity, `rg '\.error'`
+# finds every failure shape.
+
+
+class EventType:
+    RUN_START = "run.start"
+    RUN_RESUME = "run.resume"
+    RUN_END = "run.end"
+    AGENT_START = "agent.start"
+    AGENT_END = "agent.end"
+    LLM_PROMPT = "llm.prompt"
+    LLM_RESPONSE = "llm.response"
+    TOOL_CALL = "tool.call"
+    TOOL_RESULT = "tool.result"
+    TOOL_ERROR = "tool.error"
+    ERROR = "error"
+    LOG = "log"
+
+
+# Pointers live outside RUNS_DIR so a `runs/*/...` glob cannot count a run twice.
+
+DEFAULT_ROOT = ".runs"
+RUNS_DIR = "runs"
+SESSIONS_DIR = "sessions"
+LATEST_LINK = "latest"
+NAVIGATE_FILE = "NAVIGATE.md"
+INDEX_FILE = "index.jsonl"
+GITIGNORE_MARK = "# aftersight"
+
+TRANSCRIPT_FILE = "agent.logs"
+TRACE_FILE = "trace.jsonl"
+OUTLINE_FILE = "outline.md"
+ANALYTICS_FILE = "analytics.json"
+META_FILE = "meta.json"
+BLOBS_DIR = "blobs"
+ARTIFACTS_DIR = "artifacts"
+
+RUN_ID_TIME_FORMAT = "%Y%m%d_%H%M%S"
+RUN_ID_SUFFIX_LEN = 4
+
+DEFAULT_KEEP_RUNS = 50
+DEFAULT_BLOB_MAX_BYTES = 8192
+DEFAULT_BLOB_PREVIEW_LINES = 12
+
+ENV_ENABLED = "AFTERSIGHT"
+ENV_ROOT = "AFTERSIGHT_ROOT"
+ENV_KEEP_RUNS = "AFTERSIGHT_KEEP_RUNS"
+ENV_BLOB_MAX = "AFTERSIGHT_BLOB_MAX"
+ENV_OTEL = "AFTERSIGHT_OTEL"
+ENV_LOGGING = "AFTERSIGHT_LOGGING"
+ENV_REDACT = "AFTERSIGHT_REDACT"
+ENV_QUIET = "AFTERSIGHT_QUIET"
+ENV_AUTOSTART = "AFTERSIGHT_AUTOSTART"
+ENV_SESSION = "AFTERSIGHT_SESSION"
+
+TRUTHY = {"1", "true", "yes", "on"}
+FALSY = {"0", "false", "no", "off"}
+
+FRAME_WIDTH = 70
+INLINE_ARG_LIMIT = 200
+INLINE_BLOCK_LIMIT = 120
+OUTLINE_NAME_COL = 28
+
+#: Payload keys the transcript renders itself, so the generic key=value tail
+#: does not repeat them.
+RENDERED_PAYLOAD_KEYS = {"text", "text_ref", "preview", "bytes"}
+
+TRANSCRIPT_TIME_FORMAT = "%H:%M:%S"
+BANNER_TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
+
+REDACTION_MASK = "[REDACTED]"
+
+#: A pattern with a capture group masks only the group, so the surrounding
+#: `api_key=` stays readable.
+SECRET_PATTERNS = [
+    r"sk-[A-Za-z0-9_\-]{16,}",
+    r"(?i:bearer)\s+[A-Za-z0-9._\-]{16,}",
+    r"(?i:(?:api[_-]?key|secret|token|password)\"?\s*[:=]\s*\"?)([A-Za-z0-9._\-]{12,})",
+    r"gh[pousr]_[A-Za-z0-9]{16,}",
+    r"AKIA[0-9A-Z]{16}",
+]
+
+# Three are read: OTel `gen_ai.*` semantic conventions, OpenInference `llm.*`,
+# and a generic `input.value` / `output.value` fallback. Add an attribute name
+# here rather than at a call site.
+
+SPAN_KIND_ATTRS = ("openinference.span.kind", "gen_ai.operation.name",
+                   "traceloop.span.kind")
+
+LLM_SPAN_KINDS = {"LLM", "chat", "text_completion", "generate_content"}
+TOOL_SPAN_KINDS = {"TOOL", "execute_tool"}
+CONTAINER_SPAN_KINDS = {"AGENT", "CHAIN", "GRAPH", "WORKFLOW", "invoke_agent",
+                        "create_agent"}
+
+#: Indexed message attributes, joined into one text block.
+PROMPT_TEMPLATES = ("gen_ai.prompt.{i}.content", "llm.input_messages.{i}.message.content")
+COMPLETION_TEMPLATES = ("gen_ai.completion.{i}.content",
+                        "llm.output_messages.{i}.message.content")
+
+PROMPT_ATTRS = ("gen_ai.prompt", "input.value", "llm.prompts")
+COMPLETION_ATTRS = ("gen_ai.completion", "output.value")
+MODEL_ATTRS = ("gen_ai.request.model", "llm.model_name")
+TOOL_NAME_ATTRS = ("tool.name", "gen_ai.tool.name")
+TOOL_ARG_ATTRS = ("tool.parameters", "input.value")
+COST_ATTRS = ("gen_ai.usage.cost", "llm.cost.total")
+FINISH_REASON_ATTRS = ("gen_ai.response.finish_reasons",)
+INPUT_TOKEN_ATTRS = ("gen_ai.usage.input_tokens", "gen_ai.usage.prompt_tokens",
+                     "llm.token_count.prompt")
+OUTPUT_TOKEN_ATTRS = ("gen_ai.usage.output_tokens", "gen_ai.usage.completion_tokens",
+                      "llm.token_count.completion")
+
+MAX_INDEXED_MESSAGES = 64
+
+#: OpenInference instrumentors activated when already installed. Nothing is
+#: installed on the user's behalf.
+INSTRUMENTORS = [
+    ("openinference.instrumentation.agno", "AgnoInstrumentor", "agno"),
+    ("openinference.instrumentation.langchain", "LangChainInstrumentor", "langchain"),
+    ("openinference.instrumentation.crewai", "CrewAIInstrumentor", "crewai"),
+    ("openinference.instrumentation.llama_index", "LlamaIndexInstrumentor", "llama-index"),
+    ("openinference.instrumentation.openai_agents", "OpenAIAgentsInstrumentor", "openai-agents"),
+    ("openinference.instrumentation.smolagents", "SmolagentsInstrumentor", "smolagents"),
+]

aftersight/events/schemas.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+
+@dataclass
+class Event:
+    run_id: str
+    seq: int
+    attempt: int
+    ts: float
+    type: str
+    name: str | None = None
+    parent_seq: int | None = None
+    status: str | None = None
+    dur_ms: int | None = None
+    payload: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def anchor(self) -> str:
+        """`#0016`. The same string in outline.md, agent.logs and trace.jsonl."""
+        return f"#{self.seq:04d}"
+
+    def datetime(self) -> datetime:
+        """Local time with an explicit offset. A user correlating a run with their
+        own terminal reads a local clock, and the offset keeps it unambiguous."""
+        return datetime.fromtimestamp(self.ts).astimezone()
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "run_id": self.run_id,
+            "seq": self.seq,
+            "attempt": self.attempt,
+            "ts": round(self.ts, 3),
+            "iso": self.datetime().isoformat(),
+            "type": self.type,
+            "name": self.name,
+            "parent_seq": self.parent_seq,
+            "status": self.status,
+            "dur_ms": self.dur_ms,
+            "payload": self.payload,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "Event":
+        return cls(
+            run_id=data["run_id"],
+            seq=data["seq"],
+            attempt=data.get("attempt", 1),
+            ts=data["ts"],
+            type=data["type"],
+            name=data.get("name"),
+            parent_seq=data.get("parent_seq"),
+            status=data.get("status"),
+            dur_ms=data.get("dur_ms"),
+            payload=data.get("payload", {}),
+        )

aftersight/events/sink.py

@@ -0,0 +1,58 @@
+"""Sequence allocation and fan-out to the writers.
+
+`emit()` is thread-safe: concurrent agents get unique, contiguous sequence
+numbers. Sequence numbers continue across a resume, which is why `start_seq`
+exists.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from collections.abc import Callable
+from typing import Any
+
+from aftersight.events.schemas import Event
+from aftersight.utils import log_error, redact
+
+
+class EventSink:
+    def __init__(self, run_id: str, attempt: int = 1, *, start_seq: int = 0,
+                 redact_payloads: bool = True):
+        self.run_id = run_id
+        self.attempt = attempt
+        self._seq = start_seq
+        self._redact = redact_payloads
+        self._writers: list[Callable[[Event], None]] = []
+        self._lock = threading.Lock()
+
+    def add_writer(self, writer: Callable[[Event], None]) -> None:
+        self._writers.append(writer)
+
+    @property
+    def last_seq(self) -> int:
+        return self._seq
+
+    def emit(self, type: str, name: str | None = None, *, parent_seq: int | None = None,
+             status: str | None = None, dur_ms: int | None = None,
+             ts: float | None = None, **payload: Any) -> Event:
+        with self._lock:
+            self._seq += 1
+            event = Event(
+                run_id=self.run_id,
+                seq=self._seq,
+                attempt=self.attempt,
+                ts=time.time() if ts is None else ts,
+                type=type,
+                name=name,
+                parent_seq=parent_seq,
+                status=status,
+                dur_ms=dur_ms,
+                payload=redact(payload) if self._redact else payload,
+            )
+            for write in self._writers:
+                try:
+                    write(event)
+                except Exception as exc:  # noqa: BLE001 (never break the host app)
+                    log_error(f"writer {write!r} failed on {event.anchor}: {exc}")
+        return event