agentdynamics 0.4.0__py3-none-any.whl

agentdynamics/bootstrap/sitecustomize.py

@@ -0,0 +1,27 @@
+"""Loaded automatically by `agentdynamics run <command>` (placed first on PYTHONPATH).
+
+Instruments the child Python process before any user code runs, like `ddtrace-run` or `opentelemetry-instrument`.
+"""
+import os
+import sys
+
+try:
+    import agentdynamics
+
+    agentdynamics.init(quiet=os.environ.get("AGENTDYNAMICS_QUIET") == "1")
+except Exception as ex:  # never break the user's program
+    print(f"[agentdynamics] auto-instrumentation disabled: {ex}", file=sys.stderr)
+
+# Chain to any sitecustomize that was shadowed by ours.
+_here = os.path.dirname(os.path.abspath(__file__))
+for _p in sys.path:
+    if _p and os.path.abspath(_p) != _here and os.path.isfile(os.path.join(_p, "sitecustomize.py")):
+        import importlib.util
+
+        _spec = importlib.util.spec_from_file_location("_user_sitecustomize", os.path.join(_p, "sitecustomize.py"))
+        _mod = importlib.util.module_from_spec(_spec)
+        try:
+            _spec.loader.exec_module(_mod)
+        except Exception as ex:
+            print(f"[agentdynamics] user sitecustomize failed: {ex}", file=sys.stderr)
+        break

agentdynamics/collectors/aegis_audit.py

@@ -0,0 +1,72 @@
+"""Aegis audit logs -> AgentDynamics runs (out-of-process path).
+
+For agents that run Aegis without the in-process integration (a different language runtime, CI
+conformance runs, historical logs), point the inbox at the JSONL audit file (`AuditLog(path=...)`)
+or POST records to /api/ingest/records. Records are grouped into runs by the correlation id Aegis
+stamps into `details.ctx` (run_id / trace_id) when a context provider is registered, otherwise by
+grant id.
+
+Aegis records hash the arguments rather than storing them, so these runs show *what was decided*
+(tool, verdict, rule, agent, depth) but not argument values. Policy export needs argument values and
+therefore uses the in-process integration.
+"""
+from __future__ import annotations
+
+REQUIRED = {"seq", "grant_id", "prev_hash", "rule", "allowed", "tool"}
+
+
+def is_record(rec):
+    return isinstance(rec, dict) and REQUIRED <= set(rec)
+
+
+def group_key(rec):
+    ctx = (rec.get("details") or {}).get("ctx") or {}
+    return ctx.get("run_id") or ctx.get("trace_id") or f"grant:{rec.get('grant_id')}"
+
+
+def span_id(rec):
+    return rec.get("hash") or f"{rec.get('grant_id')}:{rec.get('seq')}:{rec.get('ts')}"
+
+
+def build_payload(group, records):
+    """Generic-run payload for one group of audit records."""
+    records = sorted(records, key=lambda r: (r.get("ts") or 0, r.get("seq") or 0))
+    ctx = next(((r.get("details") or {}).get("ctx") for r in records if (r.get("details") or {}).get("ctx")), {}) or {}
+    t0 = records[0].get("ts")
+    steps = [{"kind": "prompt", "ts": t0, "text": ctx.get("workflow") or f"Aegis-governed run ({records[0].get('agent')})"}]
+    pending = {}  # (grant, tool, args_digest) -> index of an admitted step a later record may deny
+    for r in records:
+        tool, rule, ts = r.get("tool"), r.get("rule"), r.get("ts")
+        det = r.get("details") or {}
+        base = {"ts": ts, "end_ts": ts, "agent": r.get("agent"), "grant_depth": r.get("depth"), "governed": True,
+                "rule": rule, "guard": r.get("guard"), "node": (det.get("ctx") or {}).get("node")}
+        if tool == "model.spend":
+            if rule == "budget.settled":
+                steps.append({**base, "kind": "llm", "model": "model", "cost": float(det.get("usd") or 0),
+                              "input_tokens": int(det.get("tokens") or 0), "output_tokens": 0})
+            elif not r.get("allowed"):
+                steps.append({**base, "kind": "llm", "model": "model", "denied": True, "error": r.get("reason")})
+            continue
+        if tool == "agent.revoke":
+            steps.append({"kind": "notice", "name": "revoked", "ts": ts, "agent": r.get("agent"), "rule": rule,
+                          "text": f"{r.get('agent')}: revoked"})
+            continue
+        key = (r.get("grant_id"), tool, r.get("args_digest"))
+        if not r.get("allowed") and key in pending:
+            # a budget charge or post-guard refused a call that had been admitted: one call, denied
+            i = pending.pop(key)
+            steps[i].update(denied=True, rule=rule, guard=r.get("guard"), error=f"[{rule}] {r.get('reason') or ''}")
+            continue
+        if tool == "agent.spawn" and r.get("allowed"):
+            steps.append({**base, "kind": "span", "span_kind": "agent", "name": "sub-agent", "rule": "spawn.granted",
+                          "start_ts": ts})
+            continue
+        step = {**base, "kind": "tool", "name": tool}
+        if r.get("allowed"):
+            pending[key] = len(steps)
+        else:
+            step.update(denied=True, error=f"[{rule}] {r.get('reason') or ''}")
+        steps.append(step)
+    return {"id": f"aegis-{group}", "source": "aegis", "workflow": ctx.get("workflow") or "aegis-governed",
+            "project": ctx.get("project") or "aegis", "agent": records[0].get("agent"), "framework": "aegis",
+            "status": "ok", "complete": True, "steps": steps}

agentdynamics/collectors/claude_code.py

@@ -0,0 +1,257 @@
+"""Collector for Claude Code session transcripts (~/.claude/projects/**/*.jsonl).
+
+Normalizes each transcript file into a *run*: session metadata plus an ordered
+list of steps (prompt / llm / tool / notice). Analysis never looks at the raw
+transcript format, so other agent frameworks only need their own collector.
+"""
+import hashlib
+import json
+import os
+from datetime import datetime
+
+from .. import pricing
+from ..phases import classify
+
+DEFAULT_ROOT = os.path.join(os.path.expanduser("~"), ".claude", "projects")
+
+
+def parse_ts(s):
+    if not s:
+        return None
+    try:
+        return datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp()
+    except ValueError:
+        return None
+
+
+def _content_len(c):
+    if c is None:
+        return 0
+    if isinstance(c, str):
+        return len(c)
+    if isinstance(c, list):
+        n = 0
+        for b in c:
+            if isinstance(b, dict):
+                if b.get("type") == "text":
+                    n += len(b.get("text", ""))
+                elif b.get("type") == "image":
+                    n += 6000  # ~1.5k tokens per image, rough
+                else:
+                    n += len(json.dumps(b))
+        return n
+    return len(json.dumps(c))
+
+
+def _text_of(c):
+    if isinstance(c, str):
+        return c
+    if isinstance(c, list):
+        return "\n".join(b.get("text", "") for b in c if isinstance(b, dict) and b.get("type") == "text")
+    return ""
+
+
+def _classify_user_text(text, entry):
+    """Return (kind, subkind) for a user text message."""
+    t = text.lstrip()
+    origin = (entry.get("origin") or {}).get("kind")
+    if entry.get("isCompactSummary") or t.startswith("This session is being continued from a previous conversation"):
+        return "notice", "compaction"
+    if t.startswith("[Request interrupted by user"):
+        return "notice", "interrupt"
+    if origin == "task-notification" or t.startswith("<task-notification"):
+        return "notice", "task_notification"
+    if t.startswith("<local-command-stdout") or t.startswith("<local-command-caveat") or t.startswith("<local-command-stderr"):
+        return "skip", None
+    if entry.get("isMeta"):
+        return "skip", None
+    if t.startswith("Base directory for this skill") or t.startswith("<system-reminder>"):
+        return "skip", None
+    if t.startswith("<scheduled-task"):
+        return "prompt", "scheduled"
+    if t.startswith("<command-name>") or t.startswith("<command-message>"):
+        return "prompt", "command"
+    return "prompt", "human"
+
+
+def _hash_input(name, inp):
+    return hashlib.sha1((name + json.dumps(inp, sort_keys=True, default=str)).encode()).hexdigest()[:16]
+
+
+def parse_file(path, root=DEFAULT_ROOT):
+    rel = os.path.relpath(path, root)
+    parts = rel.replace("\\", "/").split("/")
+    project_dir = parts[0]
+    is_sub = "subagents" in parts
+    stem = os.path.splitext(parts[-1])[0]
+    parent_id = parts[1] if is_sub else None
+    run = {
+        "id": stem if not is_sub else f"{parent_id}:{stem}",
+        "source": "claude-code",
+        "file": path,
+        "project": project_dir,
+        "cwd": None,
+        "title": None,
+        "agent_name": None,
+        "parent_id": parent_id,
+        "is_subagent": is_sub,
+        "workflow": parts[3] if is_sub and len(parts) > 4 and parts[2] == "workflows" else None,
+        "version": None,
+        "git_branch": None,
+        "entrypoint": None,
+        "steps": [],
+    }
+    steps = run["steps"]
+    llm_by_msg = {}
+    tool_by_id = {}
+    last_ts = None
+
+    with open(path, encoding="utf-8", errors="replace") as f:
+        for line in f:
+            try:
+                e = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            et = e.get("type")
+            if et == "custom-title":
+                run["title"] = e.get("customTitle")
+                continue
+            if et == "agent-name":
+                run["agent_name"] = e.get("agentName")
+                continue
+            ts = parse_ts(e.get("timestamp"))
+            if e.get("cwd") and not run["cwd"]:
+                run["cwd"] = e["cwd"]
+                run["version"] = e.get("version")
+                run["git_branch"] = e.get("gitBranch")
+                run["entrypoint"] = e.get("entrypoint")
+
+            if et == "system":
+                st = e.get("subtype")
+                if st in ("api_error", "compact_boundary"):
+                    steps.append({"kind": "notice", "name": st if st != "compact_boundary" else "compaction",
+                                  "ts": ts, "text": str(e.get("content") or e.get("error") or "")[:300]})
+                if ts:
+                    last_ts = ts
+                continue
+
+            msg = e.get("message")
+            if et == "assistant" and isinstance(msg, dict):
+                mid = msg.get("id") or e.get("uuid")
+                step = llm_by_msg.get(mid)
+                usage = msg.get("usage") or {}
+                if step is None:
+                    step = {
+                        "kind": "llm", "name": msg.get("model"), "model": msg.get("model"), "ts": ts,
+                        "start_ts": last_ts if last_ts and ts and ts - last_ts < 3600 else ts,
+                        "end_ts": ts, "text_chars": 0, "thinking_blocks": 0, "tool_calls": 0,
+                        "sidechain": bool(e.get("isSidechain")), "text": "",
+                    }
+                    llm_by_msg[mid] = step
+                    steps.append(step)
+                step["end_ts"] = ts or step["end_ts"]
+                step["stop_reason"] = msg.get("stop_reason") or step.get("stop_reason")
+                cc = usage.get("cache_creation") or {}
+                cw_total = usage.get("cache_creation_input_tokens", 0) or 0
+                cw_1h = cc.get("ephemeral_1h_input_tokens", 0) or 0
+                cw_5m = cc.get("ephemeral_5m_input_tokens", cw_total - cw_1h) or 0
+                step.update({
+                    "input_tokens": usage.get("input_tokens", 0) or 0,
+                    "output_tokens": usage.get("output_tokens", 0) or 0,
+                    "cache_read": usage.get("cache_read_input_tokens", 0) or 0,
+                    "cache_write": cw_total,
+                    "cache_write_1h": cw_1h,
+                    "cache_write_5m": cw_5m,
+                    "thinking_tokens": (usage.get("output_tokens_details") or {}).get("thinking_tokens", 0) or 0,
+                    "effort": e.get("effort") or step.get("effort"),
+                })
+                for b in msg.get("content") or []:
+                    bt = b.get("type")
+                    if bt == "text":
+                        step["text_chars"] += len(b.get("text", ""))
+                        if len(step["text"]) < 600:
+                            step["text"] += b.get("text", "")[:600]
+                    elif bt == "thinking":
+                        step["thinking_blocks"] += 1
+                    elif bt == "tool_use":
+                        step["tool_calls"] += 1
+                        name = b.get("name")
+                        inp = b.get("input") or {}
+                        phase, target = classify(name, inp)
+                        tstep = {
+                            "kind": "tool", "name": name, "ts": ts, "start_ts": ts, "end_ts": None,
+                            "phase": phase, "target": target, "input_hash": _hash_input(name, inp),
+                            "input_chars": len(json.dumps(inp)), "llm_msg": mid, "is_error": False,
+                            "output_chars": 0, "tool_use_id": b.get("id"),
+                            "input_preview": json.dumps(inp)[:400],
+                        }
+                        tool_by_id[b.get("id")] = tstep
+                        steps.append(tstep)
+                if ts:
+                    last_ts = ts
+                continue
+
+            if et == "user" and isinstance(msg, dict):
+                content = msg.get("content")
+                blocks = content if isinstance(content, list) else []
+                tool_results = [b for b in blocks if isinstance(b, dict) and b.get("type") == "tool_result"]
+                if tool_results:
+                    for b in tool_results:
+                        tstep = tool_by_id.get(b.get("tool_use_id"))
+                        if not tstep:
+                            continue
+                        tstep["end_ts"] = ts
+                        tstep["is_error"] = bool(b.get("is_error"))
+                        tstep["output_chars"] = _content_len(b.get("content"))
+                        if tstep["is_error"]:
+                            tstep["error"] = _text_of(b.get("content"))[:300] or str(b.get("content"))[:300]
+                        tur = e.get("toolUseResult")
+                        if isinstance(tur, dict):
+                            if tur.get("interrupted"):
+                                tstep["interrupted"] = True
+                            if tur.get("agentId"):
+                                tstep["subagent_id"] = tur.get("agentId")
+                                tstep["subagent_tokens"] = tur.get("totalTokens")
+                        elif isinstance(tur, str) and "rejected" in tur.lower():
+                            tstep["rejected"] = True
+                    if ts:
+                        last_ts = ts
+                    continue
+                text = _text_of(content)
+                if not text.strip():
+                    continue
+                kind, sub = _classify_user_text(text, e)
+                if kind == "skip":
+                    continue
+                steps.append({"kind": kind, "name": sub, "ts": ts, "text": text[:2000], "sidechain": bool(e.get("isSidechain"))})
+                if ts:
+                    last_ts = ts
+
+    # finalize costs and durations
+    for s in steps:
+        if s["kind"] == "llm":
+            s["cost"] = pricing.cost(s["model"], s.get("input_tokens", 0), s.get("output_tokens", 0),
+                                     s.get("cache_read", 0), s.get("cache_write_5m", 0), s.get("cache_write_1h", 0))
+            s["context_tokens"] = s.get("input_tokens", 0) + s.get("cache_read", 0) + s.get("cache_write", 0)
+            if s.get("start_ts") and s.get("end_ts"):
+                s["duration_ms"] = max(0, int((s["end_ts"] - s["start_ts"]) * 1000))
+        elif s["kind"] == "tool":
+            if s.get("end_ts") and s.get("start_ts"):
+                s["duration_ms"] = max(0, int((s["end_ts"] - s["start_ts"]) * 1000))
+    for i, s in enumerate(steps):
+        s["seq"] = i
+    tss = [s["ts"] for s in steps if s.get("ts")]
+    run["started"] = min(tss) if tss else None
+    run["ended"] = max(tss) if tss else None
+    if is_sub and steps and steps[0]["kind"] == "prompt":
+        steps[0]["name"] = "delegated"
+    return run
+
+
+def discover(root=DEFAULT_ROOT):
+    out = []
+    for dirpath, _dirs, files in os.walk(root):
+        for fn in files:
+            if fn.endswith(".jsonl"):
+                out.append(os.path.join(dirpath, fn))
+    return out

agentdynamics/collectors/generic.py

@@ -0,0 +1,103 @@
+"""Collector for runs pushed by any agent framework (SDK / HTTP ingest).
+
+Accepted format (JSON), one run per file in <data_dir>/runs/:
+{
+  "id": "run-123", "agent": "support-bot", "project": "helpdesk",
+  "steps": [
+    {"kind": "prompt", "ts": 1718000000.0, "text": "Refund order 42"},
+    {"kind": "llm", "ts": ..., "end_ts": ..., "model": "claude-opus-5",
+     "input_tokens": 1200, "output_tokens": 300, "cache_read": 0, "cache_write": 0, "stop_reason": "tool_use"},
+    {"kind": "tool", "ts": ..., "end_ts": ..., "name": "lookup_order", "input": {...},
+     "is_error": false, "output_chars": 512, "phase": "explore"}
+  ]
+}
+Missing costs are computed from pricing; missing phases are inferred.
+"""
+import hashlib
+import json
+import os
+import uuid
+
+from .. import pricing
+from ..phases import classify
+
+
+def normalize(payload):
+    rid = str(payload.get("id") or uuid.uuid4())
+    run = {
+        "id": rid, "source": payload.get("source") or "sdk", "file": None,
+        "project": payload.get("project") or payload.get("agent") or "default",
+        "cwd": payload.get("cwd"), "title": payload.get("title"), "agent_name": payload.get("agent"),
+        "parent_id": payload.get("parent_id"), "is_subagent": bool(payload.get("parent_id")),
+        "workflow": payload.get("workflow"), "version": payload.get("version"), "git_branch": None,
+        "entrypoint": payload.get("entrypoint"), "environment": payload.get("environment") or "default",
+        "framework": payload.get("framework") or "sdk", "thread_id": payload.get("thread_id"), "user_id": payload.get("user_id"),
+        "feedback": payload.get("feedback") or [], "tags": payload.get("tags") or [],
+        "root_status": payload.get("status"), "root_error": payload.get("error"), "complete": payload.get("complete", True),
+        "policy_version": payload.get("policy_version"), "policy": payload.get("policy"),
+        "steps": [],
+    }
+    last_llm = None
+    for i, s in enumerate(payload.get("steps") or []):
+        k = s.get("kind")
+        st = dict(s)
+        st["seq"] = i
+        st.setdefault("ts", st.get("start_ts"))
+        st.setdefault("start_ts", st.get("ts"))
+        if st.get("ts") and st.get("end_ts") and "duration_ms" not in st:
+            st["duration_ms"] = int((st["end_ts"] - st["ts"]) * 1000)
+        if k == "llm":
+            st.setdefault("name", st.get("model"))
+            for f in ("input_tokens", "output_tokens", "cache_read", "cache_write", "thinking_tokens"):
+                st[f] = int(st.get(f) or 0)
+            st.setdefault("cache_write_5m", st["cache_write"])
+            st.setdefault("cache_write_1h", 0)
+            if st.get("cost") is None:
+                st["cost"] = pricing.cost(st.get("model"), st["input_tokens"], st["output_tokens"], st["cache_read"],
+                                          st["cache_write_5m"], st["cache_write_1h"])
+            st["context_tokens"] = st["input_tokens"] + st["cache_read"] + st["cache_write"]
+            st.setdefault("text", "")
+            st["_id"] = st.get("id") or f"{rid}:{i}"
+            st["tool_calls"] = 0
+            last_llm = st
+        elif k == "tool":
+            inp = st.pop("input", None) or {}
+            phase, target = classify(st.get("name"), inp)
+            st.setdefault("phase", phase)
+            st.setdefault("target", target)
+            st["input_hash"] = hashlib.sha1((str(st.get("name")) + json.dumps(inp, sort_keys=True, default=str)).encode()).hexdigest()[:16]
+            st["input_preview"] = json.dumps(inp, default=str)[:400]
+            if inp:
+                st["args_json"] = json.dumps(inp, default=str)[:4000]
+            st["is_error"] = bool(st.get("is_error"))
+            st["output_chars"] = int(st.get("output_chars") or 0)
+            if last_llm is not None:
+                st["llm_msg"] = last_llm["_id"]
+                last_llm["tool_calls"] += 1
+        elif k == "prompt":
+            st.setdefault("name", "human")
+        elif k == "span":
+            st.setdefault("span_kind", "node")
+            st.setdefault("node", st.get("name"))
+        run["steps"].append(st)
+    tss = [s["ts"] for s in run["steps"] if s.get("ts")]
+    run["started"] = min(tss) if tss else None
+    run["ended"] = max(tss) if tss else None
+    return run
+
+
+def load_dir(runs_dir):
+    out = []
+    if not os.path.isdir(runs_dir):
+        return out
+    for fn in sorted(os.listdir(runs_dir)):
+        if fn.endswith(".json"):
+            p = os.path.join(runs_dir, fn)
+            try:
+                with open(p, encoding="utf-8") as f:
+                    run = normalize(json.load(f))
+                run["file"] = p
+                out.append(run)
+            except (ValueError, OSError):
+                continue
+    return out

agentdynamics/collectors/inbox.py

@@ -0,0 +1,95 @@
+"""File inbox / log tailing: the bridge for log pipelines (Fluent Bit, Vector, Logstash, OTel Collector
+file exporter, S3/GCS sync jobs...). Watches a directory of *.jsonl / *.json files, tails growing files
+by byte offset, and auto-detects each record's format:
+
+  - OTLP JSON export          {"resourceSpans": [...]}            (OTel Collector `file` exporter)
+  - LangSmith run             {"id", "run_type", "trace_id", ...} (LangSmith bulk export / run dumps)
+  - Langfuse trace            {"id", "observations": [...]}
+  - AgentDynamics generic run {"steps": [...]}
+  - Canonical span            {"trace_id", "span_id", "kind", ...}
+"""
+import json
+import os
+
+
+def detect(rec):
+    if not isinstance(rec, dict):
+        return None
+    if "resourceSpans" in rec:
+        return "otlp"
+    if "run_type" in rec and ("trace_id" in rec or "id" in rec):
+        return "langsmith"
+    if "observations" in rec and "id" in rec:
+        return "langfuse"
+    if "steps" in rec:
+        return "generic"
+    if "trace_id" in rec and "span_id" in rec:
+        return "span"
+    if {"seq", "grant_id", "prev_hash", "rule", "allowed", "tool"} <= set(rec):
+        return "aegis"
+    return None
+
+
+def unwrap(rec):
+    """Log shippers wrap payloads: {"log": "<json>"}, {"message": {...}}, {"body": ...}."""
+    for key in ("log", "message", "body", "record"):
+        if detect(rec) is None and isinstance(rec, dict) and key in rec:
+            inner = rec[key]
+            if isinstance(inner, str):
+                try:
+                    inner = json.loads(inner)
+                except ValueError:
+                    continue
+            rec = inner
+    return rec
+
+
+class Inbox:
+    def __init__(self, path, state):
+        self.path = path
+        self.state = state  # {"offsets": {file: bytes}}
+        os.makedirs(path, exist_ok=True)
+
+    def poll(self, max_bytes=64 * 1024 * 1024):
+        """Return list of (format, record) read since last poll."""
+        offs = self.state.setdefault("offsets", {})
+        out = []
+        for fn in sorted(os.listdir(self.path)):
+            p = os.path.join(self.path, fn)
+            if not os.path.isfile(p) or not fn.endswith((".jsonl", ".json", ".ndjson", ".log")):
+                continue
+            size = os.path.getsize(p)
+            start = offs.get(p, 0)
+            if size < start:  # rotated / truncated
+                start = 0
+            if size == start:
+                continue
+            with open(p, "rb") as f:
+                f.seek(start)
+                chunk = f.read(max_bytes)
+            if fn.endswith(".json") and start == 0:
+                try:
+                    doc = json.loads(chunk)
+                    recs = doc if isinstance(doc, list) else [doc]
+                    out.extend((detect(r), r) for r in recs if detect(r))
+                    offs[p] = size
+                    continue
+                except ValueError:
+                    pass
+            last_nl = chunk.rfind(b"\n")
+            if last_nl < 0:
+                continue  # wait for a complete line
+            for line in chunk[:last_nl].splitlines():
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    rec = json.loads(line)
+                except ValueError:
+                    continue
+                rec = unwrap(rec)
+                fmt = detect(rec)
+                if fmt:
+                    out.append((fmt, rec))
+            offs[p] = start + last_nl + 1
+        return out

agentdynamics/collectors/langfuse.py

@@ -0,0 +1,98 @@
+"""Langfuse pull connector (public API) -> canonical spans.
+
+Traces: GET /api/public/traces?fromTimestamp=..&page=..&limit=..
+Detail: GET /api/public/traces/{id}  (includes observations: SPAN | GENERATION | EVENT | AGENT | TOOL | RETRIEVER ...)
+Auth: HTTP Basic public_key:secret_key
+"""
+import base64
+import json
+import os
+import time
+import urllib.request
+from datetime import datetime, timezone
+from urllib.parse import urlencode
+
+from .langsmith import parse_time
+
+OBS_KIND = {"GENERATION": "llm", "SPAN": "chain", "EVENT": "span", "AGENT": "agent", "TOOL": "tool", "CHAIN": "chain",
+            "RETRIEVER": "retriever", "EMBEDDING": "embedding", "GUARDRAIL": "guardrail", "EVALUATOR": "evaluator"}
+
+
+def trace_to_spans(tr):
+    """Langfuse trace (with observations) -> canonical spans. The trace itself becomes the root span."""
+    tid = tr["id"]
+    md = tr.get("metadata") or {}
+    root = {
+        "trace_id": tid, "span_id": f"trace-{tid}", "parent_id": None, "name": tr.get("name") or "trace", "kind": "chain",
+        "start": parse_time(tr.get("timestamp")), "end": None, "status": "ok", "error": None,
+        "input": tr.get("input"), "output": tr.get("output"), "project": md.get("project") or tr.get("release"),
+        "environment": tr.get("environment") or md.get("environment"), "session_id": tr.get("sessionId"), "user_id": tr.get("userId"),
+        "tags": tr.get("tags") or [], "framework": "langfuse", "source": "langfuse",
+        "feedback": [{"key": s.get("name"), "score": s.get("value")} for s in tr.get("scores") or [] if isinstance(s.get("value"), (int, float))],
+    }
+    spans, ends = [root], []
+    for o in tr.get("observations") or []:
+        u = o.get("usageDetails") or o.get("usage") or {}
+        it = u.get("input") or u.get("promptTokens") or u.get("input_tokens") or 0
+        ot = u.get("output") or u.get("completionTokens") or u.get("output_tokens") or 0
+        cr = u.get("cache_read_input_tokens") or u.get("input_cached_tokens") or u.get("cache_read") or 0
+        cw = u.get("cache_creation_input_tokens") or 0
+        err = o.get("statusMessage") if o.get("level") == "ERROR" else None
+        end = parse_time(o.get("endTime"))
+        ends.append(end)
+        omd = o.get("metadata") or {}
+        out = o.get("output")
+        docs = None
+        if o.get("type") == "RETRIEVER" or "retriev" in (o.get("name") or "").lower():
+            d = out.get("documents") if isinstance(out, dict) else out
+            docs = len(d) if isinstance(d, list) else None
+        spans.append({
+            "trace_id": tid, "span_id": o["id"], "parent_id": o.get("parentObservationId") or root["span_id"],
+            "name": o.get("name") or o.get("type"), "kind": OBS_KIND.get(o.get("type"), "span"),
+            "start": parse_time(o.get("startTime")), "end": end, "status": "error" if err else "ok", "error": err,
+            "model": o.get("model"), "input_tokens": it, "output_tokens": ot, "cache_read": cr, "cache_write": cw,
+            "cost": o.get("calculatedTotalCost") if o.get("calculatedTotalCost") is not None else (o.get("costDetails") or {}).get("total"),
+            "ttft_ms": (parse_time(o.get("completionStartTime")) - parse_time(o.get("startTime"))) * 1000
+            if o.get("completionStartTime") and o.get("startTime") else None,
+            "stop_reason": omd.get("finish_reason"), "input": o.get("input"), "output": out,
+            "node": omd.get("langgraph_node"), "agent": omd.get("agent_name"), "framework": "langfuse", "source": "langfuse", "docs": docs,
+        })
+    valid = [e for e in ends if e]
+    root["end"] = max(valid) if valid else root["start"]
+    if any(s.get("status") == "error" for s in spans[1:]) and not tr.get("output"):
+        root["status"] = "error"
+        root["error"] = next(s["error"] for s in spans[1:] if s.get("status") == "error")
+    return spans
+
+
+class LangfusePuller:
+    def __init__(self, cfg, state):
+        self.host = (cfg.get("host") or "https://cloud.langfuse.com").rstrip("/")
+        pk = os.environ.get(cfg.get("public_key_env", "LANGFUSE_PUBLIC_KEY"), "")
+        sk = os.environ.get(cfg.get("secret_key_env", "LANGFUSE_SECRET_KEY"), "")
+        self.auth = "Basic " + base64.b64encode(f"{pk}:{sk}".encode()).decode()
+        self.state = state
+        self.lookback = float(cfg.get("lookback_hours", 24)) * 3600
+
+    def _get(self, path, params=None):
+        url = f"{self.host}{path}" + (("?" + urlencode(params)) if params else "")
+        req = urllib.request.Request(url, headers={"Authorization": self.auth, "Accept": "application/json"})
+        with urllib.request.urlopen(req, timeout=30) as r:
+            return json.loads(r.read())
+
+    def pull(self, max_traces=2000):
+        since = self.state.get("since") or (time.time() - self.lookback)
+        iso = datetime.fromtimestamp(since, timezone.utc).isoformat().replace("+00:00", "Z")
+        traces, page, newest = [], 1, since
+        while len(traces) < max_traces:
+            res = self._get("/api/public/traces", {"fromTimestamp": iso, "page": page, "limit": 50})
+            data = res.get("data") or []
+            for t in data:
+                traces.append(self._get(f"/api/public/traces/{t['id']}"))
+                newest = max(newest, parse_time(t.get("timestamp")) or 0)
+            meta = res.get("meta") or {}
+            if not data or page >= (meta.get("totalPages") or 1):
+                break
+            page += 1
+        self.state["since"] = max(since, newest - 600)
+        return traces