custos-code 0.0.1__py3-none-any.whl

custos_code/adapters/otel.py

@@ -0,0 +1,210 @@
+"""OpenTelemetry GenAI spans -> ledger.
+
+Input is OTLP/JSON (`{"resourceSpans": [...]}`, as `otel-cli`, the collector's file exporter and
+every OTLP backend emit) or a JSONL stream of individual spans. The GenAI semantic conventions
+make the interesting content *opt-in*: `gen_ai.operation.name=execute_tool` spans carry the tool
+name and call id, but `gen_ai.tool.call.arguments` / `.result` and the message events only exist
+when the instrumentation sets `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT`. A span with
+no captured result is recorded with `flags.stderr_dropped`, and when *no* span in the trace
+carries a result the adapter writes the `no_tool_log` META row, so outcome claims come out
+`unrecorded` -- the record is known-incomplete -- rather than confirmed off a bare span status.
+
+Span status maps to outcome: `STATUS_CODE_ERROR` is positive evidence of failure. `UNSET` is not
+evidence of success, and neither is `OK` on its own -- the convention lets an exporter mark a span
+OK with nothing captured, and Tier 2 needs runner output, not just a status. exit_code 0 therefore
+requires a captured `process.exit_code`, or an OK status *and* a captured tool result.
+
+Owner: Ananya.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from typing import Any
+
+from ..ledger import chain, redact
+from ..models import EventFlags, EventKind, LedgerEvent, Session
+from ..parsers import is_piped
+from .state import Builder, as_dict, as_list, as_text, no_tool_log
+
+_TOOL_ALIASES = {
+    "bash": "Bash",
+    "shell": "Bash",
+    "run_command": "Bash",
+    "terminal": "Bash",
+    "edit": "Edit",
+    "write": "Edit",
+    "apply_patch": "Edit",
+    "str_replace_editor": "Edit",
+    "read": "Read",
+    "view": "Read",
+    "web_search": "WebSearch",
+}
+
+
+def _attributes(raw: object) -> dict[str, Any]:
+    """OTLP attributes are [{key, value:{stringValue|intValue|...}}]; plain dicts also accepted."""
+    if isinstance(raw, dict):
+        return dict(raw)
+    out: dict[str, Any] = {}
+    for item in as_list(raw):
+        attr = as_dict(item)
+        key = str(attr.get("key", ""))
+        value = attr.get("value")
+        if not key:
+            continue
+        if isinstance(value, dict):
+            for field in ("stringValue", "intValue", "doubleValue", "boolValue"):
+                if field in value:
+                    out[key] = int(value[field]) if field == "intValue" else value[field]
+                    break
+            else:
+                out[key] = as_text(value.get("arrayValue") or value.get("kvlistValue") or value)
+        else:
+            out[key] = value
+    return out
+
+
+def _nano_ts(value: object, fallback: datetime) -> datetime:
+    if isinstance(value, str) and value.isdigit():
+        value = int(value)
+    if isinstance(value, int | float) and value:
+        return datetime.fromtimestamp(float(value) / 1e9, tz=UTC)
+    return fallback
+
+
+def _spans(payload: object) -> list[dict[str, Any]]:
+    """Flatten OTLP resourceSpans/scopeSpans, or accept a bare span list."""
+    if isinstance(payload, list):
+        return [as_dict(s) for s in payload]
+    root = as_dict(payload)
+    if "resourceSpans" not in root:
+        return [root] if root.get("spanId") or root.get("name") else []
+    out: list[dict[str, Any]] = []
+    for resource in as_list(root.get("resourceSpans")):
+        res = as_dict(resource)
+        res_attrs = _attributes(as_dict(res.get("resource")).get("attributes"))
+        for scope in as_list(res.get("scopeSpans")):
+            for span in as_list(as_dict(scope).get("spans")):
+                s = as_dict(span)
+                s["_resource"] = res_attrs
+                out.append(s)
+    return out
+
+
+def _tool(attrs: dict[str, Any], span_name: str) -> str:
+    name = str(attrs.get("gen_ai.tool.name") or attrs.get("tool.name") or span_name.split(" ")[-1])
+    return _TOOL_ALIASES.get(name.lower(), name or "Tool")
+
+
+def parse(path: str) -> tuple[Session, list[LedgerEvent], str | None]:
+    with open(path, encoding="utf-8") as fh:
+        text = fh.read().strip()
+    try:
+        payload: object = json.loads(text)
+    except json.JSONDecodeError:
+        payload = [json.loads(line) for line in text.splitlines() if line.strip()]
+    spans = sorted(_spans(payload), key=lambda s: int(s.get("startTimeUnixNano") or 0))
+    if not spans:
+        raise ValueError(f"{path}: no spans found")
+
+    first = _attributes(spans[0].get("attributes")) | as_dict(spans[0].get("_resource"))
+    session_id = str(
+        first.get("gen_ai.conversation.id")
+        or first.get("session.id")
+        or spans[0].get("traceId")
+        or "otel-unknown"
+    )
+    epoch = datetime.fromtimestamp(0, tz=UTC)
+    started = _nano_ts(spans[0].get("startTimeUnixNano"), epoch)
+    cwd = str(first.get("process.working_directory") or "") or None
+
+    b = Builder(session_id, cwd)
+    report: str | None = None
+    ended = started
+    model: str | None = None
+
+    for span in spans:
+        attrs = _attributes(span.get("attributes")) | as_dict(span.get("_resource"))
+        start = _nano_ts(span.get("startTimeUnixNano"), started)
+        end = _nano_ts(span.get("endTimeUnixNano"), start)
+        ended = max(ended, end)
+        model = model or (
+            str(attrs.get("gen_ai.request.model")) if attrs.get("gen_ai.request.model") else None
+        )
+        operation = str(attrs.get("gen_ai.operation.name") or span.get("name") or "")
+        status = as_dict(span.get("status"))
+        failed = str(status.get("code", "")).endswith("ERROR")
+
+        if operation != "execute_tool" and not str(attrs.get("gen_ai.tool.name") or ""):
+            # inference spans carry the model's own words: report material, never evidence
+            output = as_text(attrs.get("gen_ai.output.messages") or attrs.get("gen_ai.completion"))
+            if output:
+                b.store_output(b.add(kind=EventKind.TEXT, ts=end), output)
+                report = output
+            continue
+
+        tool = _tool(attrs, str(span.get("name", "")))
+        arguments = attrs.get("gen_ai.tool.call.arguments")
+        command = str(
+            as_dict(arguments).get("command") if isinstance(arguments, dict) else arguments or ""
+        )
+        piped = bool(command) and is_piped(command)
+        call_input = redact({"command": command} if command else {"arguments": as_text(arguments)})
+        b.add(
+            kind=EventKind.CALL,
+            ts=start,
+            tool=tool,
+            input=call_input,
+            flags=EventFlags(piped=piped, sidechain=bool(attrs.get("gen_ai.agent.id"))),
+        )
+
+        exit_raw = attrs.get("process.exit_code")
+        result = attrs.get("gen_ai.tool.call.result")
+        ok = str(status.get("code", "")).endswith("OK") and result is not None
+        exit_code = (
+            int(exit_raw) if isinstance(exit_raw, int) else (1 if failed else 0 if ok else None)
+        )
+        event = b.add(
+            kind=EventKind.RESULT,
+            ts=end,
+            tool=tool,
+            input=call_input,
+            exit_code=exit_code,
+            duration_ms=int((end - start).total_seconds() * 1000),
+            flags=EventFlags(
+                piped=piped,
+                error=failed,
+                # content capture is opt-in in the convention; say so rather than guess
+                stderr_dropped=result is None,
+            ),
+        )
+        if result is not None:
+            b.store_output(event, as_text(result))
+
+    results = sum(1 for e in b.events if e.kind is EventKind.RESULT)
+    captured = sum(1 for e in b.events if e.kind is EventKind.RESULT and not e.flags.stderr_dropped)
+    if results and not captured:
+        no_tool_log(
+            b,
+            ended,
+            "trace captured no gen_ai.tool.call.result: content capture is opt-in in the GenAI "
+            "convention, so tool outcomes are not in the record",
+            evidence_class="T",
+        )
+    events = chain(b.events)
+    meta = Session(
+        id=session_id,
+        source="otel",
+        agent=str(first.get("gen_ai.agent.name") or first.get("service.name") or "otel"),
+        model=model,
+        started=started,
+        ended=ended,
+        cwd=cwd,
+        n_events=len(events),
+        ledger_root_hash=events[-1].hash if events else "",
+        # completeness is the known weak point of trace-based evidence: score it, don't assume it
+        integrity_score=round(captured / results, 2) if results else 0.5,
+    )
+    return meta, events, report

custos_code/adapters/state.py

@@ -0,0 +1,164 @@
+"""Shared state-evidence builders for the class-R adapters (Devin cloud, Copilot, any PR bot).
+
+Class R has no tool log by default: the agent ran elsewhere and left a PR behind. What it does
+leave is state -- commits, CI conclusions, and the files in a checkout -- which is the state half
+of the two-evidence rule, so edit/create/commit claims are settleable here and shell claims are
+not. Devin and Copilot differ only in how the report and metadata are fetched, so the evidence
+side lives here once.
+
+Owner: Ananya.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from datetime import UTC, datetime
+from typing import Any
+
+from ..ledger import MAX_OUTPUT_BYTES, redact
+from ..models import EventFlags, EventKind, LedgerEvent
+
+FAILED_CONCLUSIONS = frozenset(
+    {"failure", "timed_out", "cancelled", "action_required", "startup_failure"}
+)
+
+
+def as_ts(value: object, fallback: datetime) -> datetime:
+    if isinstance(value, str):
+        try:
+            return datetime.fromisoformat(value.replace("Z", "+00:00"))
+        except ValueError:
+            return fallback
+    if isinstance(value, int | float):
+        return datetime.fromtimestamp(float(value), tz=UTC)
+    return fallback
+
+
+def as_dict(value: object) -> dict[str, Any]:
+    return dict(value) if isinstance(value, dict) else {}
+
+
+def as_list(value: object) -> list[Any]:
+    return list(value) if isinstance(value, list) else []
+
+
+def as_text(value: object) -> str:
+    if isinstance(value, str):
+        return value
+    if value is None:
+        return ""
+    return json.dumps(value, indent=2, sort_keys=True, default=str)
+
+
+def abs_path(path: str, root: str | None) -> str:
+    return path if os.path.isabs(path) or not root else os.path.normpath(os.path.join(root, path))
+
+
+class Builder:
+    """Accumulates ledger events for one session, numbering and hashing output as it goes."""
+
+    def __init__(self, session_id: str, cwd: str | None) -> None:
+        self.events: list[LedgerEvent] = []
+        self.seq = 0
+        self.session_id = session_id
+        self.cwd = cwd
+
+    def add(self, **fields: Any) -> LedgerEvent:
+        event = LedgerEvent(seq=self.seq, session_id=self.session_id, cwd=self.cwd, **fields)
+        self.events.append(event)
+        self.seq += 1
+        return event
+
+    def store_output(self, event: LedgerEvent, text: str) -> None:
+        blob = str(redact(text))
+        raw = blob.encode()
+        event.output_hash = hashlib.sha256(raw).hexdigest()
+        event.output = raw[:MAX_OUTPUT_BYTES].decode(errors="ignore")
+        event.flags.truncated = event.flags.truncated or len(raw) > MAX_OUTPUT_BYTES
+
+
+def commits(b: Builder, git: dict[str, Any], fallback: datetime) -> datetime:
+    """Commits satisfy invariant 5 for edit/create claims: the transcript alone never could."""
+    root = str(git.get("root") or b.cwd or "") or None
+    ts = fallback
+    for raw in as_list(git.get("commits")):
+        commit = as_dict(raw)
+        ts = as_ts(commit.get("ts") or commit.get("date"), ts)
+        files = [abs_path(str(f), root) for f in as_list(commit.get("files"))]
+        event = b.add(
+            kind=EventKind.RESULT,
+            ts=ts,
+            tool="Git",
+            exit_code=0,
+            paths=files,
+            input=redact(
+                {"sha": str(commit.get("sha", "")), "subject": str(commit.get("subject", ""))}
+            ),
+        )
+        b.store_output(
+            event, as_text(commit.get("stat") or commit.get("body") or commit.get("subject"))
+        )
+    return ts
+
+
+def checks(b: Builder, runs: list[Any], fallback: datetime) -> datetime:
+    """CI runs are the outcome evidence class R has; a failed conclusion is positive evidence."""
+    ts = fallback
+    for raw in runs:
+        check = as_dict(raw)
+        ts = as_ts(check.get("completed_at") or check.get("started_at"), ts)
+        conclusion = str(check.get("conclusion") or check.get("status") or "").lower()
+        failed = conclusion in FAILED_CONCLUSIONS
+        started = as_ts(check.get("started_at"), ts)
+        duration = int((ts - started).total_seconds() * 1000) if ts > started else None
+        meta = redact({"name": str(check.get("name", "")), "url": str(check.get("url", ""))})
+        b.add(kind=EventKind.CALL, ts=started, tool="CI", input=meta)
+        event = b.add(
+            kind=EventKind.RESULT,
+            ts=ts,
+            tool="CI",
+            duration_ms=duration,
+            input=meta,
+            exit_code=0 if conclusion == "success" else 1 if failed else None,
+            flags=EventFlags(error=failed),
+        )
+        b.store_output(event, as_text(check.get("output") or check.get("logs") or conclusion))
+    return ts
+
+
+def checkout(b: Builder, probes: dict[str, Any], fallback: datetime) -> None:
+    """Filesystem probes taken after the session: `exists: false` is positive evidence of absence."""
+    root = str(probes.get("root") or b.cwd or "") or None
+    for raw in as_list(probes.get("files")):
+        probe = as_dict(raw)
+        path = abs_path(str(probe.get("path", "")), root)
+        if not path:
+            continue
+        exists = bool(probe.get("exists", True))
+        event = b.add(
+            kind=EventKind.RESULT,
+            ts=as_ts(probe.get("ts"), fallback),
+            tool="Read",
+            paths=[path],
+            exit_code=0 if exists else 1,
+            flags=EventFlags(error=not exists),
+            input={"path": path, "probe": "stat"},
+        )
+        b.store_output(
+            event,
+            json.dumps(
+                {"exists": exists, "sha256": probe.get("sha256"), "bytes": probe.get("bytes")},
+                sort_keys=True,
+            ),
+        )
+
+
+def no_tool_log(b: Builder, ts: datetime, note: str, evidence_class: str = "R") -> LedgerEvent:
+    """Record the instrumentation gap explicitly, so Tier 1/2 answer `unrecorded`, not `unwitnessed`."""
+    return b.add(
+        kind=EventKind.META,
+        ts=ts,
+        input={"event": "no_tool_log", "class": evidence_class, "note": note},
+    )