agentprof 0.1.0__py3-none-any.whl

agentprof/adapters/copilot_vscode/transcript.py

@@ -0,0 +1,176 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 epicodic
+"""Parse `GitHub.copilot-chat/transcripts/<sid>.jsonl`: tool timings and LLM call attribution."""
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from agentprof.adapters.copilot_vscode.tools import is_subagent_tool_id
+from agentprof.adapters.timestamps import parse_iso_ms
+
+
+@dataclass
+class ToolTiming:
+    """Start/end time (epoch ms) and outcome of one tool call."""
+
+    start_ms: int | None = None
+    end_ms: int | None = None
+    success: bool | None = None
+    start_order: int | None = None
+    completion_order: int | None = None
+    completion_recorded: bool = False
+
+
+@dataclass(frozen=True)
+class UserMessage:
+    """One source-recorded Copilot user message."""
+
+    message_id: str | None
+    timestamp_ms: int
+    source_order: int
+
+
+@dataclass(frozen=True)
+class ToolEventRecord:
+    """One native tool start or completion snapshot, including records without an invocation ID."""
+
+    kind: str
+    tool_call_id: str | None
+    timestamp_ms: int
+    source_order: int
+    owner_id: str | None
+    tool_name: str | None = None
+    success: bool | None = None
+
+
+@dataclass
+class Transcript:
+    """Per-`toolCallId` timing and arguments, and per-agent LLM call timestamps (key `None` is the main agent).
+
+    The arguments cover subagents' tool calls too, which the session file records without them.
+    """
+
+    tool_timings: dict[str, ToolTiming] = field(default_factory=dict)
+    tool_arguments: dict[str, dict[str, object]] = field(default_factory=dict)
+    llm_call_timestamps_ms: dict[str | None, list[int]] = field(default_factory=dict)
+    llm_call_source_orders: dict[str | None, list[int]] = field(default_factory=dict)
+    llm_call_source_request_ids: dict[str | None, list[str | None]] = field(default_factory=dict)
+    tool_request_ids: dict[str, str] = field(default_factory=dict)
+    user_messages: list[UserMessage] = field(default_factory=list)
+    tool_events: list[ToolEventRecord] = field(default_factory=list)
+    session_id: str | None = None
+    malformed_lines: int = 0
+
+
+def _apply_event(transcript: Transcript, stack: list[str | None], event: dict[str, Any], source_order: int) -> None:
+    timestamp_ms = parse_iso_ms(event["timestamp"])
+    event_type = event["type"]
+
+    if event_type == "assistant.message":
+        transcript.llm_call_timestamps_ms.setdefault(stack[-1], []).append(timestamp_ms)
+        transcript.llm_call_source_orders.setdefault(stack[-1], []).append(source_order)
+        data = event["data"]
+        message_id = data.get("messageId") if isinstance(data, dict) else None
+        transcript.llm_call_source_request_ids.setdefault(stack[-1], []).append(
+            message_id if isinstance(message_id, str) else None
+        )
+        tool_requests = data.get("toolRequests", []) if isinstance(data, dict) else []
+        if isinstance(tool_requests, list) and isinstance(message_id, str):
+            for request in tool_requests:
+                if isinstance(request, dict):
+                    tool_call_id = request.get("toolCallId")
+                    if isinstance(tool_call_id, str) and tool_call_id:
+                        transcript.tool_request_ids[tool_call_id] = message_id
+
+    elif event_type == "user.message":
+        transcript.user_messages.append(
+            UserMessage(
+                event.get("id") if isinstance(event.get("id"), str) else None,
+                timestamp_ms,
+                source_order,
+            )
+        )
+        if transcript.session_id is None:
+            session_id = event.get("sessionId")
+            if isinstance(session_id, str):
+                transcript.session_id = session_id
+
+    elif event_type == "session.start":
+        data = event.get("data")
+        session_id = data.get("sessionId") if isinstance(data, dict) else None
+        if isinstance(session_id, str):
+            transcript.session_id = session_id
+
+    elif event_type == "tool.execution_start":
+        data = event["data"]
+        tool_call_id = data.get("toolCallId")
+        transcript.tool_events.append(
+            ToolEventRecord(
+                kind=event_type,
+                tool_call_id=tool_call_id if isinstance(tool_call_id, str) and tool_call_id else None,
+                timestamp_ms=timestamp_ms,
+                source_order=source_order,
+                owner_id=stack[-1],
+                tool_name=data.get("toolName") if isinstance(data.get("toolName"), str) else None,
+            )
+        )
+        if not isinstance(tool_call_id, str) or not tool_call_id:
+            return
+        timing = transcript.tool_timings.setdefault(tool_call_id, ToolTiming())
+        timing.start_ms = timestamp_ms
+        timing.start_order = source_order
+        arguments = data.get("arguments")
+        if isinstance(arguments, dict):
+            transcript.tool_arguments[tool_call_id] = arguments
+        if is_subagent_tool_id(data["toolName"]):
+            stack.append(tool_call_id)
+
+    elif event_type == "tool.execution_complete":
+        data = event["data"]
+        tool_call_id = data.get("toolCallId")
+        success = data.get("success")
+        transcript.tool_events.append(
+            ToolEventRecord(
+                kind=event_type,
+                tool_call_id=tool_call_id if isinstance(tool_call_id, str) and tool_call_id else None,
+                timestamp_ms=timestamp_ms,
+                source_order=source_order,
+                owner_id=stack[-1],
+                success=success if isinstance(success, bool) else None,
+            )
+        )
+        if not isinstance(tool_call_id, str) or not tool_call_id:
+            return
+        timing = transcript.tool_timings.setdefault(tool_call_id, ToolTiming())
+        timing.end_ms = timestamp_ms
+        timing.success = data.get("success")
+        timing.completion_order = source_order
+        timing.completion_recorded = True
+        if stack[-1] == tool_call_id:
+            stack.pop()
+
+
+def load_transcript(path: Path) -> Transcript:
+    """Parse a transcript file.
+
+    Args:
+        path: Path to the transcript file.
+
+    Returns:
+        Tool timings and LLM calls attributed to the agent whose subagent bracket was open at the time.
+    """
+    transcript = Transcript()
+    # The stack tracks which agent tool call's bracket is currently open; `None` is the main agent.
+    stack: list[str | None] = [None]
+
+    for source_order, line in enumerate(path.read_text().splitlines()):
+        if not line.strip():
+            continue
+        try:
+            _apply_event(transcript, stack, json.loads(line), source_order)
+        except (json.JSONDecodeError, KeyError, TypeError, ValueError):
+            transcript.malformed_lines += 1
+
+    return transcript

agentprof/adapters/copilot_vscode/tree.py

@@ -0,0 +1,325 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 epicodic
+"""Join a parsed Copilot session with its transcript and debug log into a neutral node tree."""
+
+from collections import Counter
+
+from agentprof.adapters.copilot_vscode.debuglog import DebugLlmCall, DebugLog
+from agentprof.adapters.copilot_vscode.session import RawRequest, RawSession, RawToolCall
+from agentprof.adapters.copilot_vscode.tools import tool_info
+from agentprof.adapters.copilot_vscode.transcript import Transcript
+from agentprof.adapters.execution import tool_execution_events
+from agentprof.adapters.turns import split_by_turn
+from agentprof.model import (
+    CostMetric,
+    EventCallLink,
+    ExecutionEvent,
+    LlmCall,
+    Metric,
+    Node,
+    NodeKind,
+    Provenance,
+    Tokens,
+    iter_nodes,
+)
+
+_CREDITS = "credits"
+_NANO_AIU_PER_CREDIT = 1_000_000_000
+
+
+def credits_cost(value: float | None, usd_per_credit: float) -> CostMetric:
+    """A cost in Copilot credits, convertible to USD at `usd_per_credit`; `n/a` if `value` is `None`."""
+    if value is None:
+        return CostMetric.not_available()
+    return CostMetric(value=value, unit=_CREDITS, provenance=Provenance.EXACT, usd_per_unit=usd_per_credit)
+
+
+def _exact_or_not_available(value: int | None) -> Metric:
+    return Metric.not_available() if value is None else Metric.exact(value)
+
+
+def _tokens(call: DebugLlmCall) -> Tokens:
+    if call.input_tokens is None:
+        return Tokens(output=_exact_or_not_available(call.output_tokens))
+    cached = call.cached_tokens or 0
+    return Tokens(
+        input=Metric.exact(call.input_tokens - cached),
+        output=_exact_or_not_available(call.output_tokens),
+        cache_read=Metric.exact(cached),
+    )
+
+
+def _llm_call_from_debug_log(
+    call: DebugLlmCall, in_context: bool, source_stream_id: str | None, usd_per_credit: float
+) -> LlmCall:
+    return LlmCall(
+        start=Metric.exact(call.start_ms),
+        duration=Metric.exact(call.duration_ms),
+        tokens=_tokens(call),
+        model=call.model,
+        cost=credits_cost(
+            call.usage_nano_aiu / _NANO_AIU_PER_CREDIT if call.usage_nano_aiu is not None else None, usd_per_credit
+        ),
+        in_context=in_context,
+        source_request_id=call.source_request_id,
+        source_stream_id=source_stream_id,
+        timing_basis="request_start",
+    )
+
+
+def _calls_from_debug_log(
+    calls: list[DebugLlmCall], source_stream_id: str | None, usd_per_credit: float
+) -> list[LlmCall]:
+    """The agent's conversation is its most frequent request kind; other kinds are side requests."""
+    conversation = Counter(call.debug_name for call in calls).most_common(1)[0][0] if calls else None
+    return [
+        _llm_call_from_debug_log(
+            call,
+            in_context=call.debug_name == conversation,
+            source_stream_id=source_stream_id,
+            usd_per_credit=usd_per_credit,
+        )
+        for call in calls
+    ]
+
+
+def _llm_calls(
+    key: str | None,
+    transcript: Transcript | None,
+    debug_log: DebugLog | None,
+    source_stream_id: str | None,
+    usd_per_credit: float,
+) -> list[LlmCall] | None:
+    """LLM calls owned by `key` (`None` = main agent), or `None` if no source covers it."""
+    if debug_log is not None and key in debug_log.calls:
+        return _calls_from_debug_log(debug_log.calls[key], source_stream_id, usd_per_credit)
+    if transcript is not None:
+        orders = transcript.llm_call_source_orders.get(key, [])
+        request_ids = transcript.llm_call_source_request_ids.get(key, [])
+        return [
+            LlmCall(
+                start=Metric.exact(ts),
+                source_order=orders[index] if index < len(orders) else None,
+                source_stream_id=source_stream_id,
+                source_request_id=request_ids[index] if index < len(request_ids) else None,
+                timing_basis="assistant_message",
+            )
+            for index, ts in enumerate(transcript.llm_call_timestamps_ms.get(key, []))
+        ]
+    return None
+
+
+def _set_llm_calls(node: Node, calls: list[LlmCall] | None) -> None:
+    if calls is None:
+        return
+    node.llm_calls = calls
+    node.llm_call_count = Metric.exact(len(calls))
+
+
+def _build_tool_node(
+    tool_call: RawToolCall, transcript: Transcript | None, usd_per_credit: float, source_stream_id: str | None
+) -> Node:
+    arguments = tool_call.arguments
+    if not arguments and transcript is not None:
+        arguments = transcript.tool_arguments.get(tool_call.tool_call_id, {})
+    node = Node(
+        node_id=tool_call.tool_call_id,
+        kind=NodeKind.AGENT if tool_call.is_agent else NodeKind.TOOL,
+        topic=tool_call.subagent_description or tool_call.topic,
+        agent_uuid=tool_call.tool_call_id if tool_call.is_agent else None,
+        model=tool_call.subagent_model,
+        prompt=tool_call.subagent_prompt or "",
+        result=tool_call.subagent_result or "",
+        tool=tool_info(tool_call.tool_id, arguments),
+        cost_total=credits_cost(tool_call.subagent_credits, usd_per_credit),
+    )
+    timing = transcript.tool_timings.get(tool_call.tool_call_id) if transcript is not None else None
+    if timing is not None:
+        if timing.start_ms is not None:
+            node.start = Metric.exact(timing.start_ms)
+        if timing.end_ms is not None:
+            node.end = Metric.exact(timing.end_ms)
+        if timing.start_ms is not None and timing.end_ms is not None:
+            node.duration = Metric.exact(timing.end_ms - timing.start_ms)
+        node.success = timing.success
+    return node
+
+
+def _build_turn(
+    request: RawRequest,
+    main_calls: list[LlmCall] | None,
+    transcript: Transcript | None,
+    debug_log: DebugLog | None,
+    usd_per_credit: float,
+    source_stream_id: str | None,
+) -> Node:
+    nodes_by_id = {
+        tc.tool_call_id: _build_tool_node(tc, transcript, usd_per_credit, source_stream_id) for tc in request.tool_calls
+    }
+    children_by_parent: dict[str | None, list[Node]] = {}
+    for tool_call in request.tool_calls:
+        # An orphan parent id (naming no tool call in this request) is treated as a direct child of the turn.
+        parent_id = tool_call.parent_tool_call_id
+        if parent_id is not None and parent_id not in nodes_by_id:
+            parent_id = None
+        children_by_parent.setdefault(parent_id, []).append(nodes_by_id[tool_call.tool_call_id])
+
+    for tool_call in request.tool_calls:
+        node = nodes_by_id[tool_call.tool_call_id]
+        node.children = children_by_parent.get(tool_call.tool_call_id, [])
+        if node.kind is NodeKind.AGENT:
+            child_stream_id = tool_call.tool_call_id
+            _set_llm_calls(
+                node, _llm_calls(tool_call.tool_call_id, transcript, debug_log, child_stream_id, usd_per_credit)
+            )
+
+    turn = Node(
+        node_id=request.request_id,
+        kind=NodeKind.TURN,
+        topic=request.text,
+        prompt=request.text,
+        model=request.model_id,
+        children=children_by_parent.get(None, []),
+        start=Metric.exact(request.timestamp_ms),
+        cost_total=credits_cost(request.credits, usd_per_credit),
+        user_wait=(
+            Metric.exact(request.time_spent_waiting_ms)
+            if request.time_spent_waiting_ms is not None
+            else Metric.not_available()
+        ),
+    )
+    if request.elapsed_ms is not None:
+        turn.end = Metric.exact(request.timestamp_ms + request.elapsed_ms)
+        turn.duration = Metric.exact(request.elapsed_ms)
+    _set_llm_calls(turn, main_calls)
+    _populate_tool_events(turn, transcript, source_stream_id, debug_log)
+    return turn
+
+
+def _populate_tool_events(
+    owner: Node, transcript: Transcript | None, source_stream_id: str | None, debug_log: DebugLog | None
+) -> None:
+    """Attach native tool execution snapshots to the node that directly owns each invocation."""
+    agent_key = owner.agent_uuid if owner.kind is NodeKind.AGENT else None
+    calls = debug_log.calls.get(agent_key, []) if debug_log is not None else []
+    for child in owner.children:
+        timing = transcript.tool_timings.get(child.node_id) if transcript is not None else None
+        if timing is not None:
+            is_delegation = child.tool is not None and child.tool.category.value == "subagent"
+            request_ids = {
+                call.source_request_id
+                for call in calls
+                if call.source_request_id and child.node_id in call.requested_tool_ids
+            }
+            request_id = (
+                next(iter(request_ids))
+                if len(request_ids) == 1
+                else transcript.tool_request_ids.get(child.node_id)
+                if transcript is not None
+                else None
+            )
+            events = tool_execution_events(
+                invocation_id=child.node_id,
+                subject_node_id=child.node_id,
+                start=child.start,
+                end=child.end,
+                result_recorded=timing.completion_recorded,
+                success=timing.success if timing.completion_recorded else None,
+                delegation=is_delegation,
+                start_order=timing.start_order,
+                result_order=timing.completion_order,
+                source_stream_id=source_stream_id,
+                request_id=request_id,
+            )
+            if timing.start_ms is not None:
+                owner.execution_events.append(events[0])
+            if timing.completion_recorded:
+                consumers = [
+                    call
+                    for call in calls
+                    if call.source_request_id
+                    and child.node_id in call.consumed_tool_ids
+                    and timing.end_ms is not None
+                    and call.start_ms >= timing.end_ms
+                ]
+                first_start = min((call.start_ms for call in consumers), default=None)
+                first_ids = dict.fromkeys(
+                    call.source_request_id
+                    for call in consumers
+                    if call.start_ms == first_start and call.source_request_id is not None
+                )
+                events[1].links.extend(
+                    EventCallLink(source_request_id=request_id, relation="consumed_by", evidence="recorded")
+                    for request_id in first_ids
+                )
+                owner.execution_events.append(events[1])
+        if child.kind is NodeKind.AGENT:
+            _populate_tool_events(child, transcript, child.agent_uuid, debug_log)
+    owner.execution_events.sort(key=lambda event: (event.source_order is None, event.source_order or 0))
+
+
+def _call_start(call: LlmCall) -> float | None:
+    return call.start.value
+
+
+def build_root(
+    raw: RawSession, transcript: Transcript | None, debug_log: DebugLog | None, usd_per_credit: float
+) -> Node:
+    """Build session -> turn -> agent -> tool; the main agent's LLM calls are split between turns by start time."""
+    stream_id = (transcript.session_id if transcript is not None else None) or raw.session_id
+    main_calls = _llm_calls(None, transcript, debug_log, stream_id, usd_per_credit)
+    groups = split_by_turn(main_calls or [], _call_start, [request.timestamp_ms for request in raw.requests])
+    turns = [
+        _build_turn(
+            request,
+            calls if main_calls is not None else None,
+            transcript,
+            debug_log,
+            usd_per_credit,
+            stream_id,
+        )
+        for request, calls in zip(raw.requests, groups, strict=True)
+    ]
+    root = Node(node_id="session", kind=NodeKind.SESSION, topic=raw.title or "session", children=turns)
+    if transcript is not None:
+        root.execution_events.extend(
+            ExecutionEvent(
+                kind="user_input",
+                event_id=f"user-input:{message.message_id}" if message.message_id else None,
+                start=Metric.exact(message.timestamp_ms),
+                source_order=message.source_order,
+                source_stream_id=stream_id,
+            )
+            for message in transcript.user_messages
+        )
+        _append_unrepresented_tool_events(root, transcript, stream_id)
+        root.execution_events.sort(key=lambda event: (event.source_order is None, event.source_order or 0))
+    return root
+
+
+def _append_unrepresented_tool_events(root: Node, transcript: Transcript, source_stream_id: str | None) -> None:
+    """Keep source tool records visible when the session tree has no matching invocation node."""
+    nodes = list(iter_nodes(root))
+    represented_ids = {node.node_id for node in nodes if node.tool is not None}
+    for record in transcript.tool_events:
+        if record.tool_call_id is not None and record.tool_call_id in represented_ids:
+            continue
+        owner = next(
+            (node for node in nodes if record.owner_id is not None and node.agent_uuid == record.owner_id),
+            root,
+        )
+        is_start = record.kind == "tool.execution_start"
+        name_category = tool_info(record.tool_name, {}).category if record.tool_name is not None else None
+        events = tool_execution_events(
+            invocation_id=record.tool_call_id,
+            subject_node_id=None,
+            start=Metric.exact(record.timestamp_ms) if is_start else Metric.not_available(),
+            end=Metric.exact(record.timestamp_ms) if not is_start else Metric.not_available(),
+            result_recorded=not is_start,
+            success=record.success,
+            delegation=name_category is not None and name_category.value == "subagent",
+            start_order=record.source_order if is_start else None,
+            result_order=record.source_order if not is_start else None,
+            source_stream_id=record.owner_id or source_stream_id,
+        )
+        owner.execution_events.append(events[0] if is_start else events[1])

agentprof/adapters/execution.py

@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 epicodic
+"""Construct neutral source events for tool invocations and their results."""
+
+from dataclasses import replace
+
+from agentprof.model import EventCallLink, ExecutionEvent, Metric
+
+
+def tool_execution_events(
+    *,
+    invocation_id: str | None,
+    subject_node_id: str | None,
+    start: Metric,
+    end: Metric,
+    result_recorded: bool,
+    success: bool | None = None,
+    delegation: bool = False,
+    start_order: int | None = None,
+    result_order: int | None = None,
+    source_stream_id: str | None = None,
+    request_id: str | None = None,
+    next_id: str | None = None,
+) -> list[ExecutionEvent]:
+    """Create start and optional result events while keeping each event's links independent."""
+    requested_links = (
+        [EventCallLink(source_request_id=request_id, relation="requested_by", evidence="recorded")]
+        if request_id is not None
+        else []
+    )
+    events = [
+        ExecutionEvent(
+            kind="delegation" if delegation else "tool_start",
+            event_id=f"tool-start:{invocation_id}" if invocation_id else None,
+            start=start,
+            source_order=start_order,
+            links=[replace(link) for link in requested_links],
+            subject_node_id=subject_node_id,
+            source_stream_id=source_stream_id,
+            execution_start=start,
+            execution_end=end,
+        )
+    ]
+    if result_recorded:
+        result_links = list(requested_links)
+        if next_id is not None:
+            result_links.append(
+                EventCallLink(source_request_id=next_id, relation="next_observed_call", evidence="observed_order")
+            )
+        events.append(
+            ExecutionEvent(
+                kind="tool_result",
+                event_id=f"tool-result:{invocation_id}" if invocation_id else None,
+                start=end,
+                source_order=result_order,
+                success=success,
+                links=[replace(link) for link in result_links],
+                subject_node_id=subject_node_id,
+                source_stream_id=source_stream_id,
+                execution_start=start,
+                execution_end=end,
+            )
+        )
+    return events

agentprof/adapters/timestamps.py

@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 epicodic
+"""Timestamp parsing shared by adapters."""
+
+from datetime import UTC, datetime
+
+
+def parse_iso_ms(iso_timestamp: str) -> int:
+    """Epoch milliseconds of an ISO 8601 timestamp; timestamps without an offset are taken as UTC."""
+    parsed = datetime.fromisoformat(iso_timestamp)
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=UTC)
+    return int(parsed.timestamp() * 1000)

agentprof/adapters/turns.py

@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 epicodic
+"""Assign items that carry no turn id (LLM calls, tool calls) to turns by start time."""
+
+from bisect import bisect_right
+from collections.abc import Callable, Sequence
+
+
+def split_by_turn[T](
+    items: Sequence[T], start_of: Callable[[T], float | None], turn_starts: Sequence[float]
+) -> list[list[T]]:
+    """Group `items` by turn; `turn_starts` must be ascending.
+
+    Each turn gets the items that start at or after its own start and before the next turn's start.
+    Items before the first turn, and items without a start, go to the first turn.
+    """
+    groups: list[list[T]] = [[] for _ in turn_starts]
+    if not groups:
+        return groups
+    for item in items:
+        start = start_of(item)
+        index = 0 if start is None else max(bisect_right(turn_starts, start) - 1, 0)
+        groups[index].append(item)
+    return groups

agentprof/analysis/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 epicodic
+"""Agent-neutral analysis: roll-ups and waste heuristics."""