traceforge-toolkit 0.1.0__py3-none-any.whl

traceforge/preprocessors/__init__.py

@@ -0,0 +1,34 @@
+"""Preprocessors for normalizing framework-specific event structures.
+
+Each preprocessor transforms raw dicts into flat dicts suitable for
+type_field-based YAML mapping. They handle compound discriminators,
+nested structures, and field-presence-based typing.
+"""
+
+from traceforge.preprocessors.registry import (
+    PreprocessorFn,
+    get_preprocessor,
+    register_preprocessor,
+)
+
+# Import all preprocessor modules to trigger registration
+import traceforge.preprocessors.cline  # noqa: F401
+import traceforge.preprocessors.continue_dev  # noqa: F401
+import traceforge.preprocessors.goose  # noqa: F401
+import traceforge.preprocessors.openhands  # noqa: F401
+import traceforge.preprocessors.pydantic_ai  # noqa: F401
+import traceforge.preprocessors.openai_agents  # noqa: F401
+import traceforge.preprocessors.claude  # noqa: F401
+import traceforge.preprocessors.smolagents  # noqa: F401
+import traceforge.preprocessors.codex  # noqa: F401
+import traceforge.preprocessors.amazonq  # noqa: F401
+import traceforge.preprocessors.maf_transcript  # noqa: F401
+import traceforge.preprocessors.opencode  # noqa: F401
+import traceforge.preprocessors.copilot_vscode  # noqa: F401
+import traceforge.preprocessors.antigravity  # noqa: F401
+
+__all__ = [
+    "PreprocessorFn",
+    "get_preprocessor",
+    "register_preprocessor",
+]

traceforge/preprocessors/amazonq.py

@@ -0,0 +1,224 @@
+"""Amazon Q Developer preprocessor — extract tool calls from SQLite conversations table."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from traceforge.preprocessors.registry import register_preprocessor
+
+
+def _amazonq_text(content: Any) -> str:
+    """Flatten an Amazon Q ToolUseResultBlock list into text."""
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        parts = []
+        for b in content:
+            if isinstance(b, dict):
+                # externally-tagged: {"Text": "..."} | {"Json": {...}}
+                if "Text" in b:
+                    parts.append(str(b["Text"]))
+                elif "Json" in b:
+                    parts.append(json.dumps(b["Json"]))
+                elif b.get("type") == "text":
+                    parts.append(b.get("text", ""))
+        return " ".join(p for p in parts if p)
+    return ""
+
+
+def _expand_amazonq_pair(entry: dict[str, Any], cid: str) -> list[dict[str, Any]]:
+    """Expand one HistoryEntry {user, assistant} pair into block events.
+
+    Current Amazon Q schema uses externally-tagged Rust enums:
+      user.content = {Prompt:{prompt}} | {ToolUseResults:{tool_use_results:[...]}}
+                     | {CancelledToolUses:{prompt, tool_use_results}}
+      assistant    = {Response:{content}} | {ToolUse:{content, tool_uses:[...]}}
+      tool_use     = {id, name, args, ...}; result = {tool_use_id, content, status}
+    """
+    out: list[dict[str, Any]] = []
+    user = entry.get("user")
+    if isinstance(user, dict):
+        content = user.get("content", {})
+        prompt = None
+        results = None
+        if isinstance(content, dict):
+            if "Prompt" in content:
+                prompt = (content["Prompt"] or {}).get("prompt")
+            elif "CancelledToolUses" in content:
+                cancelled = content["CancelledToolUses"] or {}
+                prompt = cancelled.get("prompt")
+                results = cancelled.get("tool_use_results")
+            elif "ToolUseResults" in content:
+                results = (content["ToolUseResults"] or {}).get("tool_use_results")
+        if prompt:
+            out.append({"block_type": "message.user", "conversation_id": cid, "content": prompt})
+        for r in results or []:
+            if not isinstance(r, dict):
+                continue
+            out.append(
+                {
+                    "block_type": "tool.result",
+                    "conversation_id": cid,
+                    "tool_call_id": r.get("tool_use_id", ""),
+                    "is_error": r.get("status") == "Error",
+                    "output": _amazonq_text(r.get("content", "")),
+                }
+            )
+
+    assistant = entry.get("assistant")
+    if isinstance(assistant, dict):
+        body = assistant.get("Response") or assistant.get("ToolUse") or {}
+        if isinstance(body, dict):
+            if body.get("content"):
+                out.append(
+                    {
+                        "block_type": "message.assistant",
+                        "conversation_id": cid,
+                        "content": body["content"],
+                    }
+                )
+            for tu in body.get("tool_uses", []) or []:
+                if not isinstance(tu, dict):
+                    continue
+                out.append(
+                    {
+                        "block_type": "tool.call",
+                        "conversation_id": cid,
+                        "tool_call_id": tu.get("id", ""),
+                        "tool_name": tu.get("name", ""),
+                        "arguments": tu.get("args", {}),
+                    }
+                )
+    return out
+
+
+@register_preprocessor("amazonq")
+def preprocess_amazonq(obj: dict[str, Any]) -> list[dict[str, Any]]:
+    """Normalize an Amazon Q conversation row into tool call events.
+
+    Amazon Q Developer stores conversations in:
+      ~/.local/share/amazon-q/data.sqlite3 (Linux)
+      ~/Library/Application Support/amazon-q/data.sqlite3 (macOS)
+
+    The conversations table has a JSON `value` column containing the full
+    conversation structure. This preprocessor is invoked per-row after the
+    source reads rows from SQLite.
+
+    Expected input shape (one row from conversations table):
+    {
+        "conversation_id": "...",
+        "value": "{...}",  # JSON string with messages array
+        "created_at": "...",
+        "updated_at": "..."
+    }
+
+    The JSON value structure contains messages with role/content pairs.
+    Tool calls are embedded in assistant messages as tool_use content blocks.
+    """
+    # If already preprocessed (has block_type but no value/data), pass through
+    if "block_type" in obj and "value" not in obj and "data" not in obj:
+        return [obj]
+
+    conversation_id = obj.get("conversation_id", obj.get("id", ""))
+    value_raw = obj.get("value", obj.get("data", ""))
+
+    # Parse the JSON value column
+    if isinstance(value_raw, str):
+        try:
+            value = json.loads(value_raw)
+        except (json.JSONDecodeError, TypeError):
+            return [
+                {
+                    "block_type": "raw.parse_error",
+                    "conversation_id": conversation_id,
+                    "_raw": value_raw,
+                }
+            ]
+    elif isinstance(value_raw, dict):
+        value = value_raw
+    else:
+        return [obj]
+
+    messages = value.get("messages", value.get("history", []))
+    if not isinstance(messages, list):
+        return [{"block_type": "raw.no_messages", "conversation_id": conversation_id, **value}]
+
+    # Amazon Q persists conversation_id INSIDE the value blob; the SQLite `key`
+    # column is the working directory, not the id.
+    conversation_id = value.get("conversation_id", conversation_id)
+    results: list[dict[str, Any]] = []
+
+    # Current Amazon Q format: history is a list of {user, assistant, request_metadata}
+    # pairs, with externally-tagged Rust enums for content. Detect and expand.
+    if value.get("history") and any(
+        isinstance(e, dict) and ("user" in e or "assistant" in e) for e in messages
+    ):
+        for entry in messages:
+            if not isinstance(entry, dict):
+                continue
+            results.extend(_expand_amazonq_pair(entry, conversation_id))
+        return (
+            results
+            if results
+            else [{"block_type": "raw.empty", "conversation_id": conversation_id}]
+        )
+
+    for msg in messages:
+        if not isinstance(msg, dict):
+            continue
+        role = msg.get("role", "")
+        content = msg.get("content", "")
+
+        # Content can be a string or array of content blocks
+        if isinstance(content, str):
+            results.append(
+                {
+                    "block_type": f"message.{role}" if role else "message.unknown",
+                    "conversation_id": conversation_id,
+                    "content": content,
+                }
+            )
+        elif isinstance(content, list):
+            for block in content:
+                if not isinstance(block, dict):
+                    continue
+                block_type = block.get("type", "")
+
+                if block_type == "text":
+                    results.append(
+                        {
+                            "block_type": f"message.{role}" if role else "message.unknown",
+                            "conversation_id": conversation_id,
+                            "content": block.get("text", ""),
+                        }
+                    )
+
+                elif block_type == "tool_use":
+                    results.append(
+                        {
+                            "block_type": "tool.call",
+                            "conversation_id": conversation_id,
+                            "tool_call_id": block.get("id", ""),
+                            "tool_name": block.get("name", ""),
+                            "arguments": block.get("input", {}),
+                        }
+                    )
+
+                elif block_type == "tool_result":
+                    result_content = block.get("content", "")
+                    if isinstance(result_content, list):
+                        result_content = " ".join(
+                            b.get("text", "") for b in result_content if isinstance(b, dict)
+                        )
+                    results.append(
+                        {
+                            "block_type": "tool.result",
+                            "conversation_id": conversation_id,
+                            "tool_call_id": block.get("tool_use_id", ""),
+                            "is_error": block.get("is_error", False),
+                            "output": result_content,
+                        }
+                    )
+
+    return results if results else [{"block_type": "raw.empty", "conversation_id": conversation_id}]

traceforge/preprocessors/antigravity.py

@@ -0,0 +1,116 @@
+"""Google Antigravity preprocessor — replay the SDK ``conversation.history``.
+
+The Antigravity Python SDK exposes a finished agent run as
+``agent.conversation.history``: a ``list[google.antigravity.types.Step]`` that the
+capture scripts dump verbatim with ``model_dump(mode="json")``. Each JSONL line is
+therefore one serialized ``Step`` with these fields (see the SDK's
+``local_connection.LocalConnectionStep.from_dict`` which constructs them):
+
+    id, step_index, type, source, target, status, content, content_delta,
+    thinking, thinking_delta, tool_calls[], error, is_complete_response,
+    structured_output, usage_metadata
+
+There is no single native ``type_field`` because the event identity is a compound
+of ``type`` (StepType) and ``source`` (StepSource) — e.g. a ``TEXT_RESPONSE`` is a
+user message or an assistant message depending on its source. This preprocessor
+synthesizes a flat ``event_type`` discriminator from that pair and fans a
+``TOOL_CALL`` step out into one event per entry in ``tool_calls`` (an
+Antigravity step may batch several calls).
+
+Authoritative shape note: builtin tool *output* is consumed inside the Go
+localharness and fed back to the model — it is never surfaced as a result field on
+a history Step. So history carries tool CALLS (with args) but not tool RESULTS;
+this preprocessor intentionally emits no tool-result event.
+
+Stateless: each Step is self-describing, so no cross-line reconstruction is needed
+(unlike the copilot_vscode journal). Enum fields serialize to their string values
+(``"TEXT_RESPONSE"``, ``"MODEL"``, ``"TARGET_ENVIRONMENT"``, ``"DONE"`` ...).
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from traceforge.preprocessors.registry import register_preprocessor
+
+# StepType (string) -> handler. TEXT_RESPONSE is resolved by source; TOOL_CALL
+# fans out; the rest are 1:1.
+_TEXT = "TEXT_RESPONSE"
+_THINKING = "THINKING"
+_TOOL_CALL = "TOOL_CALL"
+_SYSTEM = "SYSTEM_MESSAGE"
+_FINISH = "FINISH"
+_COMPACTION = "COMPACTION"
+
+# StepSource (string) for a TEXT_RESPONSE -> event_type.
+_TEXT_BY_SOURCE = {
+    "USER": "user_message",
+    "MODEL": "assistant_message",
+    "SYSTEM": "system_message",
+}
+
+
+def _common(obj: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "step_id": obj.get("id"),
+        "step_index": obj.get("step_index"),
+        "source": obj.get("source"),
+        "target": obj.get("target"),
+        "status": obj.get("status"),
+    }
+
+
+@register_preprocessor("antigravity")
+def preprocess_antigravity(obj: dict[str, Any]) -> list[dict[str, Any]]:
+    """Expand one serialized Antigravity ``types.Step`` into typed flat events."""
+    # Passthrough for already-typed rows (generic conformance probes feed a line
+    # keyed by the post-preprocessor type_field, with no native Step ``type``).
+    if "type" not in obj and obj.get("event_type"):
+        return [obj]
+
+    stype = obj.get("type")
+    base = _common(obj)
+
+    if stype == _TEXT:
+        event_type = _TEXT_BY_SOURCE.get(obj.get("source"), "assistant_message")
+        return [{**base, "event_type": event_type, "content": obj.get("content")}]
+
+    if stype == _THINKING:
+        return [{**base, "event_type": "thinking", "content": obj.get("thinking")}]
+
+    if stype == _SYSTEM:
+        return [{**base, "event_type": "system_message", "content": obj.get("content")}]
+
+    if stype == _TOOL_CALL:
+        out: list[dict[str, Any]] = []
+        for call in obj.get("tool_calls") or []:
+            if not isinstance(call, dict):
+                continue
+            out.append(
+                {
+                    **base,
+                    "event_type": "tool_call",
+                    "tool_name": call.get("name"),
+                    "args": call.get("args"),
+                    "tool_call_id": call.get("id"),
+                    "path": call.get("canonical_path"),
+                }
+            )
+        return out
+
+    if stype == _FINISH:
+        return [
+            {
+                **base,
+                "event_type": "finish",
+                "content": obj.get("content"),
+                "structured_output": obj.get("structured_output"),
+            }
+        ]
+
+    if stype == _COMPACTION:
+        return [{**base, "event_type": "compaction"}]
+
+    # Unknown / UNKNOWN step types deliberately fall through (no event_type) so
+    # the golden 0-raw test surfaces genuine upstream drift rather than masking it.
+    return []

traceforge/preprocessors/claude.py

@@ -0,0 +1,95 @@
+"""Claude preprocessor — flatten content blocks into individual events."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from traceforge.preprocessors.registry import register_preprocessor
+
+
+@register_preprocessor("claude")
+def preprocess_claude(obj: dict[str, Any]) -> list[dict[str, Any]]:
+    """Flatten Claude wire-format messages into per-block dicts.
+
+    Claude wire format:
+      - user:      {type: "user", message: {content: "..."}}
+      - assistant:  {type: "assistant", message: {content: [{type: "text", ...}, ...]}}
+      - result:    {type: "result", subtype: ..., usage: {...}, ...}
+      - system:    {type: "system", ...}
+
+    Assistant messages contain a list of content blocks, each of which
+    becomes a separate normalized dict with a synthesized ``block_type``
+    discriminator (e.g. "assistant.text", "assistant.tool_use").
+
+    User messages with string content get block_type "user.text".
+    Result messages pass through with block_type "result".
+    """
+    msg_type = obj.get("type")
+    if not msg_type:
+        return [obj]
+
+    if msg_type == "result":
+        normalized = dict(obj)
+        normalized["block_type"] = "result"
+        # Flatten usage dict to top level for dot-path access
+        usage = normalized.get("usage")
+        if isinstance(usage, dict):
+            for k, v in usage.items():
+                normalized[f"usage_{k}"] = v
+        return [normalized]
+
+    if msg_type == "system":
+        normalized = dict(obj)
+        normalized["block_type"] = "system"
+        return [normalized]
+
+    message = obj.get("message")
+    if not isinstance(message, dict):
+        return [obj]
+
+    content = message.get("content")
+
+    if msg_type == "user":
+        if isinstance(content, str):
+            return [{"block_type": "user.text", "content": content}]
+        if isinstance(content, list):
+            return _flatten_blocks(content, "user")
+        return [{"block_type": "user.text", "content": str(content) if content else ""}]
+
+    if msg_type == "assistant":
+        if isinstance(content, list):
+            return _flatten_blocks(content, "assistant")
+        return [obj]
+
+    return [obj]
+
+
+def _flatten_blocks(blocks: list[Any], context: str) -> list[dict[str, Any]]:
+    """Flatten a list of content blocks into normalized dicts."""
+    results: list[dict[str, Any]] = []
+    for block in blocks:
+        if not isinstance(block, dict):
+            continue
+        block_kind = block.get("type", "unknown")
+        normalized = dict(block)
+        normalized["block_type"] = f"{context}.{block_kind}"
+
+        # For tool_result blocks, handle list-of-dicts content → joined text
+        if block_kind == "tool_result":
+            normalized["success"] = not block.get("is_error", False)
+            raw_content = block.get("content")
+            if isinstance(raw_content, list):
+                parts = []
+                for item in raw_content:
+                    if isinstance(item, dict) and item.get("type") == "text":
+                        parts.append(item.get("text", ""))
+                    elif isinstance(item, str):
+                        parts.append(item)
+                normalized["content_text"] = "\n".join(parts) if parts else None
+            elif isinstance(raw_content, str):
+                normalized["content_text"] = raw_content
+            else:
+                normalized["content_text"] = None
+
+        results.append(normalized)
+    return results

traceforge/preprocessors/cline.py

@@ -0,0 +1,36 @@
+"""Cline preprocessor — compound type + say/ask subtype discriminator."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from traceforge.preprocessors.registry import register_preprocessor
+
+
+@register_preprocessor("cline")
+def preprocess_cline(obj: dict[str, Any]) -> list[dict[str, Any]]:
+    """Synthesize compound type from Cline's type + say/ask subtype.
+
+    Cline events have type="ask"|"say" with the subtype in the
+    corresponding field. Synthesizes "say.api_req_started" etc.
+    Parses JSON text field for structured subtypes into top-level fields.
+    """
+    msg_type = obj.get("type")  # "ask" or "say"
+    subtype = obj.get(msg_type) if msg_type in ("ask", "say") else None
+
+    if subtype:
+        normalized = dict(obj)
+        normalized["type"] = f"{msg_type}.{subtype}"
+
+        # Parse JSON text field for known subtypes that embed structured data
+        text = normalized.get("text")
+        if text and subtype in ("api_req_started", "api_req_finished", "tool"):
+            try:
+                parsed = json.loads(text)
+                if isinstance(parsed, dict):
+                    normalized["parsed"] = parsed
+            except (json.JSONDecodeError, ValueError):
+                pass
+        return [normalized]
+    return [obj]