traceforge-toolkit 0.1.0__py3-none-any.whl

traceforge/preprocessors/goose.py

@@ -0,0 +1,156 @@
+"""Goose preprocessor — flatten nested content_json into typed events."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from traceforge.preprocessors.registry import register_preprocessor
+
+
+@register_preprocessor("goose")
+def preprocess_goose(obj: dict[str, Any]) -> list[dict[str, Any]]:
+    """Flatten Goose nested content_json into separate typed events."""
+    results = []
+    role = obj.get("role")
+    content_json_raw = obj.get("content_json")
+    ts = obj.get("created_at") or obj.get("created_timestamp")
+
+    if not content_json_raw:
+        return [obj]
+
+    # Parse content_json if it's a string
+    if isinstance(content_json_raw, str):
+        try:
+            content_items = json.loads(content_json_raw)
+        except (json.JSONDecodeError, ValueError):
+            return [obj]
+    else:
+        content_items = content_json_raw
+
+    if not isinstance(content_items, list):
+        return [obj]
+
+    # Extract nested events from content array
+    has_text = False
+    for item in content_items:
+        if not isinstance(item, dict):
+            continue
+        item_type = item.get("type", "")
+
+        if item_type == "text":
+            has_text = True
+        elif item_type == "toolRequest":
+            tool_call = item.get("toolCall", {})
+            value = (tool_call.get("value") or {}) if isinstance(tool_call, dict) else {}
+            results.append(
+                {
+                    "role": "tool_use",
+                    "created_at": ts,
+                    "name": value.get("name", ""),
+                    "id": item.get("id", ""),
+                    "input": value.get("arguments", {}),
+                }
+            )
+        elif item_type == "toolResponse":
+            tool_result = item.get("toolResult", {})
+            results.append(
+                {
+                    "role": "tool_result",
+                    "created_at": ts,
+                    "tool_use_id": item.get("id", ""),
+                    "content": (tool_result.get("value") or {}).get("content", "")
+                    if isinstance(tool_result, dict)
+                    else "",
+                    "is_success": tool_result.get("status") == "success"
+                    if isinstance(tool_result, dict)
+                    else False,
+                }
+            )
+        elif item_type == "thinking":
+            results.append(
+                {
+                    "role": "thinking",
+                    "created_at": ts,
+                    "thinking": item.get("thinking", ""),
+                }
+            )
+        elif item_type == "redactedThinking":
+            results.append(
+                {
+                    "role": "redacted_thinking",
+                    "created_at": ts,
+                    "data": item.get("data", ""),
+                }
+            )
+        elif item_type == "image":
+            results.append(
+                {
+                    "role": "image",
+                    "created_at": ts,
+                    "data": item.get("data", ""),
+                }
+            )
+        elif item_type == "toolConfirmationRequest":
+            results.append(
+                {
+                    "role": "tool_confirmation_request",
+                    "created_at": ts,
+                    "content": item.get("prompt", ""),
+                    "name": item.get("toolName", ""),
+                    "id": item.get("id", ""),
+                }
+            )
+        elif item_type == "actionRequired":
+            data = item.get("data", {}) if isinstance(item.get("data"), dict) else {}
+            # ActionRequiredData sub-variants:
+            #   Elicitation: {actionType, id, message, requestedSchema}
+            #   ToolConfirmation: {actionType, id, toolName, arguments, prompt}
+            #   ElicitationResponse: {actionType, id, userData}
+            content = data.get("message") or data.get("prompt") or str(data.get("userData", ""))
+            results.append(
+                {
+                    "role": "action_required",
+                    "created_at": ts,
+                    "content": content,
+                    "action_type": data.get("actionType", ""),
+                }
+            )
+        elif item_type == "frontendToolRequest":
+            tool_call = item.get("toolCall", {})
+            value = (tool_call.get("value") or {}) if isinstance(tool_call, dict) else {}
+            results.append(
+                {
+                    "role": "frontend_tool_request",
+                    "created_at": ts,
+                    "name": value.get("name", ""),
+                    "id": item.get("id", ""),
+                    "input": value.get("arguments", {}),
+                }
+            )
+        elif item_type == "systemNotification":
+            results.append(
+                {
+                    "role": "system_notification",
+                    "created_at": ts,
+                    "content": item.get("msg", ""),
+                }
+            )
+
+    # Always emit the message itself (with role)
+    if has_text or not results:
+        text_parts = [
+            i.get("text", "")
+            for i in content_items
+            if isinstance(i, dict) and i.get("type") == "text"
+        ]
+        results.insert(
+            0,
+            {
+                "role": role,
+                "created_at": ts,
+                "content": "\n".join(text_parts) if text_parts else content_json_raw,
+            },
+        )
+
+    return results

traceforge/preprocessors/maf_transcript.py

@@ -0,0 +1,84 @@
+"""MAF transcript preprocessor — normalize Activity JSONL into typed events.
+
+The Microsoft 365 Agents SDK `FileTranscriptStore` writes one Activity per JSONL
+line.  Activities use `type` ("message", "typing", "event", "invoke", etc.) and
+`from.role` ("bot", "user") to distinguish direction. This preprocessor emits a
+compound `_event_type` field of the form `{type}.{from_role}` so the YAML mapping
+can discriminate assistant vs user messages.
+
+Expected input shape (each JSONL line parsed as dict):
+    {
+        "type": "message",
+        "text": "I'll look into that for you.",
+        "timestamp": "2025-01-15T10:30:00Z",
+        "from": {"id": "bot-id", "name": "Agent", "role": "bot"},
+        "recipient": {"id": "user-id", "name": "User", "role": "user"},
+        "conversation": {"id": "conv-123"},
+        "channel_id": "directline",
+        "id": "activity-456",
+        ...
+    }
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from traceforge.preprocessors.registry import register_preprocessor
+
+
+@register_preprocessor("maf_transcript")
+def preprocess_maf_transcript(obj: dict[str, Any]) -> list[dict[str, Any]]:
+    """Flatten MAF Activity dict into a single event with compound type."""
+    activity_type = obj.get("type", "unknown")
+
+    # Determine sender role — normalize to lowercase
+    from_obj = obj.get("from") or obj.get("from_") or {}
+    from_role = ""
+    if isinstance(from_obj, dict):
+        from_role = (from_obj.get("role") or "").lower()
+
+    # Compound type: e.g. "message.bot", "message.user", "typing.bot", "event.bot"
+    event_type = f"{activity_type}.{from_role}" if from_role else activity_type
+
+    # Flatten key fields to top level for YAML mapping
+    result: dict[str, Any] = {
+        "_event_type": event_type,
+        "timestamp": obj.get("timestamp"),
+        "text": obj.get("text"),
+        "activity_id": obj.get("id"),
+        "channel_id": obj.get("channel_id"),
+    }
+
+    # Conversation ID
+    conversation = obj.get("conversation")
+    if isinstance(conversation, dict):
+        result["conversation_id"] = conversation.get("id")
+
+    # From metadata
+    if isinstance(from_obj, dict):
+        result["from_id"] = from_obj.get("id")
+        result["from_name"] = from_obj.get("name")
+        result["from_role"] = from_obj.get("role")
+
+    # Recipient metadata
+    recipient = obj.get("recipient")
+    if isinstance(recipient, dict):
+        result["recipient_id"] = recipient.get("id")
+        result["recipient_name"] = recipient.get("name")
+
+    # Attachments (tool inputs/outputs often come as attachments)
+    attachments = obj.get("attachments")
+    if attachments and isinstance(attachments, list):
+        result["attachments"] = attachments
+        result["attachment_count"] = len(attachments)
+
+    # Value field (used for invoke activities — tool calls, adaptive cards)
+    if obj.get("value") is not None:
+        result["value"] = obj["value"]
+
+    # Channel data (framework-specific extensions)
+    if obj.get("channel_data") is not None:
+        result["channel_data"] = obj["channel_data"]
+
+    return [result]

traceforge/preprocessors/openai_agents.py

@@ -0,0 +1,86 @@
+"""OpenAI Agents SDK preprocessor — normalize trace/span exports."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from traceforge.preprocessors.registry import register_preprocessor
+
+
+@register_preprocessor("openai_agents")
+def preprocess_openai_agents(obj: dict[str, Any]) -> list[dict[str, Any]]:
+    """Flatten native Trace.export()/Span.export() rows into typed events."""
+    object_type = obj.get("object")
+
+    # Passthrough already-typed rows (re-ingested events or generic conformance
+    # probes that arrive keyed by the post-preprocessor type_field).
+    if object_type is None and obj.get("event_type"):
+        return [obj]
+
+    if object_type == "trace":
+        return [
+            {
+                "event_type": "trace",
+                "trace_id": obj.get("id"),
+                "workflow_name": obj.get("workflow_name"),
+                "group_id": obj.get("group_id"),
+                "metadata": obj.get("metadata"),
+            }
+        ]
+
+    if object_type != "trace.span":
+        return []
+
+    span_data = obj.get("span_data") or {}
+    if not isinstance(span_data, dict):
+        return []
+
+    span_type = span_data.get("type")
+    base = {
+        "span_id": obj.get("id"),
+        "trace_id": obj.get("trace_id"),
+        "parent_id": obj.get("parent_id"),
+        "started_at": obj.get("started_at"),
+        "ended_at": obj.get("ended_at"),
+        "error": obj.get("error"),
+        "span_data": span_data,
+    }
+
+    if span_type == "function":
+        started = {
+            **base,
+            "event_type": "function.started",
+            "timestamp": obj.get("started_at"),
+            "tool_name": span_data.get("name"),
+            "arguments": span_data.get("input"),
+        }
+        finished_type = "function.failed" if obj.get("error") else "function.completed"
+        finished = {
+            **base,
+            "event_type": finished_type,
+            "timestamp": obj.get("ended_at") or obj.get("started_at"),
+            "tool_name": span_data.get("name"),
+            "result": span_data.get("output"),
+        }
+        return [started, finished]
+
+    event_type_by_span = {
+        "agent": "agent.completed",
+        "generation": "generation.completed",
+        "response": "response.completed",
+        "handoff": "handoff.completed",
+        "guardrail": "guardrail.completed",
+        "custom": "custom.completed",
+        "speech": "speech.completed",
+        "speech_group": "speech_group.completed",
+        "transcription": "transcription.completed",
+        "mcp_tools": "mcp_tools.completed",
+    }
+    event_type = event_type_by_span.get(str(span_type), f"{span_type}.completed")
+    return [
+        {
+            **base,
+            "event_type": event_type,
+            "timestamp": obj.get("ended_at") or obj.get("started_at"),
+        }
+    ]

traceforge/preprocessors/opencode.py

@@ -0,0 +1,85 @@
+"""OpenCode preprocessor — normalize v1.17 event-sourced SQLite rows."""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+from traceforge.preprocessors.registry import register_preprocessor
+
+_VERSION_SUFFIX_RE = re.compile(r"\.\d+$")
+_MESSAGE_ROLES: dict[tuple[str, str], str] = {}
+
+
+@register_preprocessor("opencode")
+def preprocess_opencode(obj: dict[str, Any]) -> list[dict[str, Any]]:
+    """Normalize OpenCode's SQLite `event` rows to stable mapping types."""
+    data = obj.get("data")
+    if isinstance(data, str):
+        try:
+            data = json.loads(data)
+        except (json.JSONDecodeError, TypeError):
+            data = {}
+    if not isinstance(data, dict):
+        data = {}
+
+    norm = dict(obj)
+    norm["data"] = data
+    if "properties" not in norm and data:
+        norm["properties"] = data
+    norm["type"] = _VERSION_SUFFIX_RE.sub("", str(obj.get("type", "")))
+    norm["_timestamp"] = _timestamp(data, norm)
+
+    if norm["type"] == "message.updated":
+        info = data.get("info") if isinstance(data.get("info"), dict) else {}
+        session_id = str(data.get("sessionID") or info.get("sessionID") or "")
+        message_id = str(info.get("id") or "")
+        role = str(info.get("role") or "unknown")
+        if session_id and message_id and role != "unknown":
+            _MESSAGE_ROLES[(session_id, message_id)] = role
+        norm["type"] = f"message.updated.{role}"
+        norm["message_role"] = role
+        return [norm]
+
+    if norm["type"] == "message.part.updated":
+        part = data.get("part") if isinstance(data.get("part"), dict) else {}
+        part_type = str(part.get("type") or "unknown")
+        session_id = str(data.get("sessionID") or part.get("sessionID") or "")
+        message_id = str(part.get("messageID") or "")
+        role = _MESSAGE_ROLES.get((session_id, message_id), "unknown")
+        norm["message_role"] = role
+
+        if part_type == "text" and role in {"user", "assistant"}:
+            norm["type"] = f"message.part.text.{role}"
+        elif part_type == "tool":
+            state = part.get("state") if isinstance(part.get("state"), dict) else {}
+            status = str(state.get("status") or "unknown")
+            if status == "pending":
+                # OpenCode emits a provisional tool row before input/timing exists;
+                # the following running/completed rows carry the canonical signal.
+                return []
+            norm["type"] = f"message.part.tool.{status}"
+        else:
+            norm["type"] = f"message.part.{part_type}"
+        return [norm]
+
+    return [norm]
+
+
+def _timestamp(data: dict[str, Any], obj: dict[str, Any]) -> Any:
+    props = obj.get("properties") if isinstance(obj.get("properties"), dict) else {}
+    info = data.get("info") if isinstance(data.get("info"), dict) else {}
+    info_time = info.get("time") if isinstance(info.get("time"), dict) else {}
+    part = data.get("part") if isinstance(data.get("part"), dict) else {}
+    part_time = part.get("time") if isinstance(part.get("time"), dict) else {}
+
+    return (
+        data.get("timestamp")
+        or data.get("time")
+        or info_time.get("updated")
+        or info_time.get("created")
+        or part_time.get("end")
+        or part_time.get("start")
+        or props.get("timestamp")
+    )

traceforge/preprocessors/openhands.py

@@ -0,0 +1,36 @@
+"""OpenHands preprocessor — compound action/observation discriminator."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from traceforge.preprocessors.registry import register_preprocessor
+
+
+@register_preprocessor("openhands")
+def preprocess_openhands(obj: dict[str, Any]) -> list[dict[str, Any]]:
+    """Normalize OpenHands compound discriminator (legacy or SDK events).
+
+    OpenHands 0.x used an ``action``/``observation`` compound discriminator.
+    OpenHands 1.x SDK persistence uses ``kind`` plus ``source``/``tool_name``.
+    Synthesize the YAML ``action`` discriminator for both formats while
+    preserving the native nested payload for dot-path extraction.
+    """
+    kind = obj.get("kind")
+    if kind in {"SystemPromptEvent", "ActionEvent", "ObservationEvent", "MessageEvent"}:
+        normalized = dict(obj)
+        if kind in {"ActionEvent", "ObservationEvent"}:
+            normalized["action"] = f"{kind}.{obj.get('tool_name', 'unknown')}"
+        elif kind == "MessageEvent":
+            normalized["action"] = f"MessageEvent.{obj.get('source', 'unknown')}"
+        else:
+            normalized["action"] = kind
+        return [normalized]
+
+    if "action" in obj:
+        return [obj]
+    elif "observation" in obj:
+        normalized = dict(obj)
+        normalized["action"] = f"observation.{normalized['observation']}"
+        return [normalized]
+    return [obj]

traceforge/preprocessors/pydantic_ai.py

@@ -0,0 +1,62 @@
+"""PydanticAI preprocessor — multi-level discrimination to flat type field."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from traceforge.preprocessors.registry import register_preprocessor
+
+
+@register_preprocessor("pydantic_ai")
+def preprocess_pydantic_ai(obj: dict[str, Any]) -> list[dict[str, Any]]:
+    """Normalize PydanticAI multi-level discrimination to flat type field.
+
+    Preserves nested structure for _resolve_path; only synthesizes the "type"
+    discriminator and extracts text content from parts arrays.
+    """
+    # Stream and callback events have event_kind
+    if "event_kind" in obj:
+        normalized = dict(obj)
+        event_kind = normalized["event_kind"]
+
+        if event_kind == "function_tool_call":
+            normalized["type"] = "tool_call_start"
+        elif event_kind == "function_tool_result":
+            normalized["type"] = "tool_call_end"
+        elif event_kind == "model_response_stream":
+            normalized["type"] = "model_response_chunk"
+            if "chunk" not in normalized:
+                normalized["chunk"] = (normalized.get("part") or {}).get("content", "")
+        else:
+            normalized["type"] = f"stream.{event_kind}"
+        return [normalized]
+
+    # Messages have kind (request/response)
+    kind = obj.get("kind")
+    if kind == "response":
+        normalized = dict(obj)
+        normalized["type"] = "model_response"
+        # Extract text from parts for convenience
+        parts = normalized.get("parts") or []
+        text_parts = [
+            p.get("content", "")
+            for p in parts
+            if isinstance(p, dict) and p.get("part_kind") == "text"
+        ]
+        if text_parts:
+            normalized["content"] = "\n".join(text_parts)
+        return [normalized]
+    elif kind == "request":
+        normalized = dict(obj)
+        normalized["type"] = "model_request"
+        parts = normalized.get("parts") or []
+        user_parts = [
+            p.get("content", "")
+            for p in parts
+            if isinstance(p, dict) and p.get("part_kind") == "user-prompt"
+        ]
+        if user_parts:
+            normalized["content"] = "\n".join(user_parts)
+        return [normalized]
+
+    return [obj]

traceforge/preprocessors/registry.py

@@ -0,0 +1,24 @@
+"""Preprocessor registry — central registration and lookup."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+PreprocessorFn = Callable[[dict[str, Any]], list[dict[str, Any]]]
+_PREPROCESSORS: dict[str, PreprocessorFn] = {}
+
+
+def register_preprocessor(name: str) -> Callable[[PreprocessorFn], PreprocessorFn]:
+    """Decorator to register a preprocessor function."""
+
+    def decorator(fn: PreprocessorFn) -> PreprocessorFn:
+        _PREPROCESSORS[name] = fn
+        return fn
+
+    return decorator
+
+
+def get_preprocessor(name: str) -> PreprocessorFn | None:
+    """Look up a registered preprocessor by name."""
+    return _PREPROCESSORS.get(name)

traceforge/preprocessors/smolagents.py

@@ -0,0 +1,90 @@
+"""smolagents preprocessor — infer step type from field presence."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from traceforge.preprocessors.registry import register_preprocessor
+
+
+@register_preprocessor("smolagents")
+def preprocess_smolagents(obj: dict[str, Any]) -> list[dict[str, Any]]:
+    """Infer step type from field presence (smolagents has no discriminator).
+
+    Only synthesizes the step_type field and extracts timestamps from timing.
+    If step_type is already present, trusts the existing value.
+    Nested structures (token_usage, tool_calls) preserved for _resolve_path.
+    """
+    normalized = dict(obj)
+
+    # Extract timestamp from timing dict if present
+    timing = normalized.get("timing", {})
+    if isinstance(timing, dict) and "start_time" in timing:
+        normalized["timestamp"] = timing["start_time"]
+
+    # If step_type already present (e.g., from callback wrappers), trust it
+    if "step_type" in normalized:
+        # Still handle tool_calls splitting for ActionStep
+        if normalized["step_type"] == "ActionStep":
+            tool_calls = normalized.get("tool_calls", [])
+            if tool_calls and isinstance(tool_calls, list):
+                results: list[dict[str, Any]] = [normalized]
+                for tc in tool_calls:
+                    if isinstance(tc, dict):
+                        fn = tc.get("function", {})
+                        results.append(
+                            {
+                                "step_type": "ToolCall",
+                                "timestamp": normalized.get("timestamp"),
+                                "tool_name": fn.get("name", "") if isinstance(fn, dict) else "",
+                                "call_id": tc.get("id", ""),
+                                "tool_input": fn.get("arguments", "")
+                                if isinstance(fn, dict)
+                                else "",
+                            }
+                        )
+                return results
+        return [normalized]
+
+    # Determine step type from field presence
+    # Order matters: check most specific first
+    if "step_number" in normalized:
+        # ActionStep — but check if it's the final answer
+        if normalized.get("is_final_answer"):
+            # ActionStep with is_final_answer=true: action_output IS the answer
+            normalized["step_type"] = "FinalAnswer"
+            normalized["output"] = normalized.get("action_output", "")
+        else:
+            normalized["step_type"] = "ActionStep"
+        tool_calls = normalized.get("tool_calls", [])
+        if tool_calls and isinstance(tool_calls, list):
+            results = [normalized]
+            for tc in tool_calls:
+                if isinstance(tc, dict):
+                    fn = tc.get("function", {})
+                    results.append(
+                        {
+                            "step_type": "ToolCall",
+                            "timestamp": normalized.get("timestamp"),
+                            "tool_name": fn.get("name", "") if isinstance(fn, dict) else "",
+                            "call_id": tc.get("id", ""),
+                            "tool_input": fn.get("arguments", "") if isinstance(fn, dict) else "",
+                        }
+                    )
+            return results
+    elif "plan" in normalized:
+        normalized["step_type"] = "PlanningStep"
+    elif "system_prompt" in normalized:
+        normalized["step_type"] = "SystemPromptStep"
+    elif "task" in normalized:
+        normalized["step_type"] = "TaskStep"
+    elif (
+        "output" in normalized
+        and len(set(normalized.keys()) - {"output", "timestamp", "step_type"}) == 0
+    ):
+        # Bare FinalAnswerStep: only has "output" (+ maybe timestamp)
+        normalized["step_type"] = "FinalAnswer"
+    else:
+        normalized["step_type"] = "unknown"
+
+    return [normalized]