prizmkit 1.1.145 → 1.1.147

package/bundled/dev-pipeline/scripts/parse-stream-progress.py

@@ -23,7 +23,8 @@ import signal
 import sys
 import tempfile
 import time
-from collections import Counter
+from collections import Counter, deque
+from dataclasses import dataclass
 from datetime import datetime, timezone
 from pathlib import Path
 
@@ -198,6 +199,517 @@ def _normalize_status(status, default="unknown"):
     return text
 
 
+@dataclass(frozen=True)
+class SemanticLogEvent:
+    """One provider-neutral semantic fragment destined for session.log."""
+
+    kind: str
+    payload: str = ""
+    stream_id: str = ""
+    mode: str = "complete"
+    deduplicate: bool = False
+
+
+class SemanticLogRenderer:
+    """Render semantic fragments without consulting their source format."""
+
+    LABELS = {
+        "think": "[THINK]",
+        "action": "[ACTION]",
+        "observation": "[OBSERVATION]",
+        "error": "[ERROR]",
+        "final": "[FINAL]",
+        "session_end": "[SESSION_END]",
+        "output": "[OUTPUT]",
+    }
+
+    def __init__(self):
+        self._active_stream = ""
+
+    def _close_active(self):
+        if not self._active_stream:
+            return ""
+        self._active_stream = ""
+        return "\n"
+
+    def render(self, event):
+        label = self.LABELS.get(event.kind, "[OUTPUT]")
+        payload = str(event.payload or "")
+        mode = event.mode or "complete"
+        stream_id = event.stream_id or f"{event.kind}:anonymous"
+
+        if mode == "complete":
+            prefix = self._close_active()
+            suffix = "" if payload.endswith("\n") else "\n"
+            spacer = " " if payload else ""
+            return f"{prefix}{label}{spacer}{payload}{suffix}"
+
+        if mode == "start":
+            prefix = self._close_active()
+            self._active_stream = stream_id
+            return f"{prefix}{label} {payload}"
+
+        if mode == "delta":
+            if self._active_stream == stream_id:
+                return payload
+            prefix = self._close_active()
+            self._active_stream = stream_id
+            return f"{prefix}{label} {payload}"
+
+        if mode == "end":
+            if self._active_stream == stream_id:
+                self._active_stream = ""
+                return f"{payload}\n"
+            if not payload:
+                return ""
+            prefix = self._close_active()
+            suffix = "" if payload.endswith("\n") else "\n"
+            return f"{prefix}{label} {payload}{suffix}"
+
+        return self.render(SemanticLogEvent(event.kind, payload, stream_id, "complete"))
+
+    def finish(self):
+        return self._close_active()
+
+
+class SemanticLogNormalizer:
+    """Adapt supported structured shapes into one compact semantic log stream."""
+
+    _SUCCESS_STOPS = {"stop", "end_turn", "completed", "success"}
+    _ERROR_STOPS = {"error", "aborted", "failed", "cancelled", "canceled"}
+
+    def __init__(self):
+        self.renderer = SemanticLogRenderer()
+        self._scope = 0
+        self._block_streams = {}
+        self._stream_buffers = {}
+        self._snapshots = {}
+        self._recent_digests = deque()
+        self._recent_digest_set = set()
+        self._recent_action_ids = deque()
+        self._recent_action_id_set = set()
+        self._recent_output_ids = deque()
+        self._recent_output_id_set = set()
+        self._max_recent = 256
+        self._terminal_emitted = False
+
+    @staticmethod
+    def _json_text(value):
+        if isinstance(value, str):
+            return value
+        return json.dumps(value, ensure_ascii=False, sort_keys=True, default=str)
+
+    @classmethod
+    def _value_text(cls, value):
+        if value in (None, ""):
+            return ""
+        if isinstance(value, str):
+            return value
+        if isinstance(value, list):
+            return "".join(cls._value_text(item) for item in value)
+        if isinstance(value, dict):
+            for key in ("text", "output", "content"):
+                if key in value:
+                    return cls._value_text(value.get(key))
+            return cls._json_text(value)
+        return str(value)
+
+    def _digest(self, kind, payload):
+        canonical_payload = str(payload or "").rstrip()
+        return hashlib.sha1(f"{self._scope}\0{kind}\0{canonical_payload}".encode("utf-8")).hexdigest()
+
+    def _remember_digest(self, kind, payload):
+        if not payload:
+            return
+        digest = self._digest(kind, payload)
+        if digest in self._recent_digest_set:
+            return
+        if len(self._recent_digests) >= self._max_recent:
+            old = self._recent_digests.popleft()
+            self._recent_digest_set.discard(old)
+        self._recent_digests.append(digest)
+        self._recent_digest_set.add(digest)
+
+    def _is_recent(self, kind, payload):
+        return bool(payload) and self._digest(kind, payload) in self._recent_digest_set
+
+    def _remember_action_id(self, action_id):
+        action_id = str(action_id or "")
+        if not action_id or action_id in self._recent_action_id_set:
+            return
+        if len(self._recent_action_ids) >= self._max_recent:
+            old = self._recent_action_ids.popleft()
+            self._recent_action_id_set.discard(old)
+        self._recent_action_ids.append(action_id)
+        self._recent_action_id_set.add(action_id)
+
+    def _remember_output_id(self, output_id):
+        output_id = str(output_id or "")
+        if not output_id or output_id in self._recent_output_id_set:
+            return
+        if len(self._recent_output_ids) >= self._max_recent:
+            old = self._recent_output_ids.popleft()
+            self._recent_output_id_set.discard(old)
+        self._recent_output_ids.append(output_id)
+        self._recent_output_id_set.add(output_id)
+
+    def _emit(self, event):
+        payload = str(event.payload or "")
+        if event.mode == "complete":
+            if event.deduplicate and self._is_recent(event.kind, payload):
+                return ""
+            if event.deduplicate:
+                self._remember_digest(event.kind, payload)
+            return self.renderer.render(event)
+
+        stream_id = event.stream_id or f"{event.kind}:anonymous"
+        if event.mode == "start":
+            self._stream_buffers[stream_id] = payload
+        elif event.mode == "delta":
+            self._stream_buffers[stream_id] = self._stream_buffers.get(stream_id, "") + payload
+        elif event.mode == "end":
+            complete = self._stream_buffers.pop(stream_id, "") + payload
+            self._remember_digest(event.kind, complete)
+        return self.renderer.render(event)
+
+    def _complete(self, kind, payload, *, deduplicate=False):
+        payload = str(payload or "")
+        if not payload:
+            return ""
+        return self._emit(SemanticLogEvent(kind, payload, deduplicate=deduplicate))
+
+    def _session_end(self, payload):
+        if self._terminal_emitted:
+            return ""
+        self._terminal_emitted = True
+        return self._complete("session_end", payload)
+
+    def _stream(self, kind, stream_id, mode, payload=""):
+        return self._emit(SemanticLogEvent(kind, str(payload or ""), stream_id, mode))
+
+    def _snapshot(self, kind, stream_id, value, final=False):
+        """Render cumulative snapshots while retaining only one active full value."""
+        value = str(value or "")
+        previous = self._snapshots.get(stream_id)
+        rendered = ""
+        if previous is None:
+            self._snapshots[stream_id] = value
+            if value:
+                rendered += self.renderer.render(SemanticLogEvent(kind, value, stream_id, "start"))
+        elif value == previous:
+            pass
+        elif value.startswith(previous):
+            self._snapshots[stream_id] = value
+            rendered += self.renderer.render(
+                SemanticLogEvent(kind, value[len(previous):], stream_id, "delta")
+            )
+        else:
+            rendered += self.renderer.render(SemanticLogEvent(kind, "", stream_id, "end"))
+            self._snapshots[stream_id] = value
+            if value:
+                rendered += self.renderer.render(SemanticLogEvent(kind, value, stream_id, "start"))
+        if final:
+            rendered += self.renderer.render(SemanticLogEvent(kind, "", stream_id, "end"))
+            self._snapshots.pop(stream_id, None)
+        return rendered
+
+    def _message_parts(self, message):
+        if not isinstance(message, dict):
+            return ""
+        rendered = ""
+        content = message.get("content")
+        if isinstance(content, str):
+            rendered += self._complete("final", content)
+        elif isinstance(content, list):
+            for block in content:
+                if not isinstance(block, dict):
+                    continue
+                block_type = str(block.get("type") or "")
+                if block_type in {"text", "output_text"}:
+                    rendered += self._complete(
+                        "final",
+                        block.get("text") or block.get("content") or "",
+                        deduplicate=True,
+                    )
+                elif block_type in {"thinking", "reasoning"}:
+                    rendered += self._complete(
+                        "think",
+                        block.get("thinking") or block.get("text") or "",
+                        deduplicate=True,
+                    )
+                elif block_type == "tool_use":
+                    action_id = block.get("id") or block.get("tool_use_id")
+                    self._remember_action_id(action_id)
+                    rendered += self._complete(
+                        "action",
+                        f"{block.get('name') or 'tool'} {self._json_text(block.get('input') or {})}",
+                    )
+                elif block_type == "tool_result":
+                    rendered += self._complete("observation", self._value_text(block.get("content")))
+        return rendered
+
+    def _tool_result_parts(self, message):
+        if not isinstance(message, dict):
+            return ""
+        rendered = ""
+        content = message.get("content")
+        if not isinstance(content, list):
+            return rendered
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == "tool_result":
+                rendered += self._complete("observation", self._value_text(block.get("content")))
+        return rendered
+
+    def _pi_message_update(self, event):
+        update = event.get("assistantMessageEvent")
+        if not isinstance(update, dict):
+            return ""
+        subtype = str(update.get("type") or "")
+        index = update.get("contentIndex", 0)
+        stream_id = f"turn:{self._scope}:content:{index}"
+        if subtype == "thinking_start":
+            return self._stream("think", f"{stream_id}:think", "start")
+        if subtype == "thinking_delta":
+            return self._stream("think", f"{stream_id}:think", "delta", update.get("delta"))
+        if subtype == "thinking_end":
+            return self._stream("think", f"{stream_id}:think", "end")
+        if subtype == "text_start":
+            return self._stream("final", f"{stream_id}:final", "start")
+        if subtype == "text_delta":
+            return self._stream("final", f"{stream_id}:final", "delta", update.get("delta"))
+        if subtype == "text_end":
+            return self._stream("final", f"{stream_id}:final", "end")
+        if subtype == "toolcall_start":
+            partial = update.get("partial")
+            blocks = partial.get("content") if isinstance(partial, dict) else []
+            block = blocks[index] if isinstance(blocks, list) and isinstance(index, int) and index < len(blocks) else {}
+            action_id = block.get("id") if isinstance(block, dict) else ""
+            self._remember_action_id(action_id)
+            prefix = f"{block.get('name') or 'tool'} " if isinstance(block, dict) else "tool "
+            return self._stream("action", f"{stream_id}:action", "start", prefix)
+        if subtype == "toolcall_delta":
+            return self._stream("action", f"{stream_id}:action", "delta", update.get("delta"))
+        if subtype == "toolcall_end":
+            tool_call = update.get("toolCall")
+            if isinstance(tool_call, dict):
+                self._remember_action_id(tool_call.get("id"))
+            return self._stream("action", f"{stream_id}:action", "end")
+        return ""
+
+    def _content_block_event(self, event):
+        event_type = str(event.get("type") or "")
+        index = event.get("index", 0)
+        stream_id = f"block:{self._scope}:{index}"
+        if event_type == "content_block_start":
+            block = event.get("content_block") if isinstance(event.get("content_block"), dict) else {}
+            block_type = str(block.get("type") or "")
+            if block_type in {"thinking", "reasoning"}:
+                self._block_streams[index] = ("think", stream_id)
+                return self._stream("think", stream_id, "start", block.get("thinking") or "")
+            if block_type == "text":
+                self._block_streams[index] = ("final", stream_id)
+                return self._stream("final", stream_id, "start", block.get("text") or "")
+            if block_type == "tool_use":
+                self._block_streams[index] = ("action", stream_id)
+                self._remember_action_id(block.get("id"))
+                return self._stream("action", stream_id, "start", f"{block.get('name') or 'tool'} ")
+            return ""
+        if event_type == "content_block_delta":
+            delta = event.get("delta") if isinstance(event.get("delta"), dict) else {}
+            delta_type = str(delta.get("type") or "")
+            if delta_type in {"thinking_delta", "reasoning_delta"}:
+                kind = "think"
+                payload = delta.get("thinking") or delta.get("text") or ""
+            elif delta_type == "text_delta":
+                kind = "final"
+                payload = delta.get("text") or ""
+            elif delta_type == "input_json_delta":
+                kind = "action"
+                payload = delta.get("partial_json") or ""
+            else:
+                return ""
+            active_kind, active_id = self._block_streams.get(index, (kind, stream_id))
+            return self._stream(active_kind, active_id, "delta", payload)
+        if event_type == "content_block_stop":
+            kind, active_id = self._block_streams.pop(index, ("output", stream_id))
+            return self._stream(kind, active_id, "end")
+        return ""
+
+    def process_text(self, text):
+        text = str(text or "")
+        return self._complete("output", text.rstrip("\n")) if text.strip() else ""
+
+    def process_event(self, event):
+        if not isinstance(event, dict):
+            return self.process_text(event)
+        event_type = str(event.get("type") or "")
+        rendered = ""
+
+        if event_type == "turn_start" or event_type == "turn.started":
+            self._scope += 1
+
+        if event_type == "message_update" and isinstance(event.get("assistantMessageEvent"), dict):
+            return self._pi_message_update(event)
+
+        if event_type == "message_end":
+            message = event.get("message")
+            if isinstance(message, dict):
+                role = str(message.get("role") or "")
+                if role == "assistant":
+                    rendered += self._message_parts(message)
+                elif role == "toolResult":
+                    output_id = str(message.get("toolCallId") or "")
+                    if output_id not in self._recent_output_id_set:
+                        self._remember_output_id(output_id)
+                        rendered += self._complete("observation", self._value_text(message.get("content")))
+                stop_reason = str(message.get("stopReason") or "")
+                if stop_reason in self._ERROR_STOPS:
+                    rendered += self._complete("error", message.get("errorMessage") or stop_reason)
+            return rendered
+
+        if event_type in {"turn_end", "agent_settled", "agent_start", "session", "thread.started", "turn.started"}:
+            return ""
+
+        if event_type == "tool_execution_start":
+            action_id = str(event.get("toolCallId") or event.get("id") or "")
+            if action_id in self._recent_action_id_set:
+                return ""
+            self._remember_action_id(action_id)
+            return self._complete(
+                "action",
+                f"{event.get('toolName') or 'tool'} {self._json_text(event.get('args') or {})}",
+            )
+
+        if event_type == "tool_execution_update":
+            stream_id = f"tool:{event.get('toolCallId') or event.get('id') or 'active'}:output"
+            return self._snapshot("observation", stream_id, self._value_text(event.get("partialResult")))
+
+        if event_type == "tool_execution_end":
+            output_id = str(event.get("toolCallId") or event.get("id") or "")
+            self._remember_output_id(output_id)
+            stream_id = f"tool:{output_id or 'active'}:output"
+            result_text = self._value_text(event.get("result"))
+            had_output = stream_id in self._snapshots and bool(self._snapshots.get(stream_id))
+            rendered += self._snapshot("observation", stream_id, result_text, final=True)
+            if event.get("isError"):
+                error_payload = f"{event.get('toolName') or 'tool'} failed"
+                if not had_output and result_text:
+                    error_payload = f"{error_payload}: {result_text}"
+                rendered += self._complete("error", error_payload)
+            return rendered
+
+        if event_type == "auto_retry_start":
+            message = event.get("errorMessage") or event.get("finalError") or "retrying transient error"
+            return self._complete("error", f"retry attempt {event.get('attempt') or '?'}: {message}")
+        if event_type in {"auto_retry_end", "queue_update"}:
+            return ""
+        if event_type == "compaction_start":
+            return self._complete("action", "context compaction started")
+        if event_type == "compaction_end":
+            if event.get("errorMessage") or event.get("finalError"):
+                return self._complete("error", event.get("errorMessage") or event.get("finalError"))
+            return self._complete("observation", "context compaction completed")
+
+        if event_type == "agent_end":
+            messages = event.get("messages") if isinstance(event.get("messages"), list) else []
+            last_message = messages[-1] if messages and isinstance(messages[-1], dict) else {}
+            stop_reason = str(last_message.get("stopReason") or "")
+            will_retry = bool(event.get("willRetry"))
+            if will_retry:
+                return ""
+            status = "error" if stop_reason in self._ERROR_STOPS else "success"
+            details = f"status={status}"
+            if stop_reason:
+                details += f" stop_reason={stop_reason}"
+            return self._session_end(details)
+
+        if event_type in {"content_block_start", "content_block_delta", "content_block_stop"}:
+            return self._content_block_event(event)
+        if event_type == "message_start":
+            self._scope += 1
+            return ""
+        if event_type == "message_stop":
+            return self.renderer.finish()
+
+        if event_type == "assistant":
+            self._scope += 1
+            return self._message_parts(event.get("message"))
+        if event_type == "message" and isinstance(event.get("sessionId"), str):
+            if event.get("role") == "assistant":
+                self._scope += 1
+                return self._message_parts(event)
+            return ""
+        if event_type == "tool_result":
+            return self._complete("observation", self._value_text(event.get("content")))
+        if event_type == "user" and not isinstance(event.get("sessionId"), str):
+            return self._tool_result_parts(event.get("message"))
+        if event_type == "function_call":
+            self._remember_action_id(event.get("id") or event.get("call_id"))
+            return self._complete(
+                "action",
+                f"{event.get('name') or 'tool'} {self._value_text(event.get('arguments'))}",
+            )
+        if event_type == "function_call_result":
+            return self._complete("observation", self._value_text(event.get("output") or event.get("result")))
+
+        if event_type in {"item.started", "item.completed"}:
+            item = event.get("item") if isinstance(event.get("item"), dict) else {}
+            item_type = str(item.get("type") or "")
+            item_id = str(item.get("id") or item.get("call_id") or _stable_subagent_digest(item))
+            if item_type in {"reasoning", "analysis"} and event_type == "item.completed":
+                return self._complete("think", self._value_text(item.get("text") or item.get("summary") or ""))
+            if item_type == "agent_message" and event_type == "item.completed":
+                return self._complete("final", item.get("text") or "")
+            if item_type in {"command_execution", "mcp_tool_call", "collab_tool_call"}:
+                command = item.get("command") or item.get("tool") or item.get("name") or item_type
+                if event_type == "item.started":
+                    self._remember_action_id(item_id)
+                    return self._complete("action", self._value_text(command))
+                if item_id not in self._recent_action_id_set:
+                    self._remember_action_id(item_id)
+                    rendered += self._complete("action", self._value_text(command))
+                output = item.get("aggregated_output") or item.get("output") or item.get("result")
+                rendered += self._complete("observation", self._value_text(output))
+                if str(item.get("status") or "") in self._ERROR_STOPS:
+                    rendered += self._complete("error", self._value_text(item.get("error") or "command failed"))
+                return rendered
+            if event_type == "item.completed" and item_type in {"file_change", "web_search"}:
+                return self._complete("action", self._value_text(item))
+            return ""
+
+        if event_type == "turn.completed":
+            return self._session_end("status=success")
+        if event_type == "turn.failed":
+            error_text = self._value_text(event.get("error") or event.get("message") or "turn failed")
+            return self._complete("error", error_text) + self._session_end("status=error")
+        if event_type == "result":
+            subtype = str(event.get("subtype") or event.get("result_subtype") or "")
+            normalized_subtype = subtype.lower()
+            is_error = bool(event.get("is_error")) or normalized_subtype in self._ERROR_STOPS or normalized_subtype.startswith(("error", "failed", "cancel"))
+            result_text = self._value_text(event.get("result") or event.get("message"))
+            if result_text:
+                rendered += self._complete(
+                    "error" if is_error else "final",
+                    result_text,
+                    deduplicate=not is_error,
+                )
+            rendered += self._session_end(
+                f"status={'error' if is_error else 'success'}" + (f" stop_reason={event.get('stop_reason')}" if event.get("stop_reason") else ""),
+            )
+            return rendered
+        if event_type == "error" or event.get("error"):
+            return self._complete("error", self._value_text(event.get("error") or event.get("message") or "runtime error"))
+        return ""
+
+    def finish(self):
+        rendered = self.renderer.finish()
+        if not self._terminal_emitted:
+            rendered += self._session_end("status=stream_closed")
+        self._stream_buffers.clear()
+        self._snapshots.clear()
+        return rendered
+
+
 class ProgressTracker:
     """Tracks progress state from stream-json events."""
 
@@ -254,6 +766,7 @@ class ProgressTracker:
         self._in_tool_use = False
         self._current_tool_input_parts = []
         self._current_tool_id = ""
+        self._pi_retrying_terminal_error = False
 
     def process_event(self, event):
         """Process a single stream-json event and update state.
@@ -294,6 +807,15 @@ class ProgressTracker:
             self.event_format = "pi-json"
             if event_type in {"agent_start", "turn_start", "message_start", "message_update", "tool_execution_start", "tool_execution_update", "compaction_start", "auto_retry_start"}:
                 self.is_active = True
+            if event_type == "agent_start" and self._pi_retrying_terminal_error:
+                self.result_subtype = ""
+                self.stop_reason = ""
+                self.last_result_is_error = False
+                self.api_error_status = None
+                self.api_error_code = ""
+                self.fatal_error_code = ""
+                self.terminal_result_text = ""
+                self.terminal_success_at = ""
             if event_type == "turn_start":
                 self.message_count += 1
             elif event_type in {"message_update", "message_end"}:
@@ -355,13 +877,42 @@ class ProgressTracker:
             elif event_type == "agent_end":
                 self.current_tool = None
                 self.is_active = False
-                if (
+                messages = event.get("messages") if isinstance(event.get("messages"), list) else []
+                final_message = messages[-1] if messages and isinstance(messages[-1], dict) else {}
+                final_stop_reason = str(final_message.get("stopReason") or self.stop_reason or "")
+                will_retry = bool(event.get("willRetry"))
+                if will_retry:
+                    self._pi_retrying_terminal_error = bool(
+                        self.result_subtype == "error"
+                        or self.last_result_is_error
+                        or self.fatal_error_code
+                    )
+                elif final_stop_reason not in {"error", "aborted", "failed", "cancelled", "canceled"} and (
                     self.result_subtype != "error"
-                    and not self.last_result_is_error
-                    and not self.fatal_error_code
+                    or self._pi_retrying_terminal_error
                 ):
                     self.result_subtype = "success"
+                    self.stop_reason = final_stop_reason or self.stop_reason
+                    self.last_result_is_error = False
+                    if self._pi_retrying_terminal_error:
+                        self.api_error_status = None
+                        self.api_error_code = ""
+                        self.fatal_error_code = ""
+                    final_text = ""
+                    content = final_message.get("content") if isinstance(final_message, dict) else None
+                    if isinstance(content, str):
+                        final_text = content
+                    elif isinstance(content, list):
+                        final_text = "\n".join(
+                            str(item.get("text") or "")
+                            for item in content
+                            if isinstance(item, dict) and item.get("type") == "text"
+                        )
+                    if final_text.strip():
+                        self.terminal_result_text = final_text[:1000]
+                        self.last_text_snippet = final_text.strip()[-120:]
                     self.terminal_success_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+                    self._pi_retrying_terminal_error = False
             return
 
         # ── Codex exec --json JSONL format ──────────────────────────
@@ -412,12 +963,23 @@ class ProgressTracker:
 
             elif event_type == "turn.completed":
                 self.current_tool = None
+                self.is_active = False
+                if not self.fatal_error_code:
+                    self.result_subtype = "success"
+                    self.stop_reason = "completed"
+                    self.last_result_is_error = False
+                    self.terminal_success_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
             elif event_type == "turn.failed":
                 error = event.get("error") or event.get("message") or "Codex turn failed"
                 self.errors.append(str(error))
                 self._detect_terminal_error(str(error))
+                self.result_subtype = "error"
+                self.stop_reason = "failed"
+                self.last_result_is_error = True
+                self.terminal_result_text = str(error)[:1000]
                 self.current_tool = None
+                self.is_active = False
 
             elif event_type == "error":
                 error = event.get("error") or event.get("message") or "Unknown error"