prizmkit 1.1.99 → 1.1.101

package/bundled/dev-pipeline/lib/heartbeat.sh

@@ -1,469 +0,0 @@
-#!/usr/bin/env bash
-# ============================================================
-# dev-pipeline/lib/heartbeat.sh - Shared heartbeat monitoring
-#
-# Provides start_heartbeat / stop_heartbeat functions that read
-# structured progress from progress.json (written by
-# parse-stream-progress.py) and fall back to tail-based monitoring.
-#
-# When stale_kill_threshold is set (>0), the heartbeat monitor will
-# automatically kill the AI CLI process if it shows no progress for
-# the specified duration. This prevents sessions from hanging forever
-# when the AI CLI process doesn't exit after completing its work.
-#
-# Usage:
-#   source "$SCRIPT_DIR/lib/heartbeat.sh"
-#   start_heartbeat "$cli_pid" "$session_log" "$progress_json" "$interval" ["$stale_kill_threshold"]
-#   # ... wait for CLI to finish ...
-#   stop_heartbeat "$_HEARTBEAT_PID"
-#
-# Requires: colors (GREEN, YELLOW, BLUE, NC) and log functions
-# to be defined before sourcing.
-# ============================================================
-
-# Start a heartbeat monitor in the background.
-# Sets _HEARTBEAT_PID to the background process PID.
-#
-# Arguments:
-#   $1 - cli_pid              PID of the AI CLI process to monitor
-#   $2 - session_log          Path to session.log
-#   $3 - progress_json        Path to progress.json (may not exist if stream-json disabled)
-#   $4 - interval             Heartbeat interval in seconds
-#   $5 - stale_kill_threshold (optional) Seconds of no progress before auto-killing the process.
-#                              0 = disabled (default). Recommended: 900 (15 minutes).
-start_heartbeat() {
-    local cli_pid="$1"
-    local session_log="$2"
-    local progress_json="$3"
-    local heartbeat_interval="$4"
-    local stale_kill_threshold="${5:-0}"
-
-    (
-        local elapsed=0
-        local prev_size=0
-        local prev_max_size=0
-        local prev_child_activity_signature=""
-        local prev_progress_signature=""
-        local stale_seconds=0
-        local stale_reason="stale_session"
-        while kill -0 "$cli_pid" 2>/dev/null; do
-            sleep "$heartbeat_interval"
-            elapsed=$((elapsed + heartbeat_interval))
-            kill -0 "$cli_pid" 2>/dev/null || break
-
-            # Get log file size. If session_log was truncated (e.g. by subagent
-            # stdout redirection severing the file descriptor), detect and flag it.
-            local cur_size=0
-            if [[ -f "$session_log" ]]; then
-                cur_size=$(wc -c < "$session_log" 2>/dev/null || echo 0)
-                cur_size=$(echo "$cur_size" | tr -d ' ')
-            fi
-
-            # Detect log truncation: size dropped to 0 or near-zero after previously
-            # accumulating significant content (>100KB). This indicates the file was
-            # unlinked/truncated by something outside our control while the AI CLI
-            # process is still running.
-            local log_truncated=false
-            if [[ $cur_size -lt 1024 && $prev_max_size -gt 102400 ]]; then
-                log_truncated=true
-            fi
-            # Track the maximum size we've seen for truncation detection
-            local prev_max_size=$prev_max_size
-            if [[ $cur_size -gt $prev_max_size ]]; then
-                prev_max_size=$cur_size
-            fi
-
-            local growth=$((cur_size - prev_size))
-            prev_size=$cur_size
-
-            local child_activity_signature=""
-            local child_total_bytes=0
-            local child_session_count=0
-            local progress_signature=""
-            if [[ -f "$progress_json" ]]; then
-                local progress_activity_data
-                progress_activity_data=$(python3 - "$progress_json" <<'PY' 2>/dev/null || true
-import json
-import sys
-
-try:
-    with open(sys.argv[1], "r", encoding="utf-8") as fh:
-        progress = json.load(fh)
-except Exception:
-    sys.exit(0)
-
-child_signature = str(progress.get("child_activity_signature") or "")
-total_bytes = int(progress.get("child_total_bytes") or 0)
-session_count = len(progress.get("child_session_files") or [])
-progress_signature = json.dumps(
-    [
-        progress.get("current_phase") or "",
-        progress.get("current_tool") or "",
-        int(progress.get("message_count") or 0),
-        int(progress.get("total_tool_calls") or 0),
-        int(progress.get("active_subagent_count") or 0),
-        int(progress.get("subagent_spawn_count") or 0),
-        child_signature,
-        total_bytes,
-        session_count,
-        progress.get("fatal_error_code") or "",
-        progress.get("terminal_result_text") or "",
-    ],
-    separators=(",", ":"),
-)
-sep = "\x1f"
-print(sep.join([child_signature, str(total_bytes), str(session_count), progress_signature]))
-PY
-                )
-                if [[ -n "$progress_activity_data" ]]; then
-                    IFS=$'\037' read -r child_activity_signature child_total_bytes child_session_count progress_signature <<< "$progress_activity_data"
-                fi
-            fi
-
-            local child_growth=0
-            if [[ -n "$child_activity_signature" && "$child_activity_signature" != "$prev_child_activity_signature" ]]; then
-                child_growth=1
-            fi
-            prev_child_activity_signature="$child_activity_signature"
-
-            local progress_advanced=0
-            if [[ -n "$progress_signature" && "$progress_signature" != "$prev_progress_signature" ]]; then
-                progress_advanced=1
-            fi
-            prev_progress_signature="$progress_signature"
-
-            local effective_stale_kill_threshold="$stale_kill_threshold"
-            if [[ $stale_kill_threshold -gt 0 && -f "$progress_json" ]]; then
-                local extended_threshold
-                extended_threshold=$(python3 - "$progress_json" "$stale_kill_threshold" <<'PY' 2>/dev/null || true
-import json
-import os
-import sys
-
-progress_path = sys.argv[1]
-base_threshold = int(sys.argv[2])
-
-with open(progress_path, "r", encoding="utf-8") as fh:
-    progress = json.load(fh)
-
-spawn_count = 0
-for tool in progress.get("tool_calls", []):
-    if isinstance(tool, dict) and tool.get("name") in ("spawn_agent", "Agent", "TaskCreate"):
-        try:
-            spawn_count += int(tool.get("count", 0))
-        except (TypeError, ValueError):
-            pass
-
-# Also check the subagent_spawn_count field (set by _record_cb_agent_tool_call)
-if not spawn_count:
-    spawn_count = int(progress.get("subagent_spawn_count", 0))
-
-fmt = progress.get("event_format", "")
-
-# Codex: current_tool == "wait" means parent is blocked on spawn_agent completion
-if (
-    fmt == "codex-json"
-    and progress.get("current_tool") == "wait"
-    and spawn_count > 0
-):
-    configured = os.environ.get("CODEX_WAIT_STALE_KILL_THRESHOLD", "")
-    try:
-        wait_threshold = int(configured)
-    except ValueError:
-        wait_threshold = max(base_threshold * 4, 3600)
-    if wait_threshold > base_threshold:
-        print(wait_threshold)
-
-# CodeBuddy: Agent tool blocks synchronously; Task* tools imply bg agents.
-# Extend the stale window when sub-agents have been spawned so the heartbeat
-# doesn't kill the parent while children are still running.
-if (
-    fmt == "stream-json"
-    and spawn_count > 0
-    and progress.get("cb_session_id", "")
-):
-    configured = os.environ.get("CB_SUBAGENT_STALE_KILL_THRESHOLD", "")
-    try:
-        cb_threshold = int(configured)
-    except ValueError:
-        cb_threshold = max(base_threshold * 4, 3600)
-    if cb_threshold > base_threshold:
-        print(cb_threshold)
-PY
-                )
-                if [[ "$extended_threshold" =~ ^[0-9]+$ && "$extended_threshold" -gt "$stale_kill_threshold" ]]; then
-                    effective_stale_kill_threshold="$extended_threshold"
-                fi
-            fi
-
-            # Track progress staleness. Parent sessions can sit in a wait/polling
-            # tool while child transcripts or structured progress counters keep
-            # advancing, so both child activity and progress.json state changes
-            # count as progress.
-            #
-            # When the main session log has been truncated (e.g. by subagent stdout
-            # redirection severing the file descriptor), use structured progress and
-            # child activity as the primary progress signals — the parent process is
-            # still running and children may still be producing output.
-            if [[ "$log_truncated" == "true" ]]; then
-                if [[ $progress_advanced -gt 0 || $child_growth -gt 0 ]]; then
-                    stale_seconds=0
-                else
-                    stale_seconds=$((stale_seconds + heartbeat_interval))
-                fi
-            elif [[ $growth -le 0 && $child_growth -eq 0 && $progress_advanced -eq 0 ]]; then
-                stale_seconds=$((stale_seconds + heartbeat_interval))
-            else
-                stale_seconds=0
-            fi
-
-            local size_display
-            if $log_truncated; then
-                size_display="${cur_size}B/TRUNCATED(from ${prev_max_size}B)"
-            elif [[ $cur_size -gt 1048576 ]]; then
-                size_display="$((cur_size / 1048576))MB"
-            elif [[ $cur_size -gt 1024 ]]; then
-                size_display="$((cur_size / 1024))KB"
-            else
-                size_display="${cur_size}B"
-            fi
-            local child_display=""
-            if [[ ${child_total_bytes:-0} -gt 0 ]]; then
-                local child_size_display
-                if [[ $child_total_bytes -gt 1048576 ]]; then
-                    child_size_display="$((child_total_bytes / 1048576))MB"
-                elif [[ $child_total_bytes -gt 1024 ]]; then
-                    child_size_display="$((child_total_bytes / 1024))KB"
-                else
-                    child_size_display="${child_total_bytes}B"
-                fi
-                child_display=" | child: ${child_size_display}"
-                if [[ ${child_session_count:-0} -gt 1 ]]; then
-                    child_display="${child_display}/${child_session_count}"
-                fi
-            fi
-
-            local mins=$((elapsed / 60))
-            local secs=$((elapsed % 60))
-
-            local status_icon
-            if $log_truncated; then
-                status_icon="${RED}⚠${NC}"
-            elif [[ $growth -gt 0 || $child_growth -gt 0 || $progress_advanced -gt 0 ]]; then
-                status_icon="${GREEN}▶${NC}"
-            else
-                status_icon="${YELLOW}⏸${NC}"
-            fi
-
-            # Fatal provider/runtime errors are terminal; do not wait for the
-            # stale window when progress.json already proves the model cannot
-            # continue (for example context_too_large).
-            if [[ -f "$progress_json" ]]; then
-                local fatal_error_code=""
-                fatal_error_code=$(python3 - "$progress_json" <<'PY' 2>/dev/null || true
-import json
-import sys
-try:
-    with open(sys.argv[1], encoding="utf-8") as fh:
-        progress = json.load(fh)
-except Exception:
-    raise SystemExit(0)
-code = progress.get("fatal_error_code") or ""
-if code:
-    print(code)
-PY
-)
-                if [[ -n "$fatal_error_code" ]]; then
-                    echo -e "  ${RED}[HEARTBEAT]${NC} ${mins}m${secs}s | log: ${size_display} | ${RED}FATAL: ${fatal_error_code}${NC}"
-                    local _marker_dir
-                    _marker_dir="$(dirname "$session_log")"
-                    echo "{\"killed_at\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\", \"reason\": \"${fatal_error_code}\", \"fatal_error_code\": \"${fatal_error_code}\", \"stale_seconds\": $stale_seconds, \"threshold\": $effective_stale_kill_threshold}" > "$_marker_dir/fatal-error.json" 2>/dev/null || true
-                    echo "{\"killed_at\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\", \"reason\": \"${fatal_error_code}\", \"fatal_error_code\": \"${fatal_error_code}\", \"stale_seconds\": $stale_seconds, \"threshold\": $effective_stale_kill_threshold}" > "$_marker_dir/stale-kill.json" 2>/dev/null || true
-                    kill -TERM "$cli_pid" 2>/dev/null || true
-                    local fatal_kill_grace_seconds="${STALE_KILL_GRACE_SECONDS:-10}"
-                    if [[ $fatal_kill_grace_seconds -gt 0 ]]; then
-                        sleep "$fatal_kill_grace_seconds"
-                    fi
-                    if kill -0 "$cli_pid" 2>/dev/null; then
-                        kill -9 "$cli_pid" 2>/dev/null || true
-                    fi
-                    break
-                fi
-            fi
-
-            # Stale-kill: auto-terminate process if no progress for too long.
-            # Parent sessions can wait on spawned work; child transcript growth
-            # counts as progress above, while silent waits still use the active
-            # stale window to surface stuck agents promptly.
-            if [[ $effective_stale_kill_threshold -gt 0 && $stale_seconds -ge $effective_stale_kill_threshold ]]; then
-                local stale_mins=$((stale_seconds / 60))
-                echo -e "  ${RED}[HEARTBEAT]${NC} ${mins}m${secs}s | log: ${size_display} | ${RED}STALE-KILL: no progress for ${stale_mins}m (threshold: ${effective_stale_kill_threshold}s)${NC}"
-                echo -e "  ${RED}[HEARTBEAT]${NC} Killing AI CLI process $cli_pid (stale session)..."
-                # Write the marker before killing. Some CLIs exit quickly, and the
-                # parent runner may stop this heartbeat process immediately after
-                # wait(1) returns.
-                local _marker_dir
-                _marker_dir="$(dirname "$session_log")"
-                echo "{\"killed_at\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\", \"reason\": \"${stale_reason}\", \"stale_seconds\": $stale_seconds, \"threshold\": $effective_stale_kill_threshold}" > "$_marker_dir/stale-kill.json" 2>/dev/null || true
-                kill -TERM "$cli_pid" 2>/dev/null || true
-                # Give process 10s to exit gracefully, then force kill
-                local stale_kill_grace_seconds="${STALE_KILL_GRACE_SECONDS:-10}"
-                if [[ $stale_kill_grace_seconds -gt 0 ]]; then
-                    sleep "$stale_kill_grace_seconds"
-                fi
-                if kill -0 "$cli_pid" 2>/dev/null; then
-                    echo -e "  ${RED}[HEARTBEAT]${NC} Process still alive after SIGTERM, sending SIGKILL..."
-                    kill -9 "$cli_pid" 2>/dev/null || true
-                fi
-                break
-            fi
-
-            # Build staleness hint for display
-            local stale_hint=""
-            if [[ $effective_stale_kill_threshold -gt 0 && $stale_seconds -gt 0 ]]; then
-                local stale_mins=$((stale_seconds / 60))
-                local threshold_mins=$((effective_stale_kill_threshold / 60))
-                stale_hint=" | stale: ${stale_mins}m/${threshold_mins}m"
-            fi
-
-            # Try structured progress from progress.json
-            if [[ -f "$progress_json" ]]; then
-                local phase tool msgs tools_total
-                phase=$(python3 -c "
-import json, sys
-try:
-    with open(sys.argv[1]) as f:
-        d = json.load(f)
-    parts = []
-    if d.get('current_phase'):
-        parts.append('phase: ' + d['current_phase'])
-    if d.get('current_tool'):
-        parts.append('tool: ' + d['current_tool'])
-    parts.append('msgs: ' + str(d.get('message_count', 0)))
-    parts.append(str(d.get('total_tool_calls', 0)) + ' tool calls')
-    print(' | '.join(parts))
-except Exception:
-    sys.exit(1)
-" "$progress_json" 2>/dev/null) && {
-                    echo -e "  ${status_icon} ${BLUE}[HEARTBEAT]${NC} ${mins}m${secs}s | log: ${size_display}${child_display} | ${phase}${stale_hint}"
-                    continue
-                }
-            fi
-
-            # Fallback: tail-based activity detection
-            local last_activity=""
-            if [[ -f "$session_log" ]]; then
-                last_activity=$(tail -20 "$session_log" 2>/dev/null | grep -v '^$' | tail -1 | cut -c1-80 || echo "")
-            fi
-
-            echo -e "  ${status_icon} ${BLUE}[HEARTBEAT]${NC} ${mins}m${secs}s elapsed | log: ${size_display}${child_display} (+${growth}B) | ${last_activity}${stale_hint}"
-        done
-    ) &
-    _HEARTBEAT_PID=$!
-}
-
-# Stop a heartbeat monitor process.
-#
-# Arguments:
-#   $1 - heartbeat_pid   PID returned by start_heartbeat
-stop_heartbeat() {
-    local heartbeat_pid="$1"
-    if [[ -n "$heartbeat_pid" ]]; then
-        kill "$heartbeat_pid" 2>/dev/null || true
-        wait "$heartbeat_pid" 2>/dev/null || true
-    fi
-}
-
-# Start the stream-json progress parser as a background process.
-# Sets _PARSER_PID to the background process PID.
-# No-op if USE_STREAM_JSON is not "true".
-#
-# Arguments:
-#   $1 - session_log     Path to session.log
-#   $2 - progress_json   Path to write progress.json
-#   $3 - scripts_dir     Path to scripts/ directory
-start_progress_parser() {
-    local session_log="$1"
-    local progress_json="$2"
-    local scripts_dir="$3"
-
-    _PARSER_PID=""
-
-    if [[ "${USE_STREAM_JSON:-}" != "true" ]]; then
-        return 0
-    fi
-
-    local parser_script="$scripts_dir/parse-stream-progress.py"
-    if [[ ! -f "$parser_script" ]]; then
-        return 0
-    fi
-
-    python3 "$parser_script" \
-        --session-log "$session_log" \
-        --progress-file "$progress_json" &
-    _PARSER_PID=$!
-}
-
-# Stop the progress parser process.
-#
-# Arguments:
-#   $1 - parser_pid   PID returned by start_progress_parser
-stop_progress_parser() {
-    local parser_pid="$1"
-    if [[ -n "$parser_pid" ]]; then
-        kill "$parser_pid" 2>/dev/null || true
-        wait "$parser_pid" 2>/dev/null || true
-    fi
-}
-
-# Detect whether the AI CLI supports structured JSON progress.
-# Sets USE_STREAM_JSON to "true" or "false".
-#
-# Arguments:
-#   $1 - cli_cmd   The AI CLI command
-detect_stream_json_support() {
-    local cli_cmd="$1"
-    USE_STREAM_JSON="false"
-    STREAM_JSON_FORMAT=""
-
-    local session_platform=""
-    session_platform="$(_prizm_known_platform_from_cli "$cli_cmd" 2>/dev/null || true)"
-    if [[ -z "$session_platform" ]]; then
-        case "$(_prizm_normalize_platform "${PRIZMKIT_PLATFORM:-${PLATFORM:-}}")" in
-            codex|claude|codebuddy)
-                session_platform="$(_prizm_normalize_platform "${PRIZMKIT_PLATFORM:-${PLATFORM:-}}")"
-                ;;
-        esac
-    fi
-
-    # CodeBuddy (cbc) always supports stream-json
-    if [[ "$session_platform" == "codebuddy" || "$cli_cmd" == "cbc" ]]; then
-        USE_STREAM_JSON="true"
-        STREAM_JSON_FORMAT="stream-json"
-        export USE_STREAM_JSON STREAM_JSON_FORMAT
-        return 0
-    fi
-
-    # Codex uses `codex exec --json` JSONL, not `--output-format stream-json`.
-    if [[ "$session_platform" == "codex" ]]; then
-        local codex_help
-        codex_help=$("$cli_cmd" exec --help 2>&1) || true
-        if echo "$codex_help" | grep -q -- "--json"; then
-            USE_STREAM_JSON="true"
-            STREAM_JSON_FORMAT="codex-json"
-        fi
-        export USE_STREAM_JSON STREAM_JSON_FORMAT
-        return 0
-    fi
-
-    # For other CLIs, try to detect support via --help output
-    # Use explicit file descriptor to avoid issues in background processes
-    local help_output
-    help_output=$("$cli_cmd" --help 2>&1) || true
-
-    if echo "$help_output" | grep -q "stream-json"; then
-        USE_STREAM_JSON="true"
-        STREAM_JSON_FORMAT="stream-json"
-    fi
-    export USE_STREAM_JSON STREAM_JSON_FORMAT
-}