htmlgraph 0.26.25__py3-none-any.whl → 0.27.1__py3-none-any.whl

htmlgraph/api/main.py

@@ -20,7 +20,7 @@ import logging
 import random
 import sqlite3
 import time
-from datetime import datetime
+from datetime import datetime, timedelta
 from pathlib import Path
 from typing import Any
 
@@ -204,7 +204,37 @@ def get_app(db_path: str) -> FastAPI:
             return "0"
         return f"{value:,}"
 
+    def format_duration(seconds: float | int | None) -> str:
+        """Format duration in seconds to human-readable string."""
+        if seconds is None:
+            return "0.00s"
+        return f"{float(seconds):.2f}s"
+
+    def format_bytes(bytes_size: int | float | None) -> str:
+        """Format bytes to MB with 2 decimal places."""
+        if bytes_size is None:
+            return "0.00MB"
+        return f"{int(bytes_size) / (1024 * 1024):.2f}MB"
+
+    def truncate_text(text: str | None, length: int = 50) -> str:
+        """Truncate text to specified length with ellipsis."""
+        if text is None:
+            return ""
+        return text[:length] + "..." if len(text) > length else text
+
+    def format_timestamp(ts: Any) -> str:
+        """Format timestamp to readable string."""
+        if ts is None:
+            return ""
+        if hasattr(ts, "strftime"):
+            return str(ts.strftime("%Y-%m-%d %H:%M:%S"))
+        return str(ts)
+
     templates.env.filters["format_number"] = format_number
+    templates.env.filters["format_duration"] = format_duration
+    templates.env.filters["format_bytes"] = format_bytes
+    templates.env.filters["truncate"] = truncate_text
+    templates.env.filters["format_timestamp"] = format_timestamp
 
     # Setup static files
     static_dir = Path(__file__).parent / "static"
@@ -287,8 +317,8 @@ def get_app(db_path: str) -> FastAPI:
 
                 # Execute query with timing
                 exec_start = time.time()
-                cursor = await db.execute(query)
-                rows = await cursor.fetchall()
+                async with db.execute(query) as cursor:
+                    rows = await cursor.fetchall()
                 exec_time_ms = (time.time() - exec_start) * 1000
 
                 agents = []
@@ -434,8 +464,8 @@ def get_app(db_path: str) -> FastAPI:
 
                 # Execute query with timing
                 exec_start = time.time()
-                cursor = await db.execute(query, params)
-                rows = await cursor.fetchall()
+                async with db.execute(query, params) as cursor:
+                    rows = await cursor.fetchall()
                 exec_time_ms = (time.time() - exec_start) * 1000
 
                 # Build result models
@@ -485,15 +515,15 @@ def get_app(db_path: str) -> FastAPI:
                     (SELECT COUNT(DISTINCT agent_id) FROM agent_events) as total_agents,
                     (SELECT COUNT(*) FROM sessions) as total_sessions
             """
-            cursor = await db.execute(stats_query)
-            row = await cursor.fetchone()
+            async with db.execute(stats_query) as cursor:
+                row = await cursor.fetchone()
 
             # Query distinct agent IDs for the agent set
             agents_query = (
                 "SELECT DISTINCT agent_id FROM agent_events WHERE agent_id IS NOT NULL"
             )
-            agents_cursor = await db.execute(agents_query)
-            agents_rows = await agents_cursor.fetchall()
+            async with db.execute(agents_query) as agents_cursor:
+                agents_rows = await agents_cursor.fetchall()
             agents = [row[0] for row in agents_rows]
 
             if row is None:
@@ -616,8 +646,8 @@ def get_app(db_path: str) -> FastAPI:
             parent_query += " ORDER BY timestamp DESC LIMIT ?"
             parent_params.append(limit)
 
-            cursor = await db.execute(parent_query, parent_params)
-            parent_rows = await cursor.fetchall()
+            async with db.execute(parent_query, parent_params) as cursor:
+                parent_rows = await cursor.fetchall()
 
             traces: list[dict[str, Any]] = []
 
@@ -655,8 +685,8 @@ def get_app(db_path: str) -> FastAPI:
                     WHERE parent_event_id = ?
                     ORDER BY timestamp ASC
                 """
-                child_cursor = await db.execute(child_query, (parent_event_id,))
-                child_rows = await child_cursor.fetchall()
+                async with db.execute(child_query, (parent_event_id,)) as child_cursor:
+                    child_rows = await child_cursor.fetchall()
 
                 child_events = []
                 for child_row in child_rows:
@@ -806,8 +836,8 @@ def get_app(db_path: str) -> FastAPI:
             params.append(limit)
 
             exec_start = time.time()
-            cursor = await db.execute(query, params)
-            rows = await cursor.fetchall()
+            async with db.execute(query, params) as cursor:
+                rows = await cursor.fetchall()
 
             for row in rows:
                 events.append(
@@ -851,8 +881,8 @@ def get_app(db_path: str) -> FastAPI:
                     spike_query += " ORDER BY created_at DESC LIMIT ?"
                     spike_params.append(limit)
 
-                    spike_cursor = await db.execute(spike_query, spike_params)
-                    spike_rows = await spike_cursor.fetchall()
+                    async with db.execute(spike_query, spike_params) as spike_cursor:
+                        spike_rows = await spike_cursor.fetchall()
 
                     for row in spike_rows:
                         events.append(
@@ -902,8 +932,8 @@ def get_app(db_path: str) -> FastAPI:
                     collab_query += " ORDER BY timestamp DESC LIMIT ?"
                     collab_params.append(limit)
 
-                    collab_cursor = await db.execute(collab_query, collab_params)
-                    collab_rows = await collab_cursor.fetchall()
+                    async with db.execute(collab_query, collab_params) as collab_cursor:
+                        collab_rows = await collab_cursor.fetchall()
 
                     for row in collab_rows:
                         events.append(
@@ -1016,8 +1046,8 @@ def get_app(db_path: str) -> FastAPI:
                 LIMIT ?
             """
 
-            cursor = await db.execute(user_query_query, [limit])
-            user_query_rows = await cursor.fetchall()
+            async with db.execute(user_query_query, [limit]) as cursor:
+                user_query_rows = await cursor.fetchall()
 
             conversation_turns: list[dict[str, Any]] = []
 
@@ -1061,8 +1091,8 @@ def get_app(db_path: str) -> FastAPI:
                     if depth >= max_depth:
                         return [], 0.0, 0, 0
 
-                    cursor = await db.execute(children_query, [parent_id])
-                    rows = await cursor.fetchall()
+                    async with db.execute(children_query, [parent_id]) as cursor:
+                        rows = await cursor.fetchall()
 
                     children_list: list[dict[str, Any]] = []
                     total_dur = 0.0
@@ -1316,16 +1346,16 @@ def get_app(db_path: str) -> FastAPI:
             exec_start = time.time()
 
             # Build query with optional status filter
-            # Note: Database uses agent_assigned but started_at/ended_at (partial migration)
+            # Note: Database uses agent_assigned, created_at, and completed_at
             query = """
                 SELECT
                     session_id,
                     agent_assigned,
                     continued_from,
-                    started_at,
+                    created_at,
                     status,
                     start_commit,
-                    ended_at
+                    completed_at
                 FROM sessions
                 WHERE 1=1
             """
@@ -1335,11 +1365,11 @@ def get_app(db_path: str) -> FastAPI:
                 query += " AND status = ?"
                 params.append(status)
 
-            query += " ORDER BY started_at DESC LIMIT ? OFFSET ?"
+            query += " ORDER BY created_at DESC LIMIT ? OFFSET ?"
             params.extend([limit, offset])
 
-            cursor = await db.execute(query, params)
-            rows = await cursor.fetchall()
+            async with db.execute(query, params) as cursor:
+                rows = await cursor.fetchall()
 
             # Get total count for pagination
             count_query = "SELECT COUNT(*) FROM sessions WHERE 1=1"
@@ -1348,8 +1378,8 @@ def get_app(db_path: str) -> FastAPI:
                 count_query += " AND status = ?"
                 count_params.append(status)
 
-            count_cursor = await db.execute(count_query, count_params)
-            count_row = await count_cursor.fetchone()
+            async with db.execute(count_query, count_params) as count_cursor:
+                count_row = await count_cursor.fetchone()
             total = int(count_row[0]) if count_row else 0
 
             # Build session objects
@@ -1360,11 +1390,11 @@ def get_app(db_path: str) -> FastAPI:
                     {
                         "session_id": row[0],
                         "agent": row[1],  # agent_assigned -> agent for API compat
-                        "continued_from": row[2],  # parent_session_id
-                        "started_at": row[3],  # created_at -> started_at for API compat
+                        "continued_from": row[2],
+                        "created_at": row[3],  # created_at timestamp
                         "status": row[4] or "unknown",
                         "start_commit": row[5],
-                        "ended_at": row[6],  # completed_at -> ended_at for API compat
+                        "completed_at": row[6],  # completed_at timestamp
                     }
                 )
 
@@ -1416,8 +1446,8 @@ def get_app(db_path: str) -> FastAPI:
                 LIMIT 50
             """
 
-            cursor = await db.execute(query)
-            rows = list(await cursor.fetchall())
+            async with db.execute(query) as cursor:
+                rows = list(await cursor.fetchall())
             logger.debug(f"orchestration_view: Query executed, got {len(rows)} rows")
 
             delegations = []
@@ -2193,23 +2223,97 @@ def get_app(db_path: str) -> FastAPI:
     # ========== WEBSOCKET FOR REAL-TIME UPDATES ==========
 
     @app.websocket("/ws/events")
-    async def websocket_events(websocket: WebSocket) -> None:
+    async def websocket_events(websocket: WebSocket, since: str | None = None) -> None:
         """WebSocket endpoint for real-time event streaming.
 
         OPTIMIZATION: Uses timestamp-based filtering to minimize data transfers.
         The timestamp > ? filter with DESC index makes queries O(log n) instead of O(n).
 
-        IMPORTANT: Initializes last_timestamp to current time to only stream NEW events.
-        Historical events are already counted in /api/initial-stats, so streaming them
-        again would cause double-counting in the header stats.
+        FIX 3: Now supports loading historical events via 'since' parameter.
+        - If 'since' provided: Load events from that timestamp onwards
+        - If 'since' not provided: Load events from last 1 hour (default)
+        - After historical load: Continue streaming real-time events
 
         LIVE EVENTS: Also polls live_events table for real-time spawner activity
         streaming. These events are marked as broadcast after sending and cleaned up.
+
+        Args:
+            since: Optional ISO timestamp to start streaming from (e.g., "2025-01-16T10:00:00")
+                   If not provided, defaults to 1 hour ago
         """
         await websocket.accept()
-        # Initialize to current time - only stream events created AFTER connection
-        # This prevents double-counting: initial-stats already includes historical events
-        last_timestamp: str = datetime.now().isoformat()
+
+        # FIX 3: Determine starting timestamp
+        if since:
+            try:
+                # Validate timestamp format
+                datetime.fromisoformat(since.replace("Z", "+00:00"))
+                last_timestamp = since
+            except (ValueError, AttributeError):
+                # Invalid timestamp - default to 1 hour ago
+                last_timestamp = (datetime.now() - timedelta(hours=1)).isoformat()
+        else:
+            # Default: Load events from last 1 hour
+            last_timestamp = (datetime.now() - timedelta(hours=1)).isoformat()
+
+        # FIX 3: Load historical events first (before real-time streaming)
+        db = await get_db()
+        try:
+            historical_query = """
+                SELECT event_id, agent_id, event_type, timestamp, tool_name,
+                       input_summary, output_summary, session_id, status, model,
+                       parent_event_id, execution_duration_seconds, context,
+                       cost_tokens
+                FROM agent_events
+                WHERE timestamp >= ? AND timestamp < datetime('now')
+                ORDER BY timestamp ASC
+                LIMIT 1000
+            """
+            cursor = await db.execute(historical_query, [last_timestamp])
+            historical_rows = await cursor.fetchall()
+
+            # Send historical events first
+            if historical_rows:
+                historical_rows_list = list(historical_rows)
+                for row in historical_rows_list:
+                    row_list = list(row)
+                    # Parse context JSON if present
+                    context_data = {}
+                    if row_list[12]:  # context column
+                        try:
+                            context_data = json.loads(row_list[12])
+                        except (json.JSONDecodeError, TypeError):
+                            pass
+
+                    event_data = {
+                        "type": "event",
+                        "event_id": row_list[0],
+                        "agent_id": row_list[1] or "unknown",
+                        "event_type": row_list[2],
+                        "timestamp": row_list[3],
+                        "tool_name": row_list[4],
+                        "input_summary": row_list[5],
+                        "output_summary": row_list[6],
+                        "session_id": row_list[7],
+                        "status": row_list[8],
+                        "model": row_list[9],
+                        "parent_event_id": row_list[10],
+                        "execution_duration_seconds": row_list[11] or 0.0,
+                        "cost_tokens": row_list[13] or 0,
+                        "context": context_data,
+                    }
+                    await websocket.send_json(event_data)
+
+                # Update last_timestamp to last historical event
+                last_timestamp = historical_rows_list[-1][3]
+
+        except Exception as e:
+            logger.error(f"Error loading historical events: {e}")
+        finally:
+            await db.close()
+
+        # Update to current time for real-time streaming
+        last_timestamp = datetime.now().isoformat()
         poll_interval = 0.5  # OPTIMIZATION: Adaptive polling (reduced from 1s)
         last_live_event_id = 0  # Track last broadcast live event ID
 
@@ -2238,35 +2342,35 @@ def get_app(db_path: str) -> FastAPI:
 
                     if rows:
                         has_activity = True
-                        rows_list = [list(row) for row in rows]
+                        rows_list: list[list[Any]] = [list(row) for row in rows]
                         # Update last timestamp (last row since ORDER BY ts ASC)
                         last_timestamp = rows_list[-1][3]
 
                         # Send events in order (no need to reverse with ASC)
-                        for row in rows_list:
+                        for event_row in rows_list:
                             # Parse context JSON if present
                             context_data = {}
-                            if row[12]:  # context column
+                            if event_row[12]:  # context column
                                 try:
-                                    context_data = json.loads(row[12])
+                                    context_data = json.loads(event_row[12])
                                 except (json.JSONDecodeError, TypeError):
                                     pass
 
                             event_data = {
                                 "type": "event",
-                                "event_id": row[0],
-                                "agent_id": row[1] or "unknown",
-                                "event_type": row[2],
-                                "timestamp": row[3],
-                                "tool_name": row[4],
-                                "input_summary": row[5],
-                                "output_summary": row[6],
-                                "session_id": row[7],
-                                "status": row[8],
-                                "model": row[9],
-                                "parent_event_id": row[10],
-                                "execution_duration_seconds": row[11] or 0.0,
-                                "cost_tokens": row[13] or 0,
+                                "event_id": event_row[0],
+                                "agent_id": event_row[1] or "unknown",
+                                "event_type": event_row[2],
+                                "timestamp": event_row[3],
+                                "tool_name": event_row[4],
+                                "input_summary": event_row[5],
+                                "output_summary": event_row[6],
+                                "session_id": event_row[7],
+                                "status": event_row[8],
+                                "model": event_row[9],
+                                "parent_event_id": event_row[10],
+                                "execution_duration_seconds": event_row[11] or 0.0,
+                                "cost_tokens": event_row[13] or 0,
                                 "context": context_data,
                             }
                             await websocket.send_json(event_data)
@@ -2383,8 +2487,9 @@ def get_app(db_path: str) -> FastAPI:
 def create_app(db_path: str | None = None) -> FastAPI:
     """Create FastAPI app with default database path."""
     if db_path is None:
-        # Use default database location - htmlgraph.db is the unified database
-        db_path = str(Path.home() / ".htmlgraph" / "htmlgraph.db")
+        # Use index.sqlite - this is where AnalyticsIndex writes events
+        # Note: index.sqlite is the rebuildable analytics cache, not htmlgraph.db
+        db_path = str(Path.home() / ".htmlgraph" / "index.sqlite")
 
     return get_app(db_path)