code-data-ark 2.0.11__tar.gz → 2.0.12__tar.gz

{code_data_ark-2.0.11 → code_data_ark-2.0.12}/.flake8

@@ -17,3 +17,5 @@ extend-ignore =
 per-file-ignores =
     # Conditional import at bottom of watcher.py main guard
     cda/pipeline/watcher.py:E402
+    # HTML/SVG template strings are inherently long
+    cda/ui/templates.py:E501

{code_data_ark-2.0.11 → code_data_ark-2.0.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-data-ark
-Version: 2.0.11
+Version: 2.0.12
 Summary: Code Data Ark — local observability and intelligence platform for VS Code + Copilot Chat sessions
 Project-URL: Homepage, https://github.com/goCosmix/cda
 Project-URL: Repository, https://github.com/goCosmix/cda.git

{code_data_ark-2.0.11 → code_data_ark-2.0.12}/cda/__init__.py

@@ -1,3 +1,3 @@
 """Code Data Ark — local observability and intelligence platform for VS Code + Copilot Chat sessions."""
 
-__version__ = "2.0.11"
+__version__ = "2.0.12"

code_data_ark-2.0.12/cda/ui/actions.py

@@ -0,0 +1,64 @@
+import subprocess
+import sys
+import threading
+from datetime import datetime
+from typing import Any, Dict
+
+
+ACTION_STATE: Dict[str, Any] = {}
+ACTION_LOCK = threading.Lock()
+
+
+def run_action_background(action_id, action_name):
+    """Execute pipeline action in background thread."""
+    with ACTION_LOCK:
+        ACTION_STATE[action_id] = {
+            "status": "running",
+            "action": action_name,
+            "started_at": datetime.now().isoformat(),
+            "output": ""
+        }
+
+    try:
+        if action_name == "sync":
+            result = subprocess.run(
+                [sys.executable, "-m", "cda.pipeline.ingest"],
+                capture_output=True,
+                text=True,
+                timeout=300
+            )
+        elif action_name == "reconstruct":
+            result = subprocess.run(
+                [sys.executable, "-m", "cda.pipeline.reconstruct"],
+                capture_output=True,
+                text=True,
+                timeout=300
+            )
+        elif action_name == "embed-build":
+            result = subprocess.run(
+                [sys.executable, "-m", "cda.pipeline.embed", "build"],
+                capture_output=True,
+                text=True,
+                timeout=600
+            )
+        elif action_name == "watch-start":
+            result = subprocess.run(
+                [sys.executable, "-m", "cda.pipeline.watcher", "start"],
+                capture_output=True,
+                text=True,
+                timeout=30
+            )
+        else:
+            result = None
+
+        with ACTION_LOCK:
+            if result:
+                ACTION_STATE[action_id]["status"] = "completed" if result.returncode == 0 else "failed"
+                ACTION_STATE[action_id]["output"] = result.stdout + result.stderr
+                ACTION_STATE[action_id]["returncode"] = result.returncode
+            ACTION_STATE[action_id]["completed_at"] = datetime.now().isoformat()
+    except Exception as e:
+        with ACTION_LOCK:
+            ACTION_STATE[action_id]["status"] = "error"
+            ACTION_STATE[action_id]["output"] = str(e)
+            ACTION_STATE[action_id]["completed_at"] = datetime.now().isoformat()

code_data_ark-2.0.12/cda/ui/db/__init__.py

@@ -0,0 +1,31 @@
+from .base import get_db, query_rows, query_one, safe_rows, safe_one, table_exists, execute_stmt
+from .overview import get_overview
+from .sessions import get_sessions, get_session_detail
+from .search import get_search_results
+from .workspaces import get_workspaces, get_workspace_detail
+from .memory import get_memory
+from .tools import get_tool_calls, get_vfs
+from .signals import get_alerts, get_behavioral_signals
+from .tokens import get_tokens
+
+__all__ = [
+    "get_db",
+    "query_rows",
+    "query_one",
+    "safe_rows",
+    "safe_one",
+    "table_exists",
+    "execute_stmt",
+    "get_overview",
+    "get_sessions",
+    "get_session_detail",
+    "get_search_results",
+    "get_workspaces",
+    "get_workspace_detail",
+    "get_memory",
+    "get_tool_calls",
+    "get_vfs",
+    "get_alerts",
+    "get_behavioral_signals",
+    "get_tokens",
+]

code_data_ark-2.0.12/cda/ui/db/base.py

@@ -0,0 +1,69 @@
+import sqlite3
+from cda.kernel.paths import DB_PATH
+
+
+def get_db():
+    """Get database connection with proper settings."""
+    conn = sqlite3.connect(str(DB_PATH), timeout=10)
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA synchronous=NORMAL")
+    return conn
+
+
+def query_rows(sql, params=()):
+    """Execute SELECT and return rows as dicts."""
+    try:
+        conn = get_db()
+        cursor = conn.execute(sql, params)
+        rows = [dict(row) for row in cursor.fetchall()]
+        conn.close()
+        return rows
+    except Exception as e:
+        return {"error": str(e)}
+
+
+def query_one(sql, params=()):
+    """Execute SELECT and return single row or None."""
+    rows = query_rows(sql, params)
+    if isinstance(rows, dict) and "error" in rows:
+        return rows
+    return rows[0] if rows else None
+
+
+def safe_rows(rows):
+    """Normalize query_rows output to an array for APIs."""
+    if isinstance(rows, dict) and "error" in rows:
+        return []
+    return rows or []
+
+
+def safe_one(row):
+    """Normalize query_one output to a dict or None."""
+    if isinstance(row, dict) and "error" in row:
+        return None
+    return row
+
+
+def table_exists(table_name):
+    """Return True if a table exists in the current database."""
+    try:
+        row = query_one(
+            "SELECT name FROM sqlite_master WHERE type='table' AND name=?",
+            (table_name,)
+        )
+        return bool(row)
+    except Exception:
+        return False
+
+
+def execute_stmt(sql, params=()):
+    """Execute INSERT/UPDATE/DELETE statement."""
+    try:
+        conn = get_db()
+        conn.execute(sql, params)
+        conn.commit()
+        conn.close()
+        return {"ok": True}
+    except Exception as e:
+        return {"error": str(e)}

code_data_ark-2.0.12/cda/ui/db/memory.py

@@ -0,0 +1,14 @@
+from .base import query_rows
+
+
+def get_memory():
+    """Get all memory files."""
+    try:
+        memory = query_rows("""
+            SELECT id, scope, workspace_id, session_id, filename, size_bytes, ingested_at
+            FROM memory_files
+            ORDER BY ingested_at DESC
+        """)
+        return {"memory": memory}
+    except Exception as e:
+        return {"error": str(e)}

code_data_ark-2.0.12/cda/ui/db/overview.py

@@ -0,0 +1,84 @@
+from .base import query_rows, query_one, safe_rows, safe_one, table_exists
+
+
+def get_overview():
+    """Dashboard overview stats."""
+    try:
+        has_analysis = table_exists('session_analysis')
+        has_exchanges = table_exists('exchanges')
+        has_signals = table_exists('exchange_signals')
+        has_alerts = table_exists('anomaly_alerts')
+
+        stats = query_one(f"""
+            SELECT
+                (SELECT COUNT(*) FROM sessions) as total_sessions,
+                {("(SELECT COUNT(*) FROM exchanges)" if has_exchanges else "0")} as total_exchanges,
+                {("(SELECT AVG(heat_score) FROM session_analysis WHERE heat_score IS NOT NULL)" if has_analysis else "0")} as avg_heat,
+                {("(SELECT COUNT(*) FROM session_analysis WHERE heat_score >= 50)" if has_analysis else "0")} as critical_sessions,
+                {("(SELECT COUNT(*) FROM anomaly_alerts)" if has_alerts else "0")} as alert_count,
+                (SELECT COUNT(*) FROM workspaces) as workspace_count,
+                (SELECT MAX(created_at) FROM sessions) as last_session
+        """)
+
+        heat_dist = safe_rows(query_rows("""
+            SELECT
+                CASE
+                    WHEN heat_score < 20 THEN '0-19'
+                    WHEN heat_score < 40 THEN '20-39'
+                    WHEN heat_score < 60 THEN '40-59'
+                    WHEN heat_score < 80 THEN '60-79'
+                    ELSE '80-100'
+                END as range,
+                COUNT(*) as count
+            FROM session_analysis
+            WHERE heat_score IS NOT NULL
+            GROUP BY range
+            ORDER BY range
+        """)) if has_analysis else []
+
+        keywords = safe_rows(query_rows("""
+            SELECT matched_keyword as keyword, SUM(count) as total_count
+            FROM (
+                SELECT matched_keyword, COUNT(*) as count
+                FROM exchange_signals
+                WHERE matched_keyword IS NOT NULL
+                GROUP BY matched_keyword
+            )
+            GROUP BY matched_keyword
+            ORDER BY total_count DESC
+            LIMIT 15
+        """)) if has_signals else []
+
+        exchange_count_expr = (
+            "(SELECT COUNT(*) FROM exchanges WHERE exchanges.session_id = s.session_id)"
+            if has_exchanges else "0"
+        )
+        if has_analysis:
+            recent = safe_rows(query_rows(f"""
+                SELECT s.session_id as id, s.title, sa.heat_score,
+                       {exchange_count_expr} as exchange_count,
+                       s.created_at
+                FROM sessions s
+                LEFT JOIN session_analysis sa ON sa.session_id = s.session_id
+                ORDER BY s.created_at DESC
+                LIMIT 10
+            """))
+        else:
+            recent = safe_rows(query_rows(f"""
+                SELECT s.session_id as id, s.title, NULL as heat_score,
+                       {exchange_count_expr} as exchange_count,
+                       s.created_at
+                FROM sessions s
+                ORDER BY s.created_at DESC
+                LIMIT 10
+            """))
+
+        stats = safe_one(stats)
+        return {
+            "stats": dict(stats) if stats else {},
+            "heat_distribution": heat_dist,
+            "keywords": keywords,
+            "recent_sessions": recent
+        }
+    except Exception as e:
+        return {"error": str(e)}

code_data_ark-2.0.12/cda/ui/db/search.py

@@ -0,0 +1,27 @@
+from .base import query_rows
+
+
+def get_search_results(query, limit=50):
+    """Full-text search across exchanges."""
+    try:
+        results = query_rows("""
+            SELECT
+                e.session_id,
+                s.title,
+                sa.heat_score,
+                e.id as exchange_id,
+                e.exchange_index,
+                e.user_message,
+                e.response_text,
+                e.user_ts
+            FROM fts_exchanges fts
+            JOIN exchanges e ON fts.rowid = e.id
+            JOIN sessions s ON e.session_id = s.session_id
+            LEFT JOIN session_analysis sa ON sa.session_id = e.session_id
+            WHERE fts_exchanges MATCH ?
+            ORDER BY rank
+            LIMIT ?
+        """, (query, limit))
+        return {"results": results, "query": query, "count": len(results)}
+    except Exception as e:
+        return {"error": str(e)}

code_data_ark-2.0.12/cda/ui/db/sessions.py

@@ -0,0 +1,125 @@
+from .base import query_rows, query_one, safe_rows, safe_one, table_exists
+
+
+def get_sessions(limit=50, offset=0):
+    """List all sessions with heat scores."""
+    try:
+        has_analysis = table_exists('session_analysis')
+        has_exchanges = table_exists('exchanges')
+        exchange_count_expr = (
+            "(SELECT COUNT(*) FROM exchanges WHERE exchanges.session_id = s.session_id)"
+            if has_exchanges else "0"
+        )
+
+        if has_analysis:
+            sessions = safe_rows(query_rows(f"""
+                SELECT s.session_id as id, s.title, sa.heat_score, s.workspace_id,
+                       {exchange_count_expr} as exchange_count,
+                       s.created_at
+                FROM sessions s
+                LEFT JOIN session_analysis sa ON sa.session_id = s.session_id
+                ORDER BY s.created_at DESC
+                LIMIT ? OFFSET ?
+            """, (limit, offset)))
+        else:
+            sessions = safe_rows(query_rows(f"""
+                SELECT s.session_id as id, s.title, NULL as heat_score, s.workspace_id,
+                       {exchange_count_expr} as exchange_count,
+                       s.created_at
+                FROM sessions s
+                ORDER BY s.created_at DESC
+                LIMIT ? OFFSET ?
+            """, (limit, offset)))
+
+        total = safe_one(query_one("SELECT COUNT(*) as count FROM sessions"))
+
+        return {
+            "sessions": sessions,
+            "total": total["count"] if total else 0,
+            "limit": limit,
+            "offset": offset
+        }
+    except Exception as e:
+        return {"error": str(e)}
+
+
+def get_session_detail(session_id):
+    """Get full session with all exchanges and signals."""
+    if not session_id:
+        return {"error": "Missing session_id"}
+
+    try:
+        has_exchanges = table_exists('exchanges')
+        has_tool_calls = table_exists('tool_calls')
+        has_vfs = table_exists('vfs')
+        has_alerts = table_exists('anomaly_alerts')
+        has_signals = table_exists('exchange_signals')
+        has_analysis = table_exists('session_analysis')
+
+        session = safe_one(query_one(
+            "SELECT * FROM sessions WHERE session_id = ?", (session_id,)
+        ))
+        if not session:
+            return {"error": "Session not found"}
+
+        exchanges = safe_rows(query_rows("""
+            SELECT id, exchange_index, user_message as user_input, response_text as assistant_response,
+                   tool_calls, tool_call_count, ingested_at as created_at
+            FROM exchanges
+            WHERE session_id = ?
+            ORDER BY ingested_at ASC
+        """, (session_id,))) if has_exchanges else []
+
+        tool_calls = safe_rows(query_rows("""
+            SELECT id, session_id, exchange_index, request_id, tool_call_id, tool_name,
+                   file_path, arguments_json, has_output, ingested_at
+            FROM tool_calls
+            WHERE session_id = ?
+            ORDER BY ingested_at ASC
+        """, (session_id,))) if has_tool_calls else []
+
+        vfs_entries = safe_rows(query_rows("""
+            SELECT id, source_type, source_path, filename, content_type, size_bytes, sha256, ingested_at
+            FROM vfs
+            WHERE session_id = ?
+            ORDER BY filename ASC
+        """, (session_id,))) if has_vfs else []
+
+        alerts = safe_rows(query_rows("""
+            SELECT id, alert_type, severity, message, created_at
+            FROM anomaly_alerts
+            WHERE session_id = ?
+            ORDER BY created_at DESC
+        """, (session_id,))) if has_alerts else []
+
+        signals = safe_rows(query_rows("""
+            SELECT * FROM exchange_signals
+            WHERE session_id = ?
+            ORDER BY ts DESC
+        """, (session_id,))) if has_signals else []
+
+        signal_summary = safe_rows(query_rows("""
+            SELECT signal_type, COUNT(*) as count
+            FROM exchange_signals
+            WHERE session_id = ?
+            GROUP BY signal_type
+        """, (session_id,))) if has_signals else []
+
+        analysis = safe_one(query_one("""
+            SELECT * FROM session_analysis
+            WHERE session_id = ?
+            LIMIT 1
+        """, (session_id,))) if has_analysis else None
+
+        return {
+            "session": dict(session),
+            "analysis": analysis,
+            "exchanges": exchanges,
+            "tool_calls": tool_calls,
+            "vfs": vfs_entries,
+            "alerts": alerts,
+            "signals": signals,
+            "signal_summary": signal_summary
+        }
+    except Exception as e:
+        return {"error": str(e)}

code_data_ark-2.0.12/cda/ui/db/signals.py

@@ -0,0 +1,49 @@
+from .base import query_rows, query_one
+
+
+def get_alerts(limit=50):
+    """Get anomaly alerts."""
+    try:
+        alerts = query_rows("""
+            SELECT id, session_id, alert_type, message, severity, created_at
+            FROM anomaly_alerts
+            ORDER BY created_at DESC
+            LIMIT ?
+        """, (limit,))
+
+        session_titles = {}
+        for alert in alerts:
+            if alert["session_id"] not in session_titles:
+                sess = query_one(
+                    "SELECT title FROM sessions WHERE session_id = ?",
+                    (alert["session_id"],)
+                )
+                session_titles[alert["session_id"]] = sess["title"] if sess else "Unknown"
+
+        for alert in alerts:
+            alert["session_title"] = session_titles.get(alert["session_id"], "Unknown")
+
+        return {"alerts": alerts}
+    except Exception as e:
+        return {"error": str(e)}
+
+
+def get_behavioral_signals(session_id=None):
+    """Get behavioral signal analysis."""
+    try:
+        if session_id:
+            signals = query_rows("""
+                SELECT signal_type, COUNT(*) as count
+                FROM exchange_signals
+                WHERE session_id = ?
+                GROUP BY signal_type
+            """, (session_id,))
+        else:
+            signals = query_rows("""
+                SELECT signal_type, COUNT(*) as count
+                FROM exchange_signals
+                GROUP BY signal_type
+            """)
+        return {"signals": signals}
+    except Exception as e:
+        return {"error": str(e)}

code_data_ark-2.0.12/cda/ui/db/tokens.py

@@ -0,0 +1,33 @@
+from .base import query_rows
+
+
+def get_tokens(session_id=None):
+    """Get token usage analysis."""
+    try:
+        if session_id:
+            tokens = query_rows("""
+                SELECT
+                    SUM(prompt_tokens) as total_prompt,
+                    SUM(completion_tokens) as total_completion,
+                    SUM(cached_tokens) as total_cached,
+                    SUM(prompt_tokens + completion_tokens) as total_tokens,
+                    COUNT(*) as turn_count,
+                    GROUP_CONCAT(DISTINCT model_id) as models
+                FROM token_usage
+                WHERE session_id = ?
+            """, (session_id,))
+        else:
+            tokens = query_rows("""
+                SELECT
+                    SUM(prompt_tokens) as total_prompt,
+                    SUM(completion_tokens) as total_completion,
+                    SUM(cached_tokens) as total_cached,
+                    SUM(prompt_tokens + completion_tokens) as total_tokens,
+                    COUNT(*) as turn_count,
+                    COUNT(DISTINCT session_id) as session_count,
+                    GROUP_CONCAT(DISTINCT model_id) as models
+                FROM token_usage
+            """)
+        return {"tokens": tokens}
+    except Exception as e:
+        return {"error": str(e)}

code_data_ark-2.0.12/cda/ui/db/tools.py

@@ -0,0 +1,47 @@
+from .base import query_rows
+
+
+def get_tool_calls(query_str=None, limit=50):
+    """Search tool calls."""
+    try:
+        if query_str:
+            results = query_rows("""
+                SELECT tc.id, tc.session_id, tc.exchange_index, tc.request_id,
+                       tc.tool_call_id, tc.tool_name, tc.file_path,
+                       tc.arguments_json, tc.has_output, tc.ingested_at,
+                       s.title as session_title
+                FROM tool_calls tc
+                JOIN sessions s ON tc.session_id = s.session_id
+                WHERE tc.tool_name LIKE ? OR tc.arguments_json LIKE ?
+                ORDER BY tc.ingested_at DESC
+                LIMIT ?
+            """, (f"%{query_str}%", f"%{query_str}%", limit))
+        else:
+            results = query_rows("""
+                SELECT tc.id, tc.session_id, tc.exchange_index, tc.request_id,
+                       tc.tool_call_id, tc.tool_name, tc.file_path,
+                       tc.arguments_json, tc.has_output, tc.ingested_at,
+                       s.title as session_title
+                FROM tool_calls tc
+                JOIN sessions s ON tc.session_id = s.session_id
+                ORDER BY tc.ingested_at DESC
+                LIMIT ?
+            """, (limit,))
+        return {"tool_calls": results, "query": query_str, "count": len(results)}
+    except Exception as e:
+        return {"error": str(e)}
+
+
+def get_vfs(session_id):
+    """List VFS files for a session."""
+    try:
+        vfs = query_rows("""
+            SELECT id, session_id, source_type, source_path, filename,
+                   content_type, size_bytes, sha256, ingested_at
+            FROM vfs
+            WHERE session_id = ?
+            ORDER BY filename
+        """, (session_id,))
+        return {"vfs": vfs, "session_id": session_id}
+    except Exception as e:
+        return {"error": str(e)}

code_data_ark-2.0.12/cda/ui/db/workspaces.py

@@ -0,0 +1,33 @@
+from .base import query_rows
+
+
+def get_workspaces():
+    """List all workspaces with session counts."""
+    try:
+        workspaces = query_rows("""
+            SELECT w.workspace_id, w.uri, w.name, w.type, w.session_count,
+                   (SELECT MAX(s.created_at) FROM sessions s
+                    WHERE s.workspace_id = w.workspace_id) as last_session
+            FROM workspaces w
+            ORDER BY w.session_count DESC
+        """)
+        return {"workspaces": workspaces}
+    except Exception as e:
+        return {"error": str(e)}
+
+
+def get_workspace_detail(workspace_id):
+    """Get all sessions for a workspace."""
+    try:
+        sessions = query_rows("""
+            SELECT s.session_id as id, s.title, sa.heat_score,
+                   (SELECT COUNT(*) FROM exchanges WHERE exchanges.session_id = s.session_id) as exchange_count,
+                   s.created_at
+            FROM sessions s
+            LEFT JOIN session_analysis sa ON sa.session_id = s.session_id
+            WHERE s.workspace_id = ?
+            ORDER BY s.created_at DESC
+        """, (workspace_id,))
+        return {"workspace_id": workspace_id, "sessions": sessions}
+    except Exception as e:
+        return {"error": str(e)}