cseq 0.0.1__py3-none-any.whl

cseq/event_store.py

@@ -0,0 +1,378 @@
+from __future__ import annotations
+
+from dataclasses import asdict
+import json
+import sqlite3
+from pathlib import Path
+from typing import Iterable
+
+from .runtime import RuntimeEvent, TraceSession
+
+SCHEMA_VERSION = 4
+
+
+class EventStore:
+    """Small SQLite-backed runtime event store.
+
+    This is the first persistent Event Store layer. It preserves ingest order and
+    indexes marker/cpu/task for later viewport/query work. It does not infer a
+    cross-CPU total order.
+    """
+
+    def __init__(self, path: str | Path):
+        self.path = Path(path)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.db = sqlite3.connect(self.path)
+        self.db.row_factory = sqlite3.Row
+        self._init_schema()
+
+    def _init_schema(self) -> None:
+        self.db.executescript(
+            """
+            CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT NOT NULL);
+            CREATE TABLE IF NOT EXISTS events(
+              session TEXT NOT NULL,
+              event_index INTEGER NOT NULL,
+              marker_id TEXT NOT NULL,
+              raw_line TEXT NOT NULL,
+              timestamp_raw TEXT,
+              cpu_hint TEXT,
+              task_hint TEXT,
+              clock_domain TEXT,
+              message TEXT,
+              payload_json TEXT NOT NULL,
+              PRIMARY KEY(session, event_index)
+            );
+            CREATE TABLE IF NOT EXISTS sessions(
+              session TEXT PRIMARY KEY,
+              status TEXT NOT NULL,
+              event_count INTEGER NOT NULL DEFAULT 0
+            );
+            CREATE TABLE IF NOT EXISTS event_chunks(
+              session TEXT NOT NULL,
+              chunk_index INTEGER NOT NULL,
+              start_event_index INTEGER NOT NULL,
+              end_event_index INTEGER NOT NULL,
+              min_timestamp_raw TEXT,
+              max_timestamp_raw TEXT,
+              cpu_json TEXT NOT NULL,
+              task_json TEXT NOT NULL,
+              PRIMARY KEY(session, chunk_index)
+            );
+            CREATE INDEX IF NOT EXISTS idx_chunks_range ON event_chunks(session,start_event_index,end_event_index);
+            """
+        )
+        self.db.execute("INSERT OR REPLACE INTO meta(key,value) VALUES('schema_version',?)", (str(SCHEMA_VERSION),))
+        importing = self.db.execute("SELECT COUNT(*) FROM sessions WHERE status='IMPORTING'").fetchone()[0]
+        if not importing:
+            self._create_secondary_indexes()
+        self.db.commit()
+
+    def _drop_secondary_indexes(self) -> None:
+        self.db.executescript(
+            """
+            DROP INDEX IF EXISTS idx_events_marker;
+            DROP INDEX IF EXISTS idx_events_cpu;
+            DROP INDEX IF EXISTS idx_events_task;
+            """
+        )
+
+    def _create_secondary_indexes(self) -> None:
+        self.db.executescript(
+            """
+            CREATE INDEX IF NOT EXISTS idx_events_marker ON events(session, marker_id, event_index);
+            CREATE INDEX IF NOT EXISTS idx_events_cpu ON events(session, cpu_hint, event_index);
+            CREATE INDEX IF NOT EXISTS idx_events_task ON events(session, task_hint, event_index);
+            """
+        )
+
+    def close(self) -> None:
+        self.db.close()
+
+    def __enter__(self) -> "EventStore":
+        return self
+
+    def __exit__(self, *_exc) -> None:
+        self.close()
+
+    def replace_session(self, session: TraceSession, *, chunk_size: int = 1000, batch_size: int = 10000) -> int:
+        return self.replace_session_events(
+            session.name, session.events, chunk_size=chunk_size, batch_size=batch_size
+        )
+
+    def replace_session_events(
+        self,
+        session: str,
+        events: Iterable[RuntimeEvent],
+        *,
+        chunk_size: int = 1000,
+        batch_size: int = 10000,
+        bulk_rebuild_indexes: bool = False,
+    ) -> int:
+        """Replace one session from an iterable without materializing all events.
+
+        This is the large-runtime ingest path. Rows are inserted in bounded batches
+        while chunk metadata is accumulated independently. The caller may pass a
+        generator for multi-million-event imports.
+        """
+        chunk_size = max(1, int(chunk_size))
+        batch_size = max(1, int(batch_size))
+        count = 0
+        batch: list[tuple[object, ...]] = []
+        chunk_index = 0
+        chunk_count = 0
+        chunk_start_index: int | None = None
+        chunk_end_index: int | None = None
+        chunk_min_time: str | None = None
+        chunk_max_time: str | None = None
+        chunk_cpus: set[str] = set()
+        chunk_tasks: set[str] = set()
+
+        def flush_batch() -> None:
+            nonlocal batch
+            if not batch:
+                return
+            self.db.executemany(
+                """INSERT INTO events(session,event_index,marker_id,raw_line,timestamp_raw,cpu_hint,task_hint,clock_domain,message,payload_json)
+                   VALUES(?,?,?,?,?,?,?,?,?,?)""",
+                batch,
+            )
+            batch = []
+
+        def flush_chunk() -> None:
+            nonlocal chunk_index, chunk_count, chunk_start_index, chunk_end_index
+            nonlocal chunk_min_time, chunk_max_time, chunk_cpus, chunk_tasks
+            if chunk_count == 0 or chunk_start_index is None or chunk_end_index is None:
+                return
+            self.db.execute(
+                """INSERT INTO event_chunks(session,chunk_index,start_event_index,end_event_index,min_timestamp_raw,max_timestamp_raw,cpu_json,task_json)
+                   VALUES(?,?,?,?,?,?,?,?)""",
+                (
+                    session, chunk_index, chunk_start_index, chunk_end_index,
+                    chunk_min_time, chunk_max_time,
+                    json.dumps(sorted(chunk_cpus)), json.dumps(sorted(chunk_tasks)),
+                ),
+            )
+            chunk_index += 1
+            chunk_count = 0
+            chunk_start_index = None
+            chunk_end_index = None
+            chunk_min_time = None
+            chunk_max_time = None
+            chunk_cpus = set()
+            chunk_tasks = set()
+
+        if bulk_rebuild_indexes:
+            self._drop_secondary_indexes()
+        try:
+            with self.db:
+                self.db.execute("DELETE FROM events WHERE session=?", (session,))
+                self.db.execute("DELETE FROM event_chunks WHERE session=?", (session,))
+                for e in events:
+                    batch.append((
+                        session, e.event_index, e.marker_id, e.raw_line, e.timestamp_raw,
+                        e.cpu_hint, e.task_hint, e.clock_domain, e.message,
+                        json.dumps(e.payload_dict(), ensure_ascii=False, sort_keys=True),
+                    ))
+                    if chunk_count == 0:
+                        chunk_start_index = e.event_index
+                    chunk_end_index = e.event_index
+                    chunk_count += 1
+                    if e.timestamp_raw is not None:
+                        chunk_min_time = e.timestamp_raw if chunk_min_time is None else min(chunk_min_time, e.timestamp_raw)
+                        chunk_max_time = e.timestamp_raw if chunk_max_time is None else max(chunk_max_time, e.timestamp_raw)
+                    if e.cpu_hint:
+                        chunk_cpus.add(e.cpu_hint)
+                    if e.task_hint:
+                        chunk_tasks.add(e.task_hint)
+                    count += 1
+                    if len(batch) >= batch_size:
+                        flush_batch()
+                    if chunk_count >= chunk_size:
+                        flush_chunk()
+                flush_batch()
+                flush_chunk()
+        finally:
+            if bulk_rebuild_indexes:
+                self._create_secondary_indexes()
+                self.db.commit()
+        return count
+
+    def replace_session_events_chunked(
+        self,
+        session: str,
+        events: Iterable[RuntimeEvent],
+        *,
+        chunk_size: int = 10000,
+        batch_size: int = 50000,
+        commit_every: int = 250000,
+        rebuild_indexes: bool = True,
+    ) -> int:
+        """Import a very large session using bounded transactions.
+
+        Partial data is marked IMPORTING and only becomes READY after the full
+        import (and optional secondary-index rebuild) completes.
+        """
+        chunk_size=max(1,int(chunk_size)); batch_size=max(1,int(batch_size)); commit_every=max(batch_size,int(commit_every))
+        if rebuild_indexes:
+            self._drop_secondary_indexes()
+        self.db.execute("INSERT OR REPLACE INTO sessions(session,status,event_count) VALUES(?, 'IMPORTING', 0)",(session,))
+        self.db.execute("DELETE FROM events WHERE session=?",(session,))
+        self.db.execute("DELETE FROM event_chunks WHERE session=?",(session,))
+        self.db.commit()
+        count=0; since_commit=0; batch=[]
+        chunk_index=0; chunk_count=0; chunk_start=None; chunk_end=None; min_time=None; max_time=None; cpus=set(); tasks=set()
+        def flush_batch():
+            nonlocal batch
+            if batch:
+                self.db.executemany("INSERT INTO events(session,event_index,marker_id,raw_line,timestamp_raw,cpu_hint,task_hint,clock_domain,message,payload_json) VALUES(?,?,?,?,?,?,?,?,?,?)",batch); batch=[]
+        def flush_chunk():
+            nonlocal chunk_index,chunk_count,chunk_start,chunk_end,min_time,max_time,cpus,tasks
+            if not chunk_count:return
+            self.db.execute("INSERT INTO event_chunks(session,chunk_index,start_event_index,end_event_index,min_timestamp_raw,max_timestamp_raw,cpu_json,task_json) VALUES(?,?,?,?,?,?,?,?)",(session,chunk_index,chunk_start,chunk_end,min_time,max_time,json.dumps(sorted(cpus)),json.dumps(sorted(tasks))))
+            chunk_index+=1; chunk_count=0; chunk_start=None; chunk_end=None; min_time=None; max_time=None; cpus=set(); tasks=set()
+        try:
+            for e in events:
+                batch.append((session,e.event_index,e.marker_id,e.raw_line,e.timestamp_raw,e.cpu_hint,e.task_hint,e.clock_domain,e.message,json.dumps(e.payload_dict(),ensure_ascii=False,sort_keys=True)))
+                if chunk_count==0: chunk_start=e.event_index
+                chunk_end=e.event_index; chunk_count+=1; count+=1; since_commit+=1
+                if e.timestamp_raw is not None:
+                    min_time=e.timestamp_raw if min_time is None else min(min_time,e.timestamp_raw); max_time=e.timestamp_raw if max_time is None else max(max_time,e.timestamp_raw)
+                if e.cpu_hint: cpus.add(e.cpu_hint)
+                if e.task_hint: tasks.add(e.task_hint)
+                if len(batch)>=batch_size: flush_batch()
+                if chunk_count>=chunk_size: flush_chunk()
+                if since_commit>=commit_every:
+                    flush_batch(); self.db.execute("UPDATE sessions SET event_count=? WHERE session=?",(count,session)); self.db.commit(); since_commit=0
+            flush_batch(); flush_chunk(); self.db.execute("UPDATE sessions SET event_count=? WHERE session=?",(count,session)); self.db.commit()
+            if rebuild_indexes:
+                self._create_secondary_indexes(); self.db.commit()
+            self.db.execute("UPDATE sessions SET status='READY', event_count=? WHERE session=?",(count,session)); self.db.commit()
+            return count
+        except BaseException:
+            try:
+                flush_batch(); self.db.execute("UPDATE sessions SET status='IMPORTING', event_count=? WHERE session=?",(count,session)); self.db.commit()
+            finally:
+                pass
+            raise
+
+    def resume_session_events_chunked(
+        self,
+        session: str,
+        events: Iterable[RuntimeEvent],
+        *,
+        chunk_size: int = 10000,
+        batch_size: int = 50000,
+        commit_every: int = 250000,
+        finalize: bool = False,
+    ) -> int:
+        """Append to an IMPORTING session and optionally finalize it."""
+        row=self.db.execute("SELECT status,event_count FROM sessions WHERE session=?",(session,)).fetchone()
+        if row is None or row[0] != 'IMPORTING':
+            raise ValueError(f"session {session!r} is not IMPORTING")
+        existing=int(row[1])
+        chunk_size=max(1,int(chunk_size)); batch_size=max(1,int(batch_size)); commit_every=max(batch_size,int(commit_every))
+        chunk_row=self.db.execute("SELECT COALESCE(MAX(chunk_index),-1) FROM event_chunks WHERE session=?",(session,)).fetchone()
+        chunk_index=int(chunk_row[0])+1
+        count=existing; added=0; since_commit=0; batch=[]
+        chunk_count=0; chunk_start=None; chunk_end=None; min_time=None; max_time=None; cpus=set(); tasks=set()
+        def flush_batch():
+            nonlocal batch
+            if batch:
+                self.db.executemany("INSERT INTO events(session,event_index,marker_id,raw_line,timestamp_raw,cpu_hint,task_hint,clock_domain,message,payload_json) VALUES(?,?,?,?,?,?,?,?,?,?)",batch); batch=[]
+        def flush_chunk():
+            nonlocal chunk_index,chunk_count,chunk_start,chunk_end,min_time,max_time,cpus,tasks
+            if not chunk_count:return
+            self.db.execute("INSERT INTO event_chunks(session,chunk_index,start_event_index,end_event_index,min_timestamp_raw,max_timestamp_raw,cpu_json,task_json) VALUES(?,?,?,?,?,?,?,?)",(session,chunk_index,chunk_start,chunk_end,min_time,max_time,json.dumps(sorted(cpus)),json.dumps(sorted(tasks))))
+            chunk_index+=1; chunk_count=0; chunk_start=None; chunk_end=None; min_time=None; max_time=None; cpus=set(); tasks=set()
+        for e in events:
+            batch.append((session,e.event_index,e.marker_id,e.raw_line,e.timestamp_raw,e.cpu_hint,e.task_hint,e.clock_domain,e.message,json.dumps(e.payload_dict(),ensure_ascii=False,sort_keys=True)))
+            if chunk_count==0: chunk_start=e.event_index
+            chunk_end=e.event_index; chunk_count+=1; added+=1; count+=1; since_commit+=1
+            if e.timestamp_raw is not None:
+                min_time=e.timestamp_raw if min_time is None else min(min_time,e.timestamp_raw); max_time=e.timestamp_raw if max_time is None else max(max_time,e.timestamp_raw)
+            if e.cpu_hint: cpus.add(e.cpu_hint)
+            if e.task_hint: tasks.add(e.task_hint)
+            if len(batch)>=batch_size: flush_batch()
+            if chunk_count>=chunk_size: flush_chunk()
+            if since_commit>=commit_every:
+                flush_batch(); self.db.execute("UPDATE sessions SET event_count=? WHERE session=?",(count,session)); self.db.commit(); since_commit=0
+        flush_batch(); flush_chunk(); self.db.execute("UPDATE sessions SET event_count=? WHERE session=?",(count,session)); self.db.commit()
+        if finalize:
+            self._create_secondary_indexes(); self.db.commit()
+            self.db.execute("UPDATE sessions SET status='READY' WHERE session=?",(session,)); self.db.commit()
+        return added
+
+    def session_status(self, session: str) -> str | None:
+        row=self.db.execute("SELECT status FROM sessions WHERE session=?",(session,)).fetchone()
+        return None if row is None else str(row[0])
+
+    def count(self, session: str) -> int:
+        return int(self.db.execute("SELECT COUNT(*) FROM events WHERE session=?", (session,)).fetchone()[0])
+
+    def fetch_window(self, session: str, *, offset: int = 0, limit: int = 500) -> list[RuntimeEvent]:
+        rows = self.db.execute(
+            "SELECT * FROM events WHERE session=? ORDER BY event_index LIMIT ? OFFSET ?",
+            (session, int(limit), int(offset)),
+        ).fetchall()
+        return [self._row_to_event(r) for r in rows]
+
+    def by_marker(self, session: str, marker_id: str, *, limit: int = 500) -> list[RuntimeEvent]:
+        rows = self.db.execute(
+            "SELECT * FROM events WHERE session=? AND marker_id=? ORDER BY event_index LIMIT ?",
+            (session, marker_id, int(limit)),
+        ).fetchall()
+        return [self._row_to_event(r) for r in rows]
+
+    def summary(self, session: str) -> dict[str, object]:
+        count = self.count(session)
+        cpus = [r[0] for r in self.db.execute("SELECT DISTINCT cpu_hint FROM events WHERE session=? AND cpu_hint IS NOT NULL ORDER BY cpu_hint", (session,))]
+        tasks = [r[0] for r in self.db.execute("SELECT DISTINCT task_hint FROM events WHERE session=? AND task_hint IS NOT NULL ORDER BY task_hint", (session,))]
+        return {"session": session, "count": count, "cpus": cpus, "tasks": tasks, "schema_version": SCHEMA_VERSION}
+
+
+    def chunk_summary(self, session: str) -> list[dict[str, object]]:
+        rows = self.db.execute(
+            "SELECT * FROM event_chunks WHERE session=? ORDER BY chunk_index", (session,)
+        ).fetchall()
+        return [
+            {
+                "chunk_index": r["chunk_index"],
+                "start_event_index": r["start_event_index"],
+                "end_event_index": r["end_event_index"],
+                "min_timestamp_raw": r["min_timestamp_raw"],
+                "max_timestamp_raw": r["max_timestamp_raw"],
+                "cpus": json.loads(r["cpu_json"]),
+                "tasks": json.loads(r["task_json"]),
+            }
+            for r in rows
+        ]
+
+    def fetch_event_index_range(self, session: str, start_index: int, end_index: int) -> list[RuntimeEvent]:
+        rows = self.db.execute(
+            "SELECT * FROM events WHERE session=? AND event_index BETWEEN ? AND ? ORDER BY event_index",
+            (session, int(start_index), int(end_index)),
+        ).fetchall()
+        return [self._row_to_event(r) for r in rows]
+
+    def iter_events(self, session: str, *, batch_size: int = 5000):
+        """Stream events in ingest order without OFFSET-based pagination."""
+        cur = self.db.execute(
+            "SELECT * FROM events WHERE session=? ORDER BY event_index", (session,)
+        )
+        size = max(1, int(batch_size))
+        while True:
+            rows = cur.fetchmany(size)
+            if not rows:
+                break
+            for row in rows:
+                yield self._row_to_event(row)
+
+    @staticmethod
+    def _row_to_event(r: sqlite3.Row) -> RuntimeEvent:
+        payload = tuple(sorted(json.loads(r["payload_json"]).items()))
+        return RuntimeEvent(
+            event_index=r["event_index"], marker_id=r["marker_id"], raw_line=r["raw_line"],
+            timestamp_raw=r["timestamp_raw"], cpu_hint=r["cpu_hint"], task_hint=r["task_hint"],
+            clock_domain=r["clock_domain"], message=r["message"], payload=payload,
+        )

cseq/explain.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+from dataclasses import asdict
+
+from .model import ProjectIndex
+
+
+def explain_call(index: ProjectIndex, *, caller: str, callee_text: str | None = None) -> dict:
+    matches = [c for c in index.callsites if c.caller_name == caller]
+    if callee_text is not None:
+        matches = [c for c in matches if c.callee_name == callee_text or callee_text in c.raw_text]
+    if not matches:
+        return {"found": False, "caller": caller, "query": callee_text, "calls": []}
+
+    functions = {f.qualified_id: f for f in index.functions()}
+    flow_by_target: dict[str, list] = {}
+    for edge in index.flow_edges:
+        flow_by_target.setdefault(edge.target_key, []).append(edge)
+
+    calls = []
+    for call in matches:
+        targets = []
+        for tid in call.target_function_ids:
+            fn = functions.get(tid)
+            targets.append({
+                "function_id": tid,
+                "name": fn.name if fn else None,
+                "source": fn.source_path if fn else None,
+            })
+        if call.target_function_id and call.target_function_id not in call.target_function_ids:
+            fn = functions.get(call.target_function_id)
+            targets.append({
+                "function_id": call.target_function_id,
+                "name": fn.name if fn else None,
+                "source": fn.source_path if fn else None,
+            })
+
+        flow = []
+        if call.callee_slot_key:
+            queue = [call.callee_slot_key]
+            seen = set()
+            while queue and len(flow) < 64:
+                target = queue.pop(0)
+                if target in seen:
+                    continue
+                seen.add(target)
+                for edge in flow_by_target.get(target, []):
+                    flow.append({
+                        "kind": edge.kind.value,
+                        "from": edge.source_key,
+                        "to": edge.target_key,
+                        "source": edge.source_path,
+                    })
+                    queue.append(edge.source_key)
+
+        cpu = []
+        target_ids = set(call.target_function_ids)
+        if call.target_function_id:
+            target_ids.add(call.target_function_id)
+        for ev in index.cpu_evidence:
+            if ev.target_function_id in target_ids:
+                cpu.append({
+                    "function_id": ev.target_function_id,
+                    "layer": ev.layer.value,
+                    "kind": ev.value_kind.value,
+                    "value": ev.value,
+                    "provenance": ev.provenance,
+                    "explicit_override": ev.explicit_override,
+                })
+
+        linker = []
+        for ev in index.linker_evidence:
+            if ev.symbol_name == (call.callee_name or "") or target_ids.intersection(ev.matched_function_ids):
+                linker.append({
+                    "symbol": ev.symbol_name, "address": ev.address,
+                    "object_file": ev.object_file, "section": ev.section,
+                    "matched_function_ids": list(ev.matched_function_ids),
+                    "provenance": ev.provenance,
+                })
+
+        binary = []
+        for ev in index.binary_evidence:
+            if ev.symbol_name == (call.callee_name or "") or target_ids.intersection(ev.matched_function_ids):
+                binary.append({
+                    "symbol": ev.symbol_name, "address": ev.address, "size": ev.size,
+                    "kind": ev.symbol_kind, "matched_function_ids": list(ev.matched_function_ids),
+                    "binary_hash": ev.binary_hash, "provenance": ev.provenance,
+                })
+
+        dwarf = []
+        for ev in index.dwarf_evidence:
+            if ev.function_id in target_ids:
+                dwarf.append({
+                    "function_id": ev.function_id, "source": ev.source_path,
+                    "line": ev.line, "address": ev.address,
+                    "dwarf_file": ev.dwarf_file, "provenance": ev.provenance,
+                })
+
+        calls.append({
+            "source": call.source_path,
+            "line": call.source_range.start_line,
+            "raw": call.raw_text,
+            "resolution": call.target_kind.value,
+            "unknown_possible": call.unknown_possible,
+            "callee_slot": call.callee_slot_key,
+            "targets": targets,
+            "value_flow": flow,
+            "cpu_evidence": cpu,
+            "linker_evidence": linker,
+            "binary_evidence": binary,
+            "dwarf_evidence": dwarf,
+        })
+    return {"found": True, "caller": caller, "query": callee_text, "calls": calls}