traceforge-toolkit 0.1.0__py3-none-any.whl

traceforge/migrations/env.py

@@ -0,0 +1,60 @@
+"""Alembic environment configuration for traceforge system storage.
+
+This is used programmatically — not via the `alembic` CLI — so there is no
+alembic.ini file. Instead, SystemStore calls run_migrations() directly.
+"""
+
+from __future__ import annotations
+
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+
+from traceforge.migrations.models import metadata
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode (emit SQL without a live connection)."""
+    url = context.config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "qmark"},
+    )
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode (with a live connection)."""
+    connectable = context.config.attributes.get("connection")
+
+    if connectable is not None:
+        # Connection already provided by the caller (SystemStore)
+        context.configure(connection=connectable, target_metadata=metadata)
+        with context.begin_transaction():
+            context.run_migrations()
+    else:
+        # Fallback: create engine from config
+        configuration = context.config.get_section(context.config.config_ini_section, {})
+        connectable = engine_from_config(
+            configuration,
+            prefix="sqlalchemy.",
+            poolclass=pool.NullPool,
+        )
+        with connectable.connect() as connection:
+            context.configure(connection=connection, target_metadata=metadata)
+            with context.begin_transaction():
+                context.run_migrations()
+
+
+try:
+    if context.is_offline_mode():
+        run_migrations_offline()
+    else:
+        run_migrations_online()
+except (AttributeError, NameError):
+    # Guard against accidental direct import outside Alembic context.
+    # When imported normally (not via Alembic's ScriptDirectory), the
+    # context proxy raises NameError (Alembic's ModuleClsProxy machinery).
+    pass

traceforge/migrations/models.py

@@ -0,0 +1,145 @@
+"""SQLAlchemy table definitions for the traceforge system database.
+
+These models serve as the migration target metadata. They are NOT used at
+runtime for queries — the raw sqlite3 API remains for performance. Alembic
+uses these to auto-generate diffs when new migrations are needed.
+"""
+
+from __future__ import annotations
+
+from sqlalchemy import Column, Float, Integer, MetaData, PrimaryKeyConstraint, Table, Text
+
+metadata = MetaData()
+
+session_state = Table(
+    "session_state",
+    metadata,
+    Column("session_id", Text, primary_key=True),
+    Column("total_tool_calls", Integer, nullable=False, server_default="0"),
+    Column("total_tokens", Integer, nullable=False, server_default="0"),
+    Column("elapsed_seconds", Float, nullable=False, server_default="0.0"),
+    Column("pressure", Integer, nullable=False, server_default="0"),
+    Column("phase_window_json", Text, nullable=False, server_default="[]"),
+    Column("last_assistant_json", Text),
+    Column("last_user_json", Text),
+    Column("event_count", Integer, nullable=False, server_default="0"),
+    Column("dropped_events", Integer, nullable=False, server_default="0"),
+    Column("last_sequence", Integer),
+    Column("last_event_id", Text),
+    Column("updated_at", Text, nullable=False, server_default=""),
+)
+
+processed_events = Table(
+    "processed_events",
+    metadata,
+    Column("source_event_key", Text, primary_key=True),
+    Column("session_id", Text, nullable=False),
+    Column("session_meta_json", Text),
+    Column("processed_at", Text, nullable=False),
+)
+
+drift_baselines = Table(
+    "drift_baselines",
+    metadata,
+    Column("agent_model", Text, nullable=False),
+    Column("repo", Text, nullable=False),
+    Column("phase_counts_json", Text, nullable=False),
+    Column("total_events", Integer, nullable=False),
+    Column("updated_at", Text, nullable=False),
+    PrimaryKeyConstraint("agent_model", "repo"),
+)
+
+content_hashes = Table(
+    "content_hashes",
+    metadata,
+    Column("repo", Text, nullable=False),
+    Column("file_path", Text, nullable=False),
+    Column("sha256", Text, nullable=False),
+    Column("updated_at", Text, nullable=False),
+    Column("updated_by_session", Text),
+    PrimaryKeyConstraint("repo", "file_path"),
+)
+
+session_summaries = Table(
+    "session_summaries",
+    metadata,
+    Column("session_id", Text, primary_key=True),
+    Column("repo", Text),
+    Column("agent_model", Text),
+    Column("started_at", Text, nullable=False),
+    Column("ended_at", Text),
+    Column("total_events", Integer),
+    Column("dropped_events", Integer, server_default="0"),
+    Column("budget_snapshot_json", Text),
+    Column("recommendation_counts_json", Text),
+    Column("drift_max", Float),
+)
+
+gate_endpoints = Table(
+    "gate_endpoints",
+    metadata,
+    Column("session_id", Text, primary_key=True),
+    Column("sock_path", Text, nullable=False),
+    Column("pid", Integer, nullable=False),
+    Column("registered_at", Text, nullable=False, server_default="(datetime('now'))"),
+)
+
+# ── Normalized 1NF tables ───────────────────────────────────────────────────
+# Budget dimensions, the taint ledger, and MCP tool attributes are stored in
+# first normal form from the outset — there is no JSON-blob/JSON-array shape to
+# migrate away from. These are the single source of truth for their data.
+
+# One row per (session, budget dimension, key). The atomic budget scalars
+# (total_tool_calls, total_tokens, elapsed_seconds, pressure) are columns on
+# session_state — they are already atomic and need no decomposition.
+budget_counters = Table(
+    "budget_counters",
+    metadata,
+    Column("session_id", Text, nullable=False),
+    Column("dimension", Text, nullable=False),
+    Column("key", Text, nullable=False),
+    Column("count", Integer, nullable=False),
+    PrimaryKeyConstraint("session_id", "dimension", "key"),
+)
+
+# One row per taint-ledger entry. ``ordinal`` preserves append order so the
+# bounded ring-buffer semantics survive a reload.
+taint_entries = Table(
+    "taint_entries",
+    metadata,
+    Column("session_id", Text, nullable=False),
+    Column("ordinal", Integer, nullable=False),
+    Column("event_id", Text, nullable=False),
+    Column("source_event_key", Text, nullable=False),
+    Column("clearance", Text, nullable=False),
+    Column("source", Text, nullable=False),
+    Column("payload_pointer", Text, nullable=False),
+    PrimaryKeyConstraint("session_id", "ordinal"),
+)
+
+# Scalar MCP tool fingerprint. The many-valued role/capability/scope attributes
+# live in mcp_profile_attributes below (1NF), one row per value.
+mcp_profiles = Table(
+    "mcp_profiles",
+    metadata,
+    Column("server", Text, nullable=False),
+    Column("tool_name", Text, nullable=False),
+    Column("description_hash", Text, nullable=False),
+    Column("schema_hash", Text, nullable=False),
+    Column("registered_effect", Text),
+    Column("clearance", Text),
+    Column("first_seen", Text, nullable=False),
+    Column("last_seen", Text, nullable=False),
+    PrimaryKeyConstraint("server", "tool_name"),
+)
+
+# One row per many-valued MCP attribute (role / capability / scope).
+mcp_profile_attributes = Table(
+    "mcp_profile_attributes",
+    metadata,
+    Column("server", Text, nullable=False),
+    Column("tool_name", Text, nullable=False),
+    Column("attr_type", Text, nullable=False),  # 'role' | 'capability' | 'scope'
+    Column("attr_value", Text, nullable=False),
+    PrimaryKeyConstraint("server", "tool_name", "attr_type", "attr_value"),
+)

traceforge/migrations/runner.py

@@ -0,0 +1,44 @@
+"""Programmatic Alembic migration runner for traceforge system storage.
+
+Called by SystemStore on initialization to bring the database to HEAD.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from alembic import command
+from alembic.config import Config
+from sqlalchemy import Connection
+
+logger = logging.getLogger(__name__)
+
+_MIGRATIONS_DIR = Path(__file__).resolve().parent
+
+
+def _make_config(connection: Connection) -> Config:
+    """Build an Alembic Config that uses the provided connection."""
+    cfg = Config()
+    cfg.set_main_option("script_location", str(_MIGRATIONS_DIR))
+    cfg.attributes["connection"] = connection
+    return cfg
+
+
+def run_migrations(connection: Connection) -> None:
+    """Run all pending migrations against the given connection.
+
+    This is called once per SystemStore initialization. It is idempotent —
+    if the database is already at HEAD, this is a no-op.
+    """
+    cfg = _make_config(connection)
+    command.upgrade(cfg, "head")
+    logger.debug("Migrations applied successfully")
+
+
+def current_revision(connection: Connection) -> str | None:
+    """Return the current alembic revision (or None if unversioned)."""
+    from alembic.runtime.migration import MigrationContext
+
+    ctx = MigrationContext.configure(connection)
+    return ctx.get_current_revision()

traceforge/migrations/script.py.mako

@@ -0,0 +1,25 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+"""
+from __future__ import annotations
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: str | None = ${repr(down_revision)}
+branch_labels: tuple[str, ...] | None = ${repr(branch_labels)}
+depends_on: tuple[str, ...] | None = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

traceforge/migrations/versions/0001_initial.py

@@ -0,0 +1,176 @@
+"""Initial schema — the single, normalized system database.
+
+Budget dimensions, the taint ledger, and MCP tool profiles are stored in
+first-normal-form tables from the outset (``budget_counters``,
+``taint_entries``, ``mcp_profiles`` + ``mcp_profile_attributes``); the atomic
+budget scalars are plain columns on ``session_state``. No JSON-blob or
+JSON-array representation of these ever exists.
+
+Revision ID: 0001_initial
+Revises: None
+Create Date: 2026-06-15
+"""
+
+from __future__ import annotations
+
+from alembic import op
+
+revision: str = "0001_initial"
+down_revision: str | None = None
+branch_labels: tuple[str, ...] | None = None
+depends_on: tuple[str, ...] | None = None
+
+
+def upgrade() -> None:
+    # PRAGMAs are set by SystemStore.__init__ before migrations run.
+    # They are non-transactional and must not appear inside a migration
+    # (they execute outside rollback scope and would confuse partial-failure recovery).
+
+    # Use IF NOT EXISTS to handle concurrent first-run race: two processes
+    # creating a fresh system.db simultaneously won't crash each other.
+    # We use exec_driver_sql() to bypass SQLAlchemy's text() parameter parsing,
+    # which misinterprets colons in JSON default values as bind parameters.
+    conn = op.get_bind()
+
+    conn.exec_driver_sql("""
+        CREATE TABLE IF NOT EXISTS session_state (
+            session_id TEXT PRIMARY KEY,
+            total_tool_calls INTEGER NOT NULL DEFAULT 0,
+            total_tokens INTEGER NOT NULL DEFAULT 0,
+            elapsed_seconds REAL NOT NULL DEFAULT 0.0,
+            pressure INTEGER NOT NULL DEFAULT 0,
+            phase_window_json TEXT NOT NULL DEFAULT '[]',
+            last_assistant_json TEXT,
+            last_user_json TEXT,
+            event_count INTEGER NOT NULL DEFAULT 0,
+            dropped_events INTEGER NOT NULL DEFAULT 0,
+            last_sequence INTEGER,
+            last_event_id TEXT,
+            updated_at TEXT NOT NULL DEFAULT ''
+        )
+    """)
+
+    conn.exec_driver_sql("""
+        CREATE TABLE IF NOT EXISTS processed_events (
+            source_event_key TEXT PRIMARY KEY,
+            session_id TEXT NOT NULL,
+            session_meta_json TEXT,
+            processed_at TEXT NOT NULL
+        )
+    """)
+
+    # ── Normalized MCP tool profiles ──
+    # Scalar fingerprint per (server, tool). The many-valued role/capability/
+    # scope attributes live in mcp_profile_attributes (1NF), one row per value.
+    conn.exec_driver_sql("""
+        CREATE TABLE IF NOT EXISTS mcp_profiles (
+            server TEXT NOT NULL,
+            tool_name TEXT NOT NULL,
+            description_hash TEXT NOT NULL,
+            schema_hash TEXT NOT NULL,
+            registered_effect TEXT,
+            clearance TEXT,
+            first_seen TEXT NOT NULL,
+            last_seen TEXT NOT NULL,
+            PRIMARY KEY (server, tool_name)
+        )
+    """)
+
+    conn.exec_driver_sql("""
+        CREATE TABLE IF NOT EXISTS mcp_profile_attributes (
+            server TEXT NOT NULL,
+            tool_name TEXT NOT NULL,
+            attr_type TEXT NOT NULL,
+            attr_value TEXT NOT NULL,
+            PRIMARY KEY (server, tool_name, attr_type, attr_value)
+        )
+    """)
+
+    # ── Normalized budget counters ──
+    # One row per (session, dimension, key). Replaces the by_* maps that a JSON
+    # budget blob would otherwise carry; the atomic scalars are columns on
+    # session_state.
+    conn.exec_driver_sql("""
+        CREATE TABLE IF NOT EXISTS budget_counters (
+            session_id TEXT NOT NULL,
+            dimension TEXT NOT NULL,
+            key TEXT NOT NULL,
+            count INTEGER NOT NULL,
+            PRIMARY KEY (session_id, dimension, key)
+        )
+    """)
+
+    # ── Normalized taint ledger ──
+    # One row per taint entry; ordinal preserves append order so the bounded
+    # ring-buffer semantics survive a reload.
+    conn.exec_driver_sql("""
+        CREATE TABLE IF NOT EXISTS taint_entries (
+            session_id TEXT NOT NULL,
+            ordinal INTEGER NOT NULL,
+            event_id TEXT NOT NULL,
+            source_event_key TEXT NOT NULL,
+            clearance TEXT NOT NULL,
+            source TEXT NOT NULL,
+            payload_pointer TEXT NOT NULL,
+            PRIMARY KEY (session_id, ordinal)
+        )
+    """)
+
+    conn.exec_driver_sql("""
+        CREATE TABLE IF NOT EXISTS drift_baselines (
+            agent_model TEXT NOT NULL,
+            repo TEXT NOT NULL,
+            phase_counts_json TEXT NOT NULL,
+            total_events INTEGER NOT NULL,
+            updated_at TEXT NOT NULL,
+            PRIMARY KEY (agent_model, repo)
+        )
+    """)
+
+    conn.exec_driver_sql("""
+        CREATE TABLE IF NOT EXISTS content_hashes (
+            repo TEXT NOT NULL,
+            file_path TEXT NOT NULL,
+            sha256 TEXT NOT NULL,
+            updated_at TEXT NOT NULL,
+            updated_by_session TEXT,
+            PRIMARY KEY (repo, file_path)
+        )
+    """)
+
+    conn.exec_driver_sql("""
+        CREATE TABLE IF NOT EXISTS session_summaries (
+            session_id TEXT PRIMARY KEY,
+            repo TEXT,
+            agent_model TEXT,
+            started_at TEXT NOT NULL,
+            ended_at TEXT,
+            total_events INTEGER,
+            dropped_events INTEGER DEFAULT 0,
+            budget_snapshot_json TEXT,
+            recommendation_counts_json TEXT,
+            drift_max REAL
+        )
+    """)
+
+    conn.exec_driver_sql("""
+        CREATE TABLE IF NOT EXISTS gate_endpoints (
+            session_id TEXT PRIMARY KEY,
+            sock_path TEXT NOT NULL,
+            pid INTEGER NOT NULL,
+            registered_at TEXT NOT NULL DEFAULT (datetime('now'))
+        )
+    """)
+
+
+def downgrade() -> None:
+    op.drop_table("gate_endpoints")
+    op.drop_table("session_summaries")
+    op.drop_table("content_hashes")
+    op.drop_table("drift_baselines")
+    op.drop_table("taint_entries")
+    op.drop_table("budget_counters")
+    op.drop_table("mcp_profile_attributes")
+    op.drop_table("mcp_profiles")
+    op.drop_table("processed_events")
+    op.drop_table("session_state")

traceforge/migrations/versions/__init__.py

@@ -0,0 +1,4 @@
+"""Alembic migration versions for traceforge system storage."""
+
+# This must be updated whenever a new migration is added.
+LATEST_REVISION = "0001_initial"

traceforge/models.py

@@ -0,0 +1,29 @@
+"""Base Pydantic model classes for all traceforge models.
+
+Defines two base classes that encode structural policies:
+
+- ``StrictModel``: Rejects unknown fields (``extra="forbid"``).
+  Used for all configuration and mapping models where typos must be caught.
+
+- ``FrozenModel``: Immutable after construction (``frozen=True``).
+  Used for domain value objects that flow through the pipeline.
+
+All traceforge models inherit from one of these — never from raw ``BaseModel``.
+This eliminates per-class ``model_config`` repetition and enforces consistency.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict
+
+
+class StrictModel(BaseModel):
+    """Base for configuration / schema models. Rejects unknown fields."""
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class FrozenModel(BaseModel):
+    """Base for immutable domain objects. Cannot be mutated after construction."""
+
+    model_config = ConfigDict(frozen=True)

traceforge/parsers/__init__.py

@@ -0,0 +1,21 @@
+"""Pre-parsers that convert non-JSONL agent logs into structured event dicts."""
+
+from traceforge.parsers.aider import AiderPreParser
+from traceforge.parsers.base import (
+    Block,
+    MarkdownPreParser,
+    node_text,
+    strip_blockquote_markers,
+    try_parse_json,
+)
+from traceforge.parsers.copilot import CopilotPreParser
+
+__all__ = [
+    "AiderPreParser",
+    "Block",
+    "CopilotPreParser",
+    "MarkdownPreParser",
+    "node_text",
+    "strip_blockquote_markers",
+    "try_parse_json",
+]