traceforge-toolkit 0.1.0__py3-none-any.whl

traceforge/cli/watch.py

@@ -0,0 +1,297 @@
+"""Watch command — primary daemon that auto-detects, watches sources, governs, and sinks."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import signal
+import sys
+from pathlib import Path
+
+import click
+import yaml
+
+from traceforge.cli.runner import (
+    ResolvedPipeline,
+    load_mapping_path,
+    resolve_pipelines,
+    watch_directory,
+    watch_jsonl_file,
+)
+from traceforge.cli.score import ScoreServer
+from traceforge.sources.auto_detect import detect_frameworks
+
+logger = logging.getLogger(__name__)
+
+
+@click.command()
+@click.option("--config", "config_path", type=click.Path(exists=True), default=None)
+@click.option(
+    "--frameworks",
+    default=None,
+    help="Comma-separated frameworks to watch (default: all detected).",
+)
+@click.option("--once", is_flag=True, help="Process existing files then exit (no watching).")
+@click.option("--no-score", is_flag=True, help="Don't start the Score API server.")
+@click.option(
+    "--log-level", default="INFO", type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR"])
+)
+def watch(
+    config_path: str | None, frameworks: str | None, once: bool, no_score: bool, log_level: str
+) -> None:
+    """Watch detected frameworks, run governance pipeline, emit to sinks."""
+    logging.basicConfig(
+        level=getattr(logging, log_level),
+        format="%(asctime)s %(levelname)-5s %(name)s: %(message)s",
+        datefmt="%H:%M:%S",
+    )
+
+    config = _load_config(config_path)
+    fw_list = [f.strip() for f in frameworks.split(",")] if frameworks else None
+
+    # Auto-detect
+    detected = detect_frameworks(fw_list)
+    if not detected:
+        click.echo("No frameworks detected. Nothing to watch.")
+        sys.exit(0)
+
+    click.echo(f"Detected {len(detected)} framework(s): {', '.join(d.name for d in detected)}")
+
+    # Resolve pipelines
+    pipelines = resolve_pipelines(detected)
+    if not pipelines:
+        click.echo("No pipelines could be resolved.")
+        sys.exit(0)
+
+    # Initialize governance
+    from traceforge.cli.factory import create_default_pipeline
+    from traceforge.governance.persistence import SystemStore
+
+    db_path = Path.home() / ".traceforge" / "system.db"
+    db_path.parent.mkdir(parents=True, exist_ok=True)
+    store = SystemStore(db_path)
+
+    # Load policy from config if available
+    policy = None
+    if config and config.get("governance", {}).get("tool_preflight_gate"):
+        from traceforge.sdk.gate_policy import GatePolicy
+
+        dotted = config["governance"]["tool_preflight_gate"]
+        from traceforge.governance.pipeline import _import_dotted
+
+        gate_fn = _import_dotted(dotted)
+        policy = GatePolicy().preflight(gate_fn)
+
+    pipeline = create_default_pipeline(store, policy=policy)
+
+    # Start Gate IPC server (for CLI-based frameworks: Claude Code, Copilot CLI, etc.)
+    from traceforge.gate.server import GateServer
+
+    gate_server = GateServer(pipeline)
+    gate_server.start()
+    click.echo(f"Gate IPC server listening on {gate_server.sock_path}")
+
+    # Register a default session so CLI clients can find us without knowing session_id upfront.
+    # Detected framework sessions are also registered individually.
+    gate_server.register_session("_default")
+    for p in pipelines:
+        gate_server.register_session(p.name)
+
+    # Start Score API
+    score_server: ScoreServer | None = None
+    if not no_score:
+        listen = (
+            config.get("score", {}).get("listen", "localhost:7331") if config else "localhost:7331"
+        )
+        score_server = ScoreServer(pipeline, listen=listen)
+        score_server.start_background()
+
+    # Run async event loop
+    try:
+        if once:
+            asyncio.run(_run_once(pipelines, pipeline, store))
+        else:
+            asyncio.run(_run_watch(pipelines, pipeline, store))
+    except KeyboardInterrupt:
+        click.echo("\nShutting down...")
+    finally:
+        from traceforge.gate.registry import unregister_pid
+
+        gate_server.stop()
+        unregister_pid()
+        if score_server:
+            score_server.stop()
+        store.close()
+
+
+async def _run_watch(
+    pipelines: list[ResolvedPipeline],
+    governance: "GovernancePipeline",
+    store: "SystemStore",
+) -> None:
+    """Run all pipeline watchers concurrently."""
+    tasks = []
+    for p in pipelines:
+        tasks.append(asyncio.create_task(_watch_pipeline(p, governance), name=f"watch-{p.name}"))
+
+    # Handle graceful shutdown
+    loop = asyncio.get_event_loop()
+    stop_event = asyncio.Event()
+
+    def _signal_handler():
+        stop_event.set()
+
+    for sig in (signal.SIGINT, signal.SIGTERM):
+        try:
+            loop.add_signal_handler(sig, _signal_handler)
+        except NotImplementedError:
+            # Windows doesn't support add_signal_handler
+            pass
+
+    click.echo(f"Watching {len(pipelines)} pipeline(s). Press Ctrl+C to stop.")
+
+    # Wait until stop signal or all tasks complete
+    done, pending = await asyncio.wait(
+        [*tasks, asyncio.create_task(stop_event.wait())],
+        return_when=asyncio.FIRST_COMPLETED,
+    )
+
+    # Cancel remaining tasks
+    for t in pending:
+        t.cancel()
+    await asyncio.gather(*pending, return_exceptions=True)
+
+
+async def _run_once(
+    pipelines: list[ResolvedPipeline],
+    governance: "GovernancePipeline",
+    store: "SystemStore",
+) -> None:
+    """Process existing files once and exit."""
+    for p in pipelines:
+        await _process_pipeline_once(p, governance)
+
+
+async def _watch_pipeline(pipeline: ResolvedPipeline, governance: "GovernancePipeline") -> None:
+    """Watch a single pipeline's source and feed events through the unified pipeline."""
+    logger.info("Starting watcher for %s at %s", pipeline.name, pipeline.source_path)
+
+    if not (pipeline.source_path.is_dir() or pipeline.source_path.is_file()):
+        logger.warning("Source path does not exist: %s", pipeline.source_path)
+        return
+
+    adapter, event_pipeline = _build_pipeline_runtime(pipeline, governance)
+    try:
+        if pipeline.source_path.is_dir():
+            # Watch directory for JSONL files
+            pattern = "*.jsonl" if pipeline.name != "continue" else "*.json"
+            async for _file_path, line in watch_directory(pipeline.source_path, pattern=pattern):
+                await _feed_line(line, adapter, event_pipeline)
+        else:
+            # Watch single file
+            async for line in watch_jsonl_file(pipeline.source_path, start_at="end"):
+                await _feed_line(line, adapter, event_pipeline)
+    finally:
+        # Emit any buffered (unpaired tool-start) events and release sink resources.
+        await event_pipeline.close()
+
+
+async def _process_pipeline_once(
+    pipeline: ResolvedPipeline, governance: "GovernancePipeline"
+) -> None:
+    """Process existing content in a pipeline's source once (no watching)."""
+    logger.info("Processing %s at %s", pipeline.name, pipeline.source_path)
+
+    if not (pipeline.source_path.is_dir() or pipeline.source_path.is_file()):
+        return
+
+    adapter, event_pipeline = _build_pipeline_runtime(pipeline, governance)
+    try:
+        if pipeline.source_path.is_dir():
+            pattern = "*.jsonl" if pipeline.name != "continue" else "*.json"
+            for f in pipeline.source_path.rglob(pattern):
+                if f.is_file():
+                    for line in f.read_text(encoding="utf-8").splitlines():
+                        stripped = line.strip()
+                        if stripped:
+                            await _feed_line(stripped, adapter, event_pipeline)
+        else:
+            for line in pipeline.source_path.read_text(encoding="utf-8").splitlines():
+                stripped = line.strip()
+                if stripped:
+                    await _feed_line(stripped, adapter, event_pipeline)
+    finally:
+        await event_pipeline.close()
+
+
+def _build_sinks(pipeline: ResolvedPipeline) -> list:
+    """Instantiate the configured sink objects for a resolved pipeline (once).
+
+    Delegates to the shared :func:`traceforge.sinks.factory.build_sinks` so the
+    daemon and the SDK build sinks from one mapping of the full ``SinkConfig``
+    union.
+    """
+    from traceforge.sinks.factory import build_sinks
+
+    return build_sinks(pipeline.sinks)
+
+
+def _build_pipeline_runtime(pipeline: ResolvedPipeline, governance: "GovernancePipeline"):
+    """Build the ``(adapter, EventPipeline)`` runtime for a resolved pipeline.
+
+    The daemon runs the same unified pipeline as the SDK: adapt -> enrich ->
+    classify -> govern -> sinks, with ``governance`` wired in as a *stage* so every
+    emitted event carries ``metadata.governance``. Governance is one stage here, not
+    the pipeline itself. The shared governance engine also backs the Gate/Score IPC
+    servers, so session budget/drift state stays unified across observation and
+    preflight. Live ML structuring (phase + boundary labeling) is enabled by
+    default, matching the SDK ``Pipeline``.
+    """
+    from traceforge.adapters.mapped_json import MappedJsonAdapter
+    from traceforge.enricher import Enricher
+    from traceforge.pipeline import EventPipeline
+
+    mapping_path = load_mapping_path(pipeline.adapter.mapping)
+    adapter = MappedJsonAdapter.from_yaml(str(mapping_path), session_id=pipeline.name)
+    event_pipeline = EventPipeline(
+        sinks=_build_sinks(pipeline),
+        enricher=Enricher(),
+        governance=governance,
+    )
+    return adapter, event_pipeline
+
+
+async def _feed_line(line: str, adapter, event_pipeline) -> None:
+    """Parse one raw line and push each resulting event through the pipeline."""
+    try:
+        data = json.loads(line)
+    except json.JSONDecodeError:
+        logger.debug("Skipping non-JSON line")
+        return
+
+    for event in adapter.parse_dict(data):
+        await event_pipeline.push(event)
+
+
+def _load_config(config_path: str | None) -> dict | None:
+    """Load config from file path or default locations."""
+    import os
+
+    if config_path:
+        path = Path(config_path)
+    else:
+        env = os.environ.get("TRACEFORGE_CONFIG")
+        if env:
+            path = Path(env)
+        elif Path("traceforge.yaml").exists():
+            path = Path("traceforge.yaml")
+        elif (Path.home() / ".traceforge" / "config.yaml").exists():
+            path = Path.home() / ".traceforge" / "config.yaml"
+        else:
+            return None
+
+    if not path.exists():
+        return None
+
+    return yaml.safe_load(path.read_text(encoding="utf-8"))

traceforge/config/__init__.py

@@ -0,0 +1,80 @@
+"""Traceforge configuration — Pydantic Settings with hierarchical YAML config."""
+
+from traceforge.config.models import (
+    AdapterConfig,
+    BudgetConfig,
+    FileWatchSourceConfig,
+    FilePollSourceConfig,
+    GovernanceConfig,
+    HttpGateConfig,
+    HttpPollSourceConfig,
+    MappedJsonAdapterConfig,
+    OtelSpanAdapterConfig,
+    PipelineConfig,
+    ReplaySourceConfig,
+    SDKConfig,
+    SessionNamingApiConfig,
+    SessionNamingConfig,
+    SessionNamingHeuristicConfig,
+    SinkConfig,
+    SourceConfig,
+    SSESourceConfig,
+    SqliteSinkConfig,
+    SubprocessGateConfig,
+    JsonlSinkConfig,
+    S3SinkConfig,
+    ExternalGateConfig,
+    TitleApiConfig,
+    TitleConfig,
+    ActivityTitlingConfig,
+    ActivityTitlingApiConfig,
+    TraceforgeConfig,
+)
+from traceforge.config.loader import load_config, get_config, reset_config
+from traceforge.config.mappings import resolve_mapping_path, list_available_mappings
+
+__all__ = [
+    # Root config
+    "TraceforgeConfig",
+    # Source configs
+    "SourceConfig",
+    "FileWatchSourceConfig",
+    "FilePollSourceConfig",
+    "HttpPollSourceConfig",
+    "SSESourceConfig",
+    "ReplaySourceConfig",
+    # Adapter configs
+    "AdapterConfig",
+    "MappedJsonAdapterConfig",
+    "OtelSpanAdapterConfig",
+    # Sink configs
+    "SinkConfig",
+    "SqliteSinkConfig",
+    "JsonlSinkConfig",
+    "S3SinkConfig",
+    # Pipeline
+    "PipelineConfig",
+    # SDK
+    "SDKConfig",
+    # Governance
+    "GovernanceConfig",
+    "BudgetConfig",
+    "ExternalGateConfig",
+    "SubprocessGateConfig",
+    "HttpGateConfig",
+    # Title
+    "TitleConfig",
+    "TitleApiConfig",
+    "SessionNamingConfig",
+    "SessionNamingHeuristicConfig",
+    "SessionNamingApiConfig",
+    "ActivityTitlingConfig",
+    "ActivityTitlingApiConfig",
+    # Loading
+    "load_config",
+    "get_config",
+    "reset_config",
+    # Mappings
+    "resolve_mapping_path",
+    "list_available_mappings",
+]

traceforge/config/defaults.py

@@ -0,0 +1,149 @@
+"""Default config template written to ~/.traceforge/config.yaml on first access."""
+
+DEFAULT_CONFIG_YAML = """\
+# Traceforge configuration
+# Docs: https://github.com/dfinson/traceforge
+#
+# This file was auto-created on first use. Edit to customize.
+# Precedence: constructor args > env vars > ./traceforge.yaml > this file > defaults
+#
+# Environment variable override: set TRACEFORGE_CONFIG=/path/to/config.yaml
+
+log_level: INFO
+
+# ─── Auto-detection ─────────────────────────────────────────────────────────
+# On startup, traceforge scans well-known paths for installed AI coding agents.
+# Detected frameworks are watched automatically — no explicit pipeline config needed.
+auto_detect:
+  enabled: true
+  # Restrict to specific frameworks (empty = detect all):
+  # frameworks: [claude, codex, continue, cline, goose, amazonq, aider]
+
+# ─── Score API (preflight scoring endpoint) ─────────────────────────────────
+# Always available for gate integrations to ask "should this tool call proceed?"
+score:
+  enabled: true
+  listen: localhost:7331
+
+# ─── Sinks (where governance results go) ────────────────────────────────────
+# Default: SQLite (queryable history) + Console (real-time alerts)
+# Uncomment additional sinks as needed.
+#
+# sinks:
+#   - type: sqlite
+#     path: ~/.traceforge/traceforge.db
+#   - type: console
+#     filter: [warn, deny, escalate]
+#   - type: jsonl
+#     path: ~/.traceforge/output/{session_id}.jsonl
+#   - type: webhook
+#     url: https://hooks.slack.com/services/...
+#     filter: [deny, escalate]
+#   - type: otel
+#     endpoint: http://localhost:4318/v1/traces
+#     service_name: traceforge
+
+# ─── Governance ─────────────────────────────────────────────────────────────
+governance:
+  pii_scanning: true
+  integrity_verification: true
+  # budget:
+  #   max_tool_calls: 200
+  #   max_by_effect:
+  #     destructive: 10
+  #
+  # ── External preflight gate (out-of-process tool-call decider) ──────────────
+  # Delegate the ALLOW/DENY decision to an external Policy Decision Point instead
+  # of an in-process Python callback. Fail-CLOSED by default (any error/timeout/
+  # non-2xx/bad output => DENY). Mutually exclusive with `tool_preflight_gate`.
+  # Choose ONE of the two forms below.
+  #
+  # HTTP PDP (recommended — e.g. an OPA REST server):
+  # preflight_gate:
+  #   type: http
+  #   endpoint: http://localhost:8181/v1/data/traceforge/verdict
+  #   timeout: 2.0
+  #   fail_open: false        # false = fail-closed (DENY on error). Keep false.
+  #   headers:                # optional, e.g. auth tokens
+  #     Authorization: "Bearer ${PDP_TOKEN}"
+  #   max_input_bytes: 65536  # per-string cap on tool input sent to the decider
+  #
+  # Subprocess decider (portable / air-gapped — e.g. `opa eval`):
+  # preflight_gate:
+  #   type: subprocess
+  #   command: "opa eval -I -f raw data.traceforge.verdict"
+  #   timeout: 10.0
+  #   fail_open: false
+  #   max_input_bytes: 65536
+
+# ─── Phase tracker (session-level phase segmentation) ───────────────────────
+# Smooths per-event activity labels into stable workflow phases via a debounced
+# majority vote. Defaults are literature-seeded; recalibrate with the
+# phase-tracker-window-sweep experiment.
+# phase_tracker:
+#   enabled: true
+#   window_size: 3      # sliding window whose mode is the current phase
+#   debounce: 2         # consecutive events before a boundary commits
+#   phase_root_depth: 1 # dot-path depth grouped into the boundary root
+
+# ─── SDK configuration (in-process push mode) ───────────────────────────────
+sdk:
+  batch_size: 64
+  flush_interval: 5.0
+  max_queue_size: 10000
+
+# ─── Titling ────────────────────────────────────────────────────────────────
+# Two title surfaces, each with a free/offline floor and an opt-in LLM API tier
+# that engages only when the provider's API key is present in the environment
+# (e.g. export OPENAI_API_KEY=...). No key => it silently stays on the offline
+# floor. The floor is emitted immediately; the API upgrade (when configured and
+# keyed) is applied later, off the hot path, so live emission is never blocked.
+#
+#   * session_naming  — the title from the first substantive user message; floor
+#                       is a free heuristic over the user's own words.
+#   * activity_titling — the activity/step (span) titles; floor is the packaged,
+#                       offline ONNX model shipped with every install.
+#
+# title:
+#   session_naming:
+#     strategy: heuristic     # heuristic | api
+#     heuristic:
+#       method: hybrid        # clip | imperative | keyphrase | hybrid
+#       max_words: 8
+#       max_chars: 60
+#     api:
+#       model: gpt-4o-mini    # any LiteLLM model, e.g. anthropic/claude-3-5-haiku,
+#                             #   azure/<deployment>, ollama/llama3 (+ api_base)
+#       api_base: null        # for azure / ollama / vllm / openai-compatible
+#       api_key_env: null     # override which env var holds the key
+#       timeout: 10
+#       max_tokens: 24
+#   activity_titling:
+#     strategy: model         # model | api  (model = packaged offline ONNX titler)
+#     api:
+#       model: gpt-4o-mini    # any LiteLLM model string (as above)
+#       api_base: null        # for azure / ollama / vllm / openai-compatible
+#       api_key_env: null     # override which env var holds the key
+#       timeout: 10
+#       max_tokens: 256       # one call returns the activity + all step titles
+
+# ─── Additional mapping directories ────────────────────────────────────────
+mappings_dirs:
+  - ~/.traceforge/mappings
+
+# ─── Explicit pipelines (advanced — overrides auto-detect) ──────────────────
+# Define explicit source → adapter → sink pipelines when auto-detect isn't enough.
+#
+# pipelines:
+#   - name: claude-local
+#     source:
+#       type: file_watch
+#       path: ~/.claude/projects/-Users-me-myproject/latest.jsonl
+#       start_at: end
+#     adapter:
+#       type: mapped_json
+#       mapping: claude
+#     sinks:
+#       - type: sqlite
+#         path: ~/.traceforge/traces.db
+"""