flock-core 0.5.0b53__py3-none-any.whl → 0.5.0b54__py3-none-any.whl

flock/logging/auto_trace.py

@@ -0,0 +1,159 @@
+"""Metaclass for automatic method tracing via OpenTelemetry."""
+
+from __future__ import annotations
+
+import json
+import os
+
+from flock.logging.trace_and_logged import _trace_filter_config, traced_and_logged
+
+
+# Check if auto-tracing is enabled via environment variable
+ENABLE_AUTO_TRACE = os.getenv("FLOCK_AUTO_TRACE", "true").lower() in {"true", "1", "yes", "on"}
+
+
+# Parse trace filter configuration from environment variables
+def _parse_trace_filters():
+    """Parse FLOCK_TRACE_SERVICES and FLOCK_TRACE_IGNORE from environment."""
+    # Parse FLOCK_TRACE_SERVICES (whitelist)
+    services_env = os.getenv("FLOCK_TRACE_SERVICES", "")
+    if services_env:
+        try:
+            services_list = json.loads(services_env)
+            if isinstance(services_list, list):
+                # Store as lowercase set for case-insensitive matching
+                _trace_filter_config.services = {
+                    s.lower() for s in services_list if isinstance(s, str)
+                }
+        except (json.JSONDecodeError, ValueError):
+            print(f"Warning: Invalid FLOCK_TRACE_SERVICES format: {services_env}")
+
+    # Parse FLOCK_TRACE_IGNORE (blacklist)
+    ignore_env = os.getenv("FLOCK_TRACE_IGNORE", "")
+    if ignore_env:
+        try:
+            ignore_list = json.loads(ignore_env)
+            if isinstance(ignore_list, list):
+                _trace_filter_config.ignore_operations = {
+                    op for op in ignore_list if isinstance(op, str)
+                }
+        except (json.JSONDecodeError, ValueError):
+            print(f"Warning: Invalid FLOCK_TRACE_IGNORE format: {ignore_env}")
+
+
+# Auto-configure logging and telemetry when auto-tracing is enabled
+if ENABLE_AUTO_TRACE:
+    # Configure trace filters first
+    _parse_trace_filters()
+    from flock.logging.logging import configure_logging
+    from flock.logging.telemetry import TelemetryConfig
+
+    # Configure logging to DEBUG
+    configure_logging(
+        flock_level="DEBUG",
+        external_level="WARNING",
+        specific_levels={
+            "tools": "DEBUG",
+            "agent": "DEBUG",
+            "flock": "DEBUG",
+        },
+    )
+
+    # Initialize telemetry for OTEL trace context
+    # Only enable exporters if explicitly configured via env vars
+    enable_file_export = os.getenv("FLOCK_TRACE_FILE", "false").lower() in {
+        "true",
+        "1",
+        "yes",
+        "on",
+    }
+    enable_otlp_export = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT") is not None
+
+    # Parse TTL (Time To Live) for trace cleanup
+    trace_ttl_days = None
+    ttl_env = os.getenv("FLOCK_TRACE_TTL_DAYS", "")
+    if ttl_env:
+        try:
+            trace_ttl_days = int(ttl_env)
+        except ValueError:
+            print(f"Warning: Invalid FLOCK_TRACE_TTL_DAYS value: {ttl_env}")
+
+    telemetry_config = TelemetryConfig(
+        service_name="flock-auto-trace",
+        enable_jaeger=False,
+        enable_file=False,  # Disable file export, use DuckDB instead
+        enable_sql=False,
+        enable_duckdb=enable_file_export,  # Use DuckDB when file export is enabled
+        enable_otlp=enable_otlp_export,
+        otlp_endpoint=os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317"),
+        local_logging_dir=".flock",
+        duckdb_name="traces.duckdb",
+        duckdb_ttl_days=trace_ttl_days,
+    )
+    telemetry_config.setup_tracing()
+
+
+class AutoTracedMeta(type):
+    """Metaclass that automatically applies @traced_and_logged to all public methods.
+
+    This enables automatic OpenTelemetry span creation and debug logging for all
+    method calls on classes using this metaclass.
+
+    Control via environment variable:
+        FLOCK_AUTO_TRACE=true   - Enable auto-tracing (default)
+        FLOCK_AUTO_TRACE=false  - Disable auto-tracing
+
+    Example:
+        class Agent(metaclass=AutoTracedMeta):
+            def execute(self, ctx, artifacts):
+                # Automatically traced and logged
+                ...
+    """
+
+    def __new__(mcs, name, bases, namespace, **kwargs):
+        """Create a new class with auto-traced methods."""
+        if not ENABLE_AUTO_TRACE:
+            # If auto-tracing is disabled, return the class unchanged
+            return super().__new__(mcs, name, bases, namespace, **kwargs)
+
+        # Apply @traced_and_logged to all public methods
+        for attr_name, attr_value in list(namespace.items()):
+            # Skip private methods (starting with _)
+            if attr_name.startswith("_"):
+                continue
+
+            # Skip non-callables
+            if not callable(attr_value):
+                continue
+
+            # Skip if already traced
+            if getattr(attr_value, "_traced", False):
+                continue
+
+            # Skip if explicitly marked to skip tracing
+            if getattr(attr_value, "_skip_trace", False):
+                continue
+
+            # Apply the decorator
+            traced_func = traced_and_logged(attr_value)
+            traced_func._traced = True  # Mark as traced to avoid double-wrapping
+            namespace[attr_name] = traced_func
+
+        return super().__new__(mcs, name, bases, namespace, **kwargs)
+
+
+def skip_trace(func):
+    """Decorator to mark a method to skip auto-tracing.
+
+    Use this for methods that are called very frequently or are not
+    interesting for debugging purposes.
+
+    Example:
+        class Agent(metaclass=AutoTracedMeta):
+            @skip_trace
+            def _internal_helper(self):
+                # Not traced
+                ...
+    """
+    func._skip_trace = True
+    return func

flock/logging/telemetry.py

@@ -12,6 +12,9 @@ from opentelemetry.sdk.trace.export import SimpleSpanProcessor
 from flock.logging.span_middleware.baggage_span_processor import (
     BaggageAttributeSpanProcessor,
 )
+from flock.logging.telemetry_exporter.duckdb_exporter import (
+    DuckDBSpanExporter,
+)
 
 # with workflow.unsafe.imports_passed_through():
 from flock.logging.telemetry_exporter.file_exporter import (
@@ -40,9 +43,12 @@ class TelemetryConfig:
         local_logging_dir: str | None = None,
         file_export_name: str | None = None,
         sqlite_db_name: str | None = None,
+        duckdb_name: str | None = None,
+        duckdb_ttl_days: int | None = None,
         enable_jaeger: bool = True,
         enable_file: bool = True,
         enable_sql: bool = True,
+        enable_duckdb: bool = True,
         enable_otlp: bool = True,
         otlp_protocol: str = "grpc",
         otlp_endpoint: str = "http://localhost:4317",
@@ -53,6 +59,7 @@ class TelemetryConfig:
         :param jaeger_endpoint: The Jaeger collector gRPC endpoint (e.g., "localhost:14250").
         :param file_export_path: If provided, spans will be written to this file.
         :param sqlite_db_path: If provided, spans will be stored in this SQLite DB.
+        :param duckdb_ttl_days: Delete traces older than this many days (default: None = keep forever).
         :param batch_processor_options: Dict of options for BatchSpanProcessor (e.g., {"max_export_batch_size": 10}).
         """
         self.service_name = service_name
@@ -60,11 +67,14 @@ class TelemetryConfig:
         self.jaeger_transport = jaeger_transport
         self.file_export_name = file_export_name
         self.sqlite_db_name = sqlite_db_name
+        self.duckdb_name = duckdb_name
+        self.duckdb_ttl_days = duckdb_ttl_days
         self.local_logging_dir = local_logging_dir
         self.batch_processor_options = batch_processor_options or {}
         self.enable_jaeger = enable_jaeger
         self.enable_file = enable_file
         self.enable_sql = enable_sql
+        self.enable_duckdb = enable_duckdb
         self.enable_otlp = enable_otlp
         self.otlp_protocol = otlp_protocol
         self.otlp_endpoint = otlp_endpoint
@@ -166,6 +176,13 @@ class TelemetryConfig:
             sqlite_exporter = SqliteTelemetryExporter(self.local_logging_dir, self.sqlite_db_name)
             span_processors.append(SimpleSpanProcessor(sqlite_exporter))
 
+        # If a DuckDB database path is provided, add the DuckDB exporter.
+        if self.duckdb_name and self.enable_duckdb:
+            duckdb_exporter = DuckDBSpanExporter(
+                self.local_logging_dir, self.duckdb_name, ttl_days=self.duckdb_ttl_days
+            )
+            span_processors.append(SimpleSpanProcessor(duckdb_exporter))
+
         # Register all span processors with the provider.
         for processor in span_processors:
             provider.add_span_processor(processor)

flock/logging/telemetry_exporter/duckdb_exporter.py

@@ -0,0 +1,216 @@
+"""DuckDB exporter for OpenTelemetry spans - optimized for analytical queries."""
+
+import json
+from pathlib import Path
+
+import duckdb
+from opentelemetry.sdk.trace.export import SpanExportResult
+from opentelemetry.trace import Status, StatusCode
+
+from flock.logging.telemetry_exporter.base_exporter import TelemetryExporter
+
+
+class DuckDBSpanExporter(TelemetryExporter):
+    """Export spans to DuckDB for fast analytical queries.
+
+    DuckDB is a columnar analytical database optimized for OLAP workloads,
+    making it 10-100x faster than SQLite for trace analytics like:
+    - Aggregations (avg/p95/p99 duration)
+    - Time-range queries
+    - Service/operation filtering
+    - Complex analytical queries
+
+    The database is a single file with zero configuration required.
+    """
+
+    def __init__(self, dir: str, db_name: str = "traces.duckdb", ttl_days: int | None = None):
+        """Initialize the DuckDB exporter.
+
+        Args:
+            dir: Directory where the database file will be created
+            db_name: Name of the DuckDB file (default: traces.duckdb)
+            ttl_days: Delete traces older than this many days (default: None = keep forever)
+        """
+        super().__init__()
+        self.telemetry_path = Path(dir)
+        self.telemetry_path.mkdir(parents=True, exist_ok=True)
+        self.db_path = self.telemetry_path / db_name
+        self.ttl_days = ttl_days
+
+        # Initialize database and create schema
+        self._init_database()
+
+    def _init_database(self):
+        """Create the spans table if it doesn't exist."""
+        with duckdb.connect(str(self.db_path)) as conn:
+            conn.execute("""
+                CREATE TABLE IF NOT EXISTS spans (
+                    trace_id VARCHAR NOT NULL,
+                    span_id VARCHAR PRIMARY KEY,
+                    parent_id VARCHAR,
+                    name VARCHAR NOT NULL,
+                    service VARCHAR,
+                    operation VARCHAR,
+                    kind VARCHAR,
+                    start_time BIGINT NOT NULL,
+                    end_time BIGINT NOT NULL,
+                    duration_ms DOUBLE NOT NULL,
+                    status_code VARCHAR NOT NULL,
+                    status_description VARCHAR,
+                    attributes JSON,
+                    events JSON,
+                    links JSON,
+                    resource JSON,
+                    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                )
+            """)
+
+            # Create indexes for common query patterns
+            conn.execute("CREATE INDEX IF NOT EXISTS idx_trace_id ON spans(trace_id)")
+            conn.execute("CREATE INDEX IF NOT EXISTS idx_service ON spans(service)")
+            conn.execute("CREATE INDEX IF NOT EXISTS idx_start_time ON spans(start_time)")
+            conn.execute("CREATE INDEX IF NOT EXISTS idx_name ON spans(name)")
+            conn.execute("CREATE INDEX IF NOT EXISTS idx_created_at ON spans(created_at)")
+
+        # Cleanup old traces if TTL is configured
+        if self.ttl_days is not None:
+            self._cleanup_old_traces()
+
+    def _cleanup_old_traces(self):
+        """Delete traces older than TTL_DAYS.
+
+        This runs on exporter initialization to keep the database size manageable.
+        """
+        if self.ttl_days is None:
+            return
+
+        try:
+            with duckdb.connect(str(self.db_path)) as conn:
+                # Delete spans older than TTL
+                result = conn.execute(
+                    """
+                    DELETE FROM spans
+                    WHERE created_at < CURRENT_TIMESTAMP - INTERVAL ? DAYS
+                """,
+                    (self.ttl_days,),
+                )
+
+                deleted_count = result.fetchall()[0][0] if result else 0
+
+                if deleted_count > 0:
+                    print(
+                        f"[DuckDB TTL] Deleted {deleted_count} spans older than {self.ttl_days} days"
+                    )
+
+        except Exception as e:
+            print(f"[DuckDB TTL] Error cleaning up old traces: {e}")
+
+    def _span_to_record(self, span):
+        """Convert a ReadableSpan to a database record."""
+        context = span.get_span_context()
+        status = span.status or Status(StatusCode.UNSET)
+
+        # Extract service and operation from span name
+        # Format: "ServiceName.operation_name"
+        parts = span.name.split(".", 1)
+        service = parts[0] if len(parts) > 0 else "unknown"
+        operation = parts[1] if len(parts) > 1 else span.name
+
+        # Calculate duration in milliseconds
+        duration_ms = (span.end_time - span.start_time) / 1_000_000
+
+        # Serialize complex fields to JSON
+        attributes_json = json.dumps(dict(span.attributes or {}))
+        events_json = json.dumps(
+            [
+                {
+                    "name": event.name,
+                    "timestamp": event.timestamp,
+                    "attributes": dict(event.attributes or {}),
+                }
+                for event in span.events
+            ]
+        )
+        links_json = json.dumps(
+            [
+                {
+                    "context": {
+                        "trace_id": format(link.context.trace_id, "032x"),
+                        "span_id": format(link.context.span_id, "016x"),
+                    },
+                    "attributes": dict(link.attributes or {}),
+                }
+                for link in span.links
+            ]
+        )
+        resource_json = json.dumps(dict(span.resource.attributes.items()))
+
+        # Get parent span ID if exists
+        parent_id = None
+        if span.parent and span.parent.span_id != 0:
+            parent_id = format(span.parent.span_id, "016x")
+
+        return {
+            "trace_id": format(context.trace_id, "032x"),
+            "span_id": format(context.span_id, "016x"),
+            "parent_id": parent_id,
+            "name": span.name,
+            "service": service,
+            "operation": operation,
+            "kind": span.kind.name if span.kind else None,
+            "start_time": span.start_time,
+            "end_time": span.end_time,
+            "duration_ms": duration_ms,
+            "status_code": status.status_code.name,
+            "status_description": status.description,
+            "attributes": attributes_json,
+            "events": events_json,
+            "links": links_json,
+            "resource": resource_json,
+        }
+
+    def export(self, spans):
+        """Export spans to DuckDB."""
+        try:
+            with duckdb.connect(str(self.db_path)) as conn:
+                for span in spans:
+                    record = self._span_to_record(span)
+
+                    # Insert span record
+                    conn.execute(
+                        """
+                        INSERT OR REPLACE INTO spans (
+                            trace_id, span_id, parent_id, name, service, operation,
+                            kind, start_time, end_time, duration_ms,
+                            status_code, status_description,
+                            attributes, events, links, resource
+                        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    """,
+                        (
+                            record["trace_id"],
+                            record["span_id"],
+                            record["parent_id"],
+                            record["name"],
+                            record["service"],
+                            record["operation"],
+                            record["kind"],
+                            record["start_time"],
+                            record["end_time"],
+                            record["duration_ms"],
+                            record["status_code"],
+                            record["status_description"],
+                            record["attributes"],
+                            record["events"],
+                            record["links"],
+                            record["resource"],
+                        ),
+                    )
+
+            return SpanExportResult.SUCCESS
+        except Exception as e:
+            print(f"Error exporting spans to DuckDB: {e}")
+            return SpanExportResult.FAILURE
+
+    def shutdown(self) -> None:
+        """Cleanup resources."""
+        # DuckDB connections are managed per-transaction, no cleanup needed

flock/logging/telemetry_exporter/file_exporter.py

@@ -29,7 +29,7 @@ class FileSpanExporter(TelemetryExporter):
         context = span.get_span_context()
         status = span.status or Status(StatusCode.UNSET)
 
-        return {
+        result = {
             "name": span.name,
             "context": {
                 "trace_id": format(context.trace_id, "032x"),
@@ -66,6 +66,12 @@ class FileSpanExporter(TelemetryExporter):
             "resource": dict(span.resource.attributes.items()),
         }
 
+        # Add parent_id if this span has a parent
+        if span.parent and span.parent.span_id != 0:
+            result["parent_id"] = format(span.parent.span_id, "016x")
+
+        return result
+
     def export(self, spans):
         """Write spans to a log file."""
         try: