synth-ai 0.2.13.dev2__py3-none-any.whl → 0.2.16__py3-none-any.whl

synth_ai/task/trace_correlation_helpers.py

@@ -0,0 +1,328 @@
+"""Helpers for trace correlation ID extraction and inclusion in task apps.
+
+This module provides utilities for task apps to:
+1. Extract trace_correlation_id from rollout requests
+2. Include trace_correlation_id in rollout responses (3 required locations)
+
+See monorepo/trace_creation_and_judgement.txt "Fatal Guards" section for requirements.
+"""
+
+import importlib
+import logging
+from typing import Any, cast
+from urllib.parse import parse_qs, urlparse
+
+logger = logging.getLogger(__name__)
+
+
+def extract_trace_correlation_id(
+    policy_config: dict[str, Any],
+    inference_url: str | None = None,
+    mode: Any = None
+) -> str | None:
+    """
+    Extract trace_correlation_id from policy config or inference URL.
+    
+    This is the standardized method for all task apps to extract the correlation ID
+    that the RL trainer generates and passes to the task app.
+    
+    Args:
+        policy_config: Policy configuration dict from RolloutRequest.policy.config
+        inference_url: Inference URL (optional, used as fallback)
+        mode: RolloutMode or string ("rl" or "eval"). Controls warning behavior - 
+              warnings only logged for RL mode, not EVAL mode.
+        
+    Returns:
+        trace_correlation_id if found, None otherwise
+        
+    Extraction order:
+        1. policy_config["trace_correlation_id"] (preferred)
+        2. policy_config["trace"] (legacy fallback)
+        3. URL query param ?cid=... (fallback)
+        4. URL query param ?trace_correlation_id=... (fallback)
+    """
+    # Try policy_config first (preferred method)
+    candidates: list[Any] = [
+        policy_config.get("trace_correlation_id"),
+        policy_config.get("trace"),
+    ]
+    
+    logger.debug(
+        "extract_trace_correlation_id: policy_cfg keys=%s candidates=%s",
+        sorted(policy_config.keys()),
+        candidates,
+    )
+    
+    for candidate in candidates:
+        if isinstance(candidate, str):
+            stripped = candidate.strip()
+            if stripped:
+                logger.info(
+                    "extract_trace_correlation_id: extracted from policy_config=%s",
+                    stripped
+                )
+                return stripped
+    
+    # Determine if we're in EVAL mode (trace_correlation_id not required for eval)
+    rollout_mode_cls: Any | None = None
+    try:
+        contracts_module = importlib.import_module("synth_ai.task.contracts")
+        rollout_mode_cls = getattr(contracts_module, "RolloutMode", None)
+    except Exception:
+        rollout_mode_cls = None
+
+    is_eval_mode = False
+    if rollout_mode_cls is not None:
+        try:
+            is_eval_mode = (
+                mode == "eval"
+                or mode == rollout_mode_cls.EVAL
+                or getattr(mode, "value", None) == "eval"
+            )
+        except Exception:
+            is_eval_mode = mode == "eval"
+    else:
+        is_eval_mode = mode == "eval" or getattr(mode, "value", None) == "eval"
+    
+    # Fallback: try to extract from inference_url query params
+    if not inference_url or not isinstance(inference_url, str):
+        if is_eval_mode:
+            logger.debug(
+                "extract_trace_correlation_id: no correlation ID found in policy_config "
+                "and no inference_url provided (EVAL mode - expected)"
+            )
+        else:
+            logger.warning(
+                "extract_trace_correlation_id: no correlation ID found in policy_config "
+                "and no inference_url provided"
+            )
+        return None
+    
+    try:
+        parsed = urlparse(inference_url)
+        query_params = cast(dict[str, list[str]], parse_qs(parsed.query or ""))
+        # Try multiple possible query param names
+        for param_name in ["cid", "trace_correlation_id", "trace"]:
+            values = query_params.get(param_name)
+            if not values:
+                continue
+            for value in values:
+                if isinstance(value, str) and value.strip():
+                    correlation_id = value.strip()
+                    logger.info(
+                        "extract_trace_correlation_id: extracted from URL param %s=%s",
+                        param_name,
+                        correlation_id,
+                    )
+                    return correlation_id
+    except Exception as e:
+        logger.warning(
+            "extract_trace_correlation_id: failed to parse inference_url=%s error=%s",
+            inference_url,
+            e,
+        )
+    
+    if is_eval_mode:
+        logger.debug(
+            "extract_trace_correlation_id: no trace_correlation_id found in "
+            "policy_config or inference_url=%s (EVAL mode - expected)",
+            inference_url,
+        )
+    else:
+        logger.warning(
+            "extract_trace_correlation_id: no trace_correlation_id found in "
+            "policy_config or inference_url=%s",
+            inference_url,
+        )
+    return None
+
+
+def validate_trace_correlation_id(
+    trace_correlation_id: str | None,
+    run_id: str,
+    policy_config: dict[str, Any],
+    fatal: bool = False
+) -> str | None:
+    """
+    Validate that trace_correlation_id was successfully extracted.
+    
+    Args:
+        trace_correlation_id: The extracted correlation ID (or None)
+        run_id: Rollout run_id for logging
+        policy_config: Policy configuration for debugging
+        fatal: If True, raise ValueError on missing ID. If False, log error only.
+        
+    Returns:
+        trace_correlation_id if present, None if missing (when fatal=False)
+        
+    Raises:
+        ValueError: If trace_correlation_id is missing and fatal=True
+    """
+    if not trace_correlation_id:
+        error_msg = (
+            f"🚨 CRITICAL: Cannot extract trace_correlation_id!\n"
+            "\n"
+            f"Run ID: {run_id}\n"
+            f"Policy config keys: {sorted(policy_config.keys())}\n"
+            f"Inference URL: {policy_config.get('inference_url', 'NOT_SET')}\n"
+            "\n"
+            "Checked:\n"
+            f"1. policy_config['trace_correlation_id']: {policy_config.get('trace_correlation_id')}\n"
+            f"2. policy_config['trace']: {policy_config.get('trace')}\n"
+            f"3. inference_url query params\n"
+            "\n"
+            "Task app CANNOT proceed without trace_correlation_id.\n"
+            "This indicates the RL trainer is not sending it correctly.\n"
+            "\n"
+            "See monorepo/trace_creation_and_judgement.txt 'Fatal Guards' section.\n"
+        )
+        
+        if fatal:
+            raise ValueError(error_msg)
+        else:
+            logger.error(error_msg)
+    
+    return trace_correlation_id
+
+
+def include_trace_correlation_id_in_response(
+    response_data: dict[str, Any],
+    trace_correlation_id: str | None,
+    run_id: str
+) -> dict[str, Any]:
+    """
+    Include trace_correlation_id in all required locations of rollout response.
+    
+    Required locations (per Fatal Guards section):
+    1. Top-level response["trace_correlation_id"]
+    2. response["pipeline_metadata"]["trace_correlation_id"]
+    3. Each trajectory["trace_correlation_id"]
+    
+    Args:
+        response_data: RolloutResponse dict (from .model_dump())
+        trace_correlation_id: The correlation ID to include
+        run_id: Rollout run_id for logging
+        
+    Returns:
+        Modified response_data with trace_correlation_id in all required places
+    """
+    if not trace_correlation_id:
+        logger.error(
+            "include_trace_correlation_id_in_response: missing trace_correlation_id "
+            "for run_id=%s - cannot include in response",
+            run_id
+        )
+        return response_data
+    
+    # 1. Add to top-level (REQUIRED)
+    if "trace_correlation_id" not in response_data:
+        response_data["trace_correlation_id"] = trace_correlation_id
+        logger.info(
+            "include_trace_correlation_id: added to top-level run_id=%s cid=%s",
+            run_id,
+            trace_correlation_id
+        )
+    
+    # 2. Add to pipeline_metadata (REQUIRED)
+    pipeline_meta = response_data.get("pipeline_metadata")
+    if not isinstance(pipeline_meta, dict):
+        pipeline_meta = {}
+        response_data["pipeline_metadata"] = pipeline_meta
+    
+    if "trace_correlation_id" not in pipeline_meta:
+        pipeline_meta["trace_correlation_id"] = trace_correlation_id
+        logger.info(
+            "include_trace_correlation_id: added to pipeline_metadata run_id=%s cid=%s",
+            run_id,
+            trace_correlation_id
+        )
+    
+    # 3. Add to each trajectory (REQUIRED)
+    trajectories = response_data.get("trajectories", [])
+    if isinstance(trajectories, list):
+        for idx, traj in enumerate(trajectories):
+            if isinstance(traj, dict) and "trace_correlation_id" not in traj:
+                traj["trace_correlation_id"] = trace_correlation_id
+                logger.debug(
+                    "include_trace_correlation_id: added to trajectory[%d] run_id=%s cid=%s",
+                    idx,
+                    run_id,
+                    trace_correlation_id
+                )
+    
+    logger.info(
+        "include_trace_correlation_id: completed run_id=%s cid=%s "
+        "added to %d locations (top-level, metadata, %d trajectories)",
+        run_id,
+        trace_correlation_id,
+        2 + len(trajectories),
+        len(trajectories)
+    )
+    
+    return response_data
+
+
+def verify_trace_correlation_id_in_response(
+    response_data: dict[str, Any],
+    expected_correlation_id: str | None,
+    run_id: str
+) -> bool:
+    """
+    Verify that trace_correlation_id is present in all required locations.
+    
+    Args:
+        response_data: RolloutResponse dict to verify
+        expected_correlation_id: The correlation ID that should be present
+        run_id: Rollout run_id for logging
+        
+    Returns:
+        True if all required locations have the correlation ID, False otherwise
+    """
+    if not expected_correlation_id:
+        logger.error(
+            "verify_trace_correlation_id: no expected_correlation_id provided for run_id=%s",
+            run_id
+        )
+        return False
+    
+    errors = []
+    
+    # Check top-level
+    if response_data.get("trace_correlation_id") != expected_correlation_id:
+        errors.append(
+            f"Top-level missing or mismatch: "
+            f"expected={expected_correlation_id} actual={response_data.get('trace_correlation_id')}"
+        )
+    
+    # Check pipeline_metadata
+    pipeline_meta = response_data.get("pipeline_metadata", {})
+    if not isinstance(pipeline_meta, dict) or pipeline_meta.get("trace_correlation_id") != expected_correlation_id:
+        errors.append(
+            f"pipeline_metadata missing or mismatch: "
+            f"expected={expected_correlation_id} actual={pipeline_meta.get('trace_correlation_id') if isinstance(pipeline_meta, dict) else 'NOT_A_DICT'}"
+        )
+    
+    # Check trajectories
+    trajectories = response_data.get("trajectories", [])
+    if isinstance(trajectories, list):
+        for idx, traj in enumerate(trajectories):
+            if isinstance(traj, dict) and traj.get("trace_correlation_id") != expected_correlation_id:
+                errors.append(
+                    f"trajectory[{idx}] missing or mismatch: "
+                    f"expected={expected_correlation_id} actual={traj.get('trace_correlation_id')}"
+                )
+    
+    if errors:
+        logger.error(
+            "verify_trace_correlation_id: FAILED run_id=%s\n%s",
+            run_id,
+            "\n".join(errors)
+        )
+        return False
+    
+    logger.info(
+        "verify_trace_correlation_id: PASSED run_id=%s cid=%s",
+        run_id,
+        expected_correlation_id
+    )
+    return True

synth_ai/task/tracing_utils.py

@@ -4,9 +4,12 @@ from __future__ import annotations
 
 import os
 from collections.abc import Callable
+from datetime import datetime
 from pathlib import Path
 from typing import Any
 
+from synth_ai.tracing_v3.constants import TRACE_DB_DIR, canonical_trace_db_name
+
 
 def tracing_env_enabled(default: bool = False) -> bool:
     """Return True when tracing is enabled for task apps via environment variable."""
@@ -40,9 +43,17 @@ def resolve_tracing_db_url() -> str | None:
             path.parent.mkdir(parents=True, exist_ok=True)
             return f"sqlite+aiosqlite:///{path}"
 
-    fallback_path = Path("traces/v3/synth_ai.db").expanduser()
-    fallback_path.parent.mkdir(parents=True, exist_ok=True)
-    return f"sqlite+aiosqlite:///{fallback_path}"
+    existing = os.getenv("TASKAPP_TRACE_DB_PATH")
+    if existing:
+        path = Path(existing).expanduser()
+    else:
+        base_dir = TRACE_DB_DIR.expanduser()
+        base_dir.mkdir(parents=True, exist_ok=True)
+        path = base_dir / canonical_trace_db_name(timestamp=datetime.now())
+        os.environ["TASKAPP_TRACE_DB_PATH"] = str(path)
+        os.environ.setdefault("SQLD_DB_PATH", str(path))
+    path.parent.mkdir(parents=True, exist_ok=True)
+    return f"sqlite+aiosqlite:///{path}"
 
 
 def build_tracer_factory(

synth_ai/task/validators.py

@@ -3,14 +3,157 @@
 from __future__ import annotations
 
 import re
-from typing import Any
+from typing import Any, cast
+from urllib.parse import urlparse, urlunparse
 
 import click
 import httpx
-
 from synth_ai.task.contracts import TaskAppEndpoints  # type: ignore[attr-defined]
 
 
+def validate_rollout_response_for_rl(response_data: dict[str, Any], *, warn_only: bool = False) -> list[str]:
+    """Validate that a task app rollout response has required fields for RL training.
+    
+    The backend RL trainer requires:
+    1. pipeline_metadata["inference_url"] at top level (with ?cid= for trace correlation)
+    2. Each step's info.meta["inference_url"] must be present (nested structure!)
+    
+    Args:
+        response_data: The rollout response dict from task app
+        warn_only: If True, return warnings instead of raising exceptions
+        
+    Returns:
+        List of validation warnings/errors
+        
+    Raises:
+        ValueError: If critical fields are missing (unless warn_only=True)
+    """
+    issues = []
+    
+    # Check pipeline_metadata
+    pipeline_metadata = response_data.get("pipeline_metadata")
+    if not isinstance(pipeline_metadata, dict):
+        issues.append("Missing or invalid 'pipeline_metadata' (required for RL training)")
+    else:
+        inference_url = pipeline_metadata.get("inference_url")
+        if not inference_url:
+            issues.append(
+                "pipeline_metadata['inference_url'] is missing. "
+                "RL trainer requires this field to extract traces."
+            )
+        elif not isinstance(inference_url, str):
+            issues.append(
+                f"pipeline_metadata['inference_url'] must be a string, got: {type(inference_url).__name__}"
+            )
+        elif "?cid=" not in inference_url:
+            issues.append(
+                f"pipeline_metadata['inference_url'] should contain '?cid=' for trace correlation. "
+                f"Got: {inference_url[:80]}..."
+            )
+    
+    # Check trajectories and steps
+    trajectories = response_data.get("trajectories", [])
+    if not trajectories:
+        issues.append("No trajectories found in response")
+    
+    for traj_idx, trajectory in enumerate(trajectories):
+        if not isinstance(trajectory, dict):
+            continue
+        
+        steps = trajectory.get("steps", [])
+        for step_idx, step in enumerate(steps):
+            if not isinstance(step, dict):
+                continue
+            
+            step_info = step.get("info", {})
+            if not isinstance(step_info, dict):
+                issues.append(
+                    f"trajectory[{traj_idx}].steps[{step_idx}].info is not a dict"
+                )
+                continue
+            
+            # Check for nested meta.inference_url (backend expects this structure!)
+            step_meta = step_info.get("meta", {})
+            if not isinstance(step_meta, dict):
+                issues.append(
+                    f"trajectory[{traj_idx}].steps[{step_idx}].info.meta is missing or not a dict. "
+                    f"RL trainer expects nested structure: info.meta.inference_url"
+                )
+                continue
+            
+            step_inference_url = step_meta.get("inference_url")
+            if not step_inference_url:
+                issues.append(
+                    f"trajectory[{traj_idx}].steps[{step_idx}].info.meta['inference_url'] is missing. "
+                    f"RL trainer needs this for trace extraction (nested structure required!)"
+                )
+            elif not isinstance(step_inference_url, str):
+                issues.append(
+                    f"trajectory[{traj_idx}].steps[{step_idx}].info.meta['inference_url'] must be a string, "
+                    f"got: {type(step_inference_url).__name__}"
+                )
+    
+    if issues and not warn_only:
+        error_msg = "Task app response validation failed for RL training:\n" + "\n".join(
+            f"  - {issue}" for issue in issues
+        )
+        raise ValueError(error_msg)
+    
+    return issues
+
+
+def normalize_inference_url(url: str | None, *, default: str = "https://api.openai.com/v1/chat/completions") -> str:
+    """Normalize an inference URL to include the /v1/chat/completions path.
+    
+    This utility ensures inference URLs have the correct path structure for OpenAI-compatible
+    chat completions endpoints, while preserving query parameters (e.g., ?cid=trace_123) 
+    that may be added for tracing.
+    
+    Args:
+        url: The inference URL to normalize (may be None or incomplete)
+        default: Default URL to use if url is None/empty
+        
+    Returns:
+        Normalized URL with proper path and preserved query parameters
+        
+    Examples:
+        >>> normalize_inference_url("https://api.groq.com")
+        'https://api.groq.com/v1/chat/completions'
+        
+        >>> normalize_inference_url("https://modal.host?cid=trace_123")
+        'https://modal.host/v1/chat/completions?cid=trace_123'
+        
+        >>> normalize_inference_url("https://api.openai.com/v1")
+        'https://api.openai.com/v1/chat/completions'
+        
+        >>> normalize_inference_url("https://api.groq.com/openai/v1/chat/completions")
+        'https://api.groq.com/openai/v1/chat/completions'
+    """
+    candidate = (url or default).strip()
+    if not candidate:
+        candidate = default
+    
+    # Parse the URL to separate path and query components
+    parsed = urlparse(candidate)
+    
+    # Check if path already ends with a completions endpoint
+    path = parsed.path.rstrip('/')
+    if path.endswith("/v1/chat/completions") or path.endswith("/chat/completions"):
+        return candidate
+    
+    # Determine what to append based on existing path
+    if path.endswith("/v1"):
+        new_path = f"{path}/chat/completions"
+    elif path.endswith("/chat"):
+        new_path = f"{path}/completions"
+    else:
+        # Default: append full path
+        new_path = f"{path}/v1/chat/completions" if path else "/v1/chat/completions"
+    
+    # Reconstruct URL with new path and original query/fragment
+    return cast(str, urlunparse(parsed._replace(path=new_path)))
+
+
 def validate_task_app_url(url: str | None) -> str:
     """Validate and normalize a task app URL.
     

synth_ai/tracing_v3/config.py

@@ -3,27 +3,29 @@
 import os
 from dataclasses import dataclass
 
+from synth_ai.tracing_v3.constants import canonical_trace_db_path
+
+DEFAULT_DB_FILE = str(canonical_trace_db_path())
+
+
+def _default_sqlite_url() -> str:
+    base_path = os.path.abspath(os.getenv("SQLD_DB_PATH", DEFAULT_DB_FILE))
+    candidate = os.path.join(base_path, "dbs", "default", "data")
+    if os.path.isdir(base_path) and os.path.exists(candidate):
+        return f"sqlite+aiosqlite:///{candidate}"
+    return f"sqlite+aiosqlite:///{base_path}"
+
 
 @dataclass
 class TursoConfig:
     """Configuration for Turso/sqld connection."""
 
     # Default values matching serve.sh
-    DEFAULT_DB_FILE = "traces/v3/synth_ai.db"
+    DEFAULT_DB_FILE = DEFAULT_DB_FILE
     DEFAULT_HTTP_PORT = 8080
 
-    # Local embedded database for async SQLAlchemy
-    # Resolve to the actual SQLite file used by sqld if the base path is a directory
-    def _resolve_sqlite_db_url() -> str:  # type: ignore[no-redef]
-        base_path = os.path.abspath(os.getenv("SQLD_DB_PATH", "traces/v3/synth_ai.db"))
-        # If sqld is managing this DB, the real SQLite file lives under dbs/default/data
-        candidate = os.path.join(base_path, "dbs", "default", "data")
-        if os.path.isdir(base_path) and os.path.exists(candidate):
-            return f"sqlite+aiosqlite:///{candidate}"
-        return f"sqlite+aiosqlite:///{base_path}"
-
     # Use env override if provided; otherwise resolve based on SQLD layout
-    db_url: str = os.getenv("TURSO_LOCAL_DB_URL", _resolve_sqlite_db_url())
+    db_url: str = os.getenv("TURSO_LOCAL_DB_URL", _default_sqlite_url())
 
     # Remote database sync configuration
     sync_url: str = os.getenv("TURSO_DATABASE_URL", "")
@@ -48,7 +50,7 @@ class TursoConfig:
 
     # Daemon settings (for local sqld) - match serve.sh defaults
     sqld_binary: str = os.getenv("SQLD_BINARY", "sqld")
-    sqld_db_path: str = os.getenv("SQLD_DB_PATH", "traces/v3/synth_ai.db")
+    sqld_db_path: str = os.getenv("SQLD_DB_PATH", DEFAULT_DB_FILE)
     sqld_http_port: int = int(os.getenv("SQLD_HTTP_PORT", "8080"))
     sqld_idle_shutdown: int = int(os.getenv("SQLD_IDLE_SHUTDOWN", "0"))  # 0 = no idle shutdown
 

synth_ai/tracing_v3/constants.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from datetime import datetime
+from pathlib import Path
+
+TRACE_DB_DIR = Path("traces")
+TRACE_DB_BASENAME = "task_app_traces"
+
+
+def canonical_trace_db_name(*, timestamp: datetime | None = None) -> str:
+    """Return the canonical trace database filename (with optional timestamp suffix)."""
+
+    if timestamp is None:
+        return f"{TRACE_DB_BASENAME}.db"
+    return f"{TRACE_DB_BASENAME}_{timestamp.strftime('%Y-%m-%d_%H-%M-%S')}.db"
+
+
+def canonical_trace_db_path(*, timestamp: datetime | None = None) -> Path:
+    """Return the canonical trace database path within the default trace directory."""
+
+    return TRACE_DB_DIR / canonical_trace_db_name(timestamp=timestamp)

synth_ai/tracing_v3/db_config.py

@@ -7,6 +7,8 @@ import os
 import shutil
 from typing import TYPE_CHECKING, Optional
 
+from synth_ai.tracing_v3.constants import canonical_trace_db_path
+
 if TYPE_CHECKING:
     from .turso.daemon import SqldDaemon
 
@@ -17,7 +19,7 @@ class DatabaseConfig:
     """Centralized database configuration management."""
 
     # Default values from serve.sh
-    DEFAULT_DB_FILE = "traces/v3/synth_ai.db"
+    DEFAULT_DB_FILE = str(canonical_trace_db_path())
     DEFAULT_HTTP_PORT = 8080
 
     def __init__(

synth_ai/tracing_v3/decorators.py

@@ -29,6 +29,7 @@ import contextvars
 import functools
 import time
 from collections.abc import Awaitable, Callable, Mapping
+from contextvars import Token
 from typing import Any, TypeVar, cast, overload
 
 from .abstractions import LMCAISEvent, TimeRecord
@@ -367,11 +368,11 @@ class SessionContext:
         ```
     """
 
-    def __init__(self, session_id: str, tracer=None):
+    def __init__(self, session_id: str, tracer: Any | None = None):
         self.session_id = session_id
         self.tracer = tracer
-        self._token = None
-        self._tracer_token = None
+        self._token: Token[str | None] | None = None
+        self._tracer_token: Token[Any] | None = None
 
     def __enter__(self):
         # Store tokens to restore previous context on exit
@@ -382,8 +383,9 @@ class SessionContext:
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         # Restore previous context - this is crucial for proper isolation
-        _session_id_ctx.reset(self._token)
-        if self._tracer_token:
+        if self._token is not None:
+            _session_id_ctx.reset(self._token)
+        if self._tracer_token is not None:
             _session_tracer_ctx.reset(self._tracer_token)
 
     async def __aenter__(self):
@@ -393,6 +395,7 @@ class SessionContext:
         return self
 
     async def __aexit__(self, exc_type, exc_val, exc_tb):
-        _session_id_ctx.reset(self._token)
-        if self._tracer_token:
+        if self._token is not None:
+            _session_id_ctx.reset(self._token)
+        if self._tracer_token is not None:
             _session_tracer_ctx.reset(self._tracer_token)

synth_ai/tracing_v3/session_tracer.py

@@ -375,11 +375,21 @@ class SessionTracer:
 
             # Save if requested
             should_save = save if save is not None else self.auto_save
+            
+            # Debug logging
+            import logging
+            _logger = logging.getLogger(__name__)
+            _logger.info(f"[TRACE_DEBUG] end_session: should_save={should_save}, self.db={self.db is not None}, auto_save={self.auto_save}")
+            
             if should_save and self.db:
+                _logger.info(f"[TRACE_DEBUG] Calling insert_session_trace with {len(self._current_trace.markov_blanket_message_history)} messages")
                 await self.db.insert_session_trace(self._current_trace)
+                _logger.info("[TRACE_DEBUG] insert_session_trace completed")
 
                 # Trigger post-save hooks
                 await self.hooks.trigger("after_save", session=self._current_trace)
+            else:
+                _logger.warning(f"[TRACE_DEBUG] Skipping save: should_save={should_save}, self.db={self.db is not None}")
 
             # Trigger session end hooks
             await self.hooks.trigger("session_end", session=self._current_trace)