synth-ai 0.2.14__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/cli/local/experiment_queue/celery_app.py

@@ -0,0 +1,208 @@
+"""Celery application configuration for the experiment queue."""
+
+from __future__ import annotations
+
+import logging
+import os
+import sys
+from pathlib import Path
+
+# ALWAYS clear config cache to ensure fresh config on worker startup
+# This is critical for workers that may have cached config from previous runs
+from . import config as queue_config
+
+queue_config.reset_config_cache()
+
+# Log the database path that will be used (before any imports that might cache it)
+db_path_env = os.getenv("EXPERIMENT_QUEUE_DB_PATH")
+if db_path_env:
+    db_path_resolved = Path(db_path_env).expanduser().resolve()
+    print(f"[celery_app] EXPERIMENT_QUEUE_DB_PATH={db_path_resolved}", file=sys.stderr, flush=True)
+    # Verify the path exists or can be created
+    db_path_resolved.parent.mkdir(parents=True, exist_ok=True)
+    print(f"[celery_app] Database file will be: {db_path_resolved}", file=sys.stderr, flush=True)
+else:
+    # Will use default path from config (~/.synth_ai/experiment_queue.db)
+    print("[celery_app] EXPERIMENT_QUEUE_DB_PATH not set, will use default path", file=sys.stderr, flush=True)
+
+from celery import Celery  # noqa: E402
+
+from .config import load_config, reset_config_cache  # noqa: E402
+from .database import init_db  # noqa: E402
+
+logger = logging.getLogger(__name__)
+
+# Module-level celery app instance and its config (for change detection)
+_celery_app_instance: Celery | None = None
+_celery_app_broker_url: str | None = None
+
+
+def _create_celery_app() -> Celery:
+    """Instantiate the Celery application with shared configuration."""
+    # Load config (will use EXPERIMENT_QUEUE_DB_PATH if set, otherwise default path)
+    reset_config_cache()
+    config = load_config()
+    
+    # Log which database path is being used
+    env_db_path = os.getenv("EXPERIMENT_QUEUE_DB_PATH")
+    if env_db_path:
+        env_db_path_resolved = Path(env_db_path).expanduser().resolve()
+        config_db_path_resolved = config.sqlite_path.resolve()
+        
+        # Verify database path matches environment variable if env var is set
+        if config_db_path_resolved != env_db_path_resolved:
+            error_msg = (
+                f"Database path mismatch! "
+                f"ENV: {env_db_path_resolved} != CONFIG: {config_db_path_resolved}. "
+                f"This will cause workers to use different databases. "
+                f"Clear config cache and restart worker."
+            )
+            print(f"[celery_app] ERROR: {error_msg}", file=sys.stderr, flush=True)
+            raise RuntimeError(error_msg)
+        print(
+            f"[celery_app] Using database from EXPERIMENT_QUEUE_DB_PATH: {config.sqlite_path}",
+            file=sys.stderr,
+            flush=True,
+        )
+    else:
+        print(
+            f"[celery_app] Using default database path: {config.sqlite_path}",
+            file=sys.stderr,
+            flush=True,
+        )
+    
+    # CRITICAL: Ensure database path is unique and enforce single instance
+    # Verify no other process is using a different database path
+    db_file = str(config.sqlite_path)
+    Path(db_file).parent.mkdir(parents=True, exist_ok=True)
+    
+    # Initialize database (for our application DB only - Celery uses Redis broker)
+    init_db()
+    
+    # Log database path for debugging (especially in worker subprocess)
+    # Use print to stderr to ensure it's visible even if logging isn't configured
+    print(
+        f"[celery_app] Initializing with database: {config.sqlite_path} (broker: {config.broker_url})",
+        file=sys.stderr,
+        flush=True,
+    )
+    logger.info(
+        "Celery app initializing with database: %s (broker: %s)",
+        config.sqlite_path,
+        config.broker_url,
+    )
+
+    app = Celery(
+        "synth_ai.experiment_queue",
+        broker=config.broker_url,
+        backend=config.result_backend_url,
+        include=["synth_ai.cli.local.experiment_queue.tasks"],
+    )
+
+    app.conf.update(
+        task_serializer="json",
+        accept_content=["json"],
+        result_serializer="json",
+        timezone="UTC",
+        enable_utc=True,
+        task_track_started=True,
+        worker_prefetch_multiplier=1,
+        worker_max_tasks_per_child=25,
+        task_acks_late=True,
+        broker_pool_limit=4,
+        # Celery Beat schedule for periodic queue processing
+        beat_schedule={
+            "process-experiment-queue": {
+                "task": "synth_ai.cli.local.experiment_queue.process_experiment_queue",
+                "schedule": 5.0,  # Run every 5 seconds
+            },
+        },
+    )
+    
+    # Redis broker - no need to pre-create tables or configure WAL mode
+    # Redis handles concurrency natively, no SQLite locking issues
+    
+    return app
+
+
+def get_celery_app() -> Celery:
+    """Return the shared Celery app, recreating it if config has changed.
+    
+    This function checks if the config cache was cleared (by comparing
+    current broker URL to cached one) and recreates the app if needed.
+    This ensures CLI and worker use the same database when env vars change.
+    
+    Uses EXPERIMENT_QUEUE_DB_PATH if set, otherwise defaults to ~/.synth_ai/experiment_queue.db
+    """
+    global _celery_app_instance, _celery_app_broker_url
+    
+    # Check if config has changed by comparing broker URLs
+    # load_config() will use EXPERIMENT_QUEUE_DB_PATH if set, otherwise default path
+    current_config = load_config()
+    current_broker_url = current_config.broker_url
+    
+    if _celery_app_instance is None or _celery_app_broker_url != current_broker_url:
+        # Config changed or app not initialized - recreate it
+        logger.debug(
+            "Recreating Celery app: broker URL changed from %s to %s",
+            _celery_app_broker_url,
+            current_broker_url,
+        )
+        _celery_app_instance = _create_celery_app()
+        _celery_app_broker_url = current_broker_url
+    
+    return _celery_app_instance
+
+
+# Create app at module import time (for worker entrypoints and decorators)
+# This ensures workers get the correct config at startup
+# Uses default path (~/.synth_ai/experiment_queue.db) if EXPERIMENT_QUEUE_DB_PATH is not set
+try:
+    celery_app = get_celery_app()
+except RuntimeError as e:
+    # If config loading fails for any reason, create a placeholder that will fail on use
+    # This allows the module to be imported but will fail when celery_app is actually used
+    class _LazyCeleryApp:
+        """Placeholder that raises error when Celery app initialization fails."""
+        def __init__(self, original_error: RuntimeError):
+            self._original_error = original_error
+        
+        def __getattr__(self, name):
+            # Special handling for 'task' attribute used in decorators
+            # This allows @celery_app.task decorators to be defined without immediate error
+            if name == "task":
+                # Return a decorator factory that will raise error when task is actually called
+                def task_decorator(*args, **kwargs):
+                    def decorator(func):
+                        # Store the function, but wrap it to check env var on call
+                        def wrapper(*func_args, **func_kwargs):
+                            # Try to get real celery app now
+                            try:
+                                real_app = get_celery_app()
+                                # Re-register the task with the real app
+                                real_task = real_app.task(*args, **kwargs)(func)
+                                return real_task(*func_args, **func_kwargs)
+                            except RuntimeError as runtime_err:
+                                raise RuntimeError(
+                                    f"Failed to initialize Celery app. "
+                                    f"Original error: {self._original_error}, "
+                                    f"Runtime error: {runtime_err}"
+                                ) from runtime_err
+                        return wrapper
+                    return decorator
+                return task_decorator
+            raise RuntimeError(
+                f"Failed to initialize Celery app. "
+                f"Original error: {self._original_error}"
+            )
+        def __call__(self, *args, **kwargs):
+            raise RuntimeError(
+                f"Failed to initialize Celery app. "
+                f"Original error: {self._original_error}"
+            )
+        def __getitem__(self, key):
+            raise RuntimeError(
+                f"Failed to initialize Celery app. "
+                f"Original error: {self._original_error}"
+            )
+    celery_app = _LazyCeleryApp(e)  # type: ignore[assignment]

synth_ai/cli/local/experiment_queue/config.py

@@ -0,0 +1,128 @@
+"""Configuration helpers for the experiment queue service."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+
+# Default database path in user's home directory
+# Can be overridden via EXPERIMENT_QUEUE_DB_PATH environment variable
+DEFAULT_DB_PATH = Path.home() / ".synth_ai" / "experiment_queue.db"
+
+
+def _candidate_db_paths() -> list[Path]:
+    """Build candidate paths using env overrides."""
+    env_vars = (
+        "EXPERIMENT_QUEUE_DB_PATH",
+        "EXPERIMENT_QUEUE_SQLITE_PATH",
+        "SYNTH_AI_EXPERIMENT_DB_PATH",
+        "SQLD_DB_PATH",
+    )
+    candidates: list[Path] = []
+    for name in env_vars:
+        value = os.getenv(name)
+        if value:
+            candidates.append(Path(value).expanduser())
+    candidates.append(DEFAULT_DB_PATH)
+    return candidates
+
+
+def _resolve_sqlite_file(base_path: Path) -> Path:
+    """
+    Resolve the actual SQLite file path managed by sqld.
+
+    sqld typically creates `{db_path}/dbs/default/data`. If the directory
+    does not exist yet (e.g., sqld not started), fall back to the provided path.
+    """
+    base_path = base_path.expanduser()
+    if base_path.is_dir():
+        sqld_candidate = (base_path / "dbs" / "default" / "data").resolve()
+        if sqld_candidate.exists():
+            return sqld_candidate
+
+    resolved = base_path.resolve()
+    resolved.parent.mkdir(parents=True, exist_ok=True)
+    return resolved
+
+
+@dataclass(frozen=True)
+class ExperimentQueueConfig:
+    """Resolved configuration for the experiment queue runtime."""
+
+    sqlite_path: Path
+    broker_url: str
+    result_backend_url: str
+    backend_url: str
+
+    @property
+    def sqlalchemy_url(self) -> str:
+        """Return SQLAlchemy connection string for ORM usage."""
+        return f"sqlite:///{self.sqlite_path}"
+
+
+@lru_cache(maxsize=1)
+def load_config() -> ExperimentQueueConfig:
+    """Resolve configuration once per process.
+    
+    Backend URL Resolution:
+    - Default: Production backend (https://api.usesynth.ai/api)
+    - Set EXPERIMENT_QUEUE_LOCAL=true to use local backend (http://localhost:8000/api)
+    - Override with EXPERIMENT_QUEUE_BACKEND_URL env var for custom URL
+    - Falls back to DEV_BACKEND_URL if set (for compatibility)
+
+    Examples:
+        # Use production (default)
+        load_config()  # Uses https://api.usesynth.ai/api
+        
+        # Use local backend
+        os.environ["EXPERIMENT_QUEUE_LOCAL"] = "true"
+        load_config()  # Uses http://localhost:8000/api
+        
+        # Use custom backend
+        os.environ["EXPERIMENT_QUEUE_BACKEND_URL"] = "http://custom:9000/api"
+        load_config()  # Uses http://custom:9000/api
+    """
+    sqlite_path: Path | None = None
+    for candidate in _candidate_db_paths():
+        sqlite_path = _resolve_sqlite_file(candidate)
+        if sqlite_path:
+            break
+
+    if sqlite_path is None:
+        sqlite_path = _resolve_sqlite_file(DEFAULT_DB_PATH)
+
+    # Redis broker and backend (no longer using SQLite for Celery)
+    broker_url = os.getenv("EXPERIMENT_QUEUE_BROKER_URL", "redis://localhost:6379/0")
+    result_backend_url = os.getenv("EXPERIMENT_QUEUE_RESULT_BACKEND_URL", "redis://localhost:6379/1")
+    
+    # Backend API URL for progress polling
+    # Defaults to production: https://api.usesynth.ai/api
+    # Override with EXPERIMENT_QUEUE_BACKEND_URL or EXPERIMENT_QUEUE_LOCAL=true for localhost:8000
+    if os.getenv("EXPERIMENT_QUEUE_LOCAL", "").lower() in ("true", "1", "yes"):
+        # Local mode: use localhost:8000
+        backend_url = os.getenv("EXPERIMENT_QUEUE_BACKEND_URL", "http://localhost:8000/api")
+    else:
+        # Production mode (default): use api.usesynth.ai
+        backend_url = (
+            os.getenv("EXPERIMENT_QUEUE_BACKEND_URL")
+            or os.getenv("DEV_BACKEND_URL")
+            or "https://api.usesynth.ai/api"
+        )
+
+    return ExperimentQueueConfig(
+        sqlite_path=sqlite_path,
+        broker_url=broker_url,
+        result_backend_url=result_backend_url,
+        backend_url=backend_url,
+    )
+
+
+def reset_config_cache() -> None:
+    """Clear the cached config to force reload from environment variables.
+    
+    Call this before importing/using the Celery app if EXPERIMENT_QUEUE_DB_PATH
+    or EXPERIMENT_QUEUE_TRAIN_CMD environment variables have changed.
+    """
+    load_config.cache_clear()

synth_ai/cli/local/experiment_queue/config_utils.py

@@ -0,0 +1,272 @@
+"""Helpers for preparing TOML configs for experiment jobs."""
+
+from __future__ import annotations
+
+import copy
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Mapping, MutableMapping
+
+import tomli_w
+
+from synth_ai.core.telemetry import log_info
+
+from .validation import validate_path
+
+
+def _load_toml(config_path: Path) -> dict[str, Any]:
+    """Load TOML using tomllib/tomli depending on runtime.
+    
+    Args:
+        config_path: Path to TOML config file
+        
+    Returns:
+        Parsed TOML as dictionary
+        
+    Raises:
+        AssertionError: If config_path is invalid or file cannot be parsed
+        FileNotFoundError: If config file doesn't exist
+    """
+    # Validate input
+    assert config_path is not None, "config_path cannot be None"
+    path = validate_path(config_path, "config_path", must_exist=True)
+    
+    try:
+        import tomllib  # type: ignore[attr-defined]
+    except ModuleNotFoundError:
+        import tomli as tomllib  # type: ignore[no-redef]
+    
+    with open(path, "rb") as fh:
+        config = tomllib.load(fh)
+        assert isinstance(config, dict), (
+            f"TOML config must be dict, got {type(config).__name__}"
+        )
+        return config
+
+
+def _ensure_prompt_learning_section(config: dict[str, Any]) -> dict[str, Any]:
+    """Ensure prompt_learning section exists and is a dict.
+    
+    Args:
+        config: TOML config dictionary
+        
+    Returns:
+        prompt_learning section dictionary
+        
+    Raises:
+        AssertionError: If config is invalid or section is wrong type
+    """
+    assert isinstance(config, dict), (
+        f"config must be dict, got {type(config).__name__}"
+    )
+    section = config.setdefault("prompt_learning", {})
+    assert isinstance(section, dict), (
+        f"Expected [prompt_learning] section to be a dict, got {type(section).__name__}"
+    )
+    return section
+
+
+def _find_similar_keys(data: dict[str, Any], search_key: str, results: list[str], prefix: str = "") -> None:
+    """Recursively find keys similar to search_key in nested dict structure."""
+    if not isinstance(data, dict):
+        return
+    
+    for key, value in data.items():
+        current_path = f"{prefix}.{key}" if prefix else key
+        if search_key.lower() in key.lower() or key.lower() in search_key.lower():
+            results.append(current_path)
+        if isinstance(value, dict):
+            _find_similar_keys(value, search_key, results, current_path)
+
+
+def _deep_update(base: MutableMapping[str, Any], overrides: Mapping[str, Any]) -> MutableMapping[str, Any]:
+    """Deep update with support for dot-notation keys (e.g., 'prompt_learning.gepa.rollout.budget').
+    
+    Dot-notation keys are split and create nested dictionaries.
+    Regular keys are updated normally.
+    """
+    for key, value in overrides.items():
+        # Handle dot-notation keys (e.g., "prompt_learning.gepa.rollout.budget")
+        if "." in key:
+            keys = key.split(".")
+            # Navigate/create nested structure
+            current = base
+            for k in keys[:-1]:
+                if k not in current or not isinstance(current[k], MutableMapping):
+                    current[k] = {}
+                current = current[k]
+            # Set final value
+            final_key = keys[-1]
+            if (
+                isinstance(value, Mapping)
+                and isinstance(current.get(final_key), MutableMapping)
+                and not isinstance(value, str | bytes)
+            ):
+                nested = copy.deepcopy(dict(current[final_key]))
+                current[final_key] = _deep_update(nested, value)
+            else:
+                current[final_key] = copy.deepcopy(value)
+        else:
+            # Regular key (no dot notation)
+            if (
+                isinstance(value, Mapping)
+                and isinstance(base.get(key), MutableMapping)
+                and not isinstance(value, str | bytes)
+            ):
+                nested = copy.deepcopy(dict(base[key]))
+                base[key] = _deep_update(nested, value)
+            else:
+                base[key] = copy.deepcopy(value)
+    return base
+
+
+def _resolve_path(value: str, *, relative_to: Path) -> Path:
+    path = Path(value)
+    if not path.is_absolute():
+        path = (relative_to / value).resolve()
+    else:
+        path = path.expanduser().resolve()
+    return path
+
+
+def resolve_results_folder(config: dict[str, Any], config_path: Path) -> Path:
+    """Resolve and ensure the results folder exists."""
+    config_dir = config_path.parent.resolve()
+    prompt_section = _ensure_prompt_learning_section(config)
+    raw = prompt_section.get("results_folder") or config.get("results_folder")
+    if raw:
+        results_path = _resolve_path(str(raw), relative_to=config_dir)
+    else:
+        results_path = (config_dir / "results").resolve()
+    results_path.mkdir(parents=True, exist_ok=True)
+    prompt_section["results_folder"] = str(results_path)
+    return results_path
+
+
+def normalize_env_file_path(config: dict[str, Any], config_path: Path) -> None:
+    """Ensure env_file_path fields are absolute so subprocesses inherit the right file."""
+    config_dir = config_path.parent.resolve()
+    prompt_section = _ensure_prompt_learning_section(config)
+    raw = prompt_section.get("env_file_path") or config.get("env_file_path")
+    if not raw:
+        return
+    env_path = _resolve_path(str(raw), relative_to=config_dir)
+    prompt_section["env_file_path"] = str(env_path)
+    config["env_file_path"] = str(env_path)
+
+
+@dataclass(slots=True)
+class PreparedConfig:
+    """Container for a temporary config file plus metadata."""
+
+    path: Path
+    results_folder: Path
+    workdir: Path | None = None
+
+    def cleanup(self) -> None:
+        if self.workdir and self.workdir.exists():
+            import shutil
+
+            shutil.rmtree(self.workdir, ignore_errors=True)
+
+
+def prepare_config_file(config_path: str | Path, overrides: Mapping[str, Any] | None = None) -> PreparedConfig:
+    """
+    Load a TOML config, apply overrides, and materialize a temporary file.
+
+    Args:
+        config_path: Path to source TOML config file
+        overrides: Optional dictionary of config overrides to apply
+
+    Returns:
+        PreparedConfig with the path to the merged TOML and resolved results_folder.
+
+    Raises:
+        AssertionError: If inputs are invalid
+        FileNotFoundError: If config file doesn't exist
+    """
+    ctx: dict[str, Any] = {"config_path": str(config_path), "has_overrides": overrides is not None}
+    log_info("prepare_config_file invoked", ctx=ctx)
+    # Validate inputs
+    assert config_path is not None, "config_path cannot be None"
+    source_path = validate_path(config_path, "config_path", must_exist=True)
+    
+    if overrides is not None:
+        assert isinstance(overrides, Mapping), (
+            f"overrides must be Mapping, got {type(overrides).__name__}"
+        )
+
+    data = _load_toml(source_path)
+    assert isinstance(data, dict), (
+        f"_load_toml must return dict, got {type(data).__name__}"
+    )
+    
+    if overrides:
+        _deep_update(data, overrides)
+        # Validate after merge
+        assert isinstance(data, dict), (
+            f"Config after merge must be dict, got {type(data).__name__}"
+        )
+        
+        # VALIDATION: Verify critical overrides were actually applied
+        # This prevents silent failures where overrides don't match expected paths
+        for override_key, override_value in overrides.items():
+            # Navigate the nested dict structure (created by _deep_update)
+            keys = override_key.split(".")
+            current = data
+            found = True
+            
+            # Navigate through nested structure
+            for key in keys:
+                if not isinstance(current, dict) or key not in current:
+                    found = False
+                    break
+                current = current[key]
+            
+            if not found:
+                # Try to find similar keys for better error message
+                similar_keys = []
+                _find_similar_keys(data, keys[0], similar_keys)
+                similar_msg = f" Similar keys found: {similar_keys[:5]}" if similar_keys else ""
+                raise ValueError(
+                    f"Config override validation failed: '{override_key}' was not found in config after merge.{similar_msg} "
+                    f"Override value: {override_value!r}. This indicates the override path is incorrect or the config structure doesn't match."
+                )
+            elif isinstance(current, dict) and isinstance(override_value, dict):
+                # For dict overrides, check that override_value is a subset of current
+                # (i.e., all keys in override_value match current)
+                for key, val in override_value.items():
+                    if key not in current or current[key] != val:
+                        raise ValueError(
+                            f"Config override validation failed: '{override_key}.{key}' was set to {current.get(key)!r} "
+                            f"but override specified {val!r}. Override may not have been applied correctly."
+                        )
+            elif current != override_value:
+                # Value exists but doesn't match - this is a problem
+                # This is critical for rollout limits - if base config has 300 but override specifies 100,
+                # we MUST ensure 100 is applied, not 300
+                raise ValueError(
+                    f"Config override validation failed: '{override_key}' was set to {current!r} "
+                    f"but override specified {override_value!r}. Override may not have been applied correctly. "
+                    f"This indicates the base config value ({current!r}) was not overridden by the specified value ({override_value!r})."
+                )
+
+    normalize_env_file_path(data, source_path)
+    results_folder = resolve_results_folder(data, source_path)
+    assert results_folder.exists(), (
+        f"results_folder must exist after creation: {results_folder}"
+    )
+
+    workdir = Path(tempfile.mkdtemp(prefix="experiment_queue_cfg_"))
+    assert workdir.exists(), f"workdir must exist after creation: {workdir}"
+    
+    tmp_config_path = workdir / source_path.name
+    with open(tmp_config_path, "wb") as fh:
+        tomli_w.dump(data, fh)
+    
+    assert tmp_config_path.exists(), (
+        f"Temporary config file must exist after writing: {tmp_config_path}"
+    )
+
+    return PreparedConfig(path=tmp_config_path, results_folder=results_folder, workdir=workdir)