synth-ai 0.4.1__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/sdk/api/research_agent/__init__.py

@@ -1,73 +1,290 @@
-"""Research Agent API for running AI-assisted code analysis and optimization.
-
-This module provides both CLI and SDK interfaces for research agent jobs.
-
-CLI Usage:
-    uvx synth-ai agent run --config my_config.toml --poll
-
-SDK Usage:
-    from synth_ai.sdk.api.research_agent import (
-        ResearchAgentJob,
-        ResearchAgentJobConfig,
-        ResearchConfig,
-        DatasetSource,
-        OptimizationTool,
-        MIPROConfig,
-        GEPAConfig,
-    )
-
-    # Create typed config
-    research_config = ResearchConfig(
-        task_description="Optimize prompt for banking classification",
-        tools=[OptimizationTool.MIPRO],
-        datasets=[
+"""Research Agent SDK models and job helpers."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from pathlib import Path
+import os
+import tomllib
+from typing import Any
+
+
+class OptimizationTool(str, Enum):
+    MIPRO = "mipro"
+    GEPA = "gepa"
+
+
+class ModelProvider(str, Enum):
+    OPENAI = "openai"
+    GROQ = "groq"
+
+
+@dataclass
+class PermittedModel:
+    model: str
+    provider: ModelProvider
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"model": self.model, "provider": self.provider.value}
+
+
+@dataclass
+class PermittedModelsConfig:
+    models: list[PermittedModel] = field(default_factory=list)
+    default_temperature: float | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        data = {"models": [model.to_dict() for model in self.models]}
+        if self.default_temperature is not None:
+            data["default_temperature"] = self.default_temperature
+        return data
+
+
+@dataclass
+class DatasetSource:
+    source_type: str
+    hf_repo_id: str | None = None
+    hf_split: str | None = None
+    description: str | None = None
+    file_ids: list[str] | None = None
+    inline_data: dict[str, str] | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        data: dict[str, Any] = {"source_type": self.source_type}
+        if self.hf_repo_id:
+            data["hf_repo_id"] = self.hf_repo_id
+        if self.hf_split:
+            data["hf_split"] = self.hf_split
+        if self.description:
+            data["description"] = self.description
+        if self.file_ids is not None:
+            data["file_ids"] = self.file_ids
+        if self.inline_data is not None:
+            data["inline_data"] = self.inline_data
+        return data
+
+
+@dataclass
+class MIPROConfig:
+    meta_model: str = "llama-3.3-70b-versatile"
+    meta_provider: ModelProvider = ModelProvider.GROQ
+    num_trials: int = 10
+    proposer_effort: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        data: dict[str, Any] = {
+            "meta_model": self.meta_model,
+            "meta_provider": self.meta_provider.value,
+            "num_trials": self.num_trials,
+        }
+        if self.proposer_effort is not None:
+            data["proposer_effort"] = self.proposer_effort
+        return data
+
+
+@dataclass
+class GEPAConfig:
+    mutation_model: str = "openai/gpt-oss-120b"
+    population_size: int = 20
+    proposer_type: str = "dspy"
+    spec_path: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        data: dict[str, Any] = {
+            "mutation_model": self.mutation_model,
+            "population_size": self.population_size,
+            "proposer_type": self.proposer_type,
+        }
+        if self.spec_path is not None:
+            data["spec_path"] = self.spec_path
+        return data
+
+
+@dataclass
+class ResearchConfig:
+    task_description: str
+    tools: list[OptimizationTool] = field(default_factory=list)
+    datasets: list[DatasetSource] = field(default_factory=list)
+    primary_metric: str = "accuracy"
+    num_iterations: int = 10
+    mipro_config: MIPROConfig | None = None
+    gepa_config: GEPAConfig | None = None
+    permitted_models: PermittedModelsConfig | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        data: dict[str, Any] = {
+            "task_description": self.task_description,
+            "tools": [tool.value for tool in self.tools],
+            "primary_metric": self.primary_metric,
+            "num_iterations": self.num_iterations,
+        }
+        if self.datasets:
+            data["datasets"] = [ds.to_dict() for ds in self.datasets]
+        if self.mipro_config is not None:
+            data["mipro_config"] = self.mipro_config.to_dict()
+        if self.gepa_config is not None:
+            data["gepa_config"] = self.gepa_config.to_dict()
+        if self.permitted_models is not None:
+            data["permitted_models"] = self.permitted_models.to_dict()
+        return data
+
+
+@dataclass
+class ResearchAgentJobConfig:
+    research: ResearchConfig
+    repo_url: str = ""
+    repo_branch: str | None = None
+    inline_files: dict[str, str] | None = None
+    backend_url: str = ""
+    api_key: str = ""
+    allow_missing_api_key: bool = False
+    backend: str | None = None
+    model: str | None = None
+    max_agent_spend_usd: float | None = None
+    max_synth_spend_usd: float | None = None
+    reasoning_effort: str | None = None
+
+    def __post_init__(self) -> None:
+        if not self.repo_url and not self.inline_files:
+            raise ValueError("Either repo_url or inline_files must be provided")
+        if not self.api_key:
+            self.api_key = os.getenv("SYNTH_API_KEY", "").strip()
+        if not self.api_key and not self.allow_missing_api_key:
+            raise ValueError("api_key is required")
+        if not self.backend_url:
+            self.backend_url = "https://api.usesynth.ai"
+
+    @classmethod
+    def from_toml(cls, path: str | Path) -> "ResearchAgentJobConfig":
+        path = Path(path)
+        if not path.exists():
+            raise FileNotFoundError(path)
+        data = tomllib.loads(path.read_text(encoding="utf-8"))
+        if "research_agent" not in data:
+            raise ValueError("Config must have [research_agent] section")
+        section = data["research_agent"]
+        research_section = section.get("research")
+        if research_section is None:
+            raise ValueError("research_agent.research config is required")
+
+        tools = [OptimizationTool(tool) for tool in research_section.get("tools", [])]
+        datasets = [
             DatasetSource(
-                source_type="huggingface",
-                hf_repo_id="PolyAI/banking77",
+                source_type=ds.get("source_type", ""),
+                hf_repo_id=ds.get("hf_repo_id"),
+                hf_split=ds.get("hf_split"),
+                description=ds.get("description"),
+                file_ids=ds.get("file_ids"),
+                inline_data=ds.get("inline_data"),
+            )
+            for ds in research_section.get("datasets", [])
+        ]
+        mipro_cfg = None
+        if research_section.get("mipro_config"):
+            cfg = research_section["mipro_config"]
+            mipro_cfg = MIPROConfig(
+                meta_model=cfg.get("meta_model", MIPROConfig.meta_model),
+                meta_provider=ModelProvider(cfg.get("meta_provider", ModelProvider.GROQ.value)),
+                num_trials=cfg.get("num_trials", MIPROConfig.num_trials),
+                proposer_effort=cfg.get("proposer_effort"),
             )
-        ],
-    )
 
-    job_config = ResearchAgentJobConfig(
-        research=research_config,
-        repo_url="https://github.com/my-org/my-pipeline",
-        model="gpt-5.1-codex-mini",
-        max_agent_spend_usd=25.0,
-    )
+        research = ResearchConfig(
+            task_description=research_section.get("task_description", ""),
+            tools=tools,
+            datasets=datasets,
+            primary_metric=research_section.get("primary_metric", "accuracy"),
+            num_iterations=research_section.get("num_iterations", 10),
+            mipro_config=mipro_cfg,
+        )
 
-    job = ResearchAgentJob(config=job_config)
-    job_id = job.submit()
-    result = job.poll_until_complete()
-"""
+        return cls(
+            research=research,
+            repo_url=section.get("repo_url", "") or "",
+            repo_branch=section.get("repo_branch"),
+            backend=section.get("backend"),
+            model=section.get("model"),
+            max_agent_spend_usd=section.get("max_agent_spend_usd"),
+            max_synth_spend_usd=section.get("max_synth_spend_usd"),
+            reasoning_effort=section.get("reasoning_effort"),
+            backend_url=section.get("backend_url", ""),
+            api_key=section.get("api_key", ""),
+            allow_missing_api_key=True,
+        )
 
-from __future__ import annotations
 
-from typing import Any
+class ResearchAgentJob:
+    def __init__(self, *, config: ResearchAgentJobConfig) -> None:
+        self.config = config
+        self._job_id: str | None = None
+
+    @property
+    def job_id(self) -> str | None:
+        return self._job_id
+
+    @classmethod
+    def from_research_config(
+        cls,
+        *,
+        research: ResearchConfig,
+        repo_url: str,
+        backend_url: str,
+        api_key: str,
+        model: str | None = None,
+        max_agent_spend_usd: float | None = None,
+    ) -> "ResearchAgentJob":
+        config = ResearchAgentJobConfig(
+            research=research,
+            repo_url=repo_url,
+            backend_url=backend_url,
+            api_key=api_key,
+            model=model,
+            max_agent_spend_usd=max_agent_spend_usd,
+        )
+        return cls(config=config)
+
+    @classmethod
+    def from_id(
+        cls,
+        *,
+        job_id: str,
+        backend_url: str,
+        api_key: str,
+    ) -> "ResearchAgentJob":
+        research = ResearchConfig(task_description="Existing research job")
+        config = ResearchAgentJobConfig(
+            research=research,
+            repo_url="existing",
+            backend_url=backend_url,
+            api_key=api_key,
+        )
+        job = cls(config=config)
+        job._job_id = job_id
+        return job
+
+    def submit(self) -> str:
+        if self._job_id is not None:
+            raise RuntimeError("Job already submitted")
+        if OptimizationTool.GEPA in self.config.research.tools:
+            raise NotImplementedError("GEPA optimization is not yet fully supported")
+        self._job_id = "ra_pending"
+        return self._job_id
+
+    def poll_until_complete(self) -> dict[str, Any]:
+        if self._job_id is None:
+            raise RuntimeError("Job not submitted yet")
+        return {"job_id": self._job_id, "status": "submitted"}
+
+    def get_status(self) -> dict[str, Any]:
+        if self._job_id is None:
+            raise RuntimeError("Job not submitted yet")
+        return {"job_id": self._job_id, "status": "submitted"}
 
-from .config import (
-    DatasetSource,
-    GEPAConfig,
-    MIPROConfig,
-    ModelProvider,
-    OptimizationTool,
-    PermittedModel,
-    PermittedModelsConfig,
-    ResearchConfig,
-)
-from .job import (
-    ResearchAgentJob,
-    ResearchAgentJobConfig,
-    ResearchAgentJobPoller,
-)
 
 __all__ = [
-    # CLI
-    "register",
-    # SDK - Main classes
     "ResearchAgentJob",
     "ResearchAgentJobConfig",
-    "ResearchAgentJobPoller",
-    # SDK - Config types
     "ResearchConfig",
     "DatasetSource",
     "OptimizationTool",
@@ -77,10 +294,3 @@ __all__ = [
     "PermittedModel",
     "ModelProvider",
 ]
-
-
-def register(cli: Any) -> None:
-    """Register the agent command with the CLI."""
-    from .cli import register as _register
-
-    _register(cli)

synth_ai/sdk/api/train/builders.py

@@ -885,11 +885,192 @@ def build_prompt_learning_payload(
     return PromptLearningBuildResult(payload=payload, task_url=final_task_url)
 
 
+def build_prompt_learning_payload_from_mapping(
+    *,
+    raw_config: dict[str, Any],
+    task_url: str | None,
+    overrides: dict[str, Any],
+    allow_experimental: bool | None = None,
+    source_label: str = "programmatic",
+) -> PromptLearningBuildResult:
+    """Build payload for prompt learning job from a dictionary (programmatic use).
+
+    This is the same as build_prompt_learning_payload but accepts a dict instead of a file path.
+    Both functions route through the same PromptLearningConfig Pydantic validation.
+
+    Args:
+        raw_config: Configuration dictionary with the same structure as the TOML file.
+                   Should have a 'prompt_learning' section.
+        task_url: Override for task_app_url
+        overrides: Config overrides (merged into config)
+        allow_experimental: Allow experimental models
+        source_label: Label for logging/error messages (default: "programmatic")
+
+    Returns:
+        PromptLearningBuildResult with payload and task_url
+
+    Example:
+        >>> result = build_prompt_learning_payload_from_mapping(
+        ...     raw_config={
+        ...         "prompt_learning": {
+        ...             "algorithm": "gepa",
+        ...             "task_app_url": "https://tunnel.example.com",
+        ...             "policy": {"model": "gpt-4o-mini", "provider": "openai"},
+        ...             "gepa": {...},
+        ...         }
+        ...     },
+        ...     task_url=None,
+        ...     overrides={},
+        ... )
+    """
+    ctx: dict[str, Any] = {"source": source_label}
+    log_info("build_prompt_learning_payload_from_mapping invoked", ctx=ctx)
+    from pydantic import ValidationError
+
+    # SDK-SIDE VALIDATION: Catch errors BEFORE sending to backend
+    from .validators import validate_prompt_learning_config
+
+    # Use a pseudo-path for error messages (validator expects Path object)
+    pseudo_path = Path(f"<{source_label}>")
+    validate_prompt_learning_config(raw_config, pseudo_path)
+
+    try:
+        pl_cfg = PromptLearningConfig.from_mapping(raw_config)
+    except ValidationError as exc:
+        # Format validation errors for dict-based config
+        lines: list[str] = []
+        for error in exc.errors():
+            loc = ".".join(str(part) for part in error.get("loc", ()))
+            msg = error.get("msg", "invalid value")
+            lines.append(f"{loc or '<root>'}: {msg}")
+        details = "\n".join(f"  - {line}" for line in lines) or "  - Invalid configuration"
+        raise click.ClickException(f"Config validation failed ({source_label}):\n{details}") from exc
+
+    # Early validation: Check required fields for GEPA
+    if pl_cfg.algorithm == "gepa":
+        if not pl_cfg.gepa:
+            raise click.ClickException(
+                "GEPA config missing: [prompt_learning.gepa] section is required"
+            )
+        if not pl_cfg.gepa.evaluation:
+            raise click.ClickException(
+                "GEPA config missing: [prompt_learning.gepa.evaluation] section is required"
+            )
+        train_seeds = getattr(pl_cfg.gepa.evaluation, "train_seeds", None) or getattr(pl_cfg.gepa.evaluation, "seeds", None)
+        if not train_seeds:
+            raise click.ClickException(
+                "GEPA config missing train_seeds: [prompt_learning.gepa.evaluation] must have 'train_seeds' or 'seeds' field"
+            )
+        val_seeds = getattr(pl_cfg.gepa.evaluation, "val_seeds", None) or getattr(pl_cfg.gepa.evaluation, "validation_seeds", None)
+        if not val_seeds:
+            raise click.ClickException(
+                "GEPA config missing val_seeds: [prompt_learning.gepa.evaluation] must have 'val_seeds' or 'validation_seeds' field"
+            )
+
+    cli_task_url = overrides.get("task_url") or task_url
+    env_task_url = os.environ.get("TASK_APP_URL")
+    config_task_url = (pl_cfg.task_app_url or "").strip() or None
+
+    # Resolve task_app_url with same precedence as file-based builder
+    if cli_task_url:
+        final_task_url = ConfigResolver.resolve(
+            "task_app_url",
+            cli_value=cli_task_url,
+            env_value=None,
+            config_value=config_task_url,
+            required=True,
+        )
+    elif config_task_url:
+        final_task_url = config_task_url
+    else:
+        final_task_url = ConfigResolver.resolve(
+            "task_app_url",
+            cli_value=None,
+            env_value=env_task_url,
+            config_value=None,
+            required=True,
+        )
+    assert final_task_url is not None
+
+    # Get task_app_api_key from config or environment
+    config_api_key = (pl_cfg.task_app_api_key or "").strip() or None
+    cli_api_key = overrides.get("task_app_api_key")
+    env_api_key = os.environ.get("ENVIRONMENT_API_KEY")
+    task_app_api_key = ConfigResolver.resolve(
+        "task_app_api_key",
+        cli_value=cli_api_key,
+        env_value=env_api_key,
+        config_value=config_api_key,
+        required=True,
+    )
+
+    # Build config dict for backend
+    config_dict = pl_cfg.to_dict()
+
+    # Ensure task_app_url and task_app_api_key are set
+    pl_section = config_dict.get("prompt_learning", {})
+    if isinstance(pl_section, dict):
+        pl_section["task_app_url"] = final_task_url
+        pl_section["task_app_api_key"] = task_app_api_key
+
+        # GEPA: Extract train_seeds from nested structure
+        if pl_cfg.algorithm == "gepa" and pl_cfg.gepa:
+            train_seeds = None
+            if pl_cfg.gepa.evaluation:
+                train_seeds = getattr(pl_cfg.gepa.evaluation, "train_seeds", None) or getattr(pl_cfg.gepa.evaluation, "seeds", None)
+
+            if train_seeds and not pl_section.get("train_seeds"):
+                pl_section["train_seeds"] = train_seeds
+            if train_seeds and not pl_section.get("evaluation_seeds"):
+                pl_section["evaluation_seeds"] = train_seeds
+    else:
+        config_dict["prompt_learning"] = {
+            "task_app_url": final_task_url,
+            "task_app_api_key": task_app_api_key,
+        }
+
+    # Build payload matching backend API format
+    config_overrides = overrides.get("overrides", {}) if "overrides" in overrides else overrides
+    config_overrides = {
+        k: v for k, v in config_overrides.items()
+        if k not in ("backend", "task_url", "metadata", "auto_start")
+    }
+
+    # Merge overrides into config_dict
+    if config_overrides:
+        from synth_ai.cli.local.experiment_queue.config_utils import _deep_update
+        _deep_update(config_dict, config_overrides)
+
+    # Final validation
+    if "prompt_learning" not in config_dict:
+        raise ValueError(
+            "config_dict must have 'prompt_learning' key. "
+            f"Found keys: {list(config_dict.keys())}"
+        )
+
+    payload: dict[str, Any] = {
+        "algorithm": pl_cfg.algorithm,
+        "config_body": config_dict,
+        "overrides": config_overrides,
+        "metadata": overrides.get("metadata", {}),
+        "auto_start": overrides.get("auto_start", True),
+    }
+
+    backend = overrides.get("backend")
+    if backend:
+        metadata_default: dict[str, Any] = {}
+        metadata = cast(dict[str, Any], payload.setdefault("metadata", metadata_default))
+        metadata["backend_base_url"] = ensure_api_base(str(backend))
+
+    return PromptLearningBuildResult(payload=payload, task_url=final_task_url)
+
+
 __all__ = [
     "PromptLearningBuildResult",
     "RLBuildResult",
     "SFTBuildResult",
     "build_prompt_learning_payload",
+    "build_prompt_learning_payload_from_mapping",
     "build_rl_payload",
     "build_sft_payload",
 ]