synth-ai 0.4.1__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/products/graph_evolve/config.py

@@ -196,11 +196,11 @@ class ParetoFloorsConfig(BaseModel):
 
 
 # ============================================================================
-# ADAS Dataset Format Models
+# Graph Opt Dataset Format Models
 # ============================================================================
 
 class TaskInput(BaseModel):
-    """A single task/example in an ADAS dataset.
+    """A single task/example in a Graph Opt dataset.
 
     For POLICY graphs: Contains the problem to solve.
     For VERIFIER graphs: Contains a trace to evaluate.
@@ -267,8 +267,8 @@ class GoldOutput(BaseModel):
     score: Optional[float] = Field(default=None, ge=0.0, le=1.0, description="Gold score (0.0-1.0)")
 
 
-class ADASDatasetMetadata(BaseModel):
-    """Metadata about an ADAS dataset.
+class GraphOptDatasetMetadata(BaseModel):
+    """Metadata about a Graph Opt dataset.
 
     Provides context for graph generation and optimization.
     """
@@ -279,8 +279,8 @@ class ADASDatasetMetadata(BaseModel):
     domain: Optional[str] = Field(default=None, description="Domain (qa, code, games, etc.)")
 
 
-class ADASDataset(BaseModel):
-    """Complete ADAS dataset format for inline upload.
+class GraphOptDataset(BaseModel):
+    """Complete Graph Opt dataset format for inline upload.
 
     This is the schema for the `dataset` field in GraphOptimizationConfig
     when uploading data directly instead of using a pre-registered dataset.
@@ -303,7 +303,7 @@ class ADASDataset(BaseModel):
     """
     tasks: List[TaskInput] = Field(..., min_length=1, description="List of tasks/examples")
     gold_outputs: List[GoldOutput] = Field(..., min_length=1, description="Ground truth for each task")
-    metadata: ADASDatasetMetadata = Field(default_factory=ADASDatasetMetadata)
+    metadata: GraphOptDatasetMetadata = Field(default_factory=GraphOptDatasetMetadata)
 
     @field_validator("tasks", mode="before")
     @classmethod
@@ -443,7 +443,7 @@ class GraphOptimizationConfig(BaseModel):
     # Format: {"name": str, "task_description": str, "examples": [...]}
     dataset: Optional[Dict[str, Any]] = Field(
         default=None,
-        description="Inline dataset for upload (ADAS format). If provided, dataset_name is used as identifier."
+        description="Inline dataset for upload (GraphGen format). If provided, dataset_name is used as identifier."
     )
     
     # Task context for initial graph generation (when dataset doesn't provide it)
@@ -464,8 +464,8 @@ class GraphOptimizationConfig(BaseModel):
     )
     
     # Scoring configuration
-    scoring_strategy: str = Field(default="rubric", description="Scoring strategy: 'default', 'rubric', 'mae'")
-    judge_model: str = Field(default="gpt-4o-mini", description="Model for LLM judge scoring")
+    verifier_mode: str = Field(default="rubric", description="Verifier mode: 'rubric', 'contrastive', 'fewshot'")
+    verifier_model: str = Field(default="gpt-4o-mini", description="Model for LLM verifier scoring")
     
     @field_validator("graph_type", mode="before")
     @classmethod
@@ -529,8 +529,8 @@ class GraphOptimizationConfig(BaseModel):
             "graph_structure": self.graph_structure.value,
             "allowed_policy_models": self.allowed_policy_models,
             "dataset_config": self.dataset_config,
-            "scoring_strategy": self.scoring_strategy,
-            "judge_model": self.judge_model,
+            "verifier_mode": self.verifier_mode,
+            "verifier_model": self.verifier_model,
         }
 
         if self.max_llm_calls_per_run is not None:
@@ -551,19 +551,19 @@ class GraphOptimizationConfig(BaseModel):
         if self.dataset:
             # Validate dataset structure using Pydantic model
             try:
-                validated = ADASDataset(**self.dataset)
+                validated = GraphOptDataset(**self.dataset)
                 # Check for task ID consistency (non-fatal warnings)
                 warnings = validated.validate_task_ids()
                 if warnings:
                     import logging
                     logger = logging.getLogger(__name__)
                     for w in warnings:
-                        logger.warning(f"[ADASDataset] {w}")
+                        logger.warning(f"[GraphOptDataset] {w}")
             except Exception as e:
                 raise ValueError(
-                    f"Invalid ADAS dataset format: {e}\n"
+                    f"Invalid Graph Opt dataset format: {e}\n"
                     f"Expected format: {{'tasks': [...], 'gold_outputs': [...], 'metadata': {{...}}}}\n"
-                    f"See ADASDataset model for full schema.\n"
+                    f"See GraphOptDataset model for full schema.\n"
                     f"Got keys: {list(self.dataset.keys())}"
                 )
             request["dataset"] = self.dataset

synth_ai/products/graph_evolve/converters/__init__.py

@@ -1,7 +1,7 @@
 """Dataset converters for Graph GEPA.
 
 This module provides converters to transform common dataset formats
-into ADAS format for use with Graph GEPA optimization.
+into Graph Opt format for use with Graph GEPA optimization.
 
 Supported formats:
 - OpenAI SFT: JSONL with messages array (system, user, assistant roles)
@@ -11,13 +11,13 @@ Example:
     >>>
     >>> # Convert from file
     >>> result = convert_openai_sft("training_data.jsonl")
-    >>> adas_dataset = result.dataset
+    >>> graph_opt_dataset = result.dataset
     >>>
     >>> # Use in GraphOptimizationConfig
     >>> from synth_ai.products.graph_gepa import GraphOptimizationConfig
     >>> config = GraphOptimizationConfig(
     ...     dataset_name="my_qa_task",
-    ...     dataset=adas_dataset,
+    ...     dataset=graph_opt_dataset,
     ...     graph_type="policy",
     ...     ...
     ... )

synth_ai/products/graph_evolve/converters/openai_sft.py

@@ -1,6 +1,6 @@
-"""OpenAI SFT format to ADAS dataset converter.
+"""OpenAI SFT format to Graph Opt dataset converter.
 
-This module converts OpenAI SFT format (JSONL with messages array) to ADAS format
+This module converts OpenAI SFT format (JSONL with messages array) to Graph Opt format
 for use with Graph GEPA optimization.
 
 Example OpenAI SFT format:
@@ -10,7 +10,7 @@ Example OpenAI SFT format:
         {"role": "assistant", "content": "Paris"}
     ]}
 
-Example ADAS output:
+Example Graph Opt output:
     {
         "tasks": [{"task_id": "sft_0000", "input": {"user_message": "..."}}],
         "gold_outputs": [{"task_id": "sft_0000", "output": {"response": "..."}, "score": 1.0}],
@@ -59,10 +59,10 @@ class ConversionWarning:
 
 @dataclass
 class ConversionResult:
-    """Result of converting SFT to ADAS.
+    """Result of converting SFT to Graph Opt.
 
     Attributes:
-        dataset: The ADAS dataset dict
+        dataset: The Graph Opt dataset dict
         warnings: Non-fatal issues encountered
         stats: Conversion statistics
     """
@@ -343,7 +343,7 @@ def convert_openai_sft(
     detect_template: bool = True,
     max_examples: int | None = None,
 ) -> ConversionResult:
-    """Convert OpenAI SFT format to ADAS dataset.
+    """Convert OpenAI SFT format to Graph Opt dataset.
 
     Args:
         source: Path to JSONL file, or list of SFT example dicts
@@ -352,7 +352,7 @@ def convert_openai_sft(
         max_examples: Maximum number of examples to include (None for all)
 
     Returns:
-        ConversionResult containing the ADAS dataset, warnings, and stats
+        ConversionResult containing the Graph Opt dataset, warnings, and stats
 
     Raises:
         ConversionError: If no valid examples found

synth_ai/products/graph_evolve/examples/hotpotqa/config.toml

@@ -7,7 +7,7 @@ algorithm = "graph_gepa"
 
 # What we're optimizing
 dataset_name = "hotpotqa"
-graph_type = "policy"         # "policy" (solves tasks), "verifier" (judges results), or "rlm" (massive context via tools)
+graph_type = "policy"         # "policy" (solves tasks), "verifier" (verifies results), or "rlm" (massive context via tools)
 graph_structure = "dag"       # "single_prompt", "dag", or "conditional"
 
 # Custom topology guidance (optional - adds detail to graph_structure)

synth_ai/products/graph_gepa/__init__.py

@@ -0,0 +1,23 @@
+"""Backward-compatible Graph GEPA package alias."""
+
+from __future__ import annotations
+
+from synth_ai.products.graph_evolve import (
+    GraphOptimizationClient,
+    GraphOptimizationConfig,
+    ConversionError,
+    ConversionResult,
+    ConversionWarning,
+    convert_openai_sft,
+    preview_conversion,
+)
+
+__all__ = [
+    "GraphOptimizationConfig",
+    "GraphOptimizationClient",
+    "convert_openai_sft",
+    "preview_conversion",
+    "ConversionResult",
+    "ConversionWarning",
+    "ConversionError",
+]

synth_ai/products/graph_gepa/converters/__init__.py

@@ -0,0 +1,19 @@
+"""Graph GEPA converters (compatibility layer)."""
+
+from __future__ import annotations
+
+from synth_ai.products.graph_evolve.converters import (
+    ConversionError,
+    ConversionResult,
+    ConversionWarning,
+    convert_openai_sft,
+    preview_conversion,
+)
+
+__all__ = [
+    "convert_openai_sft",
+    "preview_conversion",
+    "ConversionResult",
+    "ConversionWarning",
+    "ConversionError",
+]

synth_ai/products/graph_gepa/converters/openai_sft.py

@@ -0,0 +1,29 @@
+"""Compatibility wrapper for OpenAI SFT converters."""
+
+from __future__ import annotations
+
+from synth_ai.products.graph_evolve.converters.openai_sft import (
+    ConversionError,
+    ConversionResult,
+    ConversionWarning,
+    convert_openai_sft,
+    detect_system_prompt,
+    extract_fields,
+    infer_template,
+    parse_sft_example,
+    preview_conversion,
+    validate_sft_examples,
+)
+
+__all__ = [
+    "ConversionError",
+    "ConversionResult",
+    "ConversionWarning",
+    "convert_openai_sft",
+    "detect_system_prompt",
+    "extract_fields",
+    "infer_template",
+    "parse_sft_example",
+    "preview_conversion",
+    "validate_sft_examples",
+]

synth_ai/sdk/__init__.py

@@ -1,19 +1,16 @@
 """Synth AI SDK Layer.
 
 This module provides the user-facing programmatic API for:
-- Training (prompt learning, SFT, RL)
+- Training (prompt learning, SFT, RL, graph generation)
 - Task apps (in-process, deployed, Modal)
-- Judging (LLM-based evaluation)
+- Graphs (verifiers, completions)
 - Inference (model inference via Synth)
-- Tracing (session traces)
-- Specs (system specifications)
-- Research agents (scaffold tuning, evaluation)
 
 Usage:
     from synth_ai.sdk import (
         PromptLearningJob,
         InProcessTaskApp,
-        JudgeClient,
+        VerifierClient,
         InferenceClient,
     )
 
@@ -24,32 +21,22 @@ Dependency rules:
 
 from __future__ import annotations
 
-# Research Agent
-from synth_ai.sdk.api.research_agent import ResearchAgentJob, ResearchAgentJobConfig
-
 # Inference
 from synth_ai.sdk.inference import InferenceClient
 
 # Jobs API Client
 from synth_ai.sdk.jobs import JobsClient
 
-# Judging
-from synth_ai.sdk.judging import JudgeClient, JudgeOptions, JudgeScoreResponse
+# Verifier types and graph clients
 from synth_ai.sdk.graphs import GraphCompletionsClient, GraphTarget, VerifierClient
-
-# Specs
-from synth_ai.sdk.specs import (
-    load_spec_from_dict,
-    load_spec_from_file,
-    spec_to_prompt_context,
-    validate_spec_dict,
-    validate_spec_file,
-)
+from synth_ai.sdk.graphs.verifier_schemas import VerifierOptions, VerifierScoreResponse
 
 # Task Apps
 from synth_ai.sdk.task import (
     InProcessJobResult,
     InProcessTaskApp,
+    LocalAPIClient,
+    LocalAPIConfig,
     TaskAppConfig,
     create_task_app,
     merge_dot_overrides,
@@ -69,10 +56,27 @@ from synth_ai.sdk.training import (
     GraphGenTask,
     GraphGenGoldOutput,
     GraphGenRubric,
-    GraphGenJudgeConfig,
+    GraphGenVerifierConfig,
     load_graphgen_taskset,
 )
 
+# Evaluation
+from synth_ai.sdk.api.eval import EvalJob, EvalJobConfig
+
+# Tunnels - commonly used functions for notebook/script usage
+from synth_ai.sdk.tunnels import (
+    rotate_tunnel,
+    open_managed_tunnel,
+    stop_tunnel,
+    track_process,
+    cleanup_all,
+    verify_tunnel_dns_resolution,
+    wait_for_health_check,
+    kill_port,
+    is_port_available,
+    find_available_port,
+)
+
 __all__ = [
     # Training
     "PromptLearningJob",
@@ -85,8 +89,11 @@ __all__ = [
     "GraphGenTask",
     "GraphGenGoldOutput",
     "GraphGenRubric",
-    "GraphGenJudgeConfig",
+    "GraphGenVerifierConfig",
     "load_graphgen_taskset",
+    # Evaluation
+    "EvalJob",
+    "EvalJobConfig",
     # Task Apps
     "InProcessTaskApp",
     "InProcessJobResult",
@@ -94,26 +101,29 @@ __all__ = [
     "resolve_backend_api_base",
     "run_in_process_job",
     "run_in_process_job_sync",
+    "LocalAPIClient",
+    "LocalAPIConfig",
     "TaskAppConfig",
     "create_task_app",
-    # Judging
-    "JudgeClient",
+    # Graphs / Verifier
     "VerifierClient",
-    "JudgeOptions",
-    "JudgeScoreResponse",
+    "VerifierOptions",
+    "VerifierScoreResponse",
     "GraphCompletionsClient",
     "GraphTarget",
     # Inference
     "InferenceClient",
-    # Specs
-    "load_spec_from_dict",
-    "load_spec_from_file",
-    "spec_to_prompt_context",
-    "validate_spec_dict",
-    "validate_spec_file",
-    # Research Agent
-    "ResearchAgentJob",
-    "ResearchAgentJobConfig",
     # Jobs API Client
     "JobsClient",
+    # Tunnels
+    "rotate_tunnel",
+    "open_managed_tunnel",
+    "stop_tunnel",
+    "track_process",
+    "cleanup_all",
+    "verify_tunnel_dns_resolution",
+    "wait_for_health_check",
+    "kill_port",
+    "is_port_available",
+    "find_available_port",
 ]

synth_ai/sdk/api/eval/__init__.py

@@ -0,0 +1,33 @@
+"""First-class SDK API for evaluation jobs.
+
+This module provides high-level abstractions for running evaluation jobs
+both via CLI and programmatically in Python scripts.
+
+Example CLI usage:
+    python -m synth_ai.cli eval --config banking77_eval.toml --backend http://localhost:8000
+
+Example SDK usage:
+    from synth_ai.sdk.api.eval import EvalJob, EvalResult
+
+    job = EvalJob(config)
+    job.submit()
+
+    # progress=True provides built-in status printing:
+    # [00:05] running | 3/10 completed
+    # [00:10] running | 7/10 completed
+    # [00:15] completed | mean_score: 0.85
+    result = job.poll_until_complete(progress=True)
+
+    # Typed result access (not raw dict)
+    if result.succeeded:
+        print(f"Mean score: {result.mean_score}")
+        print(f"Total cost: ${result.total_cost_usd:.4f}")
+
+See Also:
+    - `synth_ai.cli.commands.eval`: CLI implementation
+    - Backend API: POST /api/eval/jobs
+"""
+
+from .job import EvalJob, EvalJobConfig, EvalResult, EvalStatus
+
+__all__ = ["EvalJob", "EvalJobConfig", "EvalResult", "EvalStatus"]