synth-ai 0.0.0.dev2__tar.gz → 0.3.4__tar.gz

synth_ai-0.3.4/MANIFEST.in

@@ -0,0 +1,35 @@
+include README.md
+include LICENSE
+recursive-include synth_ai *.py
+# Only include lightweight config/data files from core packages
+recursive-include synth_ai *.toml
+# Include examples for task app discovery
+recursive-include examples *.py
+recursive-include examples *.toml
+recursive-include examples *.md
+recursive-include examples *.sh
+
+# Prune heavy example/demo/test/data trees from the sdist (keep code)
+prune tests
+prune private_tests
+prune synth_ai/tracing_v3/tests
+recursive-exclude synth_ai **/test_*.py
+recursive-exclude synth_ai **/tests/*
+
+# Remove large data files from package
+recursive-exclude synth_ai *.json
+recursive-exclude synth_ai *.db
+recursive-exclude synth_ai *.duckdb
+recursive-exclude synth_ai *.sqlite
+recursive-exclude synth_ai *.parquet
+recursive-exclude synth_ai *.csv
+recursive-exclude synth_ai *.npz
+recursive-exclude synth_ai *.npy
+recursive-exclude synth_ai *.pt
+recursive-exclude synth_ai *.pth
+recursive-exclude synth_ai *.bin
+recursive-exclude synth_ai *.zip
+recursive-exclude synth_ai *.tar
+global-exclude *.pyc
+global-exclude __pycache__
+global-exclude .DS_Store

synth_ai-0.3.4/PKG-INFO

@@ -0,0 +1,153 @@
+Metadata-Version: 2.4
+Name: synth-ai
+Version: 0.3.4
+Summary: Serverless Posttraining for Agents - Core AI functionality and tracing
+Author-email: Synth AI <josh@usesynth.ai>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/synth-laboratories/synth-ai
+Project-URL: Repository, https://github.com/synth-laboratories/synth-ai
+Project-URL: Issues, https://github.com/synth-laboratories/synth-ai/issues
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: requests>=2.32.3
+Requires-Dist: tqdm>=4.66.4
+Requires-Dist: typing_extensions>=4.0.0
+Requires-Dist: rich>=13.9.0
+Requires-Dist: openai>=1.99.0
+Requires-Dist: anthropic>=0.42.0
+Requires-Dist: langfuse<3.0.0,>=2.53.9
+Requires-Dist: opentelemetry-api>=1.26.0
+Requires-Dist: opentelemetry-sdk>=1.26.0
+Requires-Dist: groq>=0.30.0
+Requires-Dist: google-genai>=1.26.0
+Requires-Dist: together>=1.5.21
+Requires-Dist: mistralai>=1.9.2
+Requires-Dist: fastapi>=0.115.12
+Requires-Dist: uvicorn>=0.34.2
+Requires-Dist: numpy>=2.2.3
+Requires-Dist: networkx>=3.4.2
+Requires-Dist: sqlalchemy>=2.0.42
+Requires-Dist: celery>=5.4.0
+Requires-Dist: redis>=6.2.0
+Requires-Dist: aiosqlite>=0.21.0
+Requires-Dist: libsql>=0.1.8
+Requires-Dist: pynacl>=1.5.0
+Requires-Dist: click<8.2,>=8.1.7
+Requires-Dist: aiohttp>=3.8.0
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: modal<2.0.0,>=1.1.4
+Requires-Dist: docker>=7.0.0
+Requires-Dist: mcp>=1.21.0
+Requires-Dist: ruff>=0.12.9
+Requires-Dist: tomli_w>=1.0.0
+Requires-Dist: dspy>=3.0.4
+Requires-Dist: setuptools>=80.9.0
+Requires-Dist: gymnasium>=0.26.2
+Requires-Dist: gepa>=0.0.17
+Requires-Dist: datasets>=4.0.0
+Provides-Extra: dev
+Requires-Dist: build>=1.2.2.post1; extra == "dev"
+Requires-Dist: twine>=4.0.0; extra == "dev"
+Requires-Dist: keyring>=24.0.0; extra == "dev"
+Requires-Dist: pytest>=8.3.3; extra == "dev"
+Requires-Dist: pytest-xdist>=3.6.1; extra == "dev"
+Requires-Dist: pytest-timeout>=2.3.1; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.24.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
+Requires-Dist: pyright>=1.1.350; extra == "dev"
+Requires-Dist: coverage[toml]>=7.3.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Provides-Extra: research
+Requires-Dist: crafter>=1.8.3; extra == "research"
+Requires-Dist: datasets>=4.0.0; extra == "research"
+Provides-Extra: swe
+Requires-Dist: morphcloud>=0.1.3; extra == "swe"
+Requires-Dist: swebench>=2.3.0; extra == "swe"
+Provides-Extra: all
+Requires-Dist: crafter>=1.8.3; extra == "all"
+Requires-Dist: datasets>=4.0.0; extra == "all"
+Requires-Dist: morphcloud>=0.1.3; extra == "all"
+Requires-Dist: swebench>=2.3.0; extra == "all"
+Requires-Dist: pyboy>=2.6.0; extra == "all"
+Requires-Dist: transformers>=4.56.1; extra == "all"
+Requires-Dist: redis>=6.2.0; extra == "all"
+Provides-Extra: analytics
+Requires-Dist: pandas>=2.2.3; extra == "analytics"
+Dynamic: license-file
+
+# Synth
+
+[![Python](https://img.shields.io/badge/python-3.11+-blue)](https://www.python.org/)
+[![PyPI](https://img.shields.io/pypi/v/synth-ai.svg)](https://pypi.org/project/synth-ai/)
+[![PyPI Dev](https://img.shields.io/badge/dev-0.3.2.dev3-orange)](https://pypi.org/project/synth-ai/0.3.2.dev3/)
+[![License](https://img.shields.io/badge/license-MIT-green)](LICENSE)
+![Coverage](https://img.shields.io/badge/coverage-28.65%25-yellow)
+![Tests](https://img.shields.io/badge/tests-847%20passing-brightgreen)
+
+Serverless Posttraining APIs for Developers
+
+<p align="center">
+  <picture align="center">
+    <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/synth-laboratories/synth-ai/main/assets/langprobe_v2_dark.png">
+    <source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/synth-laboratories/synth-ai/main/assets/langprobe_v2_light.png">
+    <img alt="Shows a bar chart comparing prompt optimization performance across Synth GEPA, Synth MIPRO, GEPA (lib), DSPy MIPRO, and DSPy GEPA with baseline vs optimized." src="https://raw.githubusercontent.com/synth-laboratories/synth-ai/main/assets/langprobe_v2_light.png">
+  </picture>
+</p>
+
+<p align="center">
+  <i>Average accuracy on <a href="https://arxiv.org/abs/2502.20315">LangProBe</a> prompt optimization benchmarks.</i>
+</p>
+
+## Highlights
+
+- 🚀 Train across sft, RL, and prompt opt by standing up a single cloudflared Fastapi wrapper around your code. No production code churn.
+- ⚡️ Parallelize training and achieve 80% GPU util. via PipelineRL
+- 🗂️ Train prompts and models across multiple experiments
+- 🛠️ Spin up experiment queues and datastores locally for dev work
+- 🔩 Run serverless training via cli or programmatically
+- 🏢 Scales gpu-based model training to 64 H100s seemlessly
+- 💾 Use GEPA-calibrated judges for fast, accurate rubric scoring
+- 🖥️ Supports HTTP-based training across all programming languages
+- 🤖 CLI utilities tuned for use with Claude Code, Codex, Opencode
+
+## Getting Started
+
+```bash
+# Use with OpenAI Codex
+uvx synth-ai codex
+```
+
+```bash
+# Use with Opencode
+uvx synth-ai opencode
+```
+
+Synth is maintained by devs behind the [MIPROv2](https://scholar.google.com/citations?view_op=view_citation&hl=en&user=jauNVA8AAAAJ&citation_for_view=jauNVA8AAAAJ:u5HHmVD_uO8C) prompt optimizer.
+
+## Documentation
+
+**[docs.usesynth.ai](https://docs.usesynth.ai)**
+
+## In-Process Runner (SDK)
+
+Run GEPA/MIPRO/RL jobs against a tunneled task app without the CLI:
+
+```python
+import asyncio
+import os
+from synth_ai.sdk.task import run_in_process_job
+
+result = asyncio.run(
+    run_in_process_job(
+        job_type="prompt_learning",
+        config_path="configs/style_matching_gepa.toml",
+        task_app_path="task_apps/style_matching_task_app.py",
+        overrides={"prompt_learning.gepa.rollout.budget": 4},
+        backend_url=os.getenv("TARGET_BACKEND_BASE_URL"),  # resolves envs automatically
+    )
+)
+print(result.job_id, result.status.get("status"))
+```

synth_ai-0.3.4/README.md

@@ -0,0 +1,73 @@
+# Synth
+
+[![Python](https://img.shields.io/badge/python-3.11+-blue)](https://www.python.org/)
+[![PyPI](https://img.shields.io/pypi/v/synth-ai.svg)](https://pypi.org/project/synth-ai/)
+[![PyPI Dev](https://img.shields.io/badge/dev-0.3.2.dev3-orange)](https://pypi.org/project/synth-ai/0.3.2.dev3/)
+[![License](https://img.shields.io/badge/license-MIT-green)](LICENSE)
+![Coverage](https://img.shields.io/badge/coverage-28.65%25-yellow)
+![Tests](https://img.shields.io/badge/tests-847%20passing-brightgreen)
+
+Serverless Posttraining APIs for Developers
+
+<p align="center">
+  <picture align="center">
+    <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/synth-laboratories/synth-ai/main/assets/langprobe_v2_dark.png">
+    <source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/synth-laboratories/synth-ai/main/assets/langprobe_v2_light.png">
+    <img alt="Shows a bar chart comparing prompt optimization performance across Synth GEPA, Synth MIPRO, GEPA (lib), DSPy MIPRO, and DSPy GEPA with baseline vs optimized." src="https://raw.githubusercontent.com/synth-laboratories/synth-ai/main/assets/langprobe_v2_light.png">
+  </picture>
+</p>
+
+<p align="center">
+  <i>Average accuracy on <a href="https://arxiv.org/abs/2502.20315">LangProBe</a> prompt optimization benchmarks.</i>
+</p>
+
+## Highlights
+
+- 🚀 Train across sft, RL, and prompt opt by standing up a single cloudflared Fastapi wrapper around your code. No production code churn.
+- ⚡️ Parallelize training and achieve 80% GPU util. via PipelineRL
+- 🗂️ Train prompts and models across multiple experiments
+- 🛠️ Spin up experiment queues and datastores locally for dev work
+- 🔩 Run serverless training via cli or programmatically
+- 🏢 Scales gpu-based model training to 64 H100s seemlessly
+- 💾 Use GEPA-calibrated judges for fast, accurate rubric scoring
+- 🖥️ Supports HTTP-based training across all programming languages
+- 🤖 CLI utilities tuned for use with Claude Code, Codex, Opencode
+
+## Getting Started
+
+```bash
+# Use with OpenAI Codex
+uvx synth-ai codex
+```
+
+```bash
+# Use with Opencode
+uvx synth-ai opencode
+```
+
+Synth is maintained by devs behind the [MIPROv2](https://scholar.google.com/citations?view_op=view_citation&hl=en&user=jauNVA8AAAAJ&citation_for_view=jauNVA8AAAAJ:u5HHmVD_uO8C) prompt optimizer.
+
+## Documentation
+
+**[docs.usesynth.ai](https://docs.usesynth.ai)**
+
+## In-Process Runner (SDK)
+
+Run GEPA/MIPRO/RL jobs against a tunneled task app without the CLI:
+
+```python
+import asyncio
+import os
+from synth_ai.sdk.task import run_in_process_job
+
+result = asyncio.run(
+    run_in_process_job(
+        job_type="prompt_learning",
+        config_path="configs/style_matching_gepa.toml",
+        task_app_path="task_apps/style_matching_task_app.py",
+        overrides={"prompt_learning.gepa.rollout.budget": 4},
+        backend_url=os.getenv("TARGET_BACKEND_BASE_URL"),  # resolves envs automatically
+    )
+)
+print(result.job_id, result.status.get("status"))
+```

synth_ai-0.3.4/pyproject.toml

@@ -0,0 +1,228 @@
+[project]
+name = "synth-ai"
+version = "0.3.4"
+description = "Serverless Posttraining for Agents - Core AI functionality and tracing"
+authors = [{name = "Synth AI", email = "josh@usesynth.ai"}]
+license = "MIT"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    # Core dependencies
+    "pydantic>=2.0.0",
+    "python-dotenv>=1.0.1",
+    "requests>=2.32.3",
+    "tqdm>=4.66.4",
+    "typing_extensions>=4.0.0",
+    # AI/LLM providers
+    "rich>=13.9.0",
+    "openai>=1.99.0",
+    "anthropic>=0.42.0",
+    # Tracing and observability
+    "langfuse>=2.53.9,<3.0.0",
+    "opentelemetry-api>=1.26.0",
+    "opentelemetry-sdk>=1.26.0",
+    # Storage and caching
+    "groq>=0.30.0",
+    "google-genai>=1.26.0",
+    "together>=1.5.21",
+    "mistralai>=1.9.2",
+    # Environment framework - basic dependencies
+    "fastapi>=0.115.12",
+    "uvicorn>=0.34.2",
+    "numpy>=2.2.3",
+    "networkx>=3.4.2",
+    "sqlalchemy>=2.0.42",
+    "celery>=5.4.0",
+    "redis>=6.2.0",
+    "aiosqlite>=0.21.0",
+    "libsql>=0.1.8",
+    "pynacl>=1.5.0",
+    "click>=8.1.7,<8.2", # For CLI (pin to signature compatible with Typer)
+    "aiohttp>=3.8.0", # For async HTTP requests
+    "httpx>=0.28.1", # CLI preflights and HTTP clients (pulls httpcore)
+    "modal>=1.1.4,<2.0.0",
+    "docker>=7.0.0",
+    "mcp>=1.21.0",
+    "ruff>=0.12.9",
+    "tomli_w>=1.0.0",
+    "dspy>=3.0.4",
+    "setuptools>=80.9.0",
+    "gymnasium>=0.26.2",
+    "gepa>=0.0.17",
+    # Dataset loading - used by many task apps (banking77, math, hotpotqa, etc.)
+    "datasets>=4.0.0",
+]
+
+[project.scripts]
+synth-ai = "synth_ai.cli:cli"
+
+[project.optional-dependencies]
+dev = [
+    "build>=1.2.2.post1",
+    "twine>=4.0.0",
+    "keyring>=24.0.0",
+    "pytest>=8.3.3",
+    "pytest-xdist>=3.6.1",
+    "pytest-timeout>=2.3.1",
+    "pytest-asyncio>=0.24.0",
+    "pytest-cov>=4.1.0",
+    "pyright>=1.1.350",
+    "coverage[toml]>=7.3.0",
+    "ruff>=0.1.0",
+]
+research = [
+    # Heavy game environment dependencies
+    "crafter>=1.8.3",
+    "datasets>=4.0.0",  # For math environments
+]
+swe = [
+    "morphcloud>=0.1.3",
+    "swebench>=2.3.0",
+]
+all = [
+    # Include research and swe dependencies in all
+    "crafter>=1.8.3",
+    "datasets>=4.0.0",
+    "morphcloud>=0.1.3",
+    "swebench>=2.3.0",
+    "pyboy>=2.6.0",
+    "transformers>=4.56.1",
+    "redis>=6.2.0",
+]
+analytics = [
+    "pandas>=2.2.3",
+]
+
+[project.urls]
+Homepage = "https://github.com/synth-laboratories/synth-ai"
+Repository = "https://github.com/synth-laboratories/synth-ai"
+Issues = "https://github.com/synth-laboratories/synth-ai/issues"
+
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+include = ["synth_ai*"]
+exclude = [
+    "docs*",
+    "tests*",
+    "scripts*",
+    "private_tests*",
+    # Exclude heavy demo/test subpackages from distribution
+    "synth_ai.*agent_demos*",
+    "synth_ai.*old*",
+    "synth_ai.*tests*",
+    "synth_ai.*units*",
+    "synth_ai.tracing_v3.tests*",
+]
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.exclude-package-data]
+"synth_ai" = [
+    # Exclude heavy data inside agent_demos/old/tests
+    "**/agent_demos/**",
+    "**/old/**",
+    "**/tests/**",
+    "**/units/**",
+]
+
+[tool.setuptools.package-data]
+"synth_ai" = [
+    "demos/demo_task_apps/math/config.toml",
+    "demos/demo_task_apps/math/deploy_task_app.sh",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests", "private_tests"]
+python_files = "test_*.py"
+python_classes = "Test*"
+python_functions = "test_*"
+asyncio_mode = "auto"
+markers = [
+    "fast: marks tests as fast (≤5 seconds) - these are the default tests to run",
+    "slow: marks tests as slow (>5 seconds) - requires more time/resources",
+    "integration: marks tests as integration tests requiring external services",
+    "unit: marks tests as isolated unit tests",
+    "public: marks tests that can run in any environment",
+    "private: marks tests that may require credentials or special setup",
+]
+# Add timeout for long-running tests
+timeout = 300
+# Show test durations (slowest N tests)
+addopts = "--durations=10"
+
+[tool.coverage.run]
+source = ["synth_ai"]
+omit = ["*/tests/*", "*/test_*"]
+data_file = "coverage/.coverage"
+
+[tool.coverage.report]
+precision = 2
+show_missing = true
+skip_covered = false
+
+[tool.pyright]
+include = ["synth_ai", "tests"]
+exclude = ["**/__pycache__"]
+pythonVersion = "3.11"
+typeCheckingMode = "standard"
+reportMissingImports = true
+reportMissingTypeStubs = false
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+src = ["synth_ai"]
+extend-exclude = [
+    ".venv",
+    ".tmp",
+    "dev",
+    "docs",
+    "scripts",
+    "tests",
+    "private_tests",
+    "old",
+    "tauri",
+    "temp",
+    "traces",
+    "traces_*",
+    "synth_ai.db",
+    "synth_ai/tracing_v3/tests",
+]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "N", "UP", "YTT", "B", "C4", "T10", "SIM"]
+ignore = [
+    "E501",  # Line too long
+    "B008",  # Do not perform function call in argument defaults
+    "UP035", # Don't require `from __future__ import annotations` (not needed in 3.11+)
+    "UP006", # Allow typing.Tuple
+    "UP045", # Allow Optional[...] annotations
+]
+
+[tool.ruff.lint.per-file-ignores]
+"synth_ai/http.py" = ["F403"]
+"**/__init__.py" = ["N999"]
+"**/__main__.py" = ["N999"]
+
+[tool.ty]
+# Suppress conservative warnings about possibly-missing-attribute
+# These are false positives for dict.get() calls and dynamic attributes
+rules = { "possibly-missing-attribute" = "ignore" }
+
+[dependency-groups]
+dev = [
+    "pytest>=8.3.3",
+    "pytest-xdist>=3.6.1",
+    "pytest-timeout>=2.3.1",
+    "pytest-asyncio>=0.24.0",
+    "pytest-cov>=4.1.0",
+    "ruff>=0.1.0",
+    "pyright>=1.1.350",
+    "responses>=0.25.7",
+    "httpx>=0.28.1",
+    "crafter>=1.8.3",
+]

synth_ai-0.3.4/synth_ai/__init__.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from importlib import metadata as _metadata
+from importlib.metadata import PackageNotFoundError
+from pathlib import Path
+
+# Install log filter as early as possible to suppress noisy codex_otel logs
+try:
+    from synth_ai.core.log_filter import install_log_filter
+    install_log_filter()
+except Exception:
+    # Silently fail if log filter can't be installed
+    pass
+
+# Judge schemas moved to sdk/judging/schemas.py
+from synth_ai.sdk.judging.schemas import (
+    CriterionScorePayload,
+    JudgeOptions,
+    JudgeScoreRequest,
+    JudgeScoreResponse,
+    JudgeTaskApp,
+    JudgeTracePayload,
+    ReviewPayload,
+)
+
+try:  # Prefer the installed package metadata when available
+    __version__ = _metadata.version("synth-ai")
+except PackageNotFoundError:  # Fallback to pyproject version for editable installs
+    try:
+        import tomllib as _toml  # Python 3.11+
+    except ModuleNotFoundError:  # pragma: no cover - legacy interpreter guard
+        import tomli as _toml  # type: ignore[no-redef]
+
+    try:
+        pyproject_path = Path(__file__).resolve().parents[1] / "pyproject.toml"
+        with pyproject_path.open("rb") as fh:
+            _pyproject = _toml.load(fh)
+        __version__ = str(_pyproject["project"]["version"])
+    except Exception:
+        __version__ = "0.0.0.dev0"
+
+# Legacy tracing v1 is not required for v3 usage and can be unavailable in minimal envs.
+tracing = None  # type: ignore
+EventPartitionElement = RewardSignal = SystemTrace = TrainingQuestion = None  # type: ignore
+trace_event_async = trace_event_sync = upload = None  # type: ignore
+
+__all__ = [
+    # Judge API contracts
+    "JudgeScoreRequest",
+    "JudgeScoreResponse",
+    "JudgeOptions",
+    "JudgeTaskApp",
+    "JudgeTracePayload",
+    "ReviewPayload",
+    "CriterionScorePayload",
+]  # Explicitly define public API (v1 tracing omitted in minimal env)

synth_ai-0.3.4/synth_ai/__main__.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+"""
+Allow running synth_ai as a module: python -m synth_ai
+"""
+
+import sys
+import traceback
+
+# Log CLI invocation immediately
+if "train" in sys.argv:
+    sys.stderr.write(f"[SYNTH_AI_MAIN] Module invoked with args: {sys.argv}\n")
+    sys.stderr.flush()
+
+try:
+    from .cli import cli
+    
+    if "train" in sys.argv:
+        sys.stderr.write("[SYNTH_AI_MAIN] CLI imported successfully\n")
+        sys.stderr.flush()
+    
+    if __name__ == "__main__":
+        if "train" in sys.argv:
+            sys.stderr.write("[SYNTH_AI_MAIN] About to call cli()\n")
+            sys.stderr.flush()
+        try:
+            cli()
+        except Exception as e:
+            sys.stderr.write(f"[SYNTH_AI_MAIN] CLI call failed: {type(e).__name__}: {e}\n")
+            sys.stderr.flush()
+            traceback.print_exc(file=sys.stderr)
+            raise
+except Exception as e:
+    sys.stderr.write(f"[SYNTH_AI_MAIN] Import failed: {type(e).__name__}: {e}\n")
+    sys.stderr.flush()
+    traceback.print_exc(file=sys.stderr)
+    raise