synth-ai 0.2.9.dev4__py3-none-any.whl → 0.2.9.dev6__py3-none-any.whl

examples/finetuning_old/synth_qwen_v1/run_crafter_sft_job.py

@@ -1,207 +0,0 @@
-#!/usr/bin/env python3
-"""
-Crafter → SFT end-to-end runner (single script).
-
-Pipeline:
- 1) Read v3 traces DB (sqld/Turso) and filter sessions (achievements >= min)
- 2) Export OpenAI-format JSONL
- 3) Upload file, create/start SFT job, poll to terminal
- 4) (Optional) quick inference with the resulting model
-
-Usage:
-  uv run python examples/finetuning/synth_qwen_v1/run_crafter_sft_job.py --mode dev \
-    --db /Users/joshpurtell/Documents/GitHub/synth-ai/traces/v3/synth_ai.db/dbs/default/data \
-    --min-achievements 2 --output examples/finetuning/synth_qwen_v1/data/training_crafter.jsonl
-"""
-
-from __future__ import annotations
-
-import argparse
-import asyncio
-import json
-import os
-import sys
-from pathlib import Path
-from typing import Any
-
-# Repo root on sys.path for local runs
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))))
-
-from synth_ai.learning import FtClient, JobHandle, validate_training_jsonl  # type: ignore
-from synth_ai.inference import InferenceClient  # type: ignore
-from examples.finetuning.synth_qwen_v1.util import load_env, save_state  # type: ignore
-
-
-def parse_args() -> argparse.Namespace:
-    p = argparse.ArgumentParser(description="Crafter traces → SFT JSONL → FT job runner")
-    p.add_argument("--mode", choices=["local", "dev", "prod"], default=None)
-    p.add_argument(
-        "--db",
-        default=str(
-            Path(__file__).resolve().parents[3]
-            / "traces/v3/synth_ai.db/dbs/default/data"
-        ),
-        help="Path to sqld internal data file or sqlite+aiosqlite URL",
-    )
-    p.add_argument("--output", default=str(Path(__file__).parent / "data" / "training_crafter.jsonl"))
-    p.add_argument("--min-achievements", type=int, default=2)
-    p.add_argument("--max-cost", type=float, default=10.0)
-    p.add_argument("--max-tokens", type=int, default=100000)
-    p.add_argument("--model", default="Qwen/Qwen3-0.6B")
-    p.add_argument("--epochs", type=int, default=1)
-    p.add_argument("--batch-size", type=int, default=4)
-    p.add_argument("--no-infer", action="store_true")
-    p.add_argument("--models", nargs="*", help="Optional model name filter (any match)")
-    return p.parse_args()
-
-
-def _normalize_db_url(raw: str) -> str:
-    if raw.endswith(".db") and not raw.startswith("sqlite"):
-        return f"sqlite+aiosqlite:///{raw}"
-    if raw.startswith("sqlite+aiosqlite:///"):
-        return raw
-    if raw.startswith("sqlite:///") and raw.endswith(".db"):
-        return raw.replace("sqlite:///", "sqlite+aiosqlite:///")
-    return raw
-
-
-async def extract_jsonl_from_traces(db_url: str, output_path: str, cfg: dict[str, Any]) -> int:
-    # Import extractor with robust fallbacks across dist variants
-    Extractor = None
-    try:
-        from synth_ai.environments.examples.crafter_classic.agent_demos.crafter_modal_ft.filter_traces_sft_turso import (  # type: ignore
-            FinetuningDataExtractorV3 as _Ex,
-        )
-        Extractor = _Ex
-    except Exception:
-        try:
-            from synth_ai.environments.examples.crafter_classic.agent_demos.crafter_openai_ft.filter_traces_sft_turso import (  # type: ignore
-                FinetuningDataExtractorV3 as _Ex,
-            )
-            Extractor = _Ex
-        except Exception as e:
-            raise ImportError("FinetuningDataExtractorV3 not available in current build") from e
-
-    filters: dict[str, Any] = cfg.get("filters", {})
-    min_ach = int(filters.get("min_achievements", 2))
-    max_cost = float(filters.get("max_cost", 10.0))
-    max_tokens = int(filters.get("max_tokens", 100000))
-    models: list[str] = list(filters.get("models", []) or [])
-
-    kept: list[str] = []
-    async with Extractor(db_url) as ex:
-        sessions = await ex.get_all_sessions()
-        for _, row in sessions.iterrows():
-            sid = row["session_id"]
-            metrics = await ex.get_session_metrics(sid)
-            if float(metrics.get("total_cost", 0.0)) > max_cost:
-                continue
-            if int(metrics.get("total_tokens", 0) or 0) > max_tokens:
-                continue
-            # Optional model filter
-            if models:
-                model_df = await ex.db_manager.query_traces(
-                    """
-                    SELECT DISTINCT model_name
-                    FROM events
-                    WHERE session_id = :session_id
-                      AND event_type = 'cais'
-                      AND model_name IS NOT NULL
-                    """,
-                    {"session_id": sid},
-                )
-                session_models = model_df["model_name"].tolist() if model_df is not None and not model_df.empty else []
-                if not any(m in session_models for m in models):
-                    continue
-            ach = await ex.get_session_achievements(sid) or []
-            if len([a for a in ach if a]) >= min_ach:
-                kept.append(sid)
-
-        data = await ex.extract_openai_format(kept)
-        Path(output_path).parent.mkdir(parents=True, exist_ok=True)
-        with open(output_path, "w") as f:
-            for exm in data:
-                f.write(json.dumps(exm) + "\n")
-        return len(data)
-
-
-async def run(args: argparse.Namespace) -> None:
-    base_url, api_key = load_env(args.mode)
-
-    # 1) Filter and export JSONL from v3 traces
-    db_url = _normalize_db_url(args.db)
-    cfg = {
-        "mode": "trajectory",
-        "filters": {
-            "min_achievements": int(args.min_achievements),
-            "max_cost": float(args.max_cost),
-            "max_tokens": int(args.max_tokens),
-            "models": args.models or [],
-        },
-    }
-    out_path = str(Path(args.output))
-    print("Extracting SFT data from traces…")
-    n = await extract_jsonl_from_traces(db_url, out_path, cfg)
-    print(f"✅ Wrote {n} examples → {out_path}")
-
-    # 2) Validate JSONL
-    validate_training_jsonl(out_path)
-
-    # 3) Upload and create FT job
-    client = FtClient(base_url=base_url, api_key=api_key)
-    file_id = await client.upload_training_file(Path(out_path), purpose="fine-tune")
-    print(f"file_id={file_id}")
-    save_state({"file_id": file_id})
-
-    create = await client.create_sft_job(
-        model=str(args.model),
-        training_file_id=file_id,
-        hyperparameters={"n_epochs": int(args.epochs), "batch_size": int(args.batch_size)},
-        metadata={"upload_to_wasabi": True},
-    )
-    job_id = (create or {}).get("job_id")
-    if not job_id:
-        raise RuntimeError(f"create_sft_job missing job_id: {create}")
-    print(f"job_id={job_id}")
-    save_state({"job_id": job_id})
-
-    start = await client.start_job(job_id)
-    print(f"start={start}")
-
-    # 4) Poll to terminal
-    handle = JobHandle(base_url, api_key, job_id, strict=True)
-    final = await handle.poll_until_terminal(interval_seconds=2.0, max_seconds=1800)
-    status = (final or {}).get("status")
-    print(f"final_status={status}")
-    ft_model = (final or {}).get("fine_tuned_model")
-    if ft_model:
-        save_state({"fine_tuned_model": ft_model})
-        print(f"fine_tuned_model={ft_model}")
-
-    # 5) Optional inference check
-    if not args.no_infer:
-        try:
-            ic = InferenceClient(base_url=base_url, api_key=api_key)
-            model_for_infer = ft_model or str(args.model)
-            print(f"\nInference sanity check (model={model_for_infer})…")
-            resp = await ic.create_chat_completion(
-                model=model_for_infer,
-                messages=[{"role": "user", "content": "Give me a cheerful two-line greeting."}],
-                max_tokens=128,
-                temperature=0.7,
-                stream=False,
-            )
-            print(resp)
-        except Exception as e:
-            print(f"(inference skipped due to error: {e})")
-
-
-def main() -> None:
-    args = parse_args()
-    asyncio.run(run(args))
-
-
-if __name__ == "__main__":
-    main()
-
-

examples/finetuning_old/synth_qwen_v1/run_ft_job.py

@@ -1,232 +0,0 @@
-#!/usr/bin/env python3
-"""
-End-to-end SFT workflow for Qwen (single script).
-
-Steps performed:
- 1) Ensure/validate training JSONL (creates a minimal one if missing)
- 2) Upload training file → save file_id to state.json
- 3) Create SFT job (Qwen/Qwen3-0.6B by default) and start → save job_id
- 4) Poll until terminal → save fine_tuned_model when available
- 5) (Optional) Quick inference with the fine-tuned model (or base if absent)
-
-Usage:
-  uv run python examples/finetuning/synth_qwen_v1/run_ft_job.py --mode dev
-
-Options:
-  --mode {local,dev,prod}     Backend mode/environment (default: env override or prod)
-  --data PATH                 Path to training JSONL (default: ./data/training.jsonl)
-  --model NAME                Base model for SFT (default: Qwen/Qwen3-0.6B)
-  --epochs N                  Epochs (default: 1)
-  --batch-size N              Batch size (default: 4)
-  --no-infer                  Skip the post-training inference check
-"""
-
-from __future__ import annotations
-
-import argparse
-import asyncio
-import json
-import os
-import sys
-from pathlib import Path
-from typing import Any
-
-# Make repo root importable when running directly
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))))
-
-from synth_ai.config.base_url import get_backend_from_env
-from synth_ai.learning import FtClient, JobHandle, validate_training_jsonl  # type: ignore
-from synth_ai.inference import InferenceClient  # type: ignore
-from examples.finetuning.synth_qwen_v1.util import load_env, load_state, save_state  # type: ignore
-
-try:
-    from examples.common.backend import resolve_backend_url as _resolve_backend_default  # type: ignore
-except Exception:  # pragma: no cover - fallback for direct execution
-
-    def _resolve_backend_default() -> str:
-        base, _ = get_backend_from_env()
-        base = base.rstrip("/")
-        return base if base.endswith("/api") else f"{base}/api"
-
-
-def parse_args() -> argparse.Namespace:
-    p = argparse.ArgumentParser(description="Run Qwen SFT end-to-end")
-    p.add_argument("--mode", choices=["prod", "dev", "local"], default=None)
-    p.add_argument("--data", default=str(Path(__file__).parent / "data" / "training_crafter.jsonl"))
-    p.add_argument("--model", default="Qwen/Qwen3-0.6B")
-    p.add_argument("--epochs", type=int, default=1)
-    p.add_argument("--batch-size", type=int, default=4, dest="batch_size")
-    p.add_argument("--no-infer", action="store_true")
-    return p.parse_args()
-
-
-def ensure_training_jsonl(path: Path) -> Path:
-    path.parent.mkdir(parents=True, exist_ok=True)
-    if not path.exists():
-        # Minimal JSONL with a single example
-        lines: list[str] = [
-            json.dumps({
-                "messages": [
-                    {"role": "user", "content": "Write a short greeting."},
-                    {"role": "assistant", "content": "Hello there!"},
-                ]
-            })
-        ]
-        path.write_text("\n".join(lines) + "\n")
-    # Validate using shared SDK validator
-    validate_training_jsonl(path)
-    return path
-
-
-async def run(args: argparse.Namespace) -> None:
-    # Resolve backend and key
-    base_url, api_key = load_env(args.mode)
-    # Force canonical prod base when prod mode (or override) is selected
-    try:
-        if (args.mode == "prod") or (os.getenv("SYNTH_BACKEND_URL_OVERRIDE", "").strip().lower() == "prod"):
-            base_url = _resolve_backend_default()
-            # Also export for any downstream helpers that read env
-            os.environ["PROD_BACKEND_URL"] = base_url
-    except Exception:
-        pass
-
-    # Ensure/validate training JSONL
-    data_path = ensure_training_jsonl(Path(args.data))
-    print(f"Training JSONL: {data_path}")
-
-    # Upload file
-    ft = FtClient(base_url=base_url, api_key=api_key)
-    file_id = await ft.upload_training_file(data_path, purpose="fine-tune")
-    if not file_id:
-        raise RuntimeError("upload_training_file returned empty file_id")
-    print(f"file_id={file_id}")
-    save_state({"file_id": file_id})
-
-    # Create job
-    hyperparameters: dict[str, Any] = {
-        "n_epochs": int(args.epochs),
-        "batch_size": int(args.batch_size),
-    }
-    # Include explicit compute topology for billing/inference resolution.
-    # Default: 1x A10G (can be surfaced via CLI later if needed).
-    metadata = {
-        "upload_to_wasabi": True,
-        # Normalized effective config consumed by the backend SFT workflow
-        "effective_config": {
-            "compute": {
-                "gpu_type": "A10G",
-                "gpu_count": 1,
-                "nodes": 1,
-            },
-            "data": {
-                "topology": {
-                    "gpu_type": "A10G",
-                    "container_count": 1,
-                }
-            }
-        }
-    }
-
-    create_resp = await ft.create_sft_job(
-        model=str(args.model),
-        training_file_id=file_id,
-        hyperparameters=hyperparameters,
-        metadata=metadata,
-    )
-    job_id = (create_resp or {}).get("job_id")
-    if not job_id:
-        raise RuntimeError(f"create_sft_job missing job_id: {create_resp}")
-    print(f"job_id={job_id}")
-    save_state({"job_id": job_id})
-
-    # Start job
-    start_resp = await ft.start_job(job_id)
-    print(f"start={start_resp}")
-
-    # Poll until terminal with streaming event/metric logs
-    def _on_event(e: dict[str, Any]) -> None:
-        try:
-            seq = e.get("seq")
-            etype = e.get("type") or e.get("event_type")
-            msg = e.get("message")
-            print(f"event seq={seq} type={etype} msg={msg}")
-        except Exception:
-            pass
-
-    def _on_metric(p: dict[str, Any]) -> None:
-        try:
-            name = str(p.get("name") or "")
-            step = p.get("step")
-            epoch = p.get("epoch")
-            val = p.get("value")
-            print(f"metric {name} step={step} epoch={epoch} value={val}")
-        except Exception:
-            pass
-
-    handle = JobHandle(base_url, api_key, job_id, strict=True)
-    final = await handle.poll_until_terminal(
-        interval_seconds=2.0,
-        max_seconds=1800,
-        on_event=_on_event,
-        on_metric=_on_metric,
-    )
-    status = (final or {}).get("status")
-    print(f"final_status={status}")
-    ft_model = (final or {}).get("fine_tuned_model")
-    if ft_model:
-        print(f"fine_tuned_model={ft_model}")
-        save_state({"fine_tuned_model": ft_model})
-
-    # Optional: quick inference check
-    if not args.no_infer:
-        model_for_infer = ft_model or str(args.model)
-        try:
-            ic = InferenceClient(base_url=base_url, api_key=api_key, timeout=600.0)
-            print(f"\nInference sanity check (model={model_for_infer})…")
-            resp = await ic.create_chat_completion(
-                model=model_for_infer,
-                messages=[{"role": "user", "content": "Give me a cheerful two-line greeting."}],
-                max_tokens=128,
-                temperature=0.7,
-                stream=False,
-            )
-            print(resp)
-        except Exception as e:
-            # Always print full error details and traceback
-            import traceback
-            try:
-                from synth_ai.http import HTTPError  # type: ignore
-            except Exception:  # pragma: no cover - fallback if import shape changes
-                HTTPError = tuple()  # type: ignore
-            print("\n===== Inference Error =====")
-            print(f"Type: {type(e).__name__}")
-            print(f"Repr: {repr(e)}")
-            tb = traceback.format_exc()
-            if tb:
-                print("Traceback:")
-                print(tb)
-            # If HTTP error from backend, surface structured fields
-            if 'HTTPError' in str(type(e)) or (isinstance((), tuple) and False):
-                pass
-            try:
-                if HTTPError and isinstance(e, HTTPError):  # type: ignore[arg-type]
-                    print("HTTPError details:")
-                    print(f"  status={e.status}")
-                    print(f"  url={e.url}")
-                    print(f"  message={e.message}")
-                    if getattr(e, 'detail', None) is not None:
-                        print(f"  detail={e.detail}")
-                    if getattr(e, 'body_snippet', None):
-                        print(f"  body_snippet={e.body_snippet}")
-            except Exception:
-                pass
-            print("===== End Inference Error =====\n")
-
-
-def main() -> None:
-    args = parse_args()
-    asyncio.run(run(args))
-
-
-if __name__ == "__main__":
-    main()

examples/finetuning_old/synth_qwen_v1/upload_data.py

@@ -1,34 +0,0 @@
-from __future__ import annotations
-
-from pathlib import Path
-
-import asyncio
-
-import sys
-import os
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))))
-
-from synth_ai.learning import FtClient, validate_training_jsonl
-from examples.finetuning.synth_qwen_v1.util import load_env, save_state, parse_args
-
-
-async def _run(mode: str | None) -> None:
-    base, key = load_env(mode)
-    client = FtClient(base_url=base, api_key=key)
-
-    p = Path(__file__).parent / "data" / "training.jsonl"
-    # Use shared validator from synth_ai.learning.validators
-    validate_training_jsonl(p)
-    file_id = await client.upload_training_file(p, purpose="fine-tune")
-    print(f"file_id={file_id}")
-    save_state({"file_id": file_id})
-
-
-def main() -> None:
-    args = parse_args()
-    asyncio.run(_run(args.mode))
-
-
-if __name__ == "__main__":
-    main()
-

examples/finetuning_old/synth_qwen_v1/util.py

@@ -1,147 +0,0 @@
-from __future__ import annotations
-
-import os
-import json
-import argparse
-from pathlib import Path
-from typing import Any, Dict
-
-from synth_ai.config.base_url import get_backend_from_env
-
-try:
-    from dotenv import load_dotenv  # type: ignore[reportMissingImports]
-except Exception:  # pragma: no cover
-    def load_dotenv(*args, **kwargs):  # type: ignore[no-redef]
-        return False
-
-
-STATE_PATH = Path(__file__).parent / "state.json"
-
-
-def _default_backend_url() -> str:
-    base, _ = get_backend_from_env()
-    base = base.rstrip("/")
-    return base if base.endswith("/api") else f"{base}/api"
-
-
-def load_env(mode: str | None = None) -> tuple[str, str]:
-    """Resolve backend base_url and api_key.
-
-    Precedence:
-    - SYNTH_BACKEND_URL_OVERRIDE=local|dev|prod (preferred)
-    - explicit mode arg (local|dev|prod)
-    - default prod
-    """
-    load_dotenv()
-    # Prefer global override if present
-    override = (os.getenv("SYNTH_BACKEND_URL_OVERRIDE", "") or "").strip().lower()
-    if override in {"local", "dev", "prod"}:
-        base, key = get_backend_from_env()
-        base = base.rstrip("/")
-        print(f"SYNTH backend: {base} (override={override})")
-        # Print masked API key and source for clarity
-        src = ""
-        if override == "prod":
-            if key and key == os.getenv("PROD_SYNTH_API_KEY", "").strip():
-                src = "PROD_SYNTH_API_KEY"
-            elif key and key == os.getenv("TESTING_PROD_SYNTH_API_KEY", "").strip():
-                src = "TESTING_PROD_SYNTH_API_KEY"
-            elif key and key == os.getenv("SYNTH_API_KEY", "").strip():
-                src = "SYNTH_API_KEY"
-        elif override == "dev":
-            if key and key == os.getenv("DEV_SYNTH_API_KEY", "").strip():
-                src = "DEV_SYNTH_API_KEY"
-        else:  # local
-            if key and key == os.getenv("DEV_SYNTH_API_KEY", "").strip():
-                src = "DEV_SYNTH_API_KEY"
-            elif key and key == os.getenv("TESTING_LOCAL_SYNTH_API_KEY", "").strip():
-                src = "TESTING_LOCAL_SYNTH_API_KEY"
-        masked = ("*" * max(0, len(key) - 6)) + key[-6:] if key else "<empty>"
-        print(f"SYNTH api key: {masked} (len={len(key)}, src={src or '<unknown>'})")
-        return base, key
-
-    # Fallback to explicit mode
-    if mode is None:
-        mode = os.getenv("SYNTH_MODE", "prod").strip().lower()
-    if mode == "local":
-        base_url = os.getenv("LOCAL_BACKEND_URL", "").strip()
-        # Prefer DEV_SYNTH_API_KEY for local development; fall back to legacy var
-        api_key = (
-            os.getenv("DEV_SYNTH_API_KEY", "").strip()
-            or os.getenv("TESTING_LOCAL_SYNTH_API_KEY", "").strip()
-        )
-        if not base_url or not api_key:
-            raise RuntimeError("Missing LOCAL_BACKEND_URL or DEV_SYNTH_API_KEY/TESTING_LOCAL_SYNTH_API_KEY in environment/.env")
-    elif mode == "dev":
-        base_url = os.getenv("DEV_BACKEND_URL", "").strip()
-        api_key = os.getenv("DEV_SYNTH_API_KEY", "").strip()
-        if not base_url or not api_key:
-            raise RuntimeError("Missing DEV_BACKEND_URL or DEV_SYNTH_API_KEY in environment/.env")
-    else:  # prod
-        base_url = os.getenv("PROD_BACKEND_URL", "").strip() or _default_backend_url()
-        api_key = (
-            os.getenv("PROD_SYNTH_API_KEY", "").strip()
-            or os.getenv("TESTING_PROD_SYNTH_API_KEY", "").strip()
-            or os.getenv("SYNTH_API_KEY", "").strip()
-        )
-        if not api_key:
-            raise RuntimeError("Missing PROD_SYNTH_API_KEY/TESTING_PROD_SYNTH_API_KEY/SYNTH_API_KEY in environment/.env")
-    base_url = base_url.rstrip("/")
-    print(f"SYNTH backend: {base_url} (mode={mode})")
-    # Also print masked API key and source
-    src = ""
-    if mode == "prod":
-        if api_key and api_key == os.getenv("PROD_SYNTH_API_KEY", "").strip():
-            src = "PROD_SYNTH_API_KEY"
-        elif api_key and api_key == os.getenv("TESTING_PROD_SYNTH_API_KEY", "").strip():
-            src = "TESTING_PROD_SYNTH_API_KEY"
-        elif api_key and api_key == os.getenv("SYNTH_API_KEY", "").strip():
-            src = "SYNTH_API_KEY"
-    elif mode == "dev":
-        if api_key and api_key == os.getenv("DEV_SYNTH_API_KEY", "").strip():
-            src = "DEV_SYNTH_API_KEY"
-    else:
-        if api_key and api_key == os.getenv("DEV_SYNTH_API_KEY", "").strip():
-            src = "DEV_SYNTH_API_KEY"
-        elif api_key and api_key == os.getenv("TESTING_LOCAL_SYNTH_API_KEY", "").strip():
-            src = "TESTING_LOCAL_SYNTH_API_KEY"
-    masked = ("*" * max(0, len(api_key) - 6)) + api_key[-6:] if api_key else "<empty>"
-    print(f"SYNTH api key: {masked} (len={len(api_key)}, src={src or '<unknown>'})")
-    return base_url, api_key
-
-
-def parse_args() -> argparse.Namespace:
-    p = argparse.ArgumentParser()
-    p.add_argument("--mode", choices=["prod", "dev", "local"], default=None, help="Backend mode")
-    return p.parse_args()
-
-
-def save_state(obj: Dict[str, Any]) -> None:
-    prev: Dict[str, Any] = {}
-    if STATE_PATH.exists():
-        try:
-            prev = json.loads(STATE_PATH.read_text())
-        except Exception:
-            prev = {}
-    prev.update(obj)
-    STATE_PATH.write_text(json.dumps(prev, indent=2))
-
-
-def load_state() -> Dict[str, Any]:
-    if not STATE_PATH.exists():
-        return {}
-    try:
-        return json.loads(STATE_PATH.read_text())
-    except Exception:
-        return {}
-
-
-def validate_jsonl(path: str | Path) -> None:
-    """Backwards-compatible wrapper that delegates to shared SDK validator.
-
-    Prefer synth_ai.learning.validators.validate_training_jsonl to keep a single source
-    of JSONL validation rules used across examples and tests.
-    """
-    from synth_ai.learning import validate_training_jsonl
-
-    validate_training_jsonl(path)