synth-ai 0.2.9.dev2__py3-none-any.whl → 0.2.9.dev4__py3-none-any.whl

examples/finetuning_old/synth_qwen_v1/run_crafter_sft_job.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+"""
+Crafter → SFT end-to-end runner (single script).
+
+Pipeline:
+ 1) Read v3 traces DB (sqld/Turso) and filter sessions (achievements >= min)
+ 2) Export OpenAI-format JSONL
+ 3) Upload file, create/start SFT job, poll to terminal
+ 4) (Optional) quick inference with the resulting model
+
+Usage:
+  uv run python examples/finetuning/synth_qwen_v1/run_crafter_sft_job.py --mode dev \
+    --db /Users/joshpurtell/Documents/GitHub/synth-ai/traces/v3/synth_ai.db/dbs/default/data \
+    --min-achievements 2 --output examples/finetuning/synth_qwen_v1/data/training_crafter.jsonl
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+# Repo root on sys.path for local runs
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))))
+
+from synth_ai.learning import FtClient, JobHandle, validate_training_jsonl  # type: ignore
+from synth_ai.inference import InferenceClient  # type: ignore
+from examples.finetuning.synth_qwen_v1.util import load_env, save_state  # type: ignore
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description="Crafter traces → SFT JSONL → FT job runner")
+    p.add_argument("--mode", choices=["local", "dev", "prod"], default=None)
+    p.add_argument(
+        "--db",
+        default=str(
+            Path(__file__).resolve().parents[3]
+            / "traces/v3/synth_ai.db/dbs/default/data"
+        ),
+        help="Path to sqld internal data file or sqlite+aiosqlite URL",
+    )
+    p.add_argument("--output", default=str(Path(__file__).parent / "data" / "training_crafter.jsonl"))
+    p.add_argument("--min-achievements", type=int, default=2)
+    p.add_argument("--max-cost", type=float, default=10.0)
+    p.add_argument("--max-tokens", type=int, default=100000)
+    p.add_argument("--model", default="Qwen/Qwen3-0.6B")
+    p.add_argument("--epochs", type=int, default=1)
+    p.add_argument("--batch-size", type=int, default=4)
+    p.add_argument("--no-infer", action="store_true")
+    p.add_argument("--models", nargs="*", help="Optional model name filter (any match)")
+    return p.parse_args()
+
+
+def _normalize_db_url(raw: str) -> str:
+    if raw.endswith(".db") and not raw.startswith("sqlite"):
+        return f"sqlite+aiosqlite:///{raw}"
+    if raw.startswith("sqlite+aiosqlite:///"):
+        return raw
+    if raw.startswith("sqlite:///") and raw.endswith(".db"):
+        return raw.replace("sqlite:///", "sqlite+aiosqlite:///")
+    return raw
+
+
+async def extract_jsonl_from_traces(db_url: str, output_path: str, cfg: dict[str, Any]) -> int:
+    # Import extractor with robust fallbacks across dist variants
+    Extractor = None
+    try:
+        from synth_ai.environments.examples.crafter_classic.agent_demos.crafter_modal_ft.filter_traces_sft_turso import (  # type: ignore
+            FinetuningDataExtractorV3 as _Ex,
+        )
+        Extractor = _Ex
+    except Exception:
+        try:
+            from synth_ai.environments.examples.crafter_classic.agent_demos.crafter_openai_ft.filter_traces_sft_turso import (  # type: ignore
+                FinetuningDataExtractorV3 as _Ex,
+            )
+            Extractor = _Ex
+        except Exception as e:
+            raise ImportError("FinetuningDataExtractorV3 not available in current build") from e
+
+    filters: dict[str, Any] = cfg.get("filters", {})
+    min_ach = int(filters.get("min_achievements", 2))
+    max_cost = float(filters.get("max_cost", 10.0))
+    max_tokens = int(filters.get("max_tokens", 100000))
+    models: list[str] = list(filters.get("models", []) or [])
+
+    kept: list[str] = []
+    async with Extractor(db_url) as ex:
+        sessions = await ex.get_all_sessions()
+        for _, row in sessions.iterrows():
+            sid = row["session_id"]
+            metrics = await ex.get_session_metrics(sid)
+            if float(metrics.get("total_cost", 0.0)) > max_cost:
+                continue
+            if int(metrics.get("total_tokens", 0) or 0) > max_tokens:
+                continue
+            # Optional model filter
+            if models:
+                model_df = await ex.db_manager.query_traces(
+                    """
+                    SELECT DISTINCT model_name
+                    FROM events
+                    WHERE session_id = :session_id
+                      AND event_type = 'cais'
+                      AND model_name IS NOT NULL
+                    """,
+                    {"session_id": sid},
+                )
+                session_models = model_df["model_name"].tolist() if model_df is not None and not model_df.empty else []
+                if not any(m in session_models for m in models):
+                    continue
+            ach = await ex.get_session_achievements(sid) or []
+            if len([a for a in ach if a]) >= min_ach:
+                kept.append(sid)
+
+        data = await ex.extract_openai_format(kept)
+        Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+        with open(output_path, "w") as f:
+            for exm in data:
+                f.write(json.dumps(exm) + "\n")
+        return len(data)
+
+
+async def run(args: argparse.Namespace) -> None:
+    base_url, api_key = load_env(args.mode)
+
+    # 1) Filter and export JSONL from v3 traces
+    db_url = _normalize_db_url(args.db)
+    cfg = {
+        "mode": "trajectory",
+        "filters": {
+            "min_achievements": int(args.min_achievements),
+            "max_cost": float(args.max_cost),
+            "max_tokens": int(args.max_tokens),
+            "models": args.models or [],
+        },
+    }
+    out_path = str(Path(args.output))
+    print("Extracting SFT data from traces…")
+    n = await extract_jsonl_from_traces(db_url, out_path, cfg)
+    print(f"✅ Wrote {n} examples → {out_path}")
+
+    # 2) Validate JSONL
+    validate_training_jsonl(out_path)
+
+    # 3) Upload and create FT job
+    client = FtClient(base_url=base_url, api_key=api_key)
+    file_id = await client.upload_training_file(Path(out_path), purpose="fine-tune")
+    print(f"file_id={file_id}")
+    save_state({"file_id": file_id})
+
+    create = await client.create_sft_job(
+        model=str(args.model),
+        training_file_id=file_id,
+        hyperparameters={"n_epochs": int(args.epochs), "batch_size": int(args.batch_size)},
+        metadata={"upload_to_wasabi": True},
+    )
+    job_id = (create or {}).get("job_id")
+    if not job_id:
+        raise RuntimeError(f"create_sft_job missing job_id: {create}")
+    print(f"job_id={job_id}")
+    save_state({"job_id": job_id})
+
+    start = await client.start_job(job_id)
+    print(f"start={start}")
+
+    # 4) Poll to terminal
+    handle = JobHandle(base_url, api_key, job_id, strict=True)
+    final = await handle.poll_until_terminal(interval_seconds=2.0, max_seconds=1800)
+    status = (final or {}).get("status")
+    print(f"final_status={status}")
+    ft_model = (final or {}).get("fine_tuned_model")
+    if ft_model:
+        save_state({"fine_tuned_model": ft_model})
+        print(f"fine_tuned_model={ft_model}")
+
+    # 5) Optional inference check
+    if not args.no_infer:
+        try:
+            ic = InferenceClient(base_url=base_url, api_key=api_key)
+            model_for_infer = ft_model or str(args.model)
+            print(f"\nInference sanity check (model={model_for_infer})…")
+            resp = await ic.create_chat_completion(
+                model=model_for_infer,
+                messages=[{"role": "user", "content": "Give me a cheerful two-line greeting."}],
+                max_tokens=128,
+                temperature=0.7,
+                stream=False,
+            )
+            print(resp)
+        except Exception as e:
+            print(f"(inference skipped due to error: {e})")
+
+
+def main() -> None:
+    args = parse_args()
+    asyncio.run(run(args))
+
+
+if __name__ == "__main__":
+    main()
+
+

examples/finetuning_old/synth_qwen_v1/run_ft_job.py

@@ -0,0 +1,232 @@
+#!/usr/bin/env python3
+"""
+End-to-end SFT workflow for Qwen (single script).
+
+Steps performed:
+ 1) Ensure/validate training JSONL (creates a minimal one if missing)
+ 2) Upload training file → save file_id to state.json
+ 3) Create SFT job (Qwen/Qwen3-0.6B by default) and start → save job_id
+ 4) Poll until terminal → save fine_tuned_model when available
+ 5) (Optional) Quick inference with the fine-tuned model (or base if absent)
+
+Usage:
+  uv run python examples/finetuning/synth_qwen_v1/run_ft_job.py --mode dev
+
+Options:
+  --mode {local,dev,prod}     Backend mode/environment (default: env override or prod)
+  --data PATH                 Path to training JSONL (default: ./data/training.jsonl)
+  --model NAME                Base model for SFT (default: Qwen/Qwen3-0.6B)
+  --epochs N                  Epochs (default: 1)
+  --batch-size N              Batch size (default: 4)
+  --no-infer                  Skip the post-training inference check
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+# Make repo root importable when running directly
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))))
+
+from synth_ai.config.base_url import get_backend_from_env
+from synth_ai.learning import FtClient, JobHandle, validate_training_jsonl  # type: ignore
+from synth_ai.inference import InferenceClient  # type: ignore
+from examples.finetuning.synth_qwen_v1.util import load_env, load_state, save_state  # type: ignore
+
+try:
+    from examples.common.backend import resolve_backend_url as _resolve_backend_default  # type: ignore
+except Exception:  # pragma: no cover - fallback for direct execution
+
+    def _resolve_backend_default() -> str:
+        base, _ = get_backend_from_env()
+        base = base.rstrip("/")
+        return base if base.endswith("/api") else f"{base}/api"
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description="Run Qwen SFT end-to-end")
+    p.add_argument("--mode", choices=["prod", "dev", "local"], default=None)
+    p.add_argument("--data", default=str(Path(__file__).parent / "data" / "training_crafter.jsonl"))
+    p.add_argument("--model", default="Qwen/Qwen3-0.6B")
+    p.add_argument("--epochs", type=int, default=1)
+    p.add_argument("--batch-size", type=int, default=4, dest="batch_size")
+    p.add_argument("--no-infer", action="store_true")
+    return p.parse_args()
+
+
+def ensure_training_jsonl(path: Path) -> Path:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if not path.exists():
+        # Minimal JSONL with a single example
+        lines: list[str] = [
+            json.dumps({
+                "messages": [
+                    {"role": "user", "content": "Write a short greeting."},
+                    {"role": "assistant", "content": "Hello there!"},
+                ]
+            })
+        ]
+        path.write_text("\n".join(lines) + "\n")
+    # Validate using shared SDK validator
+    validate_training_jsonl(path)
+    return path
+
+
+async def run(args: argparse.Namespace) -> None:
+    # Resolve backend and key
+    base_url, api_key = load_env(args.mode)
+    # Force canonical prod base when prod mode (or override) is selected
+    try:
+        if (args.mode == "prod") or (os.getenv("SYNTH_BACKEND_URL_OVERRIDE", "").strip().lower() == "prod"):
+            base_url = _resolve_backend_default()
+            # Also export for any downstream helpers that read env
+            os.environ["PROD_BACKEND_URL"] = base_url
+    except Exception:
+        pass
+
+    # Ensure/validate training JSONL
+    data_path = ensure_training_jsonl(Path(args.data))
+    print(f"Training JSONL: {data_path}")
+
+    # Upload file
+    ft = FtClient(base_url=base_url, api_key=api_key)
+    file_id = await ft.upload_training_file(data_path, purpose="fine-tune")
+    if not file_id:
+        raise RuntimeError("upload_training_file returned empty file_id")
+    print(f"file_id={file_id}")
+    save_state({"file_id": file_id})
+
+    # Create job
+    hyperparameters: dict[str, Any] = {
+        "n_epochs": int(args.epochs),
+        "batch_size": int(args.batch_size),
+    }
+    # Include explicit compute topology for billing/inference resolution.
+    # Default: 1x A10G (can be surfaced via CLI later if needed).
+    metadata = {
+        "upload_to_wasabi": True,
+        # Normalized effective config consumed by the backend SFT workflow
+        "effective_config": {
+            "compute": {
+                "gpu_type": "A10G",
+                "gpu_count": 1,
+                "nodes": 1,
+            },
+            "data": {
+                "topology": {
+                    "gpu_type": "A10G",
+                    "container_count": 1,
+                }
+            }
+        }
+    }
+
+    create_resp = await ft.create_sft_job(
+        model=str(args.model),
+        training_file_id=file_id,
+        hyperparameters=hyperparameters,
+        metadata=metadata,
+    )
+    job_id = (create_resp or {}).get("job_id")
+    if not job_id:
+        raise RuntimeError(f"create_sft_job missing job_id: {create_resp}")
+    print(f"job_id={job_id}")
+    save_state({"job_id": job_id})
+
+    # Start job
+    start_resp = await ft.start_job(job_id)
+    print(f"start={start_resp}")
+
+    # Poll until terminal with streaming event/metric logs
+    def _on_event(e: dict[str, Any]) -> None:
+        try:
+            seq = e.get("seq")
+            etype = e.get("type") or e.get("event_type")
+            msg = e.get("message")
+            print(f"event seq={seq} type={etype} msg={msg}")
+        except Exception:
+            pass
+
+    def _on_metric(p: dict[str, Any]) -> None:
+        try:
+            name = str(p.get("name") or "")
+            step = p.get("step")
+            epoch = p.get("epoch")
+            val = p.get("value")
+            print(f"metric {name} step={step} epoch={epoch} value={val}")
+        except Exception:
+            pass
+
+    handle = JobHandle(base_url, api_key, job_id, strict=True)
+    final = await handle.poll_until_terminal(
+        interval_seconds=2.0,
+        max_seconds=1800,
+        on_event=_on_event,
+        on_metric=_on_metric,
+    )
+    status = (final or {}).get("status")
+    print(f"final_status={status}")
+    ft_model = (final or {}).get("fine_tuned_model")
+    if ft_model:
+        print(f"fine_tuned_model={ft_model}")
+        save_state({"fine_tuned_model": ft_model})
+
+    # Optional: quick inference check
+    if not args.no_infer:
+        model_for_infer = ft_model or str(args.model)
+        try:
+            ic = InferenceClient(base_url=base_url, api_key=api_key, timeout=600.0)
+            print(f"\nInference sanity check (model={model_for_infer})…")
+            resp = await ic.create_chat_completion(
+                model=model_for_infer,
+                messages=[{"role": "user", "content": "Give me a cheerful two-line greeting."}],
+                max_tokens=128,
+                temperature=0.7,
+                stream=False,
+            )
+            print(resp)
+        except Exception as e:
+            # Always print full error details and traceback
+            import traceback
+            try:
+                from synth_ai.http import HTTPError  # type: ignore
+            except Exception:  # pragma: no cover - fallback if import shape changes
+                HTTPError = tuple()  # type: ignore
+            print("\n===== Inference Error =====")
+            print(f"Type: {type(e).__name__}")
+            print(f"Repr: {repr(e)}")
+            tb = traceback.format_exc()
+            if tb:
+                print("Traceback:")
+                print(tb)
+            # If HTTP error from backend, surface structured fields
+            if 'HTTPError' in str(type(e)) or (isinstance((), tuple) and False):
+                pass
+            try:
+                if HTTPError and isinstance(e, HTTPError):  # type: ignore[arg-type]
+                    print("HTTPError details:")
+                    print(f"  status={e.status}")
+                    print(f"  url={e.url}")
+                    print(f"  message={e.message}")
+                    if getattr(e, 'detail', None) is not None:
+                        print(f"  detail={e.detail}")
+                    if getattr(e, 'body_snippet', None):
+                        print(f"  body_snippet={e.body_snippet}")
+            except Exception:
+                pass
+            print("===== End Inference Error =====\n")
+
+
+def main() -> None:
+    args = parse_args()
+    asyncio.run(run(args))
+
+
+if __name__ == "__main__":
+    main()

examples/finetuning_old/synth_qwen_v1/upload_data.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import asyncio
+
+import sys
+import os
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))))
+
+from synth_ai.learning import FtClient, validate_training_jsonl
+from examples.finetuning.synth_qwen_v1.util import load_env, save_state, parse_args
+
+
+async def _run(mode: str | None) -> None:
+    base, key = load_env(mode)
+    client = FtClient(base_url=base, api_key=key)
+
+    p = Path(__file__).parent / "data" / "training.jsonl"
+    # Use shared validator from synth_ai.learning.validators
+    validate_training_jsonl(p)
+    file_id = await client.upload_training_file(p, purpose="fine-tune")
+    print(f"file_id={file_id}")
+    save_state({"file_id": file_id})
+
+
+def main() -> None:
+    args = parse_args()
+    asyncio.run(_run(args.mode))
+
+
+if __name__ == "__main__":
+    main()
+

examples/finetuning_old/synth_qwen_v1/util.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+import os
+import json
+import argparse
+from pathlib import Path
+from typing import Any, Dict
+
+from synth_ai.config.base_url import get_backend_from_env
+
+try:
+    from dotenv import load_dotenv  # type: ignore[reportMissingImports]
+except Exception:  # pragma: no cover
+    def load_dotenv(*args, **kwargs):  # type: ignore[no-redef]
+        return False
+
+
+STATE_PATH = Path(__file__).parent / "state.json"
+
+
+def _default_backend_url() -> str:
+    base, _ = get_backend_from_env()
+    base = base.rstrip("/")
+    return base if base.endswith("/api") else f"{base}/api"
+
+
+def load_env(mode: str | None = None) -> tuple[str, str]:
+    """Resolve backend base_url and api_key.
+
+    Precedence:
+    - SYNTH_BACKEND_URL_OVERRIDE=local|dev|prod (preferred)
+    - explicit mode arg (local|dev|prod)
+    - default prod
+    """
+    load_dotenv()
+    # Prefer global override if present
+    override = (os.getenv("SYNTH_BACKEND_URL_OVERRIDE", "") or "").strip().lower()
+    if override in {"local", "dev", "prod"}:
+        base, key = get_backend_from_env()
+        base = base.rstrip("/")
+        print(f"SYNTH backend: {base} (override={override})")
+        # Print masked API key and source for clarity
+        src = ""
+        if override == "prod":
+            if key and key == os.getenv("PROD_SYNTH_API_KEY", "").strip():
+                src = "PROD_SYNTH_API_KEY"
+            elif key and key == os.getenv("TESTING_PROD_SYNTH_API_KEY", "").strip():
+                src = "TESTING_PROD_SYNTH_API_KEY"
+            elif key and key == os.getenv("SYNTH_API_KEY", "").strip():
+                src = "SYNTH_API_KEY"
+        elif override == "dev":
+            if key and key == os.getenv("DEV_SYNTH_API_KEY", "").strip():
+                src = "DEV_SYNTH_API_KEY"
+        else:  # local
+            if key and key == os.getenv("DEV_SYNTH_API_KEY", "").strip():
+                src = "DEV_SYNTH_API_KEY"
+            elif key and key == os.getenv("TESTING_LOCAL_SYNTH_API_KEY", "").strip():
+                src = "TESTING_LOCAL_SYNTH_API_KEY"
+        masked = ("*" * max(0, len(key) - 6)) + key[-6:] if key else "<empty>"
+        print(f"SYNTH api key: {masked} (len={len(key)}, src={src or '<unknown>'})")
+        return base, key
+
+    # Fallback to explicit mode
+    if mode is None:
+        mode = os.getenv("SYNTH_MODE", "prod").strip().lower()
+    if mode == "local":
+        base_url = os.getenv("LOCAL_BACKEND_URL", "").strip()
+        # Prefer DEV_SYNTH_API_KEY for local development; fall back to legacy var
+        api_key = (
+            os.getenv("DEV_SYNTH_API_KEY", "").strip()
+            or os.getenv("TESTING_LOCAL_SYNTH_API_KEY", "").strip()
+        )
+        if not base_url or not api_key:
+            raise RuntimeError("Missing LOCAL_BACKEND_URL or DEV_SYNTH_API_KEY/TESTING_LOCAL_SYNTH_API_KEY in environment/.env")
+    elif mode == "dev":
+        base_url = os.getenv("DEV_BACKEND_URL", "").strip()
+        api_key = os.getenv("DEV_SYNTH_API_KEY", "").strip()
+        if not base_url or not api_key:
+            raise RuntimeError("Missing DEV_BACKEND_URL or DEV_SYNTH_API_KEY in environment/.env")
+    else:  # prod
+        base_url = os.getenv("PROD_BACKEND_URL", "").strip() or _default_backend_url()
+        api_key = (
+            os.getenv("PROD_SYNTH_API_KEY", "").strip()
+            or os.getenv("TESTING_PROD_SYNTH_API_KEY", "").strip()
+            or os.getenv("SYNTH_API_KEY", "").strip()
+        )
+        if not api_key:
+            raise RuntimeError("Missing PROD_SYNTH_API_KEY/TESTING_PROD_SYNTH_API_KEY/SYNTH_API_KEY in environment/.env")
+    base_url = base_url.rstrip("/")
+    print(f"SYNTH backend: {base_url} (mode={mode})")
+    # Also print masked API key and source
+    src = ""
+    if mode == "prod":
+        if api_key and api_key == os.getenv("PROD_SYNTH_API_KEY", "").strip():
+            src = "PROD_SYNTH_API_KEY"
+        elif api_key and api_key == os.getenv("TESTING_PROD_SYNTH_API_KEY", "").strip():
+            src = "TESTING_PROD_SYNTH_API_KEY"
+        elif api_key and api_key == os.getenv("SYNTH_API_KEY", "").strip():
+            src = "SYNTH_API_KEY"
+    elif mode == "dev":
+        if api_key and api_key == os.getenv("DEV_SYNTH_API_KEY", "").strip():
+            src = "DEV_SYNTH_API_KEY"
+    else:
+        if api_key and api_key == os.getenv("DEV_SYNTH_API_KEY", "").strip():
+            src = "DEV_SYNTH_API_KEY"
+        elif api_key and api_key == os.getenv("TESTING_LOCAL_SYNTH_API_KEY", "").strip():
+            src = "TESTING_LOCAL_SYNTH_API_KEY"
+    masked = ("*" * max(0, len(api_key) - 6)) + api_key[-6:] if api_key else "<empty>"
+    print(f"SYNTH api key: {masked} (len={len(api_key)}, src={src or '<unknown>'})")
+    return base_url, api_key
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser()
+    p.add_argument("--mode", choices=["prod", "dev", "local"], default=None, help="Backend mode")
+    return p.parse_args()
+
+
+def save_state(obj: Dict[str, Any]) -> None:
+    prev: Dict[str, Any] = {}
+    if STATE_PATH.exists():
+        try:
+            prev = json.loads(STATE_PATH.read_text())
+        except Exception:
+            prev = {}
+    prev.update(obj)
+    STATE_PATH.write_text(json.dumps(prev, indent=2))
+
+
+def load_state() -> Dict[str, Any]:
+    if not STATE_PATH.exists():
+        return {}
+    try:
+        return json.loads(STATE_PATH.read_text())
+    except Exception:
+        return {}
+
+
+def validate_jsonl(path: str | Path) -> None:
+    """Backwards-compatible wrapper that delegates to shared SDK validator.
+
+    Prefer synth_ai.learning.validators.validate_training_jsonl to keep a single source
+    of JSONL validation rules used across examples and tests.
+    """
+    from synth_ai.learning import validate_training_jsonl
+
+    validate_training_jsonl(path)