synth-ai 0.2.9.dev5__py3-none-any.whl → 0.2.9.dev6__py3-none-any.whl

examples/finetuning_old/synth_qwen_v1/filter_traces_achievements.py

@@ -1,239 +0,0 @@
-#!/usr/bin/env python3
-"""
-Filter v3 Crafter traces into SFT JSONL requiring specific achievements.
-
-Environment:
-- CRAFTER_DB_URL (default: sqlite:///traces_v3_lm_synth/traces.db)
-- OUTPUT_JSONL (default: ft_data/qwen4b_crafter_sft_ach.jsonl)
-- REQUIRED_ACHIEVEMENTS (space-separated, default: collect_wood)
-- MIN_TOTAL_REWARD (float, default: 0.0)
-- MAX_COST (float, default: inf)
-- MAX_TOKENS (int, default: inf)
-- MODELS (optional, space-separated model names; default empty = all)
-- WINDOW_MODE=1 to emit per-turn user→assistant examples
-"""
-
-import asyncio
-import json
-import math
-import os
-import tomllib
-from pathlib import Path
-from typing import Any
-
-try:
-    # Preferred path (modal-specific)
-    from synth_ai.environments.examples.crafter_classic.agent_demos.crafter_modal_ft.filter_traces_sft_turso import (  # type: ignore
-        FinetuningDataExtractorV3,
-    )
-except Exception:  # pragma: no cover
-    try:
-        # Fallback path used in some dist builds
-        from synth_ai.environments.examples.crafter_classic.agent_demos.crafter_openai_ft.filter_traces_sft_turso import (  # type: ignore
-            FinetuningDataExtractorV3,
-        )
-    except Exception as _import_err:  # pragma: no cover
-        raise ImportError(
-            "Could not import FinetuningDataExtractorV3 from synth_ai.") from _import_err
-
-
-def env_list(name: str) -> list[str]:
-    val = os.getenv(name, "").strip()
-    return val.split() if val else []
-
-
-def normalize_db_url(raw: str) -> str:
-    # Accept file path or sqlite URLs; ensure async driver prefix
-    if raw.endswith(".db") and not raw.startswith("sqlite"):
-        return f"sqlite+aiosqlite:///{raw}"
-    if raw.startswith("sqlite+aiosqlite///"):
-        return raw
-    if raw.startswith("sqlite///") and raw.endswith(".db"):
-        return raw.replace("sqlite///", "sqlite+aiosqlite///")
-    return raw
-
-
-def build_filters() -> dict[str, Any]:
-    cfg_default = Path(__file__).with_name("config.toml")
-    cfg_path = os.getenv("CRAFTER_CONFIG", str(cfg_default))
-    cfg: dict[str, Any] = {}
-    if os.path.exists(cfg_path):
-        with open(cfg_path, "rb") as f:
-            cfg = tomllib.load(f)
-    fcfg = cfg.get("filter", {})
-    # Default: no required achievements gating unless provided via env/config
-    req = set(env_list("REQUIRED_ACHIEVEMENTS") or fcfg.get("required_achievements", []))
-    models = env_list("MODELS")
-    # Default: allow zero reward unless overridden
-    min_reward = float(os.getenv("MIN_TOTAL_REWARD", str(fcfg.get("min_total_reward", 0.0))))
-    max_cost = float(os.getenv("MAX_COST", str(fcfg.get("max_cost", math.inf))))
-    max_tokens = int(os.getenv("MAX_TOKENS", str(fcfg.get("max_tokens", 1_000_000_000))))
-    return {
-        "required_achievements": req,
-        "models": models,
-        "min_total_reward": min_reward,
-        "max_cost": max_cost,
-        "max_tokens": max_tokens,
-    }
-
-
-async def main() -> None:
-    cfg_default = Path(__file__).with_name("config.toml")
-    cfg_path = os.getenv("CRAFTER_CONFIG", str(cfg_default))
-    cfg: dict[str, Any] = {}
-    if os.path.exists(cfg_path):
-        with open(cfg_path, "rb") as f:
-            cfg = tomllib.load(f)
-    fcfg = cfg.get("filter", {})
-    tcfg = cfg.get("traces", {})
-
-    # Prefer env; else derive from config or repo-local v3 path
-    raw_db_url = os.getenv("CRAFTER_DB_URL", "")
-    if not raw_db_url:
-        db_path = fcfg.get("db_path")
-        if not db_path and tcfg.get("sqld_db_path"):
-            # derive the internal data file path from the sqld dir
-            db_path = str(Path(tcfg["sqld_db_path"]) / "dbs" / "default" / "data")
-        if db_path:
-            raw_db_url = f"sqlite+aiosqlite:///{db_path}"
-        else:
-            # Try repo-local default: traces/v3/synth_ai.db/dbs/default/data
-            repo_root = Path(__file__).resolve().parents[3]
-            candidate = repo_root / "traces" / "v3" / "synth_ai.db" / "dbs" / "default" / "data"
-            raw_db_url = f"sqlite+aiosqlite:///{candidate}"
-    db_url = normalize_db_url(raw_db_url)
-    output_path = os.getenv(
-        "OUTPUT_JSONL", fcfg.get("output_jsonl", "ft_data/qwen4b_crafter_sft_ach.jsonl")
-    )
-    filters = build_filters()
-    # Default: require >1 achievements unless explicitly specified in config/env
-    # If caller set REQUIRED_ACHIEVEMENTS or provided config 'required_achievements', we won't override.
-    # Otherwise, enforce min achievements via MIN_ACHIEVEMENTS (default 2)
-    if not filters.get("required_achievements"):
-        try:
-            min_ach = int(os.getenv("MIN_ACHIEVEMENTS", str(fcfg.get("min_achievements", 2))))
-        except Exception:
-            min_ach = 2
-        filters["min_achievements"] = min_ach
-
-    window_mode = os.getenv("WINDOW_MODE", "0") == "1"
-
-    print("🤖 Modal/Synth FT Filter (achievements)")
-    print("Using database:", db_url)
-    print("Output file:", output_path)
-    print(
-        "Filters:",
-        json.dumps(
-            {k: (list(v) if isinstance(v, set) else v) for k, v in filters.items()}, indent=2
-        ),
-    )
-    print("Window mode:", window_mode)
-
-    # Print distributions (achievements and rewards) before filtering for visibility
-    try:
-        import numpy as _np
-        from collections import Counter as _Counter
-        async with FinetuningDataExtractorV3(db_url) as _ex:
-            _sessions = await _ex.get_all_sessions()
-            _ach_counts: _Counter[str] = _Counter()
-            _rewards: list[float] = []
-            for _, _row in _sessions.iterrows():
-                _sid = _row["session_id"]
-                _ach = await _ex.get_session_achievements(_sid) or []
-                for _a in _ach:
-                    _ach_counts[_a] += 1
-                _met = await _ex.get_session_metrics(_sid)
-                try:
-                    _rewards.append(float(_met.get("total_reward", 0.0) or 0.0))
-                except Exception:
-                    pass
-            print(f"\nTotal sessions: {len(_sessions)}")
-            if _ach_counts:
-                print("\nAchievements by session (count):")
-                for _name, _c in sorted(_ach_counts.items(), key=lambda x: (-x[1], x[0])):
-                    print(f"  {_name}: {_c}")
-            if _rewards:
-                _r = _np.array(_rewards, dtype=float)
-                print("\nReward stats:")
-                print(f"  min={_r.min():.2f} median={_np.median(_r):.2f} mean={_r.mean():.2f} max={_r.max():.2f}")
-    except Exception:
-        pass
-
-    required: set[str] = filters["required_achievements"]
-    models: list[str] = filters["models"]
-    min_reward: float = filters["min_total_reward"]
-    max_cost: float = filters["max_cost"]
-    max_tokens: int = filters["max_tokens"]
-
-    stats: dict[str, Any] = {
-        "total_sessions": 0,
-        "kept_sessions": 0,
-        "total_examples": 0,
-    }
-
-    async with FinetuningDataExtractorV3(db_url) as extractor:
-        all_sessions = await extractor.get_all_sessions()
-        stats["total_sessions"] = len(all_sessions)
-
-        kept: list[str] = []
-        for _, row in all_sessions.iterrows():
-            session_id = row["session_id"]
-            metrics = await extractor.get_session_metrics(session_id)
-
-            if metrics["total_reward"] < min_reward:
-                continue
-            if metrics["total_cost"] > max_cost:
-                continue
-            if metrics["total_tokens"] > max_tokens:
-                continue
-
-            if models:
-                model_query = """
-                    SELECT DISTINCT model_name
-                    FROM events
-                    WHERE session_id = :session_id
-                      AND event_type = 'cais'
-                      AND model_name IS NOT NULL
-                    """
-                model_df = await extractor.db_manager.query_traces(
-                    model_query, {"session_id": session_id}
-                )
-                session_models = model_df["model_name"].tolist() if not model_df.empty else []
-                if not any(m in models for m in session_models):
-                    continue
-
-            # Respect either explicit required achievements OR min_achievements fallback
-            min_ach = int(filters.get("min_achievements", 0))
-            if required or min_ach > 0:
-                achievements = await extractor.get_session_achievements(session_id)
-                if not achievements:
-                    continue
-                if required:
-                    if not (required & set(achievements)):
-                        continue
-                else:
-                    if len(achievements) < min_ach:
-                        continue
-
-            kept.append(session_id)
-
-        stats["kept_sessions"] = len(kept)
-
-        if window_mode:
-            training_data = await extractor.extract_openai_window_format(kept)
-        else:
-            training_data = await extractor.extract_openai_format(kept)
-
-        Path(output_path).parent.mkdir(parents=True, exist_ok=True)
-        with open(output_path, "w") as f:
-            for ex in training_data:
-                f.write(json.dumps(ex) + "\n")
-        stats["total_examples"] = len(training_data)
-
-    print(
-        "\n✅ Wrote", stats["total_examples"], "examples from", stats["kept_sessions"], "sessions"
-    )
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

examples/finetuning_old/synth_qwen_v1/finetune.py

@@ -1,46 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-
-import sys
-import os
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))))
-
-from synth_ai.learning import FtClient
-from examples.finetuning.synth_qwen_v1.util import load_env, load_state, save_state, parse_args
-
-
-async def _run(mode: str | None) -> None:
-    base, key = load_env(mode)
-    client = FtClient(base_url=base, api_key=key)
-
-    st = load_state()
-    file_id = st.get("file_id")
-    if not file_id:
-        raise RuntimeError("state.json missing file_id; run upload_data.py first")
-
-    # Qwen3 0.6B default
-    resp = await client.create_sft_job(
-        model="Qwen/Qwen3-0.6B",
-        training_file_id=file_id,
-        hyperparameters={"n_epochs": 1, "batch_size": 4},
-        metadata={"upload_to_wasabi": True},
-    )
-    job_id = resp.get("job_id")
-    if not job_id:
-        raise RuntimeError(f"create_job missing job_id: {resp}")
-    print(f"job_id={job_id}")
-    save_state({"job_id": job_id})
-
-    start = await client.start_job(job_id)
-    print(f"start={start}")
-
-
-def main() -> None:
-    args = parse_args()
-    asyncio.run(_run(args.mode))
-
-
-if __name__ == "__main__":
-    main()
-

examples/finetuning_old/synth_qwen_v1/hello_ft_model.py

@@ -1,71 +0,0 @@
-#!/usr/bin/env python3
-from __future__ import annotations
-
-import asyncio
-import json
-import os
-from pathlib import Path
-
-import sys
-
-# Ensure repo root on path
-ROOT = Path(__file__).parents[3]
-if str(ROOT) not in sys.path:
-    sys.path.insert(0, str(ROOT))
-
-from examples.finetuning.synth_qwen_v1.util import load_env, load_state  # type: ignore
-from synth_ai.inference import InferenceClient  # type: ignore
-
-
-async def main() -> None:
-    base_url, api_key = load_env(os.getenv("MODE") or os.getenv("ENV") or "local")
-    st_path = Path(__file__).parent / "state.json"
-    if not st_path.exists():
-        raise FileNotFoundError(f"state.json not found at {st_path}")
-    state = json.loads(st_path.read_text())
-    model = state.get("fine_tuned_model")
-    if not model:
-        raise RuntimeError("fine_tuned_model missing in state.json")
-
-    print(f"Backend: {base_url}")
-    print(f"Model: {model}")
-
-    ic = InferenceClient(base_url=base_url, api_key=api_key)
-    try:
-        resp = await ic.create_chat_completion(
-            model=model,
-            messages=[{"role": "user", "content": "Hello world!"}],
-            max_tokens=64,
-            temperature=0.2,
-            stream=False,
-        )
-        print("\n===== Response =====")
-        print(json.dumps(resp, indent=2))
-        print("===== End Response =====\n")
-    except Exception as e:  # always print full failure context
-        import traceback
-        print("\n===== Inference Error =====")
-        print(f"Type: {type(e).__name__}")
-        print(f"Repr: {repr(e)}")
-        print("Traceback:")
-        print(traceback.format_exc())
-        try:
-            from synth_ai.http import HTTPError  # type: ignore
-            if isinstance(e, HTTPError):
-                print("HTTPError details:")
-                print(f"  status={e.status}")
-                print(f"  url={e.url}")
-                print(f"  message={e.message}")
-                if getattr(e, 'detail', None) is not None:
-                    print(f"  detail={e.detail}")
-                if getattr(e, 'body_snippet', None):
-                    print(f"  body_snippet={e.body_snippet}")
-        except Exception:
-            pass
-        print("===== End Inference Error =====\n")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())
-
-

examples/finetuning_old/synth_qwen_v1/infer.py

@@ -1,37 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-
-import sys
-import os
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))))
-
-from synth_ai.inference import InferenceClient
-from examples.finetuning.synth_qwen_v1.util import load_env, load_state, parse_args
-
-
-async def _run(mode: str | None) -> None:
-    base, key = load_env(mode)
-    client = InferenceClient(base_url=base, api_key=key)
-    st = load_state()
-    model = st.get("fine_tuned_model") or "Qwen/Qwen2.5-0.5B"
-    print(f"model={model}")
-    resp = await client.create_chat_completion(
-        model=model,
-        messages=[{"role": "user", "content": "Give me a cheerful two-line greeting."}],
-        max_tokens=128,
-        temperature=0.7,
-        stream=False,
-    )
-    print(resp)
-
-
-def main() -> None:
-    args = parse_args()
-    asyncio.run(_run(args.mode))
-
-
-if __name__ == "__main__":
-    main()
-
-

examples/finetuning_old/synth_qwen_v1/poll.py

@@ -1,44 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-from typing import Dict
-
-import sys
-import os
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))))
-
-from synth_ai.learning import JobHandle
-from examples.finetuning.synth_qwen_v1.util import load_env, load_state, save_state, parse_args
-
-
-def _print_event(e: Dict) -> None:
-    try:
-        print(f"event seq={e.get('seq')} type={e.get('type')} msg={e.get('message')}")
-    except Exception:
-        print(str(e))
-
-
-async def _run(mode: str | None) -> None:
-    base, key = load_env(mode)
-    st = load_state()
-    job_id = st.get("job_id")
-    if not job_id:
-        raise RuntimeError("state.json missing job_id; run finetune.py first")
-
-    # Use shared JobHandle poller abstraction (strict=True for FT)
-    handle = JobHandle(base, key, job_id, strict=True)
-    final = await handle.poll_until_terminal(interval_seconds=2.0, max_seconds=1800, on_event=_print_event)
-    print(f"final_status={final.get('status')}")
-    ft = final.get("fine_tuned_model")
-    if ft:
-        save_state({"fine_tuned_model": ft})
-
-
-def main() -> None:
-    args = parse_args()
-    asyncio.run(_run(args.mode))
-
-
-if __name__ == "__main__":
-    main()
-

examples/finetuning_old/synth_qwen_v1/prepare_data.py

@@ -1,35 +0,0 @@
-from __future__ import annotations
-
-import json
-import sys
-import os
-from pathlib import Path
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))))
-
-from examples.finetuning.synth_qwen_v1.util import validate_jsonl
-
-
-def main() -> None:
-    out_dir = Path(__file__).parent / "data"
-    out_dir.mkdir(parents=True, exist_ok=True)
-    out_path = out_dir / "training.jsonl"
-
-    # Minimal single-example JSONL
-    lines = [
-        json.dumps({
-            "messages": [
-                {"role": "user", "content": "Write a short greeting."},
-                {"role": "assistant", "content": "Hello there!"},
-            ]
-        })
-    ]
-    out_path.write_text("\n".join(lines) + "\n")
-    validate_jsonl(out_path)
-    print(f"Wrote {out_path}")
-
-
-if __name__ == "__main__":
-    main()
-
-

examples/finetuning_old/synth_qwen_v1/purge_v3_traces.py

@@ -1,109 +0,0 @@
-#!/usr/bin/env python3
-"""
-Purge v3 trace databases:
-- Find all paths matching **/traces_v3_lm_synth/traces.db under the repo
-- If the DB is inside an `old/` path → delete the DB (and -wal/-shm) outright
-- Else → delete records older than 24 hours and VACUUM to reclaim space
-
-Run with: uvpm examples.finetuning.synth_qwen.purge_v3_traces
-"""
-
-import contextlib
-import datetime
-import os
-import shutil
-import sqlite3
-from pathlib import Path
-
-
-def find_trace_dbs(repo_root: Path) -> list[Path]:
-    return list(repo_root.rglob("traces_v3_lm_synth/traces.db"))
-
-
-def delete_db_files(db_path: Path) -> None:
-    wal = db_path.with_suffix(".db-wal")
-    shm = db_path.with_suffix(".db-shm")
-    if db_path.exists():
-        os.remove(db_path)
-    if wal.exists():
-        os.remove(wal)
-    if shm.exists():
-        os.remove(shm)
-
-
-def purge_older_than_24h(db_path: Path) -> None:
-    cutoff = (datetime.datetime.utcnow() - datetime.timedelta(hours=24)).strftime(
-        "%Y-%m-%d %H:%M:%S"
-    )
-    con = sqlite3.connect(str(db_path))
-    cur = con.cursor()
-
-    # Collect session_ids to purge
-    cur.execute("SELECT session_id FROM session_traces WHERE created_at < ?", (cutoff,))
-    session_ids = [row[0] for row in cur.fetchall()]
-
-    if session_ids:
-        placeholders = ",".join(["?"] * len(session_ids))
-        cur.execute(f"DELETE FROM events WHERE session_id IN ({placeholders})", session_ids)
-        cur.execute(f"DELETE FROM messages WHERE session_id IN ({placeholders})", session_ids)
-        cur.execute(
-            f"DELETE FROM session_timesteps WHERE session_id IN ({placeholders})", session_ids
-        )
-        cur.execute(
-            f"DELETE FROM session_traces WHERE session_id IN ({placeholders}) AND created_at < ?",
-            session_ids + [cutoff],
-        )
-
-    # Commit deletions before VACUUM
-    con.commit()
-    con.close()
-
-    # Attempt VACUUM
-    try:
-        con2 = sqlite3.connect(str(db_path))
-        cur2 = con2.cursor()
-        cur2.execute("VACUUM")
-        con2.commit()
-        con2.close()
-        return
-    except sqlite3.OperationalError:
-        with contextlib.suppress(Exception):
-            con2.close()
-
-    # Fallback: VACUUM INTO a temp path (e.g., /tmp) then replace atomically
-    tmp_target = Path("/tmp") / f"{db_path.stem}_compacted.db"
-    try:
-        con3 = sqlite3.connect(str(db_path))
-        cur3 = con3.cursor()
-        cur3.execute(f"VACUUM INTO '{tmp_target.as_posix()}'")
-        con3.commit()
-        con3.close()
-
-        # Replace original DB with compacted copy
-        delete_db_files(db_path)
-        shutil.move(str(tmp_target), str(db_path))
-    finally:
-        if tmp_target.exists():
-            # Clean up if move failed
-            with contextlib.suppress(Exception):
-                os.remove(tmp_target)
-
-
-def main() -> None:
-    repo_root = Path(__file__).resolve().parents[3]
-    dbs = find_trace_dbs(repo_root)
-    print(f"🔎 Found {len(dbs)} v3 trace DB(s)")
-
-    for db in dbs:
-        db_str = str(db)
-        if "/old/" in db_str or db_str.endswith("/old/traces_v3_lm_synth/traces.db"):
-            print(f"🗑️  Deleting DB under old/: {db_str}")
-            delete_db_files(db)
-            continue
-        print(f"🧹 Purging records older than 24h: {db_str}")
-        purge_older_than_24h(db)
-        print(f"✅ Purged and compacted: {db_str}")
-
-
-if __name__ == "__main__":
-    main()