whileai 0.54__tar.gz → 0.55__tar.gz

{whileai-0.54/whileai.egg-info → whileai-0.55}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: whileai
-Version: 0.54
+Version: 0.55
 Summary: While Python SDK: trace ingestion and verification client, plus agent simulations grounded in your tools and system prompt.
 Author: While
 License: Apache-2.0

{whileai-0.54 → whileai-0.55}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "whileai"
-version = "0.54"
+version = "0.55"
 description = "While Python SDK: trace ingestion and verification client, plus agent simulations grounded in your tools and system prompt."
 readme = "README.md"
 license = {text = "Apache-2.0"}

{whileai-0.54 → whileai-0.55}/whileai/simulations/__init__.py

@@ -175,6 +175,7 @@ from .training import (
     TrainingRun,
     attach_delta,
     attach_holdout,
+    delete_model,
     delete_run,
     get_run,
     list_runs,
@@ -183,6 +184,7 @@ from .training import (
     serve,
     train,
     training_run,
+    unserve,
 )
 from .verify import Verifier, verifier
 from .world.sandbox import MockEnvironment
@@ -253,6 +255,7 @@ __all__ = [
     "dedupe_groups",
     "delete_dataset",
     "delete_empty_datasets",
+    "delete_model",
     "delete_run",
     "delta_report",
     "dimensions_from_traces",
@@ -382,6 +385,7 @@ __all__ = [
     "trim_unanimous_groups",
     "ungrounded_arguments",
     "unpublish",
+    "unserve",
     "update_dataset",
     "validate",
     "verifier",

{whileai-0.54 → whileai-0.55}/whileai/simulations/export.py

@@ -734,6 +734,7 @@ export_dataset = export_training
 _PAIR_KEYS = (
     "tie",
     "pairwise",
+    "first_turn_differs",
     "chosen_score",
     "rejected_score",
     "margin",
@@ -853,6 +854,12 @@ def export_preference(
         length_note = length_confound_warning(chosen_longer, len(deltas))
         if length_note:
             report["warnings"] = [length_note]
+    identical = sum(1 for r in out_rows if r.get("first_turn_differs") is False)
+    if identical:
+        from .score.judging import first_turn_note
+
+        report["first_turn_identical"] = identical
+        report.setdefault("warnings", []).append(first_turn_note(identical, len(out_rows)))
     margins = [r["margin"] for r in out_rows if isinstance(r.get("margin"), (int, float))]
     if margins:
         report["mean_margin"] = round(sum(margins) / len(margins), 4)

{whileai-0.54 → whileai-0.55}/whileai/simulations/generate/agents.py

@@ -10,7 +10,7 @@ import os
 import re
 import threading
 import time
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from typing import Any
 from urllib.parse import urlparse
 
@@ -531,6 +531,7 @@ def complete(
     timeout: float = 60,
     n: int = 1,
     logprobs: bool | str = False,
+    extra: Mapping[str, Any] | None = None,
 ) -> dict:
     """POST /chat/completions. Reuses a thread-local keep-alive connection.
 
@@ -577,6 +578,8 @@ def complete(
     if tools:
         payload["tools"] = _wire_tools(tools)
     payload.update(_request_extras(base_url, model))
+    if extra:
+        payload.update(dict(extra))
     headers = {"Content-Type": "application/json", "Connection": "keep-alive"}
     if key:
         headers["Authorization"] = f"Bearer {key}"
@@ -730,8 +733,21 @@ def _strip_tool_markup(text: str) -> str:
     return cleaned.strip()
 
 
+_THINK_BLOCK = re.compile(r"<think>.*?</think>\s*", re.S | re.I)
+_THINK_OPEN = re.compile(r"<think>.*\Z", re.S | re.I)
+
+
+def _strip_think(text: str) -> str:
+    """Drop a thinking model's reasoning markup. A closed block goes whole;
+    an unclosed ``<think>`` (the token cap landed inside it) goes to the
+    end. What is left is the reply, which is what a grader, a marker and
+    the next turn's history should see (#264)."""
+    text = _THINK_BLOCK.sub("", text)
+    return _THINK_OPEN.sub("", text)
+
+
 def _spoken_text(reply: dict) -> str:
-    return _strip_tool_markup(str(reply.get("content") or ""))
+    return _strip_tool_markup(_strip_think(str(reply.get("content") or "")))
 
 
 def _calls_from_reply(reply: dict) -> tuple[list[dict], dict]:
@@ -1391,9 +1407,23 @@ def local_model(
     timeout: float = 60,
     max_tokens: int | None = None,
     user_model: str | None = None,
+    thinking: bool | None = None,
 ) -> Callable:
+    """An agent that talks to an OpenAI-compatible endpoint (a served
+    adapter, a local vLLM, any chat server) for ``simulate(agent=...)``.
+
+    ``thinking`` is for reasoning bases such as Qwen3: ``False`` sends
+    ``chat_template_kwargs={"enable_thinking": False}`` so the reply is
+    the answer, not the reasoning, the way the hosted Qwen path already
+    does; ``True`` asks for it; ``None`` (the default) sends nothing and
+    leaves the server's default. Either way ``<think>`` markup never
+    reaches ``step["text"]`` or ``final_text``.
+    """
     local = threading.local()
     plans = fault_plans if fault_plans is not None else {}
+    extras: dict[str, Any] | None = (
+        None if thinking is None else {"chat_template_kwargs": {"enable_thinking": bool(thinking)}}
+    )
     # The simulated user's model. None means the agent's own model plays
     # the user (the default); a backend spec moves that role to another
     # model, with the key resolved for that endpoint.
@@ -1438,6 +1468,7 @@ def local_model(
                     temperature=temperature,
                     timeout=timeout,
                     max_tokens=120,
+                    extra=extras,
                 )
                 opener_text = (_spoken_text(greet) or "").strip()
                 if opener_text:
@@ -1477,6 +1508,7 @@ def local_model(
                 timeout=timeout,
                 max_tokens=reply_budget(max_tokens),
                 logprobs=logprobs,
+                extra=extras,
             )
             calls, assistant = _calls_from_reply(reply)
             # One agent turn, one set of sampling facts, on its first step.

{whileai-0.54 → whileai-0.55}/whileai/simulations/score/grading.py

@@ -40,6 +40,15 @@ _FAULT_ALIASES = {
     "entity already acted on": "already_done",
 }
 _CLEAN_FAULT = {"", "ok", "created", "deleted", "success", "exit_0", "no fault", "clean"}
+#: Statuses a tool result can carry that mean the call failed. A ``status``
+#: key with any other value (``paid``, ``open``, ``completed``) is the
+#: tool's own vocabulary, not a fault (#261).
+_KNOWN_FAULTS = (
+    _BAD_STATUS
+    | set(_FAULT_ALIASES)
+    | set(_FAULT_ALIASES.values())
+    | {"stale", "malformed", "already_done", "not_found", "deny", "garbled"}
+)
 _HTTP_FAIL = re.compile(r"^[45]\d\d$")
 _INFRA_STUB = re.compile(
     r"^<agent error:|returned\s+[45]\d\d\b|^(https?://\S+\s+)?[45]\d\d(\s|$)", re.I
@@ -426,10 +435,10 @@ def _fault_from_result(result) -> str:
     if status in {"already_done", "already_acted_on"} or reason == "already_acted_on":
         return "already_done"
     name = normalize_fault_name(status)
-    if name:
+    if name and (name in _KNOWN_FAULTS or status in _KNOWN_FAULTS):
         return name
     if _step_faulted(result):
-        return normalize_fault_name(status) or "error"
+        return name or "error"
     return ""
 
 

{whileai-0.54 → whileai-0.55}/whileai/simulations/score/judging.py

@@ -563,6 +563,42 @@ def length_confound_warning(chosen_longer: int, n: int) -> str | None:
     return None
 
 
+def _first_turn(row: dict) -> str:
+    """What the policy emitted first, read the way the hosted DPO trainer
+    reads it: the first tool step as its call, else the first assistant
+    text, else ``final_text``."""
+    steps = [s for s in (row.get("steps") or []) if isinstance(s, dict)]
+    for step in steps:
+        if step.get("tool"):
+            args = step.get("arguments")
+            if args is None:
+                args = step.get("args")
+            return json.dumps({"name": step["tool"], "arguments": args or {}}, sort_keys=True)
+    for step in steps:
+        if str(step.get("text") or "").strip():
+            return str(step["text"]).strip()
+    for message in row.get("messages") or []:
+        if isinstance(message, dict) and message.get("role") == "assistant":
+            text = str(message.get("content") or "").strip()
+            if text:
+                return text
+    return str(row.get("final_text") or "").strip()
+
+
+def first_turn_note(identical: int, n: int) -> str:
+    """The warning for pairs whose first assistant turns read the same."""
+    if not identical:
+        return ""
+    left = n - identical
+    return (
+        f"{identical}/{n} pairs have identical first assistant turns (same opening tool call "
+        "or line); their contrast is later in the rollout. The hosted DPO trainer compares "
+        f"first turns only and will drop them, leaving {left} (it needs at least 8). Keep "
+        "[p for p in pairs if p['first_turn_differs']] to see what it will train on, or "
+        "export_preference(pairs) for a trainer that reads whole conversations."
+    )
+
+
 def build_preference_pairs(
     rows: Sequence[dict],
     *,
@@ -651,6 +687,7 @@ def build_preference_pairs(
                     "rejected_model": r_model,
                     "same_policy": (c_model == r_model) if c_model and r_model else None,
                     "length_delta": reply_length(chosen) - reply_length(rejected),
+                    "first_turn_differs": _first_turn(chosen) != _first_turn(rejected),
                     "chosen_reason": str(chosen.get("reason") or ""),
                     "rejected_reason": str(rejected.get("reason") or ""),
                     "rejected_failure_class": rejected.get("failure_class"),
@@ -673,10 +710,13 @@ def build_preference_pairs(
         for p in pairs
         if p["chosen_score"] not in (0.0, 1.0) or p["rejected_score"] not in (0.0, 1.0)
     )
+    identical = sum(1 for p in pairs if not p["first_turn_differs"])
     warnings: list[str] = []
     length_note = length_confound_warning(chosen_longer, n)
     if length_note:
         warnings.append(length_note)
+    if identical:
+        warnings.append(first_turn_note(identical, n))
     if mixed_policy:
         warnings.append(
             f"{mixed_policy}/{n} pairs mix policies (chosen and rejected from different "
@@ -694,6 +734,8 @@ def build_preference_pairs(
         "min_margin": min_margin,
         "mean_margin": round(sum(p["margin"] for p in pairs) / n, 4) if n else None,
         "partial_score_pairs": partial,
+        "first_turn_identical": identical,
+        "trainer_pairs": n - identical,
         "same_policy_pairs": same_policy,
         "mixed_policy_pairs": mixed_policy,
         "eval_sourced": eval_pairs,

{whileai-0.54 → whileai-0.55}/whileai/simulations/training.py

@@ -474,6 +474,11 @@ class TrainingRun:
         else:
             self.finish("done")
 
+    @property
+    def id(self) -> str:
+        """The run id, the handle ``get_run``, ``serve`` and ``delete_run`` take."""
+        return self.run_id
+
     def __repr__(self) -> str:
         return (
             f"TrainingRun({self.run_id!r}, {self.name!r}, status={self.status!r}, step={self.step})"
@@ -825,11 +830,40 @@ def models(*, api_key: str | None = None) -> list[dict[str, Any]]:
     """The account's hosted models: ``name``, ``baseModel``, ``adapter``,
     ``adapterRunId``, ``version``, ``endpoint`` (an OpenAI-compatible
     base URL; send the account key as the bearer and ``name`` as the
-    model)."""
+    model).
+
+    A row here is a registry entry, not a running GPU: the endpoint
+    behind it idles to zero on its own and an unused model costs nothing.
+    The row stays until ``unserve(name)`` removes it; serving the same
+    name again bumps its ``version`` rather than adding a row."""
     out = _call("GET", "/models", api_key)
     return list(out.get("models") or []) if isinstance(out, dict) else []
 
 
+def unserve(
+    name: str,
+    *,
+    api_key: str | None = None,
+    transport: Callable[..., Any] | None = None,
+) -> dict[str, Any]:
+    """Stop hosting ``name``: removes the model row from the account, so
+    ``models()`` no longer lists it and its endpoint stops answering for
+    that name. The inverse of ``serve``, the way ``delete_dataset`` is the
+    inverse of ``push``. The adapter weights and the training run stay;
+    ``serve`` the run again to bring it back (at version 1).
+    Returns ``{"name": ..., "deleted": True}``."""
+    call = transport or _call
+    key = str(name).strip().lower()
+    if not key:
+        raise ValueError("unserve: name is the hosted model's name, as models() lists it")
+    out = call("DELETE", f"/models/{key}", api_key)
+    return dict(out) if isinstance(out, dict) else {"name": key, "deleted": True}
+
+
+#: Same call, the other spelling: symmetric with ``delete_dataset``.
+delete_model = unserve
+
+
 def serve(
     name: str,
     run: TrainingRun | str | None = None,
@@ -842,18 +876,36 @@ def serve(
     row; ``endpoint`` is the OpenAI-compatible base URL and ``name`` the
     model id to send. Posting an existing name bumps ``version``.
 
-    ``run`` is a ``TrainingRun`` or its id; the adapter and base model
-    come from the run record unless ``base_model`` is given. No ``run``
-    serves the bare base (``base_model`` required).
+    ``run`` is a ``TrainingRun``, the record ``get_run`` returns, or the
+    run id; the adapter and base model come from the run record unless
+    ``base_model`` is given. No ``run`` serves the bare base
+    (``base_model`` required). ``unserve`` is the inverse.
     """
     call = transport or _call
     adapter: str | None = None
     base = base_model
+    run_id: str | None
     if isinstance(run, TrainingRun):
         adapter = run.adapter
-        run_id: str | None = run.run_id
+        run_id = run.run_id
+    elif isinstance(run, dict):
+        # The record ``get_run`` returns. Reading the id here is what lets
+        # train in one process and serve in the next compose (#262).
+        run_id = str(run.get("runId") or run.get("run_id") or run.get("id") or "").strip() or None
+        if not run_id:
+            raise TypeError(
+                "run is a dict with no runId; pass the record wai.get_run(run_id) returns, "
+                "a TrainingRun, or the run id string"
+            )
+        adapter = str(run.get("adapter") or "").strip() or None
+        base = base or run.get("baseModel") or run.get("base_model")
+    elif run is None or isinstance(run, str):
+        run_id = (str(run).strip() or None) if run else None
     else:
-        run_id = str(run) if run else None
+        raise TypeError(
+            "run must be a TrainingRun, the record wai.get_run(run_id) returns, or the run "
+            f"id string; got {type(run).__name__}"
+        )
     if run_id and (adapter is None or base is None):
         meta = call("GET", f"/runs/{run_id}", api_key)
         meta = meta if isinstance(meta, dict) else {}

{whileai-0.54 → whileai-0.55/whileai.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: whileai
-Version: 0.54
+Version: 0.55
 Summary: While Python SDK: trace ingestion and verification client, plus agent simulations grounded in your tools and system prompt.
 Author: While
 License: Apache-2.0