simpleaudit-studio 0.4.1__tar.gz → 0.4.2__tar.gz

{simpleaudit_studio-0.4.1 → simpleaudit_studio-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: simpleaudit-studio
-Version: 0.4.1
+Version: 0.4.2
 Summary: Production AI model audit platform — one-liner demo mode via uvx
 License: MIT
 License-File: LICENSE

{simpleaudit_studio-0.4.1 → simpleaudit_studio-0.4.2}/infra/engine.py

@@ -140,6 +140,33 @@ def _split_endpoint_params(params: dict[str, Any]) -> tuple[dict | None, dict |
     return (gen_params or None), (client_kwargs or None)
 
 
+# A slow or stuck endpoint must fail and be retried, not hang a run: the
+# OpenAI client's own default is 600 s per request with 2 retries, and
+# SimpleAudit retries on top. An endpoint's default_parameters can set
+# "timeout" / "max_retries" to override these.
+CLIENT_TIMEOUT_S = 180
+CLIENT_MAX_RETRIES = 1
+_CLIENT_DEFAULTS: dict[str, dict] = {}
+
+
+def _client_defaults(provider: str) -> dict:
+    """``timeout`` / ``max_retries`` client kwargs, when the provider's client
+    takes them (probed once by building one; construction opens no connection).
+    Providers that don't, or can't be built here, get none."""
+    cached = _CLIENT_DEFAULTS.get(provider)
+    if cached is None:
+        defaults = {"timeout": CLIENT_TIMEOUT_S, "max_retries": CLIENT_MAX_RETRIES}
+        try:
+            from any_llm import AnyLLM
+
+            AnyLLM.create(provider, api_key="probe", **defaults)
+            cached = defaults
+        except Exception:  # noqa: BLE001 - unsupported kwargs or provider extras not installed
+            cached = {}
+        _CLIENT_DEFAULTS[provider] = cached
+    return dict(cached)
+
+
 def _auditor_kwargs_from_snapshot(snapshot: dict[str, Any], resolve_key=snapshot_api_key) -> dict[str, Any]:
     """Map a frozen endpoint snapshot onto ModelAuditor constructor kwargs.
 
@@ -159,12 +186,13 @@ def _auditor_kwargs_from_snapshot(snapshot: dict[str, Any], resolve_key=snapshot
     # same convention the engine's own preflight probe uses (api_key="probe").
     if not api_key:
         api_key = "no-auth"
+    provider = _normalize_provider(snapshot.get("provider"), base_url)
     return {
         "model": snapshot.get("model_id"),
-        "provider": _normalize_provider(snapshot.get("provider"), base_url),
+        "provider": provider,
         "base_url": base_url,
         "api_key": api_key,
-        "kwargs": client_kwargs,
+        "kwargs": {**_client_defaults(provider), **(client_kwargs or {})} or None,
         "gen_params": gen_params,
     }
 

{simpleaudit_studio-0.4.1 → simpleaudit_studio-0.4.2}/infra/tests/test_engine_integration.py

@@ -140,8 +140,9 @@ class AuditorKwargsTest(TestCase):
         self.assertEqual(language, "Norwegian")
         self.assertEqual((kwargs["model"], kwargs["auditor_model"], kwargs["judge_model"]), ("tgt", "aud", "jdg"))
         self.assertEqual(kwargs["judge_base_url"], "http://jdg.local/v1")
-        self.assertEqual(kwargs["auditor_kwargs"], {"timeout": 5})
-        self.assertEqual(kwargs["judge_kwargs"], {"timeout": 9})
+        # An endpoint's own timeout wins over the default; max_retries is the default.
+        self.assertEqual(kwargs["auditor_kwargs"], {"timeout": 5, "max_retries": 1})
+        self.assertEqual(kwargs["judge_kwargs"], {"timeout": 9, "max_retries": 1})
         self.assertEqual(kwargs["judge_params"], {"temperature": 0, "top_p": 1})
         self.assertEqual(kwargs["target_params"], {"temperature": 0.1})
         self.assertEqual(kwargs["system_prompt"], "Be brief.")
@@ -196,7 +197,7 @@ class AuditorKwargsTest(TestCase):
         entry = captured["entry"]
         self.assertEqual((entry["model"], entry["auditor_model"], entry["judge_model"]), ("tgt", "aud", "jdg"))
         self.assertEqual(entry["judge_base_url"], "http://jdg.local/v1")
-        self.assertEqual(entry["judge_kwargs"], {"timeout": 9})
+        self.assertEqual(entry["judge_kwargs"], {"timeout": 9, "max_retries": 1})
 
 
 class ProviderNormalizationTest(TestCase):
@@ -774,6 +775,24 @@ class ResumableExecutionTest(TestCase):
         self.assertEqual(run.status, AuditRun.Status.COMPLETED)
 
 
+class ClientDefaultsTest(TestCase):
+    """A stuck endpoint must not hang a run: requests get a bounded timeout."""
+
+    def test_default_timeout_and_endpoint_override(self):
+        from infra.engine import CLIENT_TIMEOUT_S, auditor_kwargs
+
+        kwargs, _ = auditor_kwargs(target=_snap("t"), auditor=_snap("a", default_parameters={"max_retries": 4}),
+                                   judge=_snap("j"))
+        self.assertEqual(kwargs["target_kwargs"], {"timeout": CLIENT_TIMEOUT_S, "max_retries": 1})
+        self.assertEqual(kwargs["auditor_kwargs"], {"timeout": CLIENT_TIMEOUT_S, "max_retries": 4})
+
+    def test_provider_without_support_gets_none(self):
+        from infra import engine
+
+        engine._CLIENT_DEFAULTS.pop("nosuchprovider", None)
+        self.assertEqual(engine._client_defaults("nosuchprovider"), {})
+
+
 class RealAuditorScenarioTest(TestCase):
     """engine.run_scenario against the real ModelAuditor (fake model clients):
     every scenario field Studio stores must reach SimpleAudit without a

{simpleaudit_studio-0.4.1 → simpleaudit_studio-0.4.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "simpleaudit-studio"
-version = "0.4.1"
+version = "0.4.2"
 description = "Production AI model audit platform — one-liner demo mode via uvx"
 readme = "README.md"
 requires-python = ">=3.11"

{simpleaudit_studio-0.4.1 → simpleaudit_studio-0.4.2}/uv.lock

@@ -2163,7 +2163,7 @@ wheels = [
 
 [[package]]
 name = "simpleaudit-studio"
-version = "0.4.1"
+version = "0.4.2"
 source = { editable = "." }
 dependencies = [
     { name = "cronsim" },