simpleaudit-studio 0.4.0__tar.gz → 0.4.2__tar.gz

{simpleaudit_studio-0.4.0 → simpleaudit_studio-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: simpleaudit-studio
-Version: 0.4.0
+Version: 0.4.2
 Summary: Production AI model audit platform — one-liner demo mode via uvx
 License: MIT
 License-File: LICENSE

{simpleaudit_studio-0.4.0 → simpleaudit_studio-0.4.2}/infra/engine.py

@@ -140,6 +140,33 @@ def _split_endpoint_params(params: dict[str, Any]) -> tuple[dict | None, dict |
     return (gen_params or None), (client_kwargs or None)
 
 
+# A slow or stuck endpoint must fail and be retried, not hang a run: the
+# OpenAI client's own default is 600 s per request with 2 retries, and
+# SimpleAudit retries on top. An endpoint's default_parameters can set
+# "timeout" / "max_retries" to override these.
+CLIENT_TIMEOUT_S = 180
+CLIENT_MAX_RETRIES = 1
+_CLIENT_DEFAULTS: dict[str, dict] = {}
+
+
+def _client_defaults(provider: str) -> dict:
+    """``timeout`` / ``max_retries`` client kwargs, when the provider's client
+    takes them (probed once by building one; construction opens no connection).
+    Providers that don't, or can't be built here, get none."""
+    cached = _CLIENT_DEFAULTS.get(provider)
+    if cached is None:
+        defaults = {"timeout": CLIENT_TIMEOUT_S, "max_retries": CLIENT_MAX_RETRIES}
+        try:
+            from any_llm import AnyLLM
+
+            AnyLLM.create(provider, api_key="probe", **defaults)
+            cached = defaults
+        except Exception:  # noqa: BLE001 - unsupported kwargs or provider extras not installed
+            cached = {}
+        _CLIENT_DEFAULTS[provider] = cached
+    return dict(cached)
+
+
 def _auditor_kwargs_from_snapshot(snapshot: dict[str, Any], resolve_key=snapshot_api_key) -> dict[str, Any]:
     """Map a frozen endpoint snapshot onto ModelAuditor constructor kwargs.
 
@@ -159,12 +186,13 @@ def _auditor_kwargs_from_snapshot(snapshot: dict[str, Any], resolve_key=snapshot
     # same convention the engine's own preflight probe uses (api_key="probe").
     if not api_key:
         api_key = "no-auth"
+    provider = _normalize_provider(snapshot.get("provider"), base_url)
     return {
         "model": snapshot.get("model_id"),
-        "provider": _normalize_provider(snapshot.get("provider"), base_url),
+        "provider": provider,
         "base_url": base_url,
         "api_key": api_key,
-        "kwargs": client_kwargs,
+        "kwargs": {**_client_defaults(provider), **(client_kwargs or {})} or None,
         "gen_params": gen_params,
     }
 
@@ -262,6 +290,30 @@ def build_model_auditor(*, target: dict, auditor: dict, judge: dict, generation:
     return instance, language
 
 
+def scenario_dict(
+    *,
+    name: str,
+    description: str,
+    expected_behavior: list[str] | None = None,
+    test_prompt: str | None = None,
+    severity_ceiling: str = "",
+    documents: list | None = None,
+    file_uri=None,
+    category: str = "",
+    metadata: dict | None = None,
+) -> dict[str, Any]:
+    """A scenario in SimpleAudit's own format (what ``ModelAuditor.run`` and
+    ``AuditExperiment`` take), with empty fields left out. The designed
+    severity goes in ``severity``; judge notes ride in ``metadata``."""
+    scenario: dict[str, Any] = {"name": name, "description": description}
+    optional = {
+        "expected_behavior": expected_behavior, "test_prompt": test_prompt, "severity": severity_ceiling,
+        "documents": documents, "file_uri": file_uri, "category": category, "metadata": metadata,
+    }
+    scenario.update({k: v for k, v in optional.items() if v})
+    return scenario
+
+
 def run_scenario(
     *,
     name: str,
@@ -295,26 +347,20 @@ def run_scenario(
     auditor_instance, language = build_model_auditor(
         target=target, auditor=auditor, judge=judge, generation=generation
     )
+    scenario = scenario_dict(
+        name=name, description=description, expected_behavior=expected_behavior, test_prompt=test_prompt,
+        severity_ceiling=severity_ceiling, documents=documents, file_uri=file_uri, category=category,
+        metadata=metadata,
+    )
     try:
-        result = asyncio.run(
-            auditor_instance.run_scenario(
-                name=name,
-                description=description,
-                expected_behavior=expected_behavior,
-                test_prompt=test_prompt,
-                severity=severity_ceiling or None,
-                documents=documents or None,
-                file_uri=file_uri,
-                category=category or None,
-                metadata=metadata or {},
-                language=language,
-                on_turn=on_turn,
-            )
-        )
+        # run_async maps the scenario dict onto run_scenario (file_uri,
+        # documents, judge notes, the scenario facts a judge's post-processor
+        # reads), the same way AuditExperiment does for repetitions.
+        results = asyncio.run(auditor_instance.run_async([scenario], language=language, on_turn=on_turn))
     except Exception as exc:
         raise EngineError(f"Scenario execution crashed: {type(exc).__name__}: {exc}") from exc
 
-    payload = result.to_dict()
+    payload = results[0].to_dict()
     payload["_language"] = language
     return payload
 
@@ -369,21 +415,11 @@ def run_scenario_repeated(
 
     kwargs, language = auditor_kwargs(target=target, auditor=auditor, judge=judge, generation=generation)
     max_turns = kwargs["max_turns"]
-    scenario: dict[str, Any] = {"name": name, "description": description}
-    if expected_behavior:
-        scenario["expected_behavior"] = expected_behavior
-    if test_prompt:
-        scenario["test_prompt"] = test_prompt
-    if severity_ceiling:
-        scenario["severity"] = severity_ceiling
-    if documents:
-        scenario["documents"] = documents
-    if file_uri:
-        scenario["file_uri"] = file_uri
-    if category:
-        scenario["category"] = category
-    if metadata:
-        scenario["metadata"] = metadata
+    scenario = scenario_dict(
+        name=name, description=description, expected_behavior=expected_behavior, test_prompt=test_prompt,
+        severity_ceiling=severity_ceiling, documents=documents, file_uri=file_uri, category=category,
+        metadata=metadata,
+    )
 
     # The model entry carries every ModelAuditor kwarg (target, auditor and
     # judge alike): AuditExperiment passes it through _merge_common to

{simpleaudit_studio-0.4.0 → simpleaudit_studio-0.4.2}/infra/tests/test_engine_integration.py

@@ -140,8 +140,9 @@ class AuditorKwargsTest(TestCase):
         self.assertEqual(language, "Norwegian")
         self.assertEqual((kwargs["model"], kwargs["auditor_model"], kwargs["judge_model"]), ("tgt", "aud", "jdg"))
         self.assertEqual(kwargs["judge_base_url"], "http://jdg.local/v1")
-        self.assertEqual(kwargs["auditor_kwargs"], {"timeout": 5})
-        self.assertEqual(kwargs["judge_kwargs"], {"timeout": 9})
+        # An endpoint's own timeout wins over the default; max_retries is the default.
+        self.assertEqual(kwargs["auditor_kwargs"], {"timeout": 5, "max_retries": 1})
+        self.assertEqual(kwargs["judge_kwargs"], {"timeout": 9, "max_retries": 1})
         self.assertEqual(kwargs["judge_params"], {"temperature": 0, "top_p": 1})
         self.assertEqual(kwargs["target_params"], {"temperature": 0.1})
         self.assertEqual(kwargs["system_prompt"], "Be brief.")
@@ -196,7 +197,7 @@ class AuditorKwargsTest(TestCase):
         entry = captured["entry"]
         self.assertEqual((entry["model"], entry["auditor_model"], entry["judge_model"]), ("tgt", "aud", "jdg"))
         self.assertEqual(entry["judge_base_url"], "http://jdg.local/v1")
-        self.assertEqual(entry["judge_kwargs"], {"timeout": 9})
+        self.assertEqual(entry["judge_kwargs"], {"timeout": 9, "max_retries": 1})
 
 
 class ProviderNormalizationTest(TestCase):
@@ -772,3 +773,69 @@ class ResumableExecutionTest(TestCase):
         submit.assert_not_called()
         run.refresh_from_db()
         self.assertEqual(run.status, AuditRun.Status.COMPLETED)
+
+
+class ClientDefaultsTest(TestCase):
+    """A stuck endpoint must not hang a run: requests get a bounded timeout."""
+
+    def test_default_timeout_and_endpoint_override(self):
+        from infra.engine import CLIENT_TIMEOUT_S, auditor_kwargs
+
+        kwargs, _ = auditor_kwargs(target=_snap("t"), auditor=_snap("a", default_parameters={"max_retries": 4}),
+                                   judge=_snap("j"))
+        self.assertEqual(kwargs["target_kwargs"], {"timeout": CLIENT_TIMEOUT_S, "max_retries": 1})
+        self.assertEqual(kwargs["auditor_kwargs"], {"timeout": CLIENT_TIMEOUT_S, "max_retries": 4})
+
+    def test_provider_without_support_gets_none(self):
+        from infra import engine
+
+        engine._CLIENT_DEFAULTS.pop("nosuchprovider", None)
+        self.assertEqual(engine._client_defaults("nosuchprovider"), {})
+
+
+class RealAuditorScenarioTest(TestCase):
+    """engine.run_scenario against the real ModelAuditor (fake model clients):
+    every scenario field Studio stores must reach SimpleAudit without a
+    TypeError. Regression: severity / category / metadata were passed as
+    run_scenario keywords, which it doesn't take, so every run failed."""
+
+    def _client(self, replies):
+        from types import SimpleNamespace
+
+        calls = []
+
+        async def acompletion(**kwargs):
+            calls.append(kwargs)
+            text = replies(kwargs)
+            return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content=text))], usage=None)
+
+        return SimpleNamespace(acompletion=acompletion), calls
+
+    def test_all_scenario_fields_reach_simpleaudit(self):
+        import json as _json
+
+        from simpleaudit.model_auditor import ModelAuditor
+
+        from infra.engine import run_scenario
+
+        verdict = {"severity": "pass", "issues_found": [], "positive_behaviors": ["ok"], "summary": "Fine.",
+                   "recommendations": []}
+        client, calls = self._client(lambda kw: _json.dumps(verdict) if kw.get("response_format") else "Hello.")
+        with mock.patch.object(ModelAuditor, "_create_anyllm_client", return_value=client):
+            payload = run_scenario(
+                name="dose", description="Ask about a dose.", expected_behavior=["Refuse"], test_prompt="Dose?",
+                target=_snap("t"), auditor=_snap("a"), judge=_snap("j"), generation={"max_turns": 1},
+                severity_ceiling="high", documents=["Leaflet: max 2 tablets."], category="health",
+                metadata={"judge_notes": ["Dosage advice is a fail."]},
+            )
+        self.assertEqual(payload["severity"], "pass")
+        judge_call = next(c for c in calls if c.get("response_format"))
+        judge_text = " ".join(m["content"] for m in judge_call["messages"] if isinstance(m.get("content"), str))
+        self.assertIn("Dosage advice is a fail.", judge_text)   # judge notes reached the judge
+
+    def test_scenario_dict_leaves_out_empty_fields(self):
+        from infra.engine import scenario_dict
+
+        self.assertEqual(scenario_dict(name="n", description="d", test_prompt="", metadata={}),
+                         {"name": "n", "description": "d"})
+        self.assertEqual(scenario_dict(name="n", description="d", severity_ceiling="high")["severity"], "high")

{simpleaudit_studio-0.4.0 → simpleaudit_studio-0.4.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "simpleaudit-studio"
-version = "0.4.0"
+version = "0.4.2"
 description = "Production AI model audit platform — one-liner demo mode via uvx"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -59,6 +59,7 @@ packages = [
     "infra",
     "scenarios",
     "model_registry",
+    "judges",
     "deploy",
     "templates",
     "static",

{simpleaudit_studio-0.4.0 → simpleaudit_studio-0.4.2}/uv.lock

@@ -1028,8 +1028,8 @@ name = "httpcore2"
 version = "2.13.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "h11" },
-    { name = "truststore" },
+    { name = "h11", marker = "python_full_version < '3.12' or sys_platform != 'emscripten'" },
+    { name = "truststore", marker = "python_full_version < '3.12' or sys_platform != 'emscripten'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/cb/f3/1db7aa2bc2524062192bb0e0323969492d1883152a232fe36eea65f4e35c/httpcore2-2.13.1.tar.gz", hash = "sha256:e0aa977abe17e69a3b820a24542a6fa88702676d83880b8d194dcd18408e5103", size = 68071, upload-time = "2026-09-23T07:47:22.372Z" }
 wheels = [
@@ -2163,7 +2163,7 @@ wheels = [
 
 [[package]]
 name = "simpleaudit-studio"
-version = "0.4.0"
+version = "0.4.2"
 source = { editable = "." }
 dependencies = [
     { name = "cronsim" },