hirundo 0.3.2__tar.gz → 0.3.3__tar.gz

{hirundo-0.3.2 → hirundo-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hirundo
-Version: 0.3.2
+Version: 0.3.3
 Summary: The Hirundo platform provides APIs for: LLM behavior unlearning & dataset QA.
 Author-email: Hirundo <dev@hirundo.io>
 License: MIT License

{hirundo-0.3.2 → hirundo-0.3.3}/hirundo/__init__.py

@@ -67,6 +67,7 @@ from .storage import (
     StorageS3,
 )
 from .unlearning_llm import (
+    Aggressiveness,
     BiasBehavior,
     CustomBehavior,
     HallucinationBehavior,
@@ -131,6 +132,7 @@ __all__ = [
     "StorageGit",
     "StorageConfig",
     "DatasetQAResults",
+    "Aggressiveness",
     "BiasBehavior",
     "CustomBehavior",
     "HallucinationBehavior",
@@ -152,4 +154,4 @@ __all__ = [
 ]
 
 # Keep this literal in sync with pyproject.toml; release workflows read it.
-__version__ = "0.3.2"
+__version__ = "0.3.3"

{hirundo-0.3.2 → hirundo-0.3.3}/hirundo/dataset_qa.py

@@ -326,7 +326,7 @@ class QADataset(BaseModel):
         if self.storage_config == StorageTypes.LOCAL:
             return False
         if (
-            isinstance(self.storage_config, (StorageConfig, ResponseStorageConfig))
+            isinstance(self.storage_config, StorageConfig | ResponseStorageConfig)
             and self.storage_config.type == StorageTypes.LOCAL
         ):
             return False
@@ -608,13 +608,26 @@ class QADataset(BaseModel):
 
         Args:
             dataset_id: The ID of the dataset to run QA on.
+            organization_id: The organization ID to use for this run. When set,
+                it overrides the default organization selection.
+            run_args: Optional QA run arguments. When omitted, the SDK loads
+                the dataset type and sends default classification arguments for
+                non-speech datasets, but omits `run_args` for speech-to-text
+                datasets. When supplied, the SDK serializes them into the
+                payload. Speech-to-text runs must not supply run arguments.
 
         Returns:
             ID of the run (`run_id`).
         """
-        run_info: dict[str, typing.Any] = {
-            "run_args": run_args.model_dump(mode="json") if run_args else {},
-        }
+        dataset = QADataset.get_by_id(dataset_id)
+        if run_args is None:
+            if dataset.labeling_type != LabelingType.SPEECH_TO_TEXT:
+                run_args = ClassificationRunArgs()
+        else:
+            dataset._validate_run_args(run_args)
+        run_info: dict[str, typing.Any] = {}
+        if run_args is not None:
+            run_info["run_args"] = run_args.model_dump(mode="json")
         if organization_id is not None:
             run_info["organization_id"] = organization_id
         run_response = requests.post(
@@ -661,7 +674,9 @@ class QADataset(BaseModel):
             organization_id: The ID of the organization to run the QA for.
             replace_dataset_if_exists: If True, the dataset will be replaced if it already exists
                 (this is determined by a dataset of the same name in the same organization).
-            run_args: The run arguments to use for the QA run
+            run_args: The run arguments to use for the QA run. When omitted,
+                non-speech datasets use default classification arguments, while
+                speech-to-text datasets omit run arguments.
 
         Returns:
             An ID of the run (`run_id`) and stores that `run_id` on the instance
@@ -672,6 +687,8 @@ class QADataset(BaseModel):
                     organization_id=organization_id,
                     replace_if_exists=replace_dataset_if_exists,
                 )
+            if run_args is None and self.labeling_type != LabelingType.SPEECH_TO_TEXT:
+                run_args = ClassificationRunArgs()
             if run_args is not None:
                 self._validate_run_args(run_args)
             run_id = self.launch_qa_run(self.id, organization_id, run_args)

{hirundo-0.3.2 → hirundo-0.3.3}/hirundo/external_eval.py

@@ -46,6 +46,15 @@ class ExternalEvalRunInfo(BaseModel):
     source_run_id: str | None = None
     task_ids: list[CanonicalTaskReference] = Field(min_length=1)
     sample_limit: int | None = Field(default=None, gt=0)
+    max_model_len: int | None = Field(
+        default=None,
+        gt=0,
+        strict=True,
+        description=(
+            "Maximum model context length for the evaluation. "
+            "Omit or set None to use the model's context length."
+        ),
+    )
 
     @model_validator(mode="after")
     def _validate_unique_task_ids(self) -> "ExternalEvalRunInfo":

{hirundo-0.3.2 → hirundo-0.3.3}/hirundo/llm_behavior_eval.py

@@ -7,7 +7,7 @@ from enum import Enum
 from typing import overload
 
 import httpx
-from pydantic import BaseModel, ConfigDict, model_validator
+from pydantic import BaseModel, ConfigDict, Field, model_validator
 from tqdm import tqdm
 from tqdm.contrib.logging import logging_redirect_tqdm
 
@@ -153,6 +153,31 @@ class EvalRunRecord(BaseModel):
     bias_type: BBQBiasType | UnqoverBiasType | None = None
     task_ids: list[str] | None = None
     sample_limit: int | None = None
+    attempt_timeout: int | None = Field(
+        default=None,
+        description=(
+            "Server-side Inspect timeout, in seconds, for each model generation "
+            "attempt. None for non-Inspect runs or runs created before this "
+            "setting was recorded."
+        ),
+    )
+    max_retries: int | None = Field(
+        default=None,
+        description=(
+            "Server-side Inspect retry limit for failed model generations. "
+            "Unrelated to the SDK's stream reconnection `max_retries` argument "
+            "on `check_run_by_id`. None for non-Inspect runs or runs created "
+            "before this setting was recorded."
+        ),
+    )
+    max_model_len: int | None = Field(
+        default=None,
+        description=(
+            "Maximum model context length used by the Inspect evaluation. "
+            "None when the model's context length was used, for non-Inspect "
+            "runs, or for runs created before this setting was recorded."
+        ),
+    )
     judge_model: JudgeModel | None = None
     run_id: str
     mlflow_run_id: str | None
@@ -218,6 +243,9 @@ class LlmBehaviorEval:
             bias_type=response_payload.get("bias_type"),
             task_ids=response_payload.get("task_ids"),
             sample_limit=response_payload.get("sample_limit"),
+            attempt_timeout=response_payload.get("attempt_timeout"),
+            max_retries=response_payload.get("max_retries"),
+            max_model_len=response_payload.get("max_model_len"),
             judge_model=judge_model,
             run_id=response_payload["run_id"],
             mlflow_run_id=response_payload.get("mlflow_run_id"),

{hirundo-0.3.2 → hirundo-0.3.3}/hirundo/unlearning_llm.py

@@ -319,6 +319,8 @@ OutputBehaviorOptions = (
     | CustomBehavior
 )
 
+Aggressiveness = Annotated[float, Field(gt=0, le=1, multiple_of=0.001)]
+
 
 class LlmRunInfo(BaseModel):
     model_config = ConfigDict(protected_namespaces=("model_validate", "model_dump"))
@@ -328,10 +330,11 @@ class LlmRunInfo(BaseModel):
     target_behaviors: list[TargetBehavior]
     target_utilities: list[CustomUtility] = Field(default_factory=list)
     advanced_options: UnlearningLlmAdvancedOptions | None = None
+    aggressiveness: Aggressiveness | None = None
 
     @model_validator(mode="after")
     def validate_refusal_utilities(self) -> "LlmRunInfo":
-        """Validate utility targets are compatible with the selected behaviors.
+        """Validate run options are compatible with the selected behaviors.
 
         Args:
             self: The `LlmRunInfo` instance to validate.
@@ -340,8 +343,8 @@ class LlmRunInfo(BaseModel):
             The validated `LlmRunInfo` instance.
 
         Raises:
-            ValueError: If refusal behavior is combined with non-empty target
-                utilities.
+            ValueError: If refusal behavior is combined with an unsupported
+                option.
         """
         has_refusal = any(
             isinstance(target_behavior, RefusalBehavior)
@@ -349,6 +352,8 @@ class LlmRunInfo(BaseModel):
         )
         if has_refusal and self.target_utilities:
             raise ValueError("Refusal behavior does not support target utilities")
+        if has_refusal and self.aggressiveness is not None:
+            raise ValueError("Refusal behavior does not support aggressiveness")
         return self
 
 
@@ -381,6 +386,7 @@ class OutputUnlearningLlmRun(BaseModel):
     target_behaviors: list[OutputBehaviorOptions]
     target_utilities: list[CustomUtility]
     advanced_options: UnlearningLlmAdvancedOptions | None
+    aggressiveness: float | None = None
     run_id: str
     mlflow_run_id: str | None
     status: CeleryTaskState
@@ -407,13 +413,16 @@ class LlmUnlearningRun:
             run_info: The `LlmRunInfo` request model to serialize.
 
         Returns:
-            A JSON-serializable payload derived from
-            `run_info.model_dump(mode="json")`. Bias targets include the
-            backend-only `bias_type` field set to `BBQBiasType.ALL.value`.
-            Refusal targets retain their validated empty `target_utilities`
-            field for the API request schema.
+            A JSON-serializable payload derived from a freshly validated copy
+            of `run_info`. Bias targets include the backend-only `bias_type`
+            field set to `BBQBiasType.ALL.value`. Refusal targets retain their
+            validated empty `target_utilities` field for the API request
+            schema.
         """
-        payload = run_info.model_dump(mode="json")
+        validated_run_info = LlmRunInfo.model_validate(run_info.model_dump())
+        payload = validated_run_info.model_dump(mode="json")
+        if validated_run_info.aggressiveness is None:
+            payload.pop("aggressiveness")
         for target_behavior in payload["target_behaviors"]:
             if target_behavior["type"] == "BIAS":
                 target_behavior["bias_type"] = BBQBiasType.ALL.value

{hirundo-0.3.2 → hirundo-0.3.3}/hirundo.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hirundo
-Version: 0.3.2
+Version: 0.3.3
 Summary: The Hirundo platform provides APIs for: LLM behavior unlearning & dataset QA.
 Author-email: Hirundo <dev@hirundo.io>
 License: MIT License

{hirundo-0.3.2 → hirundo-0.3.3}/pyproject.toml

@@ -7,7 +7,7 @@ packages = ["hirundo"]
 
 [project]
 name = "hirundo"
-version = "0.3.2"
+version = "0.3.3"
 description = "The Hirundo platform provides APIs for: LLM behavior unlearning & dataset QA."
 authors = [{ name = "Hirundo", email = "dev@hirundo.io" }]
 readme = "README.md"
@@ -121,7 +121,7 @@ docs = [
 ]
 
 [tool.bumpver]
-current_version = "0.3.2"
+current_version = "0.3.3"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [tool.bumpver.file_patterns]

{hirundo-0.3.2 → hirundo-0.3.3}/tests/test_external_eval.py

@@ -30,8 +30,10 @@ class _Response:
         return None
 
 
+@pytest.mark.parametrize("max_model_len", [None, 32768])
 def test_launch_external_eval_run_serializes_request(
     monkeypatch: pytest.MonkeyPatch,
+    max_model_len: int | None,
 ) -> None:
     captured_request: dict[str, Any] = {}
 
@@ -49,6 +51,7 @@ def test_launch_external_eval_run_serializes_request(
             name="Inspect AIME",
             source_run_id="unlearning-run-id",
             task_ids=["inspect_evals/aime25"],
+            max_model_len=max_model_len,
         ),
     )
 
@@ -59,9 +62,66 @@ def test_launch_external_eval_run_serializes_request(
         "name": "Inspect AIME",
         "source_run_id": "unlearning-run-id",
         "task_ids": ["inspect_evals/aime25"],
+        **({"max_model_len": max_model_len} if max_model_len is not None else {}),
     }
 
 
+def test_launch_model_external_eval_run_serializes_context_cap(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    captured_request: dict[str, Any] = {}
+
+    def fake_post(*args: Any, **kwargs: Any) -> _Response:
+        captured_request["url"] = args[0]
+        captured_request["json"] = kwargs["json"]
+        return _Response({"message": "Run launched", "run_id": "inspect-run-id"})
+
+    monkeypatch.setattr("hirundo.external_eval.get_headers", lambda: {})
+    monkeypatch.setattr("hirundo.external_eval.requests.post", fake_post)
+
+    ExternalEval.launch_eval_run(
+        ModelOrRun.MODEL,
+        ExternalEvalRunInfo(
+            model_id=123,
+            task_ids=["inspect_evals/aime25"],
+            max_model_len=4096,
+        ),
+    )
+
+    assert captured_request["url"].endswith("/external-evals/run/model")
+    assert captured_request["json"] == {
+        "model_id": 123,
+        "task_ids": ["inspect_evals/aime25"],
+        "max_model_len": 4096,
+    }
+
+
+@pytest.mark.parametrize("max_model_len", [0, -1, 1.5, 1.0, True, "4096"])
+def test_external_eval_run_info_rejects_invalid_context_cap(
+    max_model_len: int | float | bool | str,
+) -> None:
+    with pytest.raises(ValidationError):
+        ExternalEvalRunInfo.model_validate(
+            {
+                "model_id": 123,
+                "task_ids": ["inspect_evals/aime25"],
+                "max_model_len": max_model_len,
+            }
+        )
+
+
+def test_external_eval_run_info_rejects_server_owned_settings() -> None:
+    with pytest.raises(ValidationError, match="Extra inputs are not permitted"):
+        ExternalEvalRunInfo.model_validate(
+            {
+                "model_id": 123,
+                "task_ids": ["inspect_evals/aime25"],
+                "attempt_timeout": 60,
+                "max_retries": 0,
+            }
+        )
+
+
 def test_get_external_eval_catalog(monkeypatch: pytest.MonkeyPatch) -> None:
     def fake_get(*args: Any, **kwargs: Any) -> _Response:
         assert args[0].endswith("/external-evals/catalog")

{hirundo-0.3.2 → hirundo-0.3.3}/tests/test_llm_behavior_eval.py

@@ -68,7 +68,17 @@ def test_launch_eval_run_omits_removed_custom_dataset_path(
     assert "file_path" not in captured_request["json"]
 
 
-def test_parse_inspect_evaluation_run() -> None:
+@pytest.mark.parametrize(
+    "generation_settings",
+    [
+        {"attempt_timeout": 3600, "max_retries": 0, "max_model_len": 32768},
+        {"attempt_timeout": None, "max_retries": None, "max_model_len": None},
+        {},
+    ],
+)
+def test_parse_inspect_evaluation_run(
+    generation_settings: dict[str, int | None],
+) -> None:
     run_record = LlmBehaviorEval._parse_eval_run_record(
         {
             "id": 1,
@@ -82,6 +92,7 @@ def test_parse_inspect_evaluation_run() -> None:
             "bias_type": None,
             "task_ids": ["inspect_evals/aime25"],
             "sample_limit": None,
+            **generation_settings,
             "judge_model": None,
             "run_id": "eval-run-id",
             "mlflow_run_id": None,
@@ -106,6 +117,9 @@ def test_parse_inspect_evaluation_run() -> None:
     assert run_record.framework is EvalFramework.INSPECT_EVALS
     assert run_record.task_ids == ["inspect_evals/aime25"]
     assert run_record.sample_limit is None
+    assert run_record.attempt_timeout == generation_settings.get("attempt_timeout")
+    assert run_record.max_retries == generation_settings.get("max_retries")
+    assert run_record.max_model_len == generation_settings.get("max_model_len")
     assert run_record.metrics == LlmEvalMetrics(
         rows=[
             LlmEvalMetricRow(
@@ -141,6 +155,9 @@ def test_parse_legacy_evaluation_run_defaults_framework() -> None:
     )
 
     assert run_record.framework is EvalFramework.LLM_BEHAVIOR_EVAL
+    assert run_record.attempt_timeout is None
+    assert run_record.max_retries is None
+    assert run_record.max_model_len is None
 
 
 def test_parse_legacy_evaluation_run_defaults_null_framework() -> None: