calibrate-sdk 0.0.1__py3-none-any.whl

calibrate/__init__.py

@@ -0,0 +1,109 @@
+# This file was auto-generated by Fern from our API Definition.
+
+# isort: skip_file
+
+import typing
+from importlib import import_module
+
+if typing.TYPE_CHECKING:
+    from .types import (
+        BatchRunRequest,
+        BatchTestRun,
+        BatchTestRunResponse,
+        BatchTestSkip,
+        HttpValidationError,
+        JudgeResult,
+        ResolveAgentNamesResponse,
+        RoutersAgentTestsAgentResponse,
+        RoutersAgentTestsAgentResponseType,
+        TaskCreateResponse,
+        TestCaseResult,
+        TestOutput,
+        TestRunStatusResponse,
+        ToolCallOutput,
+        ValidationError,
+        ValidationErrorLocItem,
+    )
+    from .errors import UnprocessableEntityError
+    from . import agent_tests, agents
+    from ._default_clients import DefaultAioHttpClient, DefaultAsyncHttpxClient
+    from .client import AsyncCalibrate, Calibrate
+    from .environment import CalibrateEnvironment
+    from .version import __version__
+_dynamic_imports: typing.Dict[str, str] = {
+    "AsyncCalibrate": ".client",
+    "BatchRunRequest": ".types",
+    "BatchTestRun": ".types",
+    "BatchTestRunResponse": ".types",
+    "BatchTestSkip": ".types",
+    "Calibrate": ".client",
+    "CalibrateEnvironment": ".environment",
+    "DefaultAioHttpClient": "._default_clients",
+    "DefaultAsyncHttpxClient": "._default_clients",
+    "HttpValidationError": ".types",
+    "JudgeResult": ".types",
+    "ResolveAgentNamesResponse": ".types",
+    "RoutersAgentTestsAgentResponse": ".types",
+    "RoutersAgentTestsAgentResponseType": ".types",
+    "TaskCreateResponse": ".types",
+    "TestCaseResult": ".types",
+    "TestOutput": ".types",
+    "TestRunStatusResponse": ".types",
+    "ToolCallOutput": ".types",
+    "UnprocessableEntityError": ".errors",
+    "ValidationError": ".types",
+    "ValidationErrorLocItem": ".types",
+    "__version__": ".version",
+    "agent_tests": ".agent_tests",
+    "agents": ".agents",
+}
+
+
+def __getattr__(attr_name: str) -> typing.Any:
+    module_name = _dynamic_imports.get(attr_name)
+    if module_name is None:
+        raise AttributeError(f"No {attr_name} found in _dynamic_imports for module name -> {__name__}")
+    try:
+        module = import_module(module_name, __package__)
+        if module_name == f".{attr_name}":
+            return module
+        else:
+            return getattr(module, attr_name)
+    except ImportError as e:
+        raise ImportError(f"Failed to import {attr_name} from {module_name}: {e}") from e
+    except AttributeError as e:
+        raise AttributeError(f"Failed to get {attr_name} from {module_name}: {e}") from e
+
+
+def __dir__():
+    lazy_attrs = list(_dynamic_imports.keys())
+    return sorted(lazy_attrs)
+
+
+__all__ = [
+    "AsyncCalibrate",
+    "BatchRunRequest",
+    "BatchTestRun",
+    "BatchTestRunResponse",
+    "BatchTestSkip",
+    "Calibrate",
+    "CalibrateEnvironment",
+    "DefaultAioHttpClient",
+    "DefaultAsyncHttpxClient",
+    "HttpValidationError",
+    "JudgeResult",
+    "ResolveAgentNamesResponse",
+    "RoutersAgentTestsAgentResponse",
+    "RoutersAgentTestsAgentResponseType",
+    "TaskCreateResponse",
+    "TestCaseResult",
+    "TestOutput",
+    "TestRunStatusResponse",
+    "ToolCallOutput",
+    "UnprocessableEntityError",
+    "ValidationError",
+    "ValidationErrorLocItem",
+    "__version__",
+    "agent_tests",
+    "agents",
+]

calibrate/_default_clients.py

@@ -0,0 +1,32 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+import httpx
+
+SDK_DEFAULT_TIMEOUT = 60
+
+try:
+    import httpx_aiohttp  # type: ignore[import-not-found]
+except ImportError:
+
+    class DefaultAioHttpClient(httpx.AsyncClient):  # type: ignore
+        def __init__(self, **kwargs: typing.Any) -> None:
+            raise RuntimeError(
+                "To use the aiohttp client, install the aiohttp extra: pip install calibrate-sdk[aiohttp]"
+            )
+
+else:
+
+    class DefaultAioHttpClient(httpx_aiohttp.HttpxAiohttpClient):  # type: ignore
+        def __init__(self, **kwargs: typing.Any) -> None:
+            kwargs.setdefault("timeout", SDK_DEFAULT_TIMEOUT)
+            kwargs.setdefault("follow_redirects", True)
+            super().__init__(**kwargs)
+
+
+class DefaultAsyncHttpxClient(httpx.AsyncClient):
+    def __init__(self, **kwargs: typing.Any) -> None:
+        kwargs.setdefault("timeout", SDK_DEFAULT_TIMEOUT)
+        kwargs.setdefault("follow_redirects", True)
+        super().__init__(**kwargs)

calibrate/agent_tests/__init__.py

@@ -0,0 +1,4 @@
+# This file was auto-generated by Fern from our API Definition.
+
+# isort: skip_file
+

calibrate/agent_tests/client.py

@@ -0,0 +1,350 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper
+from ..core.request_options import RequestOptions
+from ..types.batch_run_request import BatchRunRequest
+from ..types.batch_test_run_response import BatchTestRunResponse
+from ..types.task_create_response import TaskCreateResponse
+from ..types.test_run_status_response import TestRunStatusResponse
+from .raw_client import AsyncRawAgentTestsClient, RawAgentTestsClient
+
+# this is used as the default value for optional parameters
+OMIT = typing.cast(typing.Any, ...)
+
+
+class AgentTestsClient:
+    def __init__(self, *, client_wrapper: SyncClientWrapper):
+        self._raw_client = RawAgentTestsClient(client_wrapper=client_wrapper)
+
+    @property
+    def with_raw_response(self) -> RawAgentTestsClient:
+        """
+        Retrieves a raw implementation of this client that returns raw responses.
+
+        Returns
+        -------
+        RawAgentTestsClient
+        """
+        return self._raw_client
+
+    def run(
+        self,
+        agent_uuid: str,
+        *,
+        test_uuids: typing.Optional[typing.Sequence[str]] = OMIT,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> TaskCreateResponse:
+        """
+        Run one or more tests for an agent.
+
+        This starts a background task that runs the calibrate LLM tests command
+        with the agent's config and the combined test cases from all specified tests.
+
+        Returns a task ID that can be used to poll for status and results.
+
+        Auth: requires either a JWT (frontend) or an `sk_` API key. The agent
+        must belong to the caller's org or this 404s.
+
+        Parameters
+        ----------
+        agent_uuid : str
+
+        test_uuids : typing.Optional[typing.Sequence[str]]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        TaskCreateResponse
+            Successful Response
+
+        Examples
+        --------
+        from calibrate import Calibrate
+
+        client = Calibrate(
+            org_uuid="YOUR_ORG_UUID",
+            api_key="YOUR_API_KEY",
+        )
+        client.agent_tests.run(
+            agent_uuid="agent_uuid",
+        )
+        """
+        _response = self._raw_client.run(agent_uuid, test_uuids=test_uuids, request_options=request_options)
+        return _response.data
+
+    def run_batch(
+        self,
+        *,
+        request: typing.Optional[BatchRunRequest] = None,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> BatchTestRunResponse:
+        """
+        Run every linked test for a set of agents, one ``llm-unit-test`` job per agent.
+
+        Scope is driven by the optional ``agent_names`` payload:
+
+        - **Provided (non-empty)** — run only those agents. Names are unique per org
+          and **all are validated up front**: if any doesn't resolve to a
+          (non-deleted) agent in the caller's org, the call 404s with the offending
+          names and NO jobs are created.
+        - **Omitted / null / empty** — run every agent in the caller's org.
+
+        For each selected agent, its linked tests are launched as one job. Agents
+        with no linked tests or an unverified connection are reported under
+        ``skipped`` instead of failing the batch. Subject to the normal per-org
+        concurrency queue, so over-limit jobs come back ``queued``.
+
+        Auth accepts a JWT (frontend) or an `sk_` API key (programmatic clients).
+        Returns one ``runs`` entry per launched agent with ``agent_name``,
+        ``agent_uuid``, ``task_id``, and ``status``.
+
+        Parameters
+        ----------
+        request : typing.Optional[BatchRunRequest]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        BatchTestRunResponse
+            Successful Response
+
+        Examples
+        --------
+        from calibrate import BatchRunRequest, Calibrate
+
+        client = Calibrate(
+            org_uuid="YOUR_ORG_UUID",
+            api_key="YOUR_API_KEY",
+        )
+        client.agent_tests.run_batch(
+            request=BatchRunRequest(),
+        )
+        """
+        _response = self._raw_client.run_batch(request=request, request_options=request_options)
+        return _response.data
+
+    def get_run(
+        self, task_id: str, *, request_options: typing.Optional[RequestOptions] = None
+    ) -> TestRunStatusResponse:
+        """
+        Get the status of an agent test run.
+
+        Requires either a JWT (frontend) or an `sk_` API key, plus org
+        ownership of the run. Unauthenticated access to a completed run is only
+        possible once it is made public, via the share-token endpoint in the public
+        router.
+
+        Returns the current status and, if done, the test results.
+
+        Parameters
+        ----------
+        task_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        TestRunStatusResponse
+            Successful Response
+
+        Examples
+        --------
+        from calibrate import Calibrate
+
+        client = Calibrate(
+            org_uuid="YOUR_ORG_UUID",
+            api_key="YOUR_API_KEY",
+        )
+        client.agent_tests.get_run(
+            task_id="task_id",
+        )
+        """
+        _response = self._raw_client.get_run(task_id, request_options=request_options)
+        return _response.data
+
+
+class AsyncAgentTestsClient:
+    def __init__(self, *, client_wrapper: AsyncClientWrapper):
+        self._raw_client = AsyncRawAgentTestsClient(client_wrapper=client_wrapper)
+
+    @property
+    def with_raw_response(self) -> AsyncRawAgentTestsClient:
+        """
+        Retrieves a raw implementation of this client that returns raw responses.
+
+        Returns
+        -------
+        AsyncRawAgentTestsClient
+        """
+        return self._raw_client
+
+    async def run(
+        self,
+        agent_uuid: str,
+        *,
+        test_uuids: typing.Optional[typing.Sequence[str]] = OMIT,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> TaskCreateResponse:
+        """
+        Run one or more tests for an agent.
+
+        This starts a background task that runs the calibrate LLM tests command
+        with the agent's config and the combined test cases from all specified tests.
+
+        Returns a task ID that can be used to poll for status and results.
+
+        Auth: requires either a JWT (frontend) or an `sk_` API key. The agent
+        must belong to the caller's org or this 404s.
+
+        Parameters
+        ----------
+        agent_uuid : str
+
+        test_uuids : typing.Optional[typing.Sequence[str]]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        TaskCreateResponse
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from calibrate import AsyncCalibrate
+
+        client = AsyncCalibrate(
+            org_uuid="YOUR_ORG_UUID",
+            api_key="YOUR_API_KEY",
+        )
+
+
+        async def main() -> None:
+            await client.agent_tests.run(
+                agent_uuid="agent_uuid",
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._raw_client.run(agent_uuid, test_uuids=test_uuids, request_options=request_options)
+        return _response.data
+
+    async def run_batch(
+        self,
+        *,
+        request: typing.Optional[BatchRunRequest] = None,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> BatchTestRunResponse:
+        """
+        Run every linked test for a set of agents, one ``llm-unit-test`` job per agent.
+
+        Scope is driven by the optional ``agent_names`` payload:
+
+        - **Provided (non-empty)** — run only those agents. Names are unique per org
+          and **all are validated up front**: if any doesn't resolve to a
+          (non-deleted) agent in the caller's org, the call 404s with the offending
+          names and NO jobs are created.
+        - **Omitted / null / empty** — run every agent in the caller's org.
+
+        For each selected agent, its linked tests are launched as one job. Agents
+        with no linked tests or an unverified connection are reported under
+        ``skipped`` instead of failing the batch. Subject to the normal per-org
+        concurrency queue, so over-limit jobs come back ``queued``.
+
+        Auth accepts a JWT (frontend) or an `sk_` API key (programmatic clients).
+        Returns one ``runs`` entry per launched agent with ``agent_name``,
+        ``agent_uuid``, ``task_id``, and ``status``.
+
+        Parameters
+        ----------
+        request : typing.Optional[BatchRunRequest]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        BatchTestRunResponse
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from calibrate import AsyncCalibrate, BatchRunRequest
+
+        client = AsyncCalibrate(
+            org_uuid="YOUR_ORG_UUID",
+            api_key="YOUR_API_KEY",
+        )
+
+
+        async def main() -> None:
+            await client.agent_tests.run_batch(
+                request=BatchRunRequest(),
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._raw_client.run_batch(request=request, request_options=request_options)
+        return _response.data
+
+    async def get_run(
+        self, task_id: str, *, request_options: typing.Optional[RequestOptions] = None
+    ) -> TestRunStatusResponse:
+        """
+        Get the status of an agent test run.
+
+        Requires either a JWT (frontend) or an `sk_` API key, plus org
+        ownership of the run. Unauthenticated access to a completed run is only
+        possible once it is made public, via the share-token endpoint in the public
+        router.
+
+        Returns the current status and, if done, the test results.
+
+        Parameters
+        ----------
+        task_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        TestRunStatusResponse
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from calibrate import AsyncCalibrate
+
+        client = AsyncCalibrate(
+            org_uuid="YOUR_ORG_UUID",
+            api_key="YOUR_API_KEY",
+        )
+
+
+        async def main() -> None:
+            await client.agent_tests.get_run(
+                task_id="task_id",
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._raw_client.get_run(task_id, request_options=request_options)
+        return _response.data