calibrate-python-sdk 0.0.1__py3-none-any.whl

artpark/agent_tests/raw_client.py

@@ -0,0 +1,455 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+from json.decoder import JSONDecodeError
+
+from ..core.api_error import ApiError
+from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper
+from ..core.http_response import AsyncHttpResponse, HttpResponse
+from ..core.jsonable_encoder import encode_path_param
+from ..core.parse_error import ParsingError
+from ..core.pydantic_utilities import parse_obj_as
+from ..core.request_options import RequestOptions
+from ..core.serialization import convert_and_respect_annotation_metadata
+from ..errors.unprocessable_entity_error import UnprocessableEntityError
+from ..types.batch_run_request import BatchRunRequest
+from ..types.batch_test_run_response import BatchTestRunResponse
+from ..types.http_validation_error import HttpValidationError
+from ..types.task_create_response import TaskCreateResponse
+from ..types.test_run_status_response import TestRunStatusResponse
+from pydantic import ValidationError
+
+# this is used as the default value for optional parameters
+OMIT = typing.cast(typing.Any, ...)
+
+
+class RawAgentTestsClient:
+    def __init__(self, *, client_wrapper: SyncClientWrapper):
+        self._client_wrapper = client_wrapper
+
+    def run(
+        self,
+        agent_uuid: str,
+        *,
+        test_uuids: typing.Optional[typing.Sequence[str]] = OMIT,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> HttpResponse[TaskCreateResponse]:
+        """
+        Run one or more tests for an agent.
+
+        This starts a background task that runs the calibrate LLM tests command
+        with the agent's config and the combined test cases from all specified tests.
+
+        Returns a task ID that can be used to poll for status and results.
+
+        Auth: requires either a JWT (frontend) or an `sk_` API key. The agent
+        must belong to the caller's org or this 404s.
+
+        Parameters
+        ----------
+        agent_uuid : str
+
+        test_uuids : typing.Optional[typing.Sequence[str]]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        HttpResponse[TaskCreateResponse]
+            Successful Response
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            f"agent-tests/agent/{encode_path_param(agent_uuid)}/run",
+            method="POST",
+            json={
+                "test_uuids": test_uuids,
+            },
+            headers={
+                "content-type": "application/json",
+            },
+            request_options=request_options,
+            omit=OMIT,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                _data = typing.cast(
+                    TaskCreateResponse,
+                    parse_obj_as(
+                        type_=TaskCreateResponse,  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+                return HttpResponse(response=_response, data=_data)
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    headers=dict(_response.headers),
+                    body=typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    ),
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text)
+        except ValidationError as e:
+            raise ParsingError(
+                status_code=_response.status_code, headers=dict(_response.headers), body=_response.json(), cause=e
+            )
+        raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json)
+
+    def run_batch(
+        self,
+        *,
+        request: typing.Optional[BatchRunRequest] = None,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> HttpResponse[BatchTestRunResponse]:
+        """
+        Run every linked test for a set of agents, one ``llm-unit-test`` job per agent.
+
+        Scope is driven by the optional ``agent_names`` payload:
+
+        - **Provided (non-empty)** — run only those agents. Names are unique per org
+          and **all are validated up front**: if any doesn't resolve to a
+          (non-deleted) agent in the caller's org, the call 404s with the offending
+          names and NO jobs are created.
+        - **Omitted / null / empty** — run every agent in the caller's org.
+
+        For each selected agent, its linked tests are launched as one job. Agents
+        with no linked tests or an unverified connection are reported under
+        ``skipped`` instead of failing the batch. Subject to the normal per-org
+        concurrency queue, so over-limit jobs come back ``queued``.
+
+        Auth accepts a JWT (frontend) or an `sk_` API key (programmatic clients).
+        Returns one ``runs`` entry per launched agent with ``agent_name``,
+        ``agent_uuid``, ``task_id``, and ``status``.
+
+        Parameters
+        ----------
+        request : typing.Optional[BatchRunRequest]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        HttpResponse[BatchTestRunResponse]
+            Successful Response
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            "agent-tests/run",
+            method="POST",
+            json=convert_and_respect_annotation_metadata(
+                object_=request, annotation=typing.Optional[BatchRunRequest], direction="write"
+            ),
+            request_options=request_options,
+            omit=OMIT,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                _data = typing.cast(
+                    BatchTestRunResponse,
+                    parse_obj_as(
+                        type_=BatchTestRunResponse,  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+                return HttpResponse(response=_response, data=_data)
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    headers=dict(_response.headers),
+                    body=typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    ),
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text)
+        except ValidationError as e:
+            raise ParsingError(
+                status_code=_response.status_code, headers=dict(_response.headers), body=_response.json(), cause=e
+            )
+        raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json)
+
+    def get_run(
+        self, task_id: str, *, request_options: typing.Optional[RequestOptions] = None
+    ) -> HttpResponse[TestRunStatusResponse]:
+        """
+        Get the status of an agent test run.
+
+        Requires either a JWT (frontend) or an `sk_` API key, plus org
+        ownership of the run. Unauthenticated access to a completed run is only
+        possible once it is made public, via the share-token endpoint in the public
+        router.
+
+        Returns the current status and, if done, the test results.
+
+        Parameters
+        ----------
+        task_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        HttpResponse[TestRunStatusResponse]
+            Successful Response
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            f"agent-tests/run/{encode_path_param(task_id)}",
+            method="GET",
+            request_options=request_options,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                _data = typing.cast(
+                    TestRunStatusResponse,
+                    parse_obj_as(
+                        type_=TestRunStatusResponse,  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+                return HttpResponse(response=_response, data=_data)
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    headers=dict(_response.headers),
+                    body=typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    ),
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text)
+        except ValidationError as e:
+            raise ParsingError(
+                status_code=_response.status_code, headers=dict(_response.headers), body=_response.json(), cause=e
+            )
+        raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json)
+
+
+class AsyncRawAgentTestsClient:
+    def __init__(self, *, client_wrapper: AsyncClientWrapper):
+        self._client_wrapper = client_wrapper
+
+    async def run(
+        self,
+        agent_uuid: str,
+        *,
+        test_uuids: typing.Optional[typing.Sequence[str]] = OMIT,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> AsyncHttpResponse[TaskCreateResponse]:
+        """
+        Run one or more tests for an agent.
+
+        This starts a background task that runs the calibrate LLM tests command
+        with the agent's config and the combined test cases from all specified tests.
+
+        Returns a task ID that can be used to poll for status and results.
+
+        Auth: requires either a JWT (frontend) or an `sk_` API key. The agent
+        must belong to the caller's org or this 404s.
+
+        Parameters
+        ----------
+        agent_uuid : str
+
+        test_uuids : typing.Optional[typing.Sequence[str]]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        AsyncHttpResponse[TaskCreateResponse]
+            Successful Response
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            f"agent-tests/agent/{encode_path_param(agent_uuid)}/run",
+            method="POST",
+            json={
+                "test_uuids": test_uuids,
+            },
+            headers={
+                "content-type": "application/json",
+            },
+            request_options=request_options,
+            omit=OMIT,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                _data = typing.cast(
+                    TaskCreateResponse,
+                    parse_obj_as(
+                        type_=TaskCreateResponse,  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+                return AsyncHttpResponse(response=_response, data=_data)
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    headers=dict(_response.headers),
+                    body=typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    ),
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text)
+        except ValidationError as e:
+            raise ParsingError(
+                status_code=_response.status_code, headers=dict(_response.headers), body=_response.json(), cause=e
+            )
+        raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json)
+
+    async def run_batch(
+        self,
+        *,
+        request: typing.Optional[BatchRunRequest] = None,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> AsyncHttpResponse[BatchTestRunResponse]:
+        """
+        Run every linked test for a set of agents, one ``llm-unit-test`` job per agent.
+
+        Scope is driven by the optional ``agent_names`` payload:
+
+        - **Provided (non-empty)** — run only those agents. Names are unique per org
+          and **all are validated up front**: if any doesn't resolve to a
+          (non-deleted) agent in the caller's org, the call 404s with the offending
+          names and NO jobs are created.
+        - **Omitted / null / empty** — run every agent in the caller's org.
+
+        For each selected agent, its linked tests are launched as one job. Agents
+        with no linked tests or an unverified connection are reported under
+        ``skipped`` instead of failing the batch. Subject to the normal per-org
+        concurrency queue, so over-limit jobs come back ``queued``.
+
+        Auth accepts a JWT (frontend) or an `sk_` API key (programmatic clients).
+        Returns one ``runs`` entry per launched agent with ``agent_name``,
+        ``agent_uuid``, ``task_id``, and ``status``.
+
+        Parameters
+        ----------
+        request : typing.Optional[BatchRunRequest]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        AsyncHttpResponse[BatchTestRunResponse]
+            Successful Response
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            "agent-tests/run",
+            method="POST",
+            json=convert_and_respect_annotation_metadata(
+                object_=request, annotation=typing.Optional[BatchRunRequest], direction="write"
+            ),
+            request_options=request_options,
+            omit=OMIT,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                _data = typing.cast(
+                    BatchTestRunResponse,
+                    parse_obj_as(
+                        type_=BatchTestRunResponse,  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+                return AsyncHttpResponse(response=_response, data=_data)
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    headers=dict(_response.headers),
+                    body=typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    ),
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text)
+        except ValidationError as e:
+            raise ParsingError(
+                status_code=_response.status_code, headers=dict(_response.headers), body=_response.json(), cause=e
+            )
+        raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json)
+
+    async def get_run(
+        self, task_id: str, *, request_options: typing.Optional[RequestOptions] = None
+    ) -> AsyncHttpResponse[TestRunStatusResponse]:
+        """
+        Get the status of an agent test run.
+
+        Requires either a JWT (frontend) or an `sk_` API key, plus org
+        ownership of the run. Unauthenticated access to a completed run is only
+        possible once it is made public, via the share-token endpoint in the public
+        router.
+
+        Returns the current status and, if done, the test results.
+
+        Parameters
+        ----------
+        task_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        AsyncHttpResponse[TestRunStatusResponse]
+            Successful Response
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            f"agent-tests/run/{encode_path_param(task_id)}",
+            method="GET",
+            request_options=request_options,
+        )
+        try:
+            if 200 <= _response.status_code < 300:
+                _data = typing.cast(
+                    TestRunStatusResponse,
+                    parse_obj_as(
+                        type_=TestRunStatusResponse,  # type: ignore
+                        object_=_response.json(),
+                    ),
+                )
+                return AsyncHttpResponse(response=_response, data=_data)
+            if _response.status_code == 422:
+                raise UnprocessableEntityError(
+                    headers=dict(_response.headers),
+                    body=typing.cast(
+                        HttpValidationError,
+                        parse_obj_as(
+                            type_=HttpValidationError,  # type: ignore
+                            object_=_response.json(),
+                        ),
+                    ),
+                )
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text)
+        except ValidationError as e:
+            raise ParsingError(
+                status_code=_response.status_code, headers=dict(_response.headers), body=_response.json(), cause=e
+            )
+        raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json)

artpark/agents/__init__.py

@@ -0,0 +1,4 @@
+# This file was auto-generated by Fern from our API Definition.
+
+# isort: skip_file
+

artpark/agents/client.py

@@ -0,0 +1,210 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper
+from ..core.request_options import RequestOptions
+from ..types.resolve_agent_names_response import ResolveAgentNamesResponse
+from ..types.routers_agent_tests_agent_response import RoutersAgentTestsAgentResponse
+from .raw_client import AsyncRawAgentsClient, RawAgentsClient
+
+# this is used as the default value for optional parameters
+OMIT = typing.cast(typing.Any, ...)
+
+
+class AgentsClient:
+    def __init__(self, *, client_wrapper: SyncClientWrapper):
+        self._raw_client = RawAgentsClient(client_wrapper=client_wrapper)
+
+    @property
+    def with_raw_response(self) -> RawAgentsClient:
+        """
+        Retrieves a raw implementation of this client that returns raw responses.
+
+        Returns
+        -------
+        RawAgentsClient
+        """
+        return self._raw_client
+
+    def resolve(
+        self, *, names: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None
+    ) -> ResolveAgentNamesResponse:
+        """
+        Resolve a list of agent names to their UUIDs within the caller's org.
+
+        Auth accepts either a JWT (frontend) or an `sk_` API key (programmatic
+        clients) via `get_org_jwt_or_api_key`, so CI tooling can map human-friendly
+        agent names to the UUIDs the run/poll endpoints expect. Agent names are
+        unique per org, so each name resolves to at most one agent. Names with no
+        matching (non-deleted) agent in the org are returned under `not_found`.
+
+        Parameters
+        ----------
+        names : typing.Sequence[str]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        ResolveAgentNamesResponse
+            Successful Response
+
+        Examples
+        --------
+        from artpark import Calibrate
+
+        client = Calibrate(
+            api_key="YOUR_API_KEY",
+            org_uuid="YOUR_ORG_UUID",
+            token="YOUR_TOKEN",
+        )
+        client.agents.resolve(
+            names=["names"],
+        )
+        """
+        _response = self._raw_client.resolve(names=names, request_options=request_options)
+        return _response.data
+
+    def list(
+        self, *, request_options: typing.Optional[RequestOptions] = None
+    ) -> typing.List[RoutersAgentTestsAgentResponse]:
+        """
+        List all agents for the caller's current org.
+
+        Auth accepts either a JWT (frontend) or an `sk_` API key (programmatic
+        clients) via `get_org_jwt_or_api_key`, so CI tooling can enumerate every
+        agent UUID in the org without knowing names up front (the run/poll and
+        `/resolve` endpoints accept the same key).
+
+        Parameters
+        ----------
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        typing.List[RoutersAgentTestsAgentResponse]
+            Successful Response
+
+        Examples
+        --------
+        from artpark import Calibrate
+
+        client = Calibrate(
+            api_key="YOUR_API_KEY",
+            org_uuid="YOUR_ORG_UUID",
+            token="YOUR_TOKEN",
+        )
+        client.agents.list()
+        """
+        _response = self._raw_client.list(request_options=request_options)
+        return _response.data
+
+
+class AsyncAgentsClient:
+    def __init__(self, *, client_wrapper: AsyncClientWrapper):
+        self._raw_client = AsyncRawAgentsClient(client_wrapper=client_wrapper)
+
+    @property
+    def with_raw_response(self) -> AsyncRawAgentsClient:
+        """
+        Retrieves a raw implementation of this client that returns raw responses.
+
+        Returns
+        -------
+        AsyncRawAgentsClient
+        """
+        return self._raw_client
+
+    async def resolve(
+        self, *, names: typing.Sequence[str], request_options: typing.Optional[RequestOptions] = None
+    ) -> ResolveAgentNamesResponse:
+        """
+        Resolve a list of agent names to their UUIDs within the caller's org.
+
+        Auth accepts either a JWT (frontend) or an `sk_` API key (programmatic
+        clients) via `get_org_jwt_or_api_key`, so CI tooling can map human-friendly
+        agent names to the UUIDs the run/poll endpoints expect. Agent names are
+        unique per org, so each name resolves to at most one agent. Names with no
+        matching (non-deleted) agent in the org are returned under `not_found`.
+
+        Parameters
+        ----------
+        names : typing.Sequence[str]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        ResolveAgentNamesResponse
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from artpark import AsyncCalibrate
+
+        client = AsyncCalibrate(
+            api_key="YOUR_API_KEY",
+            org_uuid="YOUR_ORG_UUID",
+            token="YOUR_TOKEN",
+        )
+
+
+        async def main() -> None:
+            await client.agents.resolve(
+                names=["names"],
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._raw_client.resolve(names=names, request_options=request_options)
+        return _response.data
+
+    async def list(
+        self, *, request_options: typing.Optional[RequestOptions] = None
+    ) -> typing.List[RoutersAgentTestsAgentResponse]:
+        """
+        List all agents for the caller's current org.
+
+        Auth accepts either a JWT (frontend) or an `sk_` API key (programmatic
+        clients) via `get_org_jwt_or_api_key`, so CI tooling can enumerate every
+        agent UUID in the org without knowing names up front (the run/poll and
+        `/resolve` endpoints accept the same key).
+
+        Parameters
+        ----------
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        typing.List[RoutersAgentTestsAgentResponse]
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from artpark import AsyncCalibrate
+
+        client = AsyncCalibrate(
+            api_key="YOUR_API_KEY",
+            org_uuid="YOUR_ORG_UUID",
+            token="YOUR_TOKEN",
+        )
+
+
+        async def main() -> None:
+            await client.agents.list()
+
+
+        asyncio.run(main())
+        """
+        _response = await self._raw_client.list(request_options=request_options)
+        return _response.data