agenta 0.48.8__py3-none-any.whl → 0.48.10__py3-none-any.whl

agenta/client/backend/evals/client.py

@@ -0,0 +1,1042 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+
+from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper
+from ..core.request_options import RequestOptions
+from ..types.annotation_create import AnnotationCreate
+from ..types.annotation_edit import AnnotationEdit
+from ..types.annotation_link_response import AnnotationLinkResponse
+from ..types.annotation_query_request import AnnotationQueryRequest
+from ..types.annotation_response import AnnotationResponse
+from ..types.annotations_response import AnnotationsResponse
+from ..types.evaluator import Evaluator
+from ..types.evaluator_query_request import EvaluatorQueryRequest
+from ..types.evaluator_response import EvaluatorResponse
+from ..types.evaluators_response import EvaluatorsResponse
+from .raw_client import AsyncRawEvalsClient, RawEvalsClient
+
+# this is used as the default value for optional parameters
+OMIT = typing.cast(typing.Any, ...)
+
+
+class EvalsClient:
+    def __init__(self, *, client_wrapper: SyncClientWrapper):
+        self._raw_client = RawEvalsClient(client_wrapper=client_wrapper)
+
+    @property
+    def with_raw_response(self) -> RawEvalsClient:
+        """
+        Retrieves a raw implementation of this client that returns raw responses.
+
+        Returns
+        -------
+        RawEvalsClient
+        """
+        return self._raw_client
+
+    def list_evaluators(
+        self, *, request_options: typing.Optional[RequestOptions] = None
+    ) -> EvaluatorsResponse:
+        """
+        Parameters
+        ----------
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        EvaluatorsResponse
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+        client.evals.list_evaluators()
+        """
+        _response = self._raw_client.list_evaluators(request_options=request_options)
+        return _response.data
+
+    def create_evaluator(
+        self,
+        *,
+        evaluator: Evaluator,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> EvaluatorResponse:
+        """
+        Parameters
+        ----------
+        evaluator : Evaluator
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        EvaluatorResponse
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi, Evaluator
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+        client.evals.create_evaluator(
+            evaluator=Evaluator(),
+        )
+        """
+        _response = self._raw_client.create_evaluator(
+            evaluator=evaluator, request_options=request_options
+        )
+        return _response.data
+
+    def fetch_evaluator(
+        self,
+        evaluator_id: str,
+        *,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> EvaluatorResponse:
+        """
+        Parameters
+        ----------
+        evaluator_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        EvaluatorResponse
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+        client.evals.fetch_evaluator(
+            evaluator_id="evaluator_id",
+        )
+        """
+        _response = self._raw_client.fetch_evaluator(
+            evaluator_id, request_options=request_options
+        )
+        return _response.data
+
+    def edit_evaluator(
+        self,
+        evaluator_id: str,
+        *,
+        evaluator: Evaluator,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> EvaluatorResponse:
+        """
+        Parameters
+        ----------
+        evaluator_id : str
+
+        evaluator : Evaluator
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        EvaluatorResponse
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi, Evaluator
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+        client.evals.edit_evaluator(
+            evaluator_id="evaluator_id",
+            evaluator=Evaluator(),
+        )
+        """
+        _response = self._raw_client.edit_evaluator(
+            evaluator_id, evaluator=evaluator, request_options=request_options
+        )
+        return _response.data
+
+    def archive_evaluator(
+        self,
+        evaluator_id: str,
+        *,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> EvaluatorResponse:
+        """
+        Parameters
+        ----------
+        evaluator_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        EvaluatorResponse
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+        client.evals.archive_evaluator(
+            evaluator_id="evaluator_id",
+        )
+        """
+        _response = self._raw_client.archive_evaluator(
+            evaluator_id, request_options=request_options
+        )
+        return _response.data
+
+    def unarchive_evaluator(
+        self,
+        evaluator_id: str,
+        *,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> EvaluatorResponse:
+        """
+        Parameters
+        ----------
+        evaluator_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        EvaluatorResponse
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+        client.evals.unarchive_evaluator(
+            evaluator_id="evaluator_id",
+        )
+        """
+        _response = self._raw_client.unarchive_evaluator(
+            evaluator_id, request_options=request_options
+        )
+        return _response.data
+
+    def query_evaluators(
+        self,
+        *,
+        request: typing.Optional[EvaluatorQueryRequest] = None,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> EvaluatorsResponse:
+        """
+        Parameters
+        ----------
+        request : typing.Optional[EvaluatorQueryRequest]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        EvaluatorsResponse
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+        client.evals.query_evaluators()
+        """
+        _response = self._raw_client.query_evaluators(
+            request=request, request_options=request_options
+        )
+        return _response.data
+
+    def query_annotations(
+        self,
+        *,
+        request: typing.Optional[AnnotationQueryRequest] = None,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> AnnotationsResponse:
+        """
+        Parameters
+        ----------
+        request : typing.Optional[AnnotationQueryRequest]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        AnnotationsResponse
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+        client.evals.query_annotations()
+        """
+        _response = self._raw_client.query_annotations(
+            request=request, request_options=request_options
+        )
+        return _response.data
+
+    def create_annotation(
+        self,
+        *,
+        annotation: AnnotationCreate,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> AnnotationResponse:
+        """
+        Parameters
+        ----------
+        annotation : AnnotationCreate
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        AnnotationResponse
+            Successful Response
+
+        Examples
+        --------
+        from agenta import (
+            AgentaApi,
+            AnnotationCreate,
+            AnnotationLink,
+            AnnotationReference,
+            AnnotationReferences,
+        )
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+        client.evals.create_annotation(
+            annotation=AnnotationCreate(
+                data={},
+                references=AnnotationReferences(
+                    evaluator=AnnotationReference(),
+                ),
+                links={
+                    "key": AnnotationLink(
+                        span_id="span_id",
+                        trace_id="trace_id",
+                    )
+                },
+            ),
+        )
+        """
+        _response = self._raw_client.create_annotation(
+            annotation=annotation, request_options=request_options
+        )
+        return _response.data
+
+    def fetch_annotation(
+        self,
+        trace_id: str,
+        span_id: str,
+        *,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> AnnotationResponse:
+        """
+        Parameters
+        ----------
+        trace_id : str
+
+        span_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        AnnotationResponse
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+        client.evals.fetch_annotation(
+            trace_id="trace_id",
+            span_id="span_id",
+        )
+        """
+        _response = self._raw_client.fetch_annotation(
+            trace_id, span_id, request_options=request_options
+        )
+        return _response.data
+
+    def delete_annotation(
+        self,
+        trace_id: str,
+        span_id: str,
+        *,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> AnnotationLinkResponse:
+        """
+        Parameters
+        ----------
+        trace_id : str
+
+        span_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        AnnotationLinkResponse
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+        client.evals.delete_annotation(
+            trace_id="trace_id",
+            span_id="span_id",
+        )
+        """
+        _response = self._raw_client.delete_annotation(
+            trace_id, span_id, request_options=request_options
+        )
+        return _response.data
+
+    def edit_annotation(
+        self,
+        trace_id: str,
+        span_id: str,
+        *,
+        annotation: AnnotationEdit,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> AnnotationResponse:
+        """
+        Parameters
+        ----------
+        trace_id : str
+
+        span_id : str
+
+        annotation : AnnotationEdit
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        AnnotationResponse
+            Successful Response
+
+        Examples
+        --------
+        from agenta import AgentaApi, AnnotationEdit
+
+        client = AgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+        client.evals.edit_annotation(
+            trace_id="trace_id",
+            span_id="span_id",
+            annotation=AnnotationEdit(
+                data={},
+            ),
+        )
+        """
+        _response = self._raw_client.edit_annotation(
+            trace_id, span_id, annotation=annotation, request_options=request_options
+        )
+        return _response.data
+
+
+class AsyncEvalsClient:
+    def __init__(self, *, client_wrapper: AsyncClientWrapper):
+        self._raw_client = AsyncRawEvalsClient(client_wrapper=client_wrapper)
+
+    @property
+    def with_raw_response(self) -> AsyncRawEvalsClient:
+        """
+        Retrieves a raw implementation of this client that returns raw responses.
+
+        Returns
+        -------
+        AsyncRawEvalsClient
+        """
+        return self._raw_client
+
+    async def list_evaluators(
+        self, *, request_options: typing.Optional[RequestOptions] = None
+    ) -> EvaluatorsResponse:
+        """
+        Parameters
+        ----------
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        EvaluatorsResponse
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+
+
+        async def main() -> None:
+            await client.evals.list_evaluators()
+
+
+        asyncio.run(main())
+        """
+        _response = await self._raw_client.list_evaluators(
+            request_options=request_options
+        )
+        return _response.data
+
+    async def create_evaluator(
+        self,
+        *,
+        evaluator: Evaluator,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> EvaluatorResponse:
+        """
+        Parameters
+        ----------
+        evaluator : Evaluator
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        EvaluatorResponse
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi, Evaluator
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+
+
+        async def main() -> None:
+            await client.evals.create_evaluator(
+                evaluator=Evaluator(),
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._raw_client.create_evaluator(
+            evaluator=evaluator, request_options=request_options
+        )
+        return _response.data
+
+    async def fetch_evaluator(
+        self,
+        evaluator_id: str,
+        *,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> EvaluatorResponse:
+        """
+        Parameters
+        ----------
+        evaluator_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        EvaluatorResponse
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+
+
+        async def main() -> None:
+            await client.evals.fetch_evaluator(
+                evaluator_id="evaluator_id",
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._raw_client.fetch_evaluator(
+            evaluator_id, request_options=request_options
+        )
+        return _response.data
+
+    async def edit_evaluator(
+        self,
+        evaluator_id: str,
+        *,
+        evaluator: Evaluator,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> EvaluatorResponse:
+        """
+        Parameters
+        ----------
+        evaluator_id : str
+
+        evaluator : Evaluator
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        EvaluatorResponse
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi, Evaluator
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+
+
+        async def main() -> None:
+            await client.evals.edit_evaluator(
+                evaluator_id="evaluator_id",
+                evaluator=Evaluator(),
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._raw_client.edit_evaluator(
+            evaluator_id, evaluator=evaluator, request_options=request_options
+        )
+        return _response.data
+
+    async def archive_evaluator(
+        self,
+        evaluator_id: str,
+        *,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> EvaluatorResponse:
+        """
+        Parameters
+        ----------
+        evaluator_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        EvaluatorResponse
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+
+
+        async def main() -> None:
+            await client.evals.archive_evaluator(
+                evaluator_id="evaluator_id",
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._raw_client.archive_evaluator(
+            evaluator_id, request_options=request_options
+        )
+        return _response.data
+
+    async def unarchive_evaluator(
+        self,
+        evaluator_id: str,
+        *,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> EvaluatorResponse:
+        """
+        Parameters
+        ----------
+        evaluator_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        EvaluatorResponse
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+
+
+        async def main() -> None:
+            await client.evals.unarchive_evaluator(
+                evaluator_id="evaluator_id",
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._raw_client.unarchive_evaluator(
+            evaluator_id, request_options=request_options
+        )
+        return _response.data
+
+    async def query_evaluators(
+        self,
+        *,
+        request: typing.Optional[EvaluatorQueryRequest] = None,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> EvaluatorsResponse:
+        """
+        Parameters
+        ----------
+        request : typing.Optional[EvaluatorQueryRequest]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        EvaluatorsResponse
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+
+
+        async def main() -> None:
+            await client.evals.query_evaluators()
+
+
+        asyncio.run(main())
+        """
+        _response = await self._raw_client.query_evaluators(
+            request=request, request_options=request_options
+        )
+        return _response.data
+
+    async def query_annotations(
+        self,
+        *,
+        request: typing.Optional[AnnotationQueryRequest] = None,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> AnnotationsResponse:
+        """
+        Parameters
+        ----------
+        request : typing.Optional[AnnotationQueryRequest]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        AnnotationsResponse
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+
+
+        async def main() -> None:
+            await client.evals.query_annotations()
+
+
+        asyncio.run(main())
+        """
+        _response = await self._raw_client.query_annotations(
+            request=request, request_options=request_options
+        )
+        return _response.data
+
+    async def create_annotation(
+        self,
+        *,
+        annotation: AnnotationCreate,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> AnnotationResponse:
+        """
+        Parameters
+        ----------
+        annotation : AnnotationCreate
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        AnnotationResponse
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import (
+            AnnotationCreate,
+            AnnotationLink,
+            AnnotationReference,
+            AnnotationReferences,
+            AsyncAgentaApi,
+        )
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+
+
+        async def main() -> None:
+            await client.evals.create_annotation(
+                annotation=AnnotationCreate(
+                    data={},
+                    references=AnnotationReferences(
+                        evaluator=AnnotationReference(),
+                    ),
+                    links={
+                        "key": AnnotationLink(
+                            span_id="span_id",
+                            trace_id="trace_id",
+                        )
+                    },
+                ),
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._raw_client.create_annotation(
+            annotation=annotation, request_options=request_options
+        )
+        return _response.data
+
+    async def fetch_annotation(
+        self,
+        trace_id: str,
+        span_id: str,
+        *,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> AnnotationResponse:
+        """
+        Parameters
+        ----------
+        trace_id : str
+
+        span_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        AnnotationResponse
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+
+
+        async def main() -> None:
+            await client.evals.fetch_annotation(
+                trace_id="trace_id",
+                span_id="span_id",
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._raw_client.fetch_annotation(
+            trace_id, span_id, request_options=request_options
+        )
+        return _response.data
+
+    async def delete_annotation(
+        self,
+        trace_id: str,
+        span_id: str,
+        *,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> AnnotationLinkResponse:
+        """
+        Parameters
+        ----------
+        trace_id : str
+
+        span_id : str
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        AnnotationLinkResponse
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+
+
+        async def main() -> None:
+            await client.evals.delete_annotation(
+                trace_id="trace_id",
+                span_id="span_id",
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._raw_client.delete_annotation(
+            trace_id, span_id, request_options=request_options
+        )
+        return _response.data
+
+    async def edit_annotation(
+        self,
+        trace_id: str,
+        span_id: str,
+        *,
+        annotation: AnnotationEdit,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> AnnotationResponse:
+        """
+        Parameters
+        ----------
+        trace_id : str
+
+        span_id : str
+
+        annotation : AnnotationEdit
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Returns
+        -------
+        AnnotationResponse
+            Successful Response
+
+        Examples
+        --------
+        import asyncio
+
+        from agenta import AnnotationEdit, AsyncAgentaApi
+
+        client = AsyncAgentaApi(
+            api_key="YOUR_API_KEY",
+        )
+
+
+        async def main() -> None:
+            await client.evals.edit_annotation(
+                trace_id="trace_id",
+                span_id="span_id",
+                annotation=AnnotationEdit(
+                    data={},
+                ),
+            )
+
+
+        asyncio.run(main())
+        """
+        _response = await self._raw_client.edit_annotation(
+            trace_id, span_id, annotation=annotation, request_options=request_options
+        )
+        return _response.data