llama-cloud 0.1.15__tar.gz → 0.1.16__tar.gz

{llama_cloud-0.1.15 → llama_cloud-0.1.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: llama-cloud
-Version: 0.1.15
+Version: 0.1.16
 Summary: 
 License: MIT
 Author: Logan Markewich

{llama_cloud-0.1.15 → llama_cloud-0.1.16}/llama_cloud/__init__.py

@@ -96,15 +96,7 @@ from .types import (
     EmbeddingModelConfigUpdateEmbeddingConfig_HuggingfaceApiEmbedding,
     EmbeddingModelConfigUpdateEmbeddingConfig_OpenaiEmbedding,
     EmbeddingModelConfigUpdateEmbeddingConfig_VertexaiEmbedding,
-    EvalDataset,
-    EvalDatasetJobParams,
-    EvalDatasetJobRecord,
     EvalExecutionParams,
-    EvalExecutionParamsOverride,
-    EvalMetric,
-    EvalQuestion,
-    EvalQuestionCreate,
-    EvalQuestionResult,
     ExtractAgent,
     ExtractAgentCreate,
     ExtractAgentCreateDataSchema,
@@ -169,9 +161,6 @@ from .types import (
     Llm,
     LlmModelData,
     LlmParameters,
-    LocalEval,
-    LocalEvalResults,
-    LocalEvalSets,
     ManagedIngestionStatus,
     ManagedIngestionStatusResponse,
     MarkdownElementNodeParser,
@@ -182,7 +171,6 @@ from .types import (
     MetadataFilterValue,
     MetadataFilters,
     MetadataFiltersFiltersItem,
-    MetricResult,
     NodeParser,
     NodeRelationship,
     NoneChunkingConfig,
@@ -261,8 +249,6 @@ from .types import (
     Project,
     ProjectCreate,
     PromptConf,
-    PromptMixinPrompts,
-    PromptSpec,
     PydanticProgramMode,
     RecurringCreditGrant,
     RelatedNodeInfo,
@@ -474,15 +460,7 @@ __all__ = [
     "EmbeddingModelConfigUpdateEmbeddingConfig_HuggingfaceApiEmbedding",
     "EmbeddingModelConfigUpdateEmbeddingConfig_OpenaiEmbedding",
     "EmbeddingModelConfigUpdateEmbeddingConfig_VertexaiEmbedding",
-    "EvalDataset",
-    "EvalDatasetJobParams",
-    "EvalDatasetJobRecord",
     "EvalExecutionParams",
-    "EvalExecutionParamsOverride",
-    "EvalMetric",
-    "EvalQuestion",
-    "EvalQuestionCreate",
-    "EvalQuestionResult",
     "ExtractAgent",
     "ExtractAgentCreate",
     "ExtractAgentCreateDataSchema",
@@ -551,9 +529,6 @@ __all__ = [
     "Llm",
     "LlmModelData",
     "LlmParameters",
-    "LocalEval",
-    "LocalEvalResults",
-    "LocalEvalSets",
     "ManagedIngestionStatus",
     "ManagedIngestionStatusResponse",
     "MarkdownElementNodeParser",
@@ -564,7 +539,6 @@ __all__ = [
     "MetadataFilterValue",
     "MetadataFilters",
     "MetadataFiltersFiltersItem",
-    "MetricResult",
     "NodeParser",
     "NodeRelationship",
     "NoneChunkingConfig",
@@ -653,8 +627,6 @@ __all__ = [
     "Project",
     "ProjectCreate",
     "PromptConf",
-    "PromptMixinPrompts",
-    "PromptSpec",
     "PydanticProgramMode",
     "RecurringCreditGrant",
     "RelatedNodeInfo",

llama_cloud-0.1.16/llama_cloud/resources/evals/client.py

@@ -0,0 +1,85 @@
+# This file was auto-generated by Fern from our API Definition.
+
+import typing
+import urllib.parse
+from json.decoder import JSONDecodeError
+
+from ...core.api_error import ApiError
+from ...core.client_wrapper import AsyncClientWrapper, SyncClientWrapper
+from ...errors.unprocessable_entity_error import UnprocessableEntityError
+from ...types.http_validation_error import HttpValidationError
+from ...types.supported_llm_model import SupportedLlmModel
+
+try:
+    import pydantic
+    if pydantic.__version__.startswith("1."):
+        raise ImportError
+    import pydantic.v1 as pydantic  # type: ignore
+except ImportError:
+    import pydantic  # type: ignore
+
+
+class EvalsClient:
+    def __init__(self, *, client_wrapper: SyncClientWrapper):
+        self._client_wrapper = client_wrapper
+
+    def list_supported_models(self) -> typing.List[SupportedLlmModel]:
+        """
+        List supported models.
+
+        ---
+        from llama_cloud.client import LlamaCloud
+
+        client = LlamaCloud(
+            token="YOUR_TOKEN",
+        )
+        client.evals.list_supported_models()
+        """
+        _response = self._client_wrapper.httpx_client.request(
+            "GET",
+            urllib.parse.urljoin(f"{self._client_wrapper.get_base_url()}/", "api/v1/evals/models"),
+            headers=self._client_wrapper.get_headers(),
+            timeout=60,
+        )
+        if 200 <= _response.status_code < 300:
+            return pydantic.parse_obj_as(typing.List[SupportedLlmModel], _response.json())  # type: ignore
+        if _response.status_code == 422:
+            raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json()))  # type: ignore
+        try:
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)
+
+
+class AsyncEvalsClient:
+    def __init__(self, *, client_wrapper: AsyncClientWrapper):
+        self._client_wrapper = client_wrapper
+
+    async def list_supported_models(self) -> typing.List[SupportedLlmModel]:
+        """
+        List supported models.
+
+        ---
+        from llama_cloud.client import AsyncLlamaCloud
+
+        client = AsyncLlamaCloud(
+            token="YOUR_TOKEN",
+        )
+        await client.evals.list_supported_models()
+        """
+        _response = await self._client_wrapper.httpx_client.request(
+            "GET",
+            urllib.parse.urljoin(f"{self._client_wrapper.get_base_url()}/", "api/v1/evals/models"),
+            headers=self._client_wrapper.get_headers(),
+            timeout=60,
+        )
+        if 200 <= _response.status_code < 300:
+            return pydantic.parse_obj_as(typing.List[SupportedLlmModel], _response.json())  # type: ignore
+        if _response.status_code == 422:
+            raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json()))  # type: ignore
+        try:
+            _response_json = _response.json()
+        except JSONDecodeError:
+            raise ApiError(status_code=_response.status_code, body=_response.text)
+        raise ApiError(status_code=_response.status_code, body=_response_json)

{llama_cloud-0.1.15 → llama_cloud-0.1.16}/llama_cloud/resources/pipelines/client.py

@@ -14,10 +14,7 @@ from ...types.cloud_document import CloudDocument
 from ...types.cloud_document_create import CloudDocumentCreate
 from ...types.configured_transformation_item import ConfiguredTransformationItem
 from ...types.data_sink_create import DataSinkCreate
-from ...types.eval_dataset_job_record import EvalDatasetJobRecord
 from ...types.eval_execution_params import EvalExecutionParams
-from ...types.eval_execution_params_override import EvalExecutionParamsOverride
-from ...types.eval_question_result import EvalQuestionResult
 from ...types.http_validation_error import HttpValidationError
 from ...types.input_message import InputMessage
 from ...types.llama_parse_parameters import LlamaParseParameters
@@ -423,188 +420,6 @@ class PipelinesClient:
             raise ApiError(status_code=_response.status_code, body=_response.text)
         raise ApiError(status_code=_response.status_code, body=_response_json)
 
-    def get_eval_dataset_executions(self, eval_dataset_id: str, pipeline_id: str) -> typing.List[EvalDatasetJobRecord]:
-        """
-        Get the status of an EvalDatasetExecution.
-
-        Parameters:
-            - eval_dataset_id: str.
-
-            - pipeline_id: str.
-        ---
-        from llama_cloud.client import LlamaCloud
-
-        client = LlamaCloud(
-            token="YOUR_TOKEN",
-        )
-        client.pipelines.get_eval_dataset_executions(
-            eval_dataset_id="string",
-            pipeline_id="string",
-        )
-        """
-        _response = self._client_wrapper.httpx_client.request(
-            "GET",
-            urllib.parse.urljoin(
-                f"{self._client_wrapper.get_base_url()}/",
-                f"api/v1/pipelines/{pipeline_id}/eval-datasets/{eval_dataset_id}/execute",
-            ),
-            headers=self._client_wrapper.get_headers(),
-            timeout=60,
-        )
-        if 200 <= _response.status_code < 300:
-            return pydantic.parse_obj_as(typing.List[EvalDatasetJobRecord], _response.json())  # type: ignore
-        if _response.status_code == 422:
-            raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json()))  # type: ignore
-        try:
-            _response_json = _response.json()
-        except JSONDecodeError:
-            raise ApiError(status_code=_response.status_code, body=_response.text)
-        raise ApiError(status_code=_response.status_code, body=_response_json)
-
-    def execute_eval_dataset(
-        self,
-        eval_dataset_id: str,
-        pipeline_id: str,
-        *,
-        eval_question_ids: typing.List[str],
-        params: typing.Optional[EvalExecutionParamsOverride] = OMIT,
-    ) -> EvalDatasetJobRecord:
-        """
-        Execute a dataset.
-
-        Parameters:
-            - eval_dataset_id: str.
-
-            - pipeline_id: str.
-
-            - eval_question_ids: typing.List[str].
-
-            - params: typing.Optional[EvalExecutionParamsOverride]. The parameters for the eval execution that will override the ones set in the pipeline.
-        ---
-        from llama_cloud import EvalExecutionParamsOverride, SupportedLlmModelNames
-        from llama_cloud.client import LlamaCloud
-
-        client = LlamaCloud(
-            token="YOUR_TOKEN",
-        )
-        client.pipelines.execute_eval_dataset(
-            eval_dataset_id="string",
-            pipeline_id="string",
-            eval_question_ids=[],
-            params=EvalExecutionParamsOverride(
-                llm_model=SupportedLlmModelNames.GPT_3_5_TURBO,
-            ),
-        )
-        """
-        _request: typing.Dict[str, typing.Any] = {"eval_question_ids": eval_question_ids}
-        if params is not OMIT:
-            _request["params"] = params
-        _response = self._client_wrapper.httpx_client.request(
-            "POST",
-            urllib.parse.urljoin(
-                f"{self._client_wrapper.get_base_url()}/",
-                f"api/v1/pipelines/{pipeline_id}/eval-datasets/{eval_dataset_id}/execute",
-            ),
-            json=jsonable_encoder(_request),
-            headers=self._client_wrapper.get_headers(),
-            timeout=60,
-        )
-        if 200 <= _response.status_code < 300:
-            return pydantic.parse_obj_as(EvalDatasetJobRecord, _response.json())  # type: ignore
-        if _response.status_code == 422:
-            raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json()))  # type: ignore
-        try:
-            _response_json = _response.json()
-        except JSONDecodeError:
-            raise ApiError(status_code=_response.status_code, body=_response.text)
-        raise ApiError(status_code=_response.status_code, body=_response_json)
-
-    def get_eval_dataset_execution_result(
-        self, eval_dataset_id: str, pipeline_id: str
-    ) -> typing.List[EvalQuestionResult]:
-        """
-        Get the result of an EvalDatasetExecution.
-        If eval_question_ids is specified, only the results for the specified
-        questions will be returned.
-        If any of the specified questions do not have a result, they will be ignored.
-
-        Parameters:
-            - eval_dataset_id: str.
-
-            - pipeline_id: str.
-        ---
-        from llama_cloud.client import LlamaCloud
-
-        client = LlamaCloud(
-            token="YOUR_TOKEN",
-        )
-        client.pipelines.get_eval_dataset_execution_result(
-            eval_dataset_id="string",
-            pipeline_id="string",
-        )
-        """
-        _response = self._client_wrapper.httpx_client.request(
-            "GET",
-            urllib.parse.urljoin(
-                f"{self._client_wrapper.get_base_url()}/",
-                f"api/v1/pipelines/{pipeline_id}/eval-datasets/{eval_dataset_id}/execute/result",
-            ),
-            headers=self._client_wrapper.get_headers(),
-            timeout=60,
-        )
-        if 200 <= _response.status_code < 300:
-            return pydantic.parse_obj_as(typing.List[EvalQuestionResult], _response.json())  # type: ignore
-        if _response.status_code == 422:
-            raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json()))  # type: ignore
-        try:
-            _response_json = _response.json()
-        except JSONDecodeError:
-            raise ApiError(status_code=_response.status_code, body=_response.text)
-        raise ApiError(status_code=_response.status_code, body=_response_json)
-
-    def get_eval_dataset_execution(
-        self, eval_dataset_id: str, eval_dataset_execution_id: str, pipeline_id: str
-    ) -> EvalDatasetJobRecord:
-        """
-        Get the status of an EvalDatasetExecution.
-
-        Parameters:
-            - eval_dataset_id: str.
-
-            - eval_dataset_execution_id: str.
-
-            - pipeline_id: str.
-        ---
-        from llama_cloud.client import LlamaCloud
-
-        client = LlamaCloud(
-            token="YOUR_TOKEN",
-        )
-        client.pipelines.get_eval_dataset_execution(
-            eval_dataset_id="string",
-            eval_dataset_execution_id="string",
-            pipeline_id="string",
-        )
-        """
-        _response = self._client_wrapper.httpx_client.request(
-            "GET",
-            urllib.parse.urljoin(
-                f"{self._client_wrapper.get_base_url()}/",
-                f"api/v1/pipelines/{pipeline_id}/eval-datasets/{eval_dataset_id}/execute/{eval_dataset_execution_id}",
-            ),
-            headers=self._client_wrapper.get_headers(),
-            timeout=60,
-        )
-        if 200 <= _response.status_code < 300:
-            return pydantic.parse_obj_as(EvalDatasetJobRecord, _response.json())  # type: ignore
-        if _response.status_code == 422:
-            raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json()))  # type: ignore
-        try:
-            _response_json = _response.json()
-        except JSONDecodeError:
-            raise ApiError(status_code=_response.status_code, body=_response.text)
-        raise ApiError(status_code=_response.status_code, body=_response_json)
-
     def list_pipeline_files(
         self,
         pipeline_id: str,
@@ -1675,7 +1490,11 @@ class PipelinesClient:
 
     def delete_pipeline_document(self, document_id: str, pipeline_id: str) -> None:
         """
-        Delete a document for a pipeline.
+        Delete a document from a pipeline.
+        Initiates an async job that will:
+
+        1. Delete vectors from the vector store
+        2. Delete the document from MongoDB after vectors are successfully deleted
 
         Parameters:
             - document_id: str.
@@ -2156,190 +1975,6 @@ class AsyncPipelinesClient:
             raise ApiError(status_code=_response.status_code, body=_response.text)
         raise ApiError(status_code=_response.status_code, body=_response_json)
 
-    async def get_eval_dataset_executions(
-        self, eval_dataset_id: str, pipeline_id: str
-    ) -> typing.List[EvalDatasetJobRecord]:
-        """
-        Get the status of an EvalDatasetExecution.
-
-        Parameters:
-            - eval_dataset_id: str.
-
-            - pipeline_id: str.
-        ---
-        from llama_cloud.client import AsyncLlamaCloud
-
-        client = AsyncLlamaCloud(
-            token="YOUR_TOKEN",
-        )
-        await client.pipelines.get_eval_dataset_executions(
-            eval_dataset_id="string",
-            pipeline_id="string",
-        )
-        """
-        _response = await self._client_wrapper.httpx_client.request(
-            "GET",
-            urllib.parse.urljoin(
-                f"{self._client_wrapper.get_base_url()}/",
-                f"api/v1/pipelines/{pipeline_id}/eval-datasets/{eval_dataset_id}/execute",
-            ),
-            headers=self._client_wrapper.get_headers(),
-            timeout=60,
-        )
-        if 200 <= _response.status_code < 300:
-            return pydantic.parse_obj_as(typing.List[EvalDatasetJobRecord], _response.json())  # type: ignore
-        if _response.status_code == 422:
-            raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json()))  # type: ignore
-        try:
-            _response_json = _response.json()
-        except JSONDecodeError:
-            raise ApiError(status_code=_response.status_code, body=_response.text)
-        raise ApiError(status_code=_response.status_code, body=_response_json)
-
-    async def execute_eval_dataset(
-        self,
-        eval_dataset_id: str,
-        pipeline_id: str,
-        *,
-        eval_question_ids: typing.List[str],
-        params: typing.Optional[EvalExecutionParamsOverride] = OMIT,
-    ) -> EvalDatasetJobRecord:
-        """
-        Execute a dataset.
-
-        Parameters:
-            - eval_dataset_id: str.
-
-            - pipeline_id: str.
-
-            - eval_question_ids: typing.List[str].
-
-            - params: typing.Optional[EvalExecutionParamsOverride]. The parameters for the eval execution that will override the ones set in the pipeline.
-        ---
-        from llama_cloud import EvalExecutionParamsOverride, SupportedLlmModelNames
-        from llama_cloud.client import AsyncLlamaCloud
-
-        client = AsyncLlamaCloud(
-            token="YOUR_TOKEN",
-        )
-        await client.pipelines.execute_eval_dataset(
-            eval_dataset_id="string",
-            pipeline_id="string",
-            eval_question_ids=[],
-            params=EvalExecutionParamsOverride(
-                llm_model=SupportedLlmModelNames.GPT_3_5_TURBO,
-            ),
-        )
-        """
-        _request: typing.Dict[str, typing.Any] = {"eval_question_ids": eval_question_ids}
-        if params is not OMIT:
-            _request["params"] = params
-        _response = await self._client_wrapper.httpx_client.request(
-            "POST",
-            urllib.parse.urljoin(
-                f"{self._client_wrapper.get_base_url()}/",
-                f"api/v1/pipelines/{pipeline_id}/eval-datasets/{eval_dataset_id}/execute",
-            ),
-            json=jsonable_encoder(_request),
-            headers=self._client_wrapper.get_headers(),
-            timeout=60,
-        )
-        if 200 <= _response.status_code < 300:
-            return pydantic.parse_obj_as(EvalDatasetJobRecord, _response.json())  # type: ignore
-        if _response.status_code == 422:
-            raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json()))  # type: ignore
-        try:
-            _response_json = _response.json()
-        except JSONDecodeError:
-            raise ApiError(status_code=_response.status_code, body=_response.text)
-        raise ApiError(status_code=_response.status_code, body=_response_json)
-
-    async def get_eval_dataset_execution_result(
-        self, eval_dataset_id: str, pipeline_id: str
-    ) -> typing.List[EvalQuestionResult]:
-        """
-        Get the result of an EvalDatasetExecution.
-        If eval_question_ids is specified, only the results for the specified
-        questions will be returned.
-        If any of the specified questions do not have a result, they will be ignored.
-
-        Parameters:
-            - eval_dataset_id: str.
-
-            - pipeline_id: str.
-        ---
-        from llama_cloud.client import AsyncLlamaCloud
-
-        client = AsyncLlamaCloud(
-            token="YOUR_TOKEN",
-        )
-        await client.pipelines.get_eval_dataset_execution_result(
-            eval_dataset_id="string",
-            pipeline_id="string",
-        )
-        """
-        _response = await self._client_wrapper.httpx_client.request(
-            "GET",
-            urllib.parse.urljoin(
-                f"{self._client_wrapper.get_base_url()}/",
-                f"api/v1/pipelines/{pipeline_id}/eval-datasets/{eval_dataset_id}/execute/result",
-            ),
-            headers=self._client_wrapper.get_headers(),
-            timeout=60,
-        )
-        if 200 <= _response.status_code < 300:
-            return pydantic.parse_obj_as(typing.List[EvalQuestionResult], _response.json())  # type: ignore
-        if _response.status_code == 422:
-            raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json()))  # type: ignore
-        try:
-            _response_json = _response.json()
-        except JSONDecodeError:
-            raise ApiError(status_code=_response.status_code, body=_response.text)
-        raise ApiError(status_code=_response.status_code, body=_response_json)
-
-    async def get_eval_dataset_execution(
-        self, eval_dataset_id: str, eval_dataset_execution_id: str, pipeline_id: str
-    ) -> EvalDatasetJobRecord:
-        """
-        Get the status of an EvalDatasetExecution.
-
-        Parameters:
-            - eval_dataset_id: str.
-
-            - eval_dataset_execution_id: str.
-
-            - pipeline_id: str.
-        ---
-        from llama_cloud.client import AsyncLlamaCloud
-
-        client = AsyncLlamaCloud(
-            token="YOUR_TOKEN",
-        )
-        await client.pipelines.get_eval_dataset_execution(
-            eval_dataset_id="string",
-            eval_dataset_execution_id="string",
-            pipeline_id="string",
-        )
-        """
-        _response = await self._client_wrapper.httpx_client.request(
-            "GET",
-            urllib.parse.urljoin(
-                f"{self._client_wrapper.get_base_url()}/",
-                f"api/v1/pipelines/{pipeline_id}/eval-datasets/{eval_dataset_id}/execute/{eval_dataset_execution_id}",
-            ),
-            headers=self._client_wrapper.get_headers(),
-            timeout=60,
-        )
-        if 200 <= _response.status_code < 300:
-            return pydantic.parse_obj_as(EvalDatasetJobRecord, _response.json())  # type: ignore
-        if _response.status_code == 422:
-            raise UnprocessableEntityError(pydantic.parse_obj_as(HttpValidationError, _response.json()))  # type: ignore
-        try:
-            _response_json = _response.json()
-        except JSONDecodeError:
-            raise ApiError(status_code=_response.status_code, body=_response.text)
-        raise ApiError(status_code=_response.status_code, body=_response_json)
-
     async def list_pipeline_files(
         self,
         pipeline_id: str,
@@ -3412,7 +3047,11 @@ class AsyncPipelinesClient:
 
     async def delete_pipeline_document(self, document_id: str, pipeline_id: str) -> None:
         """
-        Delete a document for a pipeline.
+        Delete a document from a pipeline.
+        Initiates an async job that will:
+
+        1. Delete vectors from the vector store
+        2. Delete the document from MongoDB after vectors are successfully deleted
 
         Parameters:
             - document_id: str.