PyPI - unstructured-ingest - Versions diffs - 0.5.23__tar.gz → 0.5.25__tar.gz - Mend

unstructured-ingest 0.5.23tar.gz → 0.5.25tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of unstructured-ingest might be problematic. Click here for more details.

Files changed (599) hide show

{unstructured_ingest-0.5.23/unstructured_ingest.egg-info → unstructured_ingest-0.5.25}/PKG-INFO RENAMED Viewed

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: unstructured-ingest
-Version: 0.5.23
+Version: 0.5.25
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -23,347 +23,347 @@ Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
 Requires-Dist: dataclasses_json
+Requires-Dist: pydantic>=2.7
 Requires-Dist: opentelemetry-sdk
 Requires-Dist: click
-Requires-Dist: pydantic>=2.7
 Requires-Dist: tqdm
 Requires-Dist: python-dateutil
-Requires-Dist: pandas
 Requires-Dist: numpy
+Requires-Dist: pandas
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.30.0; extra == "remote"
-Requires-Dist: pandas; extra == "remote"
 Requires-Dist: numpy; extra == "remote"
+Requires-Dist: pandas; extra == "remote"
 Provides-Extra: csv
 Requires-Dist: unstructured[tsv]; extra == "csv"
-Requires-Dist: pandas; extra == "csv"
 Requires-Dist: numpy; extra == "csv"
+Requires-Dist: pandas; extra == "csv"
 Provides-Extra: doc
 Requires-Dist: unstructured[docx]; extra == "doc"
-Requires-Dist: pandas; extra == "doc"
 Requires-Dist: numpy; extra == "doc"
+Requires-Dist: pandas; extra == "doc"
 Provides-Extra: docx
 Requires-Dist: unstructured[docx]; extra == "docx"
-Requires-Dist: pandas; extra == "docx"
 Requires-Dist: numpy; extra == "docx"
+Requires-Dist: pandas; extra == "docx"
 Provides-Extra: epub
 Requires-Dist: unstructured[epub]; extra == "epub"
-Requires-Dist: pandas; extra == "epub"
 Requires-Dist: numpy; extra == "epub"
+Requires-Dist: pandas; extra == "epub"
 Provides-Extra: md
 Requires-Dist: unstructured[md]; extra == "md"
-Requires-Dist: pandas; extra == "md"
 Requires-Dist: numpy; extra == "md"
+Requires-Dist: pandas; extra == "md"
 Provides-Extra: msg
 Requires-Dist: unstructured[msg]; extra == "msg"
-Requires-Dist: pandas; extra == "msg"
 Requires-Dist: numpy; extra == "msg"
+Requires-Dist: pandas; extra == "msg"
 Provides-Extra: odt
 Requires-Dist: unstructured[odt]; extra == "odt"
-Requires-Dist: pandas; extra == "odt"
 Requires-Dist: numpy; extra == "odt"
+Requires-Dist: pandas; extra == "odt"
 Provides-Extra: org
 Requires-Dist: unstructured[org]; extra == "org"
-Requires-Dist: pandas; extra == "org"
 Requires-Dist: numpy; extra == "org"
+Requires-Dist: pandas; extra == "org"
 Provides-Extra: pdf
 Requires-Dist: unstructured[pdf]; extra == "pdf"
-Requires-Dist: pandas; extra == "pdf"
 Requires-Dist: numpy; extra == "pdf"
+Requires-Dist: pandas; extra == "pdf"
 Provides-Extra: ppt
 Requires-Dist: unstructured[pptx]; extra == "ppt"
-Requires-Dist: pandas; extra == "ppt"
 Requires-Dist: numpy; extra == "ppt"
+Requires-Dist: pandas; extra == "ppt"
 Provides-Extra: pptx
 Requires-Dist: unstructured[pptx]; extra == "pptx"
-Requires-Dist: pandas; extra == "pptx"
 Requires-Dist: numpy; extra == "pptx"
+Requires-Dist: pandas; extra == "pptx"
 Provides-Extra: rtf
 Requires-Dist: unstructured[rtf]; extra == "rtf"
-Requires-Dist: pandas; extra == "rtf"
 Requires-Dist: numpy; extra == "rtf"
+Requires-Dist: pandas; extra == "rtf"
 Provides-Extra: rst
 Requires-Dist: unstructured[rst]; extra == "rst"
-Requires-Dist: pandas; extra == "rst"
 Requires-Dist: numpy; extra == "rst"
+Requires-Dist: pandas; extra == "rst"
 Provides-Extra: tsv
 Requires-Dist: unstructured[tsv]; extra == "tsv"
-Requires-Dist: pandas; extra == "tsv"
 Requires-Dist: numpy; extra == "tsv"
+Requires-Dist: pandas; extra == "tsv"
 Provides-Extra: xlsx
 Requires-Dist: unstructured[xlsx]; extra == "xlsx"
-Requires-Dist: pandas; extra == "xlsx"
 Requires-Dist: numpy; extra == "xlsx"
+Requires-Dist: pandas; extra == "xlsx"
 Provides-Extra: airtable
 Requires-Dist: pyairtable; extra == "airtable"
-Requires-Dist: pandas; extra == "airtable"
 Requires-Dist: numpy; extra == "airtable"
+Requires-Dist: pandas; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
-Requires-Dist: pandas; extra == "astradb"
 Requires-Dist: numpy; extra == "astradb"
+Requires-Dist: pandas; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: adlfs; extra == "azure"
 Requires-Dist: fsspec; extra == "azure"
-Requires-Dist: pandas; extra == "azure"
+Requires-Dist: adlfs; extra == "azure"
 Requires-Dist: numpy; extra == "azure"
+Requires-Dist: pandas; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
-Requires-Dist: pandas; extra == "azure-ai-search"
 Requires-Dist: numpy; extra == "azure-ai-search"
+Requires-Dist: pandas; extra == "azure-ai-search"
 Provides-Extra: biomed
-Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
-Requires-Dist: pandas; extra == "biomed"
+Requires-Dist: requests; extra == "biomed"
 Requires-Dist: numpy; extra == "biomed"
+Requires-Dist: pandas; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: boxfs; extra == "box"
 Requires-Dist: fsspec; extra == "box"
-Requires-Dist: pandas; extra == "box"
+Requires-Dist: boxfs; extra == "box"
 Requires-Dist: numpy; extra == "box"
+Requires-Dist: pandas; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
-Requires-Dist: pandas; extra == "chroma"
 Requires-Dist: numpy; extra == "chroma"
+Requires-Dist: pandas; extra == "chroma"
 Provides-Extra: clarifai
 Requires-Dist: clarifai; extra == "clarifai"
-Requires-Dist: pandas; extra == "clarifai"
 Requires-Dist: numpy; extra == "clarifai"
+Requires-Dist: pandas; extra == "clarifai"
 Provides-Extra: confluence
-Requires-Dist: requests; extra == "confluence"
 Requires-Dist: atlassian-python-api; extra == "confluence"
-Requires-Dist: pandas; extra == "confluence"
+Requires-Dist: requests; extra == "confluence"
 Requires-Dist: numpy; extra == "confluence"
+Requires-Dist: pandas; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
-Requires-Dist: pandas; extra == "couchbase"
 Requires-Dist: numpy; extra == "couchbase"
+Requires-Dist: pandas; extra == "couchbase"
 Provides-Extra: delta-table
-Requires-Dist: deltalake; extra == "delta-table"
 Requires-Dist: boto3; extra == "delta-table"
-Requires-Dist: pandas; extra == "delta-table"
+Requires-Dist: deltalake; extra == "delta-table"
 Requires-Dist: numpy; extra == "delta-table"
+Requires-Dist: pandas; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord.py; extra == "discord"
-Requires-Dist: pandas; extra == "discord"
 Requires-Dist: numpy; extra == "discord"
+Requires-Dist: pandas; extra == "discord"
 Provides-Extra: dropbox
-Requires-Dist: dropboxdrivefs; extra == "dropbox"
 Requires-Dist: fsspec; extra == "dropbox"
-Requires-Dist: pandas; extra == "dropbox"
+Requires-Dist: dropboxdrivefs; extra == "dropbox"
 Requires-Dist: numpy; extra == "dropbox"
+Requires-Dist: pandas; extra == "dropbox"
 Provides-Extra: duckdb
 Requires-Dist: duckdb; extra == "duckdb"
-Requires-Dist: pandas; extra == "duckdb"
 Requires-Dist: numpy; extra == "duckdb"
+Requires-Dist: pandas; extra == "duckdb"
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
-Requires-Dist: pandas; extra == "elasticsearch"
 Requires-Dist: numpy; extra == "elasticsearch"
+Requires-Dist: pandas; extra == "elasticsearch"
 Provides-Extra: gcs
-Requires-Dist: bs4; extra == "gcs"
-Requires-Dist: gcsfs; extra == "gcs"
 Requires-Dist: fsspec; extra == "gcs"
-Requires-Dist: pandas; extra == "gcs"
+Requires-Dist: gcsfs; extra == "gcs"
+Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: numpy; extra == "gcs"
+Requires-Dist: pandas; extra == "gcs"
 Provides-Extra: github
 Requires-Dist: pygithub>1.58.0; extra == "github"
 Requires-Dist: requests; extra == "github"
-Requires-Dist: pandas; extra == "github"
 Requires-Dist: numpy; extra == "github"
+Requires-Dist: pandas; extra == "github"
 Provides-Extra: gitlab
 Requires-Dist: python-gitlab; extra == "gitlab"
-Requires-Dist: pandas; extra == "gitlab"
 Requires-Dist: numpy; extra == "gitlab"
+Requires-Dist: pandas; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
-Requires-Dist: pandas; extra == "google-drive"
 Requires-Dist: numpy; extra == "google-drive"
+Requires-Dist: pandas; extra == "google-drive"
 Provides-Extra: hubspot
 Requires-Dist: hubspot-api-client; extra == "hubspot"
 Requires-Dist: urllib3; extra == "hubspot"
-Requires-Dist: pandas; extra == "hubspot"
 Requires-Dist: numpy; extra == "hubspot"
+Requires-Dist: pandas; extra == "hubspot"
 Provides-Extra: ibm-watsonx-s3
 Requires-Dist: tenacity; extra == "ibm-watsonx-s3"
+Requires-Dist: pyiceberg; extra == "ibm-watsonx-s3"
 Requires-Dist: pyarrow; extra == "ibm-watsonx-s3"
 Requires-Dist: httpx; extra == "ibm-watsonx-s3"
-Requires-Dist: pyiceberg; extra == "ibm-watsonx-s3"
-Requires-Dist: pandas; extra == "ibm-watsonx-s3"
 Requires-Dist: numpy; extra == "ibm-watsonx-s3"
+Requires-Dist: pandas; extra == "ibm-watsonx-s3"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
-Requires-Dist: pandas; extra == "jira"
 Requires-Dist: numpy; extra == "jira"
+Requires-Dist: pandas; extra == "jira"
 Provides-Extra: kafka
 Requires-Dist: confluent-kafka; extra == "kafka"
-Requires-Dist: pandas; extra == "kafka"
 Requires-Dist: numpy; extra == "kafka"
+Requires-Dist: pandas; extra == "kafka"
 Provides-Extra: kdbai
 Requires-Dist: kdbai-client>=1.4.0; extra == "kdbai"
-Requires-Dist: pandas; extra == "kdbai"
 Requires-Dist: numpy; extra == "kdbai"
+Requires-Dist: pandas; extra == "kdbai"
 Provides-Extra: lancedb
 Requires-Dist: lancedb; extra == "lancedb"
-Requires-Dist: pandas; extra == "lancedb"
 Requires-Dist: numpy; extra == "lancedb"
+Requires-Dist: pandas; extra == "lancedb"
 Provides-Extra: milvus
 Requires-Dist: pymilvus; extra == "milvus"
-Requires-Dist: pandas; extra == "milvus"
 Requires-Dist: numpy; extra == "milvus"
+Requires-Dist: pandas; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
-Requires-Dist: pandas; extra == "mongodb"
 Requires-Dist: numpy; extra == "mongodb"
+Requires-Dist: pandas; extra == "mongodb"
 Provides-Extra: neo4j
+Requires-Dist: cymple; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
 Requires-Dist: neo4j-rust-ext; extra == "neo4j"
-Requires-Dist: cymple; extra == "neo4j"
-Requires-Dist: pandas; extra == "neo4j"
 Requires-Dist: numpy; extra == "neo4j"
+Requires-Dist: pandas; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: backoff; extra == "notion"
-Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: notion-client; extra == "notion"
+Requires-Dist: backoff; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
-Requires-Dist: pandas; extra == "notion"
+Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: numpy; extra == "notion"
+Requires-Dist: pandas; extra == "notion"
 Provides-Extra: onedrive
 Requires-Dist: msal; extra == "onedrive"
-Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
-Requires-Dist: pandas; extra == "onedrive"
+Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Requires-Dist: numpy; extra == "onedrive"
+Requires-Dist: pandas; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
-Requires-Dist: pandas; extra == "opensearch"
 Requires-Dist: numpy; extra == "opensearch"
+Requires-Dist: pandas; extra == "opensearch"
 Provides-Extra: outlook
 Requires-Dist: msal; extra == "outlook"
 Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
-Requires-Dist: pandas; extra == "outlook"
 Requires-Dist: numpy; extra == "outlook"
+Requires-Dist: pandas; extra == "outlook"
 Provides-Extra: pinecone
 Requires-Dist: pinecone-client>=3.7.1; extra == "pinecone"
-Requires-Dist: pandas; extra == "pinecone"
 Requires-Dist: numpy; extra == "pinecone"
+Requires-Dist: pandas; extra == "pinecone"
 Provides-Extra: postgres
 Requires-Dist: psycopg2-binary; extra == "postgres"
-Requires-Dist: pandas; extra == "postgres"
 Requires-Dist: numpy; extra == "postgres"
+Requires-Dist: pandas; extra == "postgres"
 Provides-Extra: qdrant
 Requires-Dist: qdrant-client; extra == "qdrant"
-Requires-Dist: pandas; extra == "qdrant"
 Requires-Dist: numpy; extra == "qdrant"
+Requires-Dist: pandas; extra == "qdrant"
 Provides-Extra: reddit
 Requires-Dist: praw; extra == "reddit"
-Requires-Dist: pandas; extra == "reddit"
 Requires-Dist: numpy; extra == "reddit"
+Requires-Dist: pandas; extra == "reddit"
 Provides-Extra: redis
 Requires-Dist: redis; extra == "redis"
-Requires-Dist: pandas; extra == "redis"
 Requires-Dist: numpy; extra == "redis"
+Requires-Dist: pandas; extra == "redis"
 Provides-Extra: s3
-Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: fsspec; extra == "s3"
-Requires-Dist: pandas; extra == "s3"
+Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: numpy; extra == "s3"
+Requires-Dist: pandas; extra == "s3"
 Provides-Extra: sharepoint
 Requires-Dist: msal; extra == "sharepoint"
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
-Requires-Dist: pandas; extra == "sharepoint"
 Requires-Dist: numpy; extra == "sharepoint"
+Requires-Dist: pandas; extra == "sharepoint"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
-Requires-Dist: pandas; extra == "salesforce"
 Requires-Dist: numpy; extra == "salesforce"
+Requires-Dist: pandas; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: paramiko; extra == "sftp"
 Requires-Dist: fsspec; extra == "sftp"
-Requires-Dist: pandas; extra == "sftp"
+Requires-Dist: paramiko; extra == "sftp"
 Requires-Dist: numpy; extra == "sftp"
+Requires-Dist: pandas; extra == "sftp"
 Provides-Extra: slack
 Requires-Dist: slack_sdk[optional]; extra == "slack"
-Requires-Dist: pandas; extra == "slack"
 Requires-Dist: numpy; extra == "slack"
+Requires-Dist: pandas; extra == "slack"
 Provides-Extra: snowflake
 Requires-Dist: psycopg2-binary; extra == "snowflake"
 Requires-Dist: snowflake-connector-python; extra == "snowflake"
-Requires-Dist: pandas; extra == "snowflake"
 Requires-Dist: numpy; extra == "snowflake"
+Requires-Dist: pandas; extra == "snowflake"
 Provides-Extra: wikipedia
 Requires-Dist: wikipedia; extra == "wikipedia"
-Requires-Dist: pandas; extra == "wikipedia"
 Requires-Dist: numpy; extra == "wikipedia"
+Requires-Dist: pandas; extra == "wikipedia"
 Provides-Extra: weaviate
 Requires-Dist: weaviate-client; extra == "weaviate"
-Requires-Dist: pandas; extra == "weaviate"
 Requires-Dist: numpy; extra == "weaviate"
+Requires-Dist: pandas; extra == "weaviate"
 Provides-Extra: databricks-volumes
 Requires-Dist: databricks-sdk; extra == "databricks-volumes"
-Requires-Dist: pandas; extra == "databricks-volumes"
 Requires-Dist: numpy; extra == "databricks-volumes"
+Requires-Dist: pandas; extra == "databricks-volumes"
 Provides-Extra: databricks-delta-tables
 Requires-Dist: databricks-sql-connector; extra == "databricks-delta-tables"
-Requires-Dist: pandas; extra == "databricks-delta-tables"
 Requires-Dist: numpy; extra == "databricks-delta-tables"
+Requires-Dist: pandas; extra == "databricks-delta-tables"
 Provides-Extra: singlestore
 Requires-Dist: singlestoredb; extra == "singlestore"
-Requires-Dist: pandas; extra == "singlestore"
 Requires-Dist: numpy; extra == "singlestore"
+Requires-Dist: pandas; extra == "singlestore"
 Provides-Extra: vectara
+Requires-Dist: aiofiles; extra == "vectara"
 Requires-Dist: httpx; extra == "vectara"
 Requires-Dist: requests; extra == "vectara"
-Requires-Dist: aiofiles; extra == "vectara"
-Requires-Dist: pandas; extra == "vectara"
 Requires-Dist: numpy; extra == "vectara"
+Requires-Dist: pandas; extra == "vectara"
 Provides-Extra: vastdb
 Requires-Dist: ibis; extra == "vastdb"
 Requires-Dist: pyarrow; extra == "vastdb"
 Requires-Dist: vastdb; extra == "vastdb"
-Requires-Dist: pandas; extra == "vastdb"
 Requires-Dist: numpy; extra == "vastdb"
+Requires-Dist: pandas; extra == "vastdb"
 Provides-Extra: zendesk
 Requires-Dist: aiofiles; extra == "zendesk"
-Requires-Dist: httpx; extra == "zendesk"
 Requires-Dist: bs4; extra == "zendesk"
-Requires-Dist: pandas; extra == "zendesk"
+Requires-Dist: httpx; extra == "zendesk"
 Requires-Dist: numpy; extra == "zendesk"
+Requires-Dist: pandas; extra == "zendesk"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
-Requires-Dist: pandas; extra == "embed-huggingface"
 Requires-Dist: numpy; extra == "embed-huggingface"
+Requires-Dist: pandas; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
 Requires-Dist: openai; extra == "embed-octoai"
 Requires-Dist: tiktoken; extra == "embed-octoai"
-Requires-Dist: pandas; extra == "embed-octoai"
 Requires-Dist: numpy; extra == "embed-octoai"
+Requires-Dist: pandas; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
-Requires-Dist: pandas; extra == "embed-vertexai"
 Requires-Dist: numpy; extra == "embed-vertexai"
+Requires-Dist: pandas; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
 Requires-Dist: voyageai; extra == "embed-voyageai"
-Requires-Dist: pandas; extra == "embed-voyageai"
 Requires-Dist: numpy; extra == "embed-voyageai"
+Requires-Dist: pandas; extra == "embed-voyageai"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
-Requires-Dist: pandas; extra == "embed-mixedbreadai"
 Requires-Dist: numpy; extra == "embed-mixedbreadai"
+Requires-Dist: pandas; extra == "embed-mixedbreadai"
 Provides-Extra: openai
 Requires-Dist: openai; extra == "openai"
 Requires-Dist: tiktoken; extra == "openai"
-Requires-Dist: pandas; extra == "openai"
 Requires-Dist: numpy; extra == "openai"
+Requires-Dist: pandas; extra == "openai"
 Provides-Extra: bedrock
 Requires-Dist: boto3; extra == "bedrock"
 Requires-Dist: aioboto3; extra == "bedrock"
-Requires-Dist: pandas; extra == "bedrock"
 Requires-Dist: numpy; extra == "bedrock"
+Requires-Dist: pandas; extra == "bedrock"
 Provides-Extra: togetherai
 Requires-Dist: together; extra == "togetherai"
-Requires-Dist: pandas; extra == "togetherai"
 Requires-Dist: numpy; extra == "togetherai"
+Requires-Dist: pandas; extra == "togetherai"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{unstructured_ingest-0.5.23 → unstructured_ingest-0.5.25}/test/integration/connectors/test_vectara.py RENAMED Viewed

@@ -1,6 +1,7 @@
 import json
 import os
 import time
+from functools import lru_cache
 from pathlib import Path
 from typing import Generator
 from uuid import uuid4
@@ -25,24 +26,29 @@ from unstructured_ingest.v2.processes.connectors.vectara import (
 )
-def validate_upload(response: dict, expected_data: dict):
+def validate_upload(document: dict, expected_data: dict):
+    logger.info(f"validating document: {document}")
     element_id = expected_data["element_id"]
     expected_text = expected_data["text"]
     filename = expected_data["metadata"]["filename"]
     filetype = expected_data["metadata"]["filetype"]
     page_number = expected_data["metadata"]["page_number"]
-    response = response["search_results"][0]
-    assert response is not None
-    assert response["text"] == expected_text
-    assert response["part_metadata"]["element_id"] == element_id
-    assert response["part_metadata"]["filename"] == filename
-    assert response["part_metadata"]["filetype"] == filetype
-    assert response["part_metadata"]["page_number"] == page_number
+    assert document is not None
+    speech_parts = document["parts"]
+    assert speech_parts
+    first_part = speech_parts[0]
+    assert first_part["text"] == expected_text
+    part_metadata = first_part["metadata"]
+    assert part_metadata
+    assert part_metadata["element_id"] == element_id
+    assert part_metadata["filename"] == filename
+    assert part_metadata["filetype"] == filetype
+    assert part_metadata["page_number"] == page_number
 @requires_env("VECTARA_OAUTH_CLIENT_ID", "VECTARA_OAUTH_SECRET", "VECTARA_CUSTOMER_ID")
+@lru_cache()
 def _get_jwt_token():
     """Connect to the server and get a JWT token."""
     customer_id = os.environ["VECTARA_CUSTOMER_ID"]
@@ -65,23 +71,12 @@ def _get_jwt_token():
     return response_json.get("access_token")
-def query_data(corpus_key: str, element_id: str) -> dict:
+def list_documents(corpus_key: str) -> list[str]:
-    url = f"https://api.vectara.io/v2/corpora/{corpus_key}/query"
+    url = f"https://api.vectara.io/v2/corpora/{corpus_key}/documents"
     # the query below requires the corpus to have filter attributes for element_id
-    data = json.dumps(
-        {
-            "query": "string",
-            "search": {
-                "metadata_filter": f"part.element_id = '{element_id}'",
-                "lexical_interpolation": 1,
-                "limit": 10,
-            },
-        }
-    )
     jwt_token = _get_jwt_token()
     headers = {
         "Content-Type": "application/json",
@@ -90,11 +85,26 @@ def query_data(corpus_key: str, element_id: str) -> dict:
         "X-source": "unstructured",
     }
-    response = requests.post(url, headers=headers, data=data)
+    response = requests.get(url, headers=headers)
     response.raise_for_status()
     response_json = response.json()
+    documents = response_json.get("documents", [])
+    return documents
-    return response_json
+def fetch_document(corpus_key: str, documents_id: str) -> dict:
+    url = f"https://api.vectara.io/v2/corpora/{corpus_key}/documents/{documents_id}"
+    jwt_token = _get_jwt_token()
+    headers = {
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+        "Authorization": f"Bearer {jwt_token}",
+        "X-source": "unstructured",
+    }
+    response = requests.get(url, headers=headers)
+    response.raise_for_status()
+    return response.json()
 def create_corpora(corpus_key: str, corpus_name: str) -> None:
@@ -148,8 +158,8 @@ def delete_corpora(corpus_key: str) -> None:
     response.raise_for_status()
-def list_corpora() -> list:
-    url = "https://api.vectara.io/v2/corpora?limit=100"
+def get_metadata(corpus_key: str):
+    url = f"https://api.vectara.io/v2/corpora/{corpus_key}"
     jwt_token = _get_jwt_token()
     headers = {
         "Content-Type": "application/json",
@@ -159,35 +169,28 @@ def list_corpora() -> list:
     }
     response = requests.get(url, headers=headers)
     response.raise_for_status()
-    response_json = response.json()
-    if response_json.get("corpora"):
-        return [item["key"] for item in response_json.get("corpora")]
-    else:
-        return []
+    return response.json()
 def wait_for_ready(corpus_key: str, timeout=60, interval=2) -> None:
-    def is_ready_status():
-        corpora_list = list_corpora()
-        return corpus_key in corpora_list
     start = time.time()
-    is_ready = is_ready_status()
-    while not is_ready and time.time() - start < timeout:
-        time.sleep(interval)
-        is_ready = is_ready_status()
-    if not is_ready:
-        raise TimeoutError("time out waiting for corpus to be ready")
+    while time.time() - start < timeout:
+        try:
+            get_metadata(corpus_key)
+            return
+        except requests.HTTPError:
+            time.sleep(interval)
+    raise TimeoutError("time out waiting for corpus to be ready")
 def wait_for_delete(corpus_key: str, timeout=60, interval=2) -> None:
     start = time.time()
     while time.time() - start < timeout:
-        corpora_list = list_corpora()
-        if corpus_key not in corpora_list:
+        try:
+            get_metadata(corpus_key)
+            time.sleep(interval)
+        except requests.HTTPError:
             return
-        time.sleep(interval)
     raise TimeoutError("time out waiting for corpus to delete")
@@ -210,11 +213,23 @@ def corpora_util() -> Generator[str, None, None]:
         wait_for_delete(corpus_key=corpus_key)
+def wait_for_doc_meta(corpus_key: str, timeout=60, interval=1) -> list[str]:
+    start = time.time()
+    while time.time() - start < timeout:
+        all_document_meta = list_documents(corpus_key)
+        if not all_document_meta:
+            time.sleep(interval)
+            continue
+        else:
+            return all_document_meta
+    raise TimeoutError("time out waiting for document to be ready")
 @pytest.mark.asyncio
 @pytest.mark.tags(VECTARA_CONNECTOR_TYPE, DESTINATION_TAG, "vectara", NOSQL_TAG)
 @requires_env("VECTARA_OAUTH_CLIENT_ID", "VECTARA_OAUTH_SECRET", "VECTARA_CUSTOMER_ID")
 async def test_vectara_destination(
-    upload_file: Path, tmp_path: Path, corpora_util: str, retries=30, interval=10
+    upload_file: Path, tmp_path: Path, corpora_util: str, retries=30, interval=1
 ):
     corpus_key = corpora_util
     connection_kwargs = {
@@ -231,7 +246,7 @@ async def test_vectara_destination(
         identifier="mock-file-data",
     )
-    stager_config = VectaraUploadStagerConfig(batch_size=10)
+    stager_config = VectaraUploadStagerConfig()
     stager = VectaraUploadStager(upload_stager_config=stager_config)
     new_upload_file = stager.run(
         elements_filepath=upload_file,
@@ -260,11 +275,8 @@ async def test_vectara_destination(
         elements = json.load(upload_fp)
     first_element = elements[0]
-    for i in range(retries):
-        response = query_data(corpus_key, first_element["element_id"])
-        if not response["search_results"]:
-            time.sleep(interval)
-        else:
-            break
-    validate_upload(response=response, expected_data=first_element)
+    all_document_meta = wait_for_doc_meta(corpus_key)
+    assert len(all_document_meta) == 1
+    document_meta = all_document_meta[0]
+    document = fetch_document(corpus_key=corpus_key, documents_id=document_meta["id"])
+    validate_upload(document=document, expected_data=first_element)

unstructured_ingest-0.5.25/unstructured_ingest/__version__.py ADDED Viewed

	@@ -0,0 +1 @@
1	+ __version__ = "0.5.25" # pragma: no cover

unstructured-ingest 0.5.23__tar.gz → 0.5.25__tar.gz

Potentially problematic release.

unstructured-ingest 0.5.23tar.gz → 0.5.25tar.gz