unstructured-ingest 0.5.15__tar.gz → 0.5.17__tar.gz

{unstructured_ingest-0.5.15/unstructured_ingest.egg-info → unstructured_ingest-0.5.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: unstructured-ingest
-Version: 0.5.15
+Version: 0.5.17
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,13 +22,13 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
+Requires-Dist: opentelemetry-sdk
+Requires-Dist: pydantic>=2.7
+Requires-Dist: dataclasses_json
+Requires-Dist: pandas
 Requires-Dist: python-dateutil
 Requires-Dist: tqdm
-Requires-Dist: dataclasses_json
-Requires-Dist: pydantic>=2.7
 Requires-Dist: click
-Requires-Dist: opentelemetry-sdk
-Requires-Dist: pandas
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.30.0; extra == "remote"
 Provides-Extra: csv
@@ -71,8 +71,8 @@ Requires-Dist: fsspec; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: biomed
-Requires-Dist: bs4; extra == "biomed"
 Requires-Dist: requests; extra == "biomed"
+Requires-Dist: bs4; extra == "biomed"
 Provides-Extra: box
 Requires-Dist: boxfs; extra == "box"
 Requires-Dist: fsspec; extra == "box"
@@ -109,8 +109,8 @@ Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: hubspot-api-client; extra == "hubspot"
 Requires-Dist: urllib3; extra == "hubspot"
+Requires-Dist: hubspot-api-client; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -124,23 +124,23 @@ Requires-Dist: pymilvus; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: neo4j
-Requires-Dist: cymple; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
 Requires-Dist: neo4j-rust-ext; extra == "neo4j"
+Requires-Dist: cymple; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: htmlBuilder; extra == "notion"
-Requires-Dist: httpx; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
+Requires-Dist: httpx; extra == "notion"
 Requires-Dist: notion-client; extra == "notion"
+Requires-Dist: htmlBuilder; extra == "notion"
 Provides-Extra: onedrive
-Requires-Dist: bs4; extra == "onedrive"
-Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
+Requires-Dist: msal; extra == "onedrive"
+Requires-Dist: bs4; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: outlook
-Requires-Dist: msal; extra == "outlook"
 Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
+Requires-Dist: msal; extra == "outlook"
 Provides-Extra: pinecone
 Requires-Dist: pinecone-client>=3.7.1; extra == "pinecone"
 Provides-Extra: postgres
@@ -155,8 +155,8 @@ Provides-Extra: s3
 Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: fsspec; extra == "s3"
 Provides-Extra: sharepoint
-Requires-Dist: msal; extra == "sharepoint"
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
+Requires-Dist: msal; extra == "sharepoint"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
@@ -165,8 +165,8 @@ Requires-Dist: fsspec; extra == "sftp"
 Provides-Extra: slack
 Requires-Dist: slack_sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Requires-Dist: psycopg2-binary; extra == "snowflake"
+Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Provides-Extra: wikipedia
 Requires-Dist: wikipedia; extra == "wikipedia"
 Provides-Extra: weaviate
@@ -178,12 +178,12 @@ Requires-Dist: databricks-sql-connector; extra == "databricks-delta-tables"
 Provides-Extra: singlestore
 Requires-Dist: singlestoredb; extra == "singlestore"
 Provides-Extra: vectara
-Requires-Dist: httpx; extra == "vectara"
 Requires-Dist: requests; extra == "vectara"
 Requires-Dist: aiofiles; extra == "vectara"
+Requires-Dist: httpx; extra == "vectara"
 Provides-Extra: vastdb
-Requires-Dist: ibis; extra == "vastdb"
 Requires-Dist: vastdb; extra == "vastdb"
+Requires-Dist: ibis; extra == "vastdb"
 Requires-Dist: pyarrow; extra == "vastdb"
 Provides-Extra: zendesk
 Requires-Dist: httpx; extra == "zendesk"
@@ -192,8 +192,8 @@ Requires-Dist: aiofiles; extra == "zendesk"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
-Requires-Dist: tiktoken; extra == "embed-octoai"
 Requires-Dist: openai; extra == "embed-octoai"
+Requires-Dist: tiktoken; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
@@ -201,8 +201,8 @@ Requires-Dist: voyageai; extra == "embed-voyageai"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Provides-Extra: openai
-Requires-Dist: tiktoken; extra == "openai"
 Requires-Dist: openai; extra == "openai"
+Requires-Dist: tiktoken; extra == "openai"
 Provides-Extra: bedrock
 Requires-Dist: aioboto3; extra == "bedrock"
 Requires-Dist: boto3; extra == "bedrock"

{unstructured_ingest-0.5.15 → unstructured_ingest-0.5.17}/test/integration/connectors/test_zendesk.py

@@ -1,6 +1,5 @@
 import os
 from pathlib import Path
-from typing import Optional
 
 import pytest
 
@@ -21,20 +20,20 @@ from unstructured_ingest.v2.processes.connectors.zendesk.zendesk import (
     ZendeskIndexerConfig,
 )
 
+SUBDOMAIN = "unstructuredhelp"
+EMAIL = "test@unstructured.io"
 
-async def zendesk_source_test(
-    tmp_path: Path,
-    token: Optional[str] = None,
-    email: Optional[str] = None,
-    subdomain: Optional[str] = None,
-):
 
-    access_config = ZendeskAccessConfig(api_token=token)
+@pytest.mark.asyncio
+@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
+@requires_env("ZENDESK_TOKEN")
+async def test_zendesk_source_tickets(temp_dir: Path):
+    access_config = ZendeskAccessConfig(api_token=os.environ["ZENDESK_TOKEN"])
     connection_config = ZendeskConnectionConfig(
-        subdomain=subdomain, email=email, access_config=access_config
+        subdomain=SUBDOMAIN, email=EMAIL, access_config=access_config
     )
 
-    index_config = ZendeskIndexerConfig(batch_size=2, item_type="tickets")
+    index_config = ZendeskIndexerConfig(item_type="tickets")
 
     indexer = ZendeskIndexer(
         connection_config=connection_config,
@@ -43,7 +42,7 @@ async def zendesk_source_test(
     )
 
     # handle downloader.
-    download_config = ZendeskDownloaderConfig(download_dir=tmp_path)
+    download_config = ZendeskDownloaderConfig(download_dir=temp_dir)
 
     downloader = ZendeskDownloader(
         connection_config=connection_config,
@@ -57,26 +56,23 @@ async def zendesk_source_test(
         downloader=downloader,
         configs=SourceValidationConfigs(
             test_id="zendesk-tickets",
-            expected_num_files=4,
+            expected_num_files=8,
             validate_file_data=False,
             validate_downloaded_files=True,
         ),
     )
 
 
-async def zendesk_source_articles_test(
-    tmp_path: Path,
-    token: Optional[str] = None,
-    email: Optional[str] = None,
-    subdomain: Optional[str] = None,
-):
-
-    access_config = ZendeskAccessConfig(api_token=token)
+@pytest.mark.asyncio
+@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
+@requires_env("ZENDESK_TOKEN")
+async def test_zendesk_source_articles(temp_dir):
+    access_config = ZendeskAccessConfig(api_token=os.environ["ZENDESK_TOKEN"])
     connection_config = ZendeskConnectionConfig(
-        subdomain=subdomain, email=email, access_config=access_config
+        subdomain=SUBDOMAIN, email=EMAIL, access_config=access_config
     )
 
-    index_config = ZendeskIndexerConfig(batch_size=2, item_type="articles")
+    index_config = ZendeskIndexerConfig(item_type="articles")
 
     indexer = ZendeskIndexer(
         connection_config=connection_config,
@@ -85,7 +81,7 @@ async def zendesk_source_articles_test(
     )
 
     # handle downloader.
-    download_config = ZendeskDownloaderConfig(download_dir=tmp_path, extract_images=True)
+    download_config = ZendeskDownloaderConfig(download_dir=temp_dir, extract_images=True)
 
     downloader = ZendeskDownloader(
         connection_config=connection_config,
@@ -99,44 +95,26 @@ async def zendesk_source_articles_test(
         downloader=downloader,
         configs=SourceValidationConfigs(
             test_id="zendesk-articles",
-            expected_num_files=4,
-            validate_file_data=False,
+            expected_num_files=8,
+            validate_file_data=True,
             validate_downloaded_files=True,
         ),
     )
 
 
-@pytest.mark.asyncio
 @pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
-@requires_env("ZENDESK_TOKEN")
-async def test_zendesk_source(temp_dir):
-    await zendesk_source_test(
-        tmp_path=temp_dir,
-        token=os.environ["ZENDESK_TOKEN"],
-        email="test@unstructured.io",
-        subdomain="unstructuredhelp",
+def test_zendesk_source_articles_fail(temp_dir):
+    access_config = ZendeskAccessConfig(api_token="FAKE_TOKEN")
+    connection_config = ZendeskConnectionConfig(
+        subdomain=SUBDOMAIN, email=EMAIL, access_config=access_config
     )
 
+    index_config = ZendeskIndexerConfig(item_type="tickets")
 
-@pytest.mark.asyncio
-@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
-@requires_env("ZENDESK_TOKEN")
-async def test_zendesk_source_articles(temp_dir):
-    await zendesk_source_articles_test(
-        tmp_path=temp_dir,
-        token=os.environ["ZENDESK_TOKEN"],
-        email="test@unstructured.io",
-        subdomain="unstructuredhelp",
+    indexer = ZendeskIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+        connector_type=CONNECTOR_TYPE,
     )
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
-async def test_zendesk_source_articles_fail(temp_dir):
     with pytest.raises(expected_exception=UserAuthError):
-        await zendesk_source_articles_test(
-            tmp_path=temp_dir,
-            token="FORCE_FAIL_TOKEN",
-            email="test@unstructured.io",
-            subdomain="unstructuredhelp",
-        )
+        indexer.precheck()

{unstructured_ingest-0.5.15 → unstructured_ingest-0.5.17}/test/integration/connectors/utils/validation/source.py

@@ -103,7 +103,7 @@ def check_contents(
         file_data_path = expected_output_dir / f"{file_data.identifier}.json"
         with file_data_path.open("r") as file:
             expected_file_data_contents = json.load(file)
-        current_file_data_contents = file_data.model_dump()
+        current_file_data_contents = json.loads(file_data.model_dump_json())
         expected_file_data_contents = configs.omit_ignored_fields(expected_file_data_contents)
         current_file_data_contents = configs.omit_ignored_fields(current_file_data_contents)
         diff = DeepDiff(expected_file_data_contents, current_file_data_contents)
@@ -184,7 +184,7 @@ def update_fixtures(
         for file_data in all_file_data:
             file_data_path = file_data_output_path / f"{file_data.identifier}.json"
             with file_data_path.open(mode="w") as f:
-                json.dump(file_data.model_dump(), f, indent=2)
+                f.write(file_data.model_dump_json(indent=2))
 
     # Record file structure of download directory
     download_files = get_files(dir_path=download_dir)
@@ -216,7 +216,9 @@ def run_all_validations(
             len(predownload_file_data) == expected_number_indexed_file_data
         ), f"expected {expected_number_indexed_file_data} but got {len(predownload_file_data)}"
     if expected_num_files := configs.expected_num_files:
-        assert len(postdownload_file_data) == expected_num_files
+        assert (
+            len(postdownload_file_data) == expected_num_files
+        ), f"expected {expected_num_files} but got {len(postdownload_file_data)}"
 
     for pre_data, post_data in zip(predownload_file_data, postdownload_file_data):
         configs.run_file_data_validation(

unstructured_ingest-0.5.17/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.5.17"  # pragma: no cover

{unstructured_ingest-0.5.15 → unstructured_ingest-0.5.17}/unstructured_ingest/v2/pipeline/steps/download.py

@@ -88,9 +88,9 @@ class DownloadStep(PipelineStep):
                 f"match size of local file: {file_size_bytes}, updating"
             )
             file_data.metadata.filesize_bytes = file_size_bytes
-        logger.debug(f"updating file data with new content: {file_data.model_dump()}")
+        logger.debug(f"updating file data with new content: {file_data.model_dump_json()}")
         with file_data_path.open("w") as file:
-            json.dump(file_data.model_dump(), file, indent=2)
+            file.write(file_data.model_dump_json(indent=2))
 
     async def _run_async(self, fn: Callable, file_data_path: str) -> list[DownloadStepResponse]:
         file_data = file_data_from_file(path=file_data_path)
@@ -173,7 +173,7 @@ class DownloadStep(PipelineStep):
         filepath = (self.cache_dir / filename).resolve()
         filepath.parent.mkdir(parents=True, exist_ok=True)
         with open(str(filepath), "w") as f:
-            json.dump(file_data.model_dump(), f, indent=2)
+            f.write(file_data.model_dump_json(indent=2))
         return str(filepath)
 
     def get_hash(self, extras: Optional[list[str]]) -> str:

{unstructured_ingest-0.5.15 → unstructured_ingest-0.5.17}/unstructured_ingest/v2/pipeline/steps/index.py

@@ -37,14 +37,14 @@ class IndexStep(PipelineStep):
     @instrument(span_name=STEP_ID)
     def run(self) -> Generator[str, None, None]:
         for file_data in self.process.run():
-            logger.debug(f"generated file data: {file_data.model_dump()}")
+            logger.debug(f"generated file data: {file_data.model_dump_json()}")
             try:
                 record_hash = self.get_hash(extras=[file_data.identifier])
                 filename = f"{record_hash}.json"
                 filepath = (self.cache_dir / filename).resolve()
                 filepath.parent.mkdir(parents=True, exist_ok=True)
                 with open(str(filepath), "w") as f:
-                    json.dump(file_data.model_dump(), f, indent=2)
+                    f.write(file_data.model_dump_json(indent=2))
                 yield str(filepath)
             except Exception as e:
                 logger.error(f"failed to create index for file data: {file_data}", exc_info=True)
@@ -54,14 +54,14 @@ class IndexStep(PipelineStep):
 
     async def run_async(self) -> AsyncGenerator[str, None]:
         async for file_data in self.process.run_async():
-            logger.debug(f"generated file data: {file_data.model_dump()}")
+            logger.debug(f"generated file data: {file_data.model_dump_json()}")
             try:
                 record_hash = self.get_hash(extras=[file_data.identifier])
                 filename = f"{record_hash}.json"
                 filepath = (self.cache_dir / filename).resolve()
                 filepath.parent.mkdir(parents=True, exist_ok=True)
                 with open(str(filepath), "w") as f:
-                    json.dump(file_data.model_dump(), f, indent=2)
+                    f.write(file_data.model_dump_json(indent=2))
                 yield str(filepath)
             except Exception as e:
                 logger.error(f"failed to create index for file data: {file_data}", exc_info=True)

{unstructured_ingest-0.5.15 → unstructured_ingest-0.5.17}/unstructured_ingest/v2/processes/connectors/astradb.py

@@ -112,7 +112,6 @@ def get_astra_collection(
     collection_name: str,
     keyspace: str,
 ) -> "AstraDBCollection":
-
     astra_db = get_astra_db(connection_config=connection_config, keyspace=keyspace)
 
     # astradb will return a collection object in all cases (even if it doesn't exist)
@@ -315,6 +314,7 @@ class AstraDBUploadStager(UploadStager):
                     text_as_html, MAX_CONTENT_PARAM_BYTE_SIZE
                 )
             metadata["original_elements"] = format_and_truncate_orig_elements(element_dict)
+            metadata.pop("orig_elements", None)
 
     def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:
         self.truncate_dict_elements(element_dict)

{unstructured_ingest-0.5.15 → unstructured_ingest-0.5.17}/unstructured_ingest/v2/processes/connectors/databricks/volumes_table.py

@@ -61,7 +61,7 @@ class DatabricksVolumeDeltaTableUploader(Uploader):
                         self.upload_config.database, ", ".join(databases)
                     )
                 )
-            cursor.execute("SHOW TABLES")
+            cursor.execute(f"SHOW TABLES IN {self.upload_config.database}")
             table_names = [r[1] for r in cursor.fetchall()]
             if self.upload_config.table_name not in table_names:
                 raise ValueError(