unstructured-ingest 0.5.20__tar.gz → 0.5.21__tar.gz

{unstructured_ingest-0.5.20/unstructured_ingest.egg-info → unstructured_ingest-0.5.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: unstructured-ingest
-Version: 0.5.20
+Version: 0.5.21
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,197 +22,348 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: python-dateutil
-Requires-Dist: pandas
+Requires-Dist: dataclasses_json
+Requires-Dist: tqdm
 Requires-Dist: opentelemetry-sdk
-Requires-Dist: click
 Requires-Dist: pydantic>=2.7
-Requires-Dist: tqdm
-Requires-Dist: dataclasses_json
+Requires-Dist: python-dateutil
+Requires-Dist: click
+Requires-Dist: numpy
+Requires-Dist: pandas
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.30.0; extra == "remote"
+Requires-Dist: numpy; extra == "remote"
+Requires-Dist: pandas; extra == "remote"
 Provides-Extra: csv
 Requires-Dist: unstructured[tsv]; extra == "csv"
+Requires-Dist: numpy; extra == "csv"
+Requires-Dist: pandas; extra == "csv"
 Provides-Extra: doc
 Requires-Dist: unstructured[docx]; extra == "doc"
+Requires-Dist: numpy; extra == "doc"
+Requires-Dist: pandas; extra == "doc"
 Provides-Extra: docx
 Requires-Dist: unstructured[docx]; extra == "docx"
+Requires-Dist: numpy; extra == "docx"
+Requires-Dist: pandas; extra == "docx"
 Provides-Extra: epub
 Requires-Dist: unstructured[epub]; extra == "epub"
+Requires-Dist: numpy; extra == "epub"
+Requires-Dist: pandas; extra == "epub"
 Provides-Extra: md
 Requires-Dist: unstructured[md]; extra == "md"
+Requires-Dist: numpy; extra == "md"
+Requires-Dist: pandas; extra == "md"
 Provides-Extra: msg
 Requires-Dist: unstructured[msg]; extra == "msg"
+Requires-Dist: numpy; extra == "msg"
+Requires-Dist: pandas; extra == "msg"
 Provides-Extra: odt
 Requires-Dist: unstructured[odt]; extra == "odt"
+Requires-Dist: numpy; extra == "odt"
+Requires-Dist: pandas; extra == "odt"
 Provides-Extra: org
 Requires-Dist: unstructured[org]; extra == "org"
+Requires-Dist: numpy; extra == "org"
+Requires-Dist: pandas; extra == "org"
 Provides-Extra: pdf
 Requires-Dist: unstructured[pdf]; extra == "pdf"
+Requires-Dist: numpy; extra == "pdf"
+Requires-Dist: pandas; extra == "pdf"
 Provides-Extra: ppt
 Requires-Dist: unstructured[pptx]; extra == "ppt"
+Requires-Dist: numpy; extra == "ppt"
+Requires-Dist: pandas; extra == "ppt"
 Provides-Extra: pptx
 Requires-Dist: unstructured[pptx]; extra == "pptx"
+Requires-Dist: numpy; extra == "pptx"
+Requires-Dist: pandas; extra == "pptx"
 Provides-Extra: rtf
 Requires-Dist: unstructured[rtf]; extra == "rtf"
+Requires-Dist: numpy; extra == "rtf"
+Requires-Dist: pandas; extra == "rtf"
 Provides-Extra: rst
 Requires-Dist: unstructured[rst]; extra == "rst"
+Requires-Dist: numpy; extra == "rst"
+Requires-Dist: pandas; extra == "rst"
 Provides-Extra: tsv
 Requires-Dist: unstructured[tsv]; extra == "tsv"
+Requires-Dist: numpy; extra == "tsv"
+Requires-Dist: pandas; extra == "tsv"
 Provides-Extra: xlsx
 Requires-Dist: unstructured[xlsx]; extra == "xlsx"
+Requires-Dist: numpy; extra == "xlsx"
+Requires-Dist: pandas; extra == "xlsx"
 Provides-Extra: airtable
 Requires-Dist: pyairtable; extra == "airtable"
+Requires-Dist: numpy; extra == "airtable"
+Requires-Dist: pandas; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
+Requires-Dist: numpy; extra == "astradb"
+Requires-Dist: pandas; extra == "astradb"
 Provides-Extra: azure
 Requires-Dist: fsspec; extra == "azure"
 Requires-Dist: adlfs; extra == "azure"
+Requires-Dist: numpy; extra == "azure"
+Requires-Dist: pandas; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
+Requires-Dist: numpy; extra == "azure-ai-search"
+Requires-Dist: pandas; extra == "azure-ai-search"
 Provides-Extra: biomed
-Requires-Dist: bs4; extra == "biomed"
 Requires-Dist: requests; extra == "biomed"
+Requires-Dist: bs4; extra == "biomed"
+Requires-Dist: numpy; extra == "biomed"
+Requires-Dist: pandas; extra == "biomed"
 Provides-Extra: box
 Requires-Dist: boxfs; extra == "box"
 Requires-Dist: fsspec; extra == "box"
+Requires-Dist: numpy; extra == "box"
+Requires-Dist: pandas; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
+Requires-Dist: numpy; extra == "chroma"
+Requires-Dist: pandas; extra == "chroma"
 Provides-Extra: clarifai
 Requires-Dist: clarifai; extra == "clarifai"
+Requires-Dist: numpy; extra == "clarifai"
+Requires-Dist: pandas; extra == "clarifai"
 Provides-Extra: confluence
-Requires-Dist: atlassian-python-api; extra == "confluence"
 Requires-Dist: requests; extra == "confluence"
+Requires-Dist: atlassian-python-api; extra == "confluence"
+Requires-Dist: numpy; extra == "confluence"
+Requires-Dist: pandas; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
+Requires-Dist: numpy; extra == "couchbase"
+Requires-Dist: pandas; extra == "couchbase"
 Provides-Extra: delta-table
-Requires-Dist: deltalake; extra == "delta-table"
 Requires-Dist: boto3; extra == "delta-table"
+Requires-Dist: deltalake; extra == "delta-table"
+Requires-Dist: numpy; extra == "delta-table"
+Requires-Dist: pandas; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord.py; extra == "discord"
+Requires-Dist: numpy; extra == "discord"
+Requires-Dist: pandas; extra == "discord"
 Provides-Extra: dropbox
 Requires-Dist: fsspec; extra == "dropbox"
 Requires-Dist: dropboxdrivefs; extra == "dropbox"
+Requires-Dist: numpy; extra == "dropbox"
+Requires-Dist: pandas; extra == "dropbox"
 Provides-Extra: duckdb
 Requires-Dist: duckdb; extra == "duckdb"
+Requires-Dist: numpy; extra == "duckdb"
+Requires-Dist: pandas; extra == "duckdb"
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
+Requires-Dist: numpy; extra == "elasticsearch"
+Requires-Dist: pandas; extra == "elasticsearch"
 Provides-Extra: gcs
+Requires-Dist: fsspec; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: gcsfs; extra == "gcs"
-Requires-Dist: fsspec; extra == "gcs"
+Requires-Dist: numpy; extra == "gcs"
+Requires-Dist: pandas; extra == "gcs"
 Provides-Extra: github
-Requires-Dist: pygithub>1.58.0; extra == "github"
 Requires-Dist: requests; extra == "github"
+Requires-Dist: pygithub>1.58.0; extra == "github"
+Requires-Dist: numpy; extra == "github"
+Requires-Dist: pandas; extra == "github"
 Provides-Extra: gitlab
 Requires-Dist: python-gitlab; extra == "gitlab"
+Requires-Dist: numpy; extra == "gitlab"
+Requires-Dist: pandas; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
+Requires-Dist: numpy; extra == "google-drive"
+Requires-Dist: pandas; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: hubspot-api-client; extra == "hubspot"
 Requires-Dist: urllib3; extra == "hubspot"
+Requires-Dist: hubspot-api-client; extra == "hubspot"
+Requires-Dist: numpy; extra == "hubspot"
+Requires-Dist: pandas; extra == "hubspot"
 Provides-Extra: ibm-watsonx-s3
+Requires-Dist: httpx; extra == "ibm-watsonx-s3"
 Requires-Dist: pyarrow; extra == "ibm-watsonx-s3"
 Requires-Dist: tenacity; extra == "ibm-watsonx-s3"
 Requires-Dist: pyiceberg; extra == "ibm-watsonx-s3"
-Requires-Dist: httpx; extra == "ibm-watsonx-s3"
+Requires-Dist: numpy; extra == "ibm-watsonx-s3"
+Requires-Dist: pandas; extra == "ibm-watsonx-s3"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
+Requires-Dist: numpy; extra == "jira"
+Requires-Dist: pandas; extra == "jira"
 Provides-Extra: kafka
 Requires-Dist: confluent-kafka; extra == "kafka"
+Requires-Dist: numpy; extra == "kafka"
+Requires-Dist: pandas; extra == "kafka"
 Provides-Extra: kdbai
 Requires-Dist: kdbai-client>=1.4.0; extra == "kdbai"
+Requires-Dist: numpy; extra == "kdbai"
+Requires-Dist: pandas; extra == "kdbai"
 Provides-Extra: lancedb
 Requires-Dist: lancedb; extra == "lancedb"
+Requires-Dist: numpy; extra == "lancedb"
+Requires-Dist: pandas; extra == "lancedb"
 Provides-Extra: milvus
 Requires-Dist: pymilvus; extra == "milvus"
+Requires-Dist: numpy; extra == "milvus"
+Requires-Dist: pandas; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
+Requires-Dist: numpy; extra == "mongodb"
+Requires-Dist: pandas; extra == "mongodb"
 Provides-Extra: neo4j
 Requires-Dist: neo4j-rust-ext; extra == "neo4j"
-Requires-Dist: cymple; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
+Requires-Dist: cymple; extra == "neo4j"
+Requires-Dist: numpy; extra == "neo4j"
+Requires-Dist: pandas; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
+Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: notion-client; extra == "notion"
+Requires-Dist: numpy; extra == "notion"
+Requires-Dist: pandas; extra == "notion"
 Provides-Extra: onedrive
 Requires-Dist: msal; extra == "onedrive"
-Requires-Dist: bs4; extra == "onedrive"
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
+Requires-Dist: bs4; extra == "onedrive"
+Requires-Dist: numpy; extra == "onedrive"
+Requires-Dist: pandas; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
+Requires-Dist: numpy; extra == "opensearch"
+Requires-Dist: pandas; extra == "opensearch"
 Provides-Extra: outlook
 Requires-Dist: msal; extra == "outlook"
 Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
+Requires-Dist: numpy; extra == "outlook"
+Requires-Dist: pandas; extra == "outlook"
 Provides-Extra: pinecone
 Requires-Dist: pinecone-client>=3.7.1; extra == "pinecone"
+Requires-Dist: numpy; extra == "pinecone"
+Requires-Dist: pandas; extra == "pinecone"
 Provides-Extra: postgres
 Requires-Dist: psycopg2-binary; extra == "postgres"
+Requires-Dist: numpy; extra == "postgres"
+Requires-Dist: pandas; extra == "postgres"
 Provides-Extra: qdrant
 Requires-Dist: qdrant-client; extra == "qdrant"
+Requires-Dist: numpy; extra == "qdrant"
+Requires-Dist: pandas; extra == "qdrant"
 Provides-Extra: reddit
 Requires-Dist: praw; extra == "reddit"
+Requires-Dist: numpy; extra == "reddit"
+Requires-Dist: pandas; extra == "reddit"
 Provides-Extra: redis
 Requires-Dist: redis; extra == "redis"
+Requires-Dist: numpy; extra == "redis"
+Requires-Dist: pandas; extra == "redis"
 Provides-Extra: s3
 Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: fsspec; extra == "s3"
+Requires-Dist: numpy; extra == "s3"
+Requires-Dist: pandas; extra == "s3"
 Provides-Extra: sharepoint
 Requires-Dist: msal; extra == "sharepoint"
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
+Requires-Dist: numpy; extra == "sharepoint"
+Requires-Dist: pandas; extra == "sharepoint"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
+Requires-Dist: numpy; extra == "salesforce"
+Requires-Dist: pandas; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: fsspec; extra == "sftp"
 Requires-Dist: paramiko; extra == "sftp"
+Requires-Dist: fsspec; extra == "sftp"
+Requires-Dist: numpy; extra == "sftp"
+Requires-Dist: pandas; extra == "sftp"
 Provides-Extra: slack
 Requires-Dist: slack_sdk[optional]; extra == "slack"
+Requires-Dist: numpy; extra == "slack"
+Requires-Dist: pandas; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: psycopg2-binary; extra == "snowflake"
 Requires-Dist: snowflake-connector-python; extra == "snowflake"
+Requires-Dist: psycopg2-binary; extra == "snowflake"
+Requires-Dist: numpy; extra == "snowflake"
+Requires-Dist: pandas; extra == "snowflake"
 Provides-Extra: wikipedia
 Requires-Dist: wikipedia; extra == "wikipedia"
+Requires-Dist: numpy; extra == "wikipedia"
+Requires-Dist: pandas; extra == "wikipedia"
 Provides-Extra: weaviate
 Requires-Dist: weaviate-client; extra == "weaviate"
+Requires-Dist: numpy; extra == "weaviate"
+Requires-Dist: pandas; extra == "weaviate"
 Provides-Extra: databricks-volumes
 Requires-Dist: databricks-sdk; extra == "databricks-volumes"
+Requires-Dist: numpy; extra == "databricks-volumes"
+Requires-Dist: pandas; extra == "databricks-volumes"
 Provides-Extra: databricks-delta-tables
 Requires-Dist: databricks-sql-connector; extra == "databricks-delta-tables"
+Requires-Dist: numpy; extra == "databricks-delta-tables"
+Requires-Dist: pandas; extra == "databricks-delta-tables"
 Provides-Extra: singlestore
 Requires-Dist: singlestoredb; extra == "singlestore"
+Requires-Dist: numpy; extra == "singlestore"
+Requires-Dist: pandas; extra == "singlestore"
 Provides-Extra: vectara
-Requires-Dist: aiofiles; extra == "vectara"
 Requires-Dist: requests; extra == "vectara"
 Requires-Dist: httpx; extra == "vectara"
+Requires-Dist: aiofiles; extra == "vectara"
+Requires-Dist: numpy; extra == "vectara"
+Requires-Dist: pandas; extra == "vectara"
 Provides-Extra: vastdb
+Requires-Dist: ibis; extra == "vastdb"
 Requires-Dist: pyarrow; extra == "vastdb"
 Requires-Dist: vastdb; extra == "vastdb"
-Requires-Dist: ibis; extra == "vastdb"
+Requires-Dist: numpy; extra == "vastdb"
+Requires-Dist: pandas; extra == "vastdb"
 Provides-Extra: zendesk
-Requires-Dist: bs4; extra == "zendesk"
-Requires-Dist: aiofiles; extra == "zendesk"
 Requires-Dist: httpx; extra == "zendesk"
+Requires-Dist: aiofiles; extra == "zendesk"
+Requires-Dist: bs4; extra == "zendesk"
+Requires-Dist: numpy; extra == "zendesk"
+Requires-Dist: pandas; extra == "zendesk"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
+Requires-Dist: numpy; extra == "embed-huggingface"
+Requires-Dist: pandas; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
 Requires-Dist: tiktoken; extra == "embed-octoai"
 Requires-Dist: openai; extra == "embed-octoai"
+Requires-Dist: numpy; extra == "embed-octoai"
+Requires-Dist: pandas; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
+Requires-Dist: numpy; extra == "embed-vertexai"
+Requires-Dist: pandas; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
 Requires-Dist: voyageai; extra == "embed-voyageai"
+Requires-Dist: numpy; extra == "embed-voyageai"
+Requires-Dist: pandas; extra == "embed-voyageai"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
+Requires-Dist: numpy; extra == "embed-mixedbreadai"
+Requires-Dist: pandas; extra == "embed-mixedbreadai"
 Provides-Extra: openai
 Requires-Dist: tiktoken; extra == "openai"
 Requires-Dist: openai; extra == "openai"
+Requires-Dist: numpy; extra == "openai"
+Requires-Dist: pandas; extra == "openai"
 Provides-Extra: bedrock
 Requires-Dist: aioboto3; extra == "bedrock"
 Requires-Dist: boto3; extra == "bedrock"
+Requires-Dist: numpy; extra == "bedrock"
+Requires-Dist: pandas; extra == "bedrock"
 Provides-Extra: togetherai
 Requires-Dist: together; extra == "togetherai"
+Requires-Dist: numpy; extra == "togetherai"
+Requires-Dist: pandas; extra == "togetherai"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{unstructured_ingest-0.5.20 → unstructured_ingest-0.5.21}/setup.py

@@ -25,6 +25,8 @@ from setuptools import find_packages, setup
 
 from unstructured_ingest.__version__ import __version__
 
+_extra_reqs_filepath = "requirements/common/extras.in"
+
 
 def load_requirements(file: Union[str, Path]) -> List[str]:
     path = file if isinstance(file, Path) else Path(file)
@@ -42,8 +44,13 @@ def load_requirements(file: Union[str, Path]) -> List[str]:
         file_spec = recursive_req.split()[-1]
         file_path = Path(file_dir) / file_spec
         requirements.extend(load_requirements(file=file_path.resolve()))
+
     # Remove duplicates and any blank entries
-    return list({r for r in requirements if r})
+    result = list({r for r in requirements if r})
+
+    if file != _extra_reqs_filepath:
+        result.extend(load_requirements(_extra_reqs_filepath))
+    return result
 
 
 csv_reqs = load_requirements("requirements/local_partition/tsv.in")

{unstructured_ingest-0.5.20 → unstructured_ingest-0.5.21}/test/integration/connectors/test_astradb.py

@@ -1,3 +1,4 @@
+import contextlib
 import json
 import os
 from dataclasses import dataclass
@@ -231,6 +232,13 @@ def test_astra_create_destination():
     )
     collection_name = "system_created-123"
     formatted_collection_name = "system_created_123"
+
+    client = AstraDBClient()
+    db = client.get_database(api_endpoint=env_data.api_endpoint, token=env_data.token)
+    with contextlib.suppress(Exception):
+        # drop collection before trying to create it
+        db.drop_collection(formatted_collection_name)
+
     created = uploader.create_destination(destination_name=collection_name, vector_length=3072)
     assert created
     assert uploader.upload_config.collection_name == formatted_collection_name
@@ -239,8 +247,6 @@ def test_astra_create_destination():
     assert not created
 
     # cleanup
-    client = AstraDBClient()
-    db = client.get_database(api_endpoint=env_data.api_endpoint, token=env_data.token)
     db.drop_collection(formatted_collection_name)
 
 

unstructured_ingest-0.5.21/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.5.21"  # pragma: no cover

{unstructured_ingest-0.5.20 → unstructured_ingest-0.5.21}/unstructured_ingest/embed/interfaces.py

@@ -2,10 +2,10 @@ from abc import ABC
 from dataclasses import dataclass
 from typing import Any, Optional
 
-import numpy as np
 from pydantic import BaseModel, Field
 
 from unstructured_ingest.utils.data_prep import batch_generator
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 EMBEDDINGS_KEY = "embeddings"
 
@@ -32,7 +32,6 @@ class BaseEncoder(ABC):
 
 @dataclass
 class BaseEmbeddingEncoder(BaseEncoder, ABC):
-
     def initialize(self):
         """Initializes the embedding encoder class. Should also validate the instance
         is properly configured: e.g., embed a single a element"""
@@ -46,8 +45,11 @@ class BaseEmbeddingEncoder(BaseEncoder, ABC):
         return self.embed_query(query="Q")
 
     @property
+    @requires_dependencies(["numpy"])
     def is_unit_vector(self) -> bool:
         """Denotes if the embedding vector is a unit vector."""
+        import numpy as np
+
         exemplary_embedding = self.get_exemplary_embedding()
         return np.isclose(np.linalg.norm(exemplary_embedding), 1.0, rtol=1e-03)
 
@@ -86,7 +88,6 @@ class BaseEmbeddingEncoder(BaseEncoder, ABC):
 
 @dataclass
 class AsyncBaseEmbeddingEncoder(BaseEncoder, ABC):
-
     async def initialize(self):
         """Initializes the embedding encoder class. Should also validate the instance
         is properly configured: e.g., embed a single a element"""
@@ -100,8 +101,11 @@ class AsyncBaseEmbeddingEncoder(BaseEncoder, ABC):
         return await self.embed_query(query="Q")
 
     @property
+    @requires_dependencies(["numpy"])
     async def is_unit_vector(self) -> bool:
         """Denotes if the embedding vector is a unit vector."""
+        import numpy as np
+
         exemplary_embedding = await self.get_exemplary_embedding()
         return np.isclose(np.linalg.norm(exemplary_embedding), 1.0, rtol=1e-03)
 

{unstructured_ingest-0.5.20 → unstructured_ingest-0.5.21}/unstructured_ingest/utils/data_prep.py

@@ -2,20 +2,22 @@ import itertools
 import json
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Generator, Iterable, Optional, Sequence, TypeVar, Union, cast
-
-import pandas as pd
+from typing import TYPE_CHECKING, Any, Generator, Iterable, Optional, Sequence, TypeVar, Union, cast
 
 from unstructured_ingest.utils import ndjson
+from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.logger import logger
 
+if TYPE_CHECKING:
+    from pandas import DataFrame
+
 DATE_FORMATS = ("%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d+%H:%M:%S", "%Y-%m-%dT%H:%M:%S%z")
 
 T = TypeVar("T")
 IterableT = Iterable[T]
 
 
-def split_dataframe(df: pd.DataFrame, chunk_size: int = 100) -> Generator[pd.DataFrame, None, None]:
+def split_dataframe(df: "DataFrame", chunk_size: int = 100) -> Generator["DataFrame", None, None]:
     num_chunks = len(df) // chunk_size + 1
     for i in range(num_chunks):
         yield df[i * chunk_size : (i + 1) * chunk_size]
@@ -144,9 +146,13 @@ def get_data_by_suffix(path: Path) -> list[dict]:
         elif path.suffix == ".ndjson":
             return ndjson.load(f)
         elif path.suffix == ".csv":
+            import pandas as pd
+
             df = pd.read_csv(path)
             return df.to_dict(orient="records")
         elif path.suffix == ".parquet":
+            import pandas as pd
+
             df = pd.read_parquet(path)
             return df.to_dict(orient="records")
         else:
@@ -180,6 +186,9 @@ def get_data(path: Union[Path, str]) -> list[dict]:
             return ndjson.load(f)
         except Exception as e:
             logger.warning(f"failed to read {path} as ndjson: {e}")
+
+        import pandas as pd
+
         try:
             df = pd.read_csv(path)
             return df.to_dict(orient="records")
@@ -202,7 +211,10 @@ def get_json_data(path: Path) -> list[dict]:
             raise ValueError(f"Unsupported file type: {path}")
 
 
-def get_data_df(path: Path) -> pd.DataFrame:
+@requires_dependencies(["pandas"])
+def get_data_df(path: Path) -> "DataFrame":
+    import pandas as pd
+
     with path.open() as f:
         if path.suffix == ".json":
             data = json.load(f)

{unstructured_ingest-0.5.20 → unstructured_ingest-0.5.21}/unstructured_ingest/utils/table.py

@@ -1,11 +1,16 @@
-from typing import Any
-
-import pandas as pd
+from typing import TYPE_CHECKING, Any
 
 from unstructured_ingest.utils.data_prep import flatten_dict
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if TYPE_CHECKING:
+    from pandas import DataFrame
 
 
+@requires_dependencies(["pandas"])
 def get_default_pandas_dtypes() -> dict[str, Any]:
+    import pandas as pd
+
     return {
         "text": pd.StringDtype(),  # type: ignore
         "type": pd.StringDtype(),  # type: ignore
@@ -57,7 +62,9 @@ def get_default_pandas_dtypes() -> dict[str, Any]:
 def convert_to_pandas_dataframe(
     elements_dict: list[dict[str, Any]],
     drop_empty_cols: bool = False,
-) -> pd.DataFrame:
+) -> "DataFrame":
+    import pandas as pd
+
     # Flatten metadata if it hasn't already been flattened
     for d in elements_dict:
         if metadata := d.pop("metadata", None):

{unstructured_ingest-0.5.20 → unstructured_ingest-0.5.21}/unstructured_ingest/v2/processes/connectors/delta_table.py

@@ -3,10 +3,9 @@ import traceback
 from dataclasses import dataclass, field
 from multiprocessing import Process, Queue
 from pathlib import Path
-from typing import Any, Optional
+from typing import TYPE_CHECKING, Any, Optional
 from urllib.parse import urlparse
 
-import pandas as pd
 from pydantic import Field, Secret
 
 from unstructured_ingest.error import DestinationConnectionError
@@ -27,6 +26,9 @@ from unstructured_ingest.v2.processes.connector_registry import DestinationRegis
 
 CONNECTOR_TYPE = "delta_table"
 
+if TYPE_CHECKING:
+    from pandas import DataFrame
+
 
 @requires_dependencies(["deltalake"], extras="delta-table")
 def write_deltalake_with_error_handling(queue, **kwargs):
@@ -136,7 +138,7 @@ class DeltaTableUploader(Uploader):
                 logger.error(f"failed to validate connection: {e}", exc_info=True)
                 raise DestinationConnectionError(f"failed to validate connection: {e}")
 
-    def upload_dataframe(self, df: pd.DataFrame, file_data: FileData) -> None:
+    def upload_dataframe(self, df: "DataFrame", file_data: FileData) -> None:
         updated_upload_path = os.path.join(
             self.connection_config.table_uri, file_data.source_identifiers.relative_path
         )
@@ -172,7 +174,10 @@ class DeltaTableUploader(Uploader):
             logger.error(f"Exception occurred in write_deltalake: {error_message}")
             raise RuntimeError(f"Error in write_deltalake: {error_message}")
 
+    @requires_dependencies(["pandas"], extras="delta-table")
     def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
+        import pandas as pd
+
         df = pd.DataFrame(data=data)
         self.upload_dataframe(df=df, file_data=file_data)
 

{unstructured_ingest-0.5.20 → unstructured_ingest-0.5.21}/unstructured_ingest/v2/processes/connectors/duckdb/base.py

@@ -2,9 +2,8 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
-import pandas as pd
-
 from unstructured_ingest.utils.data_prep import get_data, write_data
+from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import FileData, UploadStager
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
@@ -55,7 +54,6 @@ _COLUMNS = (
 
 @dataclass
 class BaseDuckDBUploadStager(UploadStager):
-
     def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:
         data = element_dict.copy()
         metadata: dict[str, Any] = data.pop("metadata", {})
@@ -72,6 +70,7 @@ class BaseDuckDBUploadStager(UploadStager):
         data = {k: v for k, v in data.items() if k in _COLUMNS}
         return data
 
+    @requires_dependencies(["pandas"], extras="duckdb")
     def run(
         self,
         elements_filepath: Path,
@@ -80,6 +79,8 @@ class BaseDuckDBUploadStager(UploadStager):
         output_filename: str,
         **kwargs: Any,
     ) -> Path:
+        import pandas as pd
+
         elements_contents = get_data(path=elements_filepath)
         output_filename_suffix = Path(elements_filepath).suffix
         output_filename = f"{Path(output_filename).stem}{output_filename_suffix}"