unstructured-ingest 0.7.2__py3-none-any.whl → 1.0.1__py3-none-any.whl

unstructured_ingest/processes/connectors/sql/sql.py

@@ -36,9 +36,9 @@ from unstructured_ingest.interfaces import (
 from unstructured_ingest.logger import logger
 from unstructured_ingest.utils.constants import RECORD_ID_LABEL
 from unstructured_ingest.utils.data_prep import (
-    get_data,
     get_data_df,
     get_enhanced_element_id,
+    get_json_data,
     split_dataframe,
     write_data,
 )
@@ -122,8 +122,7 @@ class SQLIndexer(Indexer, ABC):
         id_batches: list[frozenset[str]] = [
             frozenset(
                 ids[
-                    i
-                    * self.index_config.batch_size : (i + 1)  # noqa
+                    i * self.index_config.batch_size : (i + 1)  # noqa
                     * self.index_config.batch_size
                 ]
             )
@@ -272,7 +271,7 @@ class SQLUploadStager(UploadStager):
     ) -> Path:
         import pandas as pd
 
-        elements_contents = get_data(path=elements_filepath)
+        elements_contents = get_json_data(path=elements_filepath)
 
         df = pd.DataFrame(
             data=[

unstructured_ingest/processes/connectors/sql/sqlite.py

@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any, Generator
 
 from pydantic import Field, Secret, model_validator
 
+from unstructured_ingest.data_types.file_data import FileData
 from unstructured_ingest.logger import logger
 from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,
@@ -133,6 +134,10 @@ class SQLiteUploader(SQLUploader):
     connection_config: SQLiteConnectionConfig
     connector_type: str = CONNECTOR_TYPE
 
+    @requires_dependencies(["pandas"])
+    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        super().run(path=path, file_data=file_data, **kwargs)
+
     @requires_dependencies(["pandas"])
     def prepare_data(
         self, columns: list[str], data: tuple[tuple[Any, ...], ...]

unstructured_ingest/processes/connectors/sql/vastdb.py

@@ -1,5 +1,6 @@
 from contextlib import contextmanager
 from dataclasses import dataclass, field
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional
 
 from pydantic import Field, Secret
@@ -68,9 +69,8 @@ class VastdbConnectionConfig(SQLConnectionConfig):
 
     @contextmanager
     def get_cursor(self) -> "VastdbTransaction":
-        with self.get_connection() as connection:
-            with connection.transaction() as transaction:
-                yield transaction
+        with self.get_connection() as connection, connection.transaction() as transaction:
+            yield transaction
 
     @contextmanager
     def get_table(self, table_name: str) -> "VastdbTable":
@@ -190,6 +190,10 @@ class VastdbUploader(SQLUploader):
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")
 
+    @requires_dependencies(["pandas"], extras="vastdb")
+    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        super().run(path=path, file_data=file_data, **kwargs)
+
     @requires_dependencies(["pyarrow", "pandas"], extras="vastdb")
     def upload_dataframe(self, df: "DataFrame", file_data: FileData) -> None:
         import numpy as np

unstructured_ingest/processes/connectors/vectara.py

@@ -108,7 +108,6 @@ class VectaraUploaderConfig(UploaderConfig):
 
 @dataclass
 class VectaraUploader(Uploader):
-
     connector_type: str = CONNECTOR_TYPE
     upload_config: VectaraUploaderConfig
     connection_config: VectaraConnectionConfig
@@ -336,7 +335,6 @@ class VectaraUploader(Uploader):
         file_data: FileData,
         **kwargs: Any,
     ) -> None:
-
         logger.info(f"inserting / updating {len(data)} documents to Vectara ")
         await asyncio.gather(*(self._index_document(vdoc) for vdoc in data))
 

unstructured_ingest/processes/connectors/zendesk/zendesk.py

@@ -53,7 +53,6 @@ class ZendeskConnectionConfig(ConnectionConfig):
     access_config: Secret[ZendeskAccessConfig]
 
     def get_client(self) -> ZendeskClient:
-
         access_config = self.access_config.get_secret_value()
 
         return ZendeskClient(
@@ -206,7 +205,6 @@ class ZendeskDownloader(Downloader):
                     await f.write(comment.as_text())
 
     async def run_async(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
-
         zendesk_filedata = ZendeskFileData.cast(file_data=file_data)
 
         item_type = zendesk_filedata.additional_metadata.item_type

unstructured_ingest/processes/embedder.py

@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, Literal, Optional
 from pydantic import BaseModel, Field, SecretStr
 
 from unstructured_ingest.interfaces.process import BaseProcess
-from unstructured_ingest.utils.data_prep import get_data
+from unstructured_ingest.utils.data_prep import get_json_data
 
 if TYPE_CHECKING:
     from unstructured_ingest.embed.interfaces import BaseEmbeddingEncoder
@@ -192,7 +192,7 @@ class Embedder(BaseProcess, ABC):
     def run(self, elements_filepath: Path, **kwargs: Any) -> list[dict]:
         # TODO update base embedder classes to support async
         embedder = self.config.get_embedder()
-        elements = get_data(path=elements_filepath)
+        elements = get_json_data(path=elements_filepath)
         if not elements:
             return []
         embedded_elements = embedder.embed_documents(elements=elements)

unstructured_ingest/processes/filter.py

@@ -13,7 +13,7 @@ from unstructured_ingest.logger import logger
 class FiltererConfig(BaseModel):
     file_glob: Optional[list[str]] = Field(
         default=None,
-        description="file globs to limit which data_types of " "files are accepted",
+        description="file globs to limit which data_types of files are accepted",
         examples=["*.pdf", "*.html"],
     )
     max_file_size: Optional[int] = Field(

unstructured_ingest/processes/partitioner.py

@@ -68,6 +68,9 @@ class PartitionerConfig(BaseModel):
         description="Use a remote API to partition the files."
         " Otherwise, use the function from partition.auto",
     )
+    api_timeout_ms: Optional[int] = Field(
+        default=None, description="Timeout in milliseconds for all api call during partitioning."
+    )
     api_key: Optional[SecretStr] = Field(
         default=None, description="API Key for partition endpoint."
     )
@@ -188,6 +191,7 @@ class Partitioner(BaseProcess, ABC):
             api_key=self.config.api_key.get_secret_value(),
             filename=filename,
             api_parameters=self.config.to_partition_kwargs(),
+            timeout_ms=self.config.api_timeout_ms,
         )
 
         # Append the data source metadata the auto partition does for you

unstructured_ingest/processes/utils/blob_storage.py

@@ -4,7 +4,7 @@ from typing import Any
 
 from unstructured_ingest.data_types.file_data import FileData
 from unstructured_ingest.interfaces import UploadStager, UploadStagerConfig
-from unstructured_ingest.utils.data_prep import get_data, write_data
+from unstructured_ingest.utils.data_prep import get_json_data, write_data
 
 
 class BlobStoreUploadStagerConfig(UploadStagerConfig):
@@ -27,6 +27,6 @@ class BlobStoreUploadStager(UploadStager):
     ) -> Path:
         output_file = self.get_output_path(output_filename=output_filename, output_dir=output_dir)
         # Always save as json
-        data = get_data(elements_filepath)
+        data = get_json_data(elements_filepath)
         write_data(path=output_file.with_suffix(".json"), data=data)
         return output_file.with_suffix(".json")

unstructured_ingest/unstructured_api.py

@@ -80,7 +80,11 @@ def wrap_error(e: Exception) -> Exception:
 
 
 async def call_api_async(
-    server_url: Optional[str], api_key: Optional[str], filename: Path, api_parameters: dict
+    server_url: Optional[str],
+    api_key: Optional[str],
+    filename: Path,
+    api_parameters: dict,
+    timeout_ms: Optional[int] = None,
 ) -> list[dict]:
     """Call the Unstructured API using unstructured-client.
 
@@ -94,13 +98,10 @@ async def call_api_async(
     """
     from unstructured_client import UnstructuredClient
 
-    client = UnstructuredClient(
-        server_url=server_url,
-        api_key_auth=api_key,
-    )
+    client = UnstructuredClient(server_url=server_url, api_key_auth=api_key)
     partition_request = create_partition_request(filename=filename, parameters_dict=api_parameters)
     try:
-        res = await client.general.partition_async(request=partition_request)
+        res = await client.general.partition_async(request=partition_request, timeout_ms=timeout_ms)
     except Exception as e:
         raise wrap_error(e)
 
@@ -108,7 +109,11 @@ async def call_api_async(
 
 
 def call_api(
-    server_url: Optional[str], api_key: Optional[str], filename: Path, api_parameters: dict
+    server_url: Optional[str],
+    api_key: Optional[str],
+    filename: Path,
+    api_parameters: dict,
+    timeout_ms: Optional[int] = None,
 ) -> list[dict]:
     """Call the Unstructured API using unstructured-client.
 
@@ -128,7 +133,7 @@ def call_api(
     )
     partition_request = create_partition_request(filename=filename, parameters_dict=api_parameters)
     try:
-        res = client.general.partition(request=partition_request)
+        res = client.general.partition(request=partition_request, timeout_ms=timeout_ms)
     except Exception as e:
         raise wrap_error(e)
 

unstructured_ingest/utils/data_prep.py

@@ -2,7 +2,7 @@ import itertools
 import json
 from datetime import datetime
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Generator, Iterable, Optional, Sequence, TypeVar, Union, cast
+from typing import TYPE_CHECKING, Any, Generator, Iterable, Optional, Sequence, TypeVar, cast
 from uuid import NAMESPACE_DNS, uuid5
 
 from unstructured_ingest.data_types.file_data import FileData
@@ -171,15 +171,13 @@ def write_data(path: Path, data: list[dict], indent: Optional[int] = 2) -> None:
             raise IOError("Unsupported file type: {path}")
 
 
-def get_data(path: Union[Path, str]) -> list[dict]:
-    if isinstance(path, str):
-        path = Path(path)
-    try:
-        return get_data_by_suffix(path=path)
-    except Exception as e:
-        logger.warning(f"failed to read {path} by extension: {e}")
-    # Fall back
+def get_json_data(path: Path) -> list[dict]:
     with path.open() as f:
+        # Attempt by prefix
+        if path.suffix == ".json":
+            return json.load(f)
+        elif path.suffix == ".ndjson":
+            return ndjson.load(f)
         try:
             return json.load(f)
         except Exception as e:
@@ -188,29 +186,7 @@ def get_data(path: Union[Path, str]) -> list[dict]:
             return ndjson.load(f)
         except Exception as e:
             logger.warning(f"failed to read {path} as ndjson: {e}")
-
-        import pandas as pd
-
-        try:
-            df = pd.read_csv(path)
-            return df.to_dict(orient="records")
-        except Exception as e:
-            logger.warning(f"failed to read {path} as csv: {e}")
-        try:
-            df = pd.read_parquet(path)
-            return df.to_dict(orient="records")
-        except Exception as e:
-            logger.warning(f"failed to read {path} as parquet: {e}")
-
-
-def get_json_data(path: Path) -> list[dict]:
-    with path.open() as f:
-        if path.suffix == ".json":
-            return json.load(f)
-        elif path.suffix == ".ndjson":
-            return ndjson.load(f)
-        else:
-            raise ValueError(f"Unsupported file type: {path}")
+    raise ValueError(f"Unsupported json file: {path}")
 
 
 @requires_dependencies(["pandas"])

unstructured_ingest-1.0.1.dist-info/METADATA

@@ -0,0 +1,226 @@
+Metadata-Version: 2.4
+Name: unstructured_ingest
+Version: 1.0.1
+Summary: Local ETL data pipeline to get data RAG ready
+Author-email: Unstructured Technologies <devops@unstructuredai.io>
+License-Expression: Apache-2.0
+License-File: LICENSE.md
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: <3.13,>=3.9
+Requires-Dist: click
+Requires-Dist: dataclasses-json
+Requires-Dist: opentelemetry-sdk
+Requires-Dist: pydantic>=2.7
+Requires-Dist: python-dateutil
+Requires-Dist: tqdm
+Provides-Extra: airtable
+Requires-Dist: pandas; extra == 'airtable'
+Requires-Dist: pyairtable; extra == 'airtable'
+Provides-Extra: astradb
+Requires-Dist: astrapy; extra == 'astradb'
+Provides-Extra: azure
+Requires-Dist: adlfs; extra == 'azure'
+Requires-Dist: fsspec; extra == 'azure'
+Provides-Extra: azure-ai-search
+Requires-Dist: azure-search-documents; extra == 'azure-ai-search'
+Provides-Extra: bedrock
+Requires-Dist: aioboto3; extra == 'bedrock'
+Requires-Dist: boto3; extra == 'bedrock'
+Provides-Extra: biomed
+Requires-Dist: bs4; extra == 'biomed'
+Requires-Dist: requests; extra == 'biomed'
+Provides-Extra: box
+Requires-Dist: boxfs; extra == 'box'
+Requires-Dist: fsspec; extra == 'box'
+Provides-Extra: chroma
+Requires-Dist: chromadb; extra == 'chroma'
+Provides-Extra: clarifai
+Requires-Dist: clarifai; extra == 'clarifai'
+Provides-Extra: confluence
+Requires-Dist: atlassian-python-api; extra == 'confluence'
+Requires-Dist: requests; extra == 'confluence'
+Provides-Extra: couchbase
+Requires-Dist: couchbase; extra == 'couchbase'
+Provides-Extra: databricks-delta-tables
+Requires-Dist: databricks-sql-connector; extra == 'databricks-delta-tables'
+Requires-Dist: pandas; extra == 'databricks-delta-tables'
+Provides-Extra: databricks-volumes
+Requires-Dist: databricks-sdk; extra == 'databricks-volumes'
+Provides-Extra: delta-table
+Requires-Dist: boto3; extra == 'delta-table'
+Requires-Dist: deltalake; extra == 'delta-table'
+Requires-Dist: pandas; extra == 'delta-table'
+Provides-Extra: discord
+Requires-Dist: discord-py; extra == 'discord'
+Provides-Extra: doc
+Requires-Dist: unstructured[doc]; extra == 'doc'
+Provides-Extra: docx
+Requires-Dist: unstructured[docx]; extra == 'docx'
+Provides-Extra: dropbox
+Requires-Dist: dropboxdrivefs; extra == 'dropbox'
+Requires-Dist: fsspec; extra == 'dropbox'
+Provides-Extra: duckdb
+Requires-Dist: duckdb; extra == 'duckdb'
+Requires-Dist: pandas; extra == 'duckdb'
+Provides-Extra: elasticsearch
+Requires-Dist: elasticsearch[async]; extra == 'elasticsearch'
+Provides-Extra: epub
+Requires-Dist: unstructured[epub]; extra == 'epub'
+Provides-Extra: gcs
+Requires-Dist: bs4; extra == 'gcs'
+Requires-Dist: fsspec; extra == 'gcs'
+Requires-Dist: gcsfs; extra == 'gcs'
+Provides-Extra: github
+Requires-Dist: pygithub>1.58.0; extra == 'github'
+Requires-Dist: requests; extra == 'github'
+Provides-Extra: gitlab
+Requires-Dist: python-gitlab; extra == 'gitlab'
+Provides-Extra: google-drive
+Requires-Dist: google-api-python-client; extra == 'google-drive'
+Provides-Extra: hubspot
+Requires-Dist: hubspot-api-client; extra == 'hubspot'
+Requires-Dist: urllib3; extra == 'hubspot'
+Provides-Extra: huggingface
+Requires-Dist: sentence-transformers; extra == 'huggingface'
+Provides-Extra: ibm-watsonx-s3
+Requires-Dist: httpx; extra == 'ibm-watsonx-s3'
+Requires-Dist: pandas; extra == 'ibm-watsonx-s3'
+Requires-Dist: pyarrow; extra == 'ibm-watsonx-s3'
+Requires-Dist: pyiceberg; extra == 'ibm-watsonx-s3'
+Requires-Dist: tenacity; extra == 'ibm-watsonx-s3'
+Provides-Extra: image
+Requires-Dist: unstructured[image]; extra == 'image'
+Provides-Extra: jira
+Requires-Dist: atlassian-python-api; extra == 'jira'
+Provides-Extra: kafka
+Requires-Dist: confluent-kafka; extra == 'kafka'
+Provides-Extra: kdbai
+Requires-Dist: kdbai-client>=1.4.0; extra == 'kdbai'
+Requires-Dist: pandas; extra == 'kdbai'
+Provides-Extra: lancedb
+Requires-Dist: lancedb; extra == 'lancedb'
+Provides-Extra: md
+Requires-Dist: unstructured[md]; extra == 'md'
+Provides-Extra: milvus
+Requires-Dist: pymilvus; extra == 'milvus'
+Provides-Extra: mixedbreadai
+Requires-Dist: mixedbread-ai; extra == 'mixedbreadai'
+Provides-Extra: mongodb
+Requires-Dist: pymongo; extra == 'mongodb'
+Provides-Extra: msg
+Requires-Dist: unstructured[msg]; extra == 'msg'
+Provides-Extra: neo4j
+Requires-Dist: cymple; extra == 'neo4j'
+Requires-Dist: neo4j-rust-ext; extra == 'neo4j'
+Requires-Dist: networkx; extra == 'neo4j'
+Provides-Extra: notion
+Requires-Dist: backoff; extra == 'notion'
+Requires-Dist: htmlbuilder; extra == 'notion'
+Requires-Dist: httpx; extra == 'notion'
+Requires-Dist: notion-client; extra == 'notion'
+Provides-Extra: octoai
+Requires-Dist: openai; extra == 'octoai'
+Requires-Dist: tiktoken; extra == 'octoai'
+Provides-Extra: odt
+Requires-Dist: unstructured[odt]; extra == 'odt'
+Provides-Extra: onedrive
+Requires-Dist: msal; extra == 'onedrive'
+Requires-Dist: office365-rest-python-client; extra == 'onedrive'
+Requires-Dist: requests; extra == 'onedrive'
+Provides-Extra: openai
+Requires-Dist: openai; extra == 'openai'
+Requires-Dist: tiktoken; extra == 'openai'
+Provides-Extra: opensearch
+Requires-Dist: opensearch-py; extra == 'opensearch'
+Provides-Extra: org
+Requires-Dist: unstructured[org]; extra == 'org'
+Provides-Extra: outlook
+Requires-Dist: msal; extra == 'outlook'
+Requires-Dist: office365-rest-python-client; extra == 'outlook'
+Provides-Extra: pdf
+Requires-Dist: unstructured[pdf]; extra == 'pdf'
+Provides-Extra: pinecone
+Requires-Dist: pinecone; extra == 'pinecone'
+Provides-Extra: postgres
+Requires-Dist: pandas; extra == 'postgres'
+Requires-Dist: psycopg2-binary; extra == 'postgres'
+Provides-Extra: ppt
+Requires-Dist: unstructured[ppt]; extra == 'ppt'
+Provides-Extra: pptx
+Requires-Dist: unstructured[pptx]; extra == 'pptx'
+Provides-Extra: qdrant
+Requires-Dist: qdrant-client; extra == 'qdrant'
+Provides-Extra: reddit
+Requires-Dist: praw; extra == 'reddit'
+Provides-Extra: redis
+Requires-Dist: redis; extra == 'redis'
+Provides-Extra: remote
+Requires-Dist: unstructured-client>=0.30.0; extra == 'remote'
+Provides-Extra: rst
+Requires-Dist: unstructured[rst]; extra == 'rst'
+Provides-Extra: rtf
+Requires-Dist: unstructured[rtf]; extra == 'rtf'
+Provides-Extra: s3
+Requires-Dist: fsspec; extra == 's3'
+Requires-Dist: s3fs; extra == 's3'
+Provides-Extra: salesforce
+Requires-Dist: simple-salesforce; extra == 'salesforce'
+Provides-Extra: sftp
+Requires-Dist: fsspec; extra == 'sftp'
+Requires-Dist: paramiko; extra == 'sftp'
+Provides-Extra: sharepoint
+Requires-Dist: msal; extra == 'sharepoint'
+Requires-Dist: office365-rest-python-client; extra == 'sharepoint'
+Requires-Dist: requests; extra == 'sharepoint'
+Provides-Extra: singlestore
+Requires-Dist: pandas; extra == 'singlestore'
+Requires-Dist: singlestoredb; extra == 'singlestore'
+Provides-Extra: slack
+Requires-Dist: slack-sdk[optional]; extra == 'slack'
+Provides-Extra: snowflake
+Requires-Dist: pandas; extra == 'snowflake'
+Requires-Dist: psycopg2-binary; extra == 'snowflake'
+Requires-Dist: snowflake-connector-python; extra == 'snowflake'
+Provides-Extra: togetherai
+Requires-Dist: together; extra == 'togetherai'
+Provides-Extra: tsv
+Requires-Dist: unstructured[tsv]; extra == 'tsv'
+Provides-Extra: vastdb
+Requires-Dist: ibis; extra == 'vastdb'
+Requires-Dist: pandas; extra == 'vastdb'
+Requires-Dist: pyarrow; extra == 'vastdb'
+Requires-Dist: vastdb; extra == 'vastdb'
+Provides-Extra: vectara
+Requires-Dist: aiofiles; extra == 'vectara'
+Requires-Dist: httpx; extra == 'vectara'
+Requires-Dist: requests; extra == 'vectara'
+Provides-Extra: vertexai
+Requires-Dist: vertexai; extra == 'vertexai'
+Provides-Extra: voyageai
+Requires-Dist: voyageai; extra == 'voyageai'
+Provides-Extra: weaviate
+Requires-Dist: weaviate-client; extra == 'weaviate'
+Provides-Extra: wikipedia
+Requires-Dist: wikipedia; extra == 'wikipedia'
+Provides-Extra: xlsx
+Requires-Dist: unstructured[xlsx]; extra == 'xlsx'
+Provides-Extra: zendesk
+Requires-Dist: aiofiles; extra == 'zendesk'
+Requires-Dist: bs4; extra == 'zendesk'
+Requires-Dist: httpx; extra == 'zendesk'
+Description-Content-Type: text/markdown
+
+# Unstructured Ingest  
+
+For details, see the [Unstructured Ingest overview](https://docs.unstructured.io/ingestion/overview) in the Unstructured documentation.