unstructured-ingest 0.0.2__tar.gz → 0.0.3__tar.gz

unstructured-ingest-0.0.3/PKG-INFO

@@ -0,0 +1,87 @@
+Metadata-Version: 2.1
+Name: unstructured-ingest
+Version: 0.0.3
+Summary: A library that prepares raw documents for downstream ML tasks.
+Home-page: https://github.com/Unstructured-IO/unstructured-ingest
+Author: Unstructured Technologies
+Author-email: devops@unstructuredai.io
+License: Apache-2.0
+Keywords: NLP PDF HTML CV XML parsing preprocessing
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9.0,<3.13
+Description-Content-Type: text/markdown
+Provides-Extra: csv
+Provides-Extra: doc
+Provides-Extra: docx
+Provides-Extra: epub
+Provides-Extra: md
+Provides-Extra: msg
+Provides-Extra: odt
+Provides-Extra: org
+Provides-Extra: pdf
+Provides-Extra: ppt
+Provides-Extra: pptx
+Provides-Extra: rtf
+Provides-Extra: rst
+Provides-Extra: tsv
+Provides-Extra: xlsx
+Provides-Extra: airtable
+Provides-Extra: astra
+Provides-Extra: azure
+Provides-Extra: azure-cognitive-search
+Provides-Extra: biomed
+Provides-Extra: box
+Provides-Extra: chroma
+Provides-Extra: clarifai
+Provides-Extra: confluence
+Provides-Extra: delta-table
+Provides-Extra: discord
+Provides-Extra: dropbox
+Provides-Extra: elasticsearch
+Provides-Extra: gcs
+Provides-Extra: github
+Provides-Extra: gitlab
+Provides-Extra: google-drive
+Provides-Extra: hubspot
+Provides-Extra: jira
+Provides-Extra: kafka
+Provides-Extra: milvus
+Provides-Extra: mongodb
+Provides-Extra: notion
+Provides-Extra: onedrive
+Provides-Extra: opensearch
+Provides-Extra: outlook
+Provides-Extra: pinecone
+Provides-Extra: postgres
+Provides-Extra: qdrant
+Provides-Extra: reddit
+Provides-Extra: s3
+Provides-Extra: sharepoint
+Provides-Extra: salesforce
+Provides-Extra: sftp
+Provides-Extra: slack
+Provides-Extra: wikipedia
+Provides-Extra: weaviate
+Provides-Extra: databricks-volumes
+Provides-Extra: singlestore
+Provides-Extra: embed-huggingface
+Provides-Extra: embed-octoai
+Provides-Extra: embed-vertexai
+Provides-Extra: embed-voyageai
+Provides-Extra: openai
+Provides-Extra: bedrock
+
+# Unstructured Ingest  
+
+For details, see the [Unstructured Ingest overview](https://docs.unstructured.io/ingestion/overview) in the Unstructured documentation.

unstructured-ingest-0.0.3/README.md

@@ -0,0 +1,3 @@
+# Unstructured Ingest  
+
+For details, see the [Unstructured Ingest overview](https://docs.unstructured.io/ingestion/overview) in the Unstructured documentation.

unstructured-ingest-0.0.3/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.0.3"  # pragma: no cover

{unstructured-ingest-0.0.2 → unstructured-ingest-0.0.3}/unstructured_ingest/v2/interfaces/process.py

@@ -17,7 +17,3 @@ class BaseProcess(ABC):
 
     async def run_async(self, **kwargs: Any) -> Any:
         return self.run(**kwargs)
-
-    def check_connection(self):
-        # If the process requires external connections, run a quick check
-        pass

{unstructured-ingest-0.0.2 → unstructured-ingest-0.0.3}/unstructured_ingest/v2/processes/connectors/fsspec/azure.py

@@ -75,6 +75,10 @@ class AzureIndexer(FsspecIndexer):
     index_config: AzureIndexerConfig
     connector_type: str = CONNECTOR_TYPE
 
+    @requires_dependencies(["adlfs", "fsspec"], extras="azure")
+    def precheck(self) -> None:
+        super().precheck()
+
     def sterilize_info(self, path) -> dict:
         info = self.fs.info(path=path)
         return sterilize_dict(data=info, default=azure_json_serial)
@@ -120,6 +124,10 @@ class AzureUploader(FsspecUploader):
     def __post_init__(self):
         super().__post_init__()
 
+    @requires_dependencies(["adlfs", "fsspec"], extras="azure")
+    def precheck(self) -> None:
+        super().precheck()
+
     @requires_dependencies(["adlfs", "fsspec"], extras="azure")
     def run(self, contents: list[UploadContent], **kwargs: Any) -> None:
         return super().run(contents=contents, **kwargs)

{unstructured-ingest-0.0.2 → unstructured-ingest-0.0.3}/unstructured_ingest/v2/processes/connectors/fsspec/box.py

@@ -70,6 +70,10 @@ class BoxIndexer(FsspecIndexer):
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
         return super().run(**kwargs)
 
+    @requires_dependencies(["boxfs"], extras="box")
+    def precheck(self) -> None:
+        super().precheck()
+
 
 @dataclass
 class BoxDownloaderConfig(FsspecDownloaderConfig):
@@ -107,6 +111,10 @@ class BoxUploader(FsspecUploader):
     def __post_init__(self):
         super().__post_init__()
 
+    @requires_dependencies(["boxfs"], extras="box")
+    def precheck(self) -> None:
+        super().precheck()
+
     @requires_dependencies(["boxfs"], extras="box")
     def run(self, contents: list[UploadContent], **kwargs: Any) -> None:
         return super().run(contents=contents, **kwargs)

{unstructured-ingest-0.0.2 → unstructured-ingest-0.0.3}/unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py

@@ -57,6 +57,10 @@ class DropboxIndexer(FsspecIndexer):
         if not self.index_config.path_without_protocol.startswith("/"):
             self.index_config.path_without_protocol = "/" + self.index_config.path_without_protocol
 
+    @requires_dependencies(["dropboxdrivefs", "fsspec"], extras="dropbox")
+    def precheck(self) -> None:
+        super().precheck()
+
     @requires_dependencies(["dropboxdrivefs", "fsspec"], extras="dropbox")
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
         return super().run(**kwargs)
@@ -106,6 +110,10 @@ class DropboxUploader(FsspecUploader):
     def __post_init__(self):
         super().__post_init__()
 
+    @requires_dependencies(["dropboxdrivefs", "fsspec"], extras="dropbox")
+    def precheck(self) -> None:
+        super().precheck()
+
     @requires_dependencies(["dropboxdrivefs", "fsspec"], extras="dropbox")
     def run(self, contents: list[UploadContent], **kwargs: Any) -> None:
         return super().run(contents=contents, **kwargs)

{unstructured-ingest-0.0.2 → unstructured-ingest-0.0.3}/unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py

@@ -9,7 +9,11 @@ from typing import TYPE_CHECKING, Any, Generator, Optional, TypeVar
 from uuid import NAMESPACE_DNS, uuid5
 
 from unstructured_ingest.enhanced_dataclass import enhanced_field
-from unstructured_ingest.error import SourceConnectionError, SourceConnectionNetworkError
+from unstructured_ingest.error import (
+    DestinationConnectionError,
+    SourceConnectionError,
+    SourceConnectionNetworkError,
+)
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
@@ -300,6 +304,19 @@ class FsspecUploader(Uploader):
                 f"missing 1 required positional argument: 'upload_config'"
             )
 
+    def precheck(self) -> None:
+        from fsspec import get_filesystem_class
+
+        try:
+            fs = get_filesystem_class(self.upload_config.protocol)(
+                **self.connection_config.get_access_config(),
+            )
+            root_dir = self.upload_config.path_without_protocol.split("/")[0]
+            fs.ls(path=root_dir, detail=False)
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
+
     def get_upload_path(self, file_data: FileData) -> Path:
         upload_path = (
             Path(self.upload_config.path_without_protocol)

{unstructured-ingest-0.0.2 → unstructured-ingest-0.0.3}/unstructured_ingest/v2/processes/connectors/fsspec/gcs.py

@@ -80,6 +80,10 @@ class GcsIndexer(FsspecIndexer):
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
         return super().run(**kwargs)
 
+    @requires_dependencies(["gcsfs", "fsspec"], extras="gcs")
+    def precheck(self) -> None:
+        super().precheck()
+
 
 @dataclass
 class GcsDownloaderConfig(FsspecDownloaderConfig):
@@ -117,6 +121,10 @@ class GcsUploader(FsspecUploader):
     def __post_init__(self):
         super().__post_init__()
 
+    @requires_dependencies(["gcsfs", "fsspec"], extras="gcs")
+    def precheck(self) -> None:
+        super().precheck()
+
     @requires_dependencies(["gcsfs", "fsspec"], extras="gcs")
     def run(self, contents: list[UploadContent], **kwargs: Any) -> None:
         return super().run(contents=contents, **kwargs)

{unstructured-ingest-0.0.2 → unstructured-ingest-0.0.3}/unstructured_ingest/v2/processes/connectors/fsspec/s3.py

@@ -111,6 +111,10 @@ class S3Indexer(FsspecIndexer):
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
         return super().run(**kwargs)
 
+    @requires_dependencies(["s3fs", "fsspec"], extras="s3")
+    def precheck(self) -> None:
+        super().precheck()
+
 
 @dataclass
 class S3DownloaderConfig(FsspecDownloaderConfig):
@@ -144,6 +148,10 @@ class S3Uploader(FsspecUploader):
     connection_config: S3ConnectionConfig
     upload_config: S3UploaderConfig = field(default=None)
 
+    @requires_dependencies(["s3fs", "fsspec"], extras="s3")
+    def precheck(self) -> None:
+        super().precheck()
+
     @requires_dependencies(["s3fs", "fsspec"], extras="s3")
     def __post_init__(self):
         super().__post_init__()

{unstructured-ingest-0.0.2 → unstructured-ingest-0.0.3}/unstructured_ingest/v2/processes/connectors/fsspec/sftp.py

@@ -91,6 +91,10 @@ class SftpIndexer(FsspecIndexer):
             file.identifier = new_identifier
             yield file
 
+    @requires_dependencies(["paramiko", "fsspec"], extras="sftp")
+    def precheck(self) -> None:
+        super().precheck()
+
 
 @dataclass
 class SftpDownloaderConfig(FsspecDownloaderConfig):
@@ -142,6 +146,10 @@ class SftpUploader(FsspecUploader):
     def __post_init__(self):
         super().__post_init__()
 
+    @requires_dependencies(["paramiko", "fsspec"], extras="sftp")
+    def precheck(self) -> None:
+        super().precheck()
+
     @requires_dependencies(["paramiko", "fsspec"], extras="sftp")
     def run(self, contents: list[UploadContent], **kwargs: Any) -> None:
         return super().run(contents=contents, **kwargs)

{unstructured-ingest-0.0.2 → unstructured-ingest-0.0.3}/unstructured_ingest/v2/processes/connectors/opensearch.py

@@ -100,9 +100,15 @@ class OpenSearchConnectionConfig(ConnectionConfig):
         return OpenSearch(**self.get_client_kwargs())
 
 
+@dataclass
+class OpensearchIndexerConfig(ElasticsearchIndexerConfig):
+    pass
+
+
 @dataclass
 class OpenSearchIndexer(ElasticsearchIndexer):
     connection_config: OpenSearchConnectionConfig
+    index_config: OpensearchIndexerConfig
     client: "OpenSearch" = field(init=False)
 
     @requires_dependencies(["opensearchpy"], extras="opensearch")
@@ -112,9 +118,15 @@ class OpenSearchIndexer(ElasticsearchIndexer):
         return scan
 
 
+@dataclass
+class OpensearchDownloaderConfig(ElasticsearchDownloaderConfig):
+    pass
+
+
 @dataclass
 class OpenSearchDownloader(ElasticsearchDownloader):
     connection_config: OpenSearchConnectionConfig
+    download_config: OpensearchDownloaderConfig
     connector_type: str = CONNECTOR_TYPE
 
     @requires_dependencies(["opensearchpy"], extras="opensearch")
@@ -125,9 +137,15 @@ class OpenSearchDownloader(ElasticsearchDownloader):
         return AsyncOpenSearch, async_scan
 
 
+@dataclass
+class OpensearchUploaderConfig(ElasticsearchUploaderConfig):
+    pass
+
+
 @dataclass
 class OpenSearchUploader(ElasticsearchUploader):
     connection_config: OpenSearchConnectionConfig
+    upload_config: OpensearchUploaderConfig
     connector_type: str = CONNECTOR_TYPE
 
     @requires_dependencies(["opensearchpy"], extras="opensearch")
@@ -137,19 +155,29 @@ class OpenSearchUploader(ElasticsearchUploader):
         return parallel_bulk
 
 
+@dataclass
+class OpensearchUploadStagerConfig(ElasticsearchUploadStagerConfig):
+    pass
+
+
+@dataclass
+class OpensearchUploadStager(ElasticsearchUploadStager):
+    upload_stager_config: OpensearchUploadStagerConfig
+
+
 opensearch_source_entry = SourceRegistryEntry(
     connection_config=OpenSearchConnectionConfig,
     indexer=OpenSearchIndexer,
-    indexer_config=ElasticsearchIndexerConfig,
+    indexer_config=OpensearchIndexerConfig,
     downloader=OpenSearchDownloader,
-    downloader_config=ElasticsearchDownloaderConfig,
+    downloader_config=OpensearchDownloaderConfig,
 )
 
 
 opensearch_destination_entry = DestinationRegistryEntry(
     connection_config=OpenSearchConnectionConfig,
-    upload_stager_config=ElasticsearchUploadStagerConfig,
-    upload_stager=ElasticsearchUploadStager,
-    uploader_config=ElasticsearchUploaderConfig,
+    upload_stager_config=OpensearchUploadStagerConfig,
+    upload_stager=OpensearchUploadStager,
+    uploader_config=OpensearchUploaderConfig,
     uploader=OpenSearchUploader,
 )

unstructured-ingest-0.0.3/unstructured_ingest.egg-info/PKG-INFO

@@ -0,0 +1,87 @@
+Metadata-Version: 2.1
+Name: unstructured-ingest
+Version: 0.0.3
+Summary: A library that prepares raw documents for downstream ML tasks.
+Home-page: https://github.com/Unstructured-IO/unstructured-ingest
+Author: Unstructured Technologies
+Author-email: devops@unstructuredai.io
+License: Apache-2.0
+Keywords: NLP PDF HTML CV XML parsing preprocessing
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9.0,<3.13
+Description-Content-Type: text/markdown
+Provides-Extra: csv
+Provides-Extra: doc
+Provides-Extra: docx
+Provides-Extra: epub
+Provides-Extra: md
+Provides-Extra: msg
+Provides-Extra: odt
+Provides-Extra: org
+Provides-Extra: pdf
+Provides-Extra: ppt
+Provides-Extra: pptx
+Provides-Extra: rtf
+Provides-Extra: rst
+Provides-Extra: tsv
+Provides-Extra: xlsx
+Provides-Extra: airtable
+Provides-Extra: astra
+Provides-Extra: azure
+Provides-Extra: azure-cognitive-search
+Provides-Extra: biomed
+Provides-Extra: box
+Provides-Extra: chroma
+Provides-Extra: clarifai
+Provides-Extra: confluence
+Provides-Extra: delta-table
+Provides-Extra: discord
+Provides-Extra: dropbox
+Provides-Extra: elasticsearch
+Provides-Extra: gcs
+Provides-Extra: github
+Provides-Extra: gitlab
+Provides-Extra: google-drive
+Provides-Extra: hubspot
+Provides-Extra: jira
+Provides-Extra: kafka
+Provides-Extra: milvus
+Provides-Extra: mongodb
+Provides-Extra: notion
+Provides-Extra: onedrive
+Provides-Extra: opensearch
+Provides-Extra: outlook
+Provides-Extra: pinecone
+Provides-Extra: postgres
+Provides-Extra: qdrant
+Provides-Extra: reddit
+Provides-Extra: s3
+Provides-Extra: sharepoint
+Provides-Extra: salesforce
+Provides-Extra: sftp
+Provides-Extra: slack
+Provides-Extra: wikipedia
+Provides-Extra: weaviate
+Provides-Extra: databricks-volumes
+Provides-Extra: singlestore
+Provides-Extra: embed-huggingface
+Provides-Extra: embed-octoai
+Provides-Extra: embed-vertexai
+Provides-Extra: embed-voyageai
+Provides-Extra: openai
+Provides-Extra: bedrock
+
+# Unstructured Ingest  
+
+For details, see the [Unstructured Ingest overview](https://docs.unstructured.io/ingestion/overview) in the Unstructured documentation.