unstructured-ingest 0.3.1__py3-none-any.whl → 0.3.3__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/lancedb/cloud.py

@@ -0,0 +1,42 @@
+from dataclasses import dataclass
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.v2.interfaces.connector import AccessConfig
+from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.processes.connectors.lancedb.lancedb import (
+    LanceDBRemoteConnectionConfig,
+    LanceDBUploader,
+    LanceDBUploaderConfig,
+    LanceDBUploadStager,
+    LanceDBUploadStagerConfig,
+)
+
+CONNECTOR_TYPE = "lancedb_cloud"
+
+
+class LanceDBCloudAccessConfig(AccessConfig):
+    api_key: str = Field(description="Api key associated with LanceDb cloud")
+
+
+class LanceDBCloudConnectionConfig(LanceDBRemoteConnectionConfig):
+    access_config: Secret[LanceDBCloudAccessConfig]
+
+    def get_storage_options(self) -> dict:
+        return {**self.access_config.get_secret_value().model_dump(), "timeout": self.timeout}
+
+
+@dataclass
+class LanceDBCloudUploader(LanceDBUploader):
+    upload_config: LanceDBUploaderConfig
+    connection_config: LanceDBCloudConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+lancedb_cloud_destination_entry = DestinationRegistryEntry(
+    connection_config=LanceDBCloudConnectionConfig,
+    uploader=LanceDBCloudUploader,
+    uploader_config=LanceDBUploaderConfig,
+    upload_stager_config=LanceDBUploadStagerConfig,
+    upload_stager=LanceDBUploadStager,
+)

unstructured_ingest/v2/processes/connectors/milvus.py

@@ -8,7 +8,7 @@ import pandas as pd
 from dateutil import parser
 from pydantic import Field, Secret
 
-from unstructured_ingest.error import WriteError
+from unstructured_ingest.error import DestinationConnectionError, WriteError
 from unstructured_ingest.utils.data_prep import flatten_dict
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.constants import RECORD_ID_LABEL
@@ -66,7 +66,6 @@ class MilvusConnectionConfig(ConnectionConfig):
 
 
 class MilvusUploadStagerConfig(UploadStagerConfig):
-
     fields_to_include: Optional[list[str]] = None
     """If set - list of fields to include in the output.
     Unspecified fields are removed from the elements.
@@ -174,6 +173,14 @@ class MilvusUploader(Uploader):
     upload_config: MilvusUploaderConfig
     connector_type: str = CONNECTOR_TYPE
 
+    @DestinationConnectionError.wrap
+    def precheck(self):
+        with self.get_client() as client:
+            if not client.has_collection(self.upload_config.collection_name):
+                raise DestinationConnectionError(
+                    f"Collection '{self.upload_config.collection_name}' does not exist"
+                )
+
     @contextmanager
     def get_client(self) -> Generator["MilvusClient", None, None]:
         client = self.connection_config.get_client()
@@ -218,7 +225,6 @@ class MilvusUploader(Uploader):
             f"db in collection {self.upload_config.collection_name}"
         )
         with self.get_client() as client:
-
             try:
                 res = client.insert(collection_name=self.upload_config.collection_name, data=data)
             except MilvusException as milvus_exception:

unstructured_ingest/v2/processes/connectors/mongodb.py

@@ -1,6 +1,7 @@
 import json
 import sys
-from dataclasses import dataclass, field
+from contextlib import contextmanager
+from dataclasses import dataclass, replace
 from datetime import datetime
 from pathlib import Path
 from time import time
@@ -12,6 +13,7 @@ from unstructured_ingest.__version__ import __version__ as unstructured_version
 from unstructured_ingest.error import DestinationConnectionError, SourceConnectionError
 from unstructured_ingest.utils.data_prep import batch_generator, flatten_dict
 from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
@@ -24,8 +26,6 @@ from unstructured_ingest.v2.interfaces import (
     SourceIdentifiers,
     Uploader,
     UploaderConfig,
-    UploadStager,
-    UploadStagerConfig,
     download_responses,
 )
 from unstructured_ingest.v2.logger import logger
@@ -36,6 +36,7 @@ from unstructured_ingest.v2.processes.connector_registry import (
 
 if TYPE_CHECKING:
     from pymongo import MongoClient
+    from pymongo.collection import Collection
 
 CONNECTOR_TYPE = "mongodb"
 SERVER_API_VERSION = "1"
@@ -54,18 +55,37 @@ class MongoDBConnectionConfig(ConnectionConfig):
         description="hostname or IP address or Unix domain socket path of a single mongod or "
         "mongos instance to connect to, or a list of hostnames",
     )
-    database: Optional[str] = Field(default=None, description="database name to connect to")
-    collection: Optional[str] = Field(default=None, description="collection name to connect to")
     port: int = Field(default=27017)
     connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
+    @contextmanager
+    @requires_dependencies(["pymongo"], extras="mongodb")
+    def get_client(self) -> Generator["MongoClient", None, None]:
+        from pymongo import MongoClient
+        from pymongo.driver_info import DriverInfo
+        from pymongo.server_api import ServerApi
 
-class MongoDBUploadStagerConfig(UploadStagerConfig):
-    pass
+        access_config = self.access_config.get_secret_value()
+        if uri := access_config.uri:
+            client_kwargs = {
+                "host": uri,
+                "server_api": ServerApi(version=SERVER_API_VERSION),
+                "driver": DriverInfo(name="unstructured", version=unstructured_version),
+            }
+        else:
+            client_kwargs = {
+                "host": self.host,
+                "port": self.port,
+                "server_api": ServerApi(version=SERVER_API_VERSION),
+            }
+        with MongoClient(**client_kwargs) as client:
+            yield client
 
 
 class MongoDBIndexerConfig(IndexerConfig):
     batch_size: int = Field(default=100, description="Number of records per batch")
+    database: Optional[str] = Field(default=None, description="database name to connect to")
+    collection: Optional[str] = Field(default=None, description="collection name to connect to")
 
 
 class MongoDBDownloaderConfig(DownloaderConfig):
@@ -81,42 +101,38 @@ class MongoDBIndexer(Indexer):
     def precheck(self) -> None:
         """Validates the connection to the MongoDB server."""
         try:
-            client = self.create_client()
-            client.admin.command("ping")
+            with self.connection_config.get_client() as client:
+                client.admin.command("ping")
+                database_names = client.list_database_names()
+                database_name = self.index_config.database
+                if database_name not in database_names:
+                    raise DestinationConnectionError(
+                        "database {} does not exist: {}".format(
+                            database_name, ", ".join(database_names)
+                        )
+                    )
+                database = client[database_name]
+                collection_names = database.list_collection_names()
+                collection_name = self.index_config.collection
+                if collection_name not in collection_names:
+                    raise SourceConnectionError(
+                        "collection {} does not exist: {}".format(
+                            collection_name, ", ".join(collection_names)
+                        )
+                    )
         except Exception as e:
             logger.error(f"Failed to validate connection: {e}", exc_info=True)
             raise SourceConnectionError(f"Failed to validate connection: {e}")
 
-    @requires_dependencies(["pymongo"], extras="mongodb")
-    def create_client(self) -> "MongoClient":
-        from pymongo import MongoClient
-        from pymongo.driver_info import DriverInfo
-        from pymongo.server_api import ServerApi
-
-        access_config = self.connection_config.access_config.get_secret_value()
-
-        if access_config.uri:
-            return MongoClient(
-                access_config.uri,
-                server_api=ServerApi(version=SERVER_API_VERSION),
-                driver=DriverInfo(name="unstructured", version=unstructured_version),
-            )
-        else:
-            return MongoClient(
-                host=self.connection_config.host,
-                port=self.connection_config.port,
-                server_api=ServerApi(version=SERVER_API_VERSION),
-            )
-
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
         """Generates FileData objects for each document in the MongoDB collection."""
-        client = self.create_client()
-        database = client[self.connection_config.database]
-        collection = database[self.connection_config.collection]
+        with self.connection_config.get_client() as client:
+            database = client[self.index_config.database]
+            collection = database[self.index_config.collection]
 
-        # Get list of document IDs
-        ids = collection.distinct("_id")
-        batch_size = self.index_config.batch_size if self.index_config else 100
+            # Get list of document IDs
+            ids = collection.distinct("_id")
+            batch_size = self.index_config.batch_size if self.index_config else 100
 
         for id_batch in batch_generator(ids, batch_size=batch_size):
             # Make sure the hash is always a positive number to create identifier
@@ -125,8 +141,8 @@ class MongoDBIndexer(Indexer):
             metadata = FileDataSourceMetadata(
                 date_processed=str(time()),
                 record_locator={
-                    "database": self.connection_config.database,
-                    "collection": self.connection_config.collection,
+                    "database": self.index_config.database,
+                    "collection": self.index_config.collection,
                 },
             )
 
@@ -177,8 +193,8 @@ class MongoDBDownloader(Downloader):
         from bson.objectid import ObjectId
 
         client = self.create_client()
-        database = client[self.connection_config.database]
-        collection = database[self.connection_config.collection]
+        database = client[file_data.metadata.record_locator["database"]]
+        collection = database[file_data.metadata.record_locator["collection"]]
 
         ids = file_data.additional_metadata.get("ids", [])
         if not ids:
@@ -222,14 +238,12 @@ class MongoDBDownloader(Downloader):
             concatenated_values = "\n".join(str(value) for value in flattened_dict.values())
 
             # Create a FileData object for each document with source_identifiers
-            individual_file_data = FileData(
-                identifier=str(doc_id),
-                connector_type=self.connector_type,
-                source_identifiers=SourceIdentifiers(
-                    filename=str(doc_id),
-                    fullpath=str(doc_id),
-                    rel_path=str(doc_id),
-                ),
+            individual_file_data = replace(file_data)
+            individual_file_data.identifier = str(doc_id)
+            individual_file_data.source_identifiers = SourceIdentifiers(
+                filename=str(doc_id),
+                fullpath=str(doc_id),
+                rel_path=str(doc_id),
             )
 
             # Determine the download path
@@ -247,15 +261,8 @@ class MongoDBDownloader(Downloader):
             individual_file_data.local_download_path = str(download_path)
 
             # Update metadata
-            individual_file_data.metadata = FileDataSourceMetadata(
-                date_created=date_created,  # Include date_created here
-                date_processed=str(time()),
-                record_locator={
-                    "database": self.connection_config.database,
-                    "collection": self.connection_config.collection,
-                    "document_id": str(doc_id),
-                },
-            )
+            individual_file_data.metadata.record_locator["document_id"] = str(doc_id)
+            individual_file_data.metadata.date_created = date_created
 
             download_response = self.generate_download_response(
                 file_data=individual_file_data, download_path=download_path
@@ -265,31 +272,14 @@ class MongoDBDownloader(Downloader):
         return download_responses
 
 
-@dataclass
-class MongoDBUploadStager(UploadStager):
-    upload_stager_config: MongoDBUploadStagerConfig = field(
-        default_factory=lambda: MongoDBUploadStagerConfig()
-    )
-
-    def run(
-        self,
-        elements_filepath: Path,
-        file_data: FileData,
-        output_dir: Path,
-        output_filename: str,
-        **kwargs: Any,
-    ) -> Path:
-        with open(elements_filepath) as elements_file:
-            elements_contents = json.load(elements_file)
-
-        output_path = Path(output_dir) / Path(f"{output_filename}.json")
-        with open(output_path, "w") as output_file:
-            json.dump(elements_contents, output_file)
-        return output_path
-
-
 class MongoDBUploaderConfig(UploaderConfig):
     batch_size: int = Field(default=100, description="Number of records per batch")
+    database: Optional[str] = Field(default=None, description="database name to connect to")
+    collection: Optional[str] = Field(default=None, description="collection name to connect to")
+    record_id_key: str = Field(
+        default=RECORD_ID_LABEL,
+        description="searchable key to find entries for the same record on previous runs",
+    )
 
 
 @dataclass
@@ -300,55 +290,76 @@ class MongoDBUploader(Uploader):
 
     def precheck(self) -> None:
         try:
-            client = self.create_client()
-            client.admin.command("ping")
+            with self.connection_config.get_client() as client:
+                client.admin.command("ping")
+                database_names = client.list_database_names()
+                database_name = self.upload_config.database
+                if database_name not in database_names:
+                    raise DestinationConnectionError(
+                        "database {} does not exist: {}".format(
+                            database_name, ", ".join(database_names)
+                        )
+                    )
+                database = client[database_name]
+                collection_names = database.list_collection_names()
+                collection_name = self.upload_config.collection
+                if collection_name not in collection_names:
+                    raise SourceConnectionError(
+                        "collection {} does not exist: {}".format(
+                            collection_name, ", ".join(collection_names)
+                        )
+                    )
         except Exception as e:
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")
 
-    @requires_dependencies(["pymongo"], extras="mongodb")
-    def create_client(self) -> "MongoClient":
-        from pymongo import MongoClient
-        from pymongo.driver_info import DriverInfo
-        from pymongo.server_api import ServerApi
-
-        access_config = self.connection_config.access_config.get_secret_value()
-
-        if access_config.uri:
-            return MongoClient(
-                access_config.uri,
-                server_api=ServerApi(version=SERVER_API_VERSION),
-                driver=DriverInfo(name="unstructured", version=unstructured_version),
-            )
-        else:
-            return MongoClient(
-                host=self.connection_config.host,
-                port=self.connection_config.port,
-                server_api=ServerApi(version=SERVER_API_VERSION),
-            )
+    def can_delete(self, collection: "Collection") -> bool:
+        indexed_keys = []
+        for index in collection.list_indexes():
+            key_bson = index["key"]
+            indexed_keys.extend(key_bson.keys())
+        return self.upload_config.record_id_key in indexed_keys
+
+    def delete_by_record_id(self, collection: "Collection", file_data: FileData) -> None:
+        logger.debug(
+            f"deleting any content with metadata "
+            f"{self.upload_config.record_id_key}={file_data.identifier} "
+            f"from collection: {collection.name}"
+        )
+        query = {self.upload_config.record_id_key: file_data.identifier}
+        delete_results = collection.delete_many(filter=query)
+        logger.info(
+            f"deleted {delete_results.deleted_count} records from collection {collection.name}"
+        )
 
     def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
         with path.open("r") as file:
             elements_dict = json.load(file)
         logger.info(
             f"writing {len(elements_dict)} objects to destination "
-            f"db, {self.connection_config.database}, "
-            f"collection {self.connection_config.collection} "
+            f"db, {self.upload_config.database}, "
+            f"collection {self.upload_config.collection} "
             f"at {self.connection_config.host}",
         )
-        client = self.create_client()
-        db = client[self.connection_config.database]
-        collection = db[self.connection_config.collection]
-        for chunk in batch_generator(elements_dict, self.upload_config.batch_size):
-            collection.insert_many(chunk)
+        # This would typically live in the stager but since no other manipulation
+        # is done, setting the record id field in the uploader
+        for element in elements_dict:
+            element[self.upload_config.record_id_key] = file_data.identifier
+        with self.connection_config.get_client() as client:
+            db = client[self.upload_config.database]
+            collection = db[self.upload_config.collection]
+            if self.can_delete(collection=collection):
+                self.delete_by_record_id(file_data=file_data, collection=collection)
+            else:
+                logger.warning("criteria for deleting previous content not met, skipping")
+            for chunk in batch_generator(elements_dict, self.upload_config.batch_size):
+                collection.insert_many(chunk)
 
 
 mongodb_destination_entry = DestinationRegistryEntry(
     connection_config=MongoDBConnectionConfig,
     uploader=MongoDBUploader,
     uploader_config=MongoDBUploaderConfig,
-    upload_stager=MongoDBUploadStager,
-    upload_stager_config=MongoDBUploadStagerConfig,
 )
 
 mongodb_source_entry = SourceRegistryEntry(

unstructured_ingest/v2/processes/connectors/weaviate/__init__.py

@@ -10,6 +10,8 @@ from .embedded import CONNECTOR_TYPE as EMBEDDED_WEAVIATE_CONNECTOR_TYPE
 from .embedded import weaviate_embedded_destination_entry
 from .local import CONNECTOR_TYPE as LOCAL_WEAVIATE_CONNECTOR_TYPE
 from .local import weaviate_local_destination_entry
+from .weaviate import CONNECTOR_TYPE as WEAVIATE_CONNECTOR_TYPE
+from .weaviate import weaviate_destination_entry
 
 add_destination_entry(
     destination_type=LOCAL_WEAVIATE_CONNECTOR_TYPE, entry=weaviate_local_destination_entry
@@ -20,3 +22,4 @@ add_destination_entry(
 add_destination_entry(
     destination_type=EMBEDDED_WEAVIATE_CONNECTOR_TYPE, entry=weaviate_embedded_destination_entry
 )
+add_destination_entry(destination_type=WEAVIATE_CONNECTOR_TYPE, entry=weaviate_destination_entry)

unstructured_ingest/v2/processes/connectors/weaviate/cloud.py

@@ -55,10 +55,11 @@ class CloudWeaviateConnectionConfig(WeaviateConnectionConfig):
             "client_secret": access_config.client_secret is not None,
             "client_password": access_config.password is not None and self.username is not None,
         }
-        if len(auths) == 0:
+        existing_auths = [auth_method for auth_method, flag in auths.items() if flag]
+
+        if len(existing_auths) == 0:
             raise ValueError("No auth values provided and anonymous is False")
-        if len(auths) > 1:
-            existing_auths = [auth_method for auth_method, flag in auths.items() if flag]
+        if len(existing_auths) > 1:
             raise ValueError(
                 "Multiple auth values provided, only one approach can be used: {}".format(
                     ", ".join(existing_auths)

unstructured_ingest/v2/processes/connectors/weaviate/weaviate.py

@@ -22,6 +22,7 @@ from unstructured_ingest.v2.interfaces import (
     UploadStagerConfig,
 )
 from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
 
 if TYPE_CHECKING:
     from weaviate.classes.init import Timeout
@@ -287,3 +288,12 @@ class WeaviateUploader(Uploader, ABC):
                         vector=vector,
                     )
             self.check_for_errors(client=weaviate_client)
+
+
+weaviate_destination_entry = DestinationRegistryEntry(
+    connection_config=WeaviateConnectionConfig,
+    uploader=WeaviateUploader,
+    uploader_config=WeaviateUploaderConfig,
+    upload_stager=WeaviateUploadStager,
+    upload_stager_config=WeaviateUploadStagerConfig,
+)

{unstructured_ingest-0.3.1.dist-info → unstructured_ingest-0.3.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.3.1
+Version: 0.3.3
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,27 +22,27 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: python-dateutil
-Requires-Dist: tqdm
-Requires-Dist: click
 Requires-Dist: dataclasses-json
 Requires-Dist: pydantic>=2.7
 Requires-Dist: pandas
+Requires-Dist: tqdm
+Requires-Dist: python-dateutil
+Requires-Dist: click
 Requires-Dist: opentelemetry-sdk
 Provides-Extra: airtable
 Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: fsspec; extra == "azure"
 Requires-Dist: adlfs; extra == "azure"
+Requires-Dist: fsspec; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: bedrock
 Requires-Dist: boto3; extra == "bedrock"
 Provides-Extra: biomed
-Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
+Requires-Dist: requests; extra == "biomed"
 Provides-Extra: box
 Requires-Dist: boxfs; extra == "box"
 Requires-Dist: fsspec; extra == "box"
@@ -91,21 +91,23 @@ Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: gcsfs; extra == "gcs"
 Requires-Dist: fsspec; extra == "gcs"
 Provides-Extra: github
-Requires-Dist: requests; extra == "github"
 Requires-Dist: pygithub>1.58.0; extra == "github"
+Requires-Dist: requests; extra == "github"
 Provides-Extra: gitlab
 Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: hubspot-api-client; extra == "hubspot"
 Requires-Dist: urllib3; extra == "hubspot"
+Requires-Dist: hubspot-api-client; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
 Requires-Dist: confluent-kafka; extra == "kafka"
 Provides-Extra: kdbai
 Requires-Dist: kdbai-client>=1.4.0; extra == "kdbai"
+Provides-Extra: lancedb
+Requires-Dist: lancedb; extra == "lancedb"
 Provides-Extra: md
 Requires-Dist: unstructured[md]; extra == "md"
 Provides-Extra: milvus
@@ -115,16 +117,16 @@ Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: msg
 Requires-Dist: unstructured[msg]; extra == "msg"
 Provides-Extra: notion
+Requires-Dist: backoff; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: notion-client; extra == "notion"
-Requires-Dist: backoff; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
 Provides-Extra: odt
 Requires-Dist: unstructured[odt]; extra == "odt"
 Provides-Extra: onedrive
 Requires-Dist: msal; extra == "onedrive"
-Requires-Dist: bs4; extra == "onedrive"
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
+Requires-Dist: bs4; extra == "onedrive"
 Provides-Extra: openai
 Requires-Dist: openai; extra == "openai"
 Requires-Dist: tiktoken; extra == "openai"
@@ -156,8 +158,8 @@ Requires-Dist: unstructured[rst]; extra == "rst"
 Provides-Extra: rtf
 Requires-Dist: unstructured[rtf]; extra == "rtf"
 Provides-Extra: s3
-Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: fsspec; extra == "s3"
+Requires-Dist: s3fs; extra == "s3"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
@@ -171,8 +173,8 @@ Requires-Dist: singlestoredb; extra == "singlestore"
 Provides-Extra: slack
 Requires-Dist: slack-sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: psycopg2-binary; extra == "snowflake"
 Requires-Dist: snowflake-connector-python; extra == "snowflake"
+Requires-Dist: psycopg2-binary; extra == "snowflake"
 Provides-Extra: togetherai
 Requires-Dist: together; extra == "togetherai"
 Provides-Extra: tsv