unstructured-ingest 0.3.1__py3-none-any.whl → 0.3.2__py3-none-any.whl

test/integration/connectors/test_lancedb.py

@@ -1,6 +1,7 @@
 import os
 from pathlib import Path
 from typing import Literal, Union
+from uuid import uuid4
 
 import lancedb
 import pandas as pd
@@ -150,7 +151,7 @@ def _get_uri(target: Literal["local", "s3", "gcs", "az"], local_base_path: Path)
     elif target == "az":
         base_uri = UPath(AZURE_BUCKET)
 
-    return str(base_uri / "destination" / "lancedb" / DATABASE_NAME)
+    return str(base_uri / "destination" / "lancedb" / str(uuid4()) / DATABASE_NAME)
 
 
 def _get_uploader(

test/integration/connectors/test_mongodb.py

@@ -0,0 +1,332 @@
+import json
+import os
+import time
+import uuid
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Generator
+
+import pytest
+from pydantic import BaseModel, SecretStr
+from pymongo.collection import Collection
+from pymongo.database import Database
+from pymongo.mongo_client import MongoClient
+from pymongo.operations import SearchIndexModel
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG
+from test.integration.connectors.utils.validation import (
+    ValidationConfigs,
+    source_connector_validation,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.error import DestinationConnectionError, SourceConnectionError
+from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
+from unstructured_ingest.v2.processes.connectors.mongodb import (
+    CONNECTOR_TYPE,
+    MongoDBAccessConfig,
+    MongoDBConnectionConfig,
+    MongoDBDownloader,
+    MongoDBDownloaderConfig,
+    MongoDBIndexer,
+    MongoDBIndexerConfig,
+    MongoDBUploader,
+    MongoDBUploaderConfig,
+)
+
+SOURCE_COLLECTION = "sample-mongodb-data"
+
+
+class EnvData(BaseModel):
+    uri: SecretStr
+    database: str
+
+
+def get_env_data() -> EnvData:
+    uri = os.getenv("MONGODB_URI")
+    assert uri
+    database = os.getenv("MONGODB_DATABASE")
+    assert database
+    return EnvData(uri=uri, database=database)
+
+
+@contextmanager
+def get_client() -> Generator[MongoClient, None, None]:
+    uri = get_env_data().uri.get_secret_value()
+    with MongoClient(uri) as client:
+        assert client.admin.command("ping")
+        yield client
+
+
+def wait_for_collection(
+    database: Database, collection_name: str, retries: int = 10, interval: int = 1
+):
+    collections = database.list_collection_names()
+    attempts = 0
+    while collection_name not in collections and attempts < retries:
+        attempts += 1
+        print(
+            "Waiting for collection {} to be recognized: {}".format(
+                collection_name, ", ".join(collections)
+            )
+        )
+        time.sleep(interval)
+        collections = database.list_collection_names()
+    if collection_name not in collection_name:
+        raise TimeoutError(f"Collection {collection_name} was not recognized")
+
+
+def get_search_index_status(collection: Collection, index_name: str) -> str:
+    search_indexes = collection.list_search_indexes(name=index_name)
+    search_index = list(search_indexes)[0]
+    return search_index["status"]
+
+
+def wait_for_search_index(
+    collection: Collection, index_name: str, retries: int = 60, interval: int = 1
+):
+    current_status = get_search_index_status(collection, index_name)
+    attempts = 0
+    while current_status != "READY" and attempts < retries:
+        attempts += 1
+        print(f"attempt {attempts}: waiting for search index to be READY: {current_status}")
+        time.sleep(interval)
+        current_status = get_search_index_status(collection, index_name)
+
+    if current_status != "READY":
+        raise TimeoutError("search index never detected as READY")
+
+
+@pytest.fixture
+def destination_collection() -> Collection:
+    env_data = get_env_data()
+    collection_name = f"utic-test-output-{uuid.uuid4()}"
+    with get_client() as client:
+        database = client[env_data.database]
+        print(f"creating collection in database {database}: {collection_name}")
+        collection = database.create_collection(name=collection_name)
+        search_index_name = "embeddings"
+        collection.create_search_index(
+            model=SearchIndexModel(
+                name=search_index_name,
+                definition={
+                    "mappings": {
+                        "dynamic": True,
+                        "fields": {
+                            "embeddings": [
+                                {"type": "knnVector", "dimensions": 384, "similarity": "euclidean"}
+                            ]
+                        },
+                    }
+                },
+            )
+        )
+        collection.create_index("record_id")
+        wait_for_collection(database=database, collection_name=collection_name)
+        wait_for_search_index(collection=collection, index_name=search_index_name)
+        try:
+            yield collection
+        finally:
+            print(f"deleting collection: {collection_name}")
+            collection.drop()
+
+
+def validate_collection_count(
+    collection: Collection, expected_records: int, retries: int = 10, interval: int = 1
+) -> None:
+    count = collection.count_documents(filter={})
+    attempt = 0
+    while count != expected_records and attempt < retries:
+        attempt += 1
+        print(f"attempt {attempt} to get count of collection {count} to match {expected_records}")
+        time.sleep(interval)
+        count = collection.count_documents(filter={})
+    assert (
+        count == expected_records
+    ), f"expected count ({expected_records}) does not match how many records were found: {count}"
+
+
+def validate_collection_vector(
+    collection: Collection, embedding: list[float], text: str, retries: int = 30, interval: int = 1
+) -> None:
+    pipeline = [
+        {
+            "$vectorSearch": {
+                "index": "embeddings",
+                "path": "embeddings",
+                "queryVector": embedding,
+                "numCandidates": 150,
+                "limit": 10,
+            },
+        },
+        {"$project": {"_id": 0, "text": 1, "score": {"$meta": "vectorSearchScore"}}},
+    ]
+    attempts = 0
+    results = list(collection.aggregate(pipeline=pipeline))
+    while not results and attempts < retries:
+        attempts += 1
+        print(f"attempt {attempts}, waiting for valid results: {results}")
+        time.sleep(interval)
+        results = list(collection.aggregate(pipeline=pipeline))
+    if not results:
+        raise TimeoutError("Timed out waiting for valid results")
+    print(f"found results on attempt {attempts}")
+    top_result = results[0]
+    assert top_result["score"] == 1.0, "score detected should be 1: {}".format(top_result["score"])
+    assert top_result["text"] == text, "text detected should be {}, found: {}".format(
+        text, top_result["text"]
+    )
+    for r in results[1:]:
+        assert r["score"] < 1.0, "score detected should be less than 1: {}".format(r["score"])
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@requires_env("MONGODB_URI", "MONGODB_DATABASE")
+async def test_mongodb_source(temp_dir: Path):
+    env_data = get_env_data()
+    indexer_config = MongoDBIndexerConfig(database=env_data.database, collection=SOURCE_COLLECTION)
+    download_config = MongoDBDownloaderConfig(download_dir=temp_dir)
+    connection_config = MongoDBConnectionConfig(
+        access_config=MongoDBAccessConfig(uri=env_data.uri.get_secret_value()),
+    )
+    indexer = MongoDBIndexer(connection_config=connection_config, index_config=indexer_config)
+    downloader = MongoDBDownloader(
+        connection_config=connection_config, download_config=download_config
+    )
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=ValidationConfigs(
+            test_id=CONNECTOR_TYPE, expected_num_files=4, validate_downloaded_files=True
+        ),
+    )
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+def test_mongodb_indexer_precheck_fail_no_host():
+    indexer_config = MongoDBIndexerConfig(
+        database="non-existent-database", collection="non-existent-database"
+    )
+    connection_config = MongoDBConnectionConfig(
+        access_config=MongoDBAccessConfig(uri="mongodb+srv://ingest-test.hgaig.mongodb"),
+    )
+    indexer = MongoDBIndexer(connection_config=connection_config, index_config=indexer_config)
+    with pytest.raises(SourceConnectionError):
+        indexer.precheck()
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@requires_env("MONGODB_URI", "MONGODB_DATABASE")
+def test_mongodb_indexer_precheck_fail_no_database():
+    env_data = get_env_data()
+    indexer_config = MongoDBIndexerConfig(
+        database="non-existent-database", collection=SOURCE_COLLECTION
+    )
+    connection_config = MongoDBConnectionConfig(
+        access_config=MongoDBAccessConfig(uri=env_data.uri.get_secret_value()),
+    )
+    indexer = MongoDBIndexer(connection_config=connection_config, index_config=indexer_config)
+    with pytest.raises(SourceConnectionError):
+        indexer.precheck()
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@requires_env("MONGODB_URI", "MONGODB_DATABASE")
+def test_mongodb_indexer_precheck_fail_no_collection():
+    env_data = get_env_data()
+    indexer_config = MongoDBIndexerConfig(
+        database=env_data.database, collection="non-existent-collection"
+    )
+    connection_config = MongoDBConnectionConfig(
+        access_config=MongoDBAccessConfig(uri=env_data.uri.get_secret_value()),
+    )
+    indexer = MongoDBIndexer(connection_config=connection_config, index_config=indexer_config)
+    with pytest.raises(SourceConnectionError):
+        indexer.precheck()
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@requires_env("MONGODB_URI", "MONGODB_DATABASE")
+async def test_mongodb_destination(
+    upload_file: Path,
+    destination_collection: Collection,
+    tmp_path: Path,
+):
+    env_data = get_env_data()
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
+        connector_type=CONNECTOR_TYPE,
+        identifier="mongodb_mock_id",
+    )
+    connection_config = MongoDBConnectionConfig(
+        access_config=MongoDBAccessConfig(uri=env_data.uri.get_secret_value()),
+    )
+
+    upload_config = MongoDBUploaderConfig(
+        database=env_data.database,
+        collection=destination_collection.name,
+    )
+    uploader = MongoDBUploader(connection_config=connection_config, upload_config=upload_config)
+    uploader.precheck()
+    uploader.run(path=upload_file, file_data=file_data)
+
+    with upload_file.open() as f:
+        staged_elements = json.load(f)
+    expected_records = len(staged_elements)
+    validate_collection_count(collection=destination_collection, expected_records=expected_records)
+    first_element = staged_elements[0]
+    validate_collection_vector(
+        collection=destination_collection,
+        embedding=first_element["embeddings"],
+        text=first_element["text"],
+    )
+
+    uploader.run(path=upload_file, file_data=file_data)
+    validate_collection_count(collection=destination_collection, expected_records=expected_records)
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+def test_mongodb_uploader_precheck_fail_no_host():
+    upload_config = MongoDBUploaderConfig(
+        database="database",
+        collection="collection",
+    )
+    connection_config = MongoDBConnectionConfig(
+        access_config=MongoDBAccessConfig(uri="mongodb+srv://ingest-test.hgaig.mongodb"),
+    )
+    uploader = MongoDBUploader(connection_config=connection_config, upload_config=upload_config)
+    with pytest.raises(DestinationConnectionError):
+        uploader.precheck()
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@requires_env("MONGODB_URI", "MONGODB_DATABASE")
+def test_mongodb_uploader_precheck_fail_no_database():
+    env_data = get_env_data()
+    upload_config = MongoDBUploaderConfig(
+        database="database",
+        collection="collection",
+    )
+    connection_config = MongoDBConnectionConfig(
+        access_config=MongoDBAccessConfig(uri=env_data.uri.get_secret_value()),
+    )
+    uploader = MongoDBUploader(connection_config=connection_config, upload_config=upload_config)
+    with pytest.raises(DestinationConnectionError):
+        uploader.precheck()
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@requires_env("MONGODB_URI", "MONGODB_DATABASE")
+def test_mongodb_uploader_precheck_fail_no_collection():
+    env_data = get_env_data()
+    upload_config = MongoDBUploaderConfig(
+        database=env_data.database,
+        collection="collection",
+    )
+    connection_config = MongoDBConnectionConfig(
+        access_config=MongoDBAccessConfig(uri=env_data.uri.get_secret_value()),
+    )
+    uploader = MongoDBUploader(connection_config=connection_config, upload_config=upload_config)
+    with pytest.raises(DestinationConnectionError):
+        uploader.precheck()

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.3.1"  # pragma: no cover
+__version__ = "0.3.2"  # pragma: no cover

unstructured_ingest/v2/processes/connectors/google_drive.py

@@ -161,7 +161,7 @@ class GoogleDriveIndexer(Indexer):
             and isinstance(parent_root_path, str)
         ):
             fullpath = f"{parent_path}/{filename}"
-            rel_path = fullpath.replace(parent_root_path, "")
+            rel_path = Path(fullpath).relative_to(parent_root_path).as_posix()
             source_identifiers = SourceIdentifiers(
                 filename=filename, fullpath=fullpath, rel_path=rel_path
             )

unstructured_ingest/v2/processes/connectors/mongodb.py

@@ -1,6 +1,7 @@
 import json
 import sys
-from dataclasses import dataclass, field
+from contextlib import contextmanager
+from dataclasses import dataclass, replace
 from datetime import datetime
 from pathlib import Path
 from time import time
@@ -12,6 +13,7 @@ from unstructured_ingest.__version__ import __version__ as unstructured_version
 from unstructured_ingest.error import DestinationConnectionError, SourceConnectionError
 from unstructured_ingest.utils.data_prep import batch_generator, flatten_dict
 from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
@@ -24,8 +26,6 @@ from unstructured_ingest.v2.interfaces import (
     SourceIdentifiers,
     Uploader,
     UploaderConfig,
-    UploadStager,
-    UploadStagerConfig,
     download_responses,
 )
 from unstructured_ingest.v2.logger import logger
@@ -36,6 +36,7 @@ from unstructured_ingest.v2.processes.connector_registry import (
 
 if TYPE_CHECKING:
     from pymongo import MongoClient
+    from pymongo.collection import Collection
 
 CONNECTOR_TYPE = "mongodb"
 SERVER_API_VERSION = "1"
@@ -54,18 +55,37 @@ class MongoDBConnectionConfig(ConnectionConfig):
         description="hostname or IP address or Unix domain socket path of a single mongod or "
         "mongos instance to connect to, or a list of hostnames",
     )
-    database: Optional[str] = Field(default=None, description="database name to connect to")
-    collection: Optional[str] = Field(default=None, description="collection name to connect to")
     port: int = Field(default=27017)
     connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
+    @contextmanager
+    @requires_dependencies(["pymongo"], extras="mongodb")
+    def get_client(self) -> Generator["MongoClient", None, None]:
+        from pymongo import MongoClient
+        from pymongo.driver_info import DriverInfo
+        from pymongo.server_api import ServerApi
 
-class MongoDBUploadStagerConfig(UploadStagerConfig):
-    pass
+        access_config = self.access_config.get_secret_value()
+        if uri := access_config.uri:
+            client_kwargs = {
+                "host": uri,
+                "server_api": ServerApi(version=SERVER_API_VERSION),
+                "driver": DriverInfo(name="unstructured", version=unstructured_version),
+            }
+        else:
+            client_kwargs = {
+                "host": self.host,
+                "port": self.port,
+                "server_api": ServerApi(version=SERVER_API_VERSION),
+            }
+        with MongoClient(**client_kwargs) as client:
+            yield client
 
 
 class MongoDBIndexerConfig(IndexerConfig):
     batch_size: int = Field(default=100, description="Number of records per batch")
+    database: Optional[str] = Field(default=None, description="database name to connect to")
+    collection: Optional[str] = Field(default=None, description="collection name to connect to")
 
 
 class MongoDBDownloaderConfig(DownloaderConfig):
@@ -81,42 +101,38 @@ class MongoDBIndexer(Indexer):
     def precheck(self) -> None:
         """Validates the connection to the MongoDB server."""
         try:
-            client = self.create_client()
-            client.admin.command("ping")
+            with self.connection_config.get_client() as client:
+                client.admin.command("ping")
+                database_names = client.list_database_names()
+                database_name = self.index_config.database
+                if database_name not in database_names:
+                    raise DestinationConnectionError(
+                        "database {} does not exist: {}".format(
+                            database_name, ", ".join(database_names)
+                        )
+                    )
+                database = client[database_name]
+                collection_names = database.list_collection_names()
+                collection_name = self.index_config.collection
+                if collection_name not in collection_names:
+                    raise SourceConnectionError(
+                        "collection {} does not exist: {}".format(
+                            collection_name, ", ".join(collection_names)
+                        )
+                    )
         except Exception as e:
             logger.error(f"Failed to validate connection: {e}", exc_info=True)
             raise SourceConnectionError(f"Failed to validate connection: {e}")
 
-    @requires_dependencies(["pymongo"], extras="mongodb")
-    def create_client(self) -> "MongoClient":
-        from pymongo import MongoClient
-        from pymongo.driver_info import DriverInfo
-        from pymongo.server_api import ServerApi
-
-        access_config = self.connection_config.access_config.get_secret_value()
-
-        if access_config.uri:
-            return MongoClient(
-                access_config.uri,
-                server_api=ServerApi(version=SERVER_API_VERSION),
-                driver=DriverInfo(name="unstructured", version=unstructured_version),
-            )
-        else:
-            return MongoClient(
-                host=self.connection_config.host,
-                port=self.connection_config.port,
-                server_api=ServerApi(version=SERVER_API_VERSION),
-            )
-
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
         """Generates FileData objects for each document in the MongoDB collection."""
-        client = self.create_client()
-        database = client[self.connection_config.database]
-        collection = database[self.connection_config.collection]
+        with self.connection_config.get_client() as client:
+            database = client[self.index_config.database]
+            collection = database[self.index_config.collection]
 
-        # Get list of document IDs
-        ids = collection.distinct("_id")
-        batch_size = self.index_config.batch_size if self.index_config else 100
+            # Get list of document IDs
+            ids = collection.distinct("_id")
+            batch_size = self.index_config.batch_size if self.index_config else 100
 
         for id_batch in batch_generator(ids, batch_size=batch_size):
             # Make sure the hash is always a positive number to create identifier
@@ -125,8 +141,8 @@ class MongoDBIndexer(Indexer):
             metadata = FileDataSourceMetadata(
                 date_processed=str(time()),
                 record_locator={
-                    "database": self.connection_config.database,
-                    "collection": self.connection_config.collection,
+                    "database": self.index_config.database,
+                    "collection": self.index_config.collection,
                 },
             )
 
@@ -177,8 +193,8 @@ class MongoDBDownloader(Downloader):
         from bson.objectid import ObjectId
 
         client = self.create_client()
-        database = client[self.connection_config.database]
-        collection = database[self.connection_config.collection]
+        database = client[file_data.metadata.record_locator["database"]]
+        collection = database[file_data.metadata.record_locator["collection"]]
 
         ids = file_data.additional_metadata.get("ids", [])
         if not ids:
@@ -222,14 +238,12 @@ class MongoDBDownloader(Downloader):
             concatenated_values = "\n".join(str(value) for value in flattened_dict.values())
 
             # Create a FileData object for each document with source_identifiers
-            individual_file_data = FileData(
-                identifier=str(doc_id),
-                connector_type=self.connector_type,
-                source_identifiers=SourceIdentifiers(
-                    filename=str(doc_id),
-                    fullpath=str(doc_id),
-                    rel_path=str(doc_id),
-                ),
+            individual_file_data = replace(file_data)
+            individual_file_data.identifier = str(doc_id)
+            individual_file_data.source_identifiers = SourceIdentifiers(
+                filename=str(doc_id),
+                fullpath=str(doc_id),
+                rel_path=str(doc_id),
             )
 
             # Determine the download path
@@ -247,15 +261,8 @@ class MongoDBDownloader(Downloader):
             individual_file_data.local_download_path = str(download_path)
 
             # Update metadata
-            individual_file_data.metadata = FileDataSourceMetadata(
-                date_created=date_created,  # Include date_created here
-                date_processed=str(time()),
-                record_locator={
-                    "database": self.connection_config.database,
-                    "collection": self.connection_config.collection,
-                    "document_id": str(doc_id),
-                },
-            )
+            individual_file_data.metadata.record_locator["document_id"] = str(doc_id)
+            individual_file_data.metadata.date_created = date_created
 
             download_response = self.generate_download_response(
                 file_data=individual_file_data, download_path=download_path
@@ -265,31 +272,14 @@ class MongoDBDownloader(Downloader):
         return download_responses
 
 
-@dataclass
-class MongoDBUploadStager(UploadStager):
-    upload_stager_config: MongoDBUploadStagerConfig = field(
-        default_factory=lambda: MongoDBUploadStagerConfig()
-    )
-
-    def run(
-        self,
-        elements_filepath: Path,
-        file_data: FileData,
-        output_dir: Path,
-        output_filename: str,
-        **kwargs: Any,
-    ) -> Path:
-        with open(elements_filepath) as elements_file:
-            elements_contents = json.load(elements_file)
-
-        output_path = Path(output_dir) / Path(f"{output_filename}.json")
-        with open(output_path, "w") as output_file:
-            json.dump(elements_contents, output_file)
-        return output_path
-
-
 class MongoDBUploaderConfig(UploaderConfig):
     batch_size: int = Field(default=100, description="Number of records per batch")
+    database: Optional[str] = Field(default=None, description="database name to connect to")
+    collection: Optional[str] = Field(default=None, description="collection name to connect to")
+    record_id_key: str = Field(
+        default=RECORD_ID_LABEL,
+        description="searchable key to find entries for the same record on previous runs",
+    )
 
 
 @dataclass
@@ -300,55 +290,76 @@ class MongoDBUploader(Uploader):
 
     def precheck(self) -> None:
         try:
-            client = self.create_client()
-            client.admin.command("ping")
+            with self.connection_config.get_client() as client:
+                client.admin.command("ping")
+                database_names = client.list_database_names()
+                database_name = self.upload_config.database
+                if database_name not in database_names:
+                    raise DestinationConnectionError(
+                        "database {} does not exist: {}".format(
+                            database_name, ", ".join(database_names)
+                        )
+                    )
+                database = client[database_name]
+                collection_names = database.list_collection_names()
+                collection_name = self.upload_config.collection
+                if collection_name not in collection_names:
+                    raise SourceConnectionError(
+                        "collection {} does not exist: {}".format(
+                            collection_name, ", ".join(collection_names)
+                        )
+                    )
         except Exception as e:
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")
 
-    @requires_dependencies(["pymongo"], extras="mongodb")
-    def create_client(self) -> "MongoClient":
-        from pymongo import MongoClient
-        from pymongo.driver_info import DriverInfo
-        from pymongo.server_api import ServerApi
-
-        access_config = self.connection_config.access_config.get_secret_value()
-
-        if access_config.uri:
-            return MongoClient(
-                access_config.uri,
-                server_api=ServerApi(version=SERVER_API_VERSION),
-                driver=DriverInfo(name="unstructured", version=unstructured_version),
-            )
-        else:
-            return MongoClient(
-                host=self.connection_config.host,
-                port=self.connection_config.port,
-                server_api=ServerApi(version=SERVER_API_VERSION),
-            )
+    def can_delete(self, collection: "Collection") -> bool:
+        indexed_keys = []
+        for index in collection.list_indexes():
+            key_bson = index["key"]
+            indexed_keys.extend(key_bson.keys())
+        return self.upload_config.record_id_key in indexed_keys
+
+    def delete_by_record_id(self, collection: "Collection", file_data: FileData) -> None:
+        logger.debug(
+            f"deleting any content with metadata "
+            f"{self.upload_config.record_id_key}={file_data.identifier} "
+            f"from collection: {collection.name}"
+        )
+        query = {self.upload_config.record_id_key: file_data.identifier}
+        delete_results = collection.delete_many(filter=query)
+        logger.info(
+            f"deleted {delete_results.deleted_count} records from collection {collection.name}"
+        )
 
     def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
         with path.open("r") as file:
             elements_dict = json.load(file)
         logger.info(
             f"writing {len(elements_dict)} objects to destination "
-            f"db, {self.connection_config.database}, "
-            f"collection {self.connection_config.collection} "
+            f"db, {self.upload_config.database}, "
+            f"collection {self.upload_config.collection} "
             f"at {self.connection_config.host}",
         )
-        client = self.create_client()
-        db = client[self.connection_config.database]
-        collection = db[self.connection_config.collection]
-        for chunk in batch_generator(elements_dict, self.upload_config.batch_size):
-            collection.insert_many(chunk)
+        # This would typically live in the stager but since no other manipulation
+        # is done, setting the record id field in the uploader
+        for element in elements_dict:
+            element[self.upload_config.record_id_key] = file_data.identifier
+        with self.connection_config.get_client() as client:
+            db = client[self.upload_config.database]
+            collection = db[self.upload_config.collection]
+            if self.can_delete(collection=collection):
+                self.delete_by_record_id(file_data=file_data, collection=collection)
+            else:
+                logger.warning("criteria for deleting previous content not met, skipping")
+            for chunk in batch_generator(elements_dict, self.upload_config.batch_size):
+                collection.insert_many(chunk)
 
 
 mongodb_destination_entry = DestinationRegistryEntry(
     connection_config=MongoDBConnectionConfig,
     uploader=MongoDBUploader,
     uploader_config=MongoDBUploaderConfig,
-    upload_stager=MongoDBUploadStager,
-    upload_stager_config=MongoDBUploadStagerConfig,
 )
 
 mongodb_source_entry = SourceRegistryEntry(

unstructured_ingest/v2/processes/connectors/weaviate/__init__.py

@@ -10,6 +10,8 @@ from .embedded import CONNECTOR_TYPE as EMBEDDED_WEAVIATE_CONNECTOR_TYPE
 from .embedded import weaviate_embedded_destination_entry
 from .local import CONNECTOR_TYPE as LOCAL_WEAVIATE_CONNECTOR_TYPE
 from .local import weaviate_local_destination_entry
+from .weaviate import CONNECTOR_TYPE as WEAVIATE_CONNECTOR_TYPE
+from .weaviate import weaviate_destination_entry
 
 add_destination_entry(
     destination_type=LOCAL_WEAVIATE_CONNECTOR_TYPE, entry=weaviate_local_destination_entry
@@ -20,3 +22,4 @@ add_destination_entry(
 add_destination_entry(
     destination_type=EMBEDDED_WEAVIATE_CONNECTOR_TYPE, entry=weaviate_embedded_destination_entry
 )
+add_destination_entry(destination_type=WEAVIATE_CONNECTOR_TYPE, entry=weaviate_destination_entry)

unstructured_ingest/v2/processes/connectors/weaviate/weaviate.py

@@ -22,6 +22,7 @@ from unstructured_ingest.v2.interfaces import (
     UploadStagerConfig,
 )
 from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
 
 if TYPE_CHECKING:
     from weaviate.classes.init import Timeout
@@ -287,3 +288,12 @@ class WeaviateUploader(Uploader, ABC):
                         vector=vector,
                     )
             self.check_for_errors(client=weaviate_client)
+
+
+weaviate_destination_entry = DestinationRegistryEntry(
+    connection_config=WeaviateConnectionConfig,
+    uploader=WeaviateUploader,
+    uploader_config=WeaviateUploaderConfig,
+    upload_stager=WeaviateUploadStager,
+    upload_stager_config=WeaviateUploadStagerConfig,
+)

{unstructured_ingest-0.3.1.dist-info → unstructured_ingest-0.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.3.1
+Version: 0.3.2
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -23,12 +23,12 @@ Requires-Python: >=3.9.0,<3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
 Requires-Dist: python-dateutil
-Requires-Dist: tqdm
-Requires-Dist: click
-Requires-Dist: dataclasses-json
 Requires-Dist: pydantic>=2.7
-Requires-Dist: pandas
 Requires-Dist: opentelemetry-sdk
+Requires-Dist: click
+Requires-Dist: tqdm
+Requires-Dist: pandas
+Requires-Dist: dataclasses-json
 Provides-Extra: airtable
 Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
@@ -41,8 +41,8 @@ Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: bedrock
 Requires-Dist: boto3; extra == "bedrock"
 Provides-Extra: biomed
-Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
+Requires-Dist: requests; extra == "biomed"
 Provides-Extra: box
 Requires-Dist: boxfs; extra == "box"
 Requires-Dist: fsspec; extra == "box"
@@ -78,8 +78,8 @@ Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Provides-Extra: embed-octoai
-Requires-Dist: openai; extra == "embed-octoai"
 Requires-Dist: tiktoken; extra == "embed-octoai"
+Requires-Dist: openai; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
@@ -98,8 +98,8 @@ Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: hubspot-api-client; extra == "hubspot"
 Requires-Dist: urllib3; extra == "hubspot"
+Requires-Dist: hubspot-api-client; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -115,19 +115,19 @@ Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: msg
 Requires-Dist: unstructured[msg]; extra == "msg"
 Provides-Extra: notion
+Requires-Dist: backoff; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: notion-client; extra == "notion"
-Requires-Dist: backoff; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
 Provides-Extra: odt
 Requires-Dist: unstructured[odt]; extra == "odt"
 Provides-Extra: onedrive
-Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
+Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Provides-Extra: openai
-Requires-Dist: openai; extra == "openai"
 Requires-Dist: tiktoken; extra == "openai"
+Requires-Dist: openai; extra == "openai"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: org
@@ -156,8 +156,8 @@ Requires-Dist: unstructured[rst]; extra == "rst"
 Provides-Extra: rtf
 Requires-Dist: unstructured[rtf]; extra == "rtf"
 Provides-Extra: s3
-Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: fsspec; extra == "s3"
+Requires-Dist: s3fs; extra == "s3"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp

{unstructured_ingest-0.3.1.dist-info → unstructured_ingest-0.3.2.dist-info}/RECORD

@@ -6,12 +6,13 @@ test/integration/chunkers/test_chunkers.py,sha256=pqn1Rqh36jZTJL4qpU0iuOMFAEQ-Lr
 test/integration/connectors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/integration/connectors/conftest.py,sha256=6dVNMBrL6WIO4KXA-0nf2tNrPYk_tsor8uomi6fbi3Q,727
 test/integration/connectors/test_astradb.py,sha256=QPFrODXmOHagpuKaiooxXb3OEW93w2g4fmq8BkaBCnY,5303
-test/integration/connectors/test_azure_cog_search.py,sha256=dae4GifRiKue5YpsxworDiaMQoMsxcPDBithb6OFkx4,8876
+test/integration/connectors/test_azure_ai_search.py,sha256=dae4GifRiKue5YpsxworDiaMQoMsxcPDBithb6OFkx4,8876
 test/integration/connectors/test_confluence.py,sha256=xcPmZ_vi_pkCt-tUPn10P49FH9i_9YUbrAPO6fYk5rU,3521
 test/integration/connectors/test_delta_table.py,sha256=GSzWIkbEUzOrRPt2F1uO0dabcp7kTFDj75BhhI2y-WU,6856
 test/integration/connectors/test_kafka.py,sha256=j7jsNWZumNBv9v-5Bpx8geUUXpxxad5EuA4CMRsl4R8,7104
-test/integration/connectors/test_lancedb.py,sha256=1EqdXOaA3gJqXDe1W-dHUzfOfeL1A4RB0oYwKvlfltg,7590
+test/integration/connectors/test_lancedb.py,sha256=O3YF6MVBkCsCgklXCJe8Kpy8aKGfafASVH4PspmpcYs,7628
 test/integration/connectors/test_milvus.py,sha256=CVmYw9iEeKT_0OtShxye2E6i1LbWzzDA8JtwJRkYQlA,4763
+test/integration/connectors/test_mongodb.py,sha256=YeS_DUnVYN02F76j87W8RhXGHnJMzQYb3n-L1-oWGXI,12254
 test/integration/connectors/test_onedrive.py,sha256=KIkBwKh1hnv203VCL2UABnDkS_bP4NxOFm1AL8EPGLA,3554
 test/integration/connectors/test_pinecone.py,sha256=X10OWZ6IrO6YyhuR3ydMAZOQq3u2f5u_lCjKNYUUcnI,7558
 test/integration/connectors/test_qdrant.py,sha256=ASvO-BNyhv8m8or28KljrJy27Da0uaTNeoR5w_QsvFg,5121
@@ -80,7 +81,7 @@ test/unit/v2/partitioners/test_partitioner.py,sha256=iIYg7IpftV3LusoO4H8tr1IHY1U
 test/unit/v2/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/unit/v2/utils/data_generator.py,sha256=UoYVNjG4S4wlaA9gceQ82HIpF9_6I1UTHD1_GrQBHp0,973
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=0bjUtHIzwwONNua74ouSySVzVv9qumqBMBxOWLE7Tbo,42
+unstructured_ingest/__version__.py,sha256=Js7MXQhyIj1akVjPNsLkmZxqoOHDGOr2opEPgFOSTZQ,42
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/interfaces.py,sha256=OYVUP0bzBJpT-Lz92BDyz_hLBvyfxkuSwWHhUdnUayA,31493
 unstructured_ingest/logger.py,sha256=S5nSqGcABoQyeicgRnBQFjDScCaTvFVivOCvbo-laL0,4479
@@ -400,11 +401,11 @@ unstructured_ingest/v2/processes/connectors/confluence.py,sha256=qQApDcmPBGg4tHX
 unstructured_ingest/v2/processes/connectors/couchbase.py,sha256=yhMDbpkZXs-Kis7tFlgjvNemU-MdWMdpCZDrpZNFaU4,12180
 unstructured_ingest/v2/processes/connectors/delta_table.py,sha256=1yS7ivEyiucwd_kv6LL5HQdGabT43yeG6XCdwiz89hc,8019
 unstructured_ingest/v2/processes/connectors/gitlab.py,sha256=yBgCeLy9iCVI8bBDcHHuHB0H3BO05e9E1OccbHwvKAo,9724
-unstructured_ingest/v2/processes/connectors/google_drive.py,sha256=a1BAvhX3nsgghjuR5CJ1lOwMtJ5ZJwimg6VtDYvluxA,13104
+unstructured_ingest/v2/processes/connectors/google_drive.py,sha256=EEwXK1Anlu-eXl5qxmdDIqPYW7eMSez6WGlTPG2vSn8,13121
 unstructured_ingest/v2/processes/connectors/kdbai.py,sha256=8bGHbZctJ_Tl1AUSMnI7CCZ7CgEtTRVcRuvlB1HPlqQ,5907
 unstructured_ingest/v2/processes/connectors/local.py,sha256=a3stgnIkhBbXPIQD0O-RaRM-Eb-szHj9Yy4Fz881-9c,6723
 unstructured_ingest/v2/processes/connectors/milvus.py,sha256=Bzv2fa852BcM4_Pr-I_DPvLmjPoXv0Z7BeEA8qSKCDc,9725
-unstructured_ingest/v2/processes/connectors/mongodb.py,sha256=A0pt6JcNTD5bEu79jZ8KhnHcBQ2VUJ2AjtQAtdFr_Lo,13175
+unstructured_ingest/v2/processes/connectors/mongodb.py,sha256=XLuprTCY0D9tAh_qn81MjJrDN9YaNqMlKe7BJl3eTZc,14998
 unstructured_ingest/v2/processes/connectors/onedrive.py,sha256=heZMtOIrCySi552ldIk8iH0pSRXZ0W2LeD-CcNOwCFQ,15979
 unstructured_ingest/v2/processes/connectors/outlook.py,sha256=KgNGM8hImRhy6_SpswRP2VwRD4VOrqqJoySgxf2oduI,9290
 unstructured_ingest/v2/processes/connectors/pinecone.py,sha256=hWkXgVDAzCtrBxf7A4HoexBACGAfVf_Qvn9YHbeiBSY,11505
@@ -451,14 +452,14 @@ unstructured_ingest/v2/processes/connectors/sql/singlestore.py,sha256=YrmhAL1RQ1
 unstructured_ingest/v2/processes/connectors/sql/snowflake.py,sha256=jl524VudwmFK63emCT7DmZan_EWJAMiGir5_zoO9FuY,5697
 unstructured_ingest/v2/processes/connectors/sql/sql.py,sha256=LFzGeAUagLknK07DsXg2oSG7ZAgR6VqT9wfI_tYlHUg,14782
 unstructured_ingest/v2/processes/connectors/sql/sqlite.py,sha256=9605K36nQ5-gBxzt1daYKYotON1SE85RETusqCJrbdk,5230
-unstructured_ingest/v2/processes/connectors/weaviate/__init__.py,sha256=NMiwnVWan69KnzVELvaqX34tMhCytIa-C8EDsXVKsEo,856
+unstructured_ingest/v2/processes/connectors/weaviate/__init__.py,sha256=eXamSnQdzzMvt62z80B8nmlkwDKO-Pogln_K_zLz53A,1067
 unstructured_ingest/v2/processes/connectors/weaviate/cloud.py,sha256=2g1Fm2J0ppfy2jCw4b5YtrsWrSD3VcrAaqiE7FlpIAg,6236
 unstructured_ingest/v2/processes/connectors/weaviate/embedded.py,sha256=S8Zg8StuZT-k7tCg1D5YShO1-vJYYk9-M1bE1fIqx64,3014
 unstructured_ingest/v2/processes/connectors/weaviate/local.py,sha256=LuTBKPseVewsz8VqxRPRLfGEm3BeI9nBZxpy7ZU5tOA,2201
-unstructured_ingest/v2/processes/connectors/weaviate/weaviate.py,sha256=dBDC_M8GVKupl7i9UMRCZyRIUv6gTkq8bJE_SILydAc,11291
-unstructured_ingest-0.3.1.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-0.3.1.dist-info/METADATA,sha256=gEXBJbX1y03XJgGGqXpNlkOw1PJ4IhEHmohj2CXHq9g,7326
-unstructured_ingest-0.3.1.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-unstructured_ingest-0.3.1.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-0.3.1.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
-unstructured_ingest-0.3.1.dist-info/RECORD,,
+unstructured_ingest/v2/processes/connectors/weaviate/weaviate.py,sha256=ln1p9ahFTaT-qsL7p4bgw_IqnU60As_l6vVAqUWyQVE,11655
+unstructured_ingest-0.3.2.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-0.3.2.dist-info/METADATA,sha256=rqTWqewB8eIrgrHJ-8AsNtehy35eSHKseCsveXTwN3Y,7326
+unstructured_ingest-0.3.2.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+unstructured_ingest-0.3.2.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-0.3.2.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
+unstructured_ingest-0.3.2.dist-info/RECORD,,