unstructured-ingest 0.3.8__py3-none-any.whl → 0.3.10__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/mongodb.py

@@ -1,13 +1,10 @@
-import json
-import sys
 from contextlib import contextmanager
-from dataclasses import dataclass, replace
+from dataclasses import dataclass
 from datetime import datetime
-from pathlib import Path
 from time import time
 from typing import TYPE_CHECKING, Any, Generator, Optional
 
-from pydantic import Field, Secret
+from pydantic import BaseModel, Field, Secret
 
 from unstructured_ingest.__version__ import __version__ as unstructured_version
 from unstructured_ingest.error import DestinationConnectionError, SourceConnectionError
@@ -16,9 +13,12 @@ from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
+    BatchFileData,
+    BatchItem,
     ConnectionConfig,
     Downloader,
     DownloaderConfig,
+    DownloadResponse,
     FileData,
     FileDataSourceMetadata,
     Indexer,
@@ -42,6 +42,15 @@ CONNECTOR_TYPE = "mongodb"
 SERVER_API_VERSION = "1"
 
 
+class MongoDBAdditionalMetadata(BaseModel):
+    database: str
+    collection: str
+
+
+class MongoDBBatchFileData(BatchFileData):
+    additional_metadata: MongoDBAdditionalMetadata
+
+
 class MongoDBAccessConfig(AccessConfig):
     uri: Optional[str] = Field(default=None, description="URI to user when connecting")
 
@@ -124,7 +133,7 @@ class MongoDBIndexer(Indexer):
             logger.error(f"Failed to validate connection: {e}", exc_info=True)
             raise SourceConnectionError(f"Failed to validate connection: {e}")
 
-    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+    def run(self, **kwargs: Any) -> Generator[BatchFileData, None, None]:
         """Generates FileData objects for each document in the MongoDB collection."""
         with self.connection_config.get_client() as client:
             database = client[self.index_config.database]
@@ -132,12 +141,12 @@ class MongoDBIndexer(Indexer):
 
             # Get list of document IDs
             ids = collection.distinct("_id")
-            batch_size = self.index_config.batch_size if self.index_config else 100
+
+        ids = sorted(ids)
+        batch_size = self.index_config.batch_size
 
         for id_batch in batch_generator(ids, batch_size=batch_size):
             # Make sure the hash is always a positive number to create identifier
-            batch_id = str(hash(frozenset(id_batch)) + sys.maxsize + 1)
-
             metadata = FileDataSourceMetadata(
                 date_processed=str(time()),
                 record_locator={
@@ -146,14 +155,13 @@ class MongoDBIndexer(Indexer):
                 },
             )
 
-            file_data = FileData(
-                identifier=batch_id,
-                doc_type="batch",
+            file_data = MongoDBBatchFileData(
                 connector_type=self.connector_type,
                 metadata=metadata,
-                additional_metadata={
-                    "ids": [str(doc_id) for doc_id in id_batch],
-                },
+                batch_items=[BatchItem(identifier=str(doc_id)) for doc_id in id_batch],
+                additional_metadata=MongoDBAdditionalMetadata(
+                    collection=self.index_config.collection, database=self.index_config.database
+                ),
             )
             yield file_data
 
@@ -164,26 +172,59 @@ class MongoDBDownloader(Downloader):
     connection_config: MongoDBConnectionConfig
     connector_type: str = CONNECTOR_TYPE
 
-    @requires_dependencies(["pymongo"], extras="mongodb")
-    def create_client(self) -> "MongoClient":
-        from pymongo import MongoClient
-        from pymongo.driver_info import DriverInfo
-        from pymongo.server_api import ServerApi
+    def generate_download_response(
+        self, doc: dict, file_data: MongoDBBatchFileData
+    ) -> DownloadResponse:
+        from bson.objectid import ObjectId
 
-        access_config = self.connection_config.access_config.get_secret_value()
+        doc_id = doc["_id"]
+        doc.pop("_id", None)
 
-        if access_config.uri:
-            return MongoClient(
-                access_config.uri,
-                server_api=ServerApi(version=SERVER_API_VERSION),
-                driver=DriverInfo(name="unstructured", version=unstructured_version),
-            )
-        else:
-            return MongoClient(
-                host=self.connection_config.host,
-                port=self.connection_config.port,
-                server_api=ServerApi(version=SERVER_API_VERSION),
-            )
+        # Extract date_created from the document or ObjectId
+        date_created = None
+        if "date_created" in doc:
+            # If the document has a 'date_created' field, use it
+            date_created = doc["date_created"]
+            if isinstance(date_created, datetime):
+                date_created = date_created.isoformat()
+            else:
+                # Convert to ISO format if it's a string
+                date_created = str(date_created)
+        elif isinstance(doc_id, ObjectId):
+            # Use the ObjectId's generation time
+            date_created = doc_id.generation_time.isoformat()
+
+        flattened_dict = flatten_dict(dictionary=doc)
+        concatenated_values = "\n".join(str(value) for value in flattened_dict.values())
+
+        # Create a FileData object for each document with source_identifiers
+        cast_file_data = FileData.cast(file_data=file_data)
+        cast_file_data.identifier = str(doc_id)
+        filename = f"{doc_id}.txt"
+        cast_file_data.source_identifiers = SourceIdentifiers(
+            filename=filename,
+            fullpath=filename,
+            rel_path=filename,
+        )
+
+        # Determine the download path
+        download_path = self.get_download_path(file_data=cast_file_data)
+        if download_path is None:
+            raise ValueError("Download path could not be determined")
+
+        download_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Write the concatenated values to the file
+        with open(download_path, "w", encoding="utf8") as f:
+            f.write(concatenated_values)
+
+        # Update metadata
+        cast_file_data.metadata.record_locator["document_id"] = str(doc_id)
+        cast_file_data.metadata.date_created = date_created
+
+        return super().generate_download_response(
+            file_data=cast_file_data, download_path=download_path
+        )
 
     @SourceConnectionError.wrap
     @requires_dependencies(["bson"], extras="mongodb")
@@ -192,82 +233,34 @@ class MongoDBDownloader(Downloader):
         from bson.errors import InvalidId
         from bson.objectid import ObjectId
 
-        client = self.create_client()
-        database = client[file_data.metadata.record_locator["database"]]
-        collection = database[file_data.metadata.record_locator["collection"]]
+        mongo_file_data = MongoDBBatchFileData.cast(file_data=file_data)
 
-        ids = file_data.additional_metadata.get("ids", [])
-        if not ids:
-            raise ValueError("No document IDs provided in additional_metadata")
+        with self.connection_config.get_client() as client:
+            database = client[mongo_file_data.additional_metadata.database]
+            collection = database[mongo_file_data.additional_metadata.collection]
 
-        object_ids = []
-        for doc_id in ids:
-            try:
-                object_ids.append(ObjectId(doc_id))
-            except InvalidId as e:
-                error_message = f"Invalid ObjectId for doc_id '{doc_id}': {str(e)}"
-                logger.error(error_message)
-                raise ValueError(error_message) from e
+            ids = [item.identifier for item in mongo_file_data.batch_items]
 
-        try:
-            docs = list(collection.find({"_id": {"$in": object_ids}}))
-        except Exception as e:
-            logger.error(f"Failed to fetch documents: {e}", exc_info=True)
-            raise e
+            object_ids = []
+            for doc_id in ids:
+                try:
+                    object_ids.append(ObjectId(doc_id))
+                except InvalidId as e:
+                    error_message = f"Invalid ObjectId for doc_id '{doc_id}': {str(e)}"
+                    logger.error(error_message)
+                    raise ValueError(error_message) from e
+
+            try:
+                docs = list(collection.find({"_id": {"$in": object_ids}}))
+            except Exception as e:
+                logger.error(f"Failed to fetch documents: {e}", exc_info=True)
+                raise e
 
         download_responses = []
         for doc in docs:
-            doc_id = doc["_id"]
-            doc.pop("_id", None)
-
-            # Extract date_created from the document or ObjectId
-            date_created = None
-            if "date_created" in doc:
-                # If the document has a 'date_created' field, use it
-                date_created = doc["date_created"]
-                if isinstance(date_created, datetime):
-                    date_created = date_created.isoformat()
-                else:
-                    # Convert to ISO format if it's a string
-                    date_created = str(date_created)
-            elif isinstance(doc_id, ObjectId):
-                # Use the ObjectId's generation time
-                date_created = doc_id.generation_time.isoformat()
-
-            flattened_dict = flatten_dict(dictionary=doc)
-            concatenated_values = "\n".join(str(value) for value in flattened_dict.values())
-
-            # Create a FileData object for each document with source_identifiers
-            individual_file_data = replace(file_data)
-            individual_file_data.identifier = str(doc_id)
-            individual_file_data.source_identifiers = SourceIdentifiers(
-                filename=str(doc_id),
-                fullpath=str(doc_id),
-                rel_path=str(doc_id),
-            )
-
-            # Determine the download path
-            download_path = self.get_download_path(individual_file_data)
-            if download_path is None:
-                raise ValueError("Download path could not be determined")
-
-            download_path.parent.mkdir(parents=True, exist_ok=True)
-            download_path = download_path.with_suffix(".txt")
-
-            # Write the concatenated values to the file
-            with open(download_path, "w", encoding="utf8") as f:
-                f.write(concatenated_values)
-
-            individual_file_data.local_download_path = str(download_path)
-
-            # Update metadata
-            individual_file_data.metadata.record_locator["document_id"] = str(doc_id)
-            individual_file_data.metadata.date_created = date_created
-
-            download_response = self.generate_download_response(
-                file_data=individual_file_data, download_path=download_path
+            download_responses.append(
+                self.generate_download_response(doc=doc, file_data=mongo_file_data)
             )
-            download_responses.append(download_response)
 
         return download_responses
 
@@ -332,18 +325,16 @@ class MongoDBUploader(Uploader):
             f"deleted {delete_results.deleted_count} records from collection {collection.name}"
         )
 
-    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
-        with path.open("r") as file:
-            elements_dict = json.load(file)
+    def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
         logger.info(
-            f"writing {len(elements_dict)} objects to destination "
+            f"writing {len(data)} objects to destination "
             f"db, {self.upload_config.database}, "
             f"collection {self.upload_config.collection} "
             f"at {self.connection_config.host}",
         )
         # This would typically live in the stager but since no other manipulation
         # is done, setting the record id field in the uploader
-        for element in elements_dict:
+        for element in data:
             element[self.upload_config.record_id_key] = file_data.identifier
         with self.connection_config.get_client() as client:
             db = client[self.upload_config.database]
@@ -352,7 +343,7 @@ class MongoDBUploader(Uploader):
                 self.delete_by_record_id(file_data=file_data, collection=collection)
             else:
                 logger.warning("criteria for deleting previous content not met, skipping")
-            for chunk in batch_generator(elements_dict, self.upload_config.batch_size):
+            for chunk in batch_generator(data, self.upload_config.batch_size):
                 collection.insert_many(chunk)
 
 

unstructured_ingest/v2/processes/connectors/neo4j.py

@@ -0,0 +1,383 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import uuid
+from collections import defaultdict
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, Secret
+
+from unstructured_ingest.error import DestinationConnectionError
+from unstructured_ingest.logger import logger
+from unstructured_ingest.utils.chunking import elements_from_base64_gzipped_json
+from unstructured_ingest.utils.data_prep import batch_generator
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    FileData,
+    Uploader,
+    UploaderConfig,
+    UploadStager,
+    UploadStagerConfig,
+)
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+)
+
+if TYPE_CHECKING:
+    from neo4j import AsyncDriver, Auth
+    from networkx import Graph, MultiDiGraph
+
+CONNECTOR_TYPE = "neo4j"
+
+
+class Neo4jAccessConfig(AccessConfig):
+    password: str
+
+
+class Neo4jConnectionConfig(ConnectionConfig):
+    access_config: Secret[Neo4jAccessConfig]
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+    username: str
+    uri: str = Field(description="Neo4j Connection URI <scheme>://<host>:<port>")
+    database: str = Field(description="Name of the target database")
+
+    @requires_dependencies(["neo4j"], extras="neo4j")
+    @asynccontextmanager
+    async def get_client(self) -> AsyncGenerator["AsyncDriver", None]:
+        from neo4j import AsyncGraphDatabase
+
+        driver = AsyncGraphDatabase.driver(**self._get_driver_parameters())
+        logger.info(f"Created driver connecting to the database '{self.database}' at {self.uri}.")
+        try:
+            yield driver
+        finally:
+            await driver.close()
+            logger.info(
+                f"Closed driver connecting to the database '{self.database}' at {self.uri}."
+            )
+
+    def _get_driver_parameters(self) -> dict:
+        return {
+            "uri": self.uri,
+            "auth": self._get_auth(),
+            "database": self.database,
+        }
+
+    @requires_dependencies(["neo4j"], extras="neo4j")
+    def _get_auth(self) -> "Auth":
+        from neo4j import Auth
+
+        return Auth("basic", self.username, self.access_config.get_secret_value().password)
+
+
+class Neo4jUploadStagerConfig(UploadStagerConfig):
+    pass
+
+
+@dataclass
+class Neo4jUploadStager(UploadStager):
+    upload_stager_config: Neo4jUploadStagerConfig = Field(
+        default_factory=Neo4jUploadStagerConfig, validate_default=True
+    )
+
+    def run(  # type: ignore
+        self,
+        elements_filepath: Path,
+        file_data: FileData,
+        output_dir: Path,
+        output_filename: str,
+        **kwargs: Any,
+    ) -> Path:
+        with elements_filepath.open() as file:
+            elements = json.load(file)
+
+        nx_graph = self._create_lexical_graph(
+            elements, self._create_document_node(file_data=file_data)
+        )
+        output_filepath = Path(output_dir) / f"{output_filename}.json"
+        output_filepath.parent.mkdir(parents=True, exist_ok=True)
+
+        with open(output_filepath, "w") as file:
+            json.dump(_GraphData.from_nx(nx_graph).model_dump(), file, indent=4)
+
+        return output_filepath
+
+    def _create_lexical_graph(self, elements: list[dict], document_node: _Node) -> "Graph":
+        import networkx as nx
+
+        graph = nx.MultiDiGraph()
+        graph.add_node(document_node)
+
+        previous_node: Optional[_Node] = None
+        for element in elements:
+            element_node = self._create_element_node(element)
+            order_relationship = (
+                Relationship.NEXT_CHUNK if self._is_chunk(element) else Relationship.NEXT_ELEMENT
+            )
+            if previous_node:
+                graph.add_edge(element_node, previous_node, relationship=order_relationship)
+
+            previous_node = element_node
+            graph.add_edge(element_node, document_node, relationship=Relationship.PART_OF_DOCUMENT)
+
+            if self._is_chunk(element):
+                origin_element_nodes = [
+                    self._create_element_node(origin_element)
+                    for origin_element in self._get_origin_elements(element)
+                ]
+                graph.add_edges_from(
+                    [
+                        (origin_element_node, element_node)
+                        for origin_element_node in origin_element_nodes
+                    ],
+                    relationship=Relationship.PART_OF_CHUNK,
+                )
+                graph.add_edges_from(
+                    [
+                        (origin_element_node, document_node)
+                        for origin_element_node in origin_element_nodes
+                    ],
+                    relationship=Relationship.PART_OF_DOCUMENT,
+                )
+
+        return graph
+
+    # TODO(Filip Knefel): Ensure _is_chunk is as reliable as possible, consider different checks
+    def _is_chunk(self, element: dict) -> bool:
+        return "orig_elements" in element.get("metadata", {})
+
+    def _create_document_node(self, file_data: FileData) -> _Node:
+        properties = {}
+        if file_data.source_identifiers:
+            properties["name"] = file_data.source_identifiers.filename
+        if file_data.metadata.date_created:
+            properties["date_created"] = file_data.metadata.date_created
+        if file_data.metadata.date_modified:
+            properties["date_modified"] = file_data.metadata.date_modified
+        return _Node(id_=file_data.identifier, properties=properties, labels=[Label.DOCUMENT])
+
+    def _create_element_node(self, element: dict) -> _Node:
+        properties = {"id": element["element_id"], "text": element["text"]}
+
+        if embeddings := element.get("embeddings"):
+            properties["embeddings"] = embeddings
+
+        label = Label.CHUNK if self._is_chunk(element) else Label.UNSTRUCTURED_ELEMENT
+        return _Node(id_=element["element_id"], properties=properties, labels=[label])
+
+    def _get_origin_elements(self, chunk_element: dict) -> list[dict]:
+        orig_elements = chunk_element.get("metadata", {}).get("orig_elements")
+        return elements_from_base64_gzipped_json(raw_s=orig_elements)
+
+
+class _GraphData(BaseModel):
+    nodes: list[_Node]
+    edges: list[_Edge]
+
+    @classmethod
+    def from_nx(cls, nx_graph: "MultiDiGraph") -> _GraphData:
+        nodes = list(nx_graph.nodes())
+        edges = [
+            _Edge(
+                source_id=u.id_,
+                destination_id=v.id_,
+                relationship=Relationship(data_dict["relationship"]),
+            )
+            for u, v, data_dict in nx_graph.edges(data=True)
+        ]
+        return _GraphData(nodes=nodes, edges=edges)
+
+
+class _Node(BaseModel):
+    model_config = ConfigDict(use_enum_values=True)
+
+    id_: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    labels: list[Label] = Field(default_factory=list)
+    properties: dict = Field(default_factory=dict)
+
+    def __hash__(self):
+        return hash(self.id_)
+
+
+class _Edge(BaseModel):
+    model_config = ConfigDict(use_enum_values=True)
+
+    source_id: str
+    destination_id: str
+    relationship: Relationship
+
+
+class Label(str, Enum):
+    UNSTRUCTURED_ELEMENT = "UnstructuredElement"
+    CHUNK = "Chunk"
+    DOCUMENT = "Document"
+
+
+class Relationship(str, Enum):
+    PART_OF_DOCUMENT = "PART_OF_DOCUMENT"
+    PART_OF_CHUNK = "PART_OF_CHUNK"
+    NEXT_CHUNK = "NEXT_CHUNK"
+    NEXT_ELEMENT = "NEXT_ELEMENT"
+
+
+class Neo4jUploaderConfig(UploaderConfig):
+    batch_size: int = Field(
+        default=100, description="Maximal number of nodes/relationships created per transaction."
+    )
+
+
+@dataclass
+class Neo4jUploader(Uploader):
+    upload_config: Neo4jUploaderConfig
+    connection_config: Neo4jConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    @DestinationConnectionError.wrap
+    def precheck(self) -> None:
+        async def verify_auth():
+            async with self.connection_config.get_client() as client:
+                await client.verify_connectivity()
+
+        asyncio.run(verify_auth())
+
+    def is_async(self):
+        return True
+
+    async def run_async(self, path: Path, file_data: FileData, **kwargs) -> None:  # type: ignore
+        with path.open() as file:
+            staged_data = json.load(file)
+
+        graph_data = _GraphData.model_validate(staged_data)
+        async with self.connection_config.get_client() as client:
+            await self._create_uniqueness_constraints(client)
+            await self._delete_old_data_if_exists(file_data, client=client)
+            await self._merge_graph(graph_data=graph_data, client=client)
+
+    async def _create_uniqueness_constraints(self, client: AsyncDriver) -> None:
+        for label in Label:
+            logger.info(
+                f"Adding id uniqueness constraint for nodes labeled '{label}'"
+                " if it does not already exist."
+            )
+            constraint_name = f"{label.lower()}_id"
+            await client.execute_query(
+                f"""
+                CREATE CONSTRAINT {constraint_name} IF NOT EXISTS
+                FOR (n: {label}) REQUIRE n.id IS UNIQUE
+                """
+            )
+
+    async def _delete_old_data_if_exists(self, file_data: FileData, client: AsyncDriver) -> None:
+        logger.info(f"Deleting old data for the record '{file_data.identifier}' (if present).")
+        _, summary, _ = await client.execute_query(
+            f"""
+            MATCH (n: {Label.DOCUMENT} {{id: $identifier}})
+            MATCH (n)--(m: {Label.CHUNK}|{Label.UNSTRUCTURED_ELEMENT})
+            DETACH DELETE m""",
+            identifier=file_data.identifier,
+        )
+        logger.info(
+            f"Deleted {summary.counters.nodes_deleted} nodes"
+            f" and {summary.counters.relationships_deleted} relationships."
+        )
+
+    async def _merge_graph(self, graph_data: _GraphData, client: AsyncDriver) -> None:
+        nodes_by_labels: defaultdict[tuple[Label, ...], list[_Node]] = defaultdict(list)
+        for node in graph_data.nodes:
+            nodes_by_labels[tuple(node.labels)].append(node)
+
+        logger.info(f"Merging {len(graph_data.nodes)} graph nodes.")
+        # NOTE: Processed in parallel as there's no overlap between accessed nodes
+        await self._execute_queries(
+            [
+                self._create_nodes_query(nodes_batch, labels)
+                for labels, nodes in nodes_by_labels.items()
+                for nodes_batch in batch_generator(nodes, batch_size=self.upload_config.batch_size)
+            ],
+            client=client,
+            in_parallel=True,
+        )
+        logger.info(f"Finished merging {len(graph_data.nodes)} graph nodes.")
+
+        edges_by_relationship: defaultdict[Relationship, list[_Edge]] = defaultdict(list)
+        for edge in graph_data.edges:
+            edges_by_relationship[edge.relationship].append(edge)
+
+        logger.info(f"Merging {len(graph_data.edges)} graph relationships (edges).")
+        # NOTE: Processed sequentially to avoid queries locking node access to one another
+        await self._execute_queries(
+            [
+                self._create_edges_query(edges_batch, relationship)
+                for relationship, edges in edges_by_relationship.items()
+                for edges_batch in batch_generator(edges, batch_size=self.upload_config.batch_size)
+            ],
+            client=client,
+        )
+        logger.info(f"Finished merging {len(graph_data.edges)} graph relationships (edges).")
+
+    @staticmethod
+    async def _execute_queries(
+        queries_with_parameters: list[tuple[str, dict]],
+        client: AsyncDriver,
+        in_parallel: bool = False,
+    ) -> None:
+        if in_parallel:
+            logger.info(f"Executing {len(queries_with_parameters)} queries in parallel.")
+            await asyncio.gather(
+                *[
+                    client.execute_query(query, parameters_=parameters)
+                    for query, parameters in queries_with_parameters
+                ]
+            )
+            logger.info("Finished executing parallel queries.")
+        else:
+            logger.info(f"Executing {len(queries_with_parameters)} queries sequentially.")
+            for i, (query, parameters) in enumerate(queries_with_parameters):
+                logger.info(f"Query #{i} started.")
+                await client.execute_query(query, parameters_=parameters)
+                logger.info(f"Query #{i} finished.")
+            logger.info(
+                f"Finished executing all ({len(queries_with_parameters)}) sequential queries."
+            )
+
+    @staticmethod
+    def _create_nodes_query(nodes: list[_Node], labels: tuple[Label, ...]) -> tuple[str, dict]:
+        labels_string = ", ".join(labels)
+        logger.info(f"Preparing MERGE query for {len(nodes)} nodes labeled '{labels_string}'.")
+        query_string = f"""
+            UNWIND $nodes AS node
+            MERGE (n: {labels_string} {{id: node.id}})
+            SET n += node.properties
+            """
+        parameters = {"nodes": [{"id": node.id_, "properties": node.properties} for node in nodes]}
+        return query_string, parameters
+
+    @staticmethod
+    def _create_edges_query(edges: list[_Edge], relationship: Relationship) -> tuple[str, dict]:
+        logger.info(f"Preparing MERGE query for {len(edges)} {relationship} relationships.")
+        query_string = f"""
+            UNWIND $edges AS edge
+            MATCH (u {{id: edge.source}})
+            MATCH (v {{id: edge.destination}})
+            MERGE (u)-[:{relationship}]->(v)
+            """
+        parameters = {
+            "edges": [
+                {"source": edge.source_id, "destination": edge.destination_id} for edge in edges
+            ]
+        }
+        return query_string, parameters
+
+
+neo4j_destination_entry = DestinationRegistryEntry(
+    connection_config=Neo4jConnectionConfig,
+    uploader=Neo4jUploader,
+    uploader_config=Neo4jUploaderConfig,
+)

unstructured_ingest/v2/processes/connectors/onedrive.py

@@ -202,7 +202,7 @@ class OnedriveDownloader(Downloader):
         if file_data.source_identifiers is None or not file_data.source_identifiers.fullpath:
             raise ValueError(
                 f"file data doesn't have enough information to get "
-                f"file content: {file_data.to_dict()}"
+                f"file content: {file_data.model_dump()}"
             )
 
         server_relative_path = file_data.source_identifiers.fullpath