unstructured-ingest 0.0.25__py3-none-any.whl → 0.1.1__py3-none-any.whl

unstructured_ingest/embed/bedrock.py

@@ -3,7 +3,6 @@ import os
 from dataclasses import dataclass
 from typing import TYPE_CHECKING
 
-import numpy as np
 from pydantic import Field, SecretStr
 
 from unstructured_ingest.embed.interfaces import BaseEmbeddingEncoder, EmbeddingConfig
@@ -45,17 +44,6 @@ class BedrockEmbeddingConfig(EmbeddingConfig):
 class BedrockEmbeddingEncoder(BaseEmbeddingEncoder):
     config: BedrockEmbeddingConfig
 
-    def get_exemplary_embedding(self) -> list[float]:
-        return self.embed_query(query="Q")
-
-    def num_of_dimensions(self) -> tuple[int, ...]:
-        exemplary_embedding = self.get_exemplary_embedding()
-        return np.shape(exemplary_embedding)
-
-    def is_unit_vector(self) -> bool:
-        exemplary_embedding = self.get_exemplary_embedding()
-        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
-
     def embed_query(self, query: str) -> list[float]:
         """Call out to Bedrock embedding endpoint."""
         # replace newlines, which can negatively affect performance.
@@ -97,11 +85,3 @@ class BedrockEmbeddingEncoder(BaseEmbeddingEncoder):
         embeddings = [self.embed_query(query=e.get("text", "")) for e in elements]
         elements_with_embeddings = self._add_embeddings_to_elements(elements, embeddings)
         return elements_with_embeddings
-
-    def _add_embeddings_to_elements(self, elements, embeddings) -> list[dict]:
-        assert len(elements) == len(embeddings)
-        elements_w_embedding = []
-        for i, element in enumerate(elements):
-            element["embeddings"] = embeddings[i]
-            elements_w_embedding.append(element)
-        return elements

unstructured_ingest/embed/huggingface.py

@@ -1,7 +1,6 @@
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Optional
 
-import numpy as np
 from pydantic import Field
 
 from unstructured_ingest.embed.interfaces import BaseEmbeddingEncoder, EmbeddingConfig
@@ -39,17 +38,6 @@ class HuggingFaceEmbeddingConfig(EmbeddingConfig):
 class HuggingFaceEmbeddingEncoder(BaseEmbeddingEncoder):
     config: HuggingFaceEmbeddingConfig
 
-    def get_exemplary_embedding(self) -> list[float]:
-        return self.embed_query(query="Q")
-
-    def num_of_dimensions(self) -> tuple[int, ...]:
-        exemplary_embedding = self.get_exemplary_embedding()
-        return np.shape(exemplary_embedding)
-
-    def is_unit_vector(self) -> bool:
-        exemplary_embedding = self.get_exemplary_embedding()
-        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
-
     def embed_query(self, query: str) -> list[float]:
         return self._embed_documents(texts=[query])[0]
 
@@ -62,12 +50,3 @@ class HuggingFaceEmbeddingEncoder(BaseEmbeddingEncoder):
         embeddings = self._embed_documents([e.get("text", "") for e in elements])
         elements_with_embeddings = self._add_embeddings_to_elements(elements, embeddings)
         return elements_with_embeddings
-
-    def _add_embeddings_to_elements(self, elements: list[dict], embeddings: list) -> list[dict]:
-        assert len(elements) == len(embeddings)
-        elements_w_embedding = []
-
-        for i, element in enumerate(elements):
-            element["embeddings"] = embeddings[i]
-            elements_w_embedding.append(element)
-        return elements

unstructured_ingest/embed/interfaces.py

@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 
+import numpy as np
 from pydantic import BaseModel
 
 
@@ -17,14 +18,18 @@ class BaseEmbeddingEncoder(ABC):
         is properly configured: e.g., embed a single a element"""
 
     @property
-    @abstractmethod
     def num_of_dimensions(self) -> tuple[int, ...]:
-        """Number of dimensions for the embedding vector."""
+        exemplary_embedding = self.get_exemplary_embedding()
+        return np.shape(exemplary_embedding)
+
+    def get_exemplary_embedding(self) -> list[float]:
+        return self.embed_query(query="Q")
 
     @property
-    @abstractmethod
     def is_unit_vector(self) -> bool:
         """Denotes if the embedding vector is a unit vector."""
+        exemplary_embedding = self.get_exemplary_embedding()
+        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
 
     @abstractmethod
     def embed_documents(self, elements: list[dict]) -> list[dict]:
@@ -41,3 +46,24 @@ class BaseEmbeddingEncoder(ABC):
             results.append(response)
 
         return results
+
+    @staticmethod
+    def _add_embeddings_to_elements(
+        elements: list[dict], embeddings: list[list[float]]
+    ) -> list[dict]:
+        """
+        Add embeddings to elements.
+
+        Args:
+            elements (list[Element]): List of elements.
+            embeddings (list[list[float]]): List of embeddings.
+
+        Returns:
+            list[Element]: Elements with embeddings added.
+        """
+        assert len(elements) == len(embeddings)
+        elements_w_embedding = []
+        for i, element in enumerate(elements):
+            element["embeddings"] = embeddings[i]
+            elements_w_embedding.append(element)
+        return elements

unstructured_ingest/embed/mixedbreadai.py

@@ -2,7 +2,6 @@ import os
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Optional
 
-import numpy as np
 from pydantic import Field, SecretStr
 
 from unstructured_ingest.embed.interfaces import BaseEmbeddingEncoder, EmbeddingConfig
@@ -66,8 +65,6 @@ class MixedbreadAIEmbeddingEncoder(BaseEmbeddingEncoder):
     """
 
     config: MixedbreadAIEmbeddingConfig
-
-    _exemplary_embedding: Optional[list[float]] = field(init=False, default=None)
     _request_options: Optional["RequestOptions"] = field(init=False, default=None)
 
     def get_exemplary_embedding(self) -> list[float]:
@@ -90,18 +87,6 @@ class MixedbreadAIEmbeddingEncoder(BaseEmbeddingEncoder):
             additional_headers={"User-Agent": USER_AGENT},
         )
 
-    @property
-    def num_of_dimensions(self) -> tuple[int, ...]:
-        """Get the number of dimensions for the embeddings."""
-        exemplary_embedding = self.get_exemplary_embedding()
-        return np.shape(exemplary_embedding)
-
-    @property
-    def is_unit_vector(self) -> bool:
-        """Check if the embedding is a unit vector."""
-        exemplary_embedding = self.get_exemplary_embedding()
-        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
-
     def _embed(self, texts: list[str]) -> list[list[float]]:
         """
         Embed a list of texts using the Mixedbread AI API.
@@ -130,27 +115,6 @@ class MixedbreadAIEmbeddingEncoder(BaseEmbeddingEncoder):
             responses.append(response)
         return [item.embedding for response in responses for item in response.data]
 
-    @staticmethod
-    def _add_embeddings_to_elements(
-        elements: list[dict], embeddings: list[list[float]]
-    ) -> list[dict]:
-        """
-        Add embeddings to elements.
-
-        Args:
-            elements (list[Element]): List of elements.
-            embeddings (list[list[float]]): List of embeddings.
-
-        Returns:
-            list[Element]: Elements with embeddings added.
-        """
-        assert len(elements) == len(embeddings)
-        elements_w_embedding = []
-        for i, element in enumerate(elements):
-            element["embeddings"] = embeddings[i]
-            elements_w_embedding.append(element)
-        return elements
-
     def embed_documents(self, elements: list[dict]) -> list[dict]:
         """
         Embed a list of document elements.

unstructured_ingest/embed/octoai.py

@@ -1,7 +1,6 @@
-from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Optional
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
 
-import numpy as np
 from pydantic import Field, SecretStr
 
 from unstructured_ingest.embed.interfaces import BaseEmbeddingEncoder, EmbeddingConfig
@@ -30,19 +29,6 @@ class OctoAiEmbeddingConfig(EmbeddingConfig):
 @dataclass
 class OctoAIEmbeddingEncoder(BaseEmbeddingEncoder):
     config: OctoAiEmbeddingConfig
-    # Uses the OpenAI SDK
-    _exemplary_embedding: Optional[list[float]] = field(init=False, default=None)
-
-    def get_exemplary_embedding(self) -> list[float]:
-        return self.embed_query("Q")
-
-    def num_of_dimensions(self) -> tuple[int, ...]:
-        exemplary_embedding = self.get_exemplary_embedding()
-        return np.shape(exemplary_embedding)
-
-    def is_unit_vector(self) -> bool:
-        exemplary_embedding = self.get_exemplary_embedding()
-        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
 
     def embed_query(self, query: str):
         client = self.config.get_client()
@@ -53,11 +39,3 @@ class OctoAIEmbeddingEncoder(BaseEmbeddingEncoder):
         embeddings = [self.embed_query(e.get("text", "")) for e in elements]
         elements_with_embeddings = self._add_embeddings_to_elements(elements, embeddings)
         return elements_with_embeddings
-
-    def _add_embeddings_to_elements(self, elements, embeddings) -> list[dict]:
-        assert len(elements) == len(embeddings)
-        elements_w_embedding = []
-        for i, element in enumerate(elements):
-            element["embeddings"] = embeddings[i]
-            elements_w_embedding.append(element)
-        return elements

unstructured_ingest/embed/openai.py

@@ -1,7 +1,6 @@
 from dataclasses import dataclass
 from typing import TYPE_CHECKING
 
-import numpy as np
 from pydantic import Field, SecretStr
 
 from unstructured_ingest.embed.interfaces import BaseEmbeddingEncoder, EmbeddingConfig
@@ -26,17 +25,6 @@ class OpenAIEmbeddingConfig(EmbeddingConfig):
 class OpenAIEmbeddingEncoder(BaseEmbeddingEncoder):
     config: OpenAIEmbeddingConfig
 
-    def get_exemplary_embedding(self) -> list[float]:
-        return self.embed_query(query="Q")
-
-    def num_of_dimensions(self) -> tuple[int, ...]:
-        exemplary_embedding = self.get_exemplary_embedding()
-        return np.shape(exemplary_embedding)
-
-    def is_unit_vector(self) -> bool:
-        exemplary_embedding = self.get_exemplary_embedding()
-        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
-
     def embed_query(self, query: str) -> list[float]:
         client = self.config.get_client()
         response = client.embeddings.create(input=query, model=self.config.embedder_model_name)
@@ -46,11 +34,3 @@ class OpenAIEmbeddingEncoder(BaseEmbeddingEncoder):
         embeddings = self._embed_documents([e.get("text", "") for e in elements])
         elements_with_embeddings = self._add_embeddings_to_elements(elements, embeddings)
         return elements_with_embeddings
-
-    def _add_embeddings_to_elements(self, elements, embeddings) -> list[dict]:
-        assert len(elements) == len(embeddings)
-        elements_w_embedding = []
-        for i, element in enumerate(elements):
-            element["embeddings"] = embeddings[i]
-            elements_w_embedding.append(element)
-        return elements

unstructured_ingest/embed/togetherai.py

@@ -0,0 +1,40 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from pydantic import Field, SecretStr
+
+from unstructured_ingest.embed.interfaces import BaseEmbeddingEncoder, EmbeddingConfig
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if TYPE_CHECKING:
+    from together import Together
+
+
+class TogetherAIEmbeddingConfig(EmbeddingConfig):
+    api_key: SecretStr
+    embedder_model_name: str = Field(
+        default="togethercomputer/m2-bert-80M-8k-retrieval", alias="model_name"
+    )
+
+    @requires_dependencies(["together"], extras="togetherai")
+    def get_client(self) -> "Together":
+        from together import Together
+
+        return Together(api_key=self.api_key.get_secret_value())
+
+
+@dataclass
+class TogetherAIEmbeddingEncoder(BaseEmbeddingEncoder):
+    config: TogetherAIEmbeddingConfig
+
+    def embed_query(self, query: str) -> list[float]:
+        return self._embed_documents(elements=[query])[0]
+
+    def embed_documents(self, elements: list[dict]) -> list[dict]:
+        embeddings = self._embed_documents([e.get("text", "") for e in elements])
+        return self._add_embeddings_to_elements(elements, embeddings)
+
+    def _embed_documents(self, elements: list[str]) -> list[list[float]]:
+        client = self.config.get_client()
+        outputs = client.embeddings.create(model=self.config.embedder_model_name, input=elements)
+        return [outputs.data[i].embedding for i in range(len(elements))]

unstructured_ingest/embed/vertexai.py

@@ -5,7 +5,6 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING, Annotated, Any, Optional
 
-import numpy as np
 from pydantic import Field, Secret, ValidationError
 from pydantic.functional_validators import BeforeValidator
 
@@ -56,17 +55,6 @@ class VertexAIEmbeddingConfig(EmbeddingConfig):
 class VertexAIEmbeddingEncoder(BaseEmbeddingEncoder):
     config: VertexAIEmbeddingConfig
 
-    def get_exemplary_embedding(self) -> list[float]:
-        return self.embed_query(query="A sample query.")
-
-    def num_of_dimensions(self) -> tuple[int, ...]:
-        exemplary_embedding = self.get_exemplary_embedding()
-        return np.shape(exemplary_embedding)
-
-    def is_unit_vector(self) -> bool:
-        exemplary_embedding = self.get_exemplary_embedding()
-        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
-
     def embed_query(self, query):
         return self._embed_documents(elements=[query])[0]
 
@@ -86,11 +74,3 @@ class VertexAIEmbeddingEncoder(BaseEmbeddingEncoder):
         inputs = [TextEmbeddingInput(text=element) for element in elements]
         embeddings = client.get_embeddings(inputs)
         return [e.values for e in embeddings]
-
-    def _add_embeddings_to_elements(self, elements, embeddings) -> list[dict]:
-        assert len(elements) == len(embeddings)
-        elements_w_embedding = []
-        for i, element in enumerate(elements):
-            element["embeddings"] = embeddings[i]
-            elements_w_embedding.append(element)
-        return elements

unstructured_ingest/embed/voyageai.py

@@ -1,7 +1,6 @@
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Optional
 
-import numpy as np
 from pydantic import Field, SecretStr
 
 from unstructured_ingest.embed.interfaces import BaseEmbeddingEncoder, EmbeddingConfig
@@ -13,7 +12,7 @@ if TYPE_CHECKING:
 
 class VoyageAIEmbeddingConfig(EmbeddingConfig):
     api_key: SecretStr
-    embedder_model_name: str = Field(alias="model_name")
+    embedder_model_name: str = Field(default="voyage-3", alias="model_name")
     batch_size: Optional[int] = Field(default=None)
     truncation: Optional[bool] = Field(default=None)
     max_retries: int = 0
@@ -39,19 +38,6 @@ class VoyageAIEmbeddingConfig(EmbeddingConfig):
 class VoyageAIEmbeddingEncoder(BaseEmbeddingEncoder):
     config: VoyageAIEmbeddingConfig
 
-    def get_exemplary_embedding(self) -> list[float]:
-        return self.embed_query(query="A sample query.")
-
-    @property
-    def num_of_dimensions(self) -> tuple[int, ...]:
-        exemplary_embedding = self.get_exemplary_embedding()
-        return np.shape(exemplary_embedding)
-
-    @property
-    def is_unit_vector(self) -> bool:
-        exemplary_embedding = self.get_exemplary_embedding()
-        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
-
     def _embed_documents(self, elements: list[str]) -> list[list[float]]:
         client: VoyageAIClient = self.config.get_client()
         response = client.embed(texts=elements, model=self.config.embedder_model_name)
@@ -63,12 +49,3 @@ class VoyageAIEmbeddingEncoder(BaseEmbeddingEncoder):
 
     def embed_query(self, query: str) -> list[float]:
         return self._embed_documents(elements=[query])[0]
-
-    @staticmethod
-    def _add_embeddings_to_elements(elements, embeddings) -> list[dict]:
-        assert len(elements) == len(embeddings)
-        elements_w_embedding = []
-        for i, element in enumerate(elements):
-            element["embeddings"] = embeddings[i]
-            elements_w_embedding.append(element)
-        return elements

unstructured_ingest/interfaces.py

@@ -100,7 +100,7 @@ class PartitionConfig(BaseConfig):
     flatten_metadata: bool = False
     metadata_exclude: list[str] = field(default_factory=list)
     metadata_include: list[str] = field(default_factory=list)
-    partition_endpoint: Optional[str] = "https://api.unstructured.io/general/v0/general"
+    partition_endpoint: Optional[str] = "https://api.unstructuredapp.io/general/v0/general"
     partition_by_api: bool = False
     api_key: Optional[str] = str(enhanced_field(default=None, sensitive=True)) or None
     hi_res_model_name: Optional[str] = None

unstructured_ingest/v2/cli/utils/click.py

@@ -3,7 +3,7 @@ import os.path
 from gettext import gettext, ngettext
 from gettext import gettext as _
 from pathlib import Path
-from typing import Any, Optional, Type, TypeVar
+from typing import Any, Optional, Type, TypeVar, Union
 
 import click
 from pydantic import BaseModel, ConfigDict, Secret
@@ -112,6 +112,20 @@ class DelimitedString(click.ParamType):
 BaseModelT = TypeVar("BaseModelT", bound=BaseModel)
 
 
+def unwrap_optional(val: Any) -> tuple[Any, bool]:
+    if (
+        hasattr(val, "__origin__")
+        and hasattr(val, "__args__")
+        and val.__origin__ is Union
+        and len(val.__args__) == 2
+        and type(None) in val.__args__
+    ):
+        args = val.__args__
+        args = [a for a in args if a is not None]
+        return args[0], True
+    return val, False
+
+
 def extract_config(flat_data: dict, config: Type[BaseModelT]) -> BaseModelT:
     fields = config.model_fields
     config.model_config = ConfigDict(extra="ignore")
@@ -119,6 +133,7 @@ def extract_config(flat_data: dict, config: Type[BaseModelT]) -> BaseModelT:
     data = {k: v for k, v in flat_data.items() if k in field_names and v is not None}
     if access_config := fields.get("access_config"):
         access_config_type = access_config.annotation
+        access_config_type, is_optional = unwrap_optional(access_config_type)
         # Check if raw type is wrapped by a secret
         if (
             hasattr(access_config_type, "__origin__")
@@ -132,9 +147,13 @@ def extract_config(flat_data: dict, config: Type[BaseModelT]) -> BaseModelT:
         else:
             raise TypeError(f"Unrecognized access_config type: {access_config_type}")
         ac_field_names = [v.alias or k for k, v in ac_fields.items()]
-        data["access_config"] = {
+        access_config_data = {
             k: v for k, v in flat_data.items() if k in ac_field_names and v is not None
         }
+        if not access_config_data and is_optional:
+            data["access_config"] = None
+        else:
+            data["access_config"] = access_config_data
     return config.model_validate(obj=data)
 
 

unstructured_ingest/v2/interfaces/connector.py

@@ -1,8 +1,9 @@
 from abc import ABC
 from dataclasses import dataclass
-from typing import Any, TypeVar
+from typing import Any, TypeVar, Union
 
-from pydantic import BaseModel, Secret
+from pydantic import BaseModel, Secret, model_validator
+from pydantic.types import _SecretBase
 
 
 class AccessConfig(BaseModel):
@@ -21,6 +22,25 @@ class ConnectionConfig(BaseModel):
             return {}
         return self.access_config.get_secret_value().model_dump()
 
+    @model_validator(mode="after")
+    def check_access_config(self):
+        access_config = self.access_config
+        if self._is_access_config_optional() and access_config is None:
+            return self
+        if not isinstance(access_config, _SecretBase):
+            raise ValueError("access_config must be an instance of SecretBase")
+        return self
+
+    def _is_access_config_optional(self) -> bool:
+        access_config_type = self.model_fields["access_config"].annotation
+        return (
+            hasattr(access_config_type, "__origin__")
+            and hasattr(access_config_type, "__args__")
+            and access_config_type.__origin__ is Union
+            and len(access_config_type.__args__) == 2
+            and type(None) in access_config_type.__args__
+        )
+
 
 ConnectionConfigT = TypeVar("ConnectionConfigT", bound=ConnectionConfig)
 

unstructured_ingest/v2/interfaces/downloader.py

@@ -62,6 +62,7 @@ class Downloader(BaseProcess, BaseConnector, ABC):
             date_modified = float(file_data.metadata.date_modified)
             date_created = float(file_data.metadata.date_created)
             os.utime(download_path, times=(date_created, date_modified))
+        file_data.local_download_path = str(download_path.resolve())
         return DownloadResponse(file_data=file_data, path=download_path)
 
     @property

unstructured_ingest/v2/processes/chunker.py

@@ -20,7 +20,7 @@ class ChunkerConfig(BaseModel):
         default=None, description="The rule-set to use to form chunks. Omit to disable chunking."
     )
     chunking_endpoint: Optional[str] = Field(
-        default="https://api.unstructured.io/general/v0/general",
+        default="https://api.unstructuredapp.io/general/v0/general",
         description="If chunking via api, use the following host.",
     )
     chunk_by_api: bool = Field(default=False, description="Flag to use api for chunking")

unstructured_ingest/v2/processes/connectors/__init__.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
+import unstructured_ingest.v2.processes.connectors.databricks  # noqa: F401
 import unstructured_ingest.v2.processes.connectors.fsspec  # noqa: F401
+import unstructured_ingest.v2.processes.connectors.sql  # noqa: F401
 from unstructured_ingest.v2.processes.connector_registry import (
     add_destination_entry,
     add_source_entry,
@@ -16,11 +18,6 @@ from .chroma import CONNECTOR_TYPE as CHROMA_CONNECTOR_TYPE
 from .chroma import chroma_destination_entry
 from .couchbase import CONNECTOR_TYPE as COUCHBASE_CONNECTOR_TYPE
 from .couchbase import couchbase_destination_entry, couchbase_source_entry
-from .databricks_volumes import CONNECTOR_TYPE as DATABRICKS_VOLUMES_CONNECTOR_TYPE
-from .databricks_volumes import (
-    databricks_volumes_destination_entry,
-    databricks_volumes_source_entry,
-)
 from .elasticsearch import CONNECTOR_TYPE as ELASTICSEARCH_CONNECTOR_TYPE
 from .elasticsearch import elasticsearch_destination_entry, elasticsearch_source_entry
 from .google_drive import CONNECTOR_TYPE as GOOGLE_DRIVE_CONNECTOR_TYPE
@@ -32,7 +29,7 @@ from .local import local_destination_entry, local_source_entry
 from .milvus import CONNECTOR_TYPE as MILVUS_CONNECTOR_TYPE
 from .milvus import milvus_destination_entry
 from .mongodb import CONNECTOR_TYPE as MONGODB_CONNECTOR_TYPE
-from .mongodb import mongodb_destination_entry
+from .mongodb import mongodb_destination_entry, mongodb_source_entry
 from .onedrive import CONNECTOR_TYPE as ONEDRIVE_CONNECTOR_TYPE
 from .onedrive import onedrive_source_entry
 from .opensearch import CONNECTOR_TYPE as OPENSEARCH_CONNECTOR_TYPE
@@ -47,8 +44,6 @@ from .sharepoint import CONNECTOR_TYPE as SHAREPOINT_CONNECTOR_TYPE
 from .sharepoint import sharepoint_source_entry
 from .singlestore import CONNECTOR_TYPE as SINGLESTORE_CONNECTOR_TYPE
 from .singlestore import singlestore_destination_entry
-from .sql import CONNECTOR_TYPE as SQL_CONNECTOR_TYPE
-from .sql import sql_destination_entry
 from .weaviate import CONNECTOR_TYPE as WEAVIATE_CONNECTOR_TYPE
 from .weaviate import weaviate_destination_entry
 
@@ -80,17 +75,9 @@ add_source_entry(source_type=SALESFORCE_CONNECTOR_TYPE, entry=salesforce_source_
 
 add_destination_entry(destination_type=WEAVIATE_CONNECTOR_TYPE, entry=weaviate_destination_entry)
 
-add_destination_entry(
-    destination_type=DATABRICKS_VOLUMES_CONNECTOR_TYPE, entry=databricks_volumes_destination_entry
-)
-add_source_entry(
-    source_type=DATABRICKS_VOLUMES_CONNECTOR_TYPE, entry=databricks_volumes_source_entry
-)
-
-
-add_destination_entry(destination_type=SQL_CONNECTOR_TYPE, entry=sql_destination_entry)
-
 add_destination_entry(destination_type=MONGODB_CONNECTOR_TYPE, entry=mongodb_destination_entry)
+add_source_entry(source_type=MONGODB_CONNECTOR_TYPE, entry=mongodb_source_entry)
+
 add_destination_entry(destination_type=PINECONE_CONNECTOR_TYPE, entry=pinecone_destination_entry)
 add_source_entry(source_type=SHAREPOINT_CONNECTOR_TYPE, entry=sharepoint_source_entry)
 add_destination_entry(

unstructured_ingest/v2/processes/connectors/databricks/__init__.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from unstructured_ingest.v2.processes.connector_registry import (
+    add_destination_entry,
+    add_source_entry,
+)
+
+from .volumes_aws import CONNECTOR_TYPE as VOLUMES_AWS_CONNECTOR_TYPE
+from .volumes_aws import (
+    databricks_aws_volumes_destination_entry,
+    databricks_aws_volumes_source_entry,
+)
+from .volumes_azure import CONNECTOR_TYPE as VOLUMES_AZURE_CONNECTOR_TYPE
+from .volumes_azure import (
+    databricks_azure_volumes_destination_entry,
+    databricks_azure_volumes_source_entry,
+)
+from .volumes_gcp import CONNECTOR_TYPE as VOLUMES_GCP_CONNECTOR_TYPE
+from .volumes_gcp import (
+    databricks_gcp_volumes_destination_entry,
+    databricks_gcp_volumes_source_entry,
+)
+from .volumes_native import CONNECTOR_TYPE as VOLUMES_NATIVE_CONNECTOR_TYPE
+from .volumes_native import (
+    databricks_native_volumes_destination_entry,
+    databricks_native_volumes_source_entry,
+)
+
+add_source_entry(source_type=VOLUMES_AWS_CONNECTOR_TYPE, entry=databricks_aws_volumes_source_entry)
+add_destination_entry(
+    destination_type=VOLUMES_AWS_CONNECTOR_TYPE, entry=databricks_aws_volumes_destination_entry
+)
+
+add_source_entry(source_type=VOLUMES_GCP_CONNECTOR_TYPE, entry=databricks_gcp_volumes_source_entry)
+add_destination_entry(
+    destination_type=VOLUMES_GCP_CONNECTOR_TYPE, entry=databricks_gcp_volumes_destination_entry
+)
+
+add_source_entry(
+    source_type=VOLUMES_NATIVE_CONNECTOR_TYPE, entry=databricks_native_volumes_source_entry
+)
+add_destination_entry(
+    destination_type=VOLUMES_NATIVE_CONNECTOR_TYPE,
+    entry=databricks_native_volumes_destination_entry,
+)
+
+add_source_entry(
+    source_type=VOLUMES_AZURE_CONNECTOR_TYPE, entry=databricks_azure_volumes_source_entry
+)
+add_destination_entry(
+    destination_type=VOLUMES_AZURE_CONNECTOR_TYPE, entry=databricks_azure_volumes_destination_entry
+)