unstructured-ingest 0.5.20__py3-none-any.whl → 0.5.23__py3-none-any.whl

test/integration/connectors/test_astradb.py

@@ -1,3 +1,4 @@
+import contextlib
 import json
 import os
 from dataclasses import dataclass
@@ -231,6 +232,13 @@ def test_astra_create_destination():
     )
     collection_name = "system_created-123"
     formatted_collection_name = "system_created_123"
+
+    client = AstraDBClient()
+    db = client.get_database(api_endpoint=env_data.api_endpoint, token=env_data.token)
+    with contextlib.suppress(Exception):
+        # drop collection before trying to create it
+        db.drop_collection(formatted_collection_name)
+
     created = uploader.create_destination(destination_name=collection_name, vector_length=3072)
     assert created
     assert uploader.upload_config.collection_name == formatted_collection_name
@@ -239,8 +247,6 @@ def test_astra_create_destination():
     assert not created
 
     # cleanup
-    client = AstraDBClient()
-    db = client.get_database(api_endpoint=env_data.api_endpoint, token=env_data.token)
     db.drop_collection(formatted_collection_name)
 
 

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.5.20"  # pragma: no cover
+__version__ = "0.5.23"  # pragma: no cover

unstructured_ingest/embed/interfaces.py

@@ -2,10 +2,10 @@ from abc import ABC
 from dataclasses import dataclass
 from typing import Any, Optional
 
-import numpy as np
 from pydantic import BaseModel, Field
 
 from unstructured_ingest.utils.data_prep import batch_generator
+from unstructured_ingest.utils.dep_check import requires_dependencies
 
 EMBEDDINGS_KEY = "embeddings"
 
@@ -32,7 +32,6 @@ class BaseEncoder(ABC):
 
 @dataclass
 class BaseEmbeddingEncoder(BaseEncoder, ABC):
-
     def initialize(self):
         """Initializes the embedding encoder class. Should also validate the instance
         is properly configured: e.g., embed a single a element"""
@@ -46,8 +45,11 @@ class BaseEmbeddingEncoder(BaseEncoder, ABC):
         return self.embed_query(query="Q")
 
     @property
+    @requires_dependencies(["numpy"])
     def is_unit_vector(self) -> bool:
         """Denotes if the embedding vector is a unit vector."""
+        import numpy as np
+
         exemplary_embedding = self.get_exemplary_embedding()
         return np.isclose(np.linalg.norm(exemplary_embedding), 1.0, rtol=1e-03)
 
@@ -86,7 +88,6 @@ class BaseEmbeddingEncoder(BaseEncoder, ABC):
 
 @dataclass
 class AsyncBaseEmbeddingEncoder(BaseEncoder, ABC):
-
     async def initialize(self):
         """Initializes the embedding encoder class. Should also validate the instance
         is properly configured: e.g., embed a single a element"""
@@ -100,8 +101,11 @@ class AsyncBaseEmbeddingEncoder(BaseEncoder, ABC):
         return await self.embed_query(query="Q")
 
     @property
+    @requires_dependencies(["numpy"])
     async def is_unit_vector(self) -> bool:
         """Denotes if the embedding vector is a unit vector."""
+        import numpy as np
+
         exemplary_embedding = await self.get_exemplary_embedding()
         return np.isclose(np.linalg.norm(exemplary_embedding), 1.0, rtol=1e-03)
 

unstructured_ingest/utils/data_prep.py

@@ -2,20 +2,22 @@ import itertools
 import json
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Generator, Iterable, Optional, Sequence, TypeVar, Union, cast
-
-import pandas as pd
+from typing import TYPE_CHECKING, Any, Generator, Iterable, Optional, Sequence, TypeVar, Union, cast
 
 from unstructured_ingest.utils import ndjson
+from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.logger import logger
 
+if TYPE_CHECKING:
+    from pandas import DataFrame
+
 DATE_FORMATS = ("%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d+%H:%M:%S", "%Y-%m-%dT%H:%M:%S%z")
 
 T = TypeVar("T")
 IterableT = Iterable[T]
 
 
-def split_dataframe(df: pd.DataFrame, chunk_size: int = 100) -> Generator[pd.DataFrame, None, None]:
+def split_dataframe(df: "DataFrame", chunk_size: int = 100) -> Generator["DataFrame", None, None]:
     num_chunks = len(df) // chunk_size + 1
     for i in range(num_chunks):
         yield df[i * chunk_size : (i + 1) * chunk_size]
@@ -144,9 +146,13 @@ def get_data_by_suffix(path: Path) -> list[dict]:
         elif path.suffix == ".ndjson":
             return ndjson.load(f)
         elif path.suffix == ".csv":
+            import pandas as pd
+
             df = pd.read_csv(path)
             return df.to_dict(orient="records")
         elif path.suffix == ".parquet":
+            import pandas as pd
+
             df = pd.read_parquet(path)
             return df.to_dict(orient="records")
         else:
@@ -180,6 +186,9 @@ def get_data(path: Union[Path, str]) -> list[dict]:
             return ndjson.load(f)
         except Exception as e:
             logger.warning(f"failed to read {path} as ndjson: {e}")
+
+        import pandas as pd
+
         try:
             df = pd.read_csv(path)
             return df.to_dict(orient="records")
@@ -202,7 +211,10 @@ def get_json_data(path: Path) -> list[dict]:
             raise ValueError(f"Unsupported file type: {path}")
 
 
-def get_data_df(path: Path) -> pd.DataFrame:
+@requires_dependencies(["pandas"])
+def get_data_df(path: Path) -> "DataFrame":
+    import pandas as pd
+
     with path.open() as f:
         if path.suffix == ".json":
             data = json.load(f)

unstructured_ingest/utils/table.py

@@ -1,11 +1,16 @@
-from typing import Any
-
-import pandas as pd
+from typing import TYPE_CHECKING, Any
 
 from unstructured_ingest.utils.data_prep import flatten_dict
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if TYPE_CHECKING:
+    from pandas import DataFrame
 
 
+@requires_dependencies(["pandas"])
 def get_default_pandas_dtypes() -> dict[str, Any]:
+    import pandas as pd
+
     return {
         "text": pd.StringDtype(),  # type: ignore
         "type": pd.StringDtype(),  # type: ignore
@@ -57,7 +62,9 @@ def get_default_pandas_dtypes() -> dict[str, Any]:
 def convert_to_pandas_dataframe(
     elements_dict: list[dict[str, Any]],
     drop_empty_cols: bool = False,
-) -> pd.DataFrame:
+) -> "DataFrame":
+    import pandas as pd
+
     # Flatten metadata if it hasn't already been flattened
     for d in elements_dict:
         if metadata := d.pop("metadata", None):

unstructured_ingest/v2/interfaces/__init__.py

@@ -1,6 +1,13 @@
+from unstructured_ingest.v2.types.file_data import (
+    BatchFileData,
+    BatchItem,
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
+
 from .connector import AccessConfig, BaseConnector, ConnectionConfig
 from .downloader import Downloader, DownloaderConfig, DownloadResponse, download_responses
-from .file_data import BatchFileData, BatchItem, FileData, FileDataSourceMetadata, SourceIdentifiers
 from .indexer import Indexer, IndexerConfig
 from .process import BaseProcess
 from .processor import ProcessorConfig

unstructured_ingest/v2/interfaces/file_data.py

@@ -1,116 +1,13 @@
-import json
-from pathlib import Path
-from typing import Any, Optional
-from uuid import NAMESPACE_DNS, uuid5
-
-from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator
-
-from unstructured_ingest.v2.logger import logger
-
-
-class SourceIdentifiers(BaseModel):
-    filename: str
-    fullpath: str
-    rel_path: Optional[str] = None
-
-    @property
-    def filename_stem(self) -> str:
-        return Path(self.filename).stem
-
-    @property
-    def relative_path(self) -> str:
-        return self.rel_path or self.fullpath
-
-
-class FileDataSourceMetadata(BaseModel):
-    url: Optional[str] = None
-    version: Optional[str] = None
-    record_locator: Optional[dict[str, Any]] = None
-    date_created: Optional[str] = None
-    date_modified: Optional[str] = None
-    date_processed: Optional[str] = None
-    permissions_data: Optional[list[dict[str, Any]]] = None
-    filesize_bytes: Optional[int] = None
-
-
-class FileData(BaseModel):
-    identifier: str
-    connector_type: str
-    source_identifiers: SourceIdentifiers
-    metadata: FileDataSourceMetadata = Field(default_factory=lambda: FileDataSourceMetadata())
-    additional_metadata: dict[str, Any] = Field(default_factory=dict)
-    reprocess: bool = False
-    local_download_path: Optional[str] = None
-    display_name: Optional[str] = None
-
-    @classmethod
-    def from_file(cls, path: str) -> "FileData":
-        path = Path(path).resolve()
-        if not path.exists() or not path.is_file():
-            raise ValueError(f"file path not valid: {path}")
-        with open(str(path.resolve()), "rb") as f:
-            file_data_dict = json.load(f)
-        file_data = cls.model_validate(file_data_dict)
-        return file_data
-
-    @classmethod
-    def cast(cls, file_data: "FileData", **kwargs) -> "FileData":
-        file_data_dict = file_data.model_dump()
-        return cls.model_validate(file_data_dict, **kwargs)
-
-    def to_file(self, path: str) -> None:
-        path = Path(path).resolve()
-        path.parent.mkdir(parents=True, exist_ok=True)
-        with open(str(path.resolve()), "w") as f:
-            json.dump(self.model_dump(), f, indent=2)
-
-
-class BatchItem(BaseModel):
-    identifier: str
-    version: Optional[str] = None
-
-
-class BatchFileData(FileData):
-    identifier: str = Field(init=False)
-    batch_items: list[BatchItem]
-    source_identifiers: Optional[SourceIdentifiers] = None
-
-    @field_validator("batch_items")
-    @classmethod
-    def check_batch_items(cls, v: list[BatchItem]) -> list[BatchItem]:
-        if not v:
-            raise ValueError("batch items cannot be empty")
-        all_identifiers = [item.identifier for item in v]
-        if len(all_identifiers) != len(set(all_identifiers)):
-            raise ValueError(f"duplicate identifiers: {all_identifiers}")
-        sorted_batch_items = sorted(v, key=lambda item: item.identifier)
-        return sorted_batch_items
-
-    @model_validator(mode="before")
-    @classmethod
-    def populate_identifier(cls, data: Any) -> Any:
-        if isinstance(data, dict) and "identifier" not in data:
-            batch_items = data["batch_items"]
-            identifier_data = json.dumps(
-                {item.identifier: item.version for item in batch_items}, sort_keys=True
-            )
-            data["identifier"] = str(uuid5(NAMESPACE_DNS, str(identifier_data)))
-        return data
-
-
-def file_data_from_file(path: str) -> FileData:
-    try:
-        return BatchFileData.from_file(path=path)
-    except ValidationError:
-        logger.debug(f"{path} not detected as batch file data")
-
-    return FileData.from_file(path=path)
-
-
-def file_data_from_dict(data: dict) -> FileData:
-    try:
-        return BatchFileData.model_validate(data)
-    except ValidationError:
-        logger.debug(f"{data} not valid for batch file data")
-
-    return FileData.model_validate(data)
+"""
+COMPATABILITY NOTICE:
+This file has moved to the v2/types/ module.
+The following line exists for backward compatibility.
+"""
+
+from unstructured_ingest.v2.types.file_data import *  # noqa - star imports are bad, but this is for maximal backward compatability
+
+#  Eventually this file should go away. Let's start warning users now:
+logger.warning(  # noqa - using logger from the star import
+    "Importing file_data.py through interfaces is deprecated. "
+    "Please use unstructured_ingest.v2.types.file_data instead!"
+)

unstructured_ingest/v2/processes/connectors/delta_table.py

@@ -3,10 +3,9 @@ import traceback
 from dataclasses import dataclass, field
 from multiprocessing import Process, Queue
 from pathlib import Path
-from typing import Any, Optional
+from typing import TYPE_CHECKING, Any, Optional
 from urllib.parse import urlparse
 
-import pandas as pd
 from pydantic import Field, Secret
 
 from unstructured_ingest.error import DestinationConnectionError
@@ -27,6 +26,9 @@ from unstructured_ingest.v2.processes.connector_registry import DestinationRegis
 
 CONNECTOR_TYPE = "delta_table"
 
+if TYPE_CHECKING:
+    from pandas import DataFrame
+
 
 @requires_dependencies(["deltalake"], extras="delta-table")
 def write_deltalake_with_error_handling(queue, **kwargs):
@@ -136,7 +138,7 @@ class DeltaTableUploader(Uploader):
                 logger.error(f"failed to validate connection: {e}", exc_info=True)
                 raise DestinationConnectionError(f"failed to validate connection: {e}")
 
-    def upload_dataframe(self, df: pd.DataFrame, file_data: FileData) -> None:
+    def upload_dataframe(self, df: "DataFrame", file_data: FileData) -> None:
         updated_upload_path = os.path.join(
             self.connection_config.table_uri, file_data.source_identifiers.relative_path
         )
@@ -172,7 +174,10 @@ class DeltaTableUploader(Uploader):
             logger.error(f"Exception occurred in write_deltalake: {error_message}")
             raise RuntimeError(f"Error in write_deltalake: {error_message}")
 
+    @requires_dependencies(["pandas"], extras="delta-table")
     def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
+        import pandas as pd
+
         df = pd.DataFrame(data=data)
         self.upload_dataframe(df=df, file_data=file_data)
 

unstructured_ingest/v2/processes/connectors/duckdb/base.py

@@ -2,9 +2,8 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
-import pandas as pd
-
 from unstructured_ingest.utils.data_prep import get_data, write_data
+from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import FileData, UploadStager
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
@@ -55,7 +54,6 @@ _COLUMNS = (
 
 @dataclass
 class BaseDuckDBUploadStager(UploadStager):
-
     def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:
         data = element_dict.copy()
         metadata: dict[str, Any] = data.pop("metadata", {})
@@ -72,6 +70,7 @@ class BaseDuckDBUploadStager(UploadStager):
         data = {k: v for k, v in data.items() if k in _COLUMNS}
         return data
 
+    @requires_dependencies(["pandas"], extras="duckdb")
     def run(
         self,
         elements_filepath: Path,
@@ -80,6 +79,8 @@ class BaseDuckDBUploadStager(UploadStager):
         output_filename: str,
         **kwargs: Any,
     ) -> Path:
+        import pandas as pd
+
         elements_contents = get_data(path=elements_filepath)
         output_filename_suffix = Path(elements_filepath).suffix
         output_filename = f"{Path(output_filename).stem}{output_filename_suffix}"

unstructured_ingest/v2/processes/connectors/duckdb/duckdb.py

@@ -3,7 +3,6 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generator, Optional
 
-import pandas as pd
 from pydantic import Field, Secret
 
 from unstructured_ingest.error import DestinationConnectionError
@@ -23,6 +22,7 @@ from unstructured_ingest.v2.processes.connectors.duckdb.base import BaseDuckDBUp
 
 if TYPE_CHECKING:
     from duckdb import DuckDBPyConnection as DuckDBConnection
+    from pandas import DataFrame
 
 CONNECTOR_TYPE = "duckdb"
 
@@ -101,7 +101,7 @@ class DuckDBUploader(Uploader):
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")
 
-    def upload_dataframe(self, df: pd.DataFrame) -> None:
+    def upload_dataframe(self, df: "DataFrame") -> None:
         logger.debug(f"uploading {len(df)} entries to {self.connection_config.database} ")
 
         with self.connection_config.get_client() as conn:
@@ -109,7 +109,10 @@ class DuckDBUploader(Uploader):
                 f"INSERT INTO {self.connection_config.db_schema}.{self.connection_config.table} BY NAME SELECT * FROM df"  # noqa: E501
             )
 
+    @requires_dependencies(["pandas"], extras="duckdb")
     def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
+        import pandas as pd
+
         df = pd.DataFrame(data=data)
         self.upload_dataframe(df=df)
 

unstructured_ingest/v2/processes/connectors/duckdb/motherduck.py

@@ -3,7 +3,6 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generator, Optional
 
-import pandas as pd
 from pydantic import Field, Secret
 
 from unstructured_ingest.__version__ import __version__ as unstructured_io_ingest_version
@@ -24,6 +23,7 @@ from unstructured_ingest.v2.processes.connectors.duckdb.base import BaseDuckDBUp
 
 if TYPE_CHECKING:
     from duckdb import DuckDBPyConnection as MotherDuckConnection
+    from pandas import DataFrame
 
 CONNECTOR_TYPE = "motherduck"
 
@@ -100,7 +100,7 @@ class MotherDuckUploader(Uploader):
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")
 
-    def upload_dataframe(self, df: pd.DataFrame) -> None:
+    def upload_dataframe(self, df: "DataFrame") -> None:
         logger.debug(f"uploading {len(df)} entries to {self.connection_config.database} ")
         database = self.connection_config.database
         db_schema = self.connection_config.db_schema
@@ -109,7 +109,10 @@ class MotherDuckUploader(Uploader):
         with self.connection_config.get_client() as conn:
             conn.query(f'INSERT INTO "{database}"."{db_schema}"."{table}" BY NAME SELECT * FROM df')
 
+    @requires_dependencies(["pandas"], extras="duckdb")
     def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
+        import pandas as pd
+
         df = pd.DataFrame(data=data)
         self.upload_dataframe(df=df)
 

unstructured_ingest/v2/processes/connectors/elasticsearch/elasticsearch.py

@@ -6,7 +6,7 @@ from pathlib import Path
 from time import time
 from typing import TYPE_CHECKING, Any, Generator, Optional, Union
 
-from pydantic import BaseModel, Field, Secret, SecretStr
+from pydantic import BaseModel, Field, Secret, SecretStr, field_validator
 
 from unstructured_ingest.error import (
     DestinationConnectionError,
@@ -98,6 +98,12 @@ class ElasticsearchConnectionConfig(ConnectionConfig):
     ca_certs: Optional[Path] = None
     access_config: Secret[ElasticsearchAccessConfig]
 
+    @field_validator("hosts", mode="before")
+    def to_list(cls, value):
+        if isinstance(value, str):
+            return [value]
+        return value
+
     def get_client_kwargs(self) -> dict:
         # Update auth related fields to conform to what the SDK expects based on the
         # supported methods:

unstructured_ingest/v2/processes/connectors/elasticsearch/opensearch.py

@@ -2,7 +2,7 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Optional
 
-from pydantic import BaseModel, Field, Secret
+from pydantic import BaseModel, Field, Secret, field_validator
 
 from unstructured_ingest.error import (
     DestinationConnectionError,
@@ -78,6 +78,12 @@ class OpenSearchConnectionConfig(ConnectionConfig):
 
     access_config: Secret[OpenSearchAccessConfig]
 
+    @field_validator("hosts", mode="before")
+    def to_list(cls, value):
+        if isinstance(value, str):
+            return [value]
+        return value
+
     def get_client_kwargs(self) -> dict:
         # Update auth related fields to conform to what the SDK expects based on the
         # supported methods:

unstructured_ingest/v2/processes/connectors/kdbai.py

@@ -3,7 +3,6 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generator, Optional
 
-import pandas as pd
 from pydantic import Field, Secret
 
 from unstructured_ingest.error import DestinationConnectionError
@@ -26,6 +25,7 @@ from unstructured_ingest.v2.utils import get_enhanced_element_id
 
 if TYPE_CHECKING:
     from kdbai_client import Database, Session, Table
+    from pandas import DataFrame
 
 CONNECTOR_TYPE = "kdbai"
 
@@ -118,11 +118,11 @@ class KdbaiUploader(Uploader):
             table = db.table(self.upload_config.table_name)
             yield table
 
-    def upsert_batch(self, batch: pd.DataFrame):
+    def upsert_batch(self, batch: "DataFrame"):
         with self.get_table() as table:
             table.insert(batch)
 
-    def process_dataframe(self, df: pd.DataFrame):
+    def process_dataframe(self, df: "DataFrame"):
         logger.debug(
             f"uploading {len(df)} entries to {self.connection_config.endpoint} "
             f"db {self.upload_config.database_name} in table {self.upload_config.table_name}"
@@ -130,7 +130,10 @@ class KdbaiUploader(Uploader):
         for batch_df in split_dataframe(df=df, chunk_size=self.upload_config.batch_size):
             self.upsert_batch(batch=batch_df)
 
+    @requires_dependencies(["pandas"], extras="kdbai")
     def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
+        import pandas as pd
+
         df = pd.DataFrame(data=data)
         self.process_dataframe(df=df)
 

unstructured_ingest/v2/processes/connectors/lancedb/lancedb.py

@@ -8,7 +8,6 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional
 
-import pandas as pd
 from pydantic import Field
 
 from unstructured_ingest.error import DestinationConnectionError
@@ -26,6 +25,7 @@ CONNECTOR_TYPE = "lancedb"
 if TYPE_CHECKING:
     from lancedb import AsyncConnection
     from lancedb.table import AsyncTable
+    from pandas import DataFrame
 
 
 class LanceDBConnectionConfig(ConnectionConfig, ABC):
@@ -69,6 +69,7 @@ class LanceDBUploadStager(UploadStager):
         default_factory=LanceDBUploadStagerConfig
     )
 
+    @requires_dependencies(["pandas"], extras="lancedb")
     def run(
         self,
         elements_filepath: Path,
@@ -77,6 +78,8 @@ class LanceDBUploadStager(UploadStager):
         output_filename: str,
         **kwargs: Any,
     ) -> Path:
+        import pandas as pd
+
         with open(elements_filepath) as elements_file:
             elements_contents: list[dict] = json.load(elements_file)
 
@@ -129,7 +132,10 @@ class LanceDBUploader(Uploader):
             finally:
                 table.close()
 
+    @requires_dependencies(["pandas"], extras="lancedb")
     async def run_async(self, path, file_data, **kwargs):
+        import pandas as pd
+
         df = pd.read_feather(path)
         async with self.get_table() as table:
             schema = await table.schema()
@@ -144,7 +150,9 @@ class LanceDBUploader(Uploader):
                 await table.delete(f'{RECORD_ID_LABEL} = "{file_data.identifier}"')
             await table.add(data=df)
 
-    def _fit_to_schema(self, df: pd.DataFrame, schema) -> pd.DataFrame:
+    def _fit_to_schema(self, df: "DataFrame", schema) -> "DataFrame":
+        import pandas as pd
+
         columns = set(df.columns)
         schema_fields = set(schema.names)
         columns_to_drop = columns - schema_fields

unstructured_ingest/v2/processes/connectors/sql/databricks_delta_tables.py

@@ -3,8 +3,6 @@ from contextlib import contextmanager
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Generator, Optional
 
-import numpy as np
-import pandas as pd
 from pydantic import Field, Secret
 
 from unstructured_ingest.utils.data_prep import split_dataframe
@@ -27,6 +25,7 @@ if TYPE_CHECKING:
     from databricks.sdk.core import oauth_service_principal
     from databricks.sql.client import Connection as DeltaTableConnection
     from databricks.sql.client import Cursor as DeltaTableCursor
+    from pandas import DataFrame
 
 CONNECTOR_TYPE = "databricks_delta_tables"
 
@@ -180,7 +179,10 @@ class DatabricksDeltaTablesUploader(SQLUploader):
         )
         return statement
 
-    def upload_dataframe(self, df: pd.DataFrame, file_data: FileData) -> None:
+    @requires_dependencies(["pandas"], extras="databricks-delta-tables")
+    def upload_dataframe(self, df: "DataFrame", file_data: FileData) -> None:
+        import numpy as np
+
         if self.can_delete():
             self.delete_by_record_id(file_data=file_data)
         else:

unstructured_ingest/v2/processes/connectors/sql/singlestore.py

@@ -3,9 +3,9 @@ from contextlib import contextmanager
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any, Generator, Optional
 
-import pandas as pd
 from pydantic import Field, Secret
 
+from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
@@ -46,6 +46,7 @@ class SingleStoreConnectionConfig(SQLConnectionConfig):
     database: Optional[str] = Field(default=None, description="SingleStore database")
 
     @contextmanager
+    @requires_dependencies(["singlestoredb"], extras="singlestore")
     def get_connection(self) -> Generator["SingleStoreConnection", None, None]:
         import singlestoredb as s2
 
@@ -130,9 +131,12 @@ class SingleStoreUploader(SQLUploader):
     values_delimiter: str = "%s"
     connector_type: str = CONNECTOR_TYPE
 
+    @requires_dependencies(["pandas"], extras="singlestore")
     def prepare_data(
         self, columns: list[str], data: tuple[tuple[Any, ...], ...]
     ) -> list[tuple[Any, ...]]:
+        import pandas as pd
+
         output = []
         for row in data:
             parsed = []