unstructured-ingest 0.5.19__py3-none-any.whl → 0.5.21__py3-none-any.whl

unstructured_ingest/utils/table.py

@@ -1,11 +1,16 @@
-from typing import Any
-
-import pandas as pd
+from typing import TYPE_CHECKING, Any
 
 from unstructured_ingest.utils.data_prep import flatten_dict
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if TYPE_CHECKING:
+    from pandas import DataFrame
 
 
+@requires_dependencies(["pandas"])
 def get_default_pandas_dtypes() -> dict[str, Any]:
+    import pandas as pd
+
     return {
         "text": pd.StringDtype(),  # type: ignore
         "type": pd.StringDtype(),  # type: ignore
@@ -57,7 +62,9 @@ def get_default_pandas_dtypes() -> dict[str, Any]:
 def convert_to_pandas_dataframe(
     elements_dict: list[dict[str, Any]],
     drop_empty_cols: bool = False,
-) -> pd.DataFrame:
+) -> "DataFrame":
+    import pandas as pd
+
     # Flatten metadata if it hasn't already been flattened
     for d in elements_dict:
         if metadata := d.pop("metadata", None):

unstructured_ingest/v2/processes/connectors/__init__.py

@@ -4,6 +4,7 @@ import unstructured_ingest.v2.processes.connectors.databricks  # noqa: F401
 import unstructured_ingest.v2.processes.connectors.duckdb  # noqa: F401
 import unstructured_ingest.v2.processes.connectors.elasticsearch  # noqa: F401
 import unstructured_ingest.v2.processes.connectors.fsspec  # noqa: F401
+import unstructured_ingest.v2.processes.connectors.ibm_watsonx  # noqa: F401
 import unstructured_ingest.v2.processes.connectors.kafka  # noqa: F401
 import unstructured_ingest.v2.processes.connectors.lancedb  # noqa: F401
 import unstructured_ingest.v2.processes.connectors.qdrant  # noqa: F401
@@ -121,4 +122,5 @@ add_source_entry(source_type=DISCORD_CONNECTOR_TYPE, entry=discord_source_entry)
 add_destination_entry(destination_type=REDIS_CONNECTOR_TYPE, entry=redis_destination_entry)
 
 add_source_entry(source_type=JIRA_CONNECTOR_TYPE, entry=jira_source_entry)
+
 add_source_entry(source_type=ZENDESK_CONNECTOR_TYPE, entry=zendesk_source_entry)

unstructured_ingest/v2/processes/connectors/delta_table.py

@@ -3,10 +3,9 @@ import traceback
 from dataclasses import dataclass, field
 from multiprocessing import Process, Queue
 from pathlib import Path
-from typing import Any, Optional
+from typing import TYPE_CHECKING, Any, Optional
 from urllib.parse import urlparse
 
-import pandas as pd
 from pydantic import Field, Secret
 
 from unstructured_ingest.error import DestinationConnectionError
@@ -27,6 +26,9 @@ from unstructured_ingest.v2.processes.connector_registry import DestinationRegis
 
 CONNECTOR_TYPE = "delta_table"
 
+if TYPE_CHECKING:
+    from pandas import DataFrame
+
 
 @requires_dependencies(["deltalake"], extras="delta-table")
 def write_deltalake_with_error_handling(queue, **kwargs):
@@ -136,7 +138,7 @@ class DeltaTableUploader(Uploader):
                 logger.error(f"failed to validate connection: {e}", exc_info=True)
                 raise DestinationConnectionError(f"failed to validate connection: {e}")
 
-    def upload_dataframe(self, df: pd.DataFrame, file_data: FileData) -> None:
+    def upload_dataframe(self, df: "DataFrame", file_data: FileData) -> None:
         updated_upload_path = os.path.join(
             self.connection_config.table_uri, file_data.source_identifiers.relative_path
         )
@@ -172,7 +174,10 @@ class DeltaTableUploader(Uploader):
             logger.error(f"Exception occurred in write_deltalake: {error_message}")
             raise RuntimeError(f"Error in write_deltalake: {error_message}")
 
+    @requires_dependencies(["pandas"], extras="delta-table")
     def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
+        import pandas as pd
+
         df = pd.DataFrame(data=data)
         self.upload_dataframe(df=df, file_data=file_data)
 

unstructured_ingest/v2/processes/connectors/duckdb/base.py

@@ -2,9 +2,8 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
-import pandas as pd
-
 from unstructured_ingest.utils.data_prep import get_data, write_data
+from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import FileData, UploadStager
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
@@ -55,7 +54,6 @@ _COLUMNS = (
 
 @dataclass
 class BaseDuckDBUploadStager(UploadStager):
-
     def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:
         data = element_dict.copy()
         metadata: dict[str, Any] = data.pop("metadata", {})
@@ -72,6 +70,7 @@ class BaseDuckDBUploadStager(UploadStager):
         data = {k: v for k, v in data.items() if k in _COLUMNS}
         return data
 
+    @requires_dependencies(["pandas"], extras="duckdb")
     def run(
         self,
         elements_filepath: Path,
@@ -80,6 +79,8 @@ class BaseDuckDBUploadStager(UploadStager):
         output_filename: str,
         **kwargs: Any,
     ) -> Path:
+        import pandas as pd
+
         elements_contents = get_data(path=elements_filepath)
         output_filename_suffix = Path(elements_filepath).suffix
         output_filename = f"{Path(output_filename).stem}{output_filename_suffix}"

unstructured_ingest/v2/processes/connectors/duckdb/duckdb.py

@@ -3,7 +3,6 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generator, Optional
 
-import pandas as pd
 from pydantic import Field, Secret
 
 from unstructured_ingest.error import DestinationConnectionError
@@ -23,6 +22,7 @@ from unstructured_ingest.v2.processes.connectors.duckdb.base import BaseDuckDBUp
 
 if TYPE_CHECKING:
     from duckdb import DuckDBPyConnection as DuckDBConnection
+    from pandas import DataFrame
 
 CONNECTOR_TYPE = "duckdb"
 
@@ -101,7 +101,7 @@ class DuckDBUploader(Uploader):
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")
 
-    def upload_dataframe(self, df: pd.DataFrame) -> None:
+    def upload_dataframe(self, df: "DataFrame") -> None:
         logger.debug(f"uploading {len(df)} entries to {self.connection_config.database} ")
 
         with self.connection_config.get_client() as conn:
@@ -109,7 +109,10 @@ class DuckDBUploader(Uploader):
                 f"INSERT INTO {self.connection_config.db_schema}.{self.connection_config.table} BY NAME SELECT * FROM df"  # noqa: E501
             )
 
+    @requires_dependencies(["pandas"], extras="duckdb")
     def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
+        import pandas as pd
+
         df = pd.DataFrame(data=data)
         self.upload_dataframe(df=df)
 

unstructured_ingest/v2/processes/connectors/duckdb/motherduck.py

@@ -3,7 +3,6 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generator, Optional
 
-import pandas as pd
 from pydantic import Field, Secret
 
 from unstructured_ingest.__version__ import __version__ as unstructured_io_ingest_version
@@ -24,6 +23,7 @@ from unstructured_ingest.v2.processes.connectors.duckdb.base import BaseDuckDBUp
 
 if TYPE_CHECKING:
     from duckdb import DuckDBPyConnection as MotherDuckConnection
+    from pandas import DataFrame
 
 CONNECTOR_TYPE = "motherduck"
 
@@ -100,7 +100,7 @@ class MotherDuckUploader(Uploader):
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")
 
-    def upload_dataframe(self, df: pd.DataFrame) -> None:
+    def upload_dataframe(self, df: "DataFrame") -> None:
         logger.debug(f"uploading {len(df)} entries to {self.connection_config.database} ")
         database = self.connection_config.database
         db_schema = self.connection_config.db_schema
@@ -109,7 +109,10 @@ class MotherDuckUploader(Uploader):
         with self.connection_config.get_client() as conn:
             conn.query(f'INSERT INTO "{database}"."{db_schema}"."{table}" BY NAME SELECT * FROM df')
 
+    @requires_dependencies(["pandas"], extras="duckdb")
     def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
+        import pandas as pd
+
         df = pd.DataFrame(data=data)
         self.upload_dataframe(df=df)
 

unstructured_ingest/v2/processes/connectors/ibm_watsonx/__init__.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from unstructured_ingest.v2.processes.connector_registry import add_destination_entry
+
+from .ibm_watsonx_s3 import CONNECTOR_TYPE as IBM_WATSONX_S3_CONNECTOR_TYPE
+from .ibm_watsonx_s3 import ibm_watsonx_s3_destination_entry
+
+add_destination_entry(
+    destination_type=IBM_WATSONX_S3_CONNECTOR_TYPE, entry=ibm_watsonx_s3_destination_entry
+)

unstructured_ingest/v2/processes/connectors/ibm_watsonx/ibm_watsonx_s3.py

@@ -0,0 +1,301 @@
+import logging
+import time
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Generator, Optional, Tuple
+
+import pandas as pd
+from pydantic import Field, Secret
+
+from unstructured_ingest.utils.data_prep import get_data_df
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.constants import RECORD_ID_LABEL
+from unstructured_ingest.v2.errors import ProviderError, UserAuthError, UserError
+from unstructured_ingest.v2.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    FileData,
+    UploaderConfig,
+)
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+)
+from unstructured_ingest.v2.processes.connectors.sql.sql import (
+    SQLUploader,
+    SQLUploadStager,
+    SQLUploadStagerConfig,
+)
+
+if TYPE_CHECKING:
+    from pyarrow import Table as ArrowTable
+    from pyiceberg.catalog.rest import RestCatalog
+    from pyiceberg.table import Table, Transaction
+
+CONNECTOR_TYPE = "ibm_watsonx_s3"
+
+DEFAULT_IBM_CLOUD_AUTH_URL = "https://iam.cloud.ibm.com/identity/token"
+DEFAULT_ICEBERG_URI_PATH = "/mds/iceberg"
+DEFAULT_ICEBERG_CATALOG_TYPE = "rest"
+
+
+class IcebergCommitFailedException(Exception):
+    """Failed to commit changes to the iceberg table."""
+
+
+class IbmWatsonxAccessConfig(AccessConfig):
+    iam_api_key: str = Field(description="IBM IAM API Key")
+    access_key_id: str = Field(description="Cloud Object Storage HMAC Access Key ID")
+    secret_access_key: str = Field(description="Cloud Object Storage HMAC Secret Access Key")
+
+
+class IbmWatsonxConnectionConfig(ConnectionConfig):
+    access_config: Secret[IbmWatsonxAccessConfig]
+    iceberg_endpoint: str = Field(description="Iceberg REST endpoint")
+    object_storage_endpoint: str = Field(description="Cloud Object Storage public endpoint")
+    object_storage_region: str = Field(description="Cloud Object Storage region")
+    catalog: str = Field(description="Catalog name")
+
+    _bearer_token: Optional[dict[str, Any]] = None
+
+    @property
+    def iceberg_url(self) -> str:
+        return f"https://{self.iceberg_endpoint.strip('/')}{DEFAULT_ICEBERG_URI_PATH}"
+
+    @property
+    def object_storage_url(self) -> str:
+        return f"https://{self.object_storage_endpoint.strip('/')}"
+
+    @property
+    def bearer_token(self) -> str:
+        # Add 60 seconds to deal with edge cases where the token expires before the request is made
+        timestamp = int(time.time()) + 60
+        if self._bearer_token is None or self._bearer_token.get("expiration", 0) <= timestamp:
+            self._bearer_token = self.generate_bearer_token()
+        return self._bearer_token["access_token"]
+
+    @requires_dependencies(["httpx"], extras="ibm-watsonx-s3")
+    def wrap_error(self, e: Exception) -> Exception:
+        import httpx
+
+        if not isinstance(e, httpx.HTTPStatusError):
+            logger.error(f"Unhandled exception from IBM watsonx.data connector: {e}", exc_info=True)
+            return e
+        url = e.request.url
+        response_code = e.response.status_code
+        if response_code == 401:
+            logger.error(
+                f"Failed to authenticate IBM watsonx.data user {url}, status code {response_code}"
+            )
+            return UserAuthError(e)
+        if response_code == 403:
+            logger.error(
+                f"Given IBM watsonx.data user is not authorized {url}, status code {response_code}"
+            )
+            return UserAuthError(e)
+        if 400 <= response_code < 500:
+            logger.error(
+                f"Request to {url} failed"
+                f"in IBM watsonx.data connector, status code {response_code}"
+            )
+            return UserError(e)
+        if response_code > 500:
+            logger.error(
+                f"Request to {url} failed"
+                f"in IBM watsonx.data connector, status code {response_code}"
+            )
+            return ProviderError(e)
+        logger.error(f"Unhandled exception from IBM watsonx.data connector: {e}", exc_info=True)
+        return e
+
+    @requires_dependencies(["httpx"], extras="ibm-watsonx-s3")
+    def generate_bearer_token(self) -> dict[str, Any]:
+        import httpx
+
+        headers = {
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Accept": "application/json",
+        }
+        data = {
+            "grant_type": "urn:ibm:params:oauth:grant-type:apikey",
+            "apikey": self.access_config.get_secret_value().iam_api_key,
+        }
+
+        logger.info("Generating IBM IAM Bearer Token")
+        try:
+            response = httpx.post(DEFAULT_IBM_CLOUD_AUTH_URL, headers=headers, data=data)
+            response.raise_for_status()
+        except Exception as e:
+            raise self.wrap_error(e)
+        return response.json()
+
+    def get_catalog_config(self) -> dict[str, Any]:
+        return {
+            "name": self.catalog,
+            "type": DEFAULT_ICEBERG_CATALOG_TYPE,
+            "uri": self.iceberg_url,
+            "token": self.bearer_token,
+            "warehouse": self.catalog,
+            "s3.endpoint": self.object_storage_url,
+            "s3.access-key-id": self.access_config.get_secret_value().access_key_id,
+            "s3.secret-access-key": self.access_config.get_secret_value().secret_access_key,
+            "s3.region": self.object_storage_region,
+        }
+
+    @requires_dependencies(["pyiceberg"], extras="ibm-watsonx-s3")
+    @contextmanager
+    def get_catalog(self) -> Generator["RestCatalog", None, None]:
+        from pyiceberg.catalog import load_catalog
+
+        try:
+            catalog_config = self.get_catalog_config()
+            catalog = load_catalog(**catalog_config)
+        except Exception as e:
+            logger.error(f"Failed to connect to catalog '{self.catalog}': {e}", exc_info=True)
+            raise ProviderError(f"Failed to connect to catalog '{self.catalog}': {e}")
+
+        yield catalog
+
+
+@dataclass
+class IbmWatsonxUploadStagerConfig(SQLUploadStagerConfig):
+    pass
+
+
+@dataclass
+class IbmWatsonxUploadStager(SQLUploadStager):
+    upload_stager_config: IbmWatsonxUploadStagerConfig = field(
+        default_factory=IbmWatsonxUploadStagerConfig
+    )
+
+
+class IbmWatsonxUploaderConfig(UploaderConfig):
+    namespace: str = Field(description="Namespace name")
+    table: str = Field(description="Table name")
+    max_retries: int = Field(
+        default=5, description="Maximum number of retries to upload data", ge=2, le=10
+    )
+    record_id_key: str = Field(
+        default=RECORD_ID_LABEL,
+        description="Searchable key to find entries for the same record on previous runs",
+    )
+
+    @property
+    def table_identifier(self) -> Tuple[str, str]:
+        return (self.namespace, self.table)
+
+
+@dataclass
+class IbmWatsonxUploader(SQLUploader):
+    connection_config: IbmWatsonxConnectionConfig
+    upload_config: IbmWatsonxUploaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def precheck(self) -> None:
+        with self.connection_config.get_catalog() as catalog:
+            if not catalog.namespace_exists(self.upload_config.namespace):
+                raise UserError(f"Namespace '{self.upload_config.namespace}' does not exist")
+            if not catalog.table_exists(self.upload_config.table_identifier):
+                raise UserError(
+                    f"Table '{self.upload_config.table}' does not exist in namespace '{self.upload_config.namespace}'"  # noqa: E501
+                )
+
+    @contextmanager
+    def get_table(self) -> Generator["Table", None, None]:
+        with self.connection_config.get_catalog() as catalog:
+            table = catalog.load_table(self.upload_config.table_identifier)
+            yield table
+
+    def get_table_columns(self) -> list[str]:
+        if self._columns is None:
+            with self.get_table() as table:
+                self._columns = table.schema().column_names
+        return self._columns
+
+    def can_delete(self) -> bool:
+        return self.upload_config.record_id_key in self.get_table_columns()
+
+    @requires_dependencies(["pyarrow"], extras="ibm-watsonx-s3")
+    def _df_to_arrow_table(self, df: pd.DataFrame) -> "ArrowTable":
+        import pyarrow as pa
+
+        # Iceberg will automatically fill missing columns with nulls
+        # Iceberg will throw an error if the DataFrame column has only null values
+        # because it can't infer the type of the column and match it with the table schema
+        return pa.Table.from_pandas(self._fit_to_schema(df, add_missing_columns=False))
+
+    @requires_dependencies(["pyiceberg"], extras="ibm-watsonx-s3")
+    def _delete(self, transaction: "Transaction", identifier: str) -> None:
+        from pyiceberg.expressions import EqualTo
+
+        if self.can_delete():
+            transaction.delete(delete_filter=EqualTo(self.upload_config.record_id_key, identifier))
+        else:
+            logger.warning(
+                f"Table doesn't contain expected "
+                f"record id column "
+                f"{self.upload_config.record_id_key}, skipping delete"
+            )
+
+    @requires_dependencies(["pyiceberg", "tenacity"], extras="ibm-watsonx-s3")
+    def upload_data_table(
+        self, table: "Table", data_table: "ArrowTable", file_data: FileData
+    ) -> None:
+        from pyiceberg.exceptions import CommitFailedException
+        from tenacity import (
+            before_log,
+            retry,
+            retry_if_exception_type,
+            stop_after_attempt,
+            wait_random,
+        )
+
+        @retry(
+            stop=stop_after_attempt(self.upload_config.max_retries),
+            wait=wait_random(),
+            retry=retry_if_exception_type(IcebergCommitFailedException),
+            before=before_log(logger, logging.DEBUG),
+            reraise=True,
+        )
+        def _upload_data_table(table: "Table", data_table: "ArrowTable", file_data: FileData):
+            try:
+                with table.transaction() as transaction:
+                    self._delete(transaction, file_data.identifier)
+                    transaction.append(data_table)
+            except CommitFailedException as e:
+                table.refresh()
+                logger.debug(e)
+                raise IcebergCommitFailedException(e)
+            except Exception as e:
+                raise ProviderError(f"Failed to upload data to table: {e}")
+
+        try:
+            return _upload_data_table(table, data_table, file_data)
+        except ProviderError:
+            raise
+        except Exception as e:
+            raise ProviderError(f"Failed to upload data to table: {e}")
+
+    def upload_dataframe(self, df: pd.DataFrame, file_data: FileData) -> None:
+        data_table = self._df_to_arrow_table(df)
+
+        with self.get_table() as table:
+            self.upload_data_table(table, data_table, file_data)
+
+    def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
+        df = pd.DataFrame(data)
+        self.upload_dataframe(df=df, file_data=file_data)
+
+    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        df = get_data_df(path=path)
+        self.upload_dataframe(df=df, file_data=file_data)
+
+
+ibm_watsonx_s3_destination_entry = DestinationRegistryEntry(
+    connection_config=IbmWatsonxConnectionConfig,
+    uploader=IbmWatsonxUploader,
+    uploader_config=IbmWatsonxUploaderConfig,
+    upload_stager=IbmWatsonxUploadStager,
+    upload_stager_config=IbmWatsonxUploadStagerConfig,
+)

unstructured_ingest/v2/processes/connectors/kdbai.py

@@ -3,7 +3,6 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generator, Optional
 
-import pandas as pd
 from pydantic import Field, Secret
 
 from unstructured_ingest.error import DestinationConnectionError
@@ -26,6 +25,7 @@ from unstructured_ingest.v2.utils import get_enhanced_element_id
 
 if TYPE_CHECKING:
     from kdbai_client import Database, Session, Table
+    from pandas import DataFrame
 
 CONNECTOR_TYPE = "kdbai"
 
@@ -118,11 +118,11 @@ class KdbaiUploader(Uploader):
             table = db.table(self.upload_config.table_name)
             yield table
 
-    def upsert_batch(self, batch: pd.DataFrame):
+    def upsert_batch(self, batch: "DataFrame"):
         with self.get_table() as table:
             table.insert(batch)
 
-    def process_dataframe(self, df: pd.DataFrame):
+    def process_dataframe(self, df: "DataFrame"):
         logger.debug(
             f"uploading {len(df)} entries to {self.connection_config.endpoint} "
             f"db {self.upload_config.database_name} in table {self.upload_config.table_name}"
@@ -130,7 +130,10 @@ class KdbaiUploader(Uploader):
         for batch_df in split_dataframe(df=df, chunk_size=self.upload_config.batch_size):
             self.upsert_batch(batch=batch_df)
 
+    @requires_dependencies(["pandas"], extras="kdbai")
     def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
+        import pandas as pd
+
         df = pd.DataFrame(data=data)
         self.process_dataframe(df=df)
 

unstructured_ingest/v2/processes/connectors/lancedb/lancedb.py

@@ -8,7 +8,6 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional
 
-import pandas as pd
 from pydantic import Field
 
 from unstructured_ingest.error import DestinationConnectionError
@@ -26,6 +25,7 @@ CONNECTOR_TYPE = "lancedb"
 if TYPE_CHECKING:
     from lancedb import AsyncConnection
     from lancedb.table import AsyncTable
+    from pandas import DataFrame
 
 
 class LanceDBConnectionConfig(ConnectionConfig, ABC):
@@ -69,6 +69,7 @@ class LanceDBUploadStager(UploadStager):
         default_factory=LanceDBUploadStagerConfig
     )
 
+    @requires_dependencies(["pandas"], extras="lancedb")
     def run(
         self,
         elements_filepath: Path,
@@ -77,6 +78,8 @@ class LanceDBUploadStager(UploadStager):
         output_filename: str,
         **kwargs: Any,
     ) -> Path:
+        import pandas as pd
+
         with open(elements_filepath) as elements_file:
             elements_contents: list[dict] = json.load(elements_file)
 
@@ -129,7 +132,10 @@ class LanceDBUploader(Uploader):
             finally:
                 table.close()
 
+    @requires_dependencies(["pandas"], extras="lancedb")
     async def run_async(self, path, file_data, **kwargs):
+        import pandas as pd
+
         df = pd.read_feather(path)
         async with self.get_table() as table:
             schema = await table.schema()
@@ -144,7 +150,9 @@ class LanceDBUploader(Uploader):
                 await table.delete(f'{RECORD_ID_LABEL} = "{file_data.identifier}"')
             await table.add(data=df)
 
-    def _fit_to_schema(self, df: pd.DataFrame, schema) -> pd.DataFrame:
+    def _fit_to_schema(self, df: "DataFrame", schema) -> "DataFrame":
+        import pandas as pd
+
         columns = set(df.columns)
         schema_fields = set(schema.names)
         columns_to_drop = columns - schema_fields

unstructured_ingest/v2/processes/connectors/sql/databricks_delta_tables.py

@@ -3,8 +3,6 @@ from contextlib import contextmanager
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Generator, Optional
 
-import numpy as np
-import pandas as pd
 from pydantic import Field, Secret
 
 from unstructured_ingest.utils.data_prep import split_dataframe
@@ -27,6 +25,7 @@ if TYPE_CHECKING:
     from databricks.sdk.core import oauth_service_principal
     from databricks.sql.client import Connection as DeltaTableConnection
     from databricks.sql.client import Cursor as DeltaTableCursor
+    from pandas import DataFrame
 
 CONNECTOR_TYPE = "databricks_delta_tables"
 
@@ -180,7 +179,10 @@ class DatabricksDeltaTablesUploader(SQLUploader):
         )
         return statement
 
-    def upload_dataframe(self, df: pd.DataFrame, file_data: FileData) -> None:
+    @requires_dependencies(["pandas"], extras="databricks-delta-tables")
+    def upload_dataframe(self, df: "DataFrame", file_data: FileData) -> None:
+        import numpy as np
+
         if self.can_delete():
             self.delete_by_record_id(file_data=file_data)
         else:

unstructured_ingest/v2/processes/connectors/sql/singlestore.py

@@ -3,9 +3,9 @@ from contextlib import contextmanager
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any, Generator, Optional
 
-import pandas as pd
 from pydantic import Field, Secret
 
+from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
@@ -46,6 +46,7 @@ class SingleStoreConnectionConfig(SQLConnectionConfig):
     database: Optional[str] = Field(default=None, description="SingleStore database")
 
     @contextmanager
+    @requires_dependencies(["singlestoredb"], extras="singlestore")
     def get_connection(self) -> Generator["SingleStoreConnection", None, None]:
         import singlestoredb as s2
 
@@ -130,9 +131,12 @@ class SingleStoreUploader(SQLUploader):
     values_delimiter: str = "%s"
     connector_type: str = CONNECTOR_TYPE
 
+    @requires_dependencies(["pandas"], extras="singlestore")
     def prepare_data(
         self, columns: list[str], data: tuple[tuple[Any, ...], ...]
     ) -> list[tuple[Any, ...]]:
+        import pandas as pd
+
         output = []
         for row in data:
             parsed = []