unstructured-ingest 0.5.23__py3-none-any.whl → 0.6.0__py3-none-any.whl

test/unit/v2/connectors/motherduck/test_base.py

@@ -3,10 +3,9 @@ from pathlib import Path
 import pytest
 from pytest_mock import MockerFixture
 
-from unstructured_ingest.v2.interfaces import FileData
-from unstructured_ingest.v2.interfaces.file_data import SourceIdentifiers
 from unstructured_ingest.v2.interfaces.upload_stager import UploadStagerConfig
 from unstructured_ingest.v2.processes.connectors.duckdb.base import BaseDuckDBUploadStager
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 @pytest.fixture

test/unit/v2/connectors/sql/test_sql.py

@@ -4,13 +4,13 @@ import pandas as pd
 import pytest
 from pytest_mock import MockerFixture
 
-from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.sql.sql import (
     SQLConnectionConfig,
     SQLUploader,
     SQLUploaderConfig,
     SQLUploadStager,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 @pytest.fixture

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.5.23"  # pragma: no cover
+__version__ = "0.6.0"  # pragma: no cover

unstructured_ingest/utils/html.py

@@ -7,8 +7,9 @@ from uuid import NAMESPACE_DNS, uuid5
 from pydantic import BaseModel, Field
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, SourceIdentifiers
+from unstructured_ingest.v2.interfaces import DownloadResponse
 from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 if TYPE_CHECKING:
     from bs4.element import Tag

unstructured_ingest/v2/interfaces/__init__.py

@@ -1,11 +1,3 @@
-from unstructured_ingest.v2.types.file_data import (
-    BatchFileData,
-    BatchItem,
-    FileData,
-    FileDataSourceMetadata,
-    SourceIdentifiers,
-)
-
 from .connector import AccessConfig, BaseConnector, ConnectionConfig
 from .downloader import Downloader, DownloaderConfig, DownloadResponse, download_responses
 from .indexer import Indexer, IndexerConfig
@@ -19,7 +11,6 @@ __all__ = [
     "download_responses",
     "Downloader",
     "DownloaderConfig",
-    "FileData",
     "Indexer",
     "IndexerConfig",
     "BaseProcess",
@@ -28,13 +19,9 @@ __all__ = [
     "UploadStagerConfig",
     "Uploader",
     "UploaderConfig",
-    "SourceIdentifiers",
     "UploadContent",
     "AccessConfig",
     "ConnectionConfig",
     "BaseConnector",
-    "FileDataSourceMetadata",
-    "BatchFileData",
-    "BatchItem",
     "VectorDBUploader",
 ]

unstructured_ingest/v2/interfaces/downloader.py

@@ -6,8 +6,8 @@ from typing import Any, Optional, TypedDict, TypeVar, Union
 from pydantic import BaseModel, Field
 
 from unstructured_ingest.v2.interfaces.connector import BaseConnector
-from unstructured_ingest.v2.interfaces.file_data import FileData
 from unstructured_ingest.v2.interfaces.process import BaseProcess
+from unstructured_ingest.v2.types.file_data import FileData
 
 
 class DownloaderConfig(BaseModel):

unstructured_ingest/v2/interfaces/indexer.py

@@ -4,8 +4,8 @@ from typing import Any, AsyncGenerator, Generator, Optional, TypeVar
 from pydantic import BaseModel
 
 from unstructured_ingest.v2.interfaces.connector import BaseConnector
-from unstructured_ingest.v2.interfaces.file_data import FileData
 from unstructured_ingest.v2.interfaces.process import BaseProcess
+from unstructured_ingest.v2.types.file_data import FileData
 
 
 class IndexerConfig(BaseModel):

unstructured_ingest/v2/interfaces/upload_stager.py

@@ -7,8 +7,8 @@ from pydantic import BaseModel
 
 from unstructured_ingest.utils import ndjson
 from unstructured_ingest.utils.data_prep import get_data, write_data
-from unstructured_ingest.v2.interfaces.file_data import FileData
-from unstructured_ingest.v2.interfaces.process import BaseProcess
+from unstructured_ingest.v2.interfaces import BaseProcess
+from unstructured_ingest.v2.types.file_data import FileData
 
 
 class UploadStagerConfig(BaseModel):

unstructured_ingest/v2/interfaces/uploader.py

@@ -6,9 +6,8 @@ from typing import Any, TypeVar
 from pydantic import BaseModel
 
 from unstructured_ingest.utils.data_prep import get_data
-from unstructured_ingest.v2.interfaces.connector import BaseConnector
-from unstructured_ingest.v2.interfaces.file_data import FileData
-from unstructured_ingest.v2.interfaces.process import BaseProcess
+from unstructured_ingest.v2.interfaces import BaseConnector, BaseProcess
+from unstructured_ingest.v2.types.file_data import FileData
 
 
 class UploaderConfig(BaseModel):

unstructured_ingest/v2/pipeline/steps/chunk.py

@@ -5,11 +5,10 @@ from pathlib import Path
 from typing import Callable, Optional, TypedDict
 
 from unstructured_ingest.utils.data_prep import write_data
-from unstructured_ingest.v2.interfaces import FileData
-from unstructured_ingest.v2.interfaces.file_data import file_data_from_file
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
 from unstructured_ingest.v2.processes.chunker import Chunker
+from unstructured_ingest.v2.types.file_data import FileData, file_data_from_file
 from unstructured_ingest.v2.utils import serialize_base_model_json
 
 STEP_ID = "chunk"

unstructured_ingest/v2/pipeline/steps/download.py

@@ -6,11 +6,10 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Callable, Optional, TypedDict, TypeVar
 
-from unstructured_ingest.v2.interfaces import FileData, download_responses
-from unstructured_ingest.v2.interfaces.downloader import Downloader
-from unstructured_ingest.v2.interfaces.file_data import file_data_from_file
+from unstructured_ingest.v2.interfaces import Downloader, download_responses
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
+from unstructured_ingest.v2.types.file_data import FileData, file_data_from_file
 from unstructured_ingest.v2.utils import serialize_base_model_json
 
 DownloaderT = TypeVar("DownloaderT", bound=Downloader)

unstructured_ingest/v2/pipeline/steps/embed.py

@@ -5,11 +5,10 @@ from pathlib import Path
 from typing import Callable, Optional, TypedDict
 
 from unstructured_ingest.utils.data_prep import write_data
-from unstructured_ingest.v2.interfaces import FileData
-from unstructured_ingest.v2.interfaces.file_data import file_data_from_file
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
 from unstructured_ingest.v2.processes.embedder import Embedder
+from unstructured_ingest.v2.types.file_data import FileData, file_data_from_file
 from unstructured_ingest.v2.utils import serialize_base_model_json
 
 STEP_ID = "embed"

unstructured_ingest/v2/pipeline/steps/filter.py

@@ -2,10 +2,10 @@ import asyncio
 from dataclasses import dataclass
 from typing import Callable, Optional
 
-from unstructured_ingest.v2.interfaces.file_data import file_data_from_file
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
 from unstructured_ingest.v2.processes.filter import Filterer
+from unstructured_ingest.v2.types.file_data import file_data_from_file
 
 STEP_ID = "filter"
 

unstructured_ingest/v2/pipeline/steps/partition.py

@@ -5,11 +5,10 @@ from pathlib import Path
 from typing import Callable, Optional, TypedDict
 
 from unstructured_ingest.utils.data_prep import write_data
-from unstructured_ingest.v2.interfaces import FileData
-from unstructured_ingest.v2.interfaces.file_data import file_data_from_file
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
 from unstructured_ingest.v2.processes.partitioner import Partitioner
+from unstructured_ingest.v2.types.file_data import FileData, file_data_from_file
 from unstructured_ingest.v2.utils import serialize_base_model_json
 
 STEP_ID = "partition"

unstructured_ingest/v2/pipeline/steps/stage.py

@@ -4,10 +4,10 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Callable, Optional, TypedDict
 
-from unstructured_ingest.v2.interfaces.file_data import file_data_from_file
-from unstructured_ingest.v2.interfaces.upload_stager import UploadStager
+from unstructured_ingest.v2.interfaces import UploadStager
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
+from unstructured_ingest.v2.types.file_data import file_data_from_file
 from unstructured_ingest.v2.utils import serialize_base_model_json
 
 STEP_ID = "upload_stage"

unstructured_ingest/v2/pipeline/steps/uncompress.py

@@ -3,10 +3,10 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Callable, TypedDict
 
-from unstructured_ingest.v2.interfaces.file_data import file_data_from_file
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
 from unstructured_ingest.v2.processes.uncompress import Uncompressor
+from unstructured_ingest.v2.types.file_data import file_data_from_file
 
 STEP_ID = "uncompress"
 

unstructured_ingest/v2/pipeline/steps/upload.py

@@ -3,11 +3,11 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Callable, Optional, TypedDict
 
-from unstructured_ingest.v2.interfaces.file_data import file_data_from_file
-from unstructured_ingest.v2.interfaces.uploader import UploadContent
+from unstructured_ingest.v2.interfaces import UploadContent
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import BatchPipelineStep
 from unstructured_ingest.v2.pipeline.otel import instrument
+from unstructured_ingest.v2.types.file_data import file_data_from_file
 
 STEP_ID = "upload"
 

unstructured_ingest/v2/processes/connectors/airtable.py

@@ -13,14 +13,13 @@ from unstructured_ingest.v2.interfaces import (
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
 )
 from unstructured_ingest.v2.processes.connector_registry import (
     SourceRegistryEntry,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 if TYPE_CHECKING:
     from pyairtable import Api

unstructured_ingest/v2/processes/connectors/astradb.py

@@ -21,17 +21,12 @@ from unstructured_ingest.utils.string_and_date_utils import truncate_string_byte
 from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
-    BatchFileData,
-    BatchItem,
     ConnectionConfig,
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
     Uploader,
     UploaderConfig,
     UploadStager,
@@ -44,6 +39,13 @@ from unstructured_ingest.v2.processes.connector_registry import (
     SourceRegistryEntry,
 )
 from unstructured_ingest.v2.processes.connectors.utils import format_and_truncate_orig_elements
+from unstructured_ingest.v2.types.file_data import (
+    BatchFileData,
+    BatchItem,
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 if TYPE_CHECKING:
     from astrapy import AsyncCollection as AstraDBAsyncCollection

unstructured_ingest/v2/processes/connectors/azure_ai_search.py

@@ -12,7 +12,6 @@ from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
-    FileData,
     Uploader,
     UploaderConfig,
     UploadStager,
@@ -23,6 +22,7 @@ from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
 )
 from unstructured_ingest.v2.processes.connectors.utils import parse_datetime
+from unstructured_ingest.v2.types.file_data import FileData
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
 if TYPE_CHECKING:

unstructured_ingest/v2/processes/connectors/chroma.py

@@ -12,7 +12,6 @@ from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
-    FileData,
     Uploader,
     UploaderConfig,
     UploadStager,
@@ -20,6 +19,7 @@ from unstructured_ingest.v2.interfaces import (
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.types.file_data import FileData
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
 from .utils import conform_string_to_dict

unstructured_ingest/v2/processes/connectors/confluence.py

@@ -15,17 +15,19 @@ from unstructured_ingest.v2.interfaces import (
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
     download_responses,
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     SourceRegistryEntry,
 )
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 if TYPE_CHECKING:
     from atlassian import Confluence

unstructured_ingest/v2/processes/connectors/couchbase.py

@@ -17,17 +17,12 @@ from unstructured_ingest.utils.data_prep import batch_generator, flatten_dict
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
-    BatchFileData,
-    BatchItem,
     ConnectionConfig,
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
     Uploader,
     UploaderConfig,
     UploadStager,
@@ -39,6 +34,13 @@ from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
+from unstructured_ingest.v2.types.file_data import (
+    BatchFileData,
+    BatchItem,
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 if TYPE_CHECKING:
     from couchbase.cluster import Cluster

unstructured_ingest/v2/processes/connectors/databricks/volumes.py

@@ -20,15 +20,17 @@ from unstructured_ingest.v2.interfaces import (
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
     Uploader,
     UploaderConfig,
 )
 from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 if TYPE_CHECKING:
     from databricks.sdk import WorkspaceClient

unstructured_ingest/v2/processes/connectors/databricks/volumes_table.py

@@ -1,14 +1,14 @@
-import json
 import os
+import tempfile
 from contextlib import contextmanager
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Generator
+from typing import TYPE_CHECKING, Any, Generator, Optional
 
 from pydantic import Field
 
-from unstructured_ingest.utils.data_prep import write_data
-from unstructured_ingest.v2.interfaces import FileData, Uploader, UploaderConfig
+from unstructured_ingest.utils.data_prep import get_data_df, write_data
+from unstructured_ingest.v2.interfaces import Uploader, UploaderConfig
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
@@ -19,9 +19,13 @@ from unstructured_ingest.v2.processes.connectors.sql.databricks_delta_tables imp
     DatabricksDeltaTablesUploadStager,
     DatabricksDeltaTablesUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData
 
 CONNECTOR_TYPE = "databricks_volume_delta_tables"
 
+if TYPE_CHECKING:
+    from pandas import DataFrame
+
 
 class DatabricksVolumeDeltaTableUploaderConfig(UploaderConfig, DatabricksPathMixin):
     database: str = Field(description="Database name", default="default")
@@ -30,10 +34,12 @@ class DatabricksVolumeDeltaTableUploaderConfig(UploaderConfig, DatabricksPathMix
 
 @dataclass
 class DatabricksVolumeDeltaTableStager(DatabricksDeltaTablesUploadStager):
-    def write_output(self, output_path: Path, data: list[dict]) -> None:
+    def write_output(self, output_path: Path, data: list[dict]) -> Path:
         # To avoid new line issues when migrating from volumes into delta tables, omit indenting
         # and always write it as a json file
-        write_data(path=output_path.with_suffix(".json"), data=data, indent=None)
+        final_output_path = output_path.with_suffix(".json")
+        write_data(path=final_output_path, data=data, indent=None)
+        return final_output_path
 
 
 @dataclass
@@ -41,6 +47,7 @@ class DatabricksVolumeDeltaTableUploader(Uploader):
     connection_config: DatabricksDeltaTablesConnectionConfig
     upload_config: DatabricksVolumeDeltaTableUploaderConfig
     connector_type: str = CONNECTOR_TYPE
+    _columns: Optional[dict[str, str]] = None
 
     def precheck(self) -> None:
         with self.connection_config.get_cursor() as cursor:
@@ -84,20 +91,58 @@ class DatabricksVolumeDeltaTableUploader(Uploader):
             cursor.execute(f"USE DATABASE {self.upload_config.database}")
             yield cursor
 
-    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
-        with self.get_cursor(staging_allowed_local_path=str(path.parent)) as cursor:
-            catalog_path = self.get_output_path(file_data=file_data)
-            logger.debug(f"uploading {path.as_posix()} to {catalog_path}")
-            cursor.execute(f"PUT '{path.as_posix()}' INTO '{catalog_path}' OVERWRITE")
-            logger.debug(
-                f"migrating content from {catalog_path} to table {self.upload_config.table_name}"
+    def get_table_columns(self) -> dict[str, str]:
+        if self._columns is None:
+            with self.get_cursor() as cursor:
+                cursor.execute(f"SELECT * from {self.upload_config.table_name} LIMIT 1")
+                self._columns = {desc[0]: desc[1] for desc in cursor.description}
+        return self._columns
+
+    def _fit_to_schema(self, df: "DataFrame", add_missing_columns: bool = True) -> "DataFrame":
+        import pandas as pd
+
+        table_columns = self.get_table_columns()
+        columns = set(df.columns)
+        schema_fields = set(table_columns.keys())
+        columns_to_drop = columns - schema_fields
+        missing_columns = schema_fields - columns
+
+        if columns_to_drop:
+            logger.info(
+                "Following columns will be dropped to match the table's schema: "
+                f"{', '.join(columns_to_drop)}"
+            )
+        if missing_columns and add_missing_columns:
+            logger.info(
+                "Following null filled columns will be added to match the table's schema:"
+                f" {', '.join(missing_columns)} "
             )
-            with path.open() as f:
-                data = json.load(f)
-                columns = data[0].keys()
-            column_str = ", ".join(columns)
-            sql_statment = f"INSERT INTO `{self.upload_config.table_name}` ({column_str}) SELECT {column_str} FROM json.`{catalog_path}`"  # noqa: E501
-            cursor.execute(sql_statment)
+
+        df = df.drop(columns=columns_to_drop)
+
+        if add_missing_columns:
+            for column in missing_columns:
+                df[column] = pd.Series()
+        return df
+
+    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            df = get_data_df()
+            df = self._fit_to_schema(df=df)
+            temp_path = Path(temp_dir) / path.name
+            df.to_json(temp_path, orient="records", lines=False)
+            with self.get_cursor(staging_allowed_local_path=temp_dir) as cursor:
+                catalog_path = self.get_output_path(file_data=file_data)
+                logger.debug(f"uploading {path.as_posix()} to {catalog_path}")
+                cursor.execute(f"PUT '{temp_path.as_posix()}' INTO '{catalog_path}' OVERWRITE")
+                logger.debug(
+                    f"migrating content from {catalog_path} to "
+                    f"table {self.upload_config.table_name}"
+                )
+                columns = list(df.columns)
+                column_str = ", ".join(columns)
+                sql_statment = f"INSERT INTO `{self.upload_config.table_name}` ({column_str}) SELECT {column_str} FROM json.`{catalog_path}`"  # noqa: E501
+                cursor.execute(sql_statment)
 
 
 databricks_volumes_delta_tables_destination_entry = DestinationRegistryEntry(

unstructured_ingest/v2/processes/connectors/delta_table.py

@@ -15,7 +15,6 @@ from unstructured_ingest.utils.table import convert_to_pandas_dataframe
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
-    FileData,
     Uploader,
     UploaderConfig,
     UploadStager,
@@ -23,6 +22,7 @@ from unstructured_ingest.v2.interfaces import (
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.types.file_data import FileData
 
 CONNECTOR_TYPE = "delta_table"
 

unstructured_ingest/v2/processes/connectors/discord.py

@@ -12,14 +12,16 @@ from unstructured_ingest.v2.interfaces import (
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 if TYPE_CHECKING:
     from discord import Client as DiscordClient

unstructured_ingest/v2/processes/connectors/duckdb/base.py

@@ -4,7 +4,8 @@ from typing import Any
 
 from unstructured_ingest.utils.data_prep import get_data, write_data
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import FileData, UploadStager
+from unstructured_ingest.v2.interfaces import UploadStager
+from unstructured_ingest.v2.types.file_data import FileData
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
 _COLUMNS = (

unstructured_ingest/v2/processes/connectors/duckdb/duckdb.py

@@ -11,7 +11,6 @@ from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
-    FileData,
     Uploader,
     UploaderConfig,
     UploadStagerConfig,
@@ -19,6 +18,7 @@ from unstructured_ingest.v2.interfaces import (
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
 from unstructured_ingest.v2.processes.connectors.duckdb.base import BaseDuckDBUploadStager
+from unstructured_ingest.v2.types.file_data import FileData
 
 if TYPE_CHECKING:
     from duckdb import DuckDBPyConnection as DuckDBConnection

unstructured_ingest/v2/processes/connectors/duckdb/motherduck.py

@@ -12,7 +12,6 @@ from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
-    FileData,
     Uploader,
     UploaderConfig,
     UploadStagerConfig,
@@ -20,6 +19,7 @@ from unstructured_ingest.v2.interfaces import (
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
 from unstructured_ingest.v2.processes.connectors.duckdb.base import BaseDuckDBUploadStager
+from unstructured_ingest.v2.types.file_data import FileData
 
 if TYPE_CHECKING:
     from duckdb import DuckDBPyConnection as MotherDuckConnection

unstructured_ingest/v2/processes/connectors/elasticsearch/elasticsearch.py

@@ -23,17 +23,12 @@ from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
-    BatchFileData,
-    BatchItem,
     ConnectionConfig,
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
     Uploader,
     UploaderConfig,
     UploadStager,
@@ -45,6 +40,13 @@ from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
+from unstructured_ingest.v2.types.file_data import (
+    BatchFileData,
+    BatchItem,
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
 if TYPE_CHECKING:

unstructured_ingest/v2/processes/connectors/fsspec/azure.py

@@ -9,7 +9,6 @@ from pydantic import Field, Secret
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.errors import ProviderError, UserAuthError, UserError
-from unstructured_ingest.v2.interfaces import FileDataSourceMetadata
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
@@ -30,6 +29,7 @@ from unstructured_ingest.v2.processes.utils.blob_storage import (
     BlobStoreUploadStager,
     BlobStoreUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileDataSourceMetadata
 
 if TYPE_CHECKING:
     from adlfs import AzureBlobFileSystem