unstructured-ingest 0.5.25__py3-none-any.whl → 0.6.1__py3-none-any.whl

unstructured_ingest/v2/pipeline/steps/partition.py

@@ -5,11 +5,10 @@ from pathlib import Path
 from typing import Callable, Optional, TypedDict
 
 from unstructured_ingest.utils.data_prep import write_data
-from unstructured_ingest.v2.interfaces import FileData
-from unstructured_ingest.v2.interfaces.file_data import file_data_from_file
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
 from unstructured_ingest.v2.processes.partitioner import Partitioner
+from unstructured_ingest.v2.types.file_data import FileData, file_data_from_file
 from unstructured_ingest.v2.utils import serialize_base_model_json
 
 STEP_ID = "partition"

unstructured_ingest/v2/pipeline/steps/stage.py

@@ -4,10 +4,10 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Callable, Optional, TypedDict
 
-from unstructured_ingest.v2.interfaces.file_data import file_data_from_file
-from unstructured_ingest.v2.interfaces.upload_stager import UploadStager
+from unstructured_ingest.v2.interfaces import UploadStager
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
+from unstructured_ingest.v2.types.file_data import file_data_from_file
 from unstructured_ingest.v2.utils import serialize_base_model_json
 
 STEP_ID = "upload_stage"

unstructured_ingest/v2/pipeline/steps/uncompress.py

@@ -3,10 +3,10 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Callable, TypedDict
 
-from unstructured_ingest.v2.interfaces.file_data import file_data_from_file
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
 from unstructured_ingest.v2.processes.uncompress import Uncompressor
+from unstructured_ingest.v2.types.file_data import file_data_from_file
 
 STEP_ID = "uncompress"
 

unstructured_ingest/v2/pipeline/steps/upload.py

@@ -3,11 +3,11 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Callable, Optional, TypedDict
 
-from unstructured_ingest.v2.interfaces.file_data import file_data_from_file
-from unstructured_ingest.v2.interfaces.uploader import UploadContent
+from unstructured_ingest.v2.interfaces import UploadContent
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import BatchPipelineStep
 from unstructured_ingest.v2.pipeline.otel import instrument
+from unstructured_ingest.v2.types.file_data import file_data_from_file
 
 STEP_ID = "upload"
 

unstructured_ingest/v2/processes/chunker.py

@@ -6,6 +6,7 @@ from typing import Any, Optional
 from pydantic import BaseModel, Field, SecretStr
 
 from unstructured_ingest.utils.chunking import assign_and_map_hash_ids
+from unstructured_ingest.utils.data_prep import get_json_data
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 from unstructured_ingest.v2.logger import logger
@@ -92,9 +93,11 @@ class Chunker(BaseProcess, ABC):
     @requires_dependencies(dependencies=["unstructured"])
     def run(self, elements_filepath: Path, **kwargs: Any) -> list[dict]:
         from unstructured.chunking import dispatch
-        from unstructured.staging.base import elements_from_json
+        from unstructured.staging.base import elements_from_dicts
 
-        elements = elements_from_json(filename=str(elements_filepath))
+        element_dicts = get_json_data(elements_filepath)
+
+        elements = elements_from_dicts(element_dicts=element_dicts)
         if not elements:
             return [e.to_dict() for e in elements]
         local_chunking_strategies = ("basic", "by_title")

unstructured_ingest/v2/processes/connectors/airtable.py

@@ -13,14 +13,13 @@ from unstructured_ingest.v2.interfaces import (
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
 )
 from unstructured_ingest.v2.processes.connector_registry import (
     SourceRegistryEntry,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 if TYPE_CHECKING:
     from pyairtable import Api

unstructured_ingest/v2/processes/connectors/astradb.py

@@ -21,17 +21,12 @@ from unstructured_ingest.utils.string_and_date_utils import truncate_string_byte
 from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
-    BatchFileData,
-    BatchItem,
     ConnectionConfig,
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
     Uploader,
     UploaderConfig,
     UploadStager,
@@ -44,6 +39,13 @@ from unstructured_ingest.v2.processes.connector_registry import (
     SourceRegistryEntry,
 )
 from unstructured_ingest.v2.processes.connectors.utils import format_and_truncate_orig_elements
+from unstructured_ingest.v2.types.file_data import (
+    BatchFileData,
+    BatchItem,
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 if TYPE_CHECKING:
     from astrapy import AsyncCollection as AstraDBAsyncCollection

unstructured_ingest/v2/processes/connectors/azure_ai_search.py

@@ -12,7 +12,6 @@ from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
-    FileData,
     Uploader,
     UploaderConfig,
     UploadStager,
@@ -23,6 +22,7 @@ from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
 )
 from unstructured_ingest.v2.processes.connectors.utils import parse_datetime
+from unstructured_ingest.v2.types.file_data import FileData
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
 if TYPE_CHECKING:

unstructured_ingest/v2/processes/connectors/chroma.py

@@ -12,7 +12,6 @@ from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
-    FileData,
     Uploader,
     UploaderConfig,
     UploadStager,
@@ -20,6 +19,7 @@ from unstructured_ingest.v2.interfaces import (
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.types.file_data import FileData
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
 from .utils import conform_string_to_dict

unstructured_ingest/v2/processes/connectors/confluence.py

@@ -15,17 +15,19 @@ from unstructured_ingest.v2.interfaces import (
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
     download_responses,
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     SourceRegistryEntry,
 )
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 if TYPE_CHECKING:
     from atlassian import Confluence

unstructured_ingest/v2/processes/connectors/couchbase.py

@@ -17,17 +17,12 @@ from unstructured_ingest.utils.data_prep import batch_generator, flatten_dict
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
-    BatchFileData,
-    BatchItem,
     ConnectionConfig,
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
     Uploader,
     UploaderConfig,
     UploadStager,
@@ -39,6 +34,13 @@ from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
+from unstructured_ingest.v2.types.file_data import (
+    BatchFileData,
+    BatchItem,
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 if TYPE_CHECKING:
     from couchbase.cluster import Cluster

unstructured_ingest/v2/processes/connectors/databricks/volumes.py

@@ -20,15 +20,17 @@ from unstructured_ingest.v2.interfaces import (
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
     Uploader,
     UploaderConfig,
 )
 from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 if TYPE_CHECKING:
     from databricks.sdk import WorkspaceClient

unstructured_ingest/v2/processes/connectors/databricks/volumes_table.py

@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any, Generator, Optional
 from pydantic import Field
 
 from unstructured_ingest.utils.data_prep import get_data_df, write_data
-from unstructured_ingest.v2.interfaces import FileData, Uploader, UploaderConfig
+from unstructured_ingest.v2.interfaces import Uploader, UploaderConfig
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
@@ -19,6 +19,7 @@ from unstructured_ingest.v2.processes.connectors.sql.databricks_delta_tables imp
     DatabricksDeltaTablesUploadStager,
     DatabricksDeltaTablesUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData
 
 CONNECTOR_TYPE = "databricks_volume_delta_tables"
 

unstructured_ingest/v2/processes/connectors/delta_table.py

@@ -15,7 +15,6 @@ from unstructured_ingest.utils.table import convert_to_pandas_dataframe
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
-    FileData,
     Uploader,
     UploaderConfig,
     UploadStager,
@@ -23,6 +22,7 @@ from unstructured_ingest.v2.interfaces import (
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.types.file_data import FileData
 
 CONNECTOR_TYPE = "delta_table"
 

unstructured_ingest/v2/processes/connectors/discord.py

@@ -12,14 +12,16 @@ from unstructured_ingest.v2.interfaces import (
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 if TYPE_CHECKING:
     from discord import Client as DiscordClient

unstructured_ingest/v2/processes/connectors/duckdb/base.py

@@ -4,7 +4,8 @@ from typing import Any
 
 from unstructured_ingest.utils.data_prep import get_data, write_data
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import FileData, UploadStager
+from unstructured_ingest.v2.interfaces import UploadStager
+from unstructured_ingest.v2.types.file_data import FileData
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
 _COLUMNS = (

unstructured_ingest/v2/processes/connectors/duckdb/duckdb.py

@@ -11,7 +11,6 @@ from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
-    FileData,
     Uploader,
     UploaderConfig,
     UploadStagerConfig,
@@ -19,6 +18,7 @@ from unstructured_ingest.v2.interfaces import (
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
 from unstructured_ingest.v2.processes.connectors.duckdb.base import BaseDuckDBUploadStager
+from unstructured_ingest.v2.types.file_data import FileData
 
 if TYPE_CHECKING:
     from duckdb import DuckDBPyConnection as DuckDBConnection

unstructured_ingest/v2/processes/connectors/duckdb/motherduck.py

@@ -12,7 +12,6 @@ from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
-    FileData,
     Uploader,
     UploaderConfig,
     UploadStagerConfig,
@@ -20,6 +19,7 @@ from unstructured_ingest.v2.interfaces import (
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
 from unstructured_ingest.v2.processes.connectors.duckdb.base import BaseDuckDBUploadStager
+from unstructured_ingest.v2.types.file_data import FileData
 
 if TYPE_CHECKING:
     from duckdb import DuckDBPyConnection as MotherDuckConnection

unstructured_ingest/v2/processes/connectors/elasticsearch/elasticsearch.py

@@ -23,17 +23,12 @@ from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
-    BatchFileData,
-    BatchItem,
     ConnectionConfig,
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
     Uploader,
     UploaderConfig,
     UploadStager,
@@ -45,6 +40,13 @@ from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
+from unstructured_ingest.v2.types.file_data import (
+    BatchFileData,
+    BatchItem,
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
 if TYPE_CHECKING:

unstructured_ingest/v2/processes/connectors/fsspec/azure.py

@@ -9,7 +9,6 @@ from pydantic import Field, Secret
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.errors import ProviderError, UserAuthError, UserError
-from unstructured_ingest.v2.interfaces import FileDataSourceMetadata
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
@@ -30,6 +29,7 @@ from unstructured_ingest.v2.processes.utils.blob_storage import (
     BlobStoreUploadStager,
     BlobStoreUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileDataSourceMetadata
 
 if TYPE_CHECKING:
     from adlfs import AzureBlobFileSystem

unstructured_ingest/v2/processes/connectors/fsspec/box.py

@@ -11,7 +11,6 @@ from pydantic.functional_validators import BeforeValidator
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.errors import ProviderError, UserAuthError, UserError
-from unstructured_ingest.v2.interfaces import FileDataSourceMetadata
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
@@ -32,6 +31,7 @@ from unstructured_ingest.v2.processes.utils.blob_storage import (
     BlobStoreUploadStager,
     BlobStoreUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileDataSourceMetadata
 
 if TYPE_CHECKING:
     from boxfs import BoxFileSystem

unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py

@@ -15,7 +15,6 @@ from unstructured_ingest.v2.errors import (
 from unstructured_ingest.v2.errors import (
     RateLimitError as CustomRateLimitError,
 )
-from unstructured_ingest.v2.interfaces import FileDataSourceMetadata
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
@@ -35,6 +34,7 @@ from unstructured_ingest.v2.processes.utils.blob_storage import (
     BlobStoreUploadStager,
     BlobStoreUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileDataSourceMetadata
 
 if TYPE_CHECKING:
     pass

unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py

@@ -18,16 +18,18 @@ from unstructured_ingest.v2.interfaces import (
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
     Uploader,
     UploaderConfig,
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connectors.fsspec.utils import sterilize_dict
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 if TYPE_CHECKING:
     from fsspec import AbstractFileSystem

unstructured_ingest/v2/processes/connectors/fsspec/gcs.py

@@ -12,7 +12,6 @@ from pydantic import Field, Secret
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.utils.string_and_date_utils import json_to_dict
 from unstructured_ingest.v2.errors import ProviderError, UserError
-from unstructured_ingest.v2.interfaces import FileDataSourceMetadata
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
@@ -32,6 +31,7 @@ from unstructured_ingest.v2.processes.utils.blob_storage import (
     BlobStoreUploadStager,
     BlobStoreUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileDataSourceMetadata
 
 if TYPE_CHECKING:
     from gcsfs import GCSFileSystem

unstructured_ingest/v2/processes/connectors/fsspec/s3.py

@@ -8,9 +8,6 @@ from pydantic import Field, Secret
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.errors import ProviderError, UserAuthError, UserError
-from unstructured_ingest.v2.interfaces import (
-    FileDataSourceMetadata,
-)
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
@@ -30,6 +27,9 @@ from unstructured_ingest.v2.processes.utils.blob_storage import (
     BlobStoreUploadStager,
     BlobStoreUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import (
+    FileDataSourceMetadata,
+)
 
 CONNECTOR_TYPE = "s3"
 

unstructured_ingest/v2/processes/connectors/fsspec/sftp.py

@@ -11,7 +11,6 @@ from urllib.parse import urlparse
 from pydantic import Field, Secret
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import FileData, FileDataSourceMetadata
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
@@ -30,6 +29,7 @@ from unstructured_ingest.v2.processes.utils.blob_storage import (
     BlobStoreUploadStager,
     BlobStoreUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, FileDataSourceMetadata
 
 if TYPE_CHECKING:
     from fsspec.implementations.sftp import SFTPFileSystem

unstructured_ingest/v2/processes/connectors/gitlab.py

@@ -16,14 +16,16 @@ from unstructured_ingest.v2.interfaces import (
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 CONNECTOR_TYPE = "gitlab"
 if TYPE_CHECKING:

unstructured_ingest/v2/processes/connectors/google_drive.py

@@ -21,15 +21,17 @@ from unstructured_ingest.v2.interfaces import (
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
 from unstructured_ingest.v2.processes.connectors.utils import conform_string_to_dict
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 CONNECTOR_TYPE = "google_drive"
 

unstructured_ingest/v2/processes/connectors/ibm_watsonx/ibm_watsonx_s3.py

@@ -15,7 +15,6 @@ from unstructured_ingest.v2.errors import ProviderError, UserAuthError, UserErro
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
-    FileData,
     UploaderConfig,
 )
 from unstructured_ingest.v2.logger import logger
@@ -27,6 +26,7 @@ from unstructured_ingest.v2.processes.connectors.sql.sql import (
     SQLUploadStager,
     SQLUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData
 
 if TYPE_CHECKING:
     from pyarrow import Table as ArrowTable

unstructured_ingest/v2/processes/connectors/jira.py

@@ -15,16 +15,18 @@ from unstructured_ingest.v2.interfaces import (
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     SourceRegistryEntry,
 )
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 if TYPE_CHECKING:
     from atlassian import Jira

unstructured_ingest/v2/processes/connectors/kafka/kafka.py

@@ -21,15 +21,17 @@ from unstructured_ingest.v2.interfaces import (
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
     Uploader,
     UploaderConfig,
 )
 from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 if TYPE_CHECKING:
     from confluent_kafka import Consumer, Producer

unstructured_ingest/v2/processes/connectors/kdbai.py

@@ -11,7 +11,6 @@ from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
-    FileData,
     Uploader,
     UploaderConfig,
     UploadStager,
@@ -21,6 +20,7 @@ from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
 )
+from unstructured_ingest.v2.types.file_data import FileData
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
 if TYPE_CHECKING:

unstructured_ingest/v2/processes/connectors/lancedb/lancedb.py

@@ -15,10 +15,14 @@ from unstructured_ingest.logger import logger
 from unstructured_ingest.utils.data_prep import flatten_dict
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.constants import RECORD_ID_LABEL
-from unstructured_ingest.v2.interfaces.connector import ConnectionConfig
-from unstructured_ingest.v2.interfaces.file_data import FileData
-from unstructured_ingest.v2.interfaces.upload_stager import UploadStager, UploadStagerConfig
-from unstructured_ingest.v2.interfaces.uploader import Uploader, UploaderConfig
+from unstructured_ingest.v2.interfaces import (
+    ConnectionConfig,
+    Uploader,
+    UploaderConfig,
+    UploadStager,
+    UploadStagerConfig,
+)
+from unstructured_ingest.v2.types.file_data import FileData
 
 CONNECTOR_TYPE = "lancedb"
 

unstructured_ingest/v2/processes/connectors/local.py

@@ -14,11 +14,8 @@ from unstructured_ingest.v2.interfaces import (
     Downloader,
     DownloaderConfig,
     DownloadResponse,
-    FileData,
-    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
-    SourceIdentifiers,
     Uploader,
     UploaderConfig,
 )
@@ -31,6 +28,11 @@ from unstructured_ingest.v2.processes.utils.blob_storage import (
     BlobStoreUploadStager,
     BlobStoreUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 CONNECTOR_TYPE = "local"