unstructured-ingest 0.5.25__py3-none-any.whl → 0.6.0__py3-none-any.whl

test/integration/connectors/databricks/test_volumes_native.py

@@ -21,7 +21,6 @@ from test.integration.connectors.utils.validation.source import (
 )
 from test.integration.utils import requires_env
 from unstructured_ingest.v2.errors import UserAuthError, UserError
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.databricks.volumes_native import (
     CONNECTOR_TYPE,
     DatabricksNativeVolumesAccessConfig,
@@ -33,6 +32,7 @@ from unstructured_ingest.v2.processes.connectors.databricks.volumes_native impor
     DatabricksNativeVolumesUploader,
     DatabricksNativeVolumesUploaderConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 @dataclass

test/integration/connectors/duckdb/test_duckdb.py

@@ -10,7 +10,6 @@ from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
     stager_validation,
 )
-from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.duckdb.duckdb import (
     CONNECTOR_TYPE,
     DuckDBConnectionConfig,
@@ -18,6 +17,7 @@ from unstructured_ingest.v2.processes.connectors.duckdb.duckdb import (
     DuckDBUploaderConfig,
     DuckDBUploadStager,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 @pytest.fixture

test/integration/connectors/duckdb/test_motherduck.py

@@ -9,7 +9,6 @@ import pytest
 
 from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.duckdb.motherduck import (
     CONNECTOR_TYPE,
     MotherDuckAccessConfig,
@@ -18,6 +17,7 @@ from unstructured_ingest.v2.processes.connectors.duckdb.motherduck import (
     MotherDuckUploaderConfig,
     MotherDuckUploadStager,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 @pytest.fixture

test/integration/connectors/elasticsearch/test_elasticsearch.py

@@ -22,7 +22,7 @@ from test.integration.connectors.utils.validation.source import (
     source_connector_validation,
 )
 from unstructured_ingest.error import DestinationConnectionError, SourceConnectionError
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.elasticsearch.elasticsearch import (
     CONNECTOR_TYPE,
     ElasticsearchAccessConfig,

test/integration/connectors/elasticsearch/test_opensearch.py

@@ -24,7 +24,6 @@ from unstructured_ingest.error import (
     DestinationConnectionError,
     SourceConnectionError,
 )
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.elasticsearch.opensearch import (
     CONNECTOR_TYPE,
     OpenSearchAccessConfig,
@@ -38,6 +37,7 @@ from unstructured_ingest.v2.processes.connectors.elasticsearch.opensearch import
     OpenSearchUploadStager,
     OpenSearchUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 SOURCE_INDEX_NAME = "movies"
 DESTINATION_INDEX_NAME = "elements"

test/integration/connectors/sql/test_databricks_delta_tables.py

@@ -14,7 +14,6 @@ from pytest_mock import MockerFixture
 
 from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG, env_setup_path
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connectors.sql.databricks_delta_tables import (
     CONNECTOR_TYPE,
@@ -24,6 +23,7 @@ from unstructured_ingest.v2.processes.connectors.sql.databricks_delta_tables imp
     DatabricksDeltaTablesUploaderConfig,
     DatabricksDeltaTablesUploadStager,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 CATALOG = "utic-dev-tech-fixtures"
 

test/integration/connectors/sql/test_postgres.py

@@ -20,7 +20,6 @@ from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
 )
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.sql.postgres import (
     CONNECTOR_TYPE,
     PostgresAccessConfig,
@@ -32,6 +31,7 @@ from unstructured_ingest.v2.processes.connectors.sql.postgres import (
     PostgresUploader,
     PostgresUploadStager,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 SEED_DATA_ROWS = 10
 

test/integration/connectors/sql/test_singlestore.py

@@ -20,7 +20,6 @@ from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
 )
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.sql.singlestore import (
     CONNECTOR_TYPE,
     SingleStoreAccessConfig,
@@ -33,6 +32,7 @@ from unstructured_ingest.v2.processes.connectors.sql.singlestore import (
     SingleStoreUploaderConfig,
     SingleStoreUploadStager,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 SEED_DATA_ROWS = 10
 

test/integration/connectors/sql/test_snowflake.py

@@ -22,7 +22,6 @@ from test.integration.connectors.utils.validation.source import (
     source_connector_validation,
 )
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.sql.snowflake import (
     CONNECTOR_TYPE,
     SnowflakeAccessConfig,
@@ -34,6 +33,7 @@ from unstructured_ingest.v2.processes.connectors.sql.snowflake import (
     SnowflakeUploader,
     SnowflakeUploadStager,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 SEED_DATA_ROWS = 20
 

test/integration/connectors/sql/test_sqlite.py

@@ -20,7 +20,6 @@ from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
 )
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.sql.sqlite import (
     CONNECTOR_TYPE,
     SQLiteConnectionConfig,
@@ -31,6 +30,7 @@ from unstructured_ingest.v2.processes.connectors.sql.sqlite import (
     SQLiteUploader,
     SQLiteUploadStager,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 SEED_DATA_ROWS = 10
 

test/integration/connectors/test_astradb.py

@@ -20,7 +20,6 @@ from test.integration.connectors.utils.validation.source import (
     source_connector_validation,
 )
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.astradb import (
     CONNECTOR_TYPE,
     AstraDBAccessConfig,
@@ -36,6 +35,7 @@ from unstructured_ingest.v2.processes.connectors.astradb import (
     DestinationConnectionError,
     SourceConnectionError,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 EXISTENT_COLLECTION_NAME = "ingest_test_src"
 NONEXISTENT_COLLECTION_NAME = "nonexistant"

test/integration/connectors/test_azure_ai_search.py

@@ -29,7 +29,6 @@ from test.integration.connectors.utils.validation.destination import (
     stager_validation,
 )
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.azure_ai_search import (
     CONNECTOR_TYPE,
     RECORD_ID_LABEL,
@@ -40,6 +39,7 @@ from unstructured_ingest.v2.processes.connectors.azure_ai_search import (
     AzureAISearchUploadStager,
     AzureAISearchUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 repo_path = Path(__file__).parent.resolve()
 

test/integration/connectors/test_chroma.py

@@ -27,7 +27,7 @@ from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
     stager_validation,
 )
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.chroma import (
     CONNECTOR_TYPE,
     ChromaConnectionConfig,

test/integration/connectors/test_delta_table.py

@@ -8,7 +8,6 @@ from fsspec import get_filesystem_class
 
 from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.delta_table import (
     CONNECTOR_TYPE,
     DeltaTableAccessConfig,
@@ -18,6 +17,7 @@ from unstructured_ingest.v2.processes.connectors.delta_table import (
     DeltaTableUploadStager,
     DeltaTableUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 multiprocessing.set_start_method("spawn")
 

test/integration/connectors/test_lancedb.py

@@ -13,7 +13,6 @@ from upath import UPath
 
 from test.integration.connectors.utils.constants import DESTINATION_TAG, VECTOR_DB_TAG
 from unstructured_ingest.v2.constants import RECORD_ID_LABEL
-from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.lancedb.aws import (
     LanceDBAwsAccessConfig,
     LanceDBAwsConnectionConfig,
@@ -39,6 +38,7 @@ from unstructured_ingest.v2.processes.connectors.lancedb.local import (
     LanceDBLocalConnectionConfig,
     LanceDBLocalUploader,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 DATABASE_NAME = "database"
 TABLE_NAME = "elements"

test/integration/connectors/test_milvus.py

@@ -25,7 +25,6 @@ from test.integration.connectors.utils.validation.destination import (
     stager_validation,
 )
 from unstructured_ingest.error import DestinationConnectionError
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.milvus import (
     CONNECTOR_TYPE,
     MilvusConnectionConfig,
@@ -33,6 +32,7 @@ from unstructured_ingest.v2.processes.connectors.milvus import (
     MilvusUploaderConfig,
     MilvusUploadStager,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 DB_NAME = "test_database"
 EXISTENT_COLLECTION_NAME = "test_collection"

test/integration/connectors/test_mongodb.py

@@ -20,7 +20,6 @@ from test.integration.connectors.utils.validation.source import (
 )
 from test.integration.utils import requires_env
 from unstructured_ingest.error import DestinationConnectionError, SourceConnectionError
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.mongodb import (
     CONNECTOR_TYPE,
     MongoDBAccessConfig,
@@ -32,6 +31,7 @@ from unstructured_ingest.v2.processes.connectors.mongodb import (
     MongoDBUploader,
     MongoDBUploaderConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 SOURCE_COLLECTION = "sample-mongodb-data"
 

test/integration/connectors/test_neo4j.py

@@ -13,11 +13,6 @@ from test.integration.connectors.utils.constants import DESTINATION_TAG, GRAPH_D
 from test.integration.connectors.utils.docker import container_context
 from unstructured_ingest.error import DestinationConnectionError
 from unstructured_ingest.utils.chunking import elements_from_base64_gzipped_json
-from unstructured_ingest.v2.interfaces.file_data import (
-    FileData,
-    FileDataSourceMetadata,
-    SourceIdentifiers,
-)
 from unstructured_ingest.v2.processes.connectors.neo4j import (
     CONNECTOR_TYPE,
     Label,
@@ -28,6 +23,11 @@ from unstructured_ingest.v2.processes.connectors.neo4j import (
     Neo4jUploadStager,
     Relationship,
 )
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 USERNAME = "neo4j"
 PASSWORD = "password"

test/integration/connectors/test_onedrive.py

@@ -15,7 +15,6 @@ from test.integration.connectors.utils.validation.source import (
     source_connector_validation,
 )
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.onedrive import (
     CONNECTOR_TYPE,
     OnedriveAccessConfig,
@@ -27,6 +26,7 @@ from unstructured_ingest.v2.processes.connectors.onedrive import (
     OnedriveUploader,
     OnedriveUploaderConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 @pytest.fixture

test/integration/connectors/test_pinecone.py

@@ -19,7 +19,6 @@ from test.integration.connectors.utils.validation.destination import (
 )
 from test.integration.utils import requires_env
 from unstructured_ingest.error import DestinationConnectionError
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connectors.pinecone import (
     CONNECTOR_TYPE,
@@ -31,6 +30,7 @@ from unstructured_ingest.v2.processes.connectors.pinecone import (
     PineconeUploadStager,
     PineconeUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 METADATA_BYTES_LIMIT = (
     40960  # 40KB https://docs.pinecone.io/reference/quotas-and-limits#hard-limits

test/integration/connectors/test_qdrant.py

@@ -16,7 +16,6 @@ from test.integration.connectors.utils.validation.destination import (
     stager_validation,
 )
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.qdrant.cloud import (
     CloudQdrantAccessConfig,
     CloudQdrantConnectionConfig,
@@ -45,6 +44,7 @@ from unstructured_ingest.v2.processes.connectors.qdrant.server import (
     ServerQdrantUploadStager,
     ServerQdrantUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 COLLECTION_NAME = f"test-coll-{uuid.uuid4().hex[:12]}"
 VECTORS_CONFIG = {"size": 384, "distance": "Cosine"}

test/integration/connectors/test_redis.py

@@ -11,7 +11,6 @@ from redis.asyncio import Redis, from_url
 
 from test.integration.connectors.utils.constants import DESTINATION_TAG, NOSQL_TAG
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.redisdb import (
     CONNECTOR_TYPE as REDIS_CONNECTOR_TYPE,
 )
@@ -21,6 +20,7 @@ from unstructured_ingest.v2.processes.connectors.redisdb import (
     RedisUploader,
     RedisUploaderConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 async def delete_record(client: Redis, element_id: str, key_prefix: str) -> None:

test/integration/connectors/test_s3.py

@@ -18,7 +18,6 @@ from test.integration.connectors.utils.validation.source import (
 )
 from test.integration.utils import requires_env
 from unstructured_ingest.v2.errors import UserAuthError, UserError
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.fsspec.s3 import (
     CONNECTOR_TYPE,
     S3AccessConfig,
@@ -30,6 +29,7 @@ from unstructured_ingest.v2.processes.connectors.fsspec.s3 import (
     S3Uploader,
     S3UploaderConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 def validate_predownload_file_data(file_data: FileData):

test/integration/connectors/test_vectara.py

@@ -11,7 +11,6 @@ import requests
 
 from test.integration.connectors.utils.constants import DESTINATION_TAG, NOSQL_TAG
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connectors.vectara import (
     CONNECTOR_TYPE as VECTARA_CONNECTOR_TYPE,
@@ -24,6 +23,7 @@ from unstructured_ingest.v2.processes.connectors.vectara import (
     VectaraUploadStager,
     VectaraUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 def validate_upload(document: dict, expected_data: dict):

test/integration/connectors/utils/validation/destination.py

@@ -4,7 +4,8 @@ from pathlib import Path
 
 from test.integration.connectors.utils.validation.utils import ValidationConfig
 from unstructured_ingest.utils.data_prep import get_data
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers, UploadStager
+from unstructured_ingest.v2.interfaces import UploadStager
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 class StagerValidationConfigs(ValidationConfig):

test/integration/connectors/utils/validation/source.py

@@ -8,7 +8,8 @@ from deepdiff import DeepDiff
 from pydantic import Field
 
 from test.integration.connectors.utils.validation.utils import ValidationConfig
-from unstructured_ingest.v2.interfaces import Downloader, FileData, Indexer
+from unstructured_ingest.v2.interfaces import Downloader, Indexer
+from unstructured_ingest.v2.types.file_data import FileData
 
 NONSTANDARD_METADATA_FIELDS = {
     "additional_metadata.@microsoft.graph.downloadUrl": [

test/integration/connectors/weaviate/test_local.py

@@ -9,7 +9,6 @@ from weaviate.client import WeaviateClient
 
 from test.integration.connectors.utils.constants import DESTINATION_TAG, VECTOR_DB_TAG
 from test.integration.connectors.utils.docker import container_context
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.weaviate.local import (
     CONNECTOR_TYPE,
     LocalWeaviateConnectionConfig,
@@ -17,6 +16,7 @@ from unstructured_ingest.v2.processes.connectors.weaviate.local import (
     LocalWeaviateUploaderConfig,
     LocalWeaviateUploadStager,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 COLLECTION_NAME = "elements"
 

test/unit/test_html.py

@@ -5,7 +5,7 @@ from bs4 import BeautifulSoup
 from pytest_mock import MockerFixture
 
 from unstructured_ingest.utils.html import HtmlMixin
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 def test_extract_images(mocker: MockerFixture):

test/unit/v2/connectors/ibm_watsonx/test_ibm_watsonx_s3.py

@@ -8,7 +8,6 @@ from pyiceberg.exceptions import CommitFailedException
 from pytest_mock import MockerFixture
 
 from unstructured_ingest.v2.errors import ProviderError, UserError
-from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.ibm_watsonx import IBM_WATSONX_S3_CONNECTOR_TYPE
 from unstructured_ingest.v2.processes.connectors.ibm_watsonx.ibm_watsonx_s3 import (
     IbmWatsonxAccessConfig,
@@ -16,6 +15,7 @@ from unstructured_ingest.v2.processes.connectors.ibm_watsonx.ibm_watsonx_s3 impo
     IbmWatsonxUploader,
     IbmWatsonxUploaderConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 @pytest.fixture

test/unit/v2/connectors/motherduck/test_base.py

@@ -3,10 +3,9 @@ from pathlib import Path
 import pytest
 from pytest_mock import MockerFixture
 
-from unstructured_ingest.v2.interfaces import FileData
-from unstructured_ingest.v2.interfaces.file_data import SourceIdentifiers
 from unstructured_ingest.v2.interfaces.upload_stager import UploadStagerConfig
 from unstructured_ingest.v2.processes.connectors.duckdb.base import BaseDuckDBUploadStager
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 @pytest.fixture

test/unit/v2/connectors/sql/test_sql.py

@@ -4,13 +4,13 @@ import pandas as pd
 import pytest
 from pytest_mock import MockerFixture
 
-from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.sql.sql import (
     SQLConnectionConfig,
     SQLUploader,
     SQLUploaderConfig,
     SQLUploadStager,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 @pytest.fixture

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.5.25"  # pragma: no cover
+__version__ = "0.6.0"  # pragma: no cover

unstructured_ingest/utils/html.py

@@ -7,8 +7,9 @@ from uuid import NAMESPACE_DNS, uuid5
 from pydantic import BaseModel, Field
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, SourceIdentifiers
+from unstructured_ingest.v2.interfaces import DownloadResponse
 from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 if TYPE_CHECKING:
     from bs4.element import Tag

unstructured_ingest/v2/interfaces/__init__.py

@@ -1,11 +1,3 @@
-from unstructured_ingest.v2.types.file_data import (
-    BatchFileData,
-    BatchItem,
-    FileData,
-    FileDataSourceMetadata,
-    SourceIdentifiers,
-)
-
 from .connector import AccessConfig, BaseConnector, ConnectionConfig
 from .downloader import Downloader, DownloaderConfig, DownloadResponse, download_responses
 from .indexer import Indexer, IndexerConfig
@@ -19,7 +11,6 @@ __all__ = [
     "download_responses",
     "Downloader",
     "DownloaderConfig",
-    "FileData",
     "Indexer",
     "IndexerConfig",
     "BaseProcess",
@@ -28,13 +19,9 @@ __all__ = [
     "UploadStagerConfig",
     "Uploader",
     "UploaderConfig",
-    "SourceIdentifiers",
     "UploadContent",
     "AccessConfig",
     "ConnectionConfig",
     "BaseConnector",
-    "FileDataSourceMetadata",
-    "BatchFileData",
-    "BatchItem",
     "VectorDBUploader",
 ]

unstructured_ingest/v2/interfaces/downloader.py

@@ -6,8 +6,8 @@ from typing import Any, Optional, TypedDict, TypeVar, Union
 from pydantic import BaseModel, Field
 
 from unstructured_ingest.v2.interfaces.connector import BaseConnector
-from unstructured_ingest.v2.interfaces.file_data import FileData
 from unstructured_ingest.v2.interfaces.process import BaseProcess
+from unstructured_ingest.v2.types.file_data import FileData
 
 
 class DownloaderConfig(BaseModel):

unstructured_ingest/v2/interfaces/indexer.py

@@ -4,8 +4,8 @@ from typing import Any, AsyncGenerator, Generator, Optional, TypeVar
 from pydantic import BaseModel
 
 from unstructured_ingest.v2.interfaces.connector import BaseConnector
-from unstructured_ingest.v2.interfaces.file_data import FileData
 from unstructured_ingest.v2.interfaces.process import BaseProcess
+from unstructured_ingest.v2.types.file_data import FileData
 
 
 class IndexerConfig(BaseModel):

unstructured_ingest/v2/interfaces/upload_stager.py

@@ -7,8 +7,8 @@ from pydantic import BaseModel
 
 from unstructured_ingest.utils import ndjson
 from unstructured_ingest.utils.data_prep import get_data, write_data
-from unstructured_ingest.v2.interfaces.file_data import FileData
-from unstructured_ingest.v2.interfaces.process import BaseProcess
+from unstructured_ingest.v2.interfaces import BaseProcess
+from unstructured_ingest.v2.types.file_data import FileData
 
 
 class UploadStagerConfig(BaseModel):

unstructured_ingest/v2/interfaces/uploader.py

@@ -6,9 +6,8 @@ from typing import Any, TypeVar
 from pydantic import BaseModel
 
 from unstructured_ingest.utils.data_prep import get_data
-from unstructured_ingest.v2.interfaces.connector import BaseConnector
-from unstructured_ingest.v2.interfaces.file_data import FileData
-from unstructured_ingest.v2.interfaces.process import BaseProcess
+from unstructured_ingest.v2.interfaces import BaseConnector, BaseProcess
+from unstructured_ingest.v2.types.file_data import FileData
 
 
 class UploaderConfig(BaseModel):

unstructured_ingest/v2/pipeline/steps/chunk.py

@@ -5,11 +5,10 @@ from pathlib import Path
 from typing import Callable, Optional, TypedDict
 
 from unstructured_ingest.utils.data_prep import write_data
-from unstructured_ingest.v2.interfaces import FileData
-from unstructured_ingest.v2.interfaces.file_data import file_data_from_file
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
 from unstructured_ingest.v2.processes.chunker import Chunker
+from unstructured_ingest.v2.types.file_data import FileData, file_data_from_file
 from unstructured_ingest.v2.utils import serialize_base_model_json
 
 STEP_ID = "chunk"

unstructured_ingest/v2/pipeline/steps/download.py

@@ -6,11 +6,10 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Callable, Optional, TypedDict, TypeVar
 
-from unstructured_ingest.v2.interfaces import FileData, download_responses
-from unstructured_ingest.v2.interfaces.downloader import Downloader
-from unstructured_ingest.v2.interfaces.file_data import file_data_from_file
+from unstructured_ingest.v2.interfaces import Downloader, download_responses
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
+from unstructured_ingest.v2.types.file_data import FileData, file_data_from_file
 from unstructured_ingest.v2.utils import serialize_base_model_json
 
 DownloaderT = TypeVar("DownloaderT", bound=Downloader)

unstructured_ingest/v2/pipeline/steps/embed.py

@@ -5,11 +5,10 @@ from pathlib import Path
 from typing import Callable, Optional, TypedDict
 
 from unstructured_ingest.utils.data_prep import write_data
-from unstructured_ingest.v2.interfaces import FileData
-from unstructured_ingest.v2.interfaces.file_data import file_data_from_file
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
 from unstructured_ingest.v2.processes.embedder import Embedder
+from unstructured_ingest.v2.types.file_data import FileData, file_data_from_file
 from unstructured_ingest.v2.utils import serialize_base_model_json
 
 STEP_ID = "embed"

unstructured_ingest/v2/pipeline/steps/filter.py

@@ -2,10 +2,10 @@ import asyncio
 from dataclasses import dataclass
 from typing import Callable, Optional
 
-from unstructured_ingest.v2.interfaces.file_data import file_data_from_file
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
 from unstructured_ingest.v2.processes.filter import Filterer
+from unstructured_ingest.v2.types.file_data import file_data_from_file
 
 STEP_ID = "filter"