unstructured-ingest 0.6.4__py3-none-any.whl → 0.7.1__py3-none-any.whl

examples/mongodb.py

@@ -0,0 +1,53 @@
+import random
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.local import (
+    LocalConnectionConfig,
+    LocalDownloaderConfig,
+    LocalIndexerConfig,
+)
+from unstructured_ingest.processes.connectors.mongodb import (
+    CONNECTOR_TYPE,
+    MongoDBAccessConfig,
+    MongoDBConnectionConfig,
+    MongoDBUploaderConfig,
+    MongoDBUploadStagerConfig,
+)
+from unstructured_ingest.processes.embedder import EmbedderConfig
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        indexer_config=LocalIndexerConfig(input_path=str(docs_path.resolve()) + "/multisimple/"),
+        downloader_config=LocalDownloaderConfig(download_dir=download_path),
+        source_connection_config=LocalConnectionConfig(),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(
+            chunking_strategy="by_title",
+            chunk_include_orig_elements=False,
+            chunk_max_characters=1500,
+            chunk_multipage_sections=True,
+        ),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        destination_connection_config=MongoDBConnectionConfig(
+            access_config=MongoDBAccessConfig(uri=None),
+            host="localhost",
+            port=27017,
+            collection=f"test-collection-{random.randint(1000, 9999)}",
+            database="testDatabase",
+        ),
+        stager_config=MongoDBUploadStagerConfig(),
+        uploader_config=MongoDBUploaderConfig(batch_size=10),
+    ).run()

examples/opensearch.py

@@ -0,0 +1,50 @@
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.local import (
+    LocalConnectionConfig,
+    LocalDownloaderConfig,
+    LocalIndexerConfig,
+)
+from unstructured_ingest.processes.connectors.opensearch import (
+    CONNECTOR_TYPE,
+    OpenSearchAccessConfig,
+    OpenSearchConnectionConfig,
+    OpenSearchUploaderConfig,
+    OpenSearchUploadStagerConfig,
+)
+from unstructured_ingest.processes.embedder import EmbedderConfig
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        indexer_config=LocalIndexerConfig(
+            input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt"
+        ),
+        downloader_config=LocalDownloaderConfig(download_dir=download_path),
+        source_connection_config=LocalConnectionConfig(),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        destination_connection_config=OpenSearchConnectionConfig(
+            hosts="http://localhost:9247",
+            username="admin",
+            use_ssl=True,
+            access_config=OpenSearchAccessConfig(password="admin"),
+        ),
+        stager_config=OpenSearchUploadStagerConfig(index_name="ingest-test-destination"),
+        uploader_config=OpenSearchUploaderConfig(
+            index_name="ingest-test-destination", batch_size_bytes=150
+        ),
+    ).run()

examples/pinecone.py

@@ -0,0 +1,57 @@
+import os
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.local import (
+    LocalConnectionConfig,
+    LocalDownloaderConfig,
+    LocalIndexerConfig,
+)
+from unstructured_ingest.processes.connectors.pinecone import (
+    CONNECTOR_TYPE,
+    PineconeAccessConfig,
+    PineconeConnectionConfig,
+    PineconeUploaderConfig,
+    PineconeUploadStagerConfig,
+)
+from unstructured_ingest.processes.embedder import EmbedderConfig
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        indexer_config=LocalIndexerConfig(
+            input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt"
+        ),
+        downloader_config=LocalDownloaderConfig(download_dir=download_path),
+        source_connection_config=LocalConnectionConfig(),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        destination_connection_config=PineconeConnectionConfig(
+            # You'll need to set PINECONE_API_KEY environment variable to run this example
+            access_config=PineconeAccessConfig(pinecone_api_key=os.getenv("PINECONE_API_KEY")),
+            index_name=os.getenv(
+                "PINECONE_INDEX",
+                default="your index name here. e.g. my-index,"
+                "or define in environment variable PINECONE_INDEX",
+            ),
+            environment=os.getenv(
+                "PINECONE_ENVIRONMENT",
+                default="your environment name here. e.g. us-east-1,"
+                "or define in environment variable PINECONE_ENVIRONMENT",
+            ),
+        ),
+        stager_config=PineconeUploadStagerConfig(),
+        uploader_config=PineconeUploaderConfig(batch_size=10, num_processes=2),
+    ).run()

examples/s3.py

@@ -0,0 +1,38 @@
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.fsspec.s3 import (
+    CONNECTOR_TYPE,
+    S3ConnectionConfig,
+    S3DownloaderConfig,
+    S3IndexerConfig,
+)
+from unstructured_ingest.processes.connectors.local import (
+    LocalUploaderConfig,
+)
+from unstructured_ingest.processes.embedder import EmbedderConfig
+from unstructured_ingest.processes.filter import FiltererConfig
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve()), verbose=True, iter_delete=True),
+        indexer_config=S3IndexerConfig(remote_url="s3://utic-dev-tech-fixtures/small-pdf-set/"),
+        downloader_config=S3DownloaderConfig(download_dir=download_path),
+        source_connection_config=S3ConnectionConfig(anonymous=True),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
+        filterer_config=FiltererConfig(max_file_size=900000),
+    ).run()

examples/salesforce.py

@@ -0,0 +1,44 @@
+import os
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.local import (
+    LocalUploaderConfig,
+)
+from unstructured_ingest.processes.connectors.salesforce import (
+    CONNECTOR_TYPE,
+    SalesforceAccessConfig,
+    SalesforceConnectionConfig,
+    SalesforceDownloaderConfig,
+    SalesforceIndexerConfig,
+)
+from unstructured_ingest.processes.embedder import EmbedderConfig
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        indexer_config=SalesforceIndexerConfig(categories=["Campaign", "EmailMessage"]),
+        downloader_config=SalesforceDownloaderConfig(download_dir=download_path),
+        source_connection_config=SalesforceConnectionConfig(
+            SalesforceAccessConfig(
+                consumer_key=os.getenv("SALESFORCE_CONSUMER_KEY"),
+                private_key=os.getenv("SALESFORCE_PRIVATE_KEY"),
+            ),
+            username=os.getenv("SALESFORCE_USERNAME"),
+        ),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
+    ).run()

examples/sharepoint.py

@@ -0,0 +1,47 @@
+import os
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.connectors.local import (
+    LocalUploaderConfig,
+)
+from unstructured_ingest.processes.connectors.sharepoint import (
+    CONNECTOR_TYPE,
+    SharepointAccessConfig,
+    SharepointConnectionConfig,
+    SharepointDownloaderConfig,
+    SharepointIndexerConfig,
+    SharepointPermissionsConfig,
+)
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve()), tqdm=True, verbose=True),
+        indexer_config=SharepointIndexerConfig(),
+        downloader_config=SharepointDownloaderConfig(download_dir=download_path),
+        source_connection_config=SharepointConnectionConfig(
+            client_id=os.getenv("SHAREPOINT_CLIENT_ID"),
+            site=os.getenv("SHAREPOINT_SITE"),
+            access_config=SharepointAccessConfig(client_cred=os.getenv("SHAREPOINT_CRED")),
+            permissions_config=SharepointPermissionsConfig(
+                permissions_application_id=os.getenv("SHAREPOINT_PERMISSIONS_APP_ID"),
+                permissions_client_cred=os.getenv("SHAREPOINT_PERMISSIONS_APP_CRED"),
+                permissions_tenant=os.getenv("SHAREPOINT_PERMISSIONS_TENANT"),
+            ),
+        ),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        # chunker_config=ChunkerConfig(chunking_strategy="by_title"),
+        # embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
+    ).run()

examples/singlestore.py

@@ -0,0 +1,49 @@
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.local import (
+    LocalConnectionConfig,
+    LocalDownloaderConfig,
+    LocalIndexerConfig,
+)
+from unstructured_ingest.processes.connectors.singlestore import (
+    CONNECTOR_TYPE,
+    SingleStoreAccessConfig,
+    SingleStoreConnectionConfig,
+    SingleStoreUploaderConfig,
+    SingleStoreUploadStagerConfig,
+)
+from unstructured_ingest.processes.embedder import EmbedderConfig
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve()), tqdm=True, verbose=True),
+        indexer_config=LocalIndexerConfig(
+            input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt"
+        ),
+        downloader_config=LocalDownloaderConfig(download_dir=download_path),
+        source_connection_config=LocalConnectionConfig(),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        destination_connection_config=SingleStoreConnectionConfig(
+            access_config=SingleStoreAccessConfig(password="password"),
+            host="localhost",
+            port=3306,
+            database="ingest_test",
+            user="root",
+        ),
+        stager_config=SingleStoreUploadStagerConfig(),
+        uploader_config=SingleStoreUploaderConfig(table_name="elements"),
+    ).run()

examples/sql.py

@@ -0,0 +1,90 @@
+import os
+import sqlite3
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.local import (
+    LocalConnectionConfig,
+    LocalDownloaderConfig,
+    LocalIndexerConfig,
+)
+from unstructured_ingest.processes.connectors.sql import (
+    CONNECTOR_TYPE,
+    POSTGRESQL_DB,
+    SQLITE_DB,
+    SQLAccessConfig,
+    SQLConnectionConfig,
+    SQLUploaderConfig,
+    SQLUploadStagerConfig,
+)
+from unstructured_ingest.processes.embedder import EmbedderConfig
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+SQLITE_DB_PATH = "test-sql-db.sqlite"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+
+    configs = {
+        "context": ProcessorConfig(work_dir=str(work_dir.resolve())),
+        "indexer_config": LocalIndexerConfig(input_path=str(docs_path.resolve()) + "/multisimple/"),
+        "downloader_config": LocalDownloaderConfig(download_dir=download_path),
+        "source_connection_config": LocalConnectionConfig(),
+        "partitioner_config": PartitionerConfig(strategy="fast"),
+        "chunker_config": ChunkerConfig(
+            chunking_strategy="by_title",
+            chunk_include_orig_elements=False,
+            chunk_max_characters=1500,
+            chunk_multipage_sections=True,
+        ),
+        "embedder_config": EmbedderConfig(embedding_provider="huggingface"),
+        "stager_config": SQLUploadStagerConfig(),
+        "uploader_config": SQLUploaderConfig(batch_size=10),
+    }
+
+    if os.path.exists(SQLITE_DB):
+        os.remove(SQLITE_DB)
+
+    connection = sqlite3.connect(database=SQLITE_DB)
+
+    query = None
+    script_path = (
+        Path(__file__).parent.parent.parent.parent.parent
+        / Path("test_e2e/env_setup/sql/sqlite-schema.sql")
+    ).resolve()
+    with open(script_path) as f:
+        query = f.read()
+    cursor = connection.cursor()
+    cursor.executescript(query)
+    connection.close()
+
+    # sqlite test first
+    Pipeline.from_configs(
+        destination_connection_config=SQLConnectionConfig(
+            db_type=SQLITE_DB,
+            database=SQLITE_DB_PATH,
+            access_config=SQLAccessConfig(),
+        ),
+        **configs,
+    ).run()
+
+    # now, pg with pgvector
+    Pipeline.from_configs(
+        destination_connection_config=SQLConnectionConfig(
+            db_type=POSTGRESQL_DB,
+            database="elements",
+            host="localhost",
+            port=5433,
+            access_config=SQLAccessConfig(username="unstructured", password="test"),
+        ),
+        **configs,
+    ).run()

examples/vectara.py

@@ -0,0 +1,54 @@
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.local import (
+    LocalConnectionConfig,
+    LocalDownloaderConfig,
+    LocalIndexerConfig,
+)
+from unstructured_ingest.processes.connectors.vectara import (
+    CONNECTOR_TYPE,
+    VectaraAccessConfig,
+    VectaraConnectionConfig,
+    VectaraUploaderConfig,
+    VectaraUploadStagerConfig,
+)
+from unstructured_ingest.processes.embedder import EmbedderConfig
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        indexer_config=LocalIndexerConfig(input_path=str(docs_path.resolve()) + "/multisimple/"),
+        downloader_config=LocalDownloaderConfig(download_dir=download_path),
+        source_connection_config=LocalConnectionConfig(),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(
+            chunking_strategy="by_title",
+            chunk_include_orig_elements=False,
+            chunk_max_characters=1500,
+            chunk_multipage_sections=True,
+        ),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        destination_connection_config=VectaraConnectionConfig(
+            access_config=VectaraAccessConfig(
+                oauth_client_id="fill oauth_client_id", oauth_secret="fill oauth_secret"
+            ),
+            customer_id="fill customer_id",
+            corpus_name="fill corpus_name",
+            corpus_key="fill corpus_key",
+            token_url="fill token_url",
+        ),
+        stager_config=VectaraUploadStagerConfig(batch_size=10),
+        uploader_config=VectaraUploaderConfig(),
+    ).run()

examples/weaviate.py

@@ -0,0 +1,44 @@
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.local import (
+    LocalConnectionConfig,
+    LocalDownloaderConfig,
+    LocalIndexerConfig,
+)
+from unstructured_ingest.processes.connectors.weaviate.local import (
+    CONNECTOR_TYPE,
+    LocalWeaviateConnectionConfig,
+    LocalWeaviateUploaderConfig,
+    LocalWeaviateUploadStagerConfig,
+)
+from unstructured_ingest.processes.embedder import EmbedderConfig
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        indexer_config=LocalIndexerConfig(input_path=str(docs_path.resolve()) + "/multisimple/"),
+        downloader_config=LocalDownloaderConfig(download_dir=download_path),
+        source_connection_config=LocalConnectionConfig(),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        destination_connection_config=LocalWeaviateConnectionConfig(
+            # Connects to http://localhost:8080
+        ),
+        stager_config=LocalWeaviateUploadStagerConfig(),
+        uploader_config=LocalWeaviateUploaderConfig(
+            collection="elements", batch_size=10, dynamic_batch=False
+        ),
+    ).run()

test/integration/chunkers/test_chunkers.py

@@ -4,7 +4,7 @@ from pathlib import Path
 import pytest
 
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.processes.chunker import Chunker, ChunkerConfig
+from unstructured_ingest.processes.chunker import Chunker, ChunkerConfig
 
 int_test_dir = Path(__file__).parent
 assets_dir = int_test_dir / "assets"

test/integration/connectors/conftest.py

@@ -4,7 +4,7 @@ from typing import Generator
 
 import pytest
 
-from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.logger import logger
 
 FILENAME = Path("DA-1p-with-duplicate-pages.pdf.json")
 

test/integration/connectors/databricks/test_volumes_native.py

@@ -20,8 +20,9 @@ from test.integration.connectors.utils.validation.source import (
     source_connector_validation,
 )
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.errors import UserAuthError, UserError
-from unstructured_ingest.v2.processes.connectors.databricks.volumes_native import (
+from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.errors_v2 import UserAuthError, UserError
+from unstructured_ingest.processes.connectors.databricks.volumes_native import (
     CONNECTOR_TYPE,
     DatabricksNativeVolumesAccessConfig,
     DatabricksNativeVolumesConnectionConfig,
@@ -32,7 +33,6 @@ from unstructured_ingest.v2.processes.connectors.databricks.volumes_native impor
     DatabricksNativeVolumesUploader,
     DatabricksNativeVolumesUploaderConfig,
 )
-from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 @dataclass

test/integration/connectors/discord/test_discord.py

@@ -13,7 +13,7 @@ from test.integration.connectors.utils.validation.source import (
 )
 from test.integration.utils import requires_env
 from unstructured_ingest.error import SourceConnectionError
-from unstructured_ingest.v2.processes.connectors.discord import (
+from unstructured_ingest.processes.connectors.discord import (
     CONNECTOR_TYPE,
     DiscordAccessConfig,
     DiscordConnectionConfig,

test/integration/connectors/duckdb/test_duckdb.py

@@ -10,14 +10,14 @@ from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
     stager_validation,
 )
-from unstructured_ingest.v2.processes.connectors.duckdb.duckdb import (
+from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.processes.connectors.duckdb.duckdb import (
     CONNECTOR_TYPE,
     DuckDBConnectionConfig,
     DuckDBUploader,
     DuckDBUploaderConfig,
     DuckDBUploadStager,
 )
-from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 @pytest.fixture

test/integration/connectors/duckdb/test_motherduck.py

@@ -9,7 +9,8 @@ import pytest
 
 from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.processes.connectors.duckdb.motherduck import (
+from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.processes.connectors.duckdb.motherduck import (
     CONNECTOR_TYPE,
     MotherDuckAccessConfig,
     MotherDuckConnectionConfig,
@@ -17,7 +18,6 @@ from unstructured_ingest.v2.processes.connectors.duckdb.motherduck import (
     MotherDuckUploaderConfig,
     MotherDuckUploadStager,
 )
-from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 @pytest.fixture

test/integration/connectors/elasticsearch/test_elasticsearch.py

@@ -22,8 +22,8 @@ from test.integration.connectors.utils.validation.source import (
     source_connector_validation,
 )
 from unstructured_ingest.error import DestinationConnectionError, SourceConnectionError
-from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
-from unstructured_ingest.v2.processes.connectors.elasticsearch.elasticsearch import (
+from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.processes.connectors.elasticsearch.elasticsearch import (
     CONNECTOR_TYPE,
     ElasticsearchAccessConfig,
     ElasticsearchConnectionConfig,

test/integration/connectors/elasticsearch/test_opensearch.py

@@ -20,11 +20,12 @@ from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
 )
+from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.error import (
     DestinationConnectionError,
     SourceConnectionError,
 )
-from unstructured_ingest.v2.processes.connectors.elasticsearch.opensearch import (
+from unstructured_ingest.processes.connectors.elasticsearch.opensearch import (
     CONNECTOR_TYPE,
     OpenSearchAccessConfig,
     OpenSearchConnectionConfig,
@@ -37,7 +38,6 @@ from unstructured_ingest.v2.processes.connectors.elasticsearch.opensearch import
     OpenSearchUploadStager,
     OpenSearchUploadStagerConfig,
 )
-from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 SOURCE_INDEX_NAME = "movies"
 DESTINATION_INDEX_NAME = "elements"