unstructured-ingest 0.7.2__py3-none-any.whl → 1.0.2__py3-none-any.whl

examples/azure_cognitive_search.py

@@ -1,55 +0,0 @@
-import os
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.azure_ai_search import (
-    CONNECTOR_TYPE,
-    AzureAISearchAccessConfig,
-    AzureAISearchConnectionConfig,
-    AzureAISearchUploaderConfig,
-    AzureAISearchUploadStagerConfig,
-)
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    index_name = "ingest-test-destination"
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=LocalIndexerConfig(
-            input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt"
-        ),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(
-            chunking_strategy="by_title", chunk_include_orig_elements=False
-        ),
-        embedder_config=EmbedderConfig(
-            embedding_provider="openai", embedding_api_key=os.getenv("OPENAI_API_KEY")
-        ),
-        destination_connection_config=AzureAISearchConnectionConfig(
-            access_config=AzureAISearchAccessConfig(
-                azure_ai_search_key=os.getenv("AZURE_SEARCH_API_KEY")
-            ),
-            index=os.getenv("AZURE_SEARCH_INDEX"),
-            endpoint=os.getenv("AZURE_SEARCH_ENDPOINT"),
-        ),
-        uploader_config=AzureAISearchUploaderConfig(batch_size=10),
-        stager_config=AzureAISearchUploadStagerConfig(),
-    ).run()

examples/chroma.py

@@ -1,54 +0,0 @@
-import random
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.chroma import (
-    CONNECTOR_TYPE,
-    ChromaAccessConfig,
-    ChromaConnectionConfig,
-    ChromaUploaderConfig,
-    ChromaUploadStagerConfig,
-)
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=LocalIndexerConfig(input_path=docs_path.resolve() / "multisimple"),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(
-            chunking_strategy="by_title",
-            chunk_include_orig_elements=False,
-            chunk_max_characters=1500,
-            chunk_multipage_sections=True,
-        ),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=ChromaConnectionConfig(
-            access_config=ChromaAccessConfig(settings=None, headers=None),
-            host="localhost",
-            port=8047,
-            collection_name=f"test-collection-{random.randint(1000, 9999)}",
-            tenant="default_tenant",
-            database="default_database",
-        ),
-        stager_config=ChromaUploadStagerConfig(),
-        uploader_config=ChromaUploaderConfig(batch_size=10),
-    ).run()

examples/couchbase.py

@@ -1,55 +0,0 @@
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.couchbase import (
-    CONNECTOR_TYPE,
-    CouchbaseAccessConfig,
-    CouchbaseConnectionConfig,
-    CouchbaseUploaderConfig,
-    CouchbaseUploadStagerConfig,
-)
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=LocalIndexerConfig(input_path=str(docs_path.resolve()) + "/multisimple/"),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(
-            chunking_strategy="by_title",
-            chunk_include_orig_elements=False,
-            chunk_max_characters=1500,
-            chunk_multipage_sections=True,
-        ),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=CouchbaseConnectionConfig(
-            access_config=CouchbaseAccessConfig(
-                connection_string="couchbase://localhost",
-                username="Administrator",
-                password="password",
-            ),
-            bucket="example_bucket",
-            scope="example_scope",
-            collection="example_collection",
-        ),
-        stager_config=CouchbaseUploadStagerConfig(),
-        uploader_config=CouchbaseUploaderConfig(batch_size=10),
-    ).run()

examples/databricks_volumes_dest.py

@@ -1,55 +0,0 @@
-import os
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.databricks.volumes_native import (
-    CONNECTOR_TYPE,
-    DatabricksNativeVolumesAccessConfig,
-    DatabricksNativeVolumesConnectionConfig,
-    DatabricksNativeVolumesUploaderConfig,
-)
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=LocalIndexerConfig(input_path=str(docs_path.resolve()) + "/fake-text.txt"),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(
-            chunking_strategy="basic",
-        ),
-        embedder_config=None,
-        destination_connection_config=DatabricksNativeVolumesConnectionConfig(
-            access_config=DatabricksNativeVolumesAccessConfig(
-                client_id=os.environ["DATABRICKS_CLIENT_ID"],
-                client_secret=os.environ["DATABRICKS_CLIENT_SECRET"],
-            ),
-            host=os.environ["DATABRICKS_HOST"],
-            catalog=os.environ["DATABRICKS_CATALOG"],
-            volume=os.environ["DATABRICKS_VOLUME"],
-            volume_path=os.environ["DATABRICKS_VOLUME_PATH"],
-        ),
-        uploader_config=DatabricksNativeVolumesUploaderConfig(
-            overwrite=True,
-            catalog=os.environ["DATABRICKS_CATALOG"],
-            volume=os.environ["DATABRICKS_VOLUME"],
-            volume_path=os.environ["DATABRICKS_VOLUME_PATH"],
-        ),
-    ).run()

examples/databricks_volumes_source.py

@@ -1,53 +0,0 @@
-import os
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.databricks.volumes_native import (
-    CONNECTOR_TYPE,
-    DatabricksNativeVolumesAccessConfig,
-    DatabricksNativeVolumesConnectionConfig,
-    DatabricksNativeVolumesDownloaderConfig,
-    DatabricksNativeVolumesIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.local import (
-    LocalUploaderConfig,
-)
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=DatabricksNativeVolumesIndexerConfig(
-            host=os.environ["DATABRICKS_HOST"],
-            catalog=os.environ["DATABRICKS_CATALOG"],
-            volume=os.environ["DATABRICKS_VOLUME"],
-            volume_path=os.environ["DATABRICKS_VOLUME_PATH"],
-        ),
-        downloader_config=DatabricksNativeVolumesDownloaderConfig(download_dir=download_path),
-        source_connection_config=DatabricksNativeVolumesConnectionConfig(
-            access_config=DatabricksNativeVolumesAccessConfig(
-                client_id=os.environ["DATABRICKS_CLIENT_ID"],
-                client_secret=os.environ["DATABRICKS_CLIENT_SECRET"],
-            ),
-            host=os.environ["DATABRICKS_HOST"],
-            catalog=os.environ["DATABRICKS_CATALOG"],
-            volume=os.environ["DATABRICKS_VOLUME"],
-            volume_path=os.environ["DATABRICKS_VOLUME_PATH"],
-        ),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(
-            chunking_strategy="basic",
-        ),
-        embedder_config=None,
-        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
-    ).run()

examples/delta_table.py

@@ -1,45 +0,0 @@
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.delta_table import (
-    CONNECTOR_TYPE,
-    DeltaTableAccessConfig,
-    DeltaTableConnectionConfig,
-    DeltaTableUploaderConfig,
-    DeltaTableUploadStagerConfig,
-)
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=LocalIndexerConfig(
-            input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt",
-        ),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=DeltaTableConnectionConfig(
-            access_config=DeltaTableAccessConfig(), table_uri="example_uri"
-        ),
-        stager_config=DeltaTableUploadStagerConfig(),
-        uploader_config=DeltaTableUploaderConfig(),
-    ).run()

examples/discord_example.py

@@ -1,36 +0,0 @@
-import os
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.connectors.discord import (
-    CONNECTOR_TYPE,
-    DiscordAccessConfig,
-    DiscordConnectionConfig,
-    DiscordDownloaderConfig,
-    DiscordIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.local import LocalUploaderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve()), tqdm=True, verbose=True),
-        indexer_config=DiscordIndexerConfig(channels=os.environ["DISCORD_CHANNELS"].split(",")),
-        downloader_config=DiscordDownloaderConfig(limit=int(os.getenv("DISCORD_LIMIT", 100))),
-        source_connection_config=DiscordConnectionConfig(
-            access_config=DiscordAccessConfig(token=os.environ["DISCORD_TOKEN"])
-        ),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        # chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        # embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
-    ).run()

examples/elasticsearch.py

@@ -1,49 +0,0 @@
-import os
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.elasticsearch import (
-    CONNECTOR_TYPE,
-    ElasticsearchAccessConfig,
-    ElasticsearchConnectionConfig,
-    ElasticsearchUploaderConfig,
-    ElasticsearchUploadStagerConfig,
-)
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    index_name = "ingest-test-destination"
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=LocalIndexerConfig(
-            input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt"
-        ),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=ElasticsearchConnectionConfig(
-            access_config=ElasticsearchAccessConfig(password=os.getenv("ELASTIC_PASSWORD")),
-            username=os.getenv("ELASTIC_USERNAME"),
-            hosts=["http://localhost:9200"],
-        ),
-        uploader_config=ElasticsearchUploaderConfig(index_name=index_name),
-        stager_config=ElasticsearchUploadStagerConfig(index_name=index_name),
-    ).run()

examples/google_drive.py

@@ -1,45 +0,0 @@
-import os
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.google_drive import (
-    CONNECTOR_TYPE,
-    GoogleDriveAccessConfig,
-    GoogleDriveConnectionConfig,
-    GoogleDriveDownloaderConfig,
-    GoogleDriveIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.local import (
-    LocalUploaderConfig,
-)
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-
-
-if __name__ == "__main__":
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        # You'll need to set GOOGLE_DRIVE_SERVICE_KEY and GOOGLE_DRIVE_DRIVE_ID
-        # environment variable to run this example
-        source_connection_config=GoogleDriveConnectionConfig(
-            access_config=GoogleDriveAccessConfig(
-                service_account_key=os.environ.get("GOOGLE_DRIVE_SERVICE_KEY")
-            ),
-            drive_id=os.environ.get("GOOGLE_DRIVE_DRIVE_ID"),
-        ),
-        indexer_config=GoogleDriveIndexerConfig(
-            resursive=True,
-        ),
-        downloader_config=GoogleDriveDownloaderConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(
-            chunking_strategy="basic",
-        ),
-        embedder_config=None,
-        uploader_config=LocalUploaderConfig(output_dir=output_path),
-    ).run()

examples/kdbai.py

@@ -1,54 +0,0 @@
-import os
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.kdbai import (
-    CONNECTOR_TYPE,
-    KdbaiConnectionConfig,
-    KdbaiUploaderConfig,
-    KdbaiUploadStagerConfig,
-)
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-input_path = docs_path.resolve() / "pdf" / "fake-memo.pdf"
-
-os.environ["KDBAI_API_KEY"] = "key"
-os.environ["KDBAI_ENDPOINT"] = "http://localhost"
-os.environ["KDBAI_DATABASE"] = "default"
-os.environ["KDBAI_TABLE"] = "table"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    logger.info(f"processing file(s): {input_path.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve()), tqdm=True, verbose=True),
-        indexer_config=LocalIndexerConfig(
-            input_path=docs_path.resolve() / "book-war-and-peace-1p.txt"
-        ),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=KdbaiConnectionConfig(
-            endpoint=os.environ["KDBAI_ENDPOINT"],
-        ),
-        stager_config=KdbaiUploadStagerConfig(),
-        uploader_config=KdbaiUploaderConfig(
-            database_name=os.environ["KDBAI_DATABASE"], table_name=os.environ["KDBAI_TABLE"]
-        ),
-    ).run()

examples/local.py

@@ -1,36 +0,0 @@
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.local import (
-    CONNECTOR_TYPE,
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-    LocalUploaderConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=LocalIndexerConfig(
-            input_path=str(docs_path.resolve()) + "/language-docs/UDHR_first_article_all.txt"
-        ),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
-    ).run()

examples/milvus.py

@@ -1,44 +0,0 @@
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.milvus import (
-    CONNECTOR_TYPE,
-    MilvusConnectionConfig,
-    MilvusUploaderConfig,
-    MilvusUploadStagerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve()), tqdm=True, verbose=True),
-        indexer_config=LocalIndexerConfig(
-            input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt"
-        ),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=MilvusConnectionConfig(
-            uri="http://localhost:19530", db_name="milvus"
-        ),
-        stager_config=MilvusUploadStagerConfig(),
-        uploader_config=MilvusUploaderConfig(collection_name="ingest_test"),
-    ).run()

examples/mongodb.py

@@ -1,53 +0,0 @@
-import random
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.mongodb import (
-    CONNECTOR_TYPE,
-    MongoDBAccessConfig,
-    MongoDBConnectionConfig,
-    MongoDBUploaderConfig,
-    MongoDBUploadStagerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=LocalIndexerConfig(input_path=str(docs_path.resolve()) + "/multisimple/"),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(
-            chunking_strategy="by_title",
-            chunk_include_orig_elements=False,
-            chunk_max_characters=1500,
-            chunk_multipage_sections=True,
-        ),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=MongoDBConnectionConfig(
-            access_config=MongoDBAccessConfig(uri=None),
-            host="localhost",
-            port=27017,
-            collection=f"test-collection-{random.randint(1000, 9999)}",
-            database="testDatabase",
-        ),
-        stager_config=MongoDBUploadStagerConfig(),
-        uploader_config=MongoDBUploaderConfig(batch_size=10),
-    ).run()