unstructured-ingest 0.6.2__py3-none-any.whl → 0.7.0__py3-none-any.whl

examples/airtable.py

@@ -0,0 +1,44 @@
+import os
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.airtable import (
+    CONNECTOR_TYPE,
+    AirtableAccessConfig,
+    AirtableConnectionConfig,
+    AirtableDownloaderConfig,
+    AirtableIndexerConfig,
+)
+from unstructured_ingest.processes.connectors.local import (
+    LocalUploaderConfig,
+)
+from unstructured_ingest.processes.embedder import EmbedderConfig
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve()), verbose=True),
+        indexer_config=AirtableIndexerConfig(
+            list_of_paths=["app5YQxSfp220fWtm", "appJ43QmP8I17zu88"]
+        ),
+        downloader_config=AirtableDownloaderConfig(download_dir=download_path),
+        source_connection_config=AirtableConnectionConfig(
+            access_config=AirtableAccessConfig(
+                personal_access_token=os.getenv("AIRTABLE_PERSONAL_ACCESS_TOKEN")
+            )
+        ),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
+    ).run()

examples/azure_cognitive_search.py

@@ -0,0 +1,55 @@
+import os
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.azure_ai_search import (
+    CONNECTOR_TYPE,
+    AzureAISearchAccessConfig,
+    AzureAISearchConnectionConfig,
+    AzureAISearchUploaderConfig,
+    AzureAISearchUploadStagerConfig,
+)
+from unstructured_ingest.processes.connectors.local import (
+    LocalConnectionConfig,
+    LocalDownloaderConfig,
+    LocalIndexerConfig,
+)
+from unstructured_ingest.processes.embedder import EmbedderConfig
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    index_name = "ingest-test-destination"
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        indexer_config=LocalIndexerConfig(
+            input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt"
+        ),
+        downloader_config=LocalDownloaderConfig(download_dir=download_path),
+        source_connection_config=LocalConnectionConfig(),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(
+            chunking_strategy="by_title", chunk_include_orig_elements=False
+        ),
+        embedder_config=EmbedderConfig(
+            embedding_provider="openai", embedding_api_key=os.getenv("OPENAI_API_KEY")
+        ),
+        destination_connection_config=AzureAISearchConnectionConfig(
+            access_config=AzureAISearchAccessConfig(
+                azure_ai_search_key=os.getenv("AZURE_SEARCH_API_KEY")
+            ),
+            index=os.getenv("AZURE_SEARCH_INDEX"),
+            endpoint=os.getenv("AZURE_SEARCH_ENDPOINT"),
+        ),
+        uploader_config=AzureAISearchUploaderConfig(batch_size=10),
+        stager_config=AzureAISearchUploadStagerConfig(),
+    ).run()

examples/chroma.py

@@ -0,0 +1,54 @@
+import random
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.chroma import (
+    CONNECTOR_TYPE,
+    ChromaAccessConfig,
+    ChromaConnectionConfig,
+    ChromaUploaderConfig,
+    ChromaUploadStagerConfig,
+)
+from unstructured_ingest.processes.connectors.local import (
+    LocalConnectionConfig,
+    LocalDownloaderConfig,
+    LocalIndexerConfig,
+)
+from unstructured_ingest.processes.embedder import EmbedderConfig
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        indexer_config=LocalIndexerConfig(input_path=docs_path.resolve() / "multisimple"),
+        downloader_config=LocalDownloaderConfig(download_dir=download_path),
+        source_connection_config=LocalConnectionConfig(),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(
+            chunking_strategy="by_title",
+            chunk_include_orig_elements=False,
+            chunk_max_characters=1500,
+            chunk_multipage_sections=True,
+        ),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        destination_connection_config=ChromaConnectionConfig(
+            access_config=ChromaAccessConfig(settings=None, headers=None),
+            host="localhost",
+            port=8047,
+            collection_name=f"test-collection-{random.randint(1000, 9999)}",
+            tenant="default_tenant",
+            database="default_database",
+        ),
+        stager_config=ChromaUploadStagerConfig(),
+        uploader_config=ChromaUploaderConfig(batch_size=10),
+    ).run()

examples/couchbase.py

@@ -0,0 +1,55 @@
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.couchbase import (
+    CONNECTOR_TYPE,
+    CouchbaseAccessConfig,
+    CouchbaseConnectionConfig,
+    CouchbaseUploaderConfig,
+    CouchbaseUploadStagerConfig,
+)
+from unstructured_ingest.processes.connectors.local import (
+    LocalConnectionConfig,
+    LocalDownloaderConfig,
+    LocalIndexerConfig,
+)
+from unstructured_ingest.processes.embedder import EmbedderConfig
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        indexer_config=LocalIndexerConfig(input_path=str(docs_path.resolve()) + "/multisimple/"),
+        downloader_config=LocalDownloaderConfig(download_dir=download_path),
+        source_connection_config=LocalConnectionConfig(),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(
+            chunking_strategy="by_title",
+            chunk_include_orig_elements=False,
+            chunk_max_characters=1500,
+            chunk_multipage_sections=True,
+        ),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        destination_connection_config=CouchbaseConnectionConfig(
+            access_config=CouchbaseAccessConfig(
+                connection_string="couchbase://localhost",
+                username="Administrator",
+                password="password",
+            ),
+            bucket="example_bucket",
+            scope="example_scope",
+            collection="example_collection",
+        ),
+        stager_config=CouchbaseUploadStagerConfig(),
+        uploader_config=CouchbaseUploaderConfig(batch_size=10),
+    ).run()

examples/databricks_volumes_dest.py

@@ -0,0 +1,55 @@
+import os
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.databricks.volumes_native import (
+    CONNECTOR_TYPE,
+    DatabricksNativeVolumesAccessConfig,
+    DatabricksNativeVolumesConnectionConfig,
+    DatabricksNativeVolumesUploaderConfig,
+)
+from unstructured_ingest.processes.connectors.local import (
+    LocalConnectionConfig,
+    LocalDownloaderConfig,
+    LocalIndexerConfig,
+)
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        indexer_config=LocalIndexerConfig(input_path=str(docs_path.resolve()) + "/fake-text.txt"),
+        downloader_config=LocalDownloaderConfig(download_dir=download_path),
+        source_connection_config=LocalConnectionConfig(),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(
+            chunking_strategy="basic",
+        ),
+        embedder_config=None,
+        destination_connection_config=DatabricksNativeVolumesConnectionConfig(
+            access_config=DatabricksNativeVolumesAccessConfig(
+                client_id=os.environ["DATABRICKS_CLIENT_ID"],
+                client_secret=os.environ["DATABRICKS_CLIENT_SECRET"],
+            ),
+            host=os.environ["DATABRICKS_HOST"],
+            catalog=os.environ["DATABRICKS_CATALOG"],
+            volume=os.environ["DATABRICKS_VOLUME"],
+            volume_path=os.environ["DATABRICKS_VOLUME_PATH"],
+        ),
+        uploader_config=DatabricksNativeVolumesUploaderConfig(
+            overwrite=True,
+            catalog=os.environ["DATABRICKS_CATALOG"],
+            volume=os.environ["DATABRICKS_VOLUME"],
+            volume_path=os.environ["DATABRICKS_VOLUME_PATH"],
+        ),
+    ).run()

examples/databricks_volumes_source.py

@@ -0,0 +1,53 @@
+import os
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.databricks.volumes_native import (
+    CONNECTOR_TYPE,
+    DatabricksNativeVolumesAccessConfig,
+    DatabricksNativeVolumesConnectionConfig,
+    DatabricksNativeVolumesDownloaderConfig,
+    DatabricksNativeVolumesIndexerConfig,
+)
+from unstructured_ingest.processes.connectors.local import (
+    LocalUploaderConfig,
+)
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        indexer_config=DatabricksNativeVolumesIndexerConfig(
+            host=os.environ["DATABRICKS_HOST"],
+            catalog=os.environ["DATABRICKS_CATALOG"],
+            volume=os.environ["DATABRICKS_VOLUME"],
+            volume_path=os.environ["DATABRICKS_VOLUME_PATH"],
+        ),
+        downloader_config=DatabricksNativeVolumesDownloaderConfig(download_dir=download_path),
+        source_connection_config=DatabricksNativeVolumesConnectionConfig(
+            access_config=DatabricksNativeVolumesAccessConfig(
+                client_id=os.environ["DATABRICKS_CLIENT_ID"],
+                client_secret=os.environ["DATABRICKS_CLIENT_SECRET"],
+            ),
+            host=os.environ["DATABRICKS_HOST"],
+            catalog=os.environ["DATABRICKS_CATALOG"],
+            volume=os.environ["DATABRICKS_VOLUME"],
+            volume_path=os.environ["DATABRICKS_VOLUME_PATH"],
+        ),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(
+            chunking_strategy="basic",
+        ),
+        embedder_config=None,
+        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
+    ).run()

examples/delta_table.py

@@ -0,0 +1,45 @@
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.delta_table import (
+    CONNECTOR_TYPE,
+    DeltaTableAccessConfig,
+    DeltaTableConnectionConfig,
+    DeltaTableUploaderConfig,
+    DeltaTableUploadStagerConfig,
+)
+from unstructured_ingest.processes.connectors.local import (
+    LocalConnectionConfig,
+    LocalDownloaderConfig,
+    LocalIndexerConfig,
+)
+from unstructured_ingest.processes.embedder import EmbedderConfig
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        indexer_config=LocalIndexerConfig(
+            input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt",
+        ),
+        downloader_config=LocalDownloaderConfig(download_dir=download_path),
+        source_connection_config=LocalConnectionConfig(),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        destination_connection_config=DeltaTableConnectionConfig(
+            access_config=DeltaTableAccessConfig(), table_uri="example_uri"
+        ),
+        stager_config=DeltaTableUploadStagerConfig(),
+        uploader_config=DeltaTableUploaderConfig(),
+    ).run()

examples/discord_example.py

@@ -0,0 +1,36 @@
+import os
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.connectors.discord import (
+    CONNECTOR_TYPE,
+    DiscordAccessConfig,
+    DiscordConnectionConfig,
+    DiscordDownloaderConfig,
+    DiscordIndexerConfig,
+)
+from unstructured_ingest.processes.connectors.local import LocalUploaderConfig
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve()), tqdm=True, verbose=True),
+        indexer_config=DiscordIndexerConfig(channels=os.environ["DISCORD_CHANNELS"].split(",")),
+        downloader_config=DiscordDownloaderConfig(limit=int(os.getenv("DISCORD_LIMIT", 100))),
+        source_connection_config=DiscordConnectionConfig(
+            access_config=DiscordAccessConfig(token=os.environ["DISCORD_TOKEN"])
+        ),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        # chunker_config=ChunkerConfig(chunking_strategy="by_title"),
+        # embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
+    ).run()

examples/elasticsearch.py

@@ -0,0 +1,49 @@
+import os
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.elasticsearch import (
+    CONNECTOR_TYPE,
+    ElasticsearchAccessConfig,
+    ElasticsearchConnectionConfig,
+    ElasticsearchUploaderConfig,
+    ElasticsearchUploadStagerConfig,
+)
+from unstructured_ingest.processes.connectors.local import (
+    LocalConnectionConfig,
+    LocalDownloaderConfig,
+    LocalIndexerConfig,
+)
+from unstructured_ingest.processes.embedder import EmbedderConfig
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    index_name = "ingest-test-destination"
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        indexer_config=LocalIndexerConfig(
+            input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt"
+        ),
+        downloader_config=LocalDownloaderConfig(download_dir=download_path),
+        source_connection_config=LocalConnectionConfig(),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        destination_connection_config=ElasticsearchConnectionConfig(
+            access_config=ElasticsearchAccessConfig(password=os.getenv("ELASTIC_PASSWORD")),
+            username=os.getenv("ELASTIC_USERNAME"),
+            hosts=["http://localhost:9200"],
+        ),
+        uploader_config=ElasticsearchUploaderConfig(index_name=index_name),
+        stager_config=ElasticsearchUploadStagerConfig(index_name=index_name),
+    ).run()

examples/google_drive.py

@@ -0,0 +1,45 @@
+import os
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.google_drive import (
+    CONNECTOR_TYPE,
+    GoogleDriveAccessConfig,
+    GoogleDriveConnectionConfig,
+    GoogleDriveDownloaderConfig,
+    GoogleDriveIndexerConfig,
+)
+from unstructured_ingest.processes.connectors.local import (
+    LocalUploaderConfig,
+)
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+
+
+if __name__ == "__main__":
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        # You'll need to set GOOGLE_DRIVE_SERVICE_KEY and GOOGLE_DRIVE_DRIVE_ID
+        # environment variable to run this example
+        source_connection_config=GoogleDriveConnectionConfig(
+            access_config=GoogleDriveAccessConfig(
+                service_account_key=os.environ.get("GOOGLE_DRIVE_SERVICE_KEY")
+            ),
+            drive_id=os.environ.get("GOOGLE_DRIVE_DRIVE_ID"),
+        ),
+        indexer_config=GoogleDriveIndexerConfig(
+            resursive=True,
+        ),
+        downloader_config=GoogleDriveDownloaderConfig(),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(
+            chunking_strategy="basic",
+        ),
+        embedder_config=None,
+        uploader_config=LocalUploaderConfig(output_dir=output_path),
+    ).run()

examples/kdbai.py

@@ -0,0 +1,54 @@
+import os
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.kdbai import (
+    CONNECTOR_TYPE,
+    KdbaiConnectionConfig,
+    KdbaiUploaderConfig,
+    KdbaiUploadStagerConfig,
+)
+from unstructured_ingest.processes.connectors.local import (
+    LocalConnectionConfig,
+    LocalDownloaderConfig,
+    LocalIndexerConfig,
+)
+from unstructured_ingest.processes.embedder import EmbedderConfig
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+input_path = docs_path.resolve() / "pdf" / "fake-memo.pdf"
+
+os.environ["KDBAI_API_KEY"] = "key"
+os.environ["KDBAI_ENDPOINT"] = "http://localhost"
+os.environ["KDBAI_DATABASE"] = "default"
+os.environ["KDBAI_TABLE"] = "table"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    logger.info(f"processing file(s): {input_path.resolve()}")
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve()), tqdm=True, verbose=True),
+        indexer_config=LocalIndexerConfig(
+            input_path=docs_path.resolve() / "book-war-and-peace-1p.txt"
+        ),
+        downloader_config=LocalDownloaderConfig(download_dir=download_path),
+        source_connection_config=LocalConnectionConfig(),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        destination_connection_config=KdbaiConnectionConfig(
+            endpoint=os.environ["KDBAI_ENDPOINT"],
+        ),
+        stager_config=KdbaiUploadStagerConfig(),
+        uploader_config=KdbaiUploaderConfig(
+            database_name=os.environ["KDBAI_DATABASE"], table_name=os.environ["KDBAI_TABLE"]
+        ),
+    ).run()

examples/local.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.local import (
+    CONNECTOR_TYPE,
+    LocalConnectionConfig,
+    LocalDownloaderConfig,
+    LocalIndexerConfig,
+    LocalUploaderConfig,
+)
+from unstructured_ingest.processes.embedder import EmbedderConfig
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
+        indexer_config=LocalIndexerConfig(
+            input_path=str(docs_path.resolve()) + "/language-docs/UDHR_first_article_all.txt"
+        ),
+        downloader_config=LocalDownloaderConfig(download_dir=download_path),
+        source_connection_config=LocalConnectionConfig(),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
+    ).run()

examples/milvus.py

@@ -0,0 +1,44 @@
+from pathlib import Path
+
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import ChunkerConfig
+from unstructured_ingest.processes.connectors.local import (
+    LocalConnectionConfig,
+    LocalDownloaderConfig,
+    LocalIndexerConfig,
+)
+from unstructured_ingest.processes.connectors.milvus import (
+    CONNECTOR_TYPE,
+    MilvusConnectionConfig,
+    MilvusUploaderConfig,
+    MilvusUploadStagerConfig,
+)
+from unstructured_ingest.processes.embedder import EmbedderConfig
+from unstructured_ingest.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+if __name__ == "__main__":
+    logger.info(f"writing all content in: {work_dir.resolve()}")
+    Pipeline.from_configs(
+        context=ProcessorConfig(work_dir=str(work_dir.resolve()), tqdm=True, verbose=True),
+        indexer_config=LocalIndexerConfig(
+            input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt"
+        ),
+        downloader_config=LocalDownloaderConfig(download_dir=download_path),
+        source_connection_config=LocalConnectionConfig(),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
+        destination_connection_config=MilvusConnectionConfig(
+            uri="http://localhost:19530", db_name="milvus"
+        ),
+        stager_config=MilvusUploadStagerConfig(),
+        uploader_config=MilvusUploaderConfig(collection_name="ingest_test"),
+    ).run()