unstructured-ingest 0.7.2__py3-none-any.whl → 1.0.1__py3-none-any.whl

examples/databricks_volumes_source.py

@@ -1,53 +0,0 @@
-import os
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.databricks.volumes_native import (
-    CONNECTOR_TYPE,
-    DatabricksNativeVolumesAccessConfig,
-    DatabricksNativeVolumesConnectionConfig,
-    DatabricksNativeVolumesDownloaderConfig,
-    DatabricksNativeVolumesIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.local import (
-    LocalUploaderConfig,
-)
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=DatabricksNativeVolumesIndexerConfig(
-            host=os.environ["DATABRICKS_HOST"],
-            catalog=os.environ["DATABRICKS_CATALOG"],
-            volume=os.environ["DATABRICKS_VOLUME"],
-            volume_path=os.environ["DATABRICKS_VOLUME_PATH"],
-        ),
-        downloader_config=DatabricksNativeVolumesDownloaderConfig(download_dir=download_path),
-        source_connection_config=DatabricksNativeVolumesConnectionConfig(
-            access_config=DatabricksNativeVolumesAccessConfig(
-                client_id=os.environ["DATABRICKS_CLIENT_ID"],
-                client_secret=os.environ["DATABRICKS_CLIENT_SECRET"],
-            ),
-            host=os.environ["DATABRICKS_HOST"],
-            catalog=os.environ["DATABRICKS_CATALOG"],
-            volume=os.environ["DATABRICKS_VOLUME"],
-            volume_path=os.environ["DATABRICKS_VOLUME_PATH"],
-        ),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(
-            chunking_strategy="basic",
-        ),
-        embedder_config=None,
-        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
-    ).run()

examples/delta_table.py

@@ -1,45 +0,0 @@
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.delta_table import (
-    CONNECTOR_TYPE,
-    DeltaTableAccessConfig,
-    DeltaTableConnectionConfig,
-    DeltaTableUploaderConfig,
-    DeltaTableUploadStagerConfig,
-)
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=LocalIndexerConfig(
-            input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt",
-        ),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=DeltaTableConnectionConfig(
-            access_config=DeltaTableAccessConfig(), table_uri="example_uri"
-        ),
-        stager_config=DeltaTableUploadStagerConfig(),
-        uploader_config=DeltaTableUploaderConfig(),
-    ).run()

examples/discord_example.py

@@ -1,36 +0,0 @@
-import os
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.connectors.discord import (
-    CONNECTOR_TYPE,
-    DiscordAccessConfig,
-    DiscordConnectionConfig,
-    DiscordDownloaderConfig,
-    DiscordIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.local import LocalUploaderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve()), tqdm=True, verbose=True),
-        indexer_config=DiscordIndexerConfig(channels=os.environ["DISCORD_CHANNELS"].split(",")),
-        downloader_config=DiscordDownloaderConfig(limit=int(os.getenv("DISCORD_LIMIT", 100))),
-        source_connection_config=DiscordConnectionConfig(
-            access_config=DiscordAccessConfig(token=os.environ["DISCORD_TOKEN"])
-        ),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        # chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        # embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
-    ).run()

examples/elasticsearch.py

@@ -1,49 +0,0 @@
-import os
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.elasticsearch import (
-    CONNECTOR_TYPE,
-    ElasticsearchAccessConfig,
-    ElasticsearchConnectionConfig,
-    ElasticsearchUploaderConfig,
-    ElasticsearchUploadStagerConfig,
-)
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    index_name = "ingest-test-destination"
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=LocalIndexerConfig(
-            input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt"
-        ),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=ElasticsearchConnectionConfig(
-            access_config=ElasticsearchAccessConfig(password=os.getenv("ELASTIC_PASSWORD")),
-            username=os.getenv("ELASTIC_USERNAME"),
-            hosts=["http://localhost:9200"],
-        ),
-        uploader_config=ElasticsearchUploaderConfig(index_name=index_name),
-        stager_config=ElasticsearchUploadStagerConfig(index_name=index_name),
-    ).run()

examples/google_drive.py

@@ -1,45 +0,0 @@
-import os
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.google_drive import (
-    CONNECTOR_TYPE,
-    GoogleDriveAccessConfig,
-    GoogleDriveConnectionConfig,
-    GoogleDriveDownloaderConfig,
-    GoogleDriveIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.local import (
-    LocalUploaderConfig,
-)
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-
-
-if __name__ == "__main__":
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        # You'll need to set GOOGLE_DRIVE_SERVICE_KEY and GOOGLE_DRIVE_DRIVE_ID
-        # environment variable to run this example
-        source_connection_config=GoogleDriveConnectionConfig(
-            access_config=GoogleDriveAccessConfig(
-                service_account_key=os.environ.get("GOOGLE_DRIVE_SERVICE_KEY")
-            ),
-            drive_id=os.environ.get("GOOGLE_DRIVE_DRIVE_ID"),
-        ),
-        indexer_config=GoogleDriveIndexerConfig(
-            resursive=True,
-        ),
-        downloader_config=GoogleDriveDownloaderConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(
-            chunking_strategy="basic",
-        ),
-        embedder_config=None,
-        uploader_config=LocalUploaderConfig(output_dir=output_path),
-    ).run()

examples/kdbai.py

@@ -1,54 +0,0 @@
-import os
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.kdbai import (
-    CONNECTOR_TYPE,
-    KdbaiConnectionConfig,
-    KdbaiUploaderConfig,
-    KdbaiUploadStagerConfig,
-)
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-input_path = docs_path.resolve() / "pdf" / "fake-memo.pdf"
-
-os.environ["KDBAI_API_KEY"] = "key"
-os.environ["KDBAI_ENDPOINT"] = "http://localhost"
-os.environ["KDBAI_DATABASE"] = "default"
-os.environ["KDBAI_TABLE"] = "table"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    logger.info(f"processing file(s): {input_path.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve()), tqdm=True, verbose=True),
-        indexer_config=LocalIndexerConfig(
-            input_path=docs_path.resolve() / "book-war-and-peace-1p.txt"
-        ),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=KdbaiConnectionConfig(
-            endpoint=os.environ["KDBAI_ENDPOINT"],
-        ),
-        stager_config=KdbaiUploadStagerConfig(),
-        uploader_config=KdbaiUploaderConfig(
-            database_name=os.environ["KDBAI_DATABASE"], table_name=os.environ["KDBAI_TABLE"]
-        ),
-    ).run()

examples/local.py

@@ -1,36 +0,0 @@
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.local import (
-    CONNECTOR_TYPE,
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-    LocalUploaderConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=LocalIndexerConfig(
-            input_path=str(docs_path.resolve()) + "/language-docs/UDHR_first_article_all.txt"
-        ),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
-    ).run()

examples/milvus.py

@@ -1,44 +0,0 @@
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.milvus import (
-    CONNECTOR_TYPE,
-    MilvusConnectionConfig,
-    MilvusUploaderConfig,
-    MilvusUploadStagerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve()), tqdm=True, verbose=True),
-        indexer_config=LocalIndexerConfig(
-            input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt"
-        ),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=MilvusConnectionConfig(
-            uri="http://localhost:19530", db_name="milvus"
-        ),
-        stager_config=MilvusUploadStagerConfig(),
-        uploader_config=MilvusUploaderConfig(collection_name="ingest_test"),
-    ).run()

examples/mongodb.py

@@ -1,53 +0,0 @@
-import random
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.mongodb import (
-    CONNECTOR_TYPE,
-    MongoDBAccessConfig,
-    MongoDBConnectionConfig,
-    MongoDBUploaderConfig,
-    MongoDBUploadStagerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=LocalIndexerConfig(input_path=str(docs_path.resolve()) + "/multisimple/"),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(
-            chunking_strategy="by_title",
-            chunk_include_orig_elements=False,
-            chunk_max_characters=1500,
-            chunk_multipage_sections=True,
-        ),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=MongoDBConnectionConfig(
-            access_config=MongoDBAccessConfig(uri=None),
-            host="localhost",
-            port=27017,
-            collection=f"test-collection-{random.randint(1000, 9999)}",
-            database="testDatabase",
-        ),
-        stager_config=MongoDBUploadStagerConfig(),
-        uploader_config=MongoDBUploaderConfig(batch_size=10),
-    ).run()

examples/opensearch.py

@@ -1,50 +0,0 @@
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.opensearch import (
-    CONNECTOR_TYPE,
-    OpenSearchAccessConfig,
-    OpenSearchConnectionConfig,
-    OpenSearchUploaderConfig,
-    OpenSearchUploadStagerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=LocalIndexerConfig(
-            input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt"
-        ),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=OpenSearchConnectionConfig(
-            hosts="http://localhost:9247",
-            username="admin",
-            use_ssl=True,
-            access_config=OpenSearchAccessConfig(password="admin"),
-        ),
-        stager_config=OpenSearchUploadStagerConfig(index_name="ingest-test-destination"),
-        uploader_config=OpenSearchUploaderConfig(
-            index_name="ingest-test-destination", batch_size_bytes=150
-        ),
-    ).run()

examples/pinecone.py

@@ -1,57 +0,0 @@
-import os
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.pinecone import (
-    CONNECTOR_TYPE,
-    PineconeAccessConfig,
-    PineconeConnectionConfig,
-    PineconeUploaderConfig,
-    PineconeUploadStagerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=LocalIndexerConfig(
-            input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt"
-        ),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=PineconeConnectionConfig(
-            # You'll need to set PINECONE_API_KEY environment variable to run this example
-            access_config=PineconeAccessConfig(pinecone_api_key=os.getenv("PINECONE_API_KEY")),
-            index_name=os.getenv(
-                "PINECONE_INDEX",
-                default="your index name here. e.g. my-index,"
-                "or define in environment variable PINECONE_INDEX",
-            ),
-            environment=os.getenv(
-                "PINECONE_ENVIRONMENT",
-                default="your environment name here. e.g. us-east-1,"
-                "or define in environment variable PINECONE_ENVIRONMENT",
-            ),
-        ),
-        stager_config=PineconeUploadStagerConfig(),
-        uploader_config=PineconeUploaderConfig(batch_size=10, num_processes=2),
-    ).run()

examples/s3.py

@@ -1,38 +0,0 @@
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.fsspec.s3 import (
-    CONNECTOR_TYPE,
-    S3ConnectionConfig,
-    S3DownloaderConfig,
-    S3IndexerConfig,
-)
-from unstructured_ingest.processes.connectors.local import (
-    LocalUploaderConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.filter import FiltererConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve()), verbose=True, iter_delete=True),
-        indexer_config=S3IndexerConfig(remote_url="s3://utic-dev-tech-fixtures/small-pdf-set/"),
-        downloader_config=S3DownloaderConfig(download_dir=download_path),
-        source_connection_config=S3ConnectionConfig(anonymous=True),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
-        filterer_config=FiltererConfig(max_file_size=900000),
-    ).run()

examples/salesforce.py

@@ -1,44 +0,0 @@
-import os
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.local import (
-    LocalUploaderConfig,
-)
-from unstructured_ingest.processes.connectors.salesforce import (
-    CONNECTOR_TYPE,
-    SalesforceAccessConfig,
-    SalesforceConnectionConfig,
-    SalesforceDownloaderConfig,
-    SalesforceIndexerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=SalesforceIndexerConfig(categories=["Campaign", "EmailMessage"]),
-        downloader_config=SalesforceDownloaderConfig(download_dir=download_path),
-        source_connection_config=SalesforceConnectionConfig(
-            SalesforceAccessConfig(
-                consumer_key=os.getenv("SALESFORCE_CONSUMER_KEY"),
-                private_key=os.getenv("SALESFORCE_PRIVATE_KEY"),
-            ),
-            username=os.getenv("SALESFORCE_USERNAME"),
-        ),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
-    ).run()

examples/sharepoint.py

@@ -1,47 +0,0 @@
-import os
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.connectors.local import (
-    LocalUploaderConfig,
-)
-from unstructured_ingest.processes.connectors.sharepoint import (
-    CONNECTOR_TYPE,
-    SharepointAccessConfig,
-    SharepointConnectionConfig,
-    SharepointDownloaderConfig,
-    SharepointIndexerConfig,
-    SharepointPermissionsConfig,
-)
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve()), tqdm=True, verbose=True),
-        indexer_config=SharepointIndexerConfig(),
-        downloader_config=SharepointDownloaderConfig(download_dir=download_path),
-        source_connection_config=SharepointConnectionConfig(
-            client_id=os.getenv("SHAREPOINT_CLIENT_ID"),
-            site=os.getenv("SHAREPOINT_SITE"),
-            access_config=SharepointAccessConfig(client_cred=os.getenv("SHAREPOINT_CRED")),
-            permissions_config=SharepointPermissionsConfig(
-                permissions_application_id=os.getenv("SHAREPOINT_PERMISSIONS_APP_ID"),
-                permissions_client_cred=os.getenv("SHAREPOINT_PERMISSIONS_APP_CRED"),
-                permissions_tenant=os.getenv("SHAREPOINT_PERMISSIONS_TENANT"),
-            ),
-        ),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        # chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        # embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
-    ).run()