unstructured-ingest 0.7.2__py3-none-any.whl → 1.0.2__py3-none-any.whl

examples/opensearch.py

@@ -1,50 +0,0 @@
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.opensearch import (
-    CONNECTOR_TYPE,
-    OpenSearchAccessConfig,
-    OpenSearchConnectionConfig,
-    OpenSearchUploaderConfig,
-    OpenSearchUploadStagerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=LocalIndexerConfig(
-            input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt"
-        ),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=OpenSearchConnectionConfig(
-            hosts="http://localhost:9247",
-            username="admin",
-            use_ssl=True,
-            access_config=OpenSearchAccessConfig(password="admin"),
-        ),
-        stager_config=OpenSearchUploadStagerConfig(index_name="ingest-test-destination"),
-        uploader_config=OpenSearchUploaderConfig(
-            index_name="ingest-test-destination", batch_size_bytes=150
-        ),
-    ).run()

examples/pinecone.py

@@ -1,57 +0,0 @@
-import os
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.pinecone import (
-    CONNECTOR_TYPE,
-    PineconeAccessConfig,
-    PineconeConnectionConfig,
-    PineconeUploaderConfig,
-    PineconeUploadStagerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=LocalIndexerConfig(
-            input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt"
-        ),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=PineconeConnectionConfig(
-            # You'll need to set PINECONE_API_KEY environment variable to run this example
-            access_config=PineconeAccessConfig(pinecone_api_key=os.getenv("PINECONE_API_KEY")),
-            index_name=os.getenv(
-                "PINECONE_INDEX",
-                default="your index name here. e.g. my-index,"
-                "or define in environment variable PINECONE_INDEX",
-            ),
-            environment=os.getenv(
-                "PINECONE_ENVIRONMENT",
-                default="your environment name here. e.g. us-east-1,"
-                "or define in environment variable PINECONE_ENVIRONMENT",
-            ),
-        ),
-        stager_config=PineconeUploadStagerConfig(),
-        uploader_config=PineconeUploaderConfig(batch_size=10, num_processes=2),
-    ).run()

examples/s3.py

@@ -1,38 +0,0 @@
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.fsspec.s3 import (
-    CONNECTOR_TYPE,
-    S3ConnectionConfig,
-    S3DownloaderConfig,
-    S3IndexerConfig,
-)
-from unstructured_ingest.processes.connectors.local import (
-    LocalUploaderConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.filter import FiltererConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve()), verbose=True, iter_delete=True),
-        indexer_config=S3IndexerConfig(remote_url="s3://utic-dev-tech-fixtures/small-pdf-set/"),
-        downloader_config=S3DownloaderConfig(download_dir=download_path),
-        source_connection_config=S3ConnectionConfig(anonymous=True),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
-        filterer_config=FiltererConfig(max_file_size=900000),
-    ).run()

examples/salesforce.py

@@ -1,44 +0,0 @@
-import os
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.local import (
-    LocalUploaderConfig,
-)
-from unstructured_ingest.processes.connectors.salesforce import (
-    CONNECTOR_TYPE,
-    SalesforceAccessConfig,
-    SalesforceConnectionConfig,
-    SalesforceDownloaderConfig,
-    SalesforceIndexerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=SalesforceIndexerConfig(categories=["Campaign", "EmailMessage"]),
-        downloader_config=SalesforceDownloaderConfig(download_dir=download_path),
-        source_connection_config=SalesforceConnectionConfig(
-            SalesforceAccessConfig(
-                consumer_key=os.getenv("SALESFORCE_CONSUMER_KEY"),
-                private_key=os.getenv("SALESFORCE_PRIVATE_KEY"),
-            ),
-            username=os.getenv("SALESFORCE_USERNAME"),
-        ),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
-    ).run()

examples/sharepoint.py

@@ -1,47 +0,0 @@
-import os
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.connectors.local import (
-    LocalUploaderConfig,
-)
-from unstructured_ingest.processes.connectors.sharepoint import (
-    CONNECTOR_TYPE,
-    SharepointAccessConfig,
-    SharepointConnectionConfig,
-    SharepointDownloaderConfig,
-    SharepointIndexerConfig,
-    SharepointPermissionsConfig,
-)
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve()), tqdm=True, verbose=True),
-        indexer_config=SharepointIndexerConfig(),
-        downloader_config=SharepointDownloaderConfig(download_dir=download_path),
-        source_connection_config=SharepointConnectionConfig(
-            client_id=os.getenv("SHAREPOINT_CLIENT_ID"),
-            site=os.getenv("SHAREPOINT_SITE"),
-            access_config=SharepointAccessConfig(client_cred=os.getenv("SHAREPOINT_CRED")),
-            permissions_config=SharepointPermissionsConfig(
-                permissions_application_id=os.getenv("SHAREPOINT_PERMISSIONS_APP_ID"),
-                permissions_client_cred=os.getenv("SHAREPOINT_PERMISSIONS_APP_CRED"),
-                permissions_tenant=os.getenv("SHAREPOINT_PERMISSIONS_TENANT"),
-            ),
-        ),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        # chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        # embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
-    ).run()

examples/singlestore.py

@@ -1,49 +0,0 @@
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.singlestore import (
-    CONNECTOR_TYPE,
-    SingleStoreAccessConfig,
-    SingleStoreConnectionConfig,
-    SingleStoreUploaderConfig,
-    SingleStoreUploadStagerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve()), tqdm=True, verbose=True),
-        indexer_config=LocalIndexerConfig(
-            input_path=str(docs_path.resolve()) + "/book-war-and-peace-1p.txt"
-        ),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=SingleStoreConnectionConfig(
-            access_config=SingleStoreAccessConfig(password="password"),
-            host="localhost",
-            port=3306,
-            database="ingest_test",
-            user="root",
-        ),
-        stager_config=SingleStoreUploadStagerConfig(),
-        uploader_config=SingleStoreUploaderConfig(table_name="elements"),
-    ).run()

examples/sql.py

@@ -1,90 +0,0 @@
-import os
-import sqlite3
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.sql import (
-    CONNECTOR_TYPE,
-    POSTGRESQL_DB,
-    SQLITE_DB,
-    SQLAccessConfig,
-    SQLConnectionConfig,
-    SQLUploaderConfig,
-    SQLUploadStagerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-SQLITE_DB_PATH = "test-sql-db.sqlite"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-
-    configs = {
-        "context": ProcessorConfig(work_dir=str(work_dir.resolve())),
-        "indexer_config": LocalIndexerConfig(input_path=str(docs_path.resolve()) + "/multisimple/"),
-        "downloader_config": LocalDownloaderConfig(download_dir=download_path),
-        "source_connection_config": LocalConnectionConfig(),
-        "partitioner_config": PartitionerConfig(strategy="fast"),
-        "chunker_config": ChunkerConfig(
-            chunking_strategy="by_title",
-            chunk_include_orig_elements=False,
-            chunk_max_characters=1500,
-            chunk_multipage_sections=True,
-        ),
-        "embedder_config": EmbedderConfig(embedding_provider="huggingface"),
-        "stager_config": SQLUploadStagerConfig(),
-        "uploader_config": SQLUploaderConfig(batch_size=10),
-    }
-
-    if os.path.exists(SQLITE_DB):
-        os.remove(SQLITE_DB)
-
-    connection = sqlite3.connect(database=SQLITE_DB)
-
-    query = None
-    script_path = (
-        Path(__file__).parent.parent.parent.parent.parent
-        / Path("test_e2e/env_setup/sql/sqlite-schema.sql")
-    ).resolve()
-    with open(script_path) as f:
-        query = f.read()
-    cursor = connection.cursor()
-    cursor.executescript(query)
-    connection.close()
-
-    # sqlite test first
-    Pipeline.from_configs(
-        destination_connection_config=SQLConnectionConfig(
-            db_type=SQLITE_DB,
-            database=SQLITE_DB_PATH,
-            access_config=SQLAccessConfig(),
-        ),
-        **configs,
-    ).run()
-
-    # now, pg with pgvector
-    Pipeline.from_configs(
-        destination_connection_config=SQLConnectionConfig(
-            db_type=POSTGRESQL_DB,
-            database="elements",
-            host="localhost",
-            port=5433,
-            access_config=SQLAccessConfig(username="unstructured", password="test"),
-        ),
-        **configs,
-    ).run()

examples/vectara.py

@@ -1,54 +0,0 @@
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.vectara import (
-    CONNECTOR_TYPE,
-    VectaraAccessConfig,
-    VectaraConnectionConfig,
-    VectaraUploaderConfig,
-    VectaraUploadStagerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=LocalIndexerConfig(input_path=str(docs_path.resolve()) + "/multisimple/"),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(
-            chunking_strategy="by_title",
-            chunk_include_orig_elements=False,
-            chunk_max_characters=1500,
-            chunk_multipage_sections=True,
-        ),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=VectaraConnectionConfig(
-            access_config=VectaraAccessConfig(
-                oauth_client_id="fill oauth_client_id", oauth_secret="fill oauth_secret"
-            ),
-            customer_id="fill customer_id",
-            corpus_name="fill corpus_name",
-            corpus_key="fill corpus_key",
-            token_url="fill token_url",
-        ),
-        stager_config=VectaraUploadStagerConfig(batch_size=10),
-        uploader_config=VectaraUploaderConfig(),
-    ).run()

examples/weaviate.py

@@ -1,44 +0,0 @@
-from pathlib import Path
-
-from unstructured_ingest.interfaces import ProcessorConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.pipeline.pipeline import Pipeline
-from unstructured_ingest.processes.chunker import ChunkerConfig
-from unstructured_ingest.processes.connectors.local import (
-    LocalConnectionConfig,
-    LocalDownloaderConfig,
-    LocalIndexerConfig,
-)
-from unstructured_ingest.processes.connectors.weaviate.local import (
-    CONNECTOR_TYPE,
-    LocalWeaviateConnectionConfig,
-    LocalWeaviateUploaderConfig,
-    LocalWeaviateUploadStagerConfig,
-)
-from unstructured_ingest.processes.embedder import EmbedderConfig
-from unstructured_ingest.processes.partitioner import PartitionerConfig
-
-base_path = Path(__file__).parent.parent.parent.parent
-docs_path = base_path / "example-docs"
-work_dir = base_path / "tmp_ingest" / CONNECTOR_TYPE
-output_path = work_dir / "output"
-download_path = work_dir / "download"
-
-if __name__ == "__main__":
-    logger.info(f"writing all content in: {work_dir.resolve()}")
-    Pipeline.from_configs(
-        context=ProcessorConfig(work_dir=str(work_dir.resolve())),
-        indexer_config=LocalIndexerConfig(input_path=str(docs_path.resolve()) + "/multisimple/"),
-        downloader_config=LocalDownloaderConfig(download_dir=download_path),
-        source_connection_config=LocalConnectionConfig(),
-        partitioner_config=PartitionerConfig(strategy="fast"),
-        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
-        destination_connection_config=LocalWeaviateConnectionConfig(
-            # Connects to http://localhost:8080
-        ),
-        stager_config=LocalWeaviateUploadStagerConfig(),
-        uploader_config=LocalWeaviateUploaderConfig(
-            collection="elements", batch_size=10, dynamic_batch=False
-        ),
-    ).run()

test/integration/chunkers/test_chunkers.py

@@ -1,31 +0,0 @@
-import os
-from pathlib import Path
-
-import pytest
-
-from test.integration.utils import requires_env
-from unstructured_ingest.processes.chunker import Chunker, ChunkerConfig
-
-int_test_dir = Path(__file__).parent
-assets_dir = int_test_dir / "assets"
-
-chunker_files = [path for path in assets_dir.iterdir() if path.is_file()]
-
-
-@pytest.mark.parametrize("chunker_file", chunker_files, ids=[path.name for path in chunker_files])
-@pytest.mark.parametrize("strategy", ["basic", "by_title", "by_similarity", "by_page"])
-@requires_env("UNSTRUCTURED_API_KEY", "UNSTRUCTURED_API_URL")
-@pytest.mark.asyncio
-async def test_chunker_api(chunker_file: Path, strategy: str):
-    api_key = os.getenv("UNSTRUCTURED_API_KEY")
-    api_url = os.getenv("UNSTRUCTURED_API_URL")
-
-    chunker_config = ChunkerConfig(
-        chunking_strategy=strategy,
-        chunk_by_api=True,
-        chunk_api_key=api_key,
-        chunking_endpoint=api_url,
-    )
-    chunker = Chunker(config=chunker_config)
-    results = await chunker.run_async(elements_filepath=chunker_file)
-    assert results

test/integration/connectors/conftest.py

@@ -1,38 +0,0 @@
-import tempfile
-from pathlib import Path
-from typing import Generator
-
-import pytest
-
-from unstructured_ingest.logger import logger
-
-FILENAME = Path("DA-1p-with-duplicate-pages.pdf.json")
-
-
-@pytest.fixture
-def upload_file() -> Path:
-    int_test_dir = Path(__file__).parent
-    assets_dir = int_test_dir / "assets"
-    upload_file = assets_dir / FILENAME
-    assert upload_file.exists()
-    assert upload_file.is_file()
-    return upload_file
-
-
-@pytest.fixture
-def upload_file_ndjson() -> Path:
-    int_test_dir = Path(__file__).parent
-    assets_dir = int_test_dir / "assets"
-    upload_file = assets_dir / FILENAME.with_suffix(".ndjson")
-    assert upload_file.exists()
-    assert upload_file.is_file()
-    return upload_file
-
-
-@pytest.fixture
-def temp_dir() -> Generator[Path, None, None]:
-    with tempfile.TemporaryDirectory() as temp_dir:
-        temp_path = Path(temp_dir)
-        logger.info(f"Created temp dir '{temp_path}'")
-        yield temp_path
-        logger.info(f"Removing temp dir '{temp_path}'")