unstructured-ingest 0.2.1__tar.gz → 0.2.2__tar.gz

{unstructured_ingest-0.2.1/unstructured_ingest.egg-info → unstructured_ingest-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.2.1
+Version: 0.2.2
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,15 +22,15 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
+Requires-Dist: pandas
 Requires-Dist: tqdm
-Requires-Dist: click
-Requires-Dist: pydantic>=2.7
-Requires-Dist: dataclasses_json
 Requires-Dist: opentelemetry-sdk
-Requires-Dist: pandas
+Requires-Dist: dataclasses_json
+Requires-Dist: pydantic>=2.7
+Requires-Dist: click
 Requires-Dist: python-dateutil
 Provides-Extra: remote
-Requires-Dist: unstructured-client>=0.25.8; extra == "remote"
+Requires-Dist: unstructured-client>=0.26.1; extra == "remote"
 Provides-Extra: csv
 Requires-Dist: unstructured[tsv]; extra == "csv"
 Provides-Extra: doc
@@ -66,23 +66,23 @@ Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: adlfs; extra == "azure"
 Requires-Dist: fsspec; extra == "azure"
+Requires-Dist: adlfs; extra == "azure"
 Provides-Extra: azure-cognitive-search
 Requires-Dist: azure-search-documents; extra == "azure-cognitive-search"
 Provides-Extra: biomed
-Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
+Requires-Dist: requests; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: boxfs; extra == "box"
 Requires-Dist: fsspec; extra == "box"
+Requires-Dist: boxfs; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
 Requires-Dist: clarifai; extra == "clarifai"
 Provides-Extra: confluence
-Requires-Dist: requests; extra == "confluence"
 Requires-Dist: atlassian-python-api; extra == "confluence"
+Requires-Dist: requests; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
 Provides-Extra: delta-table
@@ -91,8 +91,8 @@ Requires-Dist: deltalake; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord-py; extra == "discord"
 Provides-Extra: dropbox
-Requires-Dist: fsspec; extra == "dropbox"
 Requires-Dist: dropboxdrivefs; extra == "dropbox"
+Requires-Dist: fsspec; extra == "dropbox"
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
 Provides-Extra: gcs
@@ -100,15 +100,15 @@ Requires-Dist: fsspec; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: gcsfs; extra == "gcs"
 Provides-Extra: github
-Requires-Dist: requests; extra == "github"
 Requires-Dist: pygithub>1.58.0; extra == "github"
+Requires-Dist: requests; extra == "github"
 Provides-Extra: gitlab
 Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: urllib3; extra == "hubspot"
 Requires-Dist: hubspot-api-client; extra == "hubspot"
+Requires-Dist: urllib3; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -120,14 +120,14 @@ Requires-Dist: pymilvus; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: notion
+Requires-Dist: httpx; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
-Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: notion-client; extra == "notion"
-Requires-Dist: httpx; extra == "notion"
+Requires-Dist: htmlBuilder; extra == "notion"
 Provides-Extra: onedrive
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
-Requires-Dist: bs4; extra == "onedrive"
 Requires-Dist: msal; extra == "onedrive"
+Requires-Dist: bs4; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: outlook
@@ -155,7 +155,7 @@ Requires-Dist: fsspec; extra == "sftp"
 Provides-Extra: slack
 Requires-Dist: slack_sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: snowflake; extra == "snowflake"
+Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Provides-Extra: wikipedia
 Requires-Dist: wikipedia; extra == "wikipedia"
 Provides-Extra: weaviate
@@ -169,8 +169,8 @@ Requires-Dist: requests; extra == "vectara"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
-Requires-Dist: openai; extra == "embed-octoai"
 Requires-Dist: tiktoken; extra == "embed-octoai"
+Requires-Dist: openai; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
@@ -178,8 +178,8 @@ Requires-Dist: voyageai; extra == "embed-voyageai"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Provides-Extra: openai
-Requires-Dist: openai; extra == "openai"
 Requires-Dist: tiktoken; extra == "openai"
+Requires-Dist: openai; extra == "openai"
 Provides-Extra: bedrock
 Requires-Dist: boto3; extra == "bedrock"
 Provides-Extra: togetherai

unstructured_ingest-0.2.2/test/integration/connectors/test_confluence.py

@@ -0,0 +1,113 @@
+import os
+
+import pytest
+
+from test.integration.connectors.utils.constants import (
+    SOURCE_TAG,
+)
+from test.integration.connectors.utils.validation import (
+    ValidationConfigs,
+    source_connector_validation,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.processes.connectors.confluence import (
+    CONNECTOR_TYPE,
+    ConfluenceAccessConfig,
+    ConfluenceConnectionConfig,
+    ConfluenceDownloader,
+    ConfluenceDownloaderConfig,
+    ConfluenceIndexer,
+    ConfluenceIndexerConfig,
+)
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@requires_env("CONFLUENCE_USER_EMAIL", "CONFLUENCE_API_TOKEN")
+async def test_confluence_source(temp_dir):
+    # Retrieve environment variables
+    confluence_url = "https://unstructured-ingest-test.atlassian.net"
+    user_email = os.environ["CONFLUENCE_USER_EMAIL"]
+    api_token = os.environ["CONFLUENCE_API_TOKEN"]
+    spaces = ["testteamsp", "MFS"]
+
+    # Create connection and indexer configurations
+    access_config = ConfluenceAccessConfig(api_token=api_token)
+    connection_config = ConfluenceConnectionConfig(
+        url=confluence_url,
+        user_email=user_email,
+        access_config=access_config,
+    )
+    index_config = ConfluenceIndexerConfig(
+        max_num_of_spaces=500,
+        max_num_of_docs_from_each_space=100,
+        spaces=spaces,
+    )
+
+    download_config = ConfluenceDownloaderConfig(download_dir=temp_dir)
+
+    # Instantiate indexer and downloader
+    indexer = ConfluenceIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+    )
+    downloader = ConfluenceDownloader(
+        connection_config=connection_config,
+        download_config=download_config,
+    )
+
+    # Run the source connector validation
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=ValidationConfigs(
+            test_id="confluence",
+            expected_num_files=11,
+            validate_downloaded_files=True,
+        ),
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@requires_env("CONFLUENCE_USER_EMAIL", "CONFLUENCE_API_TOKEN")
+async def test_confluence_source_large(temp_dir):
+    # Retrieve environment variables
+    confluence_url = "https://unstructured-ingest-test.atlassian.net"
+    user_email = os.environ["CONFLUENCE_USER_EMAIL"]
+    api_token = os.environ["CONFLUENCE_API_TOKEN"]
+    spaces = ["testteamsp1"]
+
+    # Create connection and indexer configurations
+    access_config = ConfluenceAccessConfig(api_token=api_token)
+    connection_config = ConfluenceConnectionConfig(
+        url=confluence_url,
+        user_email=user_email,
+        access_config=access_config,
+    )
+    index_config = ConfluenceIndexerConfig(
+        max_num_of_spaces=10,
+        max_num_of_docs_from_each_space=250,
+        spaces=spaces,
+    )
+
+    download_config = ConfluenceDownloaderConfig(download_dir=temp_dir)
+
+    # Instantiate indexer and downloader
+    indexer = ConfluenceIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+    )
+    downloader = ConfluenceDownloader(
+        connection_config=connection_config,
+        download_config=download_config,
+    )
+
+    # Run the source connector validation
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=ValidationConfigs(
+            test_id="confluence_large", expected_num_files=250, validate_file_data=False
+        ),
+    )

unstructured_ingest-0.2.2/test/integration/connectors/test_kafka.py

@@ -0,0 +1,67 @@
+import socket
+import tempfile
+from pathlib import Path
+
+import pytest
+from confluent_kafka import Producer
+
+from test.integration.connectors.utils.constants import (
+    SOURCE_TAG,
+    env_setup_path,
+)
+from test.integration.connectors.utils.docker_compose import docker_compose_context
+from test.integration.connectors.utils.validation import (
+    ValidationConfigs,
+    source_connector_validation,
+)
+from unstructured_ingest.v2.processes.connectors.kafka.local import (
+    CONNECTOR_TYPE,
+    LocalKafkaConnectionConfig,
+    LocalKafkaDownloader,
+    LocalKafkaDownloaderConfig,
+    LocalKafkaIndexer,
+    LocalKafkaIndexerConfig,
+)
+
+SEED_MESSAGES = 10
+TOPIC = "fake-topic"
+
+
+@pytest.fixture
+def kafka_seed_topic() -> str:
+    with docker_compose_context(docker_compose_path=env_setup_path / "kafka"):
+        conf = {
+            "bootstrap.servers": "localhost:29092",
+            "client.id": socket.gethostname(),
+            "message.max.bytes": 10485760,
+        }
+        producer = Producer(conf)
+        for i in range(SEED_MESSAGES):
+            message = f"This is some text for message {i}"
+            producer.produce(topic=TOPIC, value=message)
+        producer.flush(timeout=10)
+        print(f"kafka topic {TOPIC} seeded with {SEED_MESSAGES} messages")
+        yield TOPIC
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+async def test_kafka_source_local(kafka_seed_topic: str):
+    connection_config = LocalKafkaConnectionConfig(bootstrap_server="localhost", port=29092)
+    with tempfile.TemporaryDirectory() as tempdir:
+        tempdir_path = Path(tempdir)
+        download_config = LocalKafkaDownloaderConfig(download_dir=tempdir_path)
+        indexer = LocalKafkaIndexer(
+            connection_config=connection_config,
+            index_config=LocalKafkaIndexerConfig(topic=kafka_seed_topic, num_messages_to_consume=5),
+        )
+        downloader = LocalKafkaDownloader(
+            connection_config=connection_config, download_config=download_config
+        )
+        await source_connector_validation(
+            indexer=indexer,
+            downloader=downloader,
+            configs=ValidationConfigs(
+                test_id="kafka", expected_num_files=5, validate_downloaded_files=True
+            ),
+        )

unstructured_ingest-0.2.2/test/integration/connectors/test_onedrive.py

@@ -0,0 +1,112 @@
+import os
+import uuid
+from pathlib import Path
+
+import pytest
+from office365.graph_client import GraphClient
+
+from test.integration.connectors.utils.constants import (
+    DESTINATION_TAG,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
+from unstructured_ingest.v2.processes.connectors.onedrive import (
+    CONNECTOR_TYPE,
+    OnedriveAccessConfig,
+    OnedriveConnectionConfig,
+    OnedriveUploader,
+    OnedriveUploaderConfig,
+)
+
+
+@pytest.fixture
+def onedrive_test_folder() -> str:
+    """
+    Pytest fixture that creates a test folder in OneDrive and deletes it after test run.
+    """
+    connection_config = get_connection_config()
+    user_pname = connection_config.user_pname
+
+    # Get the OneDrive client
+    client: GraphClient = connection_config.get_client()
+    drive = client.users[user_pname].drive
+
+    # Generate a unique test folder path
+    test_folder_path = f"utic-test-output-{uuid.uuid4()}"
+
+    # Create the test folder
+    root = drive.root
+    folder = root.create_folder(test_folder_path).execute_query()
+    print(f"created folder: {folder.name}")
+    try:
+        yield test_folder_path
+    finally:
+        # Teardown: delete the test folder and its contents
+        folder.delete_object().execute_query()
+        print(f"successfully deleted folder: {folder.name}")
+
+
+def get_connection_config():
+    """
+    Pytest fixture that provides the OnedriveConnectionConfig for tests.
+    """
+    client_id = os.getenv("MS_CLIENT_ID")
+    client_secret = os.getenv("MS_CLIENT_CRED")
+    tenant_id = os.getenv("MS_TENANT_ID")
+    user_pname = os.getenv("MS_USER_PNAME")
+
+    connection_config = OnedriveConnectionConfig(
+        client_id=client_id,
+        tenant=tenant_id,
+        user_pname=user_pname,
+        access_config=OnedriveAccessConfig(client_cred=client_secret),
+    )
+    return connection_config
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@requires_env("MS_CLIENT_CRED", "MS_CLIENT_ID", "MS_TENANT_ID", "MS_USER_PNAME")
+def test_onedrive_destination(upload_file: Path, onedrive_test_folder: str):
+    """
+    Integration test for the OneDrive destination connector.
+
+    This test uploads a file to OneDrive and verifies that it exists.
+    """
+    connection_config = get_connection_config()
+    # Retrieve user principal name from the connection config
+    user_pname = connection_config.user_pname
+
+    # The test folder is provided by the fixture
+    destination_folder = onedrive_test_folder
+    destination_fullpath = f"{destination_folder}/{upload_file.name}"
+
+    # Configure the uploader with remote_url
+    upload_config = OnedriveUploaderConfig(remote_url=f"onedrive://{destination_folder}")
+
+    uploader = OnedriveUploader(
+        connection_config=connection_config,
+        upload_config=upload_config,
+    )
+
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(
+            fullpath=destination_fullpath,
+            filename=upload_file.name,
+        ),
+        connector_type=CONNECTOR_TYPE,
+        identifier="mock_file_data",
+    )
+    uploader.precheck()
+    uploader.run(path=upload_file, file_data=file_data)
+
+    # Verify that the file was uploaded
+    client = connection_config.get_client()
+    drive = client.users[user_pname].drive
+
+    uploaded_file = (
+        drive.root.get_by_path(destination_fullpath).select(["id", "name"]).get().execute_query()
+    )
+
+    # Check if the file exists
+    assert uploaded_file is not None
+    assert uploaded_file.name == upload_file.name

unstructured_ingest-0.2.2/test/integration/connectors/test_qdrant.py

@@ -0,0 +1,137 @@
+import json
+import uuid
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import AsyncGenerator
+
+import pytest
+from qdrant_client import AsyncQdrantClient
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.connectors.utils.docker import container_context
+from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.v2.processes.connectors.qdrant.local import (
+    CONNECTOR_TYPE as LOCAL_CONNECTOR_TYPE,
+)
+from unstructured_ingest.v2.processes.connectors.qdrant.local import (
+    LocalQdrantConnectionConfig,
+    LocalQdrantUploader,
+    LocalQdrantUploaderConfig,
+    LocalQdrantUploadStager,
+    LocalQdrantUploadStagerConfig,
+)
+from unstructured_ingest.v2.processes.connectors.qdrant.server import (
+    CONNECTOR_TYPE as SERVER_CONNECTOR_TYPE,
+)
+from unstructured_ingest.v2.processes.connectors.qdrant.server import (
+    ServerQdrantConnectionConfig,
+    ServerQdrantUploader,
+    ServerQdrantUploaderConfig,
+    ServerQdrantUploadStager,
+    ServerQdrantUploadStagerConfig,
+)
+
+COLLECTION_NAME = f"test-coll-{uuid.uuid4().hex[:12]}"
+VECTORS_CONFIG = {"size": 384, "distance": "Cosine"}
+
+
+@asynccontextmanager
+async def qdrant_client(client_params: dict) -> AsyncGenerator[AsyncQdrantClient, None]:
+    client = AsyncQdrantClient(**client_params)
+    try:
+        yield client
+    finally:
+        await client.close()
+
+
+async def validate_upload(client: AsyncQdrantClient, upload_file: Path):
+    with upload_file.open() as upload_fp:
+        elements = json.load(upload_fp)
+    expected_point_count = len(elements)
+    first_element = elements[0]
+    expected_text = first_element["text"]
+    embeddings = first_element["embeddings"]
+    collection = await client.get_collection(COLLECTION_NAME)
+    assert collection.points_count == expected_point_count
+
+    response = await client.query_points(COLLECTION_NAME, query=embeddings, limit=1)
+    assert response.points[0].payload is not None
+    assert response.points[0].payload["text"] == expected_text
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(LOCAL_CONNECTOR_TYPE, DESTINATION_TAG, "qdrant")
+async def test_qdrant_destination_local(upload_file: Path, tmp_path: Path):
+    connection_kwargs = {"path": str(tmp_path / "qdrant")}
+    async with qdrant_client(connection_kwargs) as client:
+        await client.create_collection(COLLECTION_NAME, vectors_config=VECTORS_CONFIG)
+    AsyncQdrantClient(**connection_kwargs)
+    stager = LocalQdrantUploadStager(
+        upload_stager_config=LocalQdrantUploadStagerConfig(),
+    )
+    uploader = LocalQdrantUploader(
+        connection_config=LocalQdrantConnectionConfig(**connection_kwargs),
+        upload_config=LocalQdrantUploaderConfig(collection_name=COLLECTION_NAME),
+    )
+
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
+        connector_type=LOCAL_CONNECTOR_TYPE,
+        identifier="mock-file-data",
+    )
+
+    staged_upload_file = stager.run(
+        elements_filepath=upload_file,
+        file_data=file_data,
+        output_dir=tmp_path,
+        output_filename=upload_file.name,
+    )
+
+    if uploader.is_async():
+        await uploader.run_async(path=staged_upload_file, file_data=file_data)
+    else:
+        uploader.run(path=upload_file, file_data=file_data)
+    async with qdrant_client(connection_kwargs) as client:
+        await validate_upload(client=client, upload_file=upload_file)
+
+
+@pytest.fixture
+def docker_context():
+    with container_context(image="qdrant/qdrant:latest", ports={"6333": "6333"}) as container:
+        yield container
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(SERVER_CONNECTOR_TYPE, DESTINATION_TAG, "qdrant")
+async def test_qdrant_destination_server(upload_file: Path, tmp_path: Path, docker_context):
+    connection_kwargs = {"location": "http://localhost:6333"}
+    async with qdrant_client(connection_kwargs) as client:
+        await client.create_collection(COLLECTION_NAME, vectors_config=VECTORS_CONFIG)
+    AsyncQdrantClient(**connection_kwargs)
+    stager = ServerQdrantUploadStager(
+        upload_stager_config=ServerQdrantUploadStagerConfig(),
+    )
+    uploader = ServerQdrantUploader(
+        connection_config=ServerQdrantConnectionConfig(**connection_kwargs),
+        upload_config=ServerQdrantUploaderConfig(collection_name=COLLECTION_NAME),
+    )
+
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
+        connector_type=SERVER_CONNECTOR_TYPE,
+        identifier="mock-file-data",
+    )
+
+    staged_upload_file = stager.run(
+        elements_filepath=upload_file,
+        file_data=file_data,
+        output_dir=tmp_path,
+        output_filename=upload_file.name,
+    )
+
+    if uploader.is_async():
+        await uploader.run_async(path=staged_upload_file, file_data=file_data)
+    else:
+        uploader.run(path=upload_file, file_data=file_data)
+    async with qdrant_client(connection_kwargs) as client:
+        await validate_upload(client=client, upload_file=upload_file)

{unstructured_ingest-0.2.1 → unstructured_ingest-0.2.2}/test/integration/connectors/utils/docker.py

@@ -47,14 +47,15 @@ def healthcheck_wait(container: Container, timeout: int = 10) -> None:
 
 @contextmanager
 def container_context(
-    docker_client: docker.DockerClient,
     image: str,
     ports: dict,
     environment: Optional[dict] = None,
     volumes: Optional[dict] = None,
     healthcheck: Optional[dict] = None,
     healthcheck_timeout: int = 10,
+    docker_client: Optional[docker.DockerClient] = None,
 ):
+    docker_client = docker_client or docker.from_env()
     container: Optional[Container] = None
     try:
         container = get_container(