unstructured-ingest 0.2.0__tar.gz → 0.2.2__tar.gz

{unstructured_ingest-0.2.0/unstructured_ingest.egg-info → unstructured_ingest-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.2.0
+Version: 0.2.2
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,15 +22,15 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: pydantic>=2.7
 Requires-Dist: pandas
 Requires-Dist: tqdm
 Requires-Dist: opentelemetry-sdk
-Requires-Dist: click
 Requires-Dist: dataclasses_json
+Requires-Dist: pydantic>=2.7
+Requires-Dist: click
 Requires-Dist: python-dateutil
 Provides-Extra: remote
-Requires-Dist: unstructured-client>=0.25.8; extra == "remote"
+Requires-Dist: unstructured-client>=0.26.1; extra == "remote"
 Provides-Extra: csv
 Requires-Dist: unstructured[tsv]; extra == "csv"
 Provides-Extra: doc
@@ -66,13 +66,13 @@ Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: adlfs; extra == "azure"
 Requires-Dist: fsspec; extra == "azure"
+Requires-Dist: adlfs; extra == "azure"
 Provides-Extra: azure-cognitive-search
 Requires-Dist: azure-search-documents; extra == "azure-cognitive-search"
 Provides-Extra: biomed
-Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
+Requires-Dist: requests; extra == "biomed"
 Provides-Extra: box
 Requires-Dist: fsspec; extra == "box"
 Requires-Dist: boxfs; extra == "box"
@@ -81,8 +81,8 @@ Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
 Requires-Dist: clarifai; extra == "clarifai"
 Provides-Extra: confluence
-Requires-Dist: requests; extra == "confluence"
 Requires-Dist: atlassian-python-api; extra == "confluence"
+Requires-Dist: requests; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
 Provides-Extra: delta-table
@@ -91,24 +91,24 @@ Requires-Dist: deltalake; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord-py; extra == "discord"
 Provides-Extra: dropbox
-Requires-Dist: fsspec; extra == "dropbox"
 Requires-Dist: dropboxdrivefs; extra == "dropbox"
+Requires-Dist: fsspec; extra == "dropbox"
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
 Provides-Extra: gcs
 Requires-Dist: fsspec; extra == "gcs"
-Requires-Dist: gcsfs; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
+Requires-Dist: gcsfs; extra == "gcs"
 Provides-Extra: github
-Requires-Dist: requests; extra == "github"
 Requires-Dist: pygithub>1.58.0; extra == "github"
+Requires-Dist: requests; extra == "github"
 Provides-Extra: gitlab
 Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: urllib3; extra == "hubspot"
 Requires-Dist: hubspot-api-client; extra == "hubspot"
+Requires-Dist: urllib3; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -121,9 +121,9 @@ Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: notion
 Requires-Dist: httpx; extra == "notion"
+Requires-Dist: backoff; extra == "notion"
 Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
-Requires-Dist: backoff; extra == "notion"
 Provides-Extra: onedrive
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Requires-Dist: msal; extra == "onedrive"
@@ -142,20 +142,20 @@ Requires-Dist: qdrant-client; extra == "qdrant"
 Provides-Extra: reddit
 Requires-Dist: praw; extra == "reddit"
 Provides-Extra: s3
-Requires-Dist: fsspec; extra == "s3"
 Requires-Dist: s3fs; extra == "s3"
+Requires-Dist: fsspec; extra == "s3"
 Provides-Extra: sharepoint
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Requires-Dist: msal; extra == "sharepoint"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: fsspec; extra == "sftp"
 Requires-Dist: paramiko; extra == "sftp"
+Requires-Dist: fsspec; extra == "sftp"
 Provides-Extra: slack
 Requires-Dist: slack_sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: snowflake; extra == "snowflake"
+Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Provides-Extra: wikipedia
 Requires-Dist: wikipedia; extra == "wikipedia"
 Provides-Extra: weaviate

unstructured_ingest-0.2.2/test/integration/connectors/sql/test_singlestore.py

@@ -0,0 +1,156 @@
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+
+import pandas as pd
+import pytest
+import singlestoredb as s2
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG, env_setup_path
+from test.integration.connectors.utils.docker_compose import docker_compose_context
+from test.integration.connectors.utils.validation import (
+    ValidationConfigs,
+    source_connector_validation,
+)
+from unstructured_ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.processes.connectors.sql.singlestore import (
+    CONNECTOR_TYPE,
+    SingleStoreAccessConfig,
+    SingleStoreConnectionConfig,
+    SingleStoreDownloader,
+    SingleStoreDownloaderConfig,
+    SingleStoreIndexer,
+    SingleStoreIndexerConfig,
+    SingleStoreUploader,
+    SingleStoreUploaderConfig,
+    SingleStoreUploadStager,
+)
+
+SEED_DATA_ROWS = 20
+
+
+@contextmanager
+def singlestore_download_setup(connect_params: dict) -> None:
+    with docker_compose_context(
+        docker_compose_path=env_setup_path / "sql" / "singlestore" / "source"
+    ):
+        with s2.connect(**connect_params) as connection:
+            with connection.cursor() as cursor:
+                for i in range(SEED_DATA_ROWS):
+                    sql_statment = f"INSERT INTO cars (brand, price) VALUES " f"('brand_{i}', {i})"
+                    cursor.execute(sql_statment)
+                connection.commit()
+        yield
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, "sql")
+async def test_singlestore_source():
+    connect_params = {
+        "host": "localhost",
+        "port": 3306,
+        "database": "ingest_test",
+        "user": "root",
+        "password": "password",
+    }
+    with singlestore_download_setup(connect_params=connect_params):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            connection_config = SingleStoreConnectionConfig(
+                host=connect_params["host"],
+                port=connect_params["port"],
+                database=connect_params["database"],
+                user=connect_params["user"],
+                access_config=SingleStoreAccessConfig(password=connect_params["password"]),
+            )
+            indexer = SingleStoreIndexer(
+                connection_config=connection_config,
+                index_config=SingleStoreIndexerConfig(
+                    table_name="cars", id_column="car_id", batch_size=5
+                ),
+            )
+            downloader = SingleStoreDownloader(
+                connection_config=connection_config,
+                download_config=SingleStoreDownloaderConfig(
+                    fields=["car_id", "brand"], download_dir=Path(tmpdir)
+                ),
+            )
+            await source_connector_validation(
+                indexer=indexer,
+                downloader=downloader,
+                configs=ValidationConfigs(
+                    test_id="singlestore",
+                    expected_num_files=SEED_DATA_ROWS,
+                    expected_number_indexed_file_data=4,
+                    validate_downloaded_files=True,
+                ),
+            )
+
+
+def validate_destination(
+    connect_params: dict,
+    expected_num_elements: int,
+):
+    with s2.connect(**connect_params) as connection:
+        with connection.cursor() as cursor:
+            query = "select count(*) from elements;"
+            cursor.execute(query)
+            count = cursor.fetchone()[0]
+            assert (
+                count == expected_num_elements
+            ), f"dest check failed: got {count}, expected {expected_num_elements}"
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "sql")
+async def test_singlestore_destination(upload_file: Path):
+    mock_file_data = FileData(identifier="mock file data", connector_type=CONNECTOR_TYPE)
+    with docker_compose_context(
+        docker_compose_path=env_setup_path / "sql" / "singlestore" / "destination"
+    ):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            stager = SingleStoreUploadStager()
+            stager_params = {
+                "elements_filepath": upload_file,
+                "file_data": mock_file_data,
+                "output_dir": Path(tmpdir),
+                "output_filename": "test_db",
+            }
+            if stager.is_async():
+                staged_path = await stager.run_async(**stager_params)
+            else:
+                staged_path = stager.run(**stager_params)
+
+            # The stager should append the `.json` suffix to the output filename passed in.
+            assert staged_path.name == "test_db.json"
+
+            connect_params = {
+                "host": "localhost",
+                "port": 3306,
+                "database": "ingest_test",
+                "user": "root",
+                "password": "password",
+            }
+
+            uploader = SingleStoreUploader(
+                connection_config=SingleStoreConnectionConfig(
+                    host=connect_params["host"],
+                    port=connect_params["port"],
+                    database=connect_params["database"],
+                    user=connect_params["user"],
+                    access_config=SingleStoreAccessConfig(password=connect_params["password"]),
+                ),
+                upload_config=SingleStoreUploaderConfig(
+                    table_name="elements",
+                ),
+            )
+            if uploader.is_async():
+                await uploader.run_async(path=staged_path, file_data=mock_file_data)
+            else:
+                uploader.run(path=staged_path, file_data=mock_file_data)
+
+            staged_df = pd.read_json(staged_path, orient="records", lines=True)
+            expected_num_elements = len(staged_df)
+            validate_destination(
+                connect_params=connect_params,
+                expected_num_elements=expected_num_elements,
+            )

unstructured_ingest-0.2.2/test/integration/connectors/test_confluence.py

@@ -0,0 +1,113 @@
+import os
+
+import pytest
+
+from test.integration.connectors.utils.constants import (
+    SOURCE_TAG,
+)
+from test.integration.connectors.utils.validation import (
+    ValidationConfigs,
+    source_connector_validation,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.processes.connectors.confluence import (
+    CONNECTOR_TYPE,
+    ConfluenceAccessConfig,
+    ConfluenceConnectionConfig,
+    ConfluenceDownloader,
+    ConfluenceDownloaderConfig,
+    ConfluenceIndexer,
+    ConfluenceIndexerConfig,
+)
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@requires_env("CONFLUENCE_USER_EMAIL", "CONFLUENCE_API_TOKEN")
+async def test_confluence_source(temp_dir):
+    # Retrieve environment variables
+    confluence_url = "https://unstructured-ingest-test.atlassian.net"
+    user_email = os.environ["CONFLUENCE_USER_EMAIL"]
+    api_token = os.environ["CONFLUENCE_API_TOKEN"]
+    spaces = ["testteamsp", "MFS"]
+
+    # Create connection and indexer configurations
+    access_config = ConfluenceAccessConfig(api_token=api_token)
+    connection_config = ConfluenceConnectionConfig(
+        url=confluence_url,
+        user_email=user_email,
+        access_config=access_config,
+    )
+    index_config = ConfluenceIndexerConfig(
+        max_num_of_spaces=500,
+        max_num_of_docs_from_each_space=100,
+        spaces=spaces,
+    )
+
+    download_config = ConfluenceDownloaderConfig(download_dir=temp_dir)
+
+    # Instantiate indexer and downloader
+    indexer = ConfluenceIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+    )
+    downloader = ConfluenceDownloader(
+        connection_config=connection_config,
+        download_config=download_config,
+    )
+
+    # Run the source connector validation
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=ValidationConfigs(
+            test_id="confluence",
+            expected_num_files=11,
+            validate_downloaded_files=True,
+        ),
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@requires_env("CONFLUENCE_USER_EMAIL", "CONFLUENCE_API_TOKEN")
+async def test_confluence_source_large(temp_dir):
+    # Retrieve environment variables
+    confluence_url = "https://unstructured-ingest-test.atlassian.net"
+    user_email = os.environ["CONFLUENCE_USER_EMAIL"]
+    api_token = os.environ["CONFLUENCE_API_TOKEN"]
+    spaces = ["testteamsp1"]
+
+    # Create connection and indexer configurations
+    access_config = ConfluenceAccessConfig(api_token=api_token)
+    connection_config = ConfluenceConnectionConfig(
+        url=confluence_url,
+        user_email=user_email,
+        access_config=access_config,
+    )
+    index_config = ConfluenceIndexerConfig(
+        max_num_of_spaces=10,
+        max_num_of_docs_from_each_space=250,
+        spaces=spaces,
+    )
+
+    download_config = ConfluenceDownloaderConfig(download_dir=temp_dir)
+
+    # Instantiate indexer and downloader
+    indexer = ConfluenceIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+    )
+    downloader = ConfluenceDownloader(
+        connection_config=connection_config,
+        download_config=download_config,
+    )
+
+    # Run the source connector validation
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=ValidationConfigs(
+            test_id="confluence_large", expected_num_files=250, validate_file_data=False
+        ),
+    )

unstructured_ingest-0.2.2/test/integration/connectors/test_kafka.py

@@ -0,0 +1,67 @@
+import socket
+import tempfile
+from pathlib import Path
+
+import pytest
+from confluent_kafka import Producer
+
+from test.integration.connectors.utils.constants import (
+    SOURCE_TAG,
+    env_setup_path,
+)
+from test.integration.connectors.utils.docker_compose import docker_compose_context
+from test.integration.connectors.utils.validation import (
+    ValidationConfigs,
+    source_connector_validation,
+)
+from unstructured_ingest.v2.processes.connectors.kafka.local import (
+    CONNECTOR_TYPE,
+    LocalKafkaConnectionConfig,
+    LocalKafkaDownloader,
+    LocalKafkaDownloaderConfig,
+    LocalKafkaIndexer,
+    LocalKafkaIndexerConfig,
+)
+
+SEED_MESSAGES = 10
+TOPIC = "fake-topic"
+
+
+@pytest.fixture
+def kafka_seed_topic() -> str:
+    with docker_compose_context(docker_compose_path=env_setup_path / "kafka"):
+        conf = {
+            "bootstrap.servers": "localhost:29092",
+            "client.id": socket.gethostname(),
+            "message.max.bytes": 10485760,
+        }
+        producer = Producer(conf)
+        for i in range(SEED_MESSAGES):
+            message = f"This is some text for message {i}"
+            producer.produce(topic=TOPIC, value=message)
+        producer.flush(timeout=10)
+        print(f"kafka topic {TOPIC} seeded with {SEED_MESSAGES} messages")
+        yield TOPIC
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+async def test_kafka_source_local(kafka_seed_topic: str):
+    connection_config = LocalKafkaConnectionConfig(bootstrap_server="localhost", port=29092)
+    with tempfile.TemporaryDirectory() as tempdir:
+        tempdir_path = Path(tempdir)
+        download_config = LocalKafkaDownloaderConfig(download_dir=tempdir_path)
+        indexer = LocalKafkaIndexer(
+            connection_config=connection_config,
+            index_config=LocalKafkaIndexerConfig(topic=kafka_seed_topic, num_messages_to_consume=5),
+        )
+        downloader = LocalKafkaDownloader(
+            connection_config=connection_config, download_config=download_config
+        )
+        await source_connector_validation(
+            indexer=indexer,
+            downloader=downloader,
+            configs=ValidationConfigs(
+                test_id="kafka", expected_num_files=5, validate_downloaded_files=True
+            ),
+        )

unstructured_ingest-0.2.2/test/integration/connectors/test_onedrive.py

@@ -0,0 +1,112 @@
+import os
+import uuid
+from pathlib import Path
+
+import pytest
+from office365.graph_client import GraphClient
+
+from test.integration.connectors.utils.constants import (
+    DESTINATION_TAG,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
+from unstructured_ingest.v2.processes.connectors.onedrive import (
+    CONNECTOR_TYPE,
+    OnedriveAccessConfig,
+    OnedriveConnectionConfig,
+    OnedriveUploader,
+    OnedriveUploaderConfig,
+)
+
+
+@pytest.fixture
+def onedrive_test_folder() -> str:
+    """
+    Pytest fixture that creates a test folder in OneDrive and deletes it after test run.
+    """
+    connection_config = get_connection_config()
+    user_pname = connection_config.user_pname
+
+    # Get the OneDrive client
+    client: GraphClient = connection_config.get_client()
+    drive = client.users[user_pname].drive
+
+    # Generate a unique test folder path
+    test_folder_path = f"utic-test-output-{uuid.uuid4()}"
+
+    # Create the test folder
+    root = drive.root
+    folder = root.create_folder(test_folder_path).execute_query()
+    print(f"created folder: {folder.name}")
+    try:
+        yield test_folder_path
+    finally:
+        # Teardown: delete the test folder and its contents
+        folder.delete_object().execute_query()
+        print(f"successfully deleted folder: {folder.name}")
+
+
+def get_connection_config():
+    """
+    Pytest fixture that provides the OnedriveConnectionConfig for tests.
+    """
+    client_id = os.getenv("MS_CLIENT_ID")
+    client_secret = os.getenv("MS_CLIENT_CRED")
+    tenant_id = os.getenv("MS_TENANT_ID")
+    user_pname = os.getenv("MS_USER_PNAME")
+
+    connection_config = OnedriveConnectionConfig(
+        client_id=client_id,
+        tenant=tenant_id,
+        user_pname=user_pname,
+        access_config=OnedriveAccessConfig(client_cred=client_secret),
+    )
+    return connection_config
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@requires_env("MS_CLIENT_CRED", "MS_CLIENT_ID", "MS_TENANT_ID", "MS_USER_PNAME")
+def test_onedrive_destination(upload_file: Path, onedrive_test_folder: str):
+    """
+    Integration test for the OneDrive destination connector.
+
+    This test uploads a file to OneDrive and verifies that it exists.
+    """
+    connection_config = get_connection_config()
+    # Retrieve user principal name from the connection config
+    user_pname = connection_config.user_pname
+
+    # The test folder is provided by the fixture
+    destination_folder = onedrive_test_folder
+    destination_fullpath = f"{destination_folder}/{upload_file.name}"
+
+    # Configure the uploader with remote_url
+    upload_config = OnedriveUploaderConfig(remote_url=f"onedrive://{destination_folder}")
+
+    uploader = OnedriveUploader(
+        connection_config=connection_config,
+        upload_config=upload_config,
+    )
+
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(
+            fullpath=destination_fullpath,
+            filename=upload_file.name,
+        ),
+        connector_type=CONNECTOR_TYPE,
+        identifier="mock_file_data",
+    )
+    uploader.precheck()
+    uploader.run(path=upload_file, file_data=file_data)
+
+    # Verify that the file was uploaded
+    client = connection_config.get_client()
+    drive = client.users[user_pname].drive
+
+    uploaded_file = (
+        drive.root.get_by_path(destination_fullpath).select(["id", "name"]).get().execute_query()
+    )
+
+    # Check if the file exists
+    assert uploaded_file is not None
+    assert uploaded_file.name == upload_file.name