unstructured-ingest 0.7.2__py3-none-any.whl → 1.0.2__py3-none-any.whl

test/integration/connectors/test_redis.py

@@ -1,143 +0,0 @@
-import asyncio
-import json
-import os
-from pathlib import Path
-from typing import Optional
-
-import numpy as np
-import pytest
-from redis import exceptions as redis_exceptions
-from redis.asyncio import Redis, from_url
-
-from test.integration.connectors.utils.constants import DESTINATION_TAG, NOSQL_TAG
-from test.integration.utils import requires_env
-from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
-from unstructured_ingest.processes.connectors.redisdb import (
-    CONNECTOR_TYPE as REDIS_CONNECTOR_TYPE,
-)
-from unstructured_ingest.processes.connectors.redisdb import (
-    RedisAccessConfig,
-    RedisConnectionConfig,
-    RedisUploader,
-    RedisUploaderConfig,
-)
-
-
-async def delete_record(client: Redis, element_id: str, key_prefix: str) -> None:
-    key_with_prefix = f"{key_prefix}{element_id}"
-    await client.delete(key_with_prefix)
-
-
-async def validate_upload(client: Redis, first_element: dict, key_prefix: str) -> None:
-    element_id = first_element["element_id"]
-    key_with_prefix = f"{key_prefix}{element_id}"
-    expected_text = first_element["text"]
-    expected_embeddings = first_element["embeddings"]
-    async with client.pipeline(transaction=True) as pipe:
-        try:
-            response = await pipe.json().get(key_with_prefix, "$").execute()
-            response = response[0][0]
-        except redis_exceptions.ResponseError:
-            response = await pipe.get(key_with_prefix).execute()
-            response = json.loads(response[0])
-
-    embedding_similarity = np.linalg.norm(
-        np.array(response["embeddings"]) - np.array(expected_embeddings)
-    )
-
-    assert response is not None
-    assert response["element_id"] == element_id
-    assert response["text"] == expected_text
-    assert embedding_similarity < 1e-10
-
-
-async def redis_destination_test(
-    upload_file: Path,
-    tmp_path: Path,
-    connection_kwargs: dict,
-    uploader_config: dict,
-    uri: Optional[str] = None,
-    password: Optional[str] = None,
-):
-    uploader = RedisUploader(
-        connection_config=RedisConnectionConfig(
-            **connection_kwargs, access_config=RedisAccessConfig(uri=uri, password=password)
-        ),
-        upload_config=RedisUploaderConfig(batch_size=10, **uploader_config),
-    )
-    key_prefix = uploader.upload_config.key_prefix
-
-    file_data = FileData(
-        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
-        connector_type=REDIS_CONNECTOR_TYPE,
-        identifier="mock-file-data",
-    )
-    with upload_file.open() as upload_fp:
-        elements = json.load(upload_fp)
-    first_element = elements[0]
-
-    try:
-        if uploader.is_async():
-            await uploader.run_data_async(data=elements, file_data=file_data)
-
-        if uri:
-            async with from_url(uri) as client:
-                await validate_upload(
-                    client=client,
-                    first_element=first_element,
-                    key_prefix=key_prefix,
-                )
-        else:
-            async with Redis(**connection_kwargs, password=password) as client:
-                await validate_upload(
-                    client=client,
-                    first_element=first_element,
-                    key_prefix=key_prefix,
-                )
-    except Exception as e:
-        raise e
-    finally:
-        if uri:
-            async with from_url(uri) as client:
-                tasks = [
-                    delete_record(client, element["element_id"], key_prefix) for element in elements
-                ]
-                await asyncio.gather(*tasks)
-        else:
-            async with Redis(**connection_kwargs, password=password) as client:
-                tasks = [
-                    delete_record(client, element["element_id"], key_prefix) for element in elements
-                ]
-                await asyncio.gather(*tasks)
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(REDIS_CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
-@requires_env("AZURE_REDIS_INGEST_TEST_PASSWORD")
-async def test_redis_destination_azure_with_password(upload_file: Path, tmp_path: Path):
-    connection_kwargs = {
-        "host": "utic-dashboard-dev.redis.cache.windows.net",
-        "port": 6380,
-        "db": 0,
-        "ssl": True,
-    }
-    uploader_config = {
-        "key_prefix": "test_ingest:",
-    }
-    redis_pw = os.environ["AZURE_REDIS_INGEST_TEST_PASSWORD"]
-    await redis_destination_test(
-        upload_file, tmp_path, connection_kwargs, uploader_config, password=redis_pw
-    )
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(REDIS_CONNECTOR_TYPE, DESTINATION_TAG, "redis", NOSQL_TAG)
-@requires_env("AZURE_REDIS_INGEST_TEST_PASSWORD")
-async def test_redis_destination_azure_with_uri(upload_file: Path, tmp_path: Path):
-    connection_kwargs = {}
-    uploader_config = {
-        "key_prefix": "test_ingest:",
-    }
-    redis_pw = os.environ["AZURE_REDIS_INGEST_TEST_PASSWORD"]
-    uri = f"rediss://:{redis_pw}@utic-dashboard-dev.redis.cache.windows.net:6380/0"
-    await redis_destination_test(upload_file, tmp_path, connection_kwargs, uploader_config, uri=uri)

test/integration/connectors/test_s3.py

@@ -1,184 +0,0 @@
-import os
-import tempfile
-import uuid
-from pathlib import Path
-
-import pytest
-
-from test.integration.connectors.utils.constants import (
-    BLOB_STORAGE_TAG,
-    DESTINATION_TAG,
-    SOURCE_TAG,
-    env_setup_path,
-)
-from test.integration.connectors.utils.docker_compose import docker_compose_context
-from test.integration.connectors.utils.validation.source import (
-    SourceValidationConfigs,
-    source_connector_validation,
-)
-from test.integration.utils import requires_env
-from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
-from unstructured_ingest.errors_v2 import UserAuthError, UserError
-from unstructured_ingest.processes.connectors.fsspec.s3 import (
-    CONNECTOR_TYPE,
-    S3AccessConfig,
-    S3ConnectionConfig,
-    S3Downloader,
-    S3DownloaderConfig,
-    S3Indexer,
-    S3IndexerConfig,
-    S3Uploader,
-    S3UploaderConfig,
-)
-
-
-def validate_predownload_file_data(file_data: FileData):
-    assert file_data.connector_type == CONNECTOR_TYPE
-    assert file_data.local_download_path is None
-
-
-def validate_postdownload_file_data(file_data: FileData):
-    assert file_data.connector_type == CONNECTOR_TYPE
-    assert file_data.local_download_path is not None
-
-
-@pytest.fixture
-def anon_connection_config() -> S3ConnectionConfig:
-    return S3ConnectionConfig(access_config=S3AccessConfig(), anonymous=True)
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
-async def test_s3_source(anon_connection_config: S3ConnectionConfig):
-    indexer_config = S3IndexerConfig(remote_url="s3://utic-dev-tech-fixtures/small-pdf-set/")
-    with tempfile.TemporaryDirectory() as tempdir:
-        tempdir_path = Path(tempdir)
-        download_config = S3DownloaderConfig(download_dir=tempdir_path)
-        indexer = S3Indexer(connection_config=anon_connection_config, index_config=indexer_config)
-        downloader = S3Downloader(
-            connection_config=anon_connection_config, download_config=download_config
-        )
-        await source_connector_validation(
-            indexer=indexer,
-            downloader=downloader,
-            configs=SourceValidationConfigs(
-                test_id="s3",
-                predownload_file_data_check=validate_predownload_file_data,
-                postdownload_file_data_check=validate_postdownload_file_data,
-                expected_num_files=4,
-            ),
-        )
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
-async def test_s3_source_special_char(anon_connection_config: S3ConnectionConfig):
-    indexer_config = S3IndexerConfig(remote_url="s3://utic-dev-tech-fixtures/special-characters/")
-    with tempfile.TemporaryDirectory() as tempdir:
-        tempdir_path = Path(tempdir)
-        download_config = S3DownloaderConfig(download_dir=tempdir_path)
-        indexer = S3Indexer(connection_config=anon_connection_config, index_config=indexer_config)
-        downloader = S3Downloader(
-            connection_config=anon_connection_config, download_config=download_config
-        )
-        await source_connector_validation(
-            indexer=indexer,
-            downloader=downloader,
-            configs=SourceValidationConfigs(
-                test_id="s3-specialchar",
-                predownload_file_data_check=validate_predownload_file_data,
-                postdownload_file_data_check=validate_postdownload_file_data,
-                expected_num_files=1,
-            ),
-        )
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
-def test_s3_source_no_access(anon_connection_config: S3ConnectionConfig):
-    indexer_config = S3IndexerConfig(remote_url="s3://utic-ingest-test-fixtures/destination/")
-    indexer = S3Indexer(connection_config=anon_connection_config, index_config=indexer_config)
-    with pytest.raises(UserAuthError):
-        indexer.precheck()
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
-def test_s3_source_no_bucket(anon_connection_config: S3ConnectionConfig):
-    indexer_config = S3IndexerConfig(remote_url="s3://fake-bucket")
-    indexer = S3Indexer(connection_config=anon_connection_config, index_config=indexer_config)
-    with pytest.raises(UserError):
-        indexer.precheck()
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, "minio", BLOB_STORAGE_TAG)
-async def test_s3_minio_source(anon_connection_config: S3ConnectionConfig):
-    anon_connection_config.endpoint_url = "http://localhost:9000"
-    indexer_config = S3IndexerConfig(remote_url="s3://utic-dev-tech-fixtures/")
-    with docker_compose_context(docker_compose_path=env_setup_path / "minio" / "source"):
-        with tempfile.TemporaryDirectory() as tempdir:
-            tempdir_path = Path(tempdir)
-            download_config = S3DownloaderConfig(download_dir=tempdir_path)
-            indexer = S3Indexer(
-                connection_config=anon_connection_config, index_config=indexer_config
-            )
-            downloader = S3Downloader(
-                connection_config=anon_connection_config, download_config=download_config
-            )
-            await source_connector_validation(
-                indexer=indexer,
-                downloader=downloader,
-                configs=SourceValidationConfigs(
-                    test_id="s3-minio",
-                    predownload_file_data_check=validate_predownload_file_data,
-                    postdownload_file_data_check=validate_postdownload_file_data,
-                    expected_num_files=1,
-                    exclude_fields_extend=[
-                        "metadata.date_modified",
-                        "metadata.date_created",
-                        "additional_metadata.LastModified",
-                    ],
-                ),
-            )
-
-
-def get_aws_credentials() -> dict:
-    access_key = os.getenv("S3_INGEST_TEST_ACCESS_KEY", None)
-    assert access_key
-    secret_key = os.getenv("S3_INGEST_TEST_SECRET_KEY", None)
-    assert secret_key
-    return {"aws_access_key_id": access_key, "aws_secret_access_key": secret_key}
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, BLOB_STORAGE_TAG)
-@requires_env("S3_INGEST_TEST_ACCESS_KEY", "S3_INGEST_TEST_SECRET_KEY")
-async def test_s3_destination(upload_file: Path):
-    aws_credentials = get_aws_credentials()
-    s3_bucket = "s3://utic-ingest-test-fixtures"
-    destination_path = f"{s3_bucket}/destination/{uuid.uuid4()}"
-    connection_config = S3ConnectionConfig(
-        access_config=S3AccessConfig(
-            key=aws_credentials["aws_access_key_id"],
-            secret=aws_credentials["aws_secret_access_key"],
-        ),
-    )
-    upload_config = S3UploaderConfig(remote_url=destination_path)
-    uploader = S3Uploader(connection_config=connection_config, upload_config=upload_config)
-    s3fs = uploader.fs
-    file_data = FileData(
-        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
-        connector_type=CONNECTOR_TYPE,
-        identifier="mock file data",
-    )
-    try:
-        uploader.precheck()
-        if uploader.is_async():
-            await uploader.run_async(path=upload_file, file_data=file_data)
-        else:
-            uploader.run(path=upload_file, file_data=file_data)
-        uploaded_files = [
-            Path(file) for file in s3fs.ls(path=destination_path) if Path(file).name != "_empty"
-        ]
-        assert len(uploaded_files) == 1
-    finally:
-        s3fs.rm(path=destination_path, recursive=True)

test/integration/connectors/test_sharepoint.py

@@ -1,222 +0,0 @@
-import os
-
-import pytest
-
-from test.integration.connectors.utils.constants import BLOB_STORAGE_TAG, SOURCE_TAG
-from test.integration.connectors.utils.validation.source import (
-    SourceValidationConfigs,
-    source_connector_validation,
-)
-from test.integration.utils import requires_env
-from unstructured_ingest.processes.connectors.sharepoint import (
-    CONNECTOR_TYPE,
-    SharepointAccessConfig,
-    SharepointConnectionConfig,
-    SharepointDownloader,
-    SharepointDownloaderConfig,
-    SharepointIndexer,
-    SharepointIndexerConfig,
-)
-
-
-def sharepoint_config():
-    class SharepointTestConfig:
-        def __init__(self):
-            self.client_id = os.environ["SHAREPOINT_CLIENT_ID"]
-            self.client_cred = os.environ["SHAREPOINT_CRED"]
-            self.user_pname = os.environ["MS_USER_PNAME"]
-            self.tenant = os.environ["MS_TENANT_ID"]
-
-    return SharepointTestConfig()
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
-@requires_env("SHAREPOINT_CLIENT_ID", "SHAREPOINT_CRED", "MS_TENANT_ID", "MS_USER_PNAME")
-async def test_sharepoint_source(temp_dir):
-    site = "https://unstructuredio.sharepoint.com/sites/utic-platform-test-source"
-    config = sharepoint_config()
-
-    # Create connection and indexer configurations
-    access_config = SharepointAccessConfig(client_cred=config.client_cred)
-    connection_config = SharepointConnectionConfig(
-        client_id=config.client_id,
-        site=site,
-        tenant=config.tenant,
-        user_pname=config.user_pname,
-        access_config=access_config,
-    )
-    index_config = SharepointIndexerConfig(recursive=True)
-
-    download_config = SharepointDownloaderConfig(download_dir=temp_dir)
-
-    # Instantiate indexer and downloader
-    indexer = SharepointIndexer(
-        connection_config=connection_config,
-        index_config=index_config,
-    )
-    downloader = SharepointDownloader(
-        connection_config=connection_config,
-        download_config=download_config,
-    )
-
-    # Run the source connector validation
-    await source_connector_validation(
-        indexer=indexer,
-        downloader=downloader,
-        configs=SourceValidationConfigs(
-            test_id="sharepoint1",
-            expected_num_files=4,
-            validate_downloaded_files=True,
-            exclude_fields_extend=[
-                "metadata.date_created",
-                "metadata.date_modified",
-                "additional_metadata.LastModified",
-                "additional_metadata.@microsoft.graph.downloadUrl",
-            ],
-        ),
-    )
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
-@requires_env("SHAREPOINT_CLIENT_ID", "SHAREPOINT_CRED", "MS_TENANT_ID", "MS_USER_PNAME")
-async def test_sharepoint_source_with_path(temp_dir):
-    site = "https://unstructuredio.sharepoint.com/sites/utic-platform-test-source"
-    config = sharepoint_config()
-
-    # Create connection and indexer configurations
-    access_config = SharepointAccessConfig(client_cred=config.client_cred)
-    connection_config = SharepointConnectionConfig(
-        client_id=config.client_id,
-        site=site,
-        tenant=config.tenant,
-        user_pname=config.user_pname,
-        access_config=access_config,
-    )
-    index_config = SharepointIndexerConfig(recursive=True, path="Folder1")
-
-    download_config = SharepointDownloaderConfig(download_dir=temp_dir)
-
-    # Instantiate indexer and downloader
-    indexer = SharepointIndexer(
-        connection_config=connection_config,
-        index_config=index_config,
-    )
-    downloader = SharepointDownloader(
-        connection_config=connection_config,
-        download_config=download_config,
-    )
-
-    # Run the source connector validation
-    await source_connector_validation(
-        indexer=indexer,
-        downloader=downloader,
-        configs=SourceValidationConfigs(
-            test_id="sharepoint2",
-            expected_num_files=2,
-            validate_downloaded_files=True,
-            exclude_fields_extend=[
-                "metadata.date_created",
-                "metadata.date_modified",
-                "additional_metadata.LastModified",
-                "additional_metadata.@microsoft.graph.downloadUrl",
-            ],
-        ),
-    )
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
-@requires_env("SHAREPOINT_CLIENT_ID", "SHAREPOINT_CRED", "MS_TENANT_ID", "MS_USER_PNAME")
-async def test_sharepoint_root_with_path(temp_dir):
-    site = "https://unstructuredio.sharepoint.com/"
-    config = sharepoint_config()
-
-    # Create connection and indexer configurations
-    access_config = SharepointAccessConfig(client_cred=config.client_cred)
-    connection_config = SharepointConnectionConfig(
-        client_id=config.client_id,
-        site=site,
-        tenant=config.tenant,
-        user_pname=config.user_pname,
-        access_config=access_config,
-    )
-    index_config = SharepointIndexerConfig(recursive=True, path="e2e-test-folder")
-
-    download_config = SharepointDownloaderConfig(download_dir=temp_dir)
-
-    # Instantiate indexer and downloader
-    indexer = SharepointIndexer(
-        connection_config=connection_config,
-        index_config=index_config,
-    )
-    downloader = SharepointDownloader(
-        connection_config=connection_config,
-        download_config=download_config,
-    )
-
-    # Run the source connector validation
-    await source_connector_validation(
-        indexer=indexer,
-        downloader=downloader,
-        configs=SourceValidationConfigs(
-            test_id="sharepoint3",
-            expected_num_files=1,
-            validate_downloaded_files=True,
-            exclude_fields_extend=[
-                "metadata.date_created",
-                "metadata.date_modified",
-                "additional_metadata.LastModified",
-                "additional_metadata.@microsoft.graph.downloadUrl",
-            ],
-        ),
-    )
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
-@requires_env("SHAREPOINT_CLIENT_ID", "SHAREPOINT_CRED", "MS_TENANT_ID", "MS_USER_PNAME")
-async def test_sharepoint_shared_documents(temp_dir):
-    site = "https://unstructuredio.sharepoint.com/sites/utic-platform-test-source"
-    config = sharepoint_config()
-
-    # Create connection and indexer configurations
-    access_config = SharepointAccessConfig(client_cred=config.client_cred)
-    connection_config = SharepointConnectionConfig(
-        client_id=config.client_id,
-        site=site,
-        tenant=config.tenant,
-        user_pname=config.user_pname,
-        access_config=access_config,
-    )
-    index_config = SharepointIndexerConfig(recursive=True, path="Shared Documents")
-
-    download_config = SharepointDownloaderConfig(download_dir=temp_dir)
-
-    # Instantiate indexer and downloader
-    indexer = SharepointIndexer(
-        connection_config=connection_config,
-        index_config=index_config,
-    )
-    downloader = SharepointDownloader(
-        connection_config=connection_config,
-        download_config=download_config,
-    )
-
-    # Run the source connector validation
-    await source_connector_validation(
-        indexer=indexer,
-        downloader=downloader,
-        configs=SourceValidationConfigs(
-            test_id="sharepoint4",
-            expected_num_files=4,
-            validate_downloaded_files=True,
-            exclude_fields_extend=[
-                "metadata.date_created",
-                "metadata.date_modified",
-                "additional_metadata.LastModified",
-                "additional_metadata.@microsoft.graph.downloadUrl",
-            ],
-        ),
-    )