unstructured-ingest 0.7.2__py3-none-any.whl → 1.0.2__py3-none-any.whl

test/integration/connectors/test_confluence.py

@@ -1,111 +0,0 @@
-import os
-
-import pytest
-
-from test.integration.connectors.utils.constants import SOURCE_TAG, UNCATEGORIZED_TAG
-from test.integration.connectors.utils.validation.source import (
-    SourceValidationConfigs,
-    source_connector_validation,
-)
-from test.integration.utils import requires_env
-from unstructured_ingest.processes.connectors.confluence import (
-    CONNECTOR_TYPE,
-    ConfluenceAccessConfig,
-    ConfluenceConnectionConfig,
-    ConfluenceDownloader,
-    ConfluenceDownloaderConfig,
-    ConfluenceIndexer,
-    ConfluenceIndexerConfig,
-)
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
-@requires_env("CONFLUENCE_USER_EMAIL", "CONFLUENCE_API_TOKEN")
-async def test_confluence_source(temp_dir):
-    # Retrieve environment variables
-    confluence_url = "https://unstructured-ingest-test.atlassian.net"
-    user_email = os.environ["CONFLUENCE_USER_EMAIL"]
-    api_token = os.environ["CONFLUENCE_API_TOKEN"]
-    spaces = ["testteamsp", "MFS"]
-
-    # Create connection and indexer configurations
-    access_config = ConfluenceAccessConfig(api_token=api_token)
-    connection_config = ConfluenceConnectionConfig(
-        url=confluence_url,
-        username=user_email,
-        access_config=access_config,
-    )
-    index_config = ConfluenceIndexerConfig(
-        max_num_of_spaces=500,
-        max_num_of_docs_from_each_space=100,
-        spaces=spaces,
-    )
-
-    download_config = ConfluenceDownloaderConfig(download_dir=temp_dir)
-
-    # Instantiate indexer and downloader
-    indexer = ConfluenceIndexer(
-        connection_config=connection_config,
-        index_config=index_config,
-    )
-    downloader = ConfluenceDownloader(
-        connection_config=connection_config,
-        download_config=download_config,
-    )
-
-    # Run the source connector validation
-    await source_connector_validation(
-        indexer=indexer,
-        downloader=downloader,
-        configs=SourceValidationConfigs(
-            test_id="confluence",
-            expected_num_files=11,
-            validate_downloaded_files=True,
-        ),
-    )
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
-@requires_env("CONFLUENCE_USER_EMAIL", "CONFLUENCE_API_TOKEN")
-async def test_confluence_source_large(temp_dir):
-    # Retrieve environment variables
-    confluence_url = "https://unstructured-ingest-test.atlassian.net"
-    user_email = os.environ["CONFLUENCE_USER_EMAIL"]
-    api_token = os.environ["CONFLUENCE_API_TOKEN"]
-    spaces = ["testteamsp1"]
-
-    # Create connection and indexer configurations
-    access_config = ConfluenceAccessConfig(api_token=api_token)
-    connection_config = ConfluenceConnectionConfig(
-        url=confluence_url,
-        username=user_email,
-        access_config=access_config,
-    )
-    index_config = ConfluenceIndexerConfig(
-        max_num_of_spaces=10,
-        max_num_of_docs_from_each_space=250,
-        spaces=spaces,
-    )
-
-    download_config = ConfluenceDownloaderConfig(download_dir=temp_dir)
-
-    # Instantiate indexer and downloader
-    indexer = ConfluenceIndexer(
-        connection_config=connection_config,
-        index_config=index_config,
-    )
-    downloader = ConfluenceDownloader(
-        connection_config=connection_config,
-        download_config=download_config,
-    )
-
-    # Run the source connector validation
-    await source_connector_validation(
-        indexer=indexer,
-        downloader=downloader,
-        configs=SourceValidationConfigs(
-            test_id="confluence_large", expected_num_files=250, validate_file_data=False
-        ),
-    )

test/integration/connectors/test_delta_table.py

@@ -1,183 +0,0 @@
-import multiprocessing
-import os
-from pathlib import Path
-
-import pytest
-from deltalake import DeltaTable
-from fsspec import get_filesystem_class
-
-from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG
-from test.integration.utils import requires_env
-from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
-from unstructured_ingest.processes.connectors.delta_table import (
-    CONNECTOR_TYPE,
-    DeltaTableAccessConfig,
-    DeltaTableConnectionConfig,
-    DeltaTableUploader,
-    DeltaTableUploaderConfig,
-    DeltaTableUploadStager,
-    DeltaTableUploadStagerConfig,
-)
-
-multiprocessing.set_start_method("spawn")
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
-async def test_delta_table_destination_local(upload_file: Path, temp_dir: Path):
-    destination_path = str(temp_dir)
-    connection_config = DeltaTableConnectionConfig(
-        access_config=DeltaTableAccessConfig(),
-        table_uri=destination_path,
-    )
-    stager_config = DeltaTableUploadStagerConfig()
-    stager = DeltaTableUploadStager(upload_stager_config=stager_config)
-    new_upload_file = stager.run(
-        elements_filepath=upload_file,
-        output_dir=temp_dir,
-        output_filename=upload_file.name,
-    )
-
-    upload_config = DeltaTableUploaderConfig()
-    uploader = DeltaTableUploader(connection_config=connection_config, upload_config=upload_config)
-    file_data = FileData(
-        source_identifiers=SourceIdentifiers(
-            fullpath=upload_file.name, filename=new_upload_file.name
-        ),
-        connector_type=CONNECTOR_TYPE,
-        identifier="mock file data",
-    )
-
-    if uploader.is_async():
-        await uploader.run_async(path=new_upload_file, file_data=file_data)
-    else:
-        uploader.run(path=new_upload_file, file_data=file_data)
-    delta_table_path = os.path.join(destination_path, upload_file.name)
-    delta_table = DeltaTable(table_uri=delta_table_path)
-    df = delta_table.to_pandas()
-
-    EXPECTED_COLUMNS = 10
-    EXPECTED_ROWS = 22
-    assert (
-        len(df) == EXPECTED_ROWS
-    ), f"Number of rows in table vs expected: {len(df)}/{EXPECTED_ROWS}"
-    assert (
-        len(df.columns) == EXPECTED_COLUMNS
-    ), f"Number of columns in table vs expected: {len(df.columns)}/{EXPECTED_COLUMNS}"
-
-
-def get_aws_credentials() -> dict:
-    access_key = os.getenv("S3_INGEST_TEST_ACCESS_KEY", None)
-    assert access_key
-    secret_key = os.getenv("S3_INGEST_TEST_SECRET_KEY", None)
-    assert secret_key
-    return {
-        "AWS_ACCESS_KEY_ID": access_key,
-        "AWS_SECRET_ACCESS_KEY": secret_key,
-        "AWS_REGION": "us-east-2",
-    }
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
-@requires_env("S3_INGEST_TEST_ACCESS_KEY", "S3_INGEST_TEST_SECRET_KEY")
-async def test_delta_table_destination_s3(upload_file: Path, temp_dir: Path):
-    aws_credentials = get_aws_credentials()
-    s3_bucket = "s3://utic-platform-test-destination"
-    destination_path = f"{s3_bucket}/destination/test"
-    connection_config = DeltaTableConnectionConfig(
-        access_config=DeltaTableAccessConfig(
-            aws_access_key_id=aws_credentials["AWS_ACCESS_KEY_ID"],
-            aws_secret_access_key=aws_credentials["AWS_SECRET_ACCESS_KEY"],
-        ),
-        aws_region=aws_credentials["AWS_REGION"],
-        table_uri=destination_path,
-    )
-    stager_config = DeltaTableUploadStagerConfig()
-    stager = DeltaTableUploadStager(upload_stager_config=stager_config)
-    new_upload_file = stager.run(
-        elements_filepath=upload_file,
-        output_dir=temp_dir,
-        output_filename=upload_file.name,
-    )
-
-    upload_config = DeltaTableUploaderConfig()
-    uploader = DeltaTableUploader(connection_config=connection_config, upload_config=upload_config)
-    file_data = FileData(
-        source_identifiers=SourceIdentifiers(
-            fullpath=upload_file.name, filename=new_upload_file.name
-        ),
-        connector_type=CONNECTOR_TYPE,
-        identifier="mock file data",
-    )
-
-    try:
-        uploader.precheck()
-        if uploader.is_async():
-            await uploader.run_async(path=new_upload_file, file_data=file_data)
-        else:
-            uploader.run(path=new_upload_file, file_data=file_data)
-        delta_table_path = os.path.join(destination_path, upload_file.name)
-        delta_table = DeltaTable(table_uri=delta_table_path, storage_options=aws_credentials)
-        df = delta_table.to_pandas()
-
-        EXPECTED_COLUMNS = 10
-        EXPECTED_ROWS = 22
-        assert (
-            len(df) == EXPECTED_ROWS
-        ), f"Number of rows in table vs expected: {len(df)}/{EXPECTED_ROWS}"
-        assert (
-            len(df.columns) == EXPECTED_COLUMNS
-        ), f"Number of columns in table vs expected: {len(df.columns)}/{EXPECTED_COLUMNS}"
-    finally:
-        s3fs = get_filesystem_class("s3")(
-            key=aws_credentials["AWS_ACCESS_KEY_ID"],
-            secret=aws_credentials["AWS_SECRET_ACCESS_KEY"],
-        )
-        s3fs.rm(path=destination_path, recursive=True)
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
-@requires_env("S3_INGEST_TEST_ACCESS_KEY", "S3_INGEST_TEST_SECRET_KEY")
-async def test_delta_table_destination_s3_bad_creds(upload_file: Path, temp_dir: Path):
-    aws_credentials = {
-        "AWS_ACCESS_KEY_ID": "bad key",
-        "AWS_SECRET_ACCESS_KEY": "bad secret",
-        "AWS_REGION": "us-east-2",
-    }
-    s3_bucket = "s3://utic-platform-test-destination"
-    destination_path = f"{s3_bucket}/destination/test"
-    connection_config = DeltaTableConnectionConfig(
-        access_config=DeltaTableAccessConfig(
-            aws_access_key_id=aws_credentials["AWS_ACCESS_KEY_ID"],
-            aws_secret_access_key=aws_credentials["AWS_SECRET_ACCESS_KEY"],
-        ),
-        aws_region=aws_credentials["AWS_REGION"],
-        table_uri=destination_path,
-    )
-    stager_config = DeltaTableUploadStagerConfig()
-    stager = DeltaTableUploadStager(upload_stager_config=stager_config)
-    new_upload_file = stager.run(
-        elements_filepath=upload_file,
-        output_dir=temp_dir,
-        output_filename=upload_file.name,
-    )
-
-    upload_config = DeltaTableUploaderConfig()
-    uploader = DeltaTableUploader(connection_config=connection_config, upload_config=upload_config)
-    file_data = FileData(
-        source_identifiers=SourceIdentifiers(
-            fullpath=upload_file.name, filename=new_upload_file.name
-        ),
-        connector_type=CONNECTOR_TYPE,
-        identifier="mock file data",
-    )
-
-    with pytest.raises(Exception) as excinfo:
-        if uploader.is_async():
-            await uploader.run_async(path=new_upload_file, file_data=file_data)
-        else:
-            uploader.run(path=new_upload_file, file_data=file_data)
-
-    assert "403 Forbidden" in str(excinfo.value), f"Exception message did not match: {str(excinfo)}"

test/integration/connectors/test_dropbox.py

@@ -1,151 +0,0 @@
-import os
-
-import pytest
-import requests
-
-from test.integration.connectors.utils.constants import (
-    BLOB_STORAGE_TAG,
-    SOURCE_TAG,
-)
-from test.integration.connectors.utils.validation.source import (
-    SourceValidationConfigs,
-    source_connector_validation,
-)
-from test.integration.utils import requires_env
-from unstructured_ingest.processes.connectors.fsspec.dropbox import (
-    CONNECTOR_TYPE as DROPBOX_CONNECTOR_TYPE,
-)
-from unstructured_ingest.processes.connectors.fsspec.dropbox import (
-    DropboxAccessConfig,
-    DropboxConnectionConfig,
-    DropboxDownloader,
-    DropboxDownloaderConfig,
-    DropboxIndexer,
-    DropboxIndexerConfig,
-)
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(DROPBOX_CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
-@requires_env("DROPBOX_REFRESH_TOKEN", "DROPBOX_APP_KEY", "DROPBOX_APP_SECRET")
-async def test_dropbox_source(temp_dir):
-    """
-    Integration test for the Dropbox source connector.
-
-    This test indexes data from dropbox://test-input/ and downloads the resulting files,
-    then compares them to fixture data.
-    """
-    refresh_token = os.getenv("DROPBOX_REFRESH_TOKEN")
-    app_key = os.getenv("DROPBOX_APP_KEY")
-    app_secret = os.getenv("DROPBOX_APP_SECRET")
-
-    connection_config = DropboxConnectionConfig(
-        access_config=DropboxAccessConfig(
-            refresh_token=refresh_token,
-            app_key=app_key,
-            app_secret=app_secret,
-        )
-    )
-
-    index_config = DropboxIndexerConfig(
-        recursive=True,
-        remote_url="dropbox://test-input",
-    )
-    downloader_config = DropboxDownloaderConfig(download_dir=temp_dir)
-
-    indexer = DropboxIndexer(
-        connection_config=connection_config,
-        index_config=index_config,
-    )
-    downloader = DropboxDownloader(
-        connection_config=connection_config,
-        download_config=downloader_config,
-    )
-
-    await source_connector_validation(
-        indexer=indexer,
-        downloader=downloader,
-        configs=SourceValidationConfigs(
-            test_id="dropbox",
-            expected_num_files=4,
-            validate_downloaded_files=True,
-            exclude_fields_extend=[
-                "metadata.date_created",
-                "metadata.date_modified",
-            ],
-        ),
-    )
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(DROPBOX_CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
-@requires_env("DROPBOX_REFRESH_TOKEN", "DROPBOX_APP_KEY", "DROPBOX_APP_SECRET")
-async def test_dropbox_short_lived_token_via_refresh(temp_dir):
-    """
-    Demonstrates manually generating an access token from refresh credentials,
-    then passing ONLY the short-lived token to the Dropbox connector
-    (no app_key, app_secret, or refresh_token in the actual connection config).
-
-    This effectively mimics an external system that hands us a short-lived token.
-    """
-    refresh_token = os.getenv("DROPBOX_REFRESH_TOKEN")
-    app_key = os.getenv("DROPBOX_APP_KEY")
-    app_secret = os.getenv("DROPBOX_APP_SECRET")
-
-    #    Manually request a short-lived token from Dropbox's OAuth endpoint
-    #    This call is basically what the connector code does internally,
-    #    but we're doing it here in the test so we can pass only the short-lived token later.
-    response = requests.post(
-        "https://api.dropboxapi.com/oauth2/token",
-        data={
-            "grant_type": "refresh_token",
-            "refresh_token": refresh_token,
-        },
-        auth=(app_key, app_secret),
-        timeout=30,  # seconds
-    )
-    response.raise_for_status()
-    data = response.json()
-    short_lived_token = data["access_token"]
-    print("Acquired an access token from Dropbox")
-
-    #    Build connection config with ONLY the short-lived token
-    #    We omit refresh_token, app_key, and app_secret to confirm that
-    #    our connector can operate purely on the short-lived token.
-    connection_config = DropboxConnectionConfig(
-        access_config=DropboxAccessConfig(
-            token=short_lived_token,
-            app_key=None,
-            app_secret=None,
-            refresh_token=None,
-        )
-    )
-
-    index_config = DropboxIndexerConfig(
-        recursive=True,
-        remote_url="dropbox://test-input",
-    )
-    downloader_config = DropboxDownloaderConfig(download_dir=temp_dir)
-
-    indexer = DropboxIndexer(
-        connection_config=connection_config,
-        index_config=index_config,
-    )
-    downloader = DropboxDownloader(
-        connection_config=connection_config,
-        download_config=downloader_config,
-    )
-
-    await source_connector_validation(
-        indexer=indexer,
-        downloader=downloader,
-        configs=SourceValidationConfigs(
-            test_id="dropbox_short_lived_via_refresh",
-            expected_num_files=4,
-            validate_downloaded_files=True,
-            exclude_fields_extend=[
-                "metadata.date_created",
-                "metadata.date_modified",
-            ],
-        ),
-    )

test/integration/connectors/test_github.py

@@ -1,49 +0,0 @@
-import os
-
-import pytest
-
-from test.integration.connectors.utils.constants import SOURCE_TAG, UNCATEGORIZED_TAG
-from test.integration.connectors.utils.validation.source import (
-    SourceValidationConfigs,
-    source_connector_validation,
-)
-from test.integration.utils import requires_env
-from unstructured_ingest.processes.connectors.github import (
-    CONNECTOR_TYPE,
-    GithubAccessConfig,
-    GithubConnectionConfig,
-    GithubDownloader,
-    GithubDownloaderConfig,
-    GithubIndexer,
-    GithubIndexerConfig,
-)
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
-@pytest.mark.asyncio
-@requires_env("GH_READ_ONLY_ACCESS_TOKEN")
-async def test_github_source(temp_dir):
-    access_token = os.environ["GH_READ_ONLY_ACCESS_TOKEN"]
-    connection_config = GithubConnectionConfig(
-        access_config=GithubAccessConfig(access_token=access_token),
-        url="dcneiner/Downloadify",
-    )
-
-    indexer = GithubIndexer(
-        connection_config=connection_config,
-        index_config=GithubIndexerConfig(file_glob=["*.txt", "*.html"]),
-    )
-
-    downloader = GithubDownloader(
-        connection_config=connection_config,
-        download_config=GithubDownloaderConfig(download_dir=temp_dir),
-    )
-
-    # Run the source connector validation
-    await source_connector_validation(
-        indexer=indexer,
-        downloader=downloader,
-        configs=SourceValidationConfigs(
-            test_id="github", expected_num_files=2, validate_downloaded_files=True
-        ),
-    )