unstructured-ingest 0.7.2__py3-none-any.whl → 1.0.2__py3-none-any.whl

test/integration/connectors/test_google_drive.py

@@ -1,257 +0,0 @@
-import os
-import uuid
-
-import pytest
-from googleapiclient.errors import HttpError
-
-from test.integration.connectors.utils.constants import (
-    SOURCE_TAG,
-    UNCATEGORIZED_TAG,
-)
-from test.integration.connectors.utils.validation.source import (
-    SourceValidationConfigs,
-    get_all_file_data,
-    run_all_validations,
-    update_fixtures,
-)
-from test.integration.utils import requires_env
-from unstructured_ingest.error import (
-    SourceConnectionError,
-)
-from unstructured_ingest.interfaces import Downloader, Indexer
-from unstructured_ingest.processes.connectors.google_drive import (
-    CONNECTOR_TYPE,
-    GoogleDriveAccessConfig,
-    GoogleDriveConnectionConfig,
-    GoogleDriveDownloader,
-    GoogleDriveDownloaderConfig,
-    GoogleDriveIndexer,
-    GoogleDriveIndexerConfig,
-)
-
-
-@pytest.fixture
-def google_drive_connection_config():
-    """
-    Build a valid GoogleDriveConnectionConfig using the environment variables.
-    Expects:
-      - GOOGLE_DRIVE_ID
-      - GOOGLE_DRIVE_SERVICE_KEY
-    """
-    drive_id = os.getenv("GOOGLE_DRIVE_ID")
-    service_key = os.getenv("GOOGLE_DRIVE_SERVICE_KEY")
-    if not drive_id or not service_key:
-        pytest.skip("Google Drive credentials not provided in environment variables.")
-
-    access_config = GoogleDriveAccessConfig(service_account_key=service_key)
-    return GoogleDriveConnectionConfig(drive_id=drive_id, access_config=access_config)
-
-
-@pytest.fixture
-def google_drive_empty_folder(google_drive_connection_config):
-    """
-    Creates an empty folder on Google Drive for testing the "empty folder" case.
-    The folder is deleted after the test.
-    """
-    from google.oauth2 import service_account
-    from googleapiclient.discovery import build
-
-    access_config = google_drive_connection_config.access_config.get_secret_value()
-    creds = service_account.Credentials.from_service_account_info(access_config.service_account_key)
-    service = build("drive", "v3", credentials=creds)
-
-    # Create an empty folder.
-    file_metadata = {
-        "name": f"utic-empty-folder-{uuid.uuid4()}",
-        "mimeType": "application/vnd.google-apps.folder",
-    }
-    folder = service.files().create(body=file_metadata, fields="id, name").execute()
-    folder_id = folder.get("id")
-    try:
-        yield folder_id
-    finally:
-        service.files().delete(fileId=folder_id).execute()
-
-
-@requires_env("GOOGLE_DRIVE_SERVICE_KEY")
-@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE)
-def test_google_drive_source(temp_dir):
-    # Retrieve environment variables
-    service_account_key = os.environ["GOOGLE_DRIVE_SERVICE_KEY"]
-
-    # Create connection and indexer configurations
-    access_config = GoogleDriveAccessConfig(service_account_key=service_account_key)
-    connection_config = GoogleDriveConnectionConfig(
-        drive_id="1XidSOO76VpZ4m0i3gJN2m1X0Obol3UAi",
-        access_config=access_config,
-    )
-    index_config = GoogleDriveIndexerConfig(recursive=True)
-
-    download_config = GoogleDriveDownloaderConfig(download_dir=temp_dir)
-
-    # Instantiate indexer and downloader
-    indexer = GoogleDriveIndexer(
-        connection_config=connection_config,
-        index_config=index_config,
-    )
-    downloader = GoogleDriveDownloader(
-        connection_config=connection_config,
-        download_config=download_config,
-    )
-
-    # Run the source connector validation
-    source_connector_validation(
-        indexer=indexer,
-        downloader=downloader,
-        configs=SourceValidationConfigs(
-            test_id="google_drive_source",
-            expected_num_files=1,
-            validate_downloaded_files=True,
-        ),
-    )
-
-
-@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
-def source_connector_validation(
-    indexer: Indexer,
-    downloader: Downloader,
-    configs: SourceValidationConfigs,
-    overwrite_fixtures: bool = os.getenv("OVERWRITE_FIXTURES", "False").lower() == "true",
-) -> None:
-    # Run common validations on the process of running a source connector, supporting dynamic
-    # validators that get passed in along with comparisons on the saved expected values.
-    # If overwrite_fixtures is st to True, will ignore all validators but instead overwrite the
-    # expected values with what gets generated by this test.
-    all_predownload_file_data = []
-    all_postdownload_file_data = []
-    indexer.precheck()
-    download_dir = downloader.download_config.download_dir
-    test_output_dir = configs.test_output_dir()
-
-    for file_data in indexer.run():
-        assert file_data
-        predownload_file_data = file_data.model_copy(deep=True)
-        all_predownload_file_data.append(predownload_file_data)
-        resp = downloader.run(file_data=file_data)
-        if isinstance(resp, list):
-            for r in resp:
-                postdownload_file_data = r["file_data"].model_copy(deep=True)
-                all_postdownload_file_data.append(postdownload_file_data)
-        else:
-            postdownload_file_data = resp["file_data"].model_copy(deep=True)
-            all_postdownload_file_data.append(postdownload_file_data)
-
-    if not overwrite_fixtures:
-        print("Running validation")
-        run_all_validations(
-            configs=configs,
-            predownload_file_data=all_predownload_file_data,
-            postdownload_file_data=all_postdownload_file_data,
-            download_dir=download_dir,
-            test_output_dir=test_output_dir,
-        )
-    else:
-        print("Running fixtures update")
-        update_fixtures(
-            output_dir=test_output_dir,
-            download_dir=download_dir,
-            all_file_data=get_all_file_data(
-                all_predownload_file_data=all_predownload_file_data,
-                all_postdownload_file_data=all_postdownload_file_data,
-            ),
-            save_downloads=configs.validate_downloaded_files,
-            save_filedata=configs.validate_file_data,
-        )
-
-
-# Precheck fails when the drive ID has an appended parameter (simulate copy-paste error)
-@pytest.mark.tags("google-drive", "precheck")
-@requires_env("GOOGLE_DRIVE_ID", "GOOGLE_DRIVE_SERVICE_KEY")
-def test_google_drive_precheck_invalid_parameter(google_drive_connection_config):
-    # Append a query parameter as often happens when copying from a URL.
-    invalid_drive_id = google_drive_connection_config.drive_id + "?usp=sharing"
-    connection_config = GoogleDriveConnectionConfig(
-        drive_id=invalid_drive_id,
-        access_config=google_drive_connection_config.access_config,
-    )
-    index_config = GoogleDriveIndexerConfig(recursive=True)
-    indexer = GoogleDriveIndexer(connection_config=connection_config, index_config=index_config)
-    with pytest.raises(SourceConnectionError) as excinfo:
-        indexer.precheck()
-    assert "invalid" in str(excinfo.value).lower() or "not found" in str(excinfo.value).lower()
-
-
-# Precheck fails due to lack of permission (simulate via monkeypatching).
-@pytest.mark.tags("google-drive", "precheck")
-@requires_env("GOOGLE_DRIVE_ID", "GOOGLE_DRIVE_SERVICE_KEY")
-def test_google_drive_precheck_no_permission(google_drive_connection_config, monkeypatch):
-    index_config = GoogleDriveIndexerConfig(recursive=True)
-    indexer = GoogleDriveIndexer(
-        connection_config=google_drive_connection_config,
-        index_config=index_config,
-    )
-
-    # Monkeypatch get_root_info to always raise an HTTP 403 error.
-    def fake_get_root_info(files_client, object_id):
-        raise HttpError(
-            resp=type("Response", (), {"status": 403, "reason": "Forbidden"})(),
-            content=b"Forbidden",
-        )
-
-    monkeypatch.setattr(indexer, "get_root_info", fake_get_root_info)
-    with pytest.raises(SourceConnectionError) as excinfo:
-        indexer.precheck()
-    assert "forbidden" in str(excinfo.value).lower() or "permission" in str(excinfo.value).lower()
-
-
-# Precheck fails when the folder is empty.
-# @pytest.mark.tags("google-drive", "precheck")
-# @requires_env("GOOGLE_DRIVE_ID", "GOOGLE_DRIVE_SERVICE_KEY")
-# def test_google_drive_precheck_empty_folder(
-#     google_drive_connection_config, google_drive_empty_folder
-# ):
-#     # Use the empty folder's ID as the target.
-#     connection_config = GoogleDriveConnectionConfig(
-#         drive_id=google_drive_empty_folder,
-#         access_config=google_drive_connection_config.access_config,
-#     )
-
-#     index_config = GoogleDriveIndexerConfig(recursive=True)
-#     indexer = GoogleDriveIndexer(connection_config=connection_config, index_config=index_config)
-#     with pytest.raises(SourceConnectionError) as excinfo:
-#         indexer.precheck()
-#     assert "empty folder" in str(excinfo.value).lower()
-
-
-@pytest.mark.tags("google-drive", "count", "integration")
-@requires_env("GOOGLE_DRIVE_ID", "GOOGLE_DRIVE_SERVICE_KEY")
-def test_google_drive_count_files(google_drive_connection_config):
-    """
-    This test verifies that the count_files_recursively method returns the expected count of files.
-    According to the test credentials, there are 3 files in the root directory and 1 nested file,
-    so the total count should be 4.
-    """
-    # I assumed that we're applying the same extension filter as with other tests
-    # However there's 6 files in total in the test dir
-    extensions_filter = ["pdf", "docx"]
-    with google_drive_connection_config.get_client() as client:
-        count = GoogleDriveIndexer.count_files_recursively(
-            client, google_drive_connection_config.drive_id, extensions_filter
-        )
-    assert count == 4, f"Expected file count of 4, but got {count}"
-
-
-# Precheck fails with a completely invalid drive ID.
-@pytest.mark.tags("google-drive", "precheck")
-@requires_env("GOOGLE_DRIVE_ID", "GOOGLE_DRIVE_SERVICE_KEY")
-def test_google_drive_precheck_invalid_drive_id(google_drive_connection_config):
-    invalid_drive_id = "invalid_drive_id"
-    connection_config = GoogleDriveConnectionConfig(
-        drive_id=invalid_drive_id,
-        access_config=google_drive_connection_config.access_config,
-    )
-    index_config = GoogleDriveIndexerConfig(recursive=True)
-    indexer = GoogleDriveIndexer(connection_config=connection_config, index_config=index_config)
-    with pytest.raises(SourceConnectionError) as excinfo:
-        indexer.precheck()
-    assert "invalid" in str(excinfo.value).lower() or "not found" in str(excinfo.value).lower()

test/integration/connectors/test_jira.py

@@ -1,67 +0,0 @@
-import os
-
-import pytest
-
-from test.integration.connectors.utils.constants import SOURCE_TAG, UNCATEGORIZED_TAG
-from test.integration.connectors.utils.validation.source import (
-    SourceValidationConfigs,
-    source_connector_validation,
-)
-from test.integration.utils import requires_env
-from unstructured_ingest.processes.connectors.jira import (
-    CONNECTOR_TYPE,
-    JiraAccessConfig,
-    JiraConnectionConfig,
-    JiraDownloader,
-    JiraDownloaderConfig,
-    JiraIndexer,
-    JiraIndexerConfig,
-)
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
-@requires_env("JIRA_INGEST_USER_EMAIL", "JIRA_INGEST_API_TOKEN")
-async def test_jira_source(temp_dir):
-    # Retrieve environment variables
-    jira_url = os.environ.get(
-        "JIRA_INGEST_URL", "https://unstructured-jira-connector-test.atlassian.net"
-    )
-    user_email = os.environ["JIRA_INGEST_USER_EMAIL"]
-    api_token = os.environ["JIRA_INGEST_API_TOKEN"]
-    projects = ["JCTP1"]
-    boards = ["3"]
-    issues = ["JCTP2-1", "JCTP2-2", "JCTP2-3"]
-
-    # Create connection and indexer configurations
-    access_config = JiraAccessConfig(password=api_token)
-    connection_config = JiraConnectionConfig(
-        url=jira_url,
-        username=user_email,
-        access_config=access_config,
-    )
-    index_config = JiraIndexerConfig(projects=projects, boards=boards, issues=issues)
-
-    download_config = JiraDownloaderConfig(download_dir=temp_dir)
-
-    # Instantiate indexer and downloader
-    indexer = JiraIndexer(
-        connection_config=connection_config,
-        index_config=index_config,
-    )
-    downloader = JiraDownloader(
-        connection_config=connection_config,
-        download_config=download_config,
-    )
-
-    # Run the source connector validation
-    await source_connector_validation(
-        indexer=indexer,
-        downloader=downloader,
-        configs=SourceValidationConfigs(
-            test_id="jira",
-            expected_num_files=8,
-            validate_file_data=True,
-            validate_downloaded_files=True,
-        ),
-    )

test/integration/connectors/test_lancedb.py

@@ -1,247 +0,0 @@
-import os
-from pathlib import Path
-from typing import Literal, Union
-from uuid import uuid4
-
-import lancedb
-import pandas as pd
-import pyarrow as pa
-import pytest
-import pytest_asyncio
-from lancedb import AsyncConnection
-from upath import UPath
-
-from test.integration.connectors.utils.constants import DESTINATION_TAG, VECTOR_DB_TAG
-from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
-from unstructured_ingest.processes.connectors.lancedb.aws import (
-    LanceDBAwsAccessConfig,
-    LanceDBAwsConnectionConfig,
-    LanceDBAwsUploader,
-)
-from unstructured_ingest.processes.connectors.lancedb.azure import (
-    LanceDBAzureAccessConfig,
-    LanceDBAzureConnectionConfig,
-    LanceDBAzureUploader,
-)
-from unstructured_ingest.processes.connectors.lancedb.gcp import (
-    LanceDBGCSAccessConfig,
-    LanceDBGCSConnectionConfig,
-    LanceDBGSPUploader,
-)
-from unstructured_ingest.processes.connectors.lancedb.lancedb import (
-    CONNECTOR_TYPE,
-    LanceDBUploaderConfig,
-    LanceDBUploadStager,
-)
-from unstructured_ingest.processes.connectors.lancedb.local import (
-    LanceDBLocalAccessConfig,
-    LanceDBLocalConnectionConfig,
-    LanceDBLocalUploader,
-)
-from unstructured_ingest.utils.constants import RECORD_ID_LABEL
-
-DATABASE_NAME = "database"
-TABLE_NAME = "elements"
-DIMENSION = 384
-NUMBER_EXPECTED_ROWS = 22
-S3_BUCKET = "s3://utic-ingest-test-fixtures/"
-GS_BUCKET = "gs://utic-test-ingest-fixtures-output/"
-AZURE_BUCKET = "az://utic-ingest-test-fixtures-output/"
-REQUIRED_ENV_VARS = {
-    "s3": ("S3_INGEST_TEST_ACCESS_KEY", "S3_INGEST_TEST_SECRET_KEY"),
-    "gcs": ("GCP_INGEST_SERVICE_KEY",),
-    "az": ("AZURE_DEST_CONNECTION_STR",),
-    "local": (),
-}
-
-SCHEMA = pa.schema(
-    [
-        pa.field(RECORD_ID_LABEL, pa.string()),
-        pa.field("vector", pa.list_(pa.float16(), DIMENSION)),
-        pa.field("text", pa.string(), nullable=True),
-        pa.field("type", pa.string(), nullable=True),
-        pa.field("element_id", pa.string(), nullable=True),
-        pa.field("metadata-text_as_html", pa.string(), nullable=True),
-        pa.field("metadata-filetype", pa.string(), nullable=True),
-        pa.field("metadata-filename", pa.string(), nullable=True),
-        pa.field("metadata-languages", pa.list_(pa.string()), nullable=True),
-        pa.field("metadata-is_continuation", pa.bool_(), nullable=True),
-        pa.field("metadata-page_number", pa.int32(), nullable=True),
-    ]
-)
-NUMBER_EXPECTED_COLUMNS = len(SCHEMA.names)
-
-
-@pytest_asyncio.fixture
-async def connection_with_uri(request, tmp_path: Path):
-    target = request.param
-    uri = _get_uri(target, local_base_path=tmp_path)
-
-    unset_variables = [env for env in REQUIRED_ENV_VARS[target] if env not in os.environ]
-    if unset_variables:
-        pytest.skip(
-            reason="Following required environment variables were not set: "
-            + f"{', '.join(unset_variables)}"
-        )
-
-    storage_options = {
-        "aws_access_key_id": os.getenv("S3_INGEST_TEST_ACCESS_KEY"),
-        "aws_secret_access_key": os.getenv("S3_INGEST_TEST_SECRET_KEY"),
-        "google_service_account_key": os.getenv("GCP_INGEST_SERVICE_KEY"),
-    }
-    azure_connection_string = os.getenv("AZURE_DEST_CONNECTION_STR")
-    if azure_connection_string:
-        storage_options.update(_parse_azure_connection_string(azure_connection_string))
-
-    storage_options = {key: value for key, value in storage_options.items() if value is not None}
-    connection = await lancedb.connect_async(
-        uri=uri,
-        storage_options=storage_options,
-    )
-    await connection.create_table(name=TABLE_NAME, schema=SCHEMA)
-
-    yield connection, uri
-
-    await connection.drop_database()
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
-@pytest.mark.parametrize("connection_with_uri", ["local", "s3", "gcs", "az"], indirect=True)
-async def test_lancedb_destination(
-    upload_file: Path,
-    connection_with_uri: tuple[AsyncConnection, str],
-    tmp_path: Path,
-) -> None:
-    connection, uri = connection_with_uri
-    file_data = FileData(
-        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
-        connector_type=CONNECTOR_TYPE,
-        identifier="mock-file-data",
-    )
-    stager = LanceDBUploadStager()
-    uploader = _get_uploader(uri)
-    staged_file_path = stager.run(
-        elements_filepath=upload_file,
-        file_data=file_data,
-        output_dir=tmp_path,
-        output_filename=upload_file.name,
-    )
-
-    await uploader.run_async(path=staged_file_path, file_data=file_data)
-
-    # Test upload to empty table
-    with await connection.open_table(TABLE_NAME) as table:
-        table_df: pd.DataFrame = await table.to_pandas()
-
-    assert len(table_df) == NUMBER_EXPECTED_ROWS
-    assert len(table_df.columns) == NUMBER_EXPECTED_COLUMNS
-
-    assert table_df[RECORD_ID_LABEL][0] == file_data.identifier
-    assert table_df["element_id"][0] == "2470d8dc42215b3d68413b55bf00fed2"
-    assert table_df["type"][0] == "CompositeElement"
-    assert table_df["metadata-filename"][0] == "DA-1p-with-duplicate-pages.pdf.json"
-    assert table_df["metadata-text_as_html"][0] is None
-
-    # Test upload of the second file, rows should be appended
-    file_data.identifier = "mock-file-data-2"
-    staged_second_file_path = stager.run(
-        elements_filepath=upload_file,
-        file_data=file_data,
-        output_dir=tmp_path,
-        output_filename=f"{upload_file.stem}-2{upload_file.suffix}",
-    )
-    await uploader.run_async(path=staged_second_file_path, file_data=file_data)
-    with await connection.open_table(TABLE_NAME) as table:
-        appended_table_df: pd.DataFrame = await table.to_pandas()
-    assert len(appended_table_df) == 2 * NUMBER_EXPECTED_ROWS
-
-    # Test re-upload of the first file, rows should be overwritten, not appended
-    await uploader.run_async(path=staged_file_path, file_data=file_data)
-    with await connection.open_table(TABLE_NAME) as table:
-        overwritten_table_df: pd.DataFrame = await table.to_pandas()
-    assert len(overwritten_table_df) == 2 * NUMBER_EXPECTED_ROWS
-
-
-class TestPrecheck:
-    @pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
-    @pytest.mark.parametrize("connection_with_uri", ["local", "s3", "gcs", "az"], indirect=True)
-    def test_succeeds(
-        self,
-        upload_file: Path,
-        connection_with_uri: tuple[AsyncConnection, str],
-        tmp_path: Path,
-    ) -> None:
-        _, uri = connection_with_uri
-        uploader = _get_uploader(uri)
-        uploader.precheck()
-
-
-def _get_uri(target: Literal["local", "s3", "gcs", "az"], local_base_path: Path) -> str:
-    if target == "local":
-        return str(local_base_path / DATABASE_NAME)
-    if target == "s3":
-        base_uri = UPath(S3_BUCKET)
-    elif target == "gcs":
-        base_uri = UPath(GS_BUCKET)
-    elif target == "az":
-        base_uri = UPath(AZURE_BUCKET)
-
-    return str(base_uri / "destination" / "lancedb" / str(uuid4()) / DATABASE_NAME)
-
-
-def _get_uploader(
-    uri: str,
-) -> Union[LanceDBAzureUploader, LanceDBAzureUploader, LanceDBAwsUploader, LanceDBGSPUploader]:
-    target = uri.split("://", maxsplit=1)[0] if uri.startswith(("s3", "az", "gs")) else "local"
-    upload_config = LanceDBUploaderConfig(table_name=TABLE_NAME)
-    if target == "az":
-        azure_connection_string = os.getenv("AZURE_DEST_CONNECTION_STR")
-        access_config_kwargs = _parse_azure_connection_string(azure_connection_string)
-        return LanceDBAzureUploader(
-            upload_config=upload_config,
-            connection_config=LanceDBAzureConnectionConfig(
-                access_config=LanceDBAzureAccessConfig(**access_config_kwargs),
-                uri=uri,
-            ),
-        )
-
-    elif target == "s3":
-        return LanceDBAwsUploader(
-            upload_config=upload_config,
-            connection_config=LanceDBAwsConnectionConfig(
-                access_config=LanceDBAwsAccessConfig(
-                    aws_access_key_id=os.getenv("S3_INGEST_TEST_ACCESS_KEY"),
-                    aws_secret_access_key=os.getenv("S3_INGEST_TEST_SECRET_KEY"),
-                ),
-                uri=uri,
-            ),
-        )
-    elif target == "gs":
-        return LanceDBGSPUploader(
-            upload_config=upload_config,
-            connection_config=LanceDBGCSConnectionConfig(
-                access_config=LanceDBGCSAccessConfig(
-                    google_service_account_key=os.getenv("GCP_INGEST_SERVICE_KEY")
-                ),
-                uri=uri,
-            ),
-        )
-    else:
-        return LanceDBLocalUploader(
-            upload_config=upload_config,
-            connection_config=LanceDBLocalConnectionConfig(
-                access_config=LanceDBLocalAccessConfig(),
-                uri=uri,
-            ),
-        )
-
-
-def _parse_azure_connection_string(
-    connection_str: str,
-) -> dict[Literal["azure_storage_account_name", "azure_storage_account_key"], str]:
-    parameters = dict(keyvalue.split("=", maxsplit=1) for keyvalue in connection_str.split(";"))
-    return {
-        "azure_storage_account_name": parameters.get("AccountName"),
-        "azure_storage_account_key": parameters.get("AccountKey"),
-    }