unstructured-ingest 0.7.2__py3-none-any.whl → 1.0.1__py3-none-any.whl

test/integration/connectors/duckdb/conftest.py

@@ -1,14 +0,0 @@
-from pathlib import Path
-
-import pytest
-
-int_test_dir = Path(__file__).parent
-assets_dir = int_test_dir / "assets"
-
-
-@pytest.fixture
-def duckdb_schema() -> Path:
-    schema_file = assets_dir / "duckdb-schema.sql"
-    assert schema_file.exists()
-    assert schema_file.is_file()
-    return schema_file

test/integration/connectors/duckdb/test_duckdb.py

@@ -1,90 +0,0 @@
-import json
-from pathlib import Path
-
-import duckdb
-import pytest
-from _pytest.fixtures import TopRequest
-
-from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG
-from test.integration.connectors.utils.validation.destination import (
-    StagerValidationConfigs,
-    stager_validation,
-)
-from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
-from unstructured_ingest.processes.connectors.duckdb.duckdb import (
-    CONNECTOR_TYPE,
-    DuckDBConnectionConfig,
-    DuckDBUploader,
-    DuckDBUploaderConfig,
-    DuckDBUploadStager,
-)
-
-
-@pytest.fixture
-def provisioned_db_file(duckdb_schema: Path, temp_dir: Path) -> Path:
-    db_path = Path(temp_dir) / "temp_duck.db"
-    with duckdb.connect(database=db_path) as duckdb_connection:
-        with duckdb_schema.open("r") as f:
-            query = f.read()
-        duckdb_connection.execute(query)
-        duckdb_connection.close()
-    return db_path
-
-
-def validate_duckdb_destination(db_path: Path, expected_num_elements: int):
-    conn = None
-    try:
-        conn = duckdb.connect(db_path)
-        _results = conn.sql("select count(*) from elements").fetchall()
-        _count = _results[0][0]
-        assert (
-            _count == expected_num_elements
-        ), f"dest check failed: got {_count}, expected {expected_num_elements}"
-        conn.close()
-    finally:
-        if conn:
-            conn.close()
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "duckdb", SQL_TAG)
-def test_duckdb_destination(upload_file: Path, provisioned_db_file: Path, temp_dir: Path):
-    file_data = FileData(
-        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
-        connector_type=CONNECTOR_TYPE,
-        identifier="mock-file-data",
-    )
-
-    stager = DuckDBUploadStager()
-    staged_path = stager.run(
-        elements_filepath=upload_file,
-        file_data=file_data,
-        output_dir=temp_dir,
-        output_filename=upload_file.name,
-    )
-
-    connection_config = DuckDBConnectionConfig(database=str(provisioned_db_file))
-    upload_config = DuckDBUploaderConfig()
-    uploader = DuckDBUploader(connection_config=connection_config, upload_config=upload_config)
-
-    uploader.run(path=staged_path, file_data=file_data)
-
-    with staged_path.open() as f:
-        data = json.load(f)
-    validate_duckdb_destination(db_path=provisioned_db_file, expected_num_elements=len(data))
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "duckdb", SQL_TAG)
-@pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
-def test_duckdb_stager(
-    request: TopRequest,
-    upload_file_str: str,
-    tmp_path: Path,
-):
-    upload_file: Path = request.getfixturevalue(upload_file_str)
-    stager = DuckDBUploadStager()
-    stager_validation(
-        configs=StagerValidationConfigs(test_id=CONNECTOR_TYPE, expected_count=22),
-        input_file=upload_file,
-        stager=stager,
-        tmp_dir=tmp_path,
-    )

test/integration/connectors/duckdb/test_motherduck.py

@@ -1,95 +0,0 @@
-import os
-import uuid
-from pathlib import Path
-from typing import Generator
-
-import duckdb
-import pandas as pd
-import pytest
-
-from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG
-from test.integration.utils import requires_env
-from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
-from unstructured_ingest.processes.connectors.duckdb.motherduck import (
-    CONNECTOR_TYPE,
-    MotherDuckAccessConfig,
-    MotherDuckConnectionConfig,
-    MotherDuckUploader,
-    MotherDuckUploaderConfig,
-    MotherDuckUploadStager,
-)
-
-
-@pytest.fixture
-def md_token() -> str:
-    motherduck_token = os.getenv("MOTHERDUCK_TOKEN", None)
-    assert motherduck_token
-    return motherduck_token
-
-
-@pytest.fixture
-def provisioned_db(md_token: str, duckdb_schema: Path) -> Generator[str, None, None]:
-    database_name = f"test_{str(uuid.uuid4()).replace('-', '_')}"
-    try:
-        with duckdb.connect(f"md:?motherduck_token={md_token}") as md_conn:
-            with duckdb_schema.open("r") as f:
-                query = f.read()
-            md_conn.execute(f"CREATE DATABASE {database_name}")
-            md_conn.execute(f"USE {database_name}")
-            md_conn.execute(query)
-            md_conn.close()
-        yield database_name
-    finally:
-        with duckdb.connect(f"md:?motherduck_token={md_token}") as md_conn:
-            md_conn.execute(f"DROP DATABASE {database_name}")
-            md_conn.close()
-
-
-def validate_motherduck_destination(database: str, expected_num_elements: int, md_token: str):
-    conn = None
-    try:
-        conn = duckdb.connect(f"md:?motherduck_token={md_token}")
-        conn.execute(f"USE {database}")
-        _results = conn.sql("select count(*) from elements").fetchall()
-        _count = _results[0][0]
-        assert (
-            _count == expected_num_elements
-        ), f"dest check failed: got {_count}, expected {expected_num_elements}"
-        conn.close()
-    finally:
-        if conn:
-            conn.close()
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
-@requires_env("MOTHERDUCK_TOKEN")
-def test_motherduck_destination(
-    md_token: str, upload_file: Path, provisioned_db: str, temp_dir: Path
-):
-    file_data = FileData(
-        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
-        connector_type=CONNECTOR_TYPE,
-        identifier="mock-file-data",
-    )
-
-    stager = MotherDuckUploadStager()
-    staged_path = stager.run(
-        elements_filepath=upload_file,
-        file_data=file_data,
-        output_dir=temp_dir,
-        output_filename=upload_file.name,
-    )
-
-    access_config = MotherDuckAccessConfig(md_token=md_token)
-    connection_config = MotherDuckConnectionConfig(
-        database=provisioned_db, access_config=access_config
-    )
-    upload_config = MotherDuckUploaderConfig()
-    uploader = MotherDuckUploader(connection_config=connection_config, upload_config=upload_config)
-
-    uploader.run(path=staged_path, file_data=file_data)
-
-    staged_df = pd.read_json(staged_path, orient="records", lines=True)
-    validate_motherduck_destination(
-        database=provisioned_db, expected_num_elements=len(staged_df), md_token=md_token
-    )

test/integration/connectors/elasticsearch/conftest.py

@@ -1,34 +0,0 @@
-import json
-from pathlib import Path
-
-import pandas as pd
-import pytest
-
-int_test_dir = Path(__file__).parent
-assets_dir = int_test_dir / "assets"
-
-
-@pytest.fixture
-def movies_dataframe() -> pd.DataFrame:
-    movies_file = assets_dir / "wiki_movie_plots_small.csv"
-    assert movies_file.exists()
-    assert movies_file.is_file()
-    return pd.read_csv(movies_file).dropna().reset_index()
-
-
-@pytest.fixture
-def opensearch_elements_mapping() -> dict:
-    elements_mapping_file = assets_dir / "opensearch_elements_mappings.json"
-    assert elements_mapping_file.exists()
-    assert elements_mapping_file.is_file()
-    with elements_mapping_file.open() as fp:
-        return json.load(fp)
-
-
-@pytest.fixture
-def elasticsearch_elements_mapping() -> dict:
-    elements_mapping_file = assets_dir / "elasticsearch_elements_mappings.json"
-    assert elements_mapping_file.exists()
-    assert elements_mapping_file.is_file()
-    with elements_mapping_file.open() as fp:
-        return json.load(fp)

test/integration/connectors/elasticsearch/test_elasticsearch.py

@@ -1,331 +0,0 @@
-# ruff: noqa: I001
-import json
-import tempfile
-import time
-from contextlib import contextmanager
-from pathlib import Path
-from typing import Generator
-from test.integration.connectors.utils.validation.destination import (
-    StagerValidationConfigs,
-    stager_validation,
-)
-import pandas as pd
-import pytest
-from _pytest.fixtures import TopRequest
-from elasticsearch import Elasticsearch as ElasticsearchClient
-from elasticsearch.helpers import bulk
-
-from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG, NOSQL_TAG
-from test.integration.connectors.utils.docker import HealthCheck, container_context
-from test.integration.connectors.utils.validation.source import (
-    SourceValidationConfigs,
-    source_connector_validation,
-)
-from unstructured_ingest.error import DestinationConnectionError, SourceConnectionError
-from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
-from unstructured_ingest.processes.connectors.elasticsearch.elasticsearch import (
-    CONNECTOR_TYPE,
-    ElasticsearchAccessConfig,
-    ElasticsearchConnectionConfig,
-    ElasticsearchDownloader,
-    ElasticsearchDownloaderConfig,
-    ElasticsearchIndexer,
-    ElasticsearchIndexerConfig,
-    ElasticsearchUploader,
-    ElasticsearchUploaderConfig,
-    ElasticsearchUploadStager,
-    ElasticsearchUploadStagerConfig,
-)
-
-SOURCE_INDEX_NAME = "movies"
-DESTINATION_INDEX_NAME = "elements"
-ES_USERNAME = "elastic"
-ES_PASSWORD = "elastic_password"
-
-
-@contextmanager
-def get_client() -> Generator[ElasticsearchClient, None, None]:
-    with ElasticsearchClient(
-        hosts="http://localhost:9200", basic_auth=(ES_USERNAME, ES_PASSWORD), request_timeout=30
-    ) as client:
-        yield client
-
-
-def form_elasticsearch_doc_dict(i, csv_row):
-    return {
-        "_index": SOURCE_INDEX_NAME,
-        "_id": i,
-        "_source": {
-            "title": csv_row["Title"],
-            "ethnicity": csv_row["Origin/Ethnicity"],
-            "director": csv_row["Director"],
-            "cast": csv_row["Cast"],
-            "genre": csv_row["Genre"],
-            "plot": csv_row["Plot"],
-            "year": csv_row["Release Year"],
-            "wiki_page": csv_row["Wiki Page"],
-        },
-    }
-
-
-def dataframe_to_upload_data(df: pd.DataFrame) -> list[dict]:
-    upload_data = []
-    for index, row in df.iterrows():
-        upload_data.append(form_elasticsearch_doc_dict(index, row))
-    return upload_data
-
-
-def get_index_count(client: ElasticsearchClient, index_name: str) -> int:
-    count_resp = client.cat.count(index=index_name, format="json")
-    return int(count_resp[0]["count"])
-
-
-def validate_count(
-    client: ElasticsearchClient,
-    index_name: str,
-    expected_count: int,
-    retries: int = 10,
-    interval: int = 1,
-) -> None:
-    current_count = get_index_count(client, index_name)
-    if current_count == expected_count:
-        return
-    tries = 0
-    while tries < retries:
-        print(
-            f"retrying validation to check if expected count "
-            f"{expected_count} will match current count {current_count}"
-        )
-        time.sleep(interval)
-        current_count = get_index_count(client, index_name)
-        if current_count == expected_count:
-            break
-    assert current_count == expected_count, (
-        f"Expected count ({expected_count}) doesn't match how "
-        f"much came back from index: {current_count}"
-    )
-
-
-def seed_source_db(df: pd.DataFrame):
-    mapping = {
-        "properties": {
-            "title": {"type": "text", "analyzer": "english"},
-            "ethnicity": {"type": "text", "analyzer": "standard"},
-            "director": {"type": "text", "analyzer": "standard"},
-            "cast": {"type": "text", "analyzer": "standard"},
-            "genre": {"type": "text", "analyzer": "standard"},
-            "plot": {"type": "text", "analyzer": "english"},
-            "year": {"type": "integer"},
-            "wiki_page": {"type": "keyword"},
-        },
-    }
-    # seed content
-    with get_client() as client:
-        client.indices.create(index=SOURCE_INDEX_NAME, mappings=mapping)
-        upload_data = dataframe_to_upload_data(df=df)
-        bulk(client, upload_data)
-        client.indices.refresh(index=SOURCE_INDEX_NAME)
-        count = get_index_count(client, SOURCE_INDEX_NAME)
-        print(f"seeded {SOURCE_INDEX_NAME} index with {count} records")
-
-
-@pytest.fixture
-def source_index(movies_dataframe: pd.DataFrame) -> str:
-    with container_context(
-        image="docker.elastic.co/elasticsearch/elasticsearch:8.7.0",
-        ports={9200: 9200, 9300: 9300},
-        environment={
-            "discovery.type": "single-node",
-            "xpack.security.enabled": True,
-            "ELASTIC_PASSWORD": ES_PASSWORD,
-            "ELASTIC_USER": ES_USERNAME,
-        },
-        healthcheck=HealthCheck(
-            test="curl --silent --fail -u ${ELASTIC_USER}:${ELASTIC_PASSWORD} localhost:9200/_cluster/health || exit 1",  # noqa: E501
-            interval=1,
-            start_period=5,
-        ),
-    ):
-        seed_source_db(df=movies_dataframe)
-        yield SOURCE_INDEX_NAME
-
-
-@pytest.fixture
-def destination_index(elasticsearch_elements_mapping: dict) -> str:
-    with container_context(
-        image="docker.elastic.co/elasticsearch/elasticsearch:8.7.0",
-        ports={9200: 9200, 9300: 9300},
-        environment={
-            "discovery.type": "single-node",
-            "xpack.security.enabled": True,
-            "ELASTIC_PASSWORD": ES_PASSWORD,
-            "ELASTIC_USER": ES_USERNAME,
-        },
-        healthcheck=HealthCheck(
-            test="curl --silent --fail -u ${ELASTIC_USER}:${ELASTIC_PASSWORD} localhost:9200/_cluster/health || exit 1",  # noqa: E501
-            interval=1,
-            start_period=5,
-        ),
-    ):
-        with get_client() as client:
-            response = client.indices.create(
-                index=DESTINATION_INDEX_NAME, mappings=elasticsearch_elements_mapping
-            )
-            if not response["acknowledged"]:
-                raise RuntimeError(f"failed to create index: {response}")
-        yield DESTINATION_INDEX_NAME
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
-async def test_elasticsearch_source(source_index: str, movies_dataframe: pd.DataFrame):
-    indexer_config = ElasticsearchIndexerConfig(index_name=source_index)
-    with tempfile.TemporaryDirectory() as tempdir:
-        tempdir_path = Path(tempdir)
-        connection_config = ElasticsearchConnectionConfig(
-            access_config=ElasticsearchAccessConfig(password=ES_PASSWORD),
-            username=ES_USERNAME,
-            hosts=["http://localhost:9200"],
-        )
-        download_config = ElasticsearchDownloaderConfig(download_dir=tempdir_path)
-        indexer = ElasticsearchIndexer(
-            connection_config=connection_config, index_config=indexer_config
-        )
-        downloader = ElasticsearchDownloader(
-            connection_config=connection_config, download_config=download_config
-        )
-        expected_num_files = len(movies_dataframe)
-        await source_connector_validation(
-            indexer=indexer,
-            downloader=downloader,
-            configs=SourceValidationConfigs(
-                test_id=CONNECTOR_TYPE,
-                expected_num_files=expected_num_files,
-                expected_number_indexed_file_data=1,
-                validate_downloaded_files=True,
-            ),
-        )
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
-def test_elasticsearch_source_precheck_fail_no_cluster():
-    indexer_config = ElasticsearchIndexerConfig(index_name="index")
-
-    connection_config = ElasticsearchConnectionConfig(
-        access_config=ElasticsearchAccessConfig(password=ES_PASSWORD),
-        username=ES_USERNAME,
-        hosts=["http://localhost:9200"],
-    )
-    indexer = ElasticsearchIndexer(connection_config=connection_config, index_config=indexer_config)
-    with pytest.raises(SourceConnectionError):
-        indexer.precheck()
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
-def test_elasticsearch_source_precheck_fail_no_index(source_index: str):
-    indexer_config = ElasticsearchIndexerConfig(index_name="index")
-
-    connection_config = ElasticsearchConnectionConfig(
-        access_config=ElasticsearchAccessConfig(password=ES_PASSWORD),
-        username=ES_USERNAME,
-        hosts=["http://localhost:9200"],
-    )
-    indexer = ElasticsearchIndexer(connection_config=connection_config, index_config=indexer_config)
-    with pytest.raises(SourceConnectionError):
-        indexer.precheck()
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
-async def test_elasticsearch_destination(
-    upload_file: Path,
-    destination_index: str,
-    tmp_path: Path,
-):
-    file_data = FileData(
-        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
-        connector_type=CONNECTOR_TYPE,
-        identifier="mock file data",
-    )
-    connection_config = ElasticsearchConnectionConfig(
-        access_config=ElasticsearchAccessConfig(password=ES_PASSWORD),
-        username=ES_USERNAME,
-        hosts=["http://localhost:9200"],
-    )
-    stager = ElasticsearchUploadStager(
-        upload_stager_config=ElasticsearchUploadStagerConfig(index_name=destination_index)
-    )
-
-    uploader = ElasticsearchUploader(
-        connection_config=connection_config,
-        upload_config=ElasticsearchUploaderConfig(index_name=destination_index),
-    )
-    staged_filepath = stager.run(
-        elements_filepath=upload_file,
-        file_data=file_data,
-        output_dir=tmp_path,
-        output_filename=upload_file.name,
-    )
-    uploader.precheck()
-    uploader.run(path=staged_filepath, file_data=file_data)
-
-    # Run validation
-    with staged_filepath.open() as f:
-        staged_elements = json.load(f)
-    expected_count = len(staged_elements)
-    with get_client() as client:
-        validate_count(client=client, expected_count=expected_count, index_name=destination_index)
-
-    # Rerun and make sure the same documents get updated
-    uploader.run(path=staged_filepath, file_data=file_data)
-    with get_client() as client:
-        validate_count(client=client, expected_count=expected_count, index_name=destination_index)
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
-def test_elasticsearch_destination_precheck_fail():
-    connection_config = ElasticsearchConnectionConfig(
-        access_config=ElasticsearchAccessConfig(password=ES_PASSWORD),
-        username=ES_USERNAME,
-        hosts=["http://localhost:9200"],
-    )
-    uploader = ElasticsearchUploader(
-        connection_config=connection_config,
-        upload_config=ElasticsearchUploaderConfig(index_name="index"),
-    )
-    with pytest.raises(DestinationConnectionError):
-        uploader.precheck()
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
-def test_elasticsearch_destination_precheck_fail_no_index(destination_index: str):
-    connection_config = ElasticsearchConnectionConfig(
-        access_config=ElasticsearchAccessConfig(password=ES_PASSWORD),
-        username=ES_USERNAME,
-        hosts=["http://localhost:9200"],
-    )
-    uploader = ElasticsearchUploader(
-        connection_config=connection_config,
-        upload_config=ElasticsearchUploaderConfig(index_name="index"),
-    )
-    with pytest.raises(DestinationConnectionError):
-        uploader.precheck()
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
-@pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
-def test_elasticsearch_stager(
-    request: TopRequest,
-    upload_file_str: str,
-    tmp_path: Path,
-):
-    upload_file: Path = request.getfixturevalue(upload_file_str)
-    stager = ElasticsearchUploadStager(
-        upload_stager_config=ElasticsearchUploadStagerConfig(index_name="mock_index")
-    )
-    stager_validation(
-        configs=StagerValidationConfigs(test_id=CONNECTOR_TYPE, expected_count=22),
-        input_file=upload_file,
-        stager=stager,
-        tmp_dir=tmp_path,
-    )