unstructured-ingest 0.7.2__py3-none-any.whl → 1.0.2__py3-none-any.whl

test/integration/connectors/test_vectara.py

@@ -1,282 +0,0 @@
-import json
-import os
-import time
-from functools import lru_cache
-from pathlib import Path
-from typing import Generator
-from uuid import uuid4
-
-import pytest
-import requests
-
-from test.integration.connectors.utils.constants import DESTINATION_TAG, NOSQL_TAG
-from test.integration.utils import requires_env
-from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
-from unstructured_ingest.logger import logger
-from unstructured_ingest.processes.connectors.vectara import (
-    CONNECTOR_TYPE as VECTARA_CONNECTOR_TYPE,
-)
-from unstructured_ingest.processes.connectors.vectara import (
-    VectaraAccessConfig,
-    VectaraConnectionConfig,
-    VectaraUploader,
-    VectaraUploaderConfig,
-    VectaraUploadStager,
-    VectaraUploadStagerConfig,
-)
-
-
-def validate_upload(document: dict, expected_data: dict):
-    logger.info(f"validating document: {document}")
-    element_id = expected_data["element_id"]
-    expected_text = expected_data["text"]
-    filename = expected_data["metadata"]["filename"]
-    filetype = expected_data["metadata"]["filetype"]
-    page_number = expected_data["metadata"]["page_number"]
-
-    assert document is not None
-    speech_parts = document["parts"]
-    assert speech_parts
-    first_part = speech_parts[0]
-    assert first_part["text"] == expected_text
-    part_metadata = first_part["metadata"]
-    assert part_metadata
-    assert part_metadata["element_id"] == element_id
-    assert part_metadata["filename"] == filename
-    assert part_metadata["filetype"] == filetype
-    assert part_metadata["page_number"] == page_number
-
-
-@requires_env("VECTARA_OAUTH_CLIENT_ID", "VECTARA_OAUTH_SECRET", "VECTARA_CUSTOMER_ID")
-@lru_cache()
-def _get_jwt_token():
-    """Connect to the server and get a JWT token."""
-    customer_id = os.environ["VECTARA_CUSTOMER_ID"]
-    token_endpoint = (
-        f"https://vectara-prod-{customer_id}.auth.us-west-2.amazoncognito.com/oauth2/token"
-    )
-    headers = {
-        "Content-Type": "application/x-www-form-urlencoded",
-    }
-    data = {
-        "grant_type": "client_credentials",
-        "client_id": os.environ["VECTARA_OAUTH_CLIENT_ID"],
-        "client_secret": os.environ["VECTARA_OAUTH_SECRET"],
-    }
-
-    response = requests.post(token_endpoint, headers=headers, data=data)
-    response.raise_for_status()
-    response_json = response.json()
-
-    return response_json.get("access_token")
-
-
-def list_documents(corpus_key: str) -> list[str]:
-
-    url = f"https://api.vectara.io/v2/corpora/{corpus_key}/documents"
-
-    # the query below requires the corpus to have filter attributes for element_id
-
-    jwt_token = _get_jwt_token()
-    headers = {
-        "Content-Type": "application/json",
-        "Accept": "application/json",
-        "Authorization": f"Bearer {jwt_token}",
-        "X-source": "unstructured",
-    }
-
-    response = requests.get(url, headers=headers)
-    response.raise_for_status()
-    response_json = response.json()
-    documents = response_json.get("documents", [])
-    return documents
-
-
-def fetch_document(corpus_key: str, documents_id: str) -> dict:
-    url = f"https://api.vectara.io/v2/corpora/{corpus_key}/documents/{documents_id}"
-    jwt_token = _get_jwt_token()
-    headers = {
-        "Content-Type": "application/json",
-        "Accept": "application/json",
-        "Authorization": f"Bearer {jwt_token}",
-        "X-source": "unstructured",
-    }
-
-    response = requests.get(url, headers=headers)
-    response.raise_for_status()
-    return response.json()
-
-
-def create_corpora(corpus_key: str, corpus_name: str) -> None:
-    url = "https://api.vectara.io/v2/corpora"
-    data = json.dumps({"key": corpus_key, "name": corpus_name, "description": "integration test"})
-    jwt_token = _get_jwt_token()
-    headers = {
-        "Content-Type": "application/json",
-        "Accept": "application/json",
-        "Authorization": f"Bearer {jwt_token}",
-        "X-source": "unstructured",
-    }
-
-    response = requests.post(url, headers=headers, data=data)
-    response.raise_for_status()
-
-
-def replace_filter_attributes(corpus_key: str) -> None:
-    url = f"https://api.vectara.io/v2/corpora/{corpus_key}/replace_filter_attributes"
-    data = json.dumps(
-        {
-            "filter_attributes": [
-                {"name": "element_id", "level": "part", "indexed": True, "type": "text"}
-            ]
-        }
-    )
-    jwt_token = _get_jwt_token()
-    headers = {
-        "Content-Type": "application/json",
-        "Accept": "application/json",
-        "Authorization": f"Bearer {jwt_token}",
-        "X-source": "unstructured",
-    }
-
-    response = requests.post(url, headers=headers, data=data)
-    response.raise_for_status()
-
-
-def delete_corpora(corpus_key: str) -> None:
-    url = f"https://api.vectara.io/v2/corpora/{corpus_key}"
-
-    jwt_token = _get_jwt_token()
-    headers = {
-        "Content-Type": "application/json",
-        "Accept": "application/json",
-        "Authorization": f"Bearer {jwt_token}",
-        "X-source": "unstructured",
-    }
-
-    response = requests.delete(url, headers=headers)
-    response.raise_for_status()
-
-
-def get_metadata(corpus_key: str):
-    url = f"https://api.vectara.io/v2/corpora/{corpus_key}"
-    jwt_token = _get_jwt_token()
-    headers = {
-        "Content-Type": "application/json",
-        "Accept": "application/json",
-        "Authorization": f"Bearer {jwt_token}",
-        "X-source": "unstructured",
-    }
-    response = requests.get(url, headers=headers)
-    response.raise_for_status()
-    return response.json()
-
-
-def wait_for_ready(corpus_key: str, timeout=60, interval=2) -> None:
-    start = time.time()
-    while time.time() - start < timeout:
-        try:
-            get_metadata(corpus_key)
-            return
-        except requests.HTTPError:
-            time.sleep(interval)
-    raise TimeoutError("time out waiting for corpus to be ready")
-
-
-def wait_for_delete(corpus_key: str, timeout=60, interval=2) -> None:
-    start = time.time()
-    while time.time() - start < timeout:
-        try:
-            get_metadata(corpus_key)
-            time.sleep(interval)
-        except requests.HTTPError:
-            return
-    raise TimeoutError("time out waiting for corpus to delete")
-
-
-@pytest.fixture
-def corpora_util() -> Generator[str, None, None]:
-    random_id = str(uuid4()).split("-")[0]
-    corpus_key = f"ingest-test-{random_id}"
-    corpus_name = "ingest-test"
-    logger.info(f"Creating corpus with key: {corpus_key}")
-    try:
-        create_corpora(corpus_key, corpus_name)
-        replace_filter_attributes(corpus_key)
-        wait_for_ready(corpus_key=corpus_key)
-        yield corpus_key
-    except Exception as e:
-        logger.error(f"failed to create corpus {corpus_key}: {e}")
-    finally:
-        logger.info(f"deleting corpus: {corpus_key}")
-        delete_corpora(corpus_key)
-        wait_for_delete(corpus_key=corpus_key)
-
-
-def wait_for_doc_meta(corpus_key: str, timeout=60, interval=1) -> list[str]:
-    start = time.time()
-    while time.time() - start < timeout:
-        all_document_meta = list_documents(corpus_key)
-        if not all_document_meta:
-            time.sleep(interval)
-            continue
-        else:
-            return all_document_meta
-    raise TimeoutError("time out waiting for document to be ready")
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(VECTARA_CONNECTOR_TYPE, DESTINATION_TAG, "vectara", NOSQL_TAG)
-@requires_env("VECTARA_OAUTH_CLIENT_ID", "VECTARA_OAUTH_SECRET", "VECTARA_CUSTOMER_ID")
-async def test_vectara_destination(
-    upload_file: Path, tmp_path: Path, corpora_util: str, retries=30, interval=1
-):
-    corpus_key = corpora_util
-    connection_kwargs = {
-        "customer_id": os.environ["VECTARA_CUSTOMER_ID"],
-        "corpus_key": corpus_key,
-    }
-
-    oauth_client_id = os.environ["VECTARA_OAUTH_CLIENT_ID"]
-    oauth_secret = os.environ["VECTARA_OAUTH_SECRET"]
-
-    file_data = FileData(
-        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
-        connector_type=VECTARA_CONNECTOR_TYPE,
-        identifier="mock-file-data",
-    )
-
-    stager_config = VectaraUploadStagerConfig()
-    stager = VectaraUploadStager(upload_stager_config=stager_config)
-    new_upload_file = stager.run(
-        elements_filepath=upload_file,
-        output_dir=tmp_path,
-        output_filename=upload_file.name,
-        file_data=file_data,
-    )
-
-    uploader = VectaraUploader(
-        connection_config=VectaraConnectionConfig(
-            **connection_kwargs,
-            access_config=VectaraAccessConfig(
-                oauth_client_id=oauth_client_id, oauth_secret=oauth_secret
-            ),
-        ),
-        upload_config=VectaraUploaderConfig(),
-    )
-
-    with new_upload_file.open() as new_upload_fp:
-        elements_stager = json.load(new_upload_fp)
-
-    if uploader.is_async():
-        await uploader.run_data_async(data=elements_stager, file_data=file_data)
-
-    with upload_file.open() as upload_fp:
-        elements = json.load(upload_fp)
-    first_element = elements[0]
-
-    all_document_meta = wait_for_doc_meta(corpus_key)
-    assert len(all_document_meta) == 1
-    document_meta = all_document_meta[0]
-    document = fetch_document(corpus_key=corpus_key, documents_id=document_meta["id"])
-    validate_upload(document=document, expected_data=first_element)

test/integration/connectors/test_zendesk.py

@@ -1,120 +0,0 @@
-import os
-from pathlib import Path
-
-import pytest
-
-from test.integration.connectors.utils.constants import SOURCE_TAG, UNCATEGORIZED_TAG
-from test.integration.connectors.utils.validation.source import (
-    SourceValidationConfigs,
-    source_connector_validation,
-)
-from test.integration.utils import requires_env
-from unstructured_ingest.errors_v2 import UserAuthError
-from unstructured_ingest.processes.connectors.zendesk.zendesk import (
-    CONNECTOR_TYPE,
-    ZendeskAccessConfig,
-    ZendeskConnectionConfig,
-    ZendeskDownloader,
-    ZendeskDownloaderConfig,
-    ZendeskIndexer,
-    ZendeskIndexerConfig,
-)
-
-SUBDOMAIN = "unstructuredhelp"
-EMAIL = "test@unstructured.io"
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
-@requires_env("ZENDESK_TOKEN")
-async def test_zendesk_source_tickets(temp_dir: Path):
-    access_config = ZendeskAccessConfig(api_token=os.environ["ZENDESK_TOKEN"])
-    connection_config = ZendeskConnectionConfig(
-        subdomain=SUBDOMAIN, email=EMAIL, access_config=access_config
-    )
-
-    index_config = ZendeskIndexerConfig(item_type="tickets")
-
-    indexer = ZendeskIndexer(
-        connection_config=connection_config,
-        index_config=index_config,
-        connector_type=CONNECTOR_TYPE,
-    )
-
-    # handle downloader.
-    download_config = ZendeskDownloaderConfig(download_dir=temp_dir)
-
-    downloader = ZendeskDownloader(
-        connection_config=connection_config,
-        download_config=download_config,
-        connector_type=CONNECTOR_TYPE,
-    )
-
-    # Run the source connector validation
-    await source_connector_validation(
-        indexer=indexer,
-        downloader=downloader,
-        configs=SourceValidationConfigs(
-            test_id="zendesk-tickets",
-            expected_num_files=8,
-            validate_file_data=False,
-            validate_downloaded_files=True,
-        ),
-    )
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
-@requires_env("ZENDESK_TOKEN")
-async def test_zendesk_source_articles(temp_dir):
-    access_config = ZendeskAccessConfig(api_token=os.environ["ZENDESK_TOKEN"])
-    connection_config = ZendeskConnectionConfig(
-        subdomain=SUBDOMAIN, email=EMAIL, access_config=access_config
-    )
-
-    index_config = ZendeskIndexerConfig(item_type="articles")
-
-    indexer = ZendeskIndexer(
-        connection_config=connection_config,
-        index_config=index_config,
-        connector_type=CONNECTOR_TYPE,
-    )
-
-    # handle downloader.
-    download_config = ZendeskDownloaderConfig(download_dir=temp_dir, extract_images=True)
-
-    downloader = ZendeskDownloader(
-        connection_config=connection_config,
-        download_config=download_config,
-        connector_type=CONNECTOR_TYPE,
-    )
-
-    # Run the source connector validation
-    await source_connector_validation(
-        indexer=indexer,
-        downloader=downloader,
-        configs=SourceValidationConfigs(
-            test_id="zendesk-articles",
-            expected_num_files=8,
-            validate_file_data=True,
-            validate_downloaded_files=True,
-        ),
-    )
-
-
-@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
-def test_zendesk_source_articles_fail(temp_dir):
-    access_config = ZendeskAccessConfig(api_token="FAKE_TOKEN")
-    connection_config = ZendeskConnectionConfig(
-        subdomain=SUBDOMAIN, email=EMAIL, access_config=access_config
-    )
-
-    index_config = ZendeskIndexerConfig(item_type="tickets")
-
-    indexer = ZendeskIndexer(
-        connection_config=connection_config,
-        index_config=index_config,
-        connector_type=CONNECTOR_TYPE,
-    )
-    with pytest.raises(expected_exception=UserAuthError):
-        indexer.precheck()

test/integration/connectors/utils/constants.py

@@ -1,13 +0,0 @@
-from pathlib import Path
-
-SOURCE_TAG = "source"
-DESTINATION_TAG = "destination"
-BLOB_STORAGE_TAG = "blob_storage"
-SQL_TAG = "sql"
-NOSQL_TAG = "nosql"
-VECTOR_DB_TAG = "vector_db"
-GRAPH_DB_TAG = "graph_db"
-UNCATEGORIZED_TAG = "uncategorized"
-
-env_setup_path = Path(__file__).parents[1] / "env_setup"
-expected_results_path = Path(__file__).parents[1] / "expected_results"

test/integration/connectors/utils/docker.py

@@ -1,151 +0,0 @@
-import time
-from contextlib import contextmanager
-from typing import Optional, Union
-
-import docker
-from docker.models.containers import Container
-from pydantic import BaseModel, Field, field_serializer
-
-
-class HealthCheck(BaseModel):
-    test: Union[str, list[str]]
-    interval: int = Field(
-        gt=0, default=30, description="The time to wait between checks in seconds."
-    )
-    timeout: int = Field(
-        gt=0, default=30, description="The time to wait before considering the check to have hung."
-    )
-    retries: int = Field(
-        gt=0,
-        default=3,
-        description="The number of consecutive failures needed "
-        "to consider a container as unhealthy.",
-    )
-    start_period: int = Field(
-        gt=0,
-        default=0,
-        description="Start period for the container to initialize before starting health-retries countdown in seconds.",  # noqa: E501
-    )
-
-    @field_serializer("interval")
-    def serialize_interval(self, interval: int) -> int:
-        return int(interval * 10e8)
-
-    @field_serializer("timeout")
-    def serialize_timeout(self, timeout: int) -> int:
-        return int(timeout * 10e8)
-
-    @field_serializer("start_period")
-    def serialize_start_period(self, start_period: int) -> int:
-        return int(start_period * 10e8)
-
-
-def get_container(
-    docker_client: docker.DockerClient,
-    image: str,
-    ports: dict,
-    name: Optional[str] = None,
-    environment: Optional[dict] = None,
-    volumes: Optional[dict] = None,
-    healthcheck: Optional[HealthCheck] = None,
-) -> Container:
-    run_kwargs = {
-        "image": image,
-        "detach": True,
-        "ports": ports,
-    }
-    if environment:
-        run_kwargs["environment"] = environment
-    if volumes:
-        run_kwargs["volumes"] = volumes
-    if healthcheck:
-        run_kwargs["healthcheck"] = healthcheck.model_dump()
-    if name:
-        run_kwargs["name"] = name
-    container: Container = docker_client.containers.run(**run_kwargs)
-    return container
-
-
-def get_healthcheck(container: Container) -> Optional[HealthCheck]:
-    healthcheck_config = container.attrs.get("Config", {}).get("Healthcheck", None)
-    if not healthcheck_config:
-        return None
-    healthcheck_data = {
-        "test": healthcheck_config["Test"],
-    }
-    if interval := healthcheck_config.get("Interval"):
-        healthcheck_data["interval"] = interval / 10e8
-    if start_period := healthcheck_config.get("StartPeriod"):
-        healthcheck_data["start_period"] = start_period / 10e8
-    if retries := healthcheck_config.get("Retries"):
-        healthcheck_data["retries"] = retries
-    return HealthCheck.model_validate(healthcheck_data)
-
-
-def healthcheck_wait(
-    container: Container, retries: int = 30, interval: int = 1, start_period: Optional[int] = None
-) -> None:
-    if start_period:
-        time.sleep(start_period)
-    health = container.health
-    tries = 0
-    while health != "healthy" and tries < retries:
-        tries += 1
-        logs = container.attrs.get("State", {}).get("Health", {}).get("Log")
-        latest_log = logs[-1] if logs else None
-        print(
-            f"attempt {tries} - waiting for docker container "
-            f"to be healthy: {health} latest log: {latest_log}"
-        )
-        time.sleep(interval)
-        container.reload()
-        health = container.health
-    if health != "healthy":
-        logs = container.attrs.get("State", {}).get("Health", {}).get("Log")
-        latest_log = logs[-1] if logs else None
-        raise TimeoutError(f"Docker container never came up healthy: {latest_log}")
-
-
-@contextmanager
-def container_context(
-    image: str,
-    ports: dict,
-    environment: Optional[dict] = None,
-    volumes: Optional[dict] = None,
-    healthcheck: Optional[HealthCheck] = None,
-    healthcheck_retries: int = 30,
-    docker_client: Optional[docker.DockerClient] = None,
-    name: Optional[str] = None,
-):
-    docker_client = docker_client or docker.from_env()
-    print(f"pulling image {image}")
-    docker_client.images.pull(image)
-    container: Optional[Container] = None
-    try:
-        container = get_container(
-            docker_client=docker_client,
-            image=image,
-            ports=ports,
-            environment=environment,
-            volumes=volumes,
-            healthcheck=healthcheck,
-            name=name,
-        )
-        if healthcheck_data := get_healthcheck(container):
-            # Mirror whatever healthcheck config set on container
-            healthcheck_wait(
-                container=container,
-                retries=healthcheck_retries,
-                start_period=healthcheck_data.start_period,
-                interval=healthcheck_data.interval,
-            )
-        yield container
-    except AssertionError as e:
-        if container:
-            logs = container.logs()
-            print(logs.decode("utf-8"))
-            raise e
-    finally:
-        if container:
-            container.kill()
-            container.remove()

test/integration/connectors/utils/docker_compose.py

@@ -1,59 +0,0 @@
-import subprocess
-from contextlib import contextmanager
-from pathlib import Path
-
-
-def docker_compose_down(docker_compose_path: Path):
-    cmd = f"docker compose -f {docker_compose_path.resolve()} down --remove-orphans -v --rmi all"
-    print(f"Running command: {cmd}")
-    final_resp = subprocess.run(
-        cmd,
-        shell=True,
-        capture_output=True,
-    )
-    if final_resp.returncode != 0:
-        print("STDOUT: {}".format(final_resp.stdout.decode("utf-8")))
-        print("STDERR: {}".format(final_resp.stderr.decode("utf-8")))
-
-
-def run_cleanup(docker_compose_path: Path):
-    docker_compose_down(docker_compose_path=docker_compose_path)
-
-
-@contextmanager
-def docker_compose_context(docker_compose_path: Path):
-    # Dynamically run a specific docker compose file and make sure it gets cleanup by
-    # by leveraging a context manager. Uses subprocess to map docker compose commands
-    # to the underlying shell.
-    assert docker_compose_path.exists()
-    if docker_compose_path.is_dir():
-        if (docker_compose_path / "docker-compose.yml").exists():
-            docker_compose_path = docker_compose_path / "docker-compose.yml"
-        elif (docker_compose_path / "docker-compose.yaml").exists():
-            docker_compose_path = docker_compose_path / "docker-compose.yaml"
-    assert docker_compose_path.is_file()
-    resp = None
-    try:
-        cmd = f"docker compose -f {docker_compose_path.resolve()} up -d --wait"
-        print(f"Running command: {cmd}")
-        resp = subprocess.run(
-            cmd,
-            shell=True,
-            capture_output=True,
-        )
-        # Return code from docker compose using --wait can be 1 even if no error
-        yield
-    except Exception as e:
-        if resp:
-            print("STDOUT: {}".format(resp.stdout.decode("utf-8")))
-            print("STDERR: {}".format(resp.stderr.decode("utf-8")))
-        cmd = f"docker compose -f {docker_compose_path.resolve()} logs"
-        logs = subprocess.run(
-            cmd,
-            shell=True,
-            capture_output=True,
-        )
-        print("DOCKER LOGS: {}".format(logs.stdout.decode("utf-8")))
-        raise e
-    finally:
-        run_cleanup(docker_compose_path=docker_compose_path)