unstructured-ingest 0.1.1__py3-none-any.whl → 0.2.1__py3-none-any.whl

test/integration/connectors/utils/docker.py

@@ -0,0 +1,78 @@
+import time
+from contextlib import contextmanager
+from typing import Optional
+
+import docker
+from docker.models.containers import Container
+
+
+def get_container(
+    docker_client: docker.DockerClient,
+    image: str,
+    ports: dict,
+    environment: Optional[dict] = None,
+    volumes: Optional[dict] = None,
+    healthcheck: Optional[dict] = None,
+) -> Container:
+    run_kwargs = {
+        "image": image,
+        "detach": True,
+        "ports": ports,
+    }
+    if environment:
+        run_kwargs["environment"] = environment
+    if volumes:
+        run_kwargs["volumes"] = volumes
+    if healthcheck:
+        run_kwargs["healthcheck"] = healthcheck
+    container: Container = docker_client.containers.run(**run_kwargs)
+    return container
+
+
+def has_healthcheck(container: Container) -> bool:
+    return container.attrs.get("Config", {}).get("Healthcheck", None) is not None
+
+
+def healthcheck_wait(container: Container, timeout: int = 10) -> None:
+    health = container.health
+    start = time.time()
+    while health != "healthy" and time.time() - start < timeout:
+        time.sleep(1)
+        container.reload()
+        health = container.health
+    if health != "healthy":
+        health_dict = container.attrs.get("State", {}).get("Health", {})
+        raise TimeoutError(f"Docker container never came up healthy: {health_dict}")
+
+
+@contextmanager
+def container_context(
+    docker_client: docker.DockerClient,
+    image: str,
+    ports: dict,
+    environment: Optional[dict] = None,
+    volumes: Optional[dict] = None,
+    healthcheck: Optional[dict] = None,
+    healthcheck_timeout: int = 10,
+):
+    container: Optional[Container] = None
+    try:
+        container = get_container(
+            docker_client=docker_client,
+            image=image,
+            ports=ports,
+            environment=environment,
+            volumes=volumes,
+            healthcheck=healthcheck,
+        )
+        if has_healthcheck(container):
+            healthcheck_wait(container=container, timeout=healthcheck_timeout)
+        yield container
+    except AssertionError as e:
+        if container:
+            logs = container.logs()
+            print(logs.decode("utf-8"))
+            raise e
+    finally:
+        if container:
+            container.kill()

test/integration/connectors/utils/docker_compose.py

@@ -3,6 +3,23 @@ from contextlib import contextmanager
 from pathlib import Path
 
 
+def docker_compose_down(docker_compose_path: Path):
+    cmd = f"docker compose -f {docker_compose_path.resolve()} down --remove-orphans -v --rmi all"
+    print(f"Running command: {cmd}")
+    final_resp = subprocess.run(
+        cmd,
+        shell=True,
+        capture_output=True,
+    )
+    if final_resp.returncode != 0:
+        print("STDOUT: {}".format(final_resp.stdout.decode("utf-8")))
+        print("STDERR: {}".format(final_resp.stderr.decode("utf-8")))
+
+
+def run_cleanup(docker_compose_path: Path):
+    docker_compose_down(docker_compose_path=docker_compose_path)
+
+
 @contextmanager
 def docker_compose_context(docker_compose_path: Path):
     # Dynamically run a specific docker compose file and make sure it gets cleanup by
@@ -30,15 +47,13 @@ def docker_compose_context(docker_compose_path: Path):
         if resp:
             print("STDOUT: {}".format(resp.stdout.decode("utf-8")))
             print("STDERR: {}".format(resp.stderr.decode("utf-8")))
-        raise e
-    finally:
-        cmd = f"docker compose -f {docker_compose_path.resolve()} down --remove-orphans -v"
-        print(f"Running command: {cmd}")
-        final_resp = subprocess.run(
+        cmd = f"docker compose -f {docker_compose_path.resolve()} logs"
+        logs = subprocess.run(
             cmd,
             shell=True,
             capture_output=True,
         )
-        if final_resp.returncode != 0:
-            print("STDOUT: {}".format(final_resp.stdout.decode("utf-8")))
-            print("STDERR: {}".format(final_resp.stderr.decode("utf-8")))
+        print("DOCKER LOGS: {}".format(logs.stdout.decode("utf-8")))
+        raise e
+    finally:
+        run_cleanup(docker_compose_path=docker_compose_path)

test/integration/connectors/utils/validation.py

@@ -1,3 +1,4 @@
+import filecmp
 import json
 import os
 import shutil
@@ -5,15 +6,31 @@ from dataclasses import dataclass, field, replace
 from pathlib import Path
 from typing import Callable, Optional
 
+import pandas as pd
 from deepdiff import DeepDiff
 
 from test.integration.connectors.utils.constants import expected_results_path
 from unstructured_ingest.v2.interfaces import Downloader, FileData, Indexer
 
 
+def pandas_df_equality_check(expected_filepath: Path, current_filepath: Path) -> bool:
+    expected_df = pd.read_csv(expected_filepath)
+    current_df = pd.read_csv(current_filepath)
+    if expected_df.equals(current_df):
+        return True
+    # Print diff
+    diff = expected_df.merge(current_df, indicator=True, how="left").loc[
+        lambda x: x["_merge"] != "both"
+    ]
+    print("diff between expected and current df:")
+    print(diff)
+    return False
+
+
 @dataclass
 class ValidationConfigs:
     test_id: str
+    expected_number_indexed_file_data: Optional[int] = None
     expected_num_files: Optional[int] = None
     predownload_file_data_check: Optional[Callable[[FileData], None]] = None
     postdownload_file_data_check: Optional[Callable[[FileData], None]] = None
@@ -21,6 +38,8 @@ class ValidationConfigs:
         default_factory=lambda: ["local_download_path", "metadata.date_processed"]
     )
     exclude_fields_extend: list[str] = field(default_factory=list)
+    validate_downloaded_files: bool = False
+    downloaded_file_equality_check: Optional[Callable[[Path, Path], bool]] = None
 
     def get_exclude_fields(self) -> list[str]:
         exclude_fields = self.exclude_fields
@@ -78,6 +97,13 @@ def check_files(expected_output_dir: Path, all_file_data: list[FileData]):
     assert not diff, "diff in files that exist: {}".format(", ".join(diff))
 
 
+def check_files_in_paths(expected_output_dir: Path, current_output_dir: Path):
+    expected_files = get_files(dir_path=expected_output_dir)
+    current_files = get_files(dir_path=current_output_dir)
+    diff = set(expected_files) ^ set(current_files)
+    assert not diff, "diff in files that exist: {}".format(", ".join(diff))
+
+
 def check_contents(
     expected_output_dir: Path, all_file_data: list[FileData], configs: ValidationConfigs
 ):
@@ -96,6 +122,32 @@ def check_contents(
     assert not found_diff, f"Diffs found between files: {found_diff}"
 
 
+def check_raw_file_contents(
+    expected_output_dir: Path,
+    current_output_dir: Path,
+    configs: ValidationConfigs,
+):
+    current_files = get_files(dir_path=current_output_dir)
+    found_diff = False
+    files = []
+    for current_file in current_files:
+        current_file_path = current_output_dir / current_file
+        expected_file_path = expected_output_dir / current_file
+        if downloaded_file_equality_check := configs.downloaded_file_equality_check:
+            is_different = downloaded_file_equality_check(expected_file_path, current_file_path)
+        elif expected_file_path.suffix == ".csv" and current_file_path.suffix == ".csv":
+            is_different = not pandas_df_equality_check(
+                expected_filepath=expected_file_path, current_filepath=current_file_path
+            )
+        else:
+            is_different = not filecmp.cmp(expected_file_path, current_file_path, shallow=False)
+        if is_different:
+            found_diff = True
+            files.append(str(expected_file_path))
+            print(f"diffs between files {expected_file_path} and {current_file_path}")
+    assert not found_diff, "Diffs found between files: {}".format(", ".join(files))
+
+
 def run_expected_results_validation(
     expected_output_dir: Path, all_file_data: list[FileData], configs: ValidationConfigs
 ):
@@ -105,6 +157,21 @@ def run_expected_results_validation(
     )
 
 
+def run_expected_download_files_validation(
+    expected_output_dir: Path,
+    current_download_dir: Path,
+    configs: ValidationConfigs,
+):
+    check_files_in_paths(
+        expected_output_dir=expected_output_dir, current_output_dir=current_download_dir
+    )
+    check_raw_file_contents(
+        expected_output_dir=expected_output_dir,
+        current_output_dir=current_download_dir,
+        configs=configs,
+    )
+
+
 def run_directory_structure_validation(expected_output_dir: Path, download_files: list[str]):
     directory_record = expected_output_dir / "directory_structure.json"
     with directory_record.open("r") as directory_file:
@@ -113,13 +180,18 @@ def run_directory_structure_validation(expected_output_dir: Path, download_files
     assert directory_structure == download_files
 
 
-def update_fixtures(output_dir: Path, download_dir: Path, all_file_data: list[FileData]):
+def update_fixtures(
+    output_dir: Path,
+    download_dir: Path,
+    all_file_data: list[FileData],
+    save_downloads: bool = False,
+):
     # Delete current files
     shutil.rmtree(path=output_dir, ignore_errors=True)
     output_dir.mkdir(parents=True)
     # Rewrite the current file data
     file_data_output_path = output_dir / "file_data"
-    file_data_output_path.mkdir(parents=True)
+    file_data_output_path.mkdir(parents=True, exist_ok=True)
     for file_data in all_file_data:
         file_data_path = file_data_output_path / f"{file_data.identifier}.json"
         with file_data_path.open(mode="w") as f:
@@ -132,6 +204,11 @@ def update_fixtures(output_dir: Path, download_dir: Path, all_file_data: list[Fi
     with download_dir_record.open(mode="w") as f:
         json.dump({"directory_structure": download_files}, f, indent=2)
 
+    # If applicable, save raw downloads
+    if save_downloads:
+        raw_download_output_path = output_dir / "downloads"
+        shutil.copytree(download_dir, raw_download_output_path)
+
 
 def run_all_validations(
     configs: ValidationConfigs,
@@ -140,6 +217,13 @@ def run_all_validations(
     download_dir: Path,
     test_output_dir: Path,
 ):
+    if expected_number_indexed_file_data := configs.expected_number_indexed_file_data:
+        assert (
+            len(predownload_file_data) == expected_number_indexed_file_data
+        ), f"expected {expected_number_indexed_file_data} but got {len(predownload_file_data)}"
+    if expected_num_files := configs.expected_num_files:
+        assert len(postdownload_file_data) == expected_num_files
+
     for pre_data, post_data in zip(predownload_file_data, postdownload_file_data):
         configs.run_file_data_validation(
             predownload_file_data=pre_data, postdownload_file_data=post_data
@@ -155,6 +239,12 @@ def run_all_validations(
     run_directory_structure_validation(
         expected_output_dir=configs.test_output_dir(), download_files=download_files
     )
+    if configs.validate_downloaded_files:
+        run_expected_download_files_validation(
+            expected_output_dir=test_output_dir / "downloads",
+            current_download_dir=download_dir,
+            configs=configs,
+        )
 
 
 async def source_connector_validation(
@@ -200,4 +290,5 @@ async def source_connector_validation(
             output_dir=test_output_dir,
             download_dir=download_dir,
             all_file_data=all_postdownload_file_data,
+            save_downloads=configs.validate_downloaded_files,
         )

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.1.1"  # pragma: no cover
+__version__ = "0.2.1"  # pragma: no cover

unstructured_ingest/v2/cli/utils/click.py

@@ -1,12 +1,13 @@
 import json
 import os.path
+from datetime import date, datetime
 from gettext import gettext, ngettext
 from gettext import gettext as _
 from pathlib import Path
 from typing import Any, Optional, Type, TypeVar, Union
 
 import click
-from pydantic import BaseModel, ConfigDict, Secret
+from pydantic import BaseModel, ConfigDict, Secret, TypeAdapter, ValidationError
 
 
 def conform_click_options(options: dict):
@@ -109,6 +110,36 @@ class DelimitedString(click.ParamType):
         return split
 
 
+class PydanticDateTime(click.ParamType):
+    name = "datetime"
+
+    def convert(
+        self,
+        value: Any,
+        param: Optional[click.Parameter] = None,
+        ctx: Optional[click.Context] = None,
+    ) -> Any:
+        try:
+            return TypeAdapter(datetime).validate_strings(value)
+        except ValidationError:
+            self.fail(f"{value} is not a valid datetime", param, ctx)
+
+
+class PydanticDate(click.ParamType):
+    name = "date"
+
+    def convert(
+        self,
+        value: Any,
+        param: Optional[click.Parameter] = None,
+        ctx: Optional[click.Context] = None,
+    ) -> Any:
+        try:
+            return TypeAdapter(date).validate_strings(value)
+        except ValidationError:
+            self.fail(f"{value} is not a valid date", param, ctx)
+
+
 BaseModelT = TypeVar("BaseModelT", bound=BaseModel)
 
 

unstructured_ingest/v2/cli/utils/model_conversion.py

@@ -25,7 +25,12 @@ from pydantic.fields import FieldInfo
 from pydantic.types import _SecretBase
 from pydantic_core import PydanticUndefined
 
-from unstructured_ingest.v2.cli.utils.click import DelimitedString, Dict
+from unstructured_ingest.v2.cli.utils.click import (
+    DelimitedString,
+    Dict,
+    PydanticDate,
+    PydanticDateTime,
+)
 
 NoneType = type(None)
 
@@ -135,8 +140,10 @@ def get_type_from_annotation(field_type: Any) -> click.ParamType:
         return click.UUID
     if field_type is Path:
         return click.Path(path_type=Path)
-    if field_type in (datetime.datetime, datetime.date):
-        return click.DateTime()
+    if field_type is datetime.datetime:
+        return PydanticDateTime()
+    if field_type is datetime.date:
+        return PydanticDate()
     if field_origin is Literal:
         return click.Choice(field_args)
     if isinstance(field_type, EnumMeta):

unstructured_ingest/v2/interfaces/file_data.py

@@ -43,6 +43,7 @@ class FileData(DataClassJsonMixin):
     additional_metadata: dict[str, Any] = field(default_factory=dict)
     reprocess: bool = False
     local_download_path: Optional[str] = None
+    display_name: Optional[str] = None
 
     @classmethod
     def from_file(cls, path: str) -> "FileData":

unstructured_ingest/v2/interfaces/indexer.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import Any, Generator, Optional, TypeVar
+from typing import Any, AsyncGenerator, Generator, Optional, TypeVar
 
 from pydantic import BaseModel
 
@@ -25,3 +25,6 @@ class Indexer(BaseProcess, BaseConnector, ABC):
     @abstractmethod
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
         pass
+
+    async def run_async(self, **kwargs: Any) -> AsyncGenerator[FileData, None]:
+        raise NotImplementedError()

unstructured_ingest/v2/pipeline/pipeline.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 import logging
 import multiprocessing as mp
 import shutil
@@ -186,6 +187,14 @@ class Pipeline:
         filtered_records = [r for r in records if r["file_data_path"] in filtered_file_data_paths]
         return filtered_records
 
+    def get_indices(self) -> list[dict]:
+        if self.indexer_step.process.is_async():
+            indices = asyncio.run(self.indexer_step.run_async())
+        else:
+            indices = self.indexer_step.run()
+        indices_inputs = [{"file_data_path": i} for i in indices]
+        return indices_inputs
+
     def _run(self):
         logger.info(
             f"running local pipeline: {self} with configs: " f"{self.context.model_dump_json()}"
@@ -197,8 +206,7 @@ class Pipeline:
             self.context.status = {}
 
         # Index into data source
-        indices = self.indexer_step.run()
-        indices_inputs = [{"file_data_path": i} for i in indices]
+        indices_inputs = self.get_indices()
         if not indices_inputs:
             logger.info("No files to process after indexer, exiting")
             return

unstructured_ingest/v2/pipeline/steps/index.py

@@ -1,7 +1,7 @@
 import hashlib
 import json
 from dataclasses import dataclass
-from typing import Generator, Optional, TypeVar
+from typing import AsyncGenerator, Generator, Optional, TypeVar
 
 from unstructured_ingest.v2.interfaces.indexer import Indexer
 from unstructured_ingest.v2.logger import logger
@@ -52,6 +52,23 @@ class IndexStep(PipelineStep):
                     raise e
                 continue
 
+    async def run_async(self) -> AsyncGenerator[str, None]:
+        async for file_data in self.process.run_async():
+            logger.debug(f"generated file data: {file_data.to_dict()}")
+            try:
+                record_hash = self.get_hash(extras=[file_data.identifier])
+                filename = f"{record_hash}.json"
+                filepath = (self.cache_dir / filename).resolve()
+                filepath.parent.mkdir(parents=True, exist_ok=True)
+                with open(str(filepath), "w") as f:
+                    json.dump(file_data.to_dict(), f, indent=2)
+                yield str(filepath)
+            except Exception as e:
+                logger.error(f"failed to create index for file data: {file_data}", exc_info=True)
+                if self.context.raise_on_error:
+                    raise e
+                continue
+
     def get_hash(self, extras: Optional[list[str]]) -> str:
         index_config_dict = json.loads(
             serialize_base_model_json(model=self.process.index_config, sort_keys=True)

unstructured_ingest/v2/processes/connectors/__init__.py

@@ -11,13 +11,15 @@ from unstructured_ingest.v2.processes.connector_registry import (
 from .airtable import CONNECTOR_TYPE as AIRTABLE_CONNECTOR_TYPE
 from .airtable import airtable_source_entry
 from .astradb import CONNECTOR_TYPE as ASTRA_DB_CONNECTOR_TYPE
-from .astradb import astra_db_destination_entry
+from .astradb import astra_db_destination_entry, astra_db_source_entry
 from .azure_cognitive_search import CONNECTOR_TYPE as AZURE_COGNTIVE_SEARCH_CONNECTOR_TYPE
 from .azure_cognitive_search import azure_cognitive_search_destination_entry
 from .chroma import CONNECTOR_TYPE as CHROMA_CONNECTOR_TYPE
 from .chroma import chroma_destination_entry
 from .couchbase import CONNECTOR_TYPE as COUCHBASE_CONNECTOR_TYPE
 from .couchbase import couchbase_destination_entry, couchbase_source_entry
+from .delta_table import CONNECTOR_TYPE as DELTA_TABLE_CONNECTOR_TYPE
+from .delta_table import delta_table_destination_entry
 from .elasticsearch import CONNECTOR_TYPE as ELASTICSEARCH_CONNECTOR_TYPE
 from .elasticsearch import elasticsearch_destination_entry, elasticsearch_source_entry
 from .google_drive import CONNECTOR_TYPE as GOOGLE_DRIVE_CONNECTOR_TYPE
@@ -42,11 +44,12 @@ from .salesforce import CONNECTOR_TYPE as SALESFORCE_CONNECTOR_TYPE
 from .salesforce import salesforce_source_entry
 from .sharepoint import CONNECTOR_TYPE as SHAREPOINT_CONNECTOR_TYPE
 from .sharepoint import sharepoint_source_entry
-from .singlestore import CONNECTOR_TYPE as SINGLESTORE_CONNECTOR_TYPE
-from .singlestore import singlestore_destination_entry
+from .slack import CONNECTOR_TYPE as SLACK_CONNECTOR_TYPE
+from .slack import slack_source_entry
 from .weaviate import CONNECTOR_TYPE as WEAVIATE_CONNECTOR_TYPE
 from .weaviate import weaviate_destination_entry
 
+add_source_entry(source_type=ASTRA_DB_CONNECTOR_TYPE, entry=astra_db_source_entry)
 add_destination_entry(destination_type=ASTRA_DB_CONNECTOR_TYPE, entry=astra_db_destination_entry)
 
 add_destination_entry(destination_type=CHROMA_CONNECTOR_TYPE, entry=chroma_destination_entry)
@@ -54,6 +57,10 @@ add_destination_entry(destination_type=CHROMA_CONNECTOR_TYPE, entry=chroma_desti
 add_source_entry(source_type=COUCHBASE_CONNECTOR_TYPE, entry=couchbase_source_entry)
 add_destination_entry(destination_type=COUCHBASE_CONNECTOR_TYPE, entry=couchbase_destination_entry)
 
+add_destination_entry(
+    destination_type=DELTA_TABLE_CONNECTOR_TYPE, entry=delta_table_destination_entry
+)
+
 add_source_entry(source_type=ELASTICSEARCH_CONNECTOR_TYPE, entry=elasticsearch_source_entry)
 add_destination_entry(
     destination_type=ELASTICSEARCH_CONNECTOR_TYPE, entry=elasticsearch_destination_entry
@@ -80,9 +87,7 @@ add_source_entry(source_type=MONGODB_CONNECTOR_TYPE, entry=mongodb_source_entry)
 
 add_destination_entry(destination_type=PINECONE_CONNECTOR_TYPE, entry=pinecone_destination_entry)
 add_source_entry(source_type=SHAREPOINT_CONNECTOR_TYPE, entry=sharepoint_source_entry)
-add_destination_entry(
-    destination_type=SINGLESTORE_CONNECTOR_TYPE, entry=singlestore_destination_entry
-)
+
 add_destination_entry(destination_type=MILVUS_CONNECTOR_TYPE, entry=milvus_destination_entry)
 add_destination_entry(
     destination_type=AZURE_COGNTIVE_SEARCH_CONNECTOR_TYPE,
@@ -93,3 +98,5 @@ add_destination_entry(destination_type=KDBAI_CONNECTOR_TYPE, entry=kdbai_destina
 add_source_entry(source_type=AIRTABLE_CONNECTOR_TYPE, entry=airtable_source_entry)
 
 add_source_entry(source_type=OUTLOOK_CONNECTOR_TYPE, entry=outlook_source_entry)
+
+add_source_entry(source_type=SLACK_CONNECTOR_TYPE, entry=slack_source_entry)