unstructured-ingest 0.2.0__tar.gz → 0.2.1__tar.gz

{unstructured_ingest-0.2.0 → unstructured_ingest-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.2.0
+Version: 0.2.1
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,12 +22,12 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: pydantic>=2.7
-Requires-Dist: pandas
 Requires-Dist: tqdm
-Requires-Dist: opentelemetry-sdk
 Requires-Dist: click
+Requires-Dist: pydantic>=2.7
 Requires-Dist: dataclasses_json
+Requires-Dist: opentelemetry-sdk
+Requires-Dist: pandas
 Requires-Dist: python-dateutil
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.25.8; extra == "remote"
@@ -74,8 +74,8 @@ Provides-Extra: biomed
 Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: fsspec; extra == "box"
 Requires-Dist: boxfs; extra == "box"
+Requires-Dist: fsspec; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
@@ -97,8 +97,8 @@ Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
 Provides-Extra: gcs
 Requires-Dist: fsspec; extra == "gcs"
-Requires-Dist: gcsfs; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
+Requires-Dist: gcsfs; extra == "gcs"
 Provides-Extra: github
 Requires-Dist: requests; extra == "github"
 Requires-Dist: pygithub>1.58.0; extra == "github"
@@ -120,14 +120,14 @@ Requires-Dist: pymilvus; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: notion
-Requires-Dist: httpx; extra == "notion"
-Requires-Dist: notion-client; extra == "notion"
-Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
+Requires-Dist: htmlBuilder; extra == "notion"
+Requires-Dist: notion-client; extra == "notion"
+Requires-Dist: httpx; extra == "notion"
 Provides-Extra: onedrive
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
-Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
+Requires-Dist: msal; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: outlook
@@ -142,16 +142,16 @@ Requires-Dist: qdrant-client; extra == "qdrant"
 Provides-Extra: reddit
 Requires-Dist: praw; extra == "reddit"
 Provides-Extra: s3
-Requires-Dist: fsspec; extra == "s3"
 Requires-Dist: s3fs; extra == "s3"
+Requires-Dist: fsspec; extra == "s3"
 Provides-Extra: sharepoint
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Requires-Dist: msal; extra == "sharepoint"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: fsspec; extra == "sftp"
 Requires-Dist: paramiko; extra == "sftp"
+Requires-Dist: fsspec; extra == "sftp"
 Provides-Extra: slack
 Requires-Dist: slack_sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
@@ -169,8 +169,8 @@ Requires-Dist: requests; extra == "vectara"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
-Requires-Dist: tiktoken; extra == "embed-octoai"
 Requires-Dist: openai; extra == "embed-octoai"
+Requires-Dist: tiktoken; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
@@ -178,8 +178,8 @@ Requires-Dist: voyageai; extra == "embed-voyageai"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Provides-Extra: openai
-Requires-Dist: tiktoken; extra == "openai"
 Requires-Dist: openai; extra == "openai"
+Requires-Dist: tiktoken; extra == "openai"
 Provides-Extra: bedrock
 Requires-Dist: boto3; extra == "bedrock"
 Provides-Extra: togetherai

unstructured_ingest-0.2.1/test/integration/connectors/sql/test_singlestore.py

@@ -0,0 +1,156 @@
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+
+import pandas as pd
+import pytest
+import singlestoredb as s2
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG, env_setup_path
+from test.integration.connectors.utils.docker_compose import docker_compose_context
+from test.integration.connectors.utils.validation import (
+    ValidationConfigs,
+    source_connector_validation,
+)
+from unstructured_ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.processes.connectors.sql.singlestore import (
+    CONNECTOR_TYPE,
+    SingleStoreAccessConfig,
+    SingleStoreConnectionConfig,
+    SingleStoreDownloader,
+    SingleStoreDownloaderConfig,
+    SingleStoreIndexer,
+    SingleStoreIndexerConfig,
+    SingleStoreUploader,
+    SingleStoreUploaderConfig,
+    SingleStoreUploadStager,
+)
+
+SEED_DATA_ROWS = 20
+
+
+@contextmanager
+def singlestore_download_setup(connect_params: dict) -> None:
+    with docker_compose_context(
+        docker_compose_path=env_setup_path / "sql" / "singlestore" / "source"
+    ):
+        with s2.connect(**connect_params) as connection:
+            with connection.cursor() as cursor:
+                for i in range(SEED_DATA_ROWS):
+                    sql_statment = f"INSERT INTO cars (brand, price) VALUES " f"('brand_{i}', {i})"
+                    cursor.execute(sql_statment)
+                connection.commit()
+        yield
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, "sql")
+async def test_singlestore_source():
+    connect_params = {
+        "host": "localhost",
+        "port": 3306,
+        "database": "ingest_test",
+        "user": "root",
+        "password": "password",
+    }
+    with singlestore_download_setup(connect_params=connect_params):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            connection_config = SingleStoreConnectionConfig(
+                host=connect_params["host"],
+                port=connect_params["port"],
+                database=connect_params["database"],
+                user=connect_params["user"],
+                access_config=SingleStoreAccessConfig(password=connect_params["password"]),
+            )
+            indexer = SingleStoreIndexer(
+                connection_config=connection_config,
+                index_config=SingleStoreIndexerConfig(
+                    table_name="cars", id_column="car_id", batch_size=5
+                ),
+            )
+            downloader = SingleStoreDownloader(
+                connection_config=connection_config,
+                download_config=SingleStoreDownloaderConfig(
+                    fields=["car_id", "brand"], download_dir=Path(tmpdir)
+                ),
+            )
+            await source_connector_validation(
+                indexer=indexer,
+                downloader=downloader,
+                configs=ValidationConfigs(
+                    test_id="singlestore",
+                    expected_num_files=SEED_DATA_ROWS,
+                    expected_number_indexed_file_data=4,
+                    validate_downloaded_files=True,
+                ),
+            )
+
+
+def validate_destination(
+    connect_params: dict,
+    expected_num_elements: int,
+):
+    with s2.connect(**connect_params) as connection:
+        with connection.cursor() as cursor:
+            query = "select count(*) from elements;"
+            cursor.execute(query)
+            count = cursor.fetchone()[0]
+            assert (
+                count == expected_num_elements
+            ), f"dest check failed: got {count}, expected {expected_num_elements}"
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "sql")
+async def test_singlestore_destination(upload_file: Path):
+    mock_file_data = FileData(identifier="mock file data", connector_type=CONNECTOR_TYPE)
+    with docker_compose_context(
+        docker_compose_path=env_setup_path / "sql" / "singlestore" / "destination"
+    ):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            stager = SingleStoreUploadStager()
+            stager_params = {
+                "elements_filepath": upload_file,
+                "file_data": mock_file_data,
+                "output_dir": Path(tmpdir),
+                "output_filename": "test_db",
+            }
+            if stager.is_async():
+                staged_path = await stager.run_async(**stager_params)
+            else:
+                staged_path = stager.run(**stager_params)
+
+            # The stager should append the `.json` suffix to the output filename passed in.
+            assert staged_path.name == "test_db.json"
+
+            connect_params = {
+                "host": "localhost",
+                "port": 3306,
+                "database": "ingest_test",
+                "user": "root",
+                "password": "password",
+            }
+
+            uploader = SingleStoreUploader(
+                connection_config=SingleStoreConnectionConfig(
+                    host=connect_params["host"],
+                    port=connect_params["port"],
+                    database=connect_params["database"],
+                    user=connect_params["user"],
+                    access_config=SingleStoreAccessConfig(password=connect_params["password"]),
+                ),
+                upload_config=SingleStoreUploaderConfig(
+                    table_name="elements",
+                ),
+            )
+            if uploader.is_async():
+                await uploader.run_async(path=staged_path, file_data=mock_file_data)
+            else:
+                uploader.run(path=staged_path, file_data=mock_file_data)
+
+            staged_df = pd.read_json(staged_path, orient="records", lines=True)
+            expected_num_elements = len(staged_df)
+            validate_destination(
+                connect_params=connect_params,
+                expected_num_elements=expected_num_elements,
+            )

{unstructured_ingest-0.2.0 → unstructured_ingest-0.2.1}/test/integration/connectors/test_s3.py

@@ -85,7 +85,7 @@ async def test_s3_source_no_access(anon_connection_config: S3ConnectionConfig):
 async def test_s3_minio_source(anon_connection_config: S3ConnectionConfig):
     anon_connection_config.endpoint_url = "http://localhost:9000"
     indexer_config = S3IndexerConfig(remote_url="s3://utic-dev-tech-fixtures/")
-    with docker_compose_context(docker_compose_path=env_setup_path / "minio"):
+    with docker_compose_context(docker_compose_path=env_setup_path / "minio" / "source"):
         with tempfile.TemporaryDirectory() as tempdir:
             tempdir_path = Path(tempdir)
             download_config = S3DownloaderConfig(download_dir=tempdir_path)

{unstructured_ingest-0.2.0 → unstructured_ingest-0.2.1}/test/integration/connectors/utils/docker_compose.py

@@ -3,6 +3,23 @@ from contextlib import contextmanager
 from pathlib import Path
 
 
+def docker_compose_down(docker_compose_path: Path):
+    cmd = f"docker compose -f {docker_compose_path.resolve()} down --remove-orphans -v --rmi all"
+    print(f"Running command: {cmd}")
+    final_resp = subprocess.run(
+        cmd,
+        shell=True,
+        capture_output=True,
+    )
+    if final_resp.returncode != 0:
+        print("STDOUT: {}".format(final_resp.stdout.decode("utf-8")))
+        print("STDERR: {}".format(final_resp.stderr.decode("utf-8")))
+
+
+def run_cleanup(docker_compose_path: Path):
+    docker_compose_down(docker_compose_path=docker_compose_path)
+
+
 @contextmanager
 def docker_compose_context(docker_compose_path: Path):
     # Dynamically run a specific docker compose file and make sure it gets cleanup by
@@ -30,15 +47,13 @@ def docker_compose_context(docker_compose_path: Path):
         if resp:
             print("STDOUT: {}".format(resp.stdout.decode("utf-8")))
             print("STDERR: {}".format(resp.stderr.decode("utf-8")))
-        raise e
-    finally:
-        cmd = f"docker compose -f {docker_compose_path.resolve()} down --remove-orphans -v"
-        print(f"Running command: {cmd}")
-        final_resp = subprocess.run(
+        cmd = f"docker compose -f {docker_compose_path.resolve()} logs"
+        logs = subprocess.run(
             cmd,
             shell=True,
             capture_output=True,
         )
-        if final_resp.returncode != 0:
-            print("STDOUT: {}".format(final_resp.stdout.decode("utf-8")))
-            print("STDERR: {}".format(final_resp.stderr.decode("utf-8")))
+        print("DOCKER LOGS: {}".format(logs.stdout.decode("utf-8")))
+        raise e
+    finally:
+        run_cleanup(docker_compose_path=docker_compose_path)

unstructured_ingest-0.2.1/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.2.1"  # pragma: no cover

{unstructured_ingest-0.2.0 → unstructured_ingest-0.2.1}/unstructured_ingest/v2/interfaces/file_data.py

@@ -43,6 +43,7 @@ class FileData(DataClassJsonMixin):
     additional_metadata: dict[str, Any] = field(default_factory=dict)
     reprocess: bool = False
     local_download_path: Optional[str] = None
+    display_name: Optional[str] = None
 
     @classmethod
     def from_file(cls, path: str) -> "FileData":

{unstructured_ingest-0.2.0 → unstructured_ingest-0.2.1}/unstructured_ingest/v2/processes/connectors/__init__.py

@@ -11,7 +11,7 @@ from unstructured_ingest.v2.processes.connector_registry import (
 from .airtable import CONNECTOR_TYPE as AIRTABLE_CONNECTOR_TYPE
 from .airtable import airtable_source_entry
 from .astradb import CONNECTOR_TYPE as ASTRA_DB_CONNECTOR_TYPE
-from .astradb import astra_db_destination_entry
+from .astradb import astra_db_destination_entry, astra_db_source_entry
 from .azure_cognitive_search import CONNECTOR_TYPE as AZURE_COGNTIVE_SEARCH_CONNECTOR_TYPE
 from .azure_cognitive_search import azure_cognitive_search_destination_entry
 from .chroma import CONNECTOR_TYPE as CHROMA_CONNECTOR_TYPE
@@ -44,13 +44,12 @@ from .salesforce import CONNECTOR_TYPE as SALESFORCE_CONNECTOR_TYPE
 from .salesforce import salesforce_source_entry
 from .sharepoint import CONNECTOR_TYPE as SHAREPOINT_CONNECTOR_TYPE
 from .sharepoint import sharepoint_source_entry
-from .singlestore import CONNECTOR_TYPE as SINGLESTORE_CONNECTOR_TYPE
-from .singlestore import singlestore_destination_entry
 from .slack import CONNECTOR_TYPE as SLACK_CONNECTOR_TYPE
 from .slack import slack_source_entry
 from .weaviate import CONNECTOR_TYPE as WEAVIATE_CONNECTOR_TYPE
 from .weaviate import weaviate_destination_entry
 
+add_source_entry(source_type=ASTRA_DB_CONNECTOR_TYPE, entry=astra_db_source_entry)
 add_destination_entry(destination_type=ASTRA_DB_CONNECTOR_TYPE, entry=astra_db_destination_entry)
 
 add_destination_entry(destination_type=CHROMA_CONNECTOR_TYPE, entry=chroma_destination_entry)
@@ -88,9 +87,7 @@ add_source_entry(source_type=MONGODB_CONNECTOR_TYPE, entry=mongodb_source_entry)
 
 add_destination_entry(destination_type=PINECONE_CONNECTOR_TYPE, entry=pinecone_destination_entry)
 add_source_entry(source_type=SHAREPOINT_CONNECTOR_TYPE, entry=sharepoint_source_entry)
-add_destination_entry(
-    destination_type=SINGLESTORE_CONNECTOR_TYPE, entry=singlestore_destination_entry
-)
+
 add_destination_entry(destination_type=MILVUS_CONNECTOR_TYPE, entry=milvus_destination_entry)
 add_destination_entry(
     destination_type=AZURE_COGNTIVE_SEARCH_CONNECTOR_TYPE,