unstructured-ingest 0.1.1__tar.gz → 0.2.1__tar.gz

{unstructured_ingest-0.1.1 → unstructured_ingest-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.1.1
+Version: 0.2.1
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -23,12 +23,12 @@ Requires-Python: >=3.9.0,<3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
 Requires-Dist: tqdm
-Requires-Dist: python-dateutil
-Requires-Dist: dataclasses_json
-Requires-Dist: pandas
-Requires-Dist: pydantic>=2.7
 Requires-Dist: click
+Requires-Dist: pydantic>=2.7
+Requires-Dist: dataclasses_json
 Requires-Dist: opentelemetry-sdk
+Requires-Dist: pandas
+Requires-Dist: python-dateutil
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.25.8; extra == "remote"
 Provides-Extra: csv
@@ -86,19 +86,19 @@ Requires-Dist: atlassian-python-api; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
 Provides-Extra: delta-table
-Requires-Dist: deltalake; extra == "delta-table"
 Requires-Dist: fsspec; extra == "delta-table"
+Requires-Dist: deltalake; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord-py; extra == "discord"
 Provides-Extra: dropbox
-Requires-Dist: dropboxdrivefs; extra == "dropbox"
 Requires-Dist: fsspec; extra == "dropbox"
+Requires-Dist: dropboxdrivefs; extra == "dropbox"
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
 Provides-Extra: gcs
+Requires-Dist: fsspec; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: gcsfs; extra == "gcs"
-Requires-Dist: fsspec; extra == "gcs"
 Provides-Extra: github
 Requires-Dist: requests; extra == "github"
 Requires-Dist: pygithub>1.58.0; extra == "github"
@@ -107,8 +107,8 @@ Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: hubspot-api-client; extra == "hubspot"
 Requires-Dist: urllib3; extra == "hubspot"
+Requires-Dist: hubspot-api-client; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -120,13 +120,13 @@ Requires-Dist: pymilvus; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: notion
-Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
+Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
 Provides-Extra: onedrive
-Requires-Dist: bs4; extra == "onedrive"
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
+Requires-Dist: bs4; extra == "onedrive"
 Requires-Dist: msal; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
@@ -153,7 +153,9 @@ Provides-Extra: sftp
 Requires-Dist: paramiko; extra == "sftp"
 Requires-Dist: fsspec; extra == "sftp"
 Provides-Extra: slack
-Requires-Dist: slack_sdk; extra == "slack"
+Requires-Dist: slack_sdk[optional]; extra == "slack"
+Provides-Extra: snowflake
+Requires-Dist: snowflake; extra == "snowflake"
 Provides-Extra: wikipedia
 Requires-Dist: wikipedia; extra == "wikipedia"
 Provides-Extra: weaviate
@@ -167,8 +169,8 @@ Requires-Dist: requests; extra == "vectara"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
-Requires-Dist: tiktoken; extra == "embed-octoai"
 Requires-Dist: openai; extra == "embed-octoai"
+Requires-Dist: tiktoken; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
@@ -176,8 +178,8 @@ Requires-Dist: voyageai; extra == "embed-voyageai"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Provides-Extra: openai
-Requires-Dist: tiktoken; extra == "openai"
 Requires-Dist: openai; extra == "openai"
+Requires-Dist: tiktoken; extra == "openai"
 Provides-Extra: bedrock
 Requires-Dist: boto3; extra == "bedrock"
 Provides-Extra: togetherai

{unstructured_ingest-0.1.1 → unstructured_ingest-0.2.1}/setup.py

@@ -119,6 +119,7 @@ connectors_reqs = {
     "salesforce": load_requirements("requirements/connectors/salesforce.in"),
     "sftp": load_requirements("requirements/connectors/sftp.in"),
     "slack": load_requirements("requirements/connectors/slack.in"),
+    "snowflake": load_requirements("requirements/connectors/snowflake.in"),
     "wikipedia": load_requirements("requirements/connectors/wikipedia.in"),
     "weaviate": load_requirements("requirements/connectors/weaviate.in"),
     "databricks-volumes": load_requirements("requirements/connectors/databricks-volumes.in"),

unstructured_ingest-0.2.1/test/integration/connectors/conftest.py

@@ -0,0 +1,28 @@
+import tempfile
+from pathlib import Path
+from typing import Generator
+
+import pytest
+
+from unstructured_ingest.v2.logger import logger
+
+FILENAME = "DA-1p-with-duplicate-pages.pdf.json"
+
+
+@pytest.fixture
+def upload_file() -> Path:
+    int_test_dir = Path(__file__).parent
+    assets_dir = int_test_dir / "assets"
+    upload_file = assets_dir / FILENAME
+    assert upload_file.exists()
+    assert upload_file.is_file()
+    return upload_file
+
+
+@pytest.fixture
+def temp_dir() -> Generator[Path, None, None]:
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_path = Path(temp_dir)
+        logger.info(f"Created temp dir '{temp_path}'")
+        yield temp_path
+        logger.info(f"Removing temp dir '{temp_path}'")

{unstructured_ingest-0.1.1 → unstructured_ingest-0.2.1}/test/integration/connectors/databricks_tests/test_volumes_native.py

@@ -16,7 +16,7 @@ from test.integration.connectors.utils.validation import (
     source_connector_validation,
 )
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.databricks.volumes_native import (
     CONNECTOR_TYPE,
     DatabricksNativeVolumesAccessConfig,
@@ -139,7 +139,11 @@ def validate_upload(client: WorkspaceClient, catalog: str, volume: str, volume_p
 async def test_volumes_native_destination(upload_file: Path):
     env_data = get_env_data()
     volume_path = f"databricks-volumes-test-output-{uuid.uuid4()}"
-    mock_file_data = FileData(identifier="mock file data", connector_type=CONNECTOR_TYPE)
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
+        connector_type=CONNECTOR_TYPE,
+        identifier="mock file data",
+    )
     with databricks_destination_context(
         volume="test-platform", volume_path=volume_path, env_data=env_data
     ) as workspace_client:
@@ -153,9 +157,9 @@ async def test_volumes_native_destination(upload_file: Path):
             ),
         )
         if uploader.is_async():
-            await uploader.run_async(path=upload_file, file_data=mock_file_data)
+            await uploader.run_async(path=upload_file, file_data=file_data)
         else:
-            uploader.run(path=upload_file, file_data=mock_file_data)
+            uploader.run(path=upload_file, file_data=file_data)
 
         validate_upload(
             client=workspace_client,

{unstructured_ingest-0.1.1 → unstructured_ingest-0.2.1}/test/integration/connectors/sql/test_postgres.py

@@ -2,7 +2,6 @@ import tempfile
 from contextlib import contextmanager
 from pathlib import Path
 
-import faker
 import pandas as pd
 import pytest
 from psycopg2 import connect
@@ -26,9 +25,7 @@ from unstructured_ingest.v2.processes.connectors.sql.postgres import (
     PostgresUploadStager,
 )
 
-faker = faker.Faker()
-
-SEED_DATA_ROWS = 40
+SEED_DATA_ROWS = 20
 
 
 @contextmanager
@@ -42,11 +39,8 @@ def postgres_download_setup() -> None:
             port=5433,
         )
         with connection.cursor() as cursor:
-            for _ in range(SEED_DATA_ROWS):
-                sql_statment = (
-                    f"INSERT INTO cars (brand, price) VALUES "
-                    f"('{faker.word()}', {faker.random_int()})"
-                )
+            for i in range(SEED_DATA_ROWS):
+                sql_statment = f"INSERT INTO cars (brand, price) VALUES " f"('brand_{i}', {i})"
                 cursor.execute(sql_statment)
             connection.commit()
         yield
@@ -88,7 +82,9 @@ async def test_postgres_source():
                 downloader=downloader,
                 configs=ValidationConfigs(
                     test_id="postgres",
-                    expected_num_files=40,
+                    expected_num_files=SEED_DATA_ROWS,
+                    expected_number_indexed_file_data=4,
+                    validate_downloaded_files=True,
                 ),
             )
 

unstructured_ingest-0.2.1/test/integration/connectors/sql/test_singlestore.py

@@ -0,0 +1,156 @@
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+
+import pandas as pd
+import pytest
+import singlestoredb as s2
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG, env_setup_path
+from test.integration.connectors.utils.docker_compose import docker_compose_context
+from test.integration.connectors.utils.validation import (
+    ValidationConfigs,
+    source_connector_validation,
+)
+from unstructured_ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.processes.connectors.sql.singlestore import (
+    CONNECTOR_TYPE,
+    SingleStoreAccessConfig,
+    SingleStoreConnectionConfig,
+    SingleStoreDownloader,
+    SingleStoreDownloaderConfig,
+    SingleStoreIndexer,
+    SingleStoreIndexerConfig,
+    SingleStoreUploader,
+    SingleStoreUploaderConfig,
+    SingleStoreUploadStager,
+)
+
+SEED_DATA_ROWS = 20
+
+
+@contextmanager
+def singlestore_download_setup(connect_params: dict) -> None:
+    with docker_compose_context(
+        docker_compose_path=env_setup_path / "sql" / "singlestore" / "source"
+    ):
+        with s2.connect(**connect_params) as connection:
+            with connection.cursor() as cursor:
+                for i in range(SEED_DATA_ROWS):
+                    sql_statment = f"INSERT INTO cars (brand, price) VALUES " f"('brand_{i}', {i})"
+                    cursor.execute(sql_statment)
+                connection.commit()
+        yield
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, "sql")
+async def test_singlestore_source():
+    connect_params = {
+        "host": "localhost",
+        "port": 3306,
+        "database": "ingest_test",
+        "user": "root",
+        "password": "password",
+    }
+    with singlestore_download_setup(connect_params=connect_params):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            connection_config = SingleStoreConnectionConfig(
+                host=connect_params["host"],
+                port=connect_params["port"],
+                database=connect_params["database"],
+                user=connect_params["user"],
+                access_config=SingleStoreAccessConfig(password=connect_params["password"]),
+            )
+            indexer = SingleStoreIndexer(
+                connection_config=connection_config,
+                index_config=SingleStoreIndexerConfig(
+                    table_name="cars", id_column="car_id", batch_size=5
+                ),
+            )
+            downloader = SingleStoreDownloader(
+                connection_config=connection_config,
+                download_config=SingleStoreDownloaderConfig(
+                    fields=["car_id", "brand"], download_dir=Path(tmpdir)
+                ),
+            )
+            await source_connector_validation(
+                indexer=indexer,
+                downloader=downloader,
+                configs=ValidationConfigs(
+                    test_id="singlestore",
+                    expected_num_files=SEED_DATA_ROWS,
+                    expected_number_indexed_file_data=4,
+                    validate_downloaded_files=True,
+                ),
+            )
+
+
+def validate_destination(
+    connect_params: dict,
+    expected_num_elements: int,
+):
+    with s2.connect(**connect_params) as connection:
+        with connection.cursor() as cursor:
+            query = "select count(*) from elements;"
+            cursor.execute(query)
+            count = cursor.fetchone()[0]
+            assert (
+                count == expected_num_elements
+            ), f"dest check failed: got {count}, expected {expected_num_elements}"
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "sql")
+async def test_singlestore_destination(upload_file: Path):
+    mock_file_data = FileData(identifier="mock file data", connector_type=CONNECTOR_TYPE)
+    with docker_compose_context(
+        docker_compose_path=env_setup_path / "sql" / "singlestore" / "destination"
+    ):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            stager = SingleStoreUploadStager()
+            stager_params = {
+                "elements_filepath": upload_file,
+                "file_data": mock_file_data,
+                "output_dir": Path(tmpdir),
+                "output_filename": "test_db",
+            }
+            if stager.is_async():
+                staged_path = await stager.run_async(**stager_params)
+            else:
+                staged_path = stager.run(**stager_params)
+
+            # The stager should append the `.json` suffix to the output filename passed in.
+            assert staged_path.name == "test_db.json"
+
+            connect_params = {
+                "host": "localhost",
+                "port": 3306,
+                "database": "ingest_test",
+                "user": "root",
+                "password": "password",
+            }
+
+            uploader = SingleStoreUploader(
+                connection_config=SingleStoreConnectionConfig(
+                    host=connect_params["host"],
+                    port=connect_params["port"],
+                    database=connect_params["database"],
+                    user=connect_params["user"],
+                    access_config=SingleStoreAccessConfig(password=connect_params["password"]),
+                ),
+                upload_config=SingleStoreUploaderConfig(
+                    table_name="elements",
+                ),
+            )
+            if uploader.is_async():
+                await uploader.run_async(path=staged_path, file_data=mock_file_data)
+            else:
+                uploader.run(path=staged_path, file_data=mock_file_data)
+
+            staged_df = pd.read_json(staged_path, orient="records", lines=True)
+            expected_num_elements = len(staged_df)
+            validate_destination(
+                connect_params=connect_params,
+                expected_num_elements=expected_num_elements,
+            )

unstructured_ingest-0.2.1/test/integration/connectors/sql/test_snowflake.py

@@ -0,0 +1,205 @@
+import os
+import tempfile
+from pathlib import Path
+
+import docker
+import pandas as pd
+import pytest
+import snowflake.connector as sf
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG, env_setup_path
+from test.integration.connectors.utils.docker import container_context
+from test.integration.connectors.utils.validation import (
+    ValidationConfigs,
+    source_connector_validation,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.processes.connectors.sql.snowflake import (
+    CONNECTOR_TYPE,
+    SnowflakeAccessConfig,
+    SnowflakeConnectionConfig,
+    SnowflakeDownloader,
+    SnowflakeDownloaderConfig,
+    SnowflakeIndexer,
+    SnowflakeIndexerConfig,
+    SnowflakeUploader,
+    SnowflakeUploadStager,
+)
+
+SEED_DATA_ROWS = 20
+
+
+def seed_data():
+    conn = sf.connect(
+        user="test",
+        password="test",
+        account="test",
+        database="test",
+        host="snowflake.localhost.localstack.cloud",
+    )
+
+    file = Path(env_setup_path / "sql" / "snowflake" / "source" / "snowflake-schema.sql")
+
+    with file.open() as f:
+        sql = f.read()
+
+    cur = conn.cursor()
+    cur.execute(sql)
+    for i in range(SEED_DATA_ROWS):
+        sql_statment = f"INSERT INTO cars (brand, price) VALUES " f"('brand_{i}', {i})"
+        cur.execute(sql_statment)
+
+    cur.close()
+    conn.close()
+
+
+def init_db_destination():
+    conn = sf.connect(
+        user="test",
+        password="test",
+        account="test",
+        database="test",
+        host="snowflake.localhost.localstack.cloud",
+    )
+
+    file = Path(env_setup_path / "sql" / "snowflake" / "destination" / "snowflake-schema.sql")
+
+    with file.open() as f:
+        sql = f.read()
+
+    cur = conn.cursor()
+    cur.execute(sql)
+
+    cur.close()
+    conn.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, "sql")
+@requires_env("LOCALSTACK_AUTH_TOKEN")
+async def test_snowflake_source():
+    docker_client = docker.from_env()
+    token = os.getenv("LOCALSTACK_AUTH_TOKEN")
+    with container_context(
+        docker_client=docker_client,
+        image="localstack/snowflake",
+        environment={"LOCALSTACK_AUTH_TOKEN": token, "EXTRA_CORS_ALLOWED_ORIGINS": "*"},
+        ports={4566: 4566, 443: 443},
+        healthcheck_timeout=30,
+    ):
+        seed_data()
+        with tempfile.TemporaryDirectory() as tmpdir:
+            connection_config = SnowflakeConnectionConfig(
+                access_config=SnowflakeAccessConfig(password="test"),
+                account="test",
+                user="test",
+                database="test",
+                host="snowflake.localhost.localstack.cloud",
+            )
+            indexer = SnowflakeIndexer(
+                connection_config=connection_config,
+                index_config=SnowflakeIndexerConfig(
+                    table_name="cars", id_column="CAR_ID", batch_size=5
+                ),
+            )
+            downloader = SnowflakeDownloader(
+                connection_config=connection_config,
+                download_config=SnowflakeDownloaderConfig(
+                    fields=["CAR_ID", "BRAND"], download_dir=Path(tmpdir)
+                ),
+            )
+            await source_connector_validation(
+                indexer=indexer,
+                downloader=downloader,
+                configs=ValidationConfigs(
+                    test_id="snowflake",
+                    expected_num_files=SEED_DATA_ROWS,
+                    expected_number_indexed_file_data=4,
+                    validate_downloaded_files=True,
+                ),
+            )
+
+
+def validate_destination(
+    connect_params: dict,
+    expected_num_elements: int,
+):
+    # Run the following validations:
+    # * Check that the number of records in the table match the expected value
+    # * Given the embedding, make sure it matches the associated text it belongs to
+    conn = sf.connect(**connect_params)
+    cursor = conn.cursor()
+    try:
+        query = "select count(*) from elements;"
+        cursor.execute(query)
+        count = cursor.fetchone()[0]
+        assert (
+            count == expected_num_elements
+        ), f"dest check failed: got {count}, expected {expected_num_elements}"
+    finally:
+        cursor.close()
+        conn.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "sql")
+@requires_env("LOCALSTACK_AUTH_TOKEN")
+async def test_snowflake_destination(upload_file: Path):
+    # the postgres destination connector doesn't leverage the file data but is required as an input,
+    # mocking it with arbitrary values to meet the base requirements:
+    mock_file_data = FileData(identifier="mock file data", connector_type=CONNECTOR_TYPE)
+    docker_client = docker.from_env()
+    token = os.getenv("LOCALSTACK_AUTH_TOKEN")
+    with container_context(
+        docker_client=docker_client,
+        image="localstack/snowflake",
+        environment={"LOCALSTACK_AUTH_TOKEN": token, "EXTRA_CORS_ALLOWED_ORIGINS": "*"},
+        ports={4566: 4566, 443: 443},
+        healthcheck_timeout=30,
+    ):
+        init_db_destination()
+        with tempfile.TemporaryDirectory() as tmpdir:
+            stager = SnowflakeUploadStager()
+            stager_params = {
+                "elements_filepath": upload_file,
+                "file_data": mock_file_data,
+                "output_dir": Path(tmpdir),
+                "output_filename": "test_db",
+            }
+            if stager.is_async():
+                staged_path = await stager.run_async(**stager_params)
+            else:
+                staged_path = stager.run(**stager_params)
+
+            # The stager should append the `.json` suffix to the output filename passed in.
+            assert staged_path.name == "test_db.json"
+
+            connect_params = {
+                "user": "test",
+                "password": "test",
+                "account": "test",
+                "database": "test",
+                "host": "snowflake.localhost.localstack.cloud",
+            }
+
+            uploader = SnowflakeUploader(
+                connection_config=SnowflakeConnectionConfig(
+                    access_config=SnowflakeAccessConfig(password=connect_params["password"]),
+                    account=connect_params["account"],
+                    user=connect_params["user"],
+                    database=connect_params["database"],
+                    host=connect_params["host"],
+                )
+            )
+            if uploader.is_async():
+                await uploader.run_async(path=staged_path, file_data=mock_file_data)
+            else:
+                uploader.run(path=staged_path, file_data=mock_file_data)
+
+            staged_df = pd.read_json(staged_path, orient="records", lines=True)
+            expected_num_elements = len(staged_df)
+            validate_destination(
+                connect_params=connect_params,
+                expected_num_elements=expected_num_elements,
+            )

{unstructured_ingest-0.1.1 → unstructured_ingest-0.2.1}/test/integration/connectors/sql/test_sqlite.py

@@ -3,7 +3,6 @@ import tempfile
 from contextlib import contextmanager
 from pathlib import Path
 
-import faker
 import pandas as pd
 import pytest
 
@@ -24,9 +23,7 @@ from unstructured_ingest.v2.processes.connectors.sql.sqlite import (
     SQLiteUploadStager,
 )
 
-faker = faker.Faker()
-
-SEED_DATA_ROWS = 40
+SEED_DATA_ROWS = 20
 
 
 @contextmanager
@@ -41,11 +38,8 @@ def sqlite_download_setup() -> Path:
             with db_init_path.open("r") as f:
                 query = f.read()
             cursor.executescript(query)
-            for _ in range(SEED_DATA_ROWS):
-                sql_statment = (
-                    f"INSERT INTO cars (brand, price) "
-                    f"VALUES ('{faker.word()}', {faker.random_int()})"
-                )
+            for i in range(SEED_DATA_ROWS):
+                sql_statment = f"INSERT INTO cars (brand, price) " f"VALUES ('brand{i}', {i})"
                 cursor.execute(sql_statment)
 
             sqlite_connection.commit()
@@ -76,7 +70,9 @@ async def test_sqlite_source():
                 downloader=downloader,
                 configs=ValidationConfigs(
                     test_id="sqlite",
-                    expected_num_files=40,
+                    expected_num_files=SEED_DATA_ROWS,
+                    expected_number_indexed_file_data=4,
+                    validate_downloaded_files=True,
                 ),
             )