unstructured-ingest 0.1.0__tar.gz → 0.2.0__tar.gz

{unstructured_ingest-0.1.0 → unstructured_ingest-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.1.0
+Version: 0.2.0
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,13 +22,13 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: pandas
 Requires-Dist: pydantic>=2.7
+Requires-Dist: pandas
 Requires-Dist: tqdm
-Requires-Dist: dataclasses_json
 Requires-Dist: opentelemetry-sdk
-Requires-Dist: python-dateutil
 Requires-Dist: click
+Requires-Dist: dataclasses_json
+Requires-Dist: python-dateutil
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.25.8; extra == "remote"
 Provides-Extra: csv
@@ -91,13 +91,13 @@ Requires-Dist: deltalake; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord-py; extra == "discord"
 Provides-Extra: dropbox
-Requires-Dist: dropboxdrivefs; extra == "dropbox"
 Requires-Dist: fsspec; extra == "dropbox"
+Requires-Dist: dropboxdrivefs; extra == "dropbox"
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
 Provides-Extra: gcs
-Requires-Dist: gcsfs; extra == "gcs"
 Requires-Dist: fsspec; extra == "gcs"
+Requires-Dist: gcsfs; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
 Provides-Extra: github
 Requires-Dist: requests; extra == "github"
@@ -107,32 +107,32 @@ Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: hubspot-api-client; extra == "hubspot"
 Requires-Dist: urllib3; extra == "hubspot"
+Requires-Dist: hubspot-api-client; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
 Requires-Dist: confluent-kafka; extra == "kafka"
 Provides-Extra: kdbai
-Requires-Dist: kdbai-client; extra == "kdbai"
+Requires-Dist: kdbai-client>=1.4.0; extra == "kdbai"
 Provides-Extra: milvus
 Requires-Dist: pymilvus; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: notion
-Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
-Requires-Dist: backoff; extra == "notion"
+Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
+Requires-Dist: backoff; extra == "notion"
 Provides-Extra: onedrive
-Requires-Dist: bs4; extra == "onedrive"
-Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
+Requires-Dist: msal; extra == "onedrive"
+Requires-Dist: bs4; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: outlook
-Requires-Dist: msal; extra == "outlook"
 Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
+Requires-Dist: msal; extra == "outlook"
 Provides-Extra: pinecone
 Requires-Dist: pinecone-client>=3.7.1; extra == "pinecone"
 Provides-Extra: postgres
@@ -142,18 +142,20 @@ Requires-Dist: qdrant-client; extra == "qdrant"
 Provides-Extra: reddit
 Requires-Dist: praw; extra == "reddit"
 Provides-Extra: s3
-Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: fsspec; extra == "s3"
+Requires-Dist: s3fs; extra == "s3"
 Provides-Extra: sharepoint
-Requires-Dist: msal; extra == "sharepoint"
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
+Requires-Dist: msal; extra == "sharepoint"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
 Requires-Dist: fsspec; extra == "sftp"
 Requires-Dist: paramiko; extra == "sftp"
 Provides-Extra: slack
-Requires-Dist: slack_sdk; extra == "slack"
+Requires-Dist: slack_sdk[optional]; extra == "slack"
+Provides-Extra: snowflake
+Requires-Dist: snowflake; extra == "snowflake"
 Provides-Extra: wikipedia
 Requires-Dist: wikipedia; extra == "wikipedia"
 Provides-Extra: weaviate

{unstructured_ingest-0.1.0 → unstructured_ingest-0.2.0}/setup.py

@@ -119,6 +119,7 @@ connectors_reqs = {
     "salesforce": load_requirements("requirements/connectors/salesforce.in"),
     "sftp": load_requirements("requirements/connectors/sftp.in"),
     "slack": load_requirements("requirements/connectors/slack.in"),
+    "snowflake": load_requirements("requirements/connectors/snowflake.in"),
     "wikipedia": load_requirements("requirements/connectors/wikipedia.in"),
     "weaviate": load_requirements("requirements/connectors/weaviate.in"),
     "databricks-volumes": load_requirements("requirements/connectors/databricks-volumes.in"),

unstructured_ingest-0.2.0/test/integration/connectors/conftest.py

@@ -0,0 +1,28 @@
+import tempfile
+from pathlib import Path
+from typing import Generator
+
+import pytest
+
+from unstructured_ingest.v2.logger import logger
+
+FILENAME = "DA-1p-with-duplicate-pages.pdf.json"
+
+
+@pytest.fixture
+def upload_file() -> Path:
+    int_test_dir = Path(__file__).parent
+    assets_dir = int_test_dir / "assets"
+    upload_file = assets_dir / FILENAME
+    assert upload_file.exists()
+    assert upload_file.is_file()
+    return upload_file
+
+
+@pytest.fixture
+def temp_dir() -> Generator[Path, None, None]:
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_path = Path(temp_dir)
+        logger.info(f"Created temp dir '{temp_path}'")
+        yield temp_path
+        logger.info(f"Removing temp dir '{temp_path}'")

{unstructured_ingest-0.1.0 → unstructured_ingest-0.2.0}/test/integration/connectors/databricks_tests/test_volumes_native.py

@@ -16,7 +16,7 @@ from test.integration.connectors.utils.validation import (
     source_connector_validation,
 )
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.databricks.volumes_native import (
     CONNECTOR_TYPE,
     DatabricksNativeVolumesAccessConfig,
@@ -139,7 +139,11 @@ def validate_upload(client: WorkspaceClient, catalog: str, volume: str, volume_p
 async def test_volumes_native_destination(upload_file: Path):
     env_data = get_env_data()
     volume_path = f"databricks-volumes-test-output-{uuid.uuid4()}"
-    mock_file_data = FileData(identifier="mock file data", connector_type=CONNECTOR_TYPE)
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
+        connector_type=CONNECTOR_TYPE,
+        identifier="mock file data",
+    )
     with databricks_destination_context(
         volume="test-platform", volume_path=volume_path, env_data=env_data
     ) as workspace_client:
@@ -153,9 +157,9 @@ async def test_volumes_native_destination(upload_file: Path):
             ),
         )
         if uploader.is_async():
-            await uploader.run_async(path=upload_file, file_data=mock_file_data)
+            await uploader.run_async(path=upload_file, file_data=file_data)
         else:
-            uploader.run(path=upload_file, file_data=mock_file_data)
+            uploader.run(path=upload_file, file_data=file_data)
 
         validate_upload(
             client=workspace_client,

{unstructured_ingest-0.1.0/test/integration/connectors → unstructured_ingest-0.2.0/test/integration/connectors/sql}/test_postgres.py

@@ -1,21 +1,93 @@
 import tempfile
+from contextlib import contextmanager
 from pathlib import Path
 
 import pandas as pd
 import pytest
 from psycopg2 import connect
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG, env_setup_path
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG, env_setup_path
 from test.integration.connectors.utils.docker_compose import docker_compose_context
+from test.integration.connectors.utils.validation import (
+    ValidationConfigs,
+    source_connector_validation,
+)
 from unstructured_ingest.v2.interfaces import FileData
 from unstructured_ingest.v2.processes.connectors.sql.postgres import (
     CONNECTOR_TYPE,
     PostgresAccessConfig,
     PostgresConnectionConfig,
+    PostgresDownloader,
+    PostgresDownloaderConfig,
+    PostgresIndexer,
+    PostgresIndexerConfig,
     PostgresUploader,
     PostgresUploadStager,
 )
 
+SEED_DATA_ROWS = 20
+
+
+@contextmanager
+def postgres_download_setup() -> None:
+    with docker_compose_context(docker_compose_path=env_setup_path / "sql" / "postgres" / "source"):
+        connection = connect(
+            user="unstructured",
+            password="test",
+            dbname="test_db",
+            host="localhost",
+            port=5433,
+        )
+        with connection.cursor() as cursor:
+            for i in range(SEED_DATA_ROWS):
+                sql_statment = f"INSERT INTO cars (brand, price) VALUES " f"('brand_{i}', {i})"
+                cursor.execute(sql_statment)
+            connection.commit()
+        yield
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, "sql")
+async def test_postgres_source():
+    connect_params = {
+        "host": "localhost",
+        "port": 5433,
+        "database": "test_db",
+        "user": "unstructured",
+        "password": "test",
+    }
+    with postgres_download_setup():
+        with tempfile.TemporaryDirectory() as tmpdir:
+            connection_config = PostgresConnectionConfig(
+                host=connect_params["host"],
+                port=connect_params["port"],
+                database=connect_params["database"],
+                username=connect_params["user"],
+                access_config=PostgresAccessConfig(password=connect_params["password"]),
+            )
+            indexer = PostgresIndexer(
+                connection_config=connection_config,
+                index_config=PostgresIndexerConfig(
+                    table_name="cars", id_column="car_id", batch_size=5
+                ),
+            )
+            downloader = PostgresDownloader(
+                connection_config=connection_config,
+                download_config=PostgresDownloaderConfig(
+                    fields=["car_id", "brand"], download_dir=Path(tmpdir)
+                ),
+            )
+            await source_connector_validation(
+                indexer=indexer,
+                downloader=downloader,
+                configs=ValidationConfigs(
+                    test_id="postgres",
+                    expected_num_files=SEED_DATA_ROWS,
+                    expected_number_indexed_file_data=4,
+                    validate_downloaded_files=True,
+                ),
+            )
+
 
 def validate_destination(
     connect_params: dict,
@@ -50,7 +122,9 @@ async def test_postgres_destination(upload_file: Path):
     # the postgres destination connector doesn't leverage the file data but is required as an input,
     # mocking it with arbitrary values to meet the base requirements:
     mock_file_data = FileData(identifier="mock file data", connector_type=CONNECTOR_TYPE)
-    with docker_compose_context(docker_compose_path=env_setup_path / "sql"):
+    with docker_compose_context(
+        docker_compose_path=env_setup_path / "sql" / "postgres" / "destination"
+    ):
         with tempfile.TemporaryDirectory() as tmpdir:
             stager = PostgresUploadStager()
             stager_params = {

unstructured_ingest-0.2.0/test/integration/connectors/sql/test_snowflake.py

@@ -0,0 +1,205 @@
+import os
+import tempfile
+from pathlib import Path
+
+import docker
+import pandas as pd
+import pytest
+import snowflake.connector as sf
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG, env_setup_path
+from test.integration.connectors.utils.docker import container_context
+from test.integration.connectors.utils.validation import (
+    ValidationConfigs,
+    source_connector_validation,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.processes.connectors.sql.snowflake import (
+    CONNECTOR_TYPE,
+    SnowflakeAccessConfig,
+    SnowflakeConnectionConfig,
+    SnowflakeDownloader,
+    SnowflakeDownloaderConfig,
+    SnowflakeIndexer,
+    SnowflakeIndexerConfig,
+    SnowflakeUploader,
+    SnowflakeUploadStager,
+)
+
+SEED_DATA_ROWS = 20
+
+
+def seed_data():
+    conn = sf.connect(
+        user="test",
+        password="test",
+        account="test",
+        database="test",
+        host="snowflake.localhost.localstack.cloud",
+    )
+
+    file = Path(env_setup_path / "sql" / "snowflake" / "source" / "snowflake-schema.sql")
+
+    with file.open() as f:
+        sql = f.read()
+
+    cur = conn.cursor()
+    cur.execute(sql)
+    for i in range(SEED_DATA_ROWS):
+        sql_statment = f"INSERT INTO cars (brand, price) VALUES " f"('brand_{i}', {i})"
+        cur.execute(sql_statment)
+
+    cur.close()
+    conn.close()
+
+
+def init_db_destination():
+    conn = sf.connect(
+        user="test",
+        password="test",
+        account="test",
+        database="test",
+        host="snowflake.localhost.localstack.cloud",
+    )
+
+    file = Path(env_setup_path / "sql" / "snowflake" / "destination" / "snowflake-schema.sql")
+
+    with file.open() as f:
+        sql = f.read()
+
+    cur = conn.cursor()
+    cur.execute(sql)
+
+    cur.close()
+    conn.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, "sql")
+@requires_env("LOCALSTACK_AUTH_TOKEN")
+async def test_snowflake_source():
+    docker_client = docker.from_env()
+    token = os.getenv("LOCALSTACK_AUTH_TOKEN")
+    with container_context(
+        docker_client=docker_client,
+        image="localstack/snowflake",
+        environment={"LOCALSTACK_AUTH_TOKEN": token, "EXTRA_CORS_ALLOWED_ORIGINS": "*"},
+        ports={4566: 4566, 443: 443},
+        healthcheck_timeout=30,
+    ):
+        seed_data()
+        with tempfile.TemporaryDirectory() as tmpdir:
+            connection_config = SnowflakeConnectionConfig(
+                access_config=SnowflakeAccessConfig(password="test"),
+                account="test",
+                user="test",
+                database="test",
+                host="snowflake.localhost.localstack.cloud",
+            )
+            indexer = SnowflakeIndexer(
+                connection_config=connection_config,
+                index_config=SnowflakeIndexerConfig(
+                    table_name="cars", id_column="CAR_ID", batch_size=5
+                ),
+            )
+            downloader = SnowflakeDownloader(
+                connection_config=connection_config,
+                download_config=SnowflakeDownloaderConfig(
+                    fields=["CAR_ID", "BRAND"], download_dir=Path(tmpdir)
+                ),
+            )
+            await source_connector_validation(
+                indexer=indexer,
+                downloader=downloader,
+                configs=ValidationConfigs(
+                    test_id="snowflake",
+                    expected_num_files=SEED_DATA_ROWS,
+                    expected_number_indexed_file_data=4,
+                    validate_downloaded_files=True,
+                ),
+            )
+
+
+def validate_destination(
+    connect_params: dict,
+    expected_num_elements: int,
+):
+    # Run the following validations:
+    # * Check that the number of records in the table match the expected value
+    # * Given the embedding, make sure it matches the associated text it belongs to
+    conn = sf.connect(**connect_params)
+    cursor = conn.cursor()
+    try:
+        query = "select count(*) from elements;"
+        cursor.execute(query)
+        count = cursor.fetchone()[0]
+        assert (
+            count == expected_num_elements
+        ), f"dest check failed: got {count}, expected {expected_num_elements}"
+    finally:
+        cursor.close()
+        conn.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "sql")
+@requires_env("LOCALSTACK_AUTH_TOKEN")
+async def test_snowflake_destination(upload_file: Path):
+    # the postgres destination connector doesn't leverage the file data but is required as an input,
+    # mocking it with arbitrary values to meet the base requirements:
+    mock_file_data = FileData(identifier="mock file data", connector_type=CONNECTOR_TYPE)
+    docker_client = docker.from_env()
+    token = os.getenv("LOCALSTACK_AUTH_TOKEN")
+    with container_context(
+        docker_client=docker_client,
+        image="localstack/snowflake",
+        environment={"LOCALSTACK_AUTH_TOKEN": token, "EXTRA_CORS_ALLOWED_ORIGINS": "*"},
+        ports={4566: 4566, 443: 443},
+        healthcheck_timeout=30,
+    ):
+        init_db_destination()
+        with tempfile.TemporaryDirectory() as tmpdir:
+            stager = SnowflakeUploadStager()
+            stager_params = {
+                "elements_filepath": upload_file,
+                "file_data": mock_file_data,
+                "output_dir": Path(tmpdir),
+                "output_filename": "test_db",
+            }
+            if stager.is_async():
+                staged_path = await stager.run_async(**stager_params)
+            else:
+                staged_path = stager.run(**stager_params)
+
+            # The stager should append the `.json` suffix to the output filename passed in.
+            assert staged_path.name == "test_db.json"
+
+            connect_params = {
+                "user": "test",
+                "password": "test",
+                "account": "test",
+                "database": "test",
+                "host": "snowflake.localhost.localstack.cloud",
+            }
+
+            uploader = SnowflakeUploader(
+                connection_config=SnowflakeConnectionConfig(
+                    access_config=SnowflakeAccessConfig(password=connect_params["password"]),
+                    account=connect_params["account"],
+                    user=connect_params["user"],
+                    database=connect_params["database"],
+                    host=connect_params["host"],
+                )
+            )
+            if uploader.is_async():
+                await uploader.run_async(path=staged_path, file_data=mock_file_data)
+            else:
+                uploader.run(path=staged_path, file_data=mock_file_data)
+
+            staged_df = pd.read_json(staged_path, orient="records", lines=True)
+            expected_num_elements = len(staged_df)
+            validate_destination(
+                connect_params=connect_params,
+                expected_num_elements=expected_num_elements,
+            )

{unstructured_ingest-0.1.0/test/integration/connectors → unstructured_ingest-0.2.0/test/integration/connectors/sql}/test_sqlite.py

@@ -6,36 +6,92 @@ from pathlib import Path
 import pandas as pd
 import pytest
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG, env_setup_path
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG, env_setup_path
+from test.integration.connectors.utils.validation import (
+    ValidationConfigs,
+    source_connector_validation,
+)
 from unstructured_ingest.v2.interfaces import FileData
 from unstructured_ingest.v2.processes.connectors.sql.sqlite import (
     CONNECTOR_TYPE,
     SQLiteConnectionConfig,
+    SQLiteDownloader,
+    SQLiteDownloaderConfig,
+    SQLiteIndexer,
+    SQLiteIndexerConfig,
     SQLiteUploader,
     SQLiteUploadStager,
 )
 
+SEED_DATA_ROWS = 20
+
+
+@contextmanager
+def sqlite_download_setup() -> Path:
+    with tempfile.TemporaryDirectory() as tmpdir:
+        db_path = Path(tmpdir) / "mock_database.db"
+        db_init_path = env_setup_path / "sql" / "sqlite" / "source" / "sqlite-schema.sql"
+        assert db_init_path.exists()
+        assert db_init_path.is_file()
+        with sqlite3.connect(database=db_path) as sqlite_connection:
+            cursor = sqlite_connection.cursor()
+            with db_init_path.open("r") as f:
+                query = f.read()
+            cursor.executescript(query)
+            for i in range(SEED_DATA_ROWS):
+                sql_statment = f"INSERT INTO cars (brand, price) " f"VALUES ('brand{i}', {i})"
+                cursor.execute(sql_statment)
+
+            sqlite_connection.commit()
+            cursor.close()
+        yield db_path
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, "sql")
+async def test_sqlite_source():
+    with sqlite_download_setup() as db_path:
+        with tempfile.TemporaryDirectory() as tmpdir:
+            connection_config = SQLiteConnectionConfig(database_path=db_path)
+            indexer = SQLiteIndexer(
+                connection_config=connection_config,
+                index_config=SQLiteIndexerConfig(
+                    table_name="cars", id_column="car_id", batch_size=5
+                ),
+            )
+            downloader = SQLiteDownloader(
+                connection_config=connection_config,
+                download_config=SQLiteDownloaderConfig(
+                    fields=["car_id", "brand"], download_dir=Path(tmpdir)
+                ),
+            )
+            await source_connector_validation(
+                indexer=indexer,
+                downloader=downloader,
+                configs=ValidationConfigs(
+                    test_id="sqlite",
+                    expected_num_files=SEED_DATA_ROWS,
+                    expected_number_indexed_file_data=4,
+                    validate_downloaded_files=True,
+                ),
+            )
+
 
 @contextmanager
-def sqlite_setup() -> Path:
+def sqlite_upload_setup() -> Path:
     # Provision the local file that sqlite points to to have the desired schema for the integration
     # tests and make sure the file and connection get cleaned up by using a context manager.
     with tempfile.TemporaryDirectory() as tmpdir:
         db_path = Path(tmpdir) / "elements.db"
-        db_init_path = env_setup_path / "sql" / "sqlite-schema.sql"
+        db_init_path = env_setup_path / "sql" / "sqlite" / "destination" / "sqlite-schema.sql"
         assert db_init_path.exists()
         assert db_init_path.is_file()
-        connection = None
-        try:
-            connection = sqlite3.connect(database=db_path)
+        with sqlite3.connect(database=db_path) as sqlite_connection:
             with db_init_path.open("r") as f:
                 query = f.read()
-            cursor = connection.cursor()
+            cursor = sqlite_connection.cursor()
             cursor.executescript(query)
-            yield db_path
-        finally:
-            if connection:
-                connection.close()
+        yield db_path
 
 
 def validate_destination(db_path: Path, expected_num_elements: int):
@@ -62,7 +118,7 @@ async def test_sqlite_destination(upload_file: Path):
     # the sqlite destination connector doesn't leverage the file data but is required as an input,
     # mocking it with arbitrary values to meet the base requirements:
     mock_file_data = FileData(identifier="mock file data", connector_type=CONNECTOR_TYPE)
-    with sqlite_setup() as db_path:
+    with sqlite_upload_setup() as db_path:
         with tempfile.TemporaryDirectory() as tmpdir:
             stager = SQLiteUploadStager()
             stager_params = {