unstructured-ingest 0.3.13__tar.gz → 0.3.15__tar.gz

{unstructured_ingest-0.3.13/unstructured_ingest.egg-info → unstructured_ingest-0.3.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.3.13
+Version: 0.3.15
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -23,13 +23,13 @@ Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
 Requires-Dist: pandas
-Requires-Dist: pydantic>=2.7
-Requires-Dist: click
 Requires-Dist: tqdm
-Requires-Dist: python-dateutil
+Requires-Dist: click
 Requires-Dist: ndjson
 Requires-Dist: dataclasses_json
+Requires-Dist: pydantic>=2.7
 Requires-Dist: opentelemetry-sdk
+Requires-Dist: python-dateutil
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.26.1; extra == "remote"
 Provides-Extra: csv
@@ -67,16 +67,16 @@ Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: fsspec; extra == "azure"
 Requires-Dist: adlfs; extra == "azure"
+Requires-Dist: fsspec; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: biomed
 Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: fsspec; extra == "box"
 Requires-Dist: boxfs; extra == "box"
+Requires-Dist: fsspec; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
@@ -92,19 +92,19 @@ Requires-Dist: deltalake; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord.py; extra == "discord"
 Provides-Extra: dropbox
-Requires-Dist: fsspec; extra == "dropbox"
 Requires-Dist: dropboxdrivefs; extra == "dropbox"
+Requires-Dist: fsspec; extra == "dropbox"
 Provides-Extra: duckdb
 Requires-Dist: duckdb; extra == "duckdb"
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
 Provides-Extra: gcs
-Requires-Dist: fsspec; extra == "gcs"
 Requires-Dist: gcsfs; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
+Requires-Dist: fsspec; extra == "gcs"
 Provides-Extra: github
-Requires-Dist: pygithub>1.58.0; extra == "github"
 Requires-Dist: requests; extra == "github"
+Requires-Dist: pygithub>1.58.0; extra == "github"
 Provides-Extra: gitlab
 Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
@@ -125,23 +125,23 @@ Requires-Dist: pymilvus; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: neo4j
-Requires-Dist: cymple; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
 Requires-Dist: neo4j; extra == "neo4j"
+Requires-Dist: cymple; extra == "neo4j"
 Provides-Extra: notion
 Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
 Provides-Extra: onedrive
-Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Requires-Dist: msal; extra == "onedrive"
+Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: outlook
-Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
 Requires-Dist: msal; extra == "outlook"
+Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
 Provides-Extra: pinecone
 Requires-Dist: pinecone-client>=3.7.1; extra == "pinecone"
 Provides-Extra: postgres
@@ -153,16 +153,16 @@ Requires-Dist: praw; extra == "reddit"
 Provides-Extra: redis
 Requires-Dist: redis; extra == "redis"
 Provides-Extra: s3
-Requires-Dist: fsspec; extra == "s3"
 Requires-Dist: s3fs; extra == "s3"
+Requires-Dist: fsspec; extra == "s3"
 Provides-Extra: sharepoint
-Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Requires-Dist: msal; extra == "sharepoint"
+Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: fsspec; extra == "sftp"
 Requires-Dist: paramiko; extra == "sftp"
+Requires-Dist: fsspec; extra == "sftp"
 Provides-Extra: slack
 Requires-Dist: slack_sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
@@ -174,17 +174,19 @@ Provides-Extra: weaviate
 Requires-Dist: weaviate-client; extra == "weaviate"
 Provides-Extra: databricks-volumes
 Requires-Dist: databricks-sdk; extra == "databricks-volumes"
+Provides-Extra: databricks-delta-tables
+Requires-Dist: databricks-sql-connector; extra == "databricks-delta-tables"
 Provides-Extra: singlestore
 Requires-Dist: singlestoredb; extra == "singlestore"
 Provides-Extra: vectara
-Requires-Dist: httpx; extra == "vectara"
 Requires-Dist: requests; extra == "vectara"
+Requires-Dist: httpx; extra == "vectara"
 Requires-Dist: aiofiles; extra == "vectara"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
-Requires-Dist: tiktoken; extra == "embed-octoai"
 Requires-Dist: openai; extra == "embed-octoai"
+Requires-Dist: tiktoken; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
@@ -192,11 +194,11 @@ Requires-Dist: voyageai; extra == "embed-voyageai"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Provides-Extra: openai
-Requires-Dist: tiktoken; extra == "openai"
 Requires-Dist: openai; extra == "openai"
+Requires-Dist: tiktoken; extra == "openai"
 Provides-Extra: bedrock
-Requires-Dist: boto3; extra == "bedrock"
 Requires-Dist: aioboto3; extra == "bedrock"
+Requires-Dist: boto3; extra == "bedrock"
 Provides-Extra: togetherai
 Requires-Dist: together; extra == "togetherai"
 

{unstructured_ingest-0.3.13 → unstructured_ingest-0.3.15}/pyproject.toml

@@ -18,7 +18,7 @@ exclude= [
     "venv",
     ".python_packages",
     "build",
-    "migrations"
+    "migrations",
 ]
 
 [tool.ruff.lint]

{unstructured_ingest-0.3.13 → unstructured_ingest-0.3.15}/setup.py

@@ -127,6 +127,9 @@ connectors_reqs = {
     "wikipedia": load_requirements("requirements/connectors/wikipedia.in"),
     "weaviate": load_requirements("requirements/connectors/weaviate.in"),
     "databricks-volumes": load_requirements("requirements/connectors/databricks-volumes.in"),
+    "databricks-delta-tables": load_requirements(
+        "requirements/connectors/databricks-delta-tables.in"
+    ),
     "singlestore": load_requirements("requirements/connectors/singlestore.in"),
     "vectara": load_requirements("requirements/connectors/vectara.in"),
 }

{unstructured_ingest-0.3.13 → unstructured_ingest-0.3.15}/test/integration/connectors/databricks/test_volumes_native.py

@@ -10,7 +10,11 @@ import pytest
 from databricks.sdk import WorkspaceClient
 from databricks.sdk.errors.platform import NotFound
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG
+from test.integration.connectors.utils.constants import (
+    BLOB_STORAGE_TAG,
+    DESTINATION_TAG,
+    SOURCE_TAG,
+)
 from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
@@ -83,7 +87,7 @@ def get_pat_env_data() -> PATEnvData:
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
 @requires_env(
     "DATABRICKS_HOST", "DATABRICKS_CLIENT_ID", "DATABRICKS_CLIENT_SECRET", "DATABRICKS_CATALOG"
 )
@@ -115,7 +119,7 @@ async def test_volumes_native_source(tmp_path: Path):
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
 @requires_env("DATABRICKS_HOST", "DATABRICKS_PAT", "DATABRICKS_CATALOG")
 async def test_volumes_native_source_pat(tmp_path: Path):
     env_data = get_pat_env_data()
@@ -144,7 +148,7 @@ async def test_volumes_native_source_pat(tmp_path: Path):
         )
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
 @requires_env("DATABRICKS_HOST", "DATABRICKS_PAT", "DATABRICKS_CATALOG")
 def test_volumes_native_source_pat_invalid_catalog():
     env_data = get_pat_env_data()
@@ -162,7 +166,7 @@ def test_volumes_native_source_pat_invalid_catalog():
             _ = list(indexer.run())
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
 @requires_env("DATABRICKS_HOST")
 def test_volumes_native_source_pat_invalid_pat():
     host = os.environ["DATABRICKS_HOST"]
@@ -231,7 +235,7 @@ def validate_upload(client: WorkspaceClient, catalog: str, volume: str, volume_p
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, BLOB_STORAGE_TAG)
 @requires_env(
     "DATABRICKS_HOST", "DATABRICKS_CLIENT_ID", "DATABRICKS_CLIENT_SECRET", "DATABRICKS_CATALOG"
 )

{unstructured_ingest-0.3.13 → unstructured_ingest-0.3.15}/test/integration/connectors/discord/test_discord.py

@@ -6,7 +6,7 @@ from typing import Optional
 
 import pytest
 
-from test.integration.connectors.utils.constants import SOURCE_TAG
+from test.integration.connectors.utils.constants import SOURCE_TAG, UNCATEGORIZED_TAG
 from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
@@ -38,7 +38,7 @@ def get_env_data() -> EnvData:
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
 @requires_env("DISCORD_TOKEN", "DISCORD_CHANNELS")
 async def test_discord_source():
     env = get_env_data()
@@ -66,7 +66,7 @@ async def test_discord_source():
         )
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
 @requires_env("DISCORD_CHANNELS")
 def test_discord_source_precheck_fail_no_token():
     indexer_config = DiscordIndexerConfig(channels=get_env_data().channels)
@@ -77,7 +77,7 @@ def test_discord_source_precheck_fail_no_token():
         indexer.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
 @requires_env("DISCORD_TOKEN")
 def test_discord_source_precheck_fail_no_channels():
     indexer_config = DiscordIndexerConfig(channels=[])

{unstructured_ingest-0.3.13 → unstructured_ingest-0.3.15}/test/integration/connectors/duckdb/test_duckdb.py

@@ -5,7 +5,7 @@ import duckdb
 import pytest
 from _pytest.fixtures import TopRequest
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG
 from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
     stager_validation,
@@ -46,7 +46,7 @@ def validate_duckdb_destination(db_path: Path, expected_num_elements: int):
             conn.close()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "duckdb")
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "duckdb", SQL_TAG)
 def test_duckdb_destination(upload_file: Path, provisioned_db_file: Path, temp_dir: Path):
     file_data = FileData(
         source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
@@ -73,6 +73,7 @@ def test_duckdb_destination(upload_file: Path, provisioned_db_file: Path, temp_d
     validate_duckdb_destination(db_path=provisioned_db_file, expected_num_elements=len(data))
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "duckdb", SQL_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_duckdb_stager(
     request: TopRequest,

{unstructured_ingest-0.3.13 → unstructured_ingest-0.3.15}/test/integration/connectors/duckdb/test_motherduck.py

@@ -7,7 +7,7 @@ import duckdb
 import pandas as pd
 import pytest
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG
 from test.integration.utils import requires_env
 from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.duckdb.motherduck import (
@@ -61,7 +61,7 @@ def validate_motherduck_destination(database: str, expected_num_elements: int, m
             conn.close()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
 @requires_env("MOTHERDUCK_TOKEN")
 def test_motherduck_destination(
     md_token: str, upload_file: Path, provisioned_db: str, temp_dir: Path

{unstructured_ingest-0.3.13 → unstructured_ingest-0.3.15}/test/integration/connectors/elasticsearch/test_elasticsearch.py

@@ -15,7 +15,7 @@ from _pytest.fixtures import TopRequest
 from elasticsearch import Elasticsearch as ElasticsearchClient
 from elasticsearch.helpers import bulk
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG, NOSQL_TAG
 from test.integration.connectors.utils.docker import HealthCheck, container_context
 from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
@@ -177,7 +177,7 @@ def destination_index(elasticsearch_elements_mapping: dict) -> str:
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
 async def test_elasticsearch_source(source_index: str, movies_dataframe: pd.DataFrame):
     indexer_config = ElasticsearchIndexerConfig(index_name=source_index)
     with tempfile.TemporaryDirectory() as tempdir:
@@ -207,7 +207,7 @@ async def test_elasticsearch_source(source_index: str, movies_dataframe: pd.Data
         )
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
 def test_elasticsearch_source_precheck_fail_no_cluster():
     indexer_config = ElasticsearchIndexerConfig(index_name="index")
 
@@ -221,7 +221,7 @@ def test_elasticsearch_source_precheck_fail_no_cluster():
         indexer.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
 def test_elasticsearch_source_precheck_fail_no_index(source_index: str):
     indexer_config = ElasticsearchIndexerConfig(index_name="index")
 
@@ -236,7 +236,7 @@ def test_elasticsearch_source_precheck_fail_no_index(source_index: str):
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 async def test_elasticsearch_destination(
     upload_file: Path,
     destination_index: str,
@@ -282,7 +282,7 @@ async def test_elasticsearch_destination(
         validate_count(client=client, expected_count=expected_count, index_name=destination_index)
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 def test_elasticsearch_destination_precheck_fail():
     connection_config = ElasticsearchConnectionConfig(
         access_config=ElasticsearchAccessConfig(password=ES_PASSWORD),
@@ -297,7 +297,7 @@ def test_elasticsearch_destination_precheck_fail():
         uploader.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 def test_elasticsearch_destination_precheck_fail_no_index(destination_index: str):
     connection_config = ElasticsearchConnectionConfig(
         access_config=ElasticsearchAccessConfig(password=ES_PASSWORD),
@@ -312,6 +312,7 @@ def test_elasticsearch_destination_precheck_fail_no_index(destination_index: str
         uploader.precheck()
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_elasticsearch_stager(
     request: TopRequest,

{unstructured_ingest-0.3.13 → unstructured_ingest-0.3.15}/test/integration/connectors/elasticsearch/test_opensearch.py

@@ -10,7 +10,7 @@ import pytest
 from _pytest.fixtures import TopRequest
 from opensearchpy import Document, Keyword, OpenSearch, Text
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, NOSQL_TAG, SOURCE_TAG
 from test.integration.connectors.utils.docker import HealthCheck, container_context
 from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
@@ -166,7 +166,7 @@ def destination_index(opensearch_elements_mapping: dict) -> str:
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
 async def test_opensearch_source(source_index: str, movies_dataframe: pd.DataFrame):
     indexer_config = OpenSearchIndexerConfig(index_name=source_index)
     with tempfile.TemporaryDirectory() as tempdir:
@@ -197,7 +197,7 @@ async def test_opensearch_source(source_index: str, movies_dataframe: pd.DataFra
         )
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
 def test_opensearch_source_precheck_fail_no_cluster():
     indexer_config = OpenSearchIndexerConfig(index_name="index")
 
@@ -212,7 +212,7 @@ def test_opensearch_source_precheck_fail_no_cluster():
         indexer.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
 def test_opensearch_source_precheck_fail_no_index(source_index: str):
     indexer_config = OpenSearchIndexerConfig(index_name="index")
 
@@ -228,7 +228,7 @@ def test_opensearch_source_precheck_fail_no_index(source_index: str):
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 async def test_opensearch_destination(
     upload_file: Path,
     destination_index: str,
@@ -275,7 +275,7 @@ async def test_opensearch_destination(
         validate_count(client=client, expected_count=expected_count, index_name=destination_index)
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 def test_opensearch_destination_precheck_fail():
     connection_config = OpenSearchConnectionConfig(
         access_config=OpenSearchAccessConfig(password="admin"),
@@ -291,7 +291,7 @@ def test_opensearch_destination_precheck_fail():
         uploader.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 def test_opensearch_destination_precheck_fail_no_index(destination_index: str):
     connection_config = OpenSearchConnectionConfig(
         access_config=OpenSearchAccessConfig(password="admin"),
@@ -307,6 +307,7 @@ def test_opensearch_destination_precheck_fail_no_index(destination_index: str):
         uploader.precheck()
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_opensearch_stager(
     request: TopRequest,

unstructured_ingest-0.3.15/test/integration/connectors/sql/test_databricks_delta_tables.py

@@ -0,0 +1,142 @@
+import json
+import os
+import time
+from contextlib import contextmanager
+from pathlib import Path
+from uuid import uuid4
+
+import pytest
+from databricks.sql import connect
+from databricks.sql.client import Connection as DeltaTableConnection
+from databricks.sql.client import Cursor as DeltaTableCursor
+from pydantic import BaseModel, SecretStr
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG, env_setup_path
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connectors.sql.databricks_delta_tables import (
+    CONNECTOR_TYPE,
+    DatabrickDeltaTablesAccessConfig,
+    DatabrickDeltaTablesConnectionConfig,
+    DatabrickDeltaTablesUploader,
+    DatabrickDeltaTablesUploaderConfig,
+    DatabrickDeltaTablesUploadStager,
+)
+
+CATALOG = "utic-dev-tech-fixtures"
+
+
+class EnvData(BaseModel):
+    server_hostname: str
+    http_path: str
+    access_token: SecretStr
+
+
+def get_env_data() -> EnvData:
+    return EnvData(
+        server_hostname=os.environ["DATABRICKS_SERVER_HOSTNAME"],
+        http_path=os.environ["DATABRICKS_HTTP_PATH"],
+        access_token=os.environ["DATABRICKS_ACCESS_TOKEN"],
+    )
+
+
+def get_destination_schema(new_table_name: str) -> str:
+    p = Path(env_setup_path / "sql" / "databricks_delta_tables" / "destination" / "schema.sql")
+    with p.open() as f:
+        data_lines = f.readlines()
+    data_lines[0] = data_lines[0].replace("elements", new_table_name)
+    data = "".join([line.strip() for line in data_lines])
+    return data
+
+
+@contextmanager
+def get_connection() -> DeltaTableConnection:
+    env_data = get_env_data()
+    with connect(
+        server_hostname=env_data.server_hostname,
+        http_path=env_data.http_path,
+        access_token=env_data.access_token.get_secret_value(),
+    ) as connection:
+        yield connection
+
+
+@contextmanager
+def get_cursor() -> DeltaTableCursor:
+    with get_connection() as connection:
+        with connection.cursor() as cursor:
+            cursor.execute(f"USE CATALOG '{CATALOG}'")
+            yield cursor
+
+
+@pytest.fixture
+def destination_table() -> str:
+    random_id = str(uuid4())[:8]
+    table_name = f"elements_{random_id}"
+    destination_schema = get_destination_schema(new_table_name=table_name)
+    with get_cursor() as cursor:
+        logger.info(f"creating table: {table_name}")
+        cursor.execute(f"DROP TABLE IF EXISTS {table_name}")
+        cursor.execute(destination_schema)
+
+    yield table_name
+    with get_cursor() as cursor:
+        logger.info(f"dropping table: {table_name}")
+        cursor.execute(f"DROP TABLE IF EXISTS {table_name}")
+
+
+def validate_destination(expected_num_elements: int, table_name: str, retries=30, interval=1):
+    with get_cursor() as cursor:
+        for i in range(retries):
+            cursor.execute(f"SELECT COUNT(*) FROM {table_name}")
+            count = cursor.fetchone()[0]
+            if count == expected_num_elements:
+                break
+            logger.info(f"retry attempt {i}: expected {expected_num_elements} != count {count}")
+            time.sleep(interval)
+        assert (
+            count == expected_num_elements
+        ), f"dest check failed: got {count}, expected {expected_num_elements}"
+
+
+@pytest.mark.asyncio
+@pytest.mark.skip("Resources take too long to spin up to run in CI")
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
+@requires_env("DATABRICKS_SERVER_HOSTNAME", "DATABRICKS_HTTP_PATH", "DATABRICKS_ACCESS_TOKEN")
+async def test_databricks_delta_tables_destination(
+    upload_file: Path, temp_dir: Path, destination_table: str
+):
+    env_data = get_env_data()
+    mock_file_data = FileData(
+        identifier="mock file data",
+        connector_type=CONNECTOR_TYPE,
+        source_identifiers=SourceIdentifiers(filename=upload_file.name, fullpath=upload_file.name),
+    )
+    stager = DatabrickDeltaTablesUploadStager()
+    staged_path = stager.run(
+        elements_filepath=upload_file,
+        file_data=mock_file_data,
+        output_dir=temp_dir,
+        output_filename=upload_file.name,
+    )
+
+    assert staged_path.suffix == upload_file.suffix
+
+    uploader = DatabrickDeltaTablesUploader(
+        connection_config=DatabrickDeltaTablesConnectionConfig(
+            access_config=DatabrickDeltaTablesAccessConfig(
+                token=env_data.access_token.get_secret_value()
+            ),
+            http_path=env_data.http_path,
+            server_hostname=env_data.server_hostname,
+        ),
+        upload_config=DatabrickDeltaTablesUploaderConfig(
+            catalog=CATALOG, database="default", table_name=destination_table
+        ),
+    )
+    with staged_path.open("r") as f:
+        staged_data = json.load(f)
+    expected_num_elements = len(staged_data)
+    uploader.precheck()
+    uploader.run(path=staged_path, file_data=mock_file_data)
+    validate_destination(expected_num_elements=expected_num_elements, table_name=destination_table)

{unstructured_ingest-0.3.13 → unstructured_ingest-0.3.15}/test/integration/connectors/sql/test_postgres.py

@@ -5,7 +5,12 @@ import pytest
 from _pytest.fixtures import TopRequest
 from psycopg2 import connect
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG, env_setup_path
+from test.integration.connectors.utils.constants import (
+    DESTINATION_TAG,
+    SOURCE_TAG,
+    SQL_TAG,
+    env_setup_path,
+)
 from test.integration.connectors.utils.docker_compose import docker_compose_context
 from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
@@ -51,7 +56,7 @@ def source_database_setup() -> str:
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, "sql")
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, SQL_TAG)
 async def test_postgres_source(temp_dir: Path, source_database_setup: str):
     connect_params = {
         "host": "localhost",
@@ -115,7 +120,7 @@ def validate_destination(
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "sql")
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
 async def test_postgres_destination(upload_file: Path, temp_dir: Path):
     # the postgres destination connector doesn't leverage the file data but is required as an input,
     # mocking it with arbitrary values to meet the base requirements:
@@ -179,6 +184,7 @@ async def test_postgres_destination(upload_file: Path, temp_dir: Path):
         )
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_postgres_stager(
     request: TopRequest,

{unstructured_ingest-0.3.13 → unstructured_ingest-0.3.15}/test/integration/connectors/sql/test_singlestore.py

@@ -5,7 +5,12 @@ import pytest
 import singlestoredb as s2
 from _pytest.fixtures import TopRequest
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG, env_setup_path
+from test.integration.connectors.utils.constants import (
+    DESTINATION_TAG,
+    SOURCE_TAG,
+    SQL_TAG,
+    env_setup_path,
+)
 from test.integration.connectors.utils.docker_compose import docker_compose_context
 from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
@@ -54,7 +59,7 @@ def source_database_setup() -> dict:
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, "sql")
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, SQL_TAG)
 async def test_singlestore_source(temp_dir: Path, source_database_setup: dict):
 
     connection_config = SingleStoreConnectionConfig(
@@ -101,7 +106,7 @@ def validate_destination(
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "sql")
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
 async def test_singlestore_destination(upload_file: Path, temp_dir: Path):
     mock_file_data = FileData(
         identifier="mock file data",
@@ -160,6 +165,7 @@ async def test_singlestore_destination(upload_file: Path, temp_dir: Path):
         )
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_singlestore_stager(
     request: TopRequest,