unstructured-ingest 0.5.21__tar.gz → 0.5.25__tar.gz

{unstructured_ingest-0.5.21/unstructured_ingest.egg-info → unstructured_ingest-0.5.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: unstructured-ingest
-Version: 0.5.21
+Version: 0.5.25
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -23,11 +23,11 @@ Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
 Requires-Dist: dataclasses_json
-Requires-Dist: tqdm
-Requires-Dist: opentelemetry-sdk
 Requires-Dist: pydantic>=2.7
-Requires-Dist: python-dateutil
+Requires-Dist: opentelemetry-sdk
 Requires-Dist: click
+Requires-Dist: tqdm
+Requires-Dist: python-dateutil
 Requires-Dist: numpy
 Requires-Dist: pandas
 Provides-Extra: remote
@@ -112,13 +112,13 @@ Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Requires-Dist: numpy; extra == "azure-ai-search"
 Requires-Dist: pandas; extra == "azure-ai-search"
 Provides-Extra: biomed
-Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
+Requires-Dist: requests; extra == "biomed"
 Requires-Dist: numpy; extra == "biomed"
 Requires-Dist: pandas; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: boxfs; extra == "box"
 Requires-Dist: fsspec; extra == "box"
+Requires-Dist: boxfs; extra == "box"
 Requires-Dist: numpy; extra == "box"
 Requires-Dist: pandas; extra == "box"
 Provides-Extra: chroma
@@ -130,8 +130,8 @@ Requires-Dist: clarifai; extra == "clarifai"
 Requires-Dist: numpy; extra == "clarifai"
 Requires-Dist: pandas; extra == "clarifai"
 Provides-Extra: confluence
-Requires-Dist: requests; extra == "confluence"
 Requires-Dist: atlassian-python-api; extra == "confluence"
+Requires-Dist: requests; extra == "confluence"
 Requires-Dist: numpy; extra == "confluence"
 Requires-Dist: pandas; extra == "confluence"
 Provides-Extra: couchbase
@@ -162,13 +162,13 @@ Requires-Dist: numpy; extra == "elasticsearch"
 Requires-Dist: pandas; extra == "elasticsearch"
 Provides-Extra: gcs
 Requires-Dist: fsspec; extra == "gcs"
-Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: gcsfs; extra == "gcs"
+Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: numpy; extra == "gcs"
 Requires-Dist: pandas; extra == "gcs"
 Provides-Extra: github
-Requires-Dist: requests; extra == "github"
 Requires-Dist: pygithub>1.58.0; extra == "github"
+Requires-Dist: requests; extra == "github"
 Requires-Dist: numpy; extra == "github"
 Requires-Dist: pandas; extra == "github"
 Provides-Extra: gitlab
@@ -180,15 +180,15 @@ Requires-Dist: google-api-python-client; extra == "google-drive"
 Requires-Dist: numpy; extra == "google-drive"
 Requires-Dist: pandas; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: urllib3; extra == "hubspot"
 Requires-Dist: hubspot-api-client; extra == "hubspot"
+Requires-Dist: urllib3; extra == "hubspot"
 Requires-Dist: numpy; extra == "hubspot"
 Requires-Dist: pandas; extra == "hubspot"
 Provides-Extra: ibm-watsonx-s3
-Requires-Dist: httpx; extra == "ibm-watsonx-s3"
-Requires-Dist: pyarrow; extra == "ibm-watsonx-s3"
 Requires-Dist: tenacity; extra == "ibm-watsonx-s3"
 Requires-Dist: pyiceberg; extra == "ibm-watsonx-s3"
+Requires-Dist: pyarrow; extra == "ibm-watsonx-s3"
+Requires-Dist: httpx; extra == "ibm-watsonx-s3"
 Requires-Dist: numpy; extra == "ibm-watsonx-s3"
 Requires-Dist: pandas; extra == "ibm-watsonx-s3"
 Provides-Extra: jira
@@ -216,22 +216,22 @@ Requires-Dist: pymongo; extra == "mongodb"
 Requires-Dist: numpy; extra == "mongodb"
 Requires-Dist: pandas; extra == "mongodb"
 Provides-Extra: neo4j
-Requires-Dist: neo4j-rust-ext; extra == "neo4j"
-Requires-Dist: networkx; extra == "neo4j"
 Requires-Dist: cymple; extra == "neo4j"
+Requires-Dist: networkx; extra == "neo4j"
+Requires-Dist: neo4j-rust-ext; extra == "neo4j"
 Requires-Dist: numpy; extra == "neo4j"
 Requires-Dist: pandas; extra == "neo4j"
 Provides-Extra: notion
+Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
-Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: numpy; extra == "notion"
 Requires-Dist: pandas; extra == "notion"
 Provides-Extra: onedrive
 Requires-Dist: msal; extra == "onedrive"
-Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
+Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Requires-Dist: numpy; extra == "onedrive"
 Requires-Dist: pandas; extra == "onedrive"
 Provides-Extra: opensearch
@@ -264,8 +264,8 @@ Requires-Dist: redis; extra == "redis"
 Requires-Dist: numpy; extra == "redis"
 Requires-Dist: pandas; extra == "redis"
 Provides-Extra: s3
-Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: fsspec; extra == "s3"
+Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: numpy; extra == "s3"
 Requires-Dist: pandas; extra == "s3"
 Provides-Extra: sharepoint
@@ -278,8 +278,8 @@ Requires-Dist: simple-salesforce; extra == "salesforce"
 Requires-Dist: numpy; extra == "salesforce"
 Requires-Dist: pandas; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: paramiko; extra == "sftp"
 Requires-Dist: fsspec; extra == "sftp"
+Requires-Dist: paramiko; extra == "sftp"
 Requires-Dist: numpy; extra == "sftp"
 Requires-Dist: pandas; extra == "sftp"
 Provides-Extra: slack
@@ -287,8 +287,8 @@ Requires-Dist: slack_sdk[optional]; extra == "slack"
 Requires-Dist: numpy; extra == "slack"
 Requires-Dist: pandas; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Requires-Dist: psycopg2-binary; extra == "snowflake"
+Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Requires-Dist: numpy; extra == "snowflake"
 Requires-Dist: pandas; extra == "snowflake"
 Provides-Extra: wikipedia
@@ -312,9 +312,9 @@ Requires-Dist: singlestoredb; extra == "singlestore"
 Requires-Dist: numpy; extra == "singlestore"
 Requires-Dist: pandas; extra == "singlestore"
 Provides-Extra: vectara
-Requires-Dist: requests; extra == "vectara"
-Requires-Dist: httpx; extra == "vectara"
 Requires-Dist: aiofiles; extra == "vectara"
+Requires-Dist: httpx; extra == "vectara"
+Requires-Dist: requests; extra == "vectara"
 Requires-Dist: numpy; extra == "vectara"
 Requires-Dist: pandas; extra == "vectara"
 Provides-Extra: vastdb
@@ -324,9 +324,9 @@ Requires-Dist: vastdb; extra == "vastdb"
 Requires-Dist: numpy; extra == "vastdb"
 Requires-Dist: pandas; extra == "vastdb"
 Provides-Extra: zendesk
-Requires-Dist: httpx; extra == "zendesk"
 Requires-Dist: aiofiles; extra == "zendesk"
 Requires-Dist: bs4; extra == "zendesk"
+Requires-Dist: httpx; extra == "zendesk"
 Requires-Dist: numpy; extra == "zendesk"
 Requires-Dist: pandas; extra == "zendesk"
 Provides-Extra: embed-huggingface
@@ -334,8 +334,8 @@ Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Requires-Dist: numpy; extra == "embed-huggingface"
 Requires-Dist: pandas; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
-Requires-Dist: tiktoken; extra == "embed-octoai"
 Requires-Dist: openai; extra == "embed-octoai"
+Requires-Dist: tiktoken; extra == "embed-octoai"
 Requires-Dist: numpy; extra == "embed-octoai"
 Requires-Dist: pandas; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
@@ -351,13 +351,13 @@ Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Requires-Dist: numpy; extra == "embed-mixedbreadai"
 Requires-Dist: pandas; extra == "embed-mixedbreadai"
 Provides-Extra: openai
-Requires-Dist: tiktoken; extra == "openai"
 Requires-Dist: openai; extra == "openai"
+Requires-Dist: tiktoken; extra == "openai"
 Requires-Dist: numpy; extra == "openai"
 Requires-Dist: pandas; extra == "openai"
 Provides-Extra: bedrock
-Requires-Dist: aioboto3; extra == "bedrock"
 Requires-Dist: boto3; extra == "bedrock"
+Requires-Dist: aioboto3; extra == "bedrock"
 Requires-Dist: numpy; extra == "bedrock"
 Requires-Dist: pandas; extra == "bedrock"
 Provides-Extra: togetherai

{unstructured_ingest-0.5.21 → unstructured_ingest-0.5.25}/test/integration/connectors/test_vectara.py

@@ -1,6 +1,7 @@
 import json
 import os
 import time
+from functools import lru_cache
 from pathlib import Path
 from typing import Generator
 from uuid import uuid4
@@ -25,24 +26,29 @@ from unstructured_ingest.v2.processes.connectors.vectara import (
 )
 
 
-def validate_upload(response: dict, expected_data: dict):
+def validate_upload(document: dict, expected_data: dict):
+    logger.info(f"validating document: {document}")
     element_id = expected_data["element_id"]
     expected_text = expected_data["text"]
     filename = expected_data["metadata"]["filename"]
     filetype = expected_data["metadata"]["filetype"]
     page_number = expected_data["metadata"]["page_number"]
 
-    response = response["search_results"][0]
-
-    assert response is not None
-    assert response["text"] == expected_text
-    assert response["part_metadata"]["element_id"] == element_id
-    assert response["part_metadata"]["filename"] == filename
-    assert response["part_metadata"]["filetype"] == filetype
-    assert response["part_metadata"]["page_number"] == page_number
+    assert document is not None
+    speech_parts = document["parts"]
+    assert speech_parts
+    first_part = speech_parts[0]
+    assert first_part["text"] == expected_text
+    part_metadata = first_part["metadata"]
+    assert part_metadata
+    assert part_metadata["element_id"] == element_id
+    assert part_metadata["filename"] == filename
+    assert part_metadata["filetype"] == filetype
+    assert part_metadata["page_number"] == page_number
 
 
 @requires_env("VECTARA_OAUTH_CLIENT_ID", "VECTARA_OAUTH_SECRET", "VECTARA_CUSTOMER_ID")
+@lru_cache()
 def _get_jwt_token():
     """Connect to the server and get a JWT token."""
     customer_id = os.environ["VECTARA_CUSTOMER_ID"]
@@ -65,23 +71,12 @@ def _get_jwt_token():
     return response_json.get("access_token")
 
 
-def query_data(corpus_key: str, element_id: str) -> dict:
+def list_documents(corpus_key: str) -> list[str]:
 
-    url = f"https://api.vectara.io/v2/corpora/{corpus_key}/query"
+    url = f"https://api.vectara.io/v2/corpora/{corpus_key}/documents"
 
     # the query below requires the corpus to have filter attributes for element_id
 
-    data = json.dumps(
-        {
-            "query": "string",
-            "search": {
-                "metadata_filter": f"part.element_id = '{element_id}'",
-                "lexical_interpolation": 1,
-                "limit": 10,
-            },
-        }
-    )
-
     jwt_token = _get_jwt_token()
     headers = {
         "Content-Type": "application/json",
@@ -90,11 +85,26 @@ def query_data(corpus_key: str, element_id: str) -> dict:
         "X-source": "unstructured",
     }
 
-    response = requests.post(url, headers=headers, data=data)
+    response = requests.get(url, headers=headers)
     response.raise_for_status()
     response_json = response.json()
+    documents = response_json.get("documents", [])
+    return documents
+
 
-    return response_json
+def fetch_document(corpus_key: str, documents_id: str) -> dict:
+    url = f"https://api.vectara.io/v2/corpora/{corpus_key}/documents/{documents_id}"
+    jwt_token = _get_jwt_token()
+    headers = {
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+        "Authorization": f"Bearer {jwt_token}",
+        "X-source": "unstructured",
+    }
+
+    response = requests.get(url, headers=headers)
+    response.raise_for_status()
+    return response.json()
 
 
 def create_corpora(corpus_key: str, corpus_name: str) -> None:
@@ -148,8 +158,8 @@ def delete_corpora(corpus_key: str) -> None:
     response.raise_for_status()
 
 
-def list_corpora() -> list:
-    url = "https://api.vectara.io/v2/corpora?limit=100"
+def get_metadata(corpus_key: str):
+    url = f"https://api.vectara.io/v2/corpora/{corpus_key}"
     jwt_token = _get_jwt_token()
     headers = {
         "Content-Type": "application/json",
@@ -159,35 +169,28 @@ def list_corpora() -> list:
     }
     response = requests.get(url, headers=headers)
     response.raise_for_status()
-    response_json = response.json()
-    if response_json.get("corpora"):
-        return [item["key"] for item in response_json.get("corpora")]
-    else:
-        return []
+    return response.json()
 
 
 def wait_for_ready(corpus_key: str, timeout=60, interval=2) -> None:
-    def is_ready_status():
-        corpora_list = list_corpora()
-        return corpus_key in corpora_list
-
     start = time.time()
-    is_ready = is_ready_status()
-    while not is_ready and time.time() - start < timeout:
-        time.sleep(interval)
-        is_ready = is_ready_status()
-    if not is_ready:
-        raise TimeoutError("time out waiting for corpus to be ready")
+    while time.time() - start < timeout:
+        try:
+            get_metadata(corpus_key)
+            return
+        except requests.HTTPError:
+            time.sleep(interval)
+    raise TimeoutError("time out waiting for corpus to be ready")
 
 
 def wait_for_delete(corpus_key: str, timeout=60, interval=2) -> None:
     start = time.time()
     while time.time() - start < timeout:
-        corpora_list = list_corpora()
-        if corpus_key not in corpora_list:
+        try:
+            get_metadata(corpus_key)
+            time.sleep(interval)
+        except requests.HTTPError:
             return
-        time.sleep(interval)
-
     raise TimeoutError("time out waiting for corpus to delete")
 
 
@@ -210,11 +213,23 @@ def corpora_util() -> Generator[str, None, None]:
         wait_for_delete(corpus_key=corpus_key)
 
 
+def wait_for_doc_meta(corpus_key: str, timeout=60, interval=1) -> list[str]:
+    start = time.time()
+    while time.time() - start < timeout:
+        all_document_meta = list_documents(corpus_key)
+        if not all_document_meta:
+            time.sleep(interval)
+            continue
+        else:
+            return all_document_meta
+    raise TimeoutError("time out waiting for document to be ready")
+
+
 @pytest.mark.asyncio
 @pytest.mark.tags(VECTARA_CONNECTOR_TYPE, DESTINATION_TAG, "vectara", NOSQL_TAG)
 @requires_env("VECTARA_OAUTH_CLIENT_ID", "VECTARA_OAUTH_SECRET", "VECTARA_CUSTOMER_ID")
 async def test_vectara_destination(
-    upload_file: Path, tmp_path: Path, corpora_util: str, retries=30, interval=10
+    upload_file: Path, tmp_path: Path, corpora_util: str, retries=30, interval=1
 ):
     corpus_key = corpora_util
     connection_kwargs = {
@@ -231,7 +246,7 @@ async def test_vectara_destination(
         identifier="mock-file-data",
     )
 
-    stager_config = VectaraUploadStagerConfig(batch_size=10)
+    stager_config = VectaraUploadStagerConfig()
     stager = VectaraUploadStager(upload_stager_config=stager_config)
     new_upload_file = stager.run(
         elements_filepath=upload_file,
@@ -260,11 +275,8 @@ async def test_vectara_destination(
         elements = json.load(upload_fp)
     first_element = elements[0]
 
-    for i in range(retries):
-        response = query_data(corpus_key, first_element["element_id"])
-        if not response["search_results"]:
-            time.sleep(interval)
-        else:
-            break
-
-    validate_upload(response=response, expected_data=first_element)
+    all_document_meta = wait_for_doc_meta(corpus_key)
+    assert len(all_document_meta) == 1
+    document_meta = all_document_meta[0]
+    document = fetch_document(corpus_key=corpus_key, documents_id=document_meta["id"])
+    validate_upload(document=document, expected_data=first_element)

unstructured_ingest-0.5.25/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.5.25"  # pragma: no cover

{unstructured_ingest-0.5.21 → unstructured_ingest-0.5.25}/unstructured_ingest/v2/interfaces/__init__.py

@@ -1,6 +1,13 @@
+from unstructured_ingest.v2.types.file_data import (
+    BatchFileData,
+    BatchItem,
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
+
 from .connector import AccessConfig, BaseConnector, ConnectionConfig
 from .downloader import Downloader, DownloaderConfig, DownloadResponse, download_responses
-from .file_data import BatchFileData, BatchItem, FileData, FileDataSourceMetadata, SourceIdentifiers
 from .indexer import Indexer, IndexerConfig
 from .process import BaseProcess
 from .processor import ProcessorConfig

unstructured_ingest-0.5.25/unstructured_ingest/v2/interfaces/file_data.py

@@ -0,0 +1,13 @@
+"""
+COMPATABILITY NOTICE:
+This file has moved to the v2/types/ module.
+The following line exists for backward compatibility.
+"""
+
+from unstructured_ingest.v2.types.file_data import *  # noqa - star imports are bad, but this is for maximal backward compatability
+
+#  Eventually this file should go away. Let's start warning users now:
+logger.warning(  # noqa - using logger from the star import
+    "Importing file_data.py through interfaces is deprecated. "
+    "Please use unstructured_ingest.v2.types.file_data instead!"
+)

{unstructured_ingest-0.5.21 → unstructured_ingest-0.5.25}/unstructured_ingest/v2/processes/connectors/databricks/volumes_table.py

@@ -1,13 +1,13 @@
-import json
 import os
+import tempfile
 from contextlib import contextmanager
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Generator
+from typing import TYPE_CHECKING, Any, Generator, Optional
 
 from pydantic import Field
 
-from unstructured_ingest.utils.data_prep import write_data
+from unstructured_ingest.utils.data_prep import get_data_df, write_data
 from unstructured_ingest.v2.interfaces import FileData, Uploader, UploaderConfig
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
@@ -22,6 +22,9 @@ from unstructured_ingest.v2.processes.connectors.sql.databricks_delta_tables imp
 
 CONNECTOR_TYPE = "databricks_volume_delta_tables"
 
+if TYPE_CHECKING:
+    from pandas import DataFrame
+
 
 class DatabricksVolumeDeltaTableUploaderConfig(UploaderConfig, DatabricksPathMixin):
     database: str = Field(description="Database name", default="default")
@@ -30,10 +33,12 @@ class DatabricksVolumeDeltaTableUploaderConfig(UploaderConfig, DatabricksPathMix
 
 @dataclass
 class DatabricksVolumeDeltaTableStager(DatabricksDeltaTablesUploadStager):
-    def write_output(self, output_path: Path, data: list[dict]) -> None:
+    def write_output(self, output_path: Path, data: list[dict]) -> Path:
         # To avoid new line issues when migrating from volumes into delta tables, omit indenting
         # and always write it as a json file
-        write_data(path=output_path.with_suffix(".json"), data=data, indent=None)
+        final_output_path = output_path.with_suffix(".json")
+        write_data(path=final_output_path, data=data, indent=None)
+        return final_output_path
 
 
 @dataclass
@@ -41,6 +46,7 @@ class DatabricksVolumeDeltaTableUploader(Uploader):
     connection_config: DatabricksDeltaTablesConnectionConfig
     upload_config: DatabricksVolumeDeltaTableUploaderConfig
     connector_type: str = CONNECTOR_TYPE
+    _columns: Optional[dict[str, str]] = None
 
     def precheck(self) -> None:
         with self.connection_config.get_cursor() as cursor:
@@ -84,20 +90,58 @@ class DatabricksVolumeDeltaTableUploader(Uploader):
             cursor.execute(f"USE DATABASE {self.upload_config.database}")
             yield cursor
 
-    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
-        with self.get_cursor(staging_allowed_local_path=str(path.parent)) as cursor:
-            catalog_path = self.get_output_path(file_data=file_data)
-            logger.debug(f"uploading {path.as_posix()} to {catalog_path}")
-            cursor.execute(f"PUT '{path.as_posix()}' INTO '{catalog_path}' OVERWRITE")
-            logger.debug(
-                f"migrating content from {catalog_path} to table {self.upload_config.table_name}"
+    def get_table_columns(self) -> dict[str, str]:
+        if self._columns is None:
+            with self.get_cursor() as cursor:
+                cursor.execute(f"SELECT * from {self.upload_config.table_name} LIMIT 1")
+                self._columns = {desc[0]: desc[1] for desc in cursor.description}
+        return self._columns
+
+    def _fit_to_schema(self, df: "DataFrame", add_missing_columns: bool = True) -> "DataFrame":
+        import pandas as pd
+
+        table_columns = self.get_table_columns()
+        columns = set(df.columns)
+        schema_fields = set(table_columns.keys())
+        columns_to_drop = columns - schema_fields
+        missing_columns = schema_fields - columns
+
+        if columns_to_drop:
+            logger.info(
+                "Following columns will be dropped to match the table's schema: "
+                f"{', '.join(columns_to_drop)}"
+            )
+        if missing_columns and add_missing_columns:
+            logger.info(
+                "Following null filled columns will be added to match the table's schema:"
+                f" {', '.join(missing_columns)} "
             )
-            with path.open() as f:
-                data = json.load(f)
-                columns = data[0].keys()
-            column_str = ", ".join(columns)
-            sql_statment = f"INSERT INTO `{self.upload_config.table_name}` ({column_str}) SELECT {column_str} FROM json.`{catalog_path}`"  # noqa: E501
-            cursor.execute(sql_statment)
+
+        df = df.drop(columns=columns_to_drop)
+
+        if add_missing_columns:
+            for column in missing_columns:
+                df[column] = pd.Series()
+        return df
+
+    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            df = get_data_df()
+            df = self._fit_to_schema(df=df)
+            temp_path = Path(temp_dir) / path.name
+            df.to_json(temp_path, orient="records", lines=False)
+            with self.get_cursor(staging_allowed_local_path=temp_dir) as cursor:
+                catalog_path = self.get_output_path(file_data=file_data)
+                logger.debug(f"uploading {path.as_posix()} to {catalog_path}")
+                cursor.execute(f"PUT '{temp_path.as_posix()}' INTO '{catalog_path}' OVERWRITE")
+                logger.debug(
+                    f"migrating content from {catalog_path} to "
+                    f"table {self.upload_config.table_name}"
+                )
+                columns = list(df.columns)
+                column_str = ", ".join(columns)
+                sql_statment = f"INSERT INTO `{self.upload_config.table_name}` ({column_str}) SELECT {column_str} FROM json.`{catalog_path}`"  # noqa: E501
+                cursor.execute(sql_statment)
 
 
 databricks_volumes_delta_tables_destination_entry = DestinationRegistryEntry(

{unstructured_ingest-0.5.21 → unstructured_ingest-0.5.25}/unstructured_ingest/v2/processes/connectors/elasticsearch/elasticsearch.py

@@ -6,7 +6,7 @@ from pathlib import Path
 from time import time
 from typing import TYPE_CHECKING, Any, Generator, Optional, Union
 
-from pydantic import BaseModel, Field, Secret, SecretStr
+from pydantic import BaseModel, Field, Secret, SecretStr, field_validator
 
 from unstructured_ingest.error import (
     DestinationConnectionError,
@@ -98,6 +98,12 @@ class ElasticsearchConnectionConfig(ConnectionConfig):
     ca_certs: Optional[Path] = None
     access_config: Secret[ElasticsearchAccessConfig]
 
+    @field_validator("hosts", mode="before")
+    def to_list(cls, value):
+        if isinstance(value, str):
+            return [value]
+        return value
+
     def get_client_kwargs(self) -> dict:
         # Update auth related fields to conform to what the SDK expects based on the
         # supported methods:

{unstructured_ingest-0.5.21 → unstructured_ingest-0.5.25}/unstructured_ingest/v2/processes/connectors/elasticsearch/opensearch.py

@@ -2,7 +2,7 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Optional
 
-from pydantic import BaseModel, Field, Secret
+from pydantic import BaseModel, Field, Secret, field_validator
 
 from unstructured_ingest.error import (
     DestinationConnectionError,
@@ -78,6 +78,12 @@ class OpenSearchConnectionConfig(ConnectionConfig):
 
     access_config: Secret[OpenSearchAccessConfig]
 
+    @field_validator("hosts", mode="before")
+    def to_list(cls, value):
+        if isinstance(value, str):
+            return [value]
+        return value
+
     def get_client_kwargs(self) -> dict:
         # Update auth related fields to conform to what the SDK expects based on the
         # supported methods:

{unstructured_ingest-0.5.21 → unstructured_ingest-0.5.25}/unstructured_ingest/v2/processes/connectors/fsspec/s3.py

@@ -33,6 +33,9 @@ from unstructured_ingest.v2.processes.utils.blob_storage import (
 
 CONNECTOR_TYPE = "s3"
 
+# https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html#object-key-guidelines-avoid-characters
+CHARACTERS_TO_AVOID = ["\\", "{", "^", "}", "%", "`", "]", '"', ">", "[", "~", "<", "#", "|"]
+
 if TYPE_CHECKING:
     from s3fs import S3FileSystem
 
@@ -91,7 +94,7 @@ class S3ConnectionConfig(FsspecConnectionConfig):
         if isinstance(e, PermissionError):
             return UserAuthError(e)
         if isinstance(e, FileNotFoundError):
-            return UserError(e)
+            return UserError(f"File not found: {e}")
         if cause := getattr(e, "__cause__", None):
             error_response = cause.response
             error_meta = error_response["ResponseMetadata"]
@@ -140,6 +143,12 @@ class S3Indexer(FsspecIndexer):
         }
         if metadata:
             record_locator["metadata"] = metadata
+        issue_characters = [char for char in CHARACTERS_TO_AVOID if char in path]
+        if issue_characters:
+            logger.warning(
+                f"File path {path} contains characters "
+                f"that can cause issues with S3: {issue_characters}"
+            )
         return FileDataSourceMetadata(
             date_created=date_created,
             date_modified=date_modified,

{unstructured_ingest-0.5.21 → unstructured_ingest-0.5.25}/unstructured_ingest/v2/processes/connectors/sql/sql.py

@@ -251,8 +251,9 @@ class SQLUploadStager(UploadStager):
             df[column] = df[column].apply(str)
         return df
 
-    def write_output(self, output_path: Path, data: list[dict]) -> None:
+    def write_output(self, output_path: Path, data: list[dict]) -> Path:
         write_data(path=output_path, data=data)
+        return output_path
 
     def run(
         self,
@@ -278,8 +279,10 @@ class SQLUploadStager(UploadStager):
         output_filename = f"{Path(output_filename).stem}{output_filename_suffix}"
         output_path = self.get_output_path(output_filename=output_filename, output_dir=output_dir)
 
-        self.write_output(output_path=output_path, data=df.to_dict(orient="records"))
-        return output_path
+        final_output_path = self.write_output(
+            output_path=output_path, data=df.to_dict(orient="records")
+        )
+        return final_output_path
 
 
 class SQLUploaderConfig(UploaderConfig):