unstructured-ingest 1.0.44__py3-none-any.whl → 1.0.46__py3-none-any.whl

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.44"  # pragma: no cover
+__version__ = "1.0.46"  # pragma: no cover

unstructured_ingest/processes/connectors/delta_table.py

@@ -1,7 +1,7 @@
 import logging
 import traceback
 from dataclasses import dataclass, field
-from multiprocessing import Process, Queue, current_process
+from multiprocessing import Queue, current_process
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional
 from urllib.parse import urlparse
@@ -186,10 +186,25 @@ class DeltaTableUploader(Uploader):
         )
 
         def _is_commit_conflict(exc: BaseException) -> bool:  # noqa: ANN401
-            """Return True if exception looks like a Delta Lake commit conflict."""
-
-            return isinstance(exc, RuntimeError) and (
-                "CommitFailed" in str(exc) or "Metadata changed" in str(exc)
+            """Return True if exception looks like a Delta Lake commit conflict.
+
+            Besides the canonical *CommitFailed* / *Metadata changed* errors that
+            deltalake surfaces when two writers clash, we occasionally hit
+            messages such as *Delta transaction failed, version 0 already
+            exists* while multiple processes race to create the very first log
+            entry. These situations are equally safe to retry, so detect them
+            too.
+            """
+
+            return isinstance(exc, RuntimeError) and any(
+                marker in str(exc)
+                for marker in (
+                    "CommitFailed",
+                    "Metadata changed",
+                    "version 0 already exists",
+                    "version already exists",
+                    "Delta transaction failed",
+                )
             )
 
         @retry(
@@ -206,25 +221,39 @@ class DeltaTableUploader(Uploader):
             # cause ingest to fail, even though all tasks are completed normally. Putting the writer
             # into a process mitigates this issue by ensuring python interpreter waits properly for
             # deltalake's rust backend to finish
-            queue: Queue[str] = Queue()
+            # Use a multiprocessing context that relies on 'spawn' to avoid inheriting the
+            # parent process' Tokio runtime, which leads to `pyo3_runtime.PanicException`.
+            from multiprocessing import get_context
+
+            ctx = get_context("spawn")
+            queue: "Queue[str]" = ctx.Queue()
 
             if current_process().daemon:
                 # write_deltalake_with_error_handling will push any traceback to our queue
                 write_deltalake_with_error_handling(queue=queue, **writer_kwargs)
             else:
-                # On non-daemon processes we still guard against SIGABRT by running in a subprocess.
-                writer = Process(
+                # On non-daemon processes we still guard against SIGABRT by running in a
+                # dedicated subprocess created via the 'spawn' method.
+                writer = ctx.Process(
                     target=write_deltalake_with_error_handling,
                     kwargs={"queue": queue, **writer_kwargs},
                 )
                 writer.start()
                 writer.join()
 
-            # Check if the queue has any exception message
-            if not queue.empty():
-                error_message = queue.get()
-                logger.error("Exception occurred in write_deltalake: %s", error_message)
-                raise RuntimeError(f"Error in write_deltalake: {error_message}")
+                # First surface any traceback captured inside the subprocess so users see the real
+                # root-cause instead of a generic non-zero exit code.
+                if not queue.empty():
+                    error_message = queue.get()
+                    logger.error("Exception occurred in write_deltalake: %s", error_message)
+                    raise RuntimeError(f"Error in write_deltalake: {error_message}")
+
+                # If the subprocess terminated abnormally but produced no traceback (e.g., SIGABRT),
+                # still raise a helpful error for callers.
+                if not current_process().daemon and writer.exitcode != 0:
+                    raise RuntimeError(
+                        f"write_deltalake subprocess exited with code {writer.exitcode}"
+                    )
 
         _single_attempt()
 

unstructured_ingest/processes/connectors/fsspec/fsspec.py

@@ -274,7 +274,7 @@ class FsspecDownloader(Downloader):
         try:
             rpath = file_data.additional_metadata["original_file_path"]
             with self.connection_config.get_client(protocol=self.protocol) as client:
-                client.get(rpath=rpath, lpath=download_path.as_posix())
+                client.get_file(rpath=rpath, lpath=download_path.as_posix())
             self.handle_directory_download(lpath=download_path)
         except Exception as e:
             raise self.wrap_error(e=e)
@@ -286,7 +286,7 @@ class FsspecDownloader(Downloader):
         try:
             rpath = file_data.additional_metadata["original_file_path"]
             with self.connection_config.get_client(protocol=self.protocol) as client:
-                await client.get(rpath=rpath, lpath=download_path.as_posix())
+                await client.get_file(rpath=rpath, lpath=download_path.as_posix())
             self.handle_directory_download(lpath=download_path)
         except Exception as e:
             raise self.wrap_error(e=e)

{unstructured_ingest-1.0.44.dist-info → unstructured_ingest-1.0.46.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.44
+Version: 1.0.46
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0
@@ -60,6 +60,7 @@ Provides-Extra: delta-table
 Requires-Dist: boto3; extra == 'delta-table'
 Requires-Dist: deltalake; extra == 'delta-table'
 Requires-Dist: pandas; extra == 'delta-table'
+Requires-Dist: pyarrow; extra == 'delta-table'
 Requires-Dist: tenacity; extra == 'delta-table'
 Provides-Extra: discord
 Requires-Dist: discord-py; extra == 'discord'

{unstructured_ingest-1.0.44.dist-info → unstructured_ingest-1.0.46.dist-info}/RECORD

@@ -1,5 +1,5 @@
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=12SSwrWI8zU57pbaRSeJH9dGmuvWZXi056-PfBAhJTw,43
+unstructured_ingest/__version__.py,sha256=gy8wDmwG1gXjyS0ueUlncvuS37NRLiR1Derrpt71Y7g,43
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/errors_v2.py,sha256=9RuRCi7lbDxCguDz07y5RiHoQiFIOWwOD7xqzJ2B3Yw,436
 unstructured_ingest/logger.py,sha256=7e_7UeK6hVOd5BQ6i9NzRUAPCS_DF839Y8TjUDywraY,1428
@@ -68,7 +68,7 @@ unstructured_ingest/processes/connectors/azure_ai_search.py,sha256=szhSRXzUHk0DE
 unstructured_ingest/processes/connectors/chroma.py,sha256=q5_Fu4xb6_W_NyrPxVa3-jVwZLqVdlBNlR4dFvbd7l0,7235
 unstructured_ingest/processes/connectors/confluence.py,sha256=aA2B_FPdAjlVAJtmMldYu6lld2sR-6JL5tWh7yItiwg,22828
 unstructured_ingest/processes/connectors/couchbase.py,sha256=KCHoYDNya9B05NIB5D78zXoizFyfpJRepcYBe1nLSOs,12298
-unstructured_ingest/processes/connectors/delta_table.py,sha256=Y3yJPfwTyDdv7dqn54ZLZ4DBjg9OF2rXuUaNfbPCkvc,9993
+unstructured_ingest/processes/connectors/delta_table.py,sha256=JrpiX9V-YD1VhExKi6KFwlYatCheSs3t-xB3Td1BVFk,11487
 unstructured_ingest/processes/connectors/discord.py,sha256=CD-SBECMdr3pnmqbPvBMyPU2cBroXUhyW6F7L3laP6A,5348
 unstructured_ingest/processes/connectors/github.py,sha256=smHCz6jOH1p_hW2S25bYunBBj_pYjz8HTw6wkzaJz_A,7765
 unstructured_ingest/processes/connectors/gitlab.py,sha256=Fdq6_lk-By1JDmLGVjoKJkaHESiKTZsbvoHhMsljlE0,10114
@@ -109,7 +109,7 @@ unstructured_ingest/processes/connectors/fsspec/__init__.py,sha256=3HTdw4L4mdN4W
 unstructured_ingest/processes/connectors/fsspec/azure.py,sha256=31VNiG5YnXfhrFX7QJ2O1ubeWHxbe1sYVIztefbscAQ,7148
 unstructured_ingest/processes/connectors/fsspec/box.py,sha256=1gLS7xR2vbjgKBrQ4ZpI1fKTsJuIDfXuAzx_a4FzxG4,5873
 unstructured_ingest/processes/connectors/fsspec/dropbox.py,sha256=HwwKjQmjM7yFk9Esh_F20xDisRPXGUkFduzaasByRDE,8355
-unstructured_ingest/processes/connectors/fsspec/fsspec.py,sha256=NbId5WMq6M5kF3fYAwSUuaL2e_gutgmTATrE_X8okGY,14467
+unstructured_ingest/processes/connectors/fsspec/fsspec.py,sha256=viPp5NABSycN1RjAOyAYcHlYsd__Xc9owtvshLXFN4U,14477
 unstructured_ingest/processes/connectors/fsspec/gcs.py,sha256=ouxISCKpZTAj3T6pWGYbASu93wytJjl5WSICvQcrgfE,7172
 unstructured_ingest/processes/connectors/fsspec/s3.py,sha256=2ZV6b2E2pIsf_ab1Lty74FwpMnJZhpQUdamPgpwcKsQ,7141
 unstructured_ingest/processes/connectors/fsspec/sftp.py,sha256=pR_a2SgLjt8ffNkariHrPB1E0HVSTj5h3pt7KxTU3TI,6371
@@ -231,8 +231,8 @@ unstructured_ingest/utils/ndjson.py,sha256=nz8VUOPEgAFdhaDOpuveknvCU4x82fVwqE01q
 unstructured_ingest/utils/pydantic_models.py,sha256=BT_j15e4rX40wQbt8LUXbqfPhA3rJn1PHTI_G_A_EHY,1720
 unstructured_ingest/utils/string_and_date_utils.py,sha256=oXOI6rxXq-8ncbk7EoJK0WCcTXWj75EzKl8pfQMID3U,2522
 unstructured_ingest/utils/table.py,sha256=WZechczgVFvlodUWFcsnCGvBNh1xRm6hr0VbJTPxKAc,3669
-unstructured_ingest-1.0.44.dist-info/METADATA,sha256=PR_LHUUQP-2oayEmsoTGblqWKPmJt46QtijI7y-zni0,8795
-unstructured_ingest-1.0.44.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-unstructured_ingest-1.0.44.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-1.0.44.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-1.0.44.dist-info/RECORD,,
+unstructured_ingest-1.0.46.dist-info/METADATA,sha256=EHmSqyvPZRzxeO6_UvkHyVqPilKlOmbqHeAVJejCiLE,8842
+unstructured_ingest-1.0.46.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+unstructured_ingest-1.0.46.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-1.0.46.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-1.0.46.dist-info/RECORD,,