unstructured-ingest 1.0.45__tar.gz → 1.0.46.dev0__tar.gz

{unstructured_ingest-1.0.45 → unstructured_ingest-1.0.46.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.45
+Version: 1.0.46.dev0
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0
@@ -60,6 +60,7 @@ Provides-Extra: delta-table
 Requires-Dist: boto3; extra == 'delta-table'
 Requires-Dist: deltalake; extra == 'delta-table'
 Requires-Dist: pandas; extra == 'delta-table'
+Requires-Dist: pyarrow; extra == 'delta-table'
 Requires-Dist: tenacity; extra == 'delta-table'
 Provides-Extra: discord
 Requires-Dist: discord-py; extra == 'discord'

unstructured_ingest-1.0.46.dev0/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.0.46-dev0"  # pragma: no cover

{unstructured_ingest-1.0.45 → unstructured_ingest-1.0.46.dev0}/unstructured_ingest/processes/connectors/delta_table.py

@@ -1,7 +1,7 @@
 import logging
 import traceback
 from dataclasses import dataclass, field
-from multiprocessing import Process, Queue, current_process
+from multiprocessing import Queue, current_process
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional
 from urllib.parse import urlparse
@@ -186,10 +186,25 @@ class DeltaTableUploader(Uploader):
         )
 
         def _is_commit_conflict(exc: BaseException) -> bool:  # noqa: ANN401
-            """Return True if exception looks like a Delta Lake commit conflict."""
-
-            return isinstance(exc, RuntimeError) and (
-                "CommitFailed" in str(exc) or "Metadata changed" in str(exc)
+            """Return True if exception looks like a Delta Lake commit conflict.
+
+            Besides the canonical *CommitFailed* / *Metadata changed* errors that
+            deltalake surfaces when two writers clash, we occasionally hit
+            messages such as *Delta transaction failed, version 0 already
+            exists* while multiple processes race to create the very first log
+            entry. These situations are equally safe to retry, so detect them
+            too.
+            """
+
+            return isinstance(exc, RuntimeError) and any(
+                marker in str(exc)
+                for marker in (
+                    "CommitFailed",
+                    "Metadata changed",
+                    "version 0 already exists",
+                    "version already exists",
+                    "Delta transaction failed",
+                )
             )
 
         @retry(
@@ -206,25 +221,39 @@ class DeltaTableUploader(Uploader):
             # cause ingest to fail, even though all tasks are completed normally. Putting the writer
             # into a process mitigates this issue by ensuring python interpreter waits properly for
             # deltalake's rust backend to finish
-            queue: Queue[str] = Queue()
+            # Use a multiprocessing context that relies on 'spawn' to avoid inheriting the
+            # parent process' Tokio runtime, which leads to `pyo3_runtime.PanicException`.
+            from multiprocessing import get_context
+
+            ctx = get_context("spawn")
+            queue: "Queue[str]" = ctx.Queue()
 
             if current_process().daemon:
                 # write_deltalake_with_error_handling will push any traceback to our queue
                 write_deltalake_with_error_handling(queue=queue, **writer_kwargs)
             else:
-                # On non-daemon processes we still guard against SIGABRT by running in a subprocess.
-                writer = Process(
+                # On non-daemon processes we still guard against SIGABRT by running in a
+                # dedicated subprocess created via the 'spawn' method.
+                writer = ctx.Process(
                     target=write_deltalake_with_error_handling,
                     kwargs={"queue": queue, **writer_kwargs},
                 )
                 writer.start()
                 writer.join()
 
-            # Check if the queue has any exception message
-            if not queue.empty():
-                error_message = queue.get()
-                logger.error("Exception occurred in write_deltalake: %s", error_message)
-                raise RuntimeError(f"Error in write_deltalake: {error_message}")
+                # First surface any traceback captured inside the subprocess so users see the real
+                # root-cause instead of a generic non-zero exit code.
+                if not queue.empty():
+                    error_message = queue.get()
+                    logger.error("Exception occurred in write_deltalake: %s", error_message)
+                    raise RuntimeError(f"Error in write_deltalake: {error_message}")
+
+                # If the subprocess terminated abnormally but produced no traceback (e.g., SIGABRT),
+                # still raise a helpful error for callers.
+                if not current_process().daemon and writer.exitcode != 0:
+                    raise RuntimeError(
+                        f"write_deltalake subprocess exited with code {writer.exitcode}"
+                    )
 
         _single_attempt()
 

unstructured_ingest-1.0.45/unstructured_ingest/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.0.45"  # pragma: no cover