unstructured-ingest 1.0.41__tar.gz → 1.0.44__tar.gz

{unstructured_ingest-1.0.41 → unstructured_ingest-1.0.44}/.gitignore

@@ -108,6 +108,7 @@ celerybeat.pid
 
 # Environments
 .env
+.envrc
 .venv
 env/
 venv/

{unstructured_ingest-1.0.41 → unstructured_ingest-1.0.44}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.41
+Version: 1.0.44
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0
@@ -60,6 +60,7 @@ Provides-Extra: delta-table
 Requires-Dist: boto3; extra == 'delta-table'
 Requires-Dist: deltalake; extra == 'delta-table'
 Requires-Dist: pandas; extra == 'delta-table'
+Requires-Dist: tenacity; extra == 'delta-table'
 Provides-Extra: discord
 Requires-Dist: discord-py; extra == 'discord'
 Provides-Extra: doc

unstructured_ingest-1.0.44/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.0.44"  # pragma: no cover

{unstructured_ingest-1.0.41 → unstructured_ingest-1.0.44}/unstructured_ingest/embed/togetherai.py

@@ -22,7 +22,7 @@ if TYPE_CHECKING:
 class TogetherAIEmbeddingConfig(EmbeddingConfig):
     api_key: SecretStr = Field(description="API key for Together AI")
     embedder_model_name: str = Field(
-        default="togethercomputer/m2-bert-80M-8k-retrieval",
+        default="togethercomputer/m2-bert-80M-32k-retrieval",
         alias="model_name",
         description="Together AI model name",
     )

{unstructured_ingest-1.0.41 → unstructured_ingest-1.0.44}/unstructured_ingest/processes/connectors/astradb.py

@@ -197,8 +197,10 @@ class AstraDBIndexer(Indexer):
         id_batches = batch_generator(ids, self.index_config.batch_size)
         for batch in id_batches:
             batch_items = [BatchItem(identifier=b) for b in batch]
-            display_name = (f"{self.index_config.collection_name}-{self.index_config.keyspace}"
-                            f"-[{batch_items[0].identifier}..{batch_items[-1].identifier}]")
+            display_name = (
+                f"{self.index_config.collection_name}-{self.index_config.keyspace}"
+                f"-[{batch_items[0].identifier}..{batch_items[-1].identifier}]"
+            )
             fd = AstraDBBatchFileData(
                 connector_type=CONNECTOR_TYPE,
                 metadata=FileDataSourceMetadata(

{unstructured_ingest-1.0.41 → unstructured_ingest-1.0.44}/unstructured_ingest/processes/connectors/confluence.py

@@ -186,12 +186,15 @@ class ConfluenceIndexer(Indexer):
             pages = client.get_all_pages_from_space(
                 space=space_key,
                 start=0,
-                limit=self.index_config.max_num_of_docs_from_each_space,
                 expand=None,
                 content_type="page",  # blogpost and comment types not currently supported
                 status=None,
             )
-        doc_ids = [{"space_id": space_key, "doc_id": page["id"]} for page in pages]
+        # Limit the number of documents to max_num_of_docs_from_each_space
+        # Note: this is needed because the limit field in client.get_all_pages_from_space does
+        # not seem to work as expected
+        limited_pages = pages[: self.index_config.max_num_of_docs_from_each_space]
+        doc_ids = [{"space_id": space_key, "doc_id": page["id"]} for page in limited_pages]
         return doc_ids
 
     def run(self) -> Generator[FileData, None, None]:

{unstructured_ingest-1.0.41 → unstructured_ingest-1.0.44}/unstructured_ingest/processes/connectors/delta_table.py

@@ -1,7 +1,7 @@
-import os
+import logging
 import traceback
 from dataclasses import dataclass, field
-from multiprocessing import Process, Queue
+from multiprocessing import Process, Queue, current_process
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional
 from urllib.parse import urlparse
@@ -20,6 +20,7 @@ from unstructured_ingest.interfaces import (
 )
 from unstructured_ingest.logger import logger
 from unstructured_ingest.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.utils.constants import RECORD_ID_LABEL
 from unstructured_ingest.utils.data_prep import get_data_df, get_json_data
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.utils.table import convert_to_pandas_dataframe
@@ -47,18 +48,17 @@ class DeltaTableAccessConfig(AccessConfig):
 
 class DeltaTableConnectionConfig(ConnectionConfig):
     access_config: Secret[DeltaTableAccessConfig] = Field(
-        default=DeltaTableAccessConfig(), validate_default=True
+        default=Secret(DeltaTableAccessConfig()), validate_default=True
     )
     aws_region: Optional[str] = Field(default=None, description="AWS Region")
     table_uri: str = Field(
-        default=None,
         description=(
             "Local path or path to the target folder in the S3 bucket, "
             "formatted as s3://my-bucket/my-folder/"
         ),
     )
 
-    def update_storage_options(self, storage_options: dict) -> None:
+    def update_storage_options(self, storage_options: dict[str, str]) -> None:
         secrets = self.access_config.get_secret_value()
         if self.aws_region and secrets.aws_access_key_id and secrets.aws_secret_access_key:
             storage_options["AWS_REGION"] = self.aws_region
@@ -80,9 +80,10 @@ class DeltaTableUploadStager(UploadStager):
         default_factory=lambda: DeltaTableUploadStagerConfig()
     )
 
-    def run(
+    def run(  # type: ignore[override]
         self,
         elements_filepath: Path,
+        file_data: FileData,
         output_dir: Path,
         output_filename: str,
         **kwargs: Any,
@@ -91,6 +92,8 @@ class DeltaTableUploadStager(UploadStager):
         output_path = Path(output_dir) / Path(f"{output_filename}.parquet")
 
         df = convert_to_pandas_dataframe(elements_dict=elements_contents)
+        # Ensure per-record overwrite/delete semantics: tag each row with the record identifier
+        df[RECORD_ID_LABEL] = file_data.identifier
         df = df.dropna(axis=1, how="all")
         df.to_parquet(output_path)
 
@@ -138,41 +141,92 @@ class DeltaTableUploader(Uploader):
                 logger.error(f"failed to validate connection: {e}", exc_info=True)
                 raise DestinationConnectionError(f"failed to validate connection: {e}")
 
+    @requires_dependencies(["tenacity"], extras="delta-table")
     def upload_dataframe(self, df: "DataFrame", file_data: FileData) -> None:
-        updated_upload_path = os.path.join(
-            self.connection_config.table_uri, file_data.source_identifiers.relative_path
-        )
+        upload_path = self.connection_config.table_uri
         logger.info(
-            f"writing {len(df)} rows to destination table "
-            f"at {updated_upload_path}\ndtypes: {df.dtypes}",
+            f"writing {len(df)} rows to destination table at {upload_path}\ndtypes: {df.dtypes}",
         )
-        storage_options = {}
+        storage_options: dict[str, str] = {}
         self.connection_config.update_storage_options(storage_options=storage_options)
 
+        # Decide whether the Delta table already exists. If it does, we first delete all rows
+        # belonging to the current record and then append the fresh data. Otherwise we will
+        # create a brand-new table via an overwrite.
+
+        mode = "overwrite"
+        try:
+            from deltalake import DeltaTable  # pylint: disable=import-error
+
+            dt = DeltaTable(upload_path, storage_options=storage_options)
+            logger.debug(f"Table exists: deleting rows for {file_data.identifier}")
+            # Table exists – remove any previous rows for this record_id so that appending is
+            # effectively an idempotent overwrite for the record.
+            dt.delete(predicate=f"{RECORD_ID_LABEL} = '{file_data.identifier}'")
+            mode = "append"
+        except Exception:
+            # Table does not exist yet (or cannot be opened) – we will create it below with
+            # mode="overwrite". All other failures will be captured later by the writer.
+            logger.debug("Table does not exist: creating new table")
+
         writer_kwargs = {
-            "table_or_uri": updated_upload_path,
+            "table_or_uri": upload_path,
             "data": df,
-            "mode": "overwrite",
+            "mode": mode,
             "schema_mode": "merge",
             "storage_options": storage_options,
         }
-        queue = Queue()
-        # NOTE: deltalake writer on Linux sometimes can finish but still trigger a SIGABRT and cause
-        # ingest to fail, even though all tasks are completed normally. Putting the writer into a
-        # process mitigates this issue by ensuring python interpreter waits properly for deltalake's
-        # rust backend to finish
-        writer = Process(
-            target=write_deltalake_with_error_handling,
-            kwargs={"queue": queue, **writer_kwargs},
+
+        from tenacity import (
+            before_log,
+            retry,
+            retry_if_exception,
+            stop_after_attempt,
+            wait_random,
         )
-        writer.start()
-        writer.join()
 
-        # Check if the queue has any exception message
-        if not queue.empty():
-            error_message = queue.get()
-            logger.error(f"Exception occurred in write_deltalake: {error_message}")
-            raise RuntimeError(f"Error in write_deltalake: {error_message}")
+        def _is_commit_conflict(exc: BaseException) -> bool:  # noqa: ANN401
+            """Return True if exception looks like a Delta Lake commit conflict."""
+
+            return isinstance(exc, RuntimeError) and (
+                "CommitFailed" in str(exc) or "Metadata changed" in str(exc)
+            )
+
+        @retry(
+            stop=stop_after_attempt(10),
+            wait=wait_random(min=0.2, max=1.0),
+            before=before_log(logger, logging.DEBUG),
+            retry=retry_if_exception(_is_commit_conflict),
+            reraise=True,
+        )
+        def _single_attempt() -> None:
+            """One optimistic transaction: delete old rows, then append new ones."""
+
+            # NOTE: deltalake writer on Linux sometimes can finish but still trigger a SIGABRT and
+            # cause ingest to fail, even though all tasks are completed normally. Putting the writer
+            # into a process mitigates this issue by ensuring python interpreter waits properly for
+            # deltalake's rust backend to finish
+            queue: Queue[str] = Queue()
+
+            if current_process().daemon:
+                # write_deltalake_with_error_handling will push any traceback to our queue
+                write_deltalake_with_error_handling(queue=queue, **writer_kwargs)
+            else:
+                # On non-daemon processes we still guard against SIGABRT by running in a subprocess.
+                writer = Process(
+                    target=write_deltalake_with_error_handling,
+                    kwargs={"queue": queue, **writer_kwargs},
+                )
+                writer.start()
+                writer.join()
+
+            # Check if the queue has any exception message
+            if not queue.empty():
+                error_message = queue.get()
+                logger.error("Exception occurred in write_deltalake: %s", error_message)
+                raise RuntimeError(f"Error in write_deltalake: {error_message}")
+
+        _single_attempt()
 
     @requires_dependencies(["pandas"], extras="delta-table")
     def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
@@ -182,7 +236,7 @@ class DeltaTableUploader(Uploader):
         self.upload_dataframe(df=df, file_data=file_data)
 
     @requires_dependencies(["pandas"], extras="delta-table")
-    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:  # type: ignore[override]
         df = get_data_df(path)
         self.upload_dataframe(df=df, file_data=file_data)
 

{unstructured_ingest-1.0.41 → unstructured_ingest-1.0.44}/unstructured_ingest/processes/connectors/sql/sql.py

@@ -133,8 +133,10 @@ class SQLIndexer(Indexer, ABC):
 
         for batch in id_batches:
             batch_items = [BatchItem(identifier=str(b)) for b in batch]
-            display_name = (f"{self.index_config.table_name}-{self.index_config.id_column}"
-                            f"-[{batch_items[0].identifier}..{batch_items[-1].identifier}]")
+            display_name = (
+                f"{self.index_config.table_name}-{self.index_config.id_column}"
+                f"-[{batch_items[0].identifier}..{batch_items[-1].identifier}]"
+            )
             # Make sure the hash is always a positive number to create identified
             yield SqlBatchFileData(
                 connector_type=self.connector_type,
@@ -145,7 +147,7 @@ class SQLIndexer(Indexer, ABC):
                     table_name=self.index_config.table_name, id_column=self.index_config.id_column
                 ),
                 batch_items=batch_items,
-                display_name=display_name
+                display_name=display_name,
             )
 
 

unstructured_ingest-1.0.41/unstructured_ingest/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.0.41"  # pragma: no cover