unstructured-ingest 1.1.0__tar.gz → 1.1.3__tar.gz

{unstructured_ingest-1.1.0 → unstructured_ingest-1.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.1.0
+Version: 1.1.3
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

unstructured_ingest-1.1.3/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.1.3"  # pragma: no cover

{unstructured_ingest-1.1.0 → unstructured_ingest-1.1.3}/unstructured_ingest/processes/connectors/delta_table.py

@@ -48,7 +48,8 @@ class DeltaTableAccessConfig(AccessConfig):
 
 class DeltaTableConnectionConfig(ConnectionConfig):
     access_config: Secret[DeltaTableAccessConfig] = Field(
-        default=Secret(DeltaTableAccessConfig()), validate_default=True
+        default_factory=lambda: Secret[DeltaTableAccessConfig](DeltaTableAccessConfig()),
+        validate_default=True,
     )
     aws_region: Optional[str] = Field(default=None, description="AWS Region")
     table_uri: str = Field(

{unstructured_ingest-1.1.0 → unstructured_ingest-1.1.3}/unstructured_ingest/processes/connectors/elasticsearch/elasticsearch.py

@@ -19,7 +19,6 @@ from unstructured_ingest.error import (
     DestinationConnectionError,
     SourceConnectionError,
     SourceConnectionNetworkError,
-    WriteError,
 )
 from unstructured_ingest.interfaces import (
     AccessConfig,
@@ -336,6 +335,8 @@ class ElasticsearchUploadStager(UploadStager):
 
     def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:
         data = element_dict.copy()
+        # when _op_type is not specified, it defaults to "index": 
+        # Overwrites if exists, creates if not.
         resp = {
             "_index": self.upload_stager_config.index_name,
             "_id": get_enhanced_element_id(element_dict=data, file_data=file_data),
@@ -397,23 +398,6 @@ class ElasticsearchUploader(Uploader):
 
         return parallel_bulk
 
-    def delete_by_record_id(self, client, file_data: FileData) -> None:
-        logger.debug(
-            f"deleting any content with metadata {RECORD_ID_LABEL}={file_data.identifier} "
-            f"from {self.upload_config.index_name} index"
-        )
-        delete_resp = client.delete_by_query(
-            index=self.upload_config.index_name,
-            body={"query": {"match": {self.upload_config.record_id_key: file_data.identifier}}},
-        )
-        logger.info(
-            "deleted {} records from index {}".format(
-                delete_resp["deleted"], self.upload_config.index_name
-            )
-        )
-        if failures := delete_resp.get("failures"):
-            raise WriteError(f"failed to delete records: {failures}")
-
     @requires_dependencies(["elasticsearch"], extras="elasticsearch")
     def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:  # noqa: E501
         from elasticsearch.helpers.errors import BulkIndexError
@@ -429,7 +413,6 @@ class ElasticsearchUploader(Uploader):
         )
 
         with self.connection_config.get_client() as client:
-            self.delete_by_record_id(client=client, file_data=file_data)
             if not client.indices.exists(index=self.upload_config.index_name):
                 logger.warning(
                     f"{(self.__class__.__name__).replace('Uploader', '')} index does not exist: "
@@ -446,6 +429,10 @@ class ElasticsearchUploader(Uploader):
                         thread_count=self.upload_config.num_threads,
                     )
                     collections.deque(iterator, maxlen=0)
+                    logger.info(
+                        f"uploaded batch of {len(batch)} elements to index "
+                        f"{self.upload_config.index_name}"
+                    )
                 except BulkIndexError as e:
                     sanitized_errors = [
                         self._sanitize_bulk_index_error(error) for error in e.errors

unstructured_ingest-1.1.0/unstructured_ingest/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.1.0"  # pragma: no cover