unstructured-ingest 1.1.2__tar.gz → 1.2.0__tar.gz

{unstructured_ingest-1.1.2 → unstructured_ingest-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.1.2
+Version: 1.2.0
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0
@@ -12,12 +12,11 @@ Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
-Requires-Python: <3.13,>=3.9
+Requires-Python: <3.13,>=3.10
 Requires-Dist: certifi>=2025.7.14
 Requires-Dist: click
 Requires-Dist: opentelemetry-sdk

{unstructured_ingest-1.1.2 → unstructured_ingest-1.2.0}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "unstructured_ingest"
 description = "Local ETL data pipeline to get data RAG ready"
-requires-python = ">=3.9, <3.13"
+requires-python = ">=3.10, <3.13"
 authors = [{name = "Unstructured Technologies", email = "devops@unstructuredai.io"}]
 classifiers = [
     "Development Status :: 4 - Beta",
@@ -11,7 +11,6 @@ classifiers = [
     "License :: OSI Approved :: Apache Software License",
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
@@ -147,21 +146,13 @@ test = [
 ]
 # Add constraints needed for CI
 ci = [
-    # consistency with local-inference-pin
-    "protobuf<4.24",
     "grpcio>=1.65.5",
     # TODO: Pinned in transformers package, remove when that gets updated
     "tokenizers>=0.19,<0.20",
-    # TODO: Constaint due to boto, with python before 3.10 not requiring openssl 1.1.1, remove when that gets
-    # updated or we drop support for 3.9
-    "urllib3<1.27",
     # TODO: Constriant due to aiobotocore, remove when that gets updates:
     "botocore<1.34.132",
     # TODO: Constriant due to both 8.5.0 and 8.4.0 being installed during pip-compile
     "importlib-metadata>=8.5.0",
-    # TODO: Constraint due to boto, with python before 3.10 not requiring openssl 1.1.1, remove when that gets
-    # updated or we drop support for 3.9
-    "urllib3<1.27",
     "unstructured-client>= 0.25.8",
     "fsspec==2024.5.0",
     # python 3.12 support

unstructured_ingest-1.2.0/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.2.0"  # pragma: no cover

{unstructured_ingest-1.1.2 → unstructured_ingest-1.2.0}/unstructured_ingest/otel.py

@@ -32,21 +32,23 @@ class LogSpanExporter(ConsoleSpanExporter):
             self.log_out(self.formatter(span))
         return SpanExportResult.SUCCESS
 
+
 def get_log_out() -> Callable:
     level_names_mapping = {
-        'CRITICAL': logging.CRITICAL,
-        'FATAL': logging.FATAL,
-        'ERROR': logging.ERROR,
-        'WARN': logging.WARNING,
-        'WARNING': logging.WARNING,
-        'INFO': logging.INFO,
-        'DEBUG': logging.DEBUG,
-        'NOTSET': logging.NOTSET,
+        "CRITICAL": logging.CRITICAL,
+        "FATAL": logging.FATAL,
+        "ERROR": logging.ERROR,
+        "WARN": logging.WARNING,
+        "WARNING": logging.WARNING,
+        "INFO": logging.INFO,
+        "DEBUG": logging.DEBUG,
+        "NOTSET": logging.NOTSET,
     }
     log_level = os.getenv("OTEL_LOG_LEVEL", "DEBUG").upper()
     log_level_int = level_names_mapping.get(log_level, logging.DEBUG)
     return lambda message: logger.log(log_level_int, message)
 
+
 @dataclass
 class OtelHandler:
     otel_endpoint: Optional[str] = None

{unstructured_ingest-1.1.2 → unstructured_ingest-1.2.0}/unstructured_ingest/processes/connectors/elasticsearch/elasticsearch.py

@@ -19,7 +19,6 @@ from unstructured_ingest.error import (
     DestinationConnectionError,
     SourceConnectionError,
     SourceConnectionNetworkError,
-    WriteError,
 )
 from unstructured_ingest.interfaces import (
     AccessConfig,
@@ -336,6 +335,8 @@ class ElasticsearchUploadStager(UploadStager):
 
     def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:
         data = element_dict.copy()
+        # when _op_type is not specified, it defaults to "index":
+        # Overwrites if exists, creates if not.
         resp = {
             "_index": self.upload_stager_config.index_name,
             "_id": get_enhanced_element_id(element_dict=data, file_data=file_data),
@@ -397,23 +398,6 @@ class ElasticsearchUploader(Uploader):
 
         return parallel_bulk
 
-    def delete_by_record_id(self, client, file_data: FileData) -> None:
-        logger.debug(
-            f"deleting any content with metadata {RECORD_ID_LABEL}={file_data.identifier} "
-            f"from {self.upload_config.index_name} index"
-        )
-        delete_resp = client.delete_by_query(
-            index=self.upload_config.index_name,
-            body={"query": {"match": {self.upload_config.record_id_key: file_data.identifier}}},
-        )
-        logger.info(
-            "deleted {} records from index {}".format(
-                delete_resp["deleted"], self.upload_config.index_name
-            )
-        )
-        if failures := delete_resp.get("failures"):
-            raise WriteError(f"failed to delete records: {failures}")
-
     @requires_dependencies(["elasticsearch"], extras="elasticsearch")
     def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:  # noqa: E501
         from elasticsearch.helpers.errors import BulkIndexError
@@ -429,7 +413,6 @@ class ElasticsearchUploader(Uploader):
         )
 
         with self.connection_config.get_client() as client:
-            self.delete_by_record_id(client=client, file_data=file_data)
             if not client.indices.exists(index=self.upload_config.index_name):
                 logger.warning(
                     f"{(self.__class__.__name__).replace('Uploader', '')} index does not exist: "
@@ -446,6 +429,10 @@ class ElasticsearchUploader(Uploader):
                         thread_count=self.upload_config.num_threads,
                     )
                     collections.deque(iterator, maxlen=0)
+                    logger.info(
+                        f"uploaded batch of {len(batch)} elements to index "
+                        f"{self.upload_config.index_name}"
+                    )
                 except BulkIndexError as e:
                     sanitized_errors = [
                         self._sanitize_bulk_index_error(error) for error in e.errors

{unstructured_ingest-1.1.2 → unstructured_ingest-1.2.0}/unstructured_ingest/processes/connectors/fsspec/fsspec.py

@@ -385,7 +385,7 @@ class FsspecUploader(Uploader):
 
     def __post_init__(self):
         super().__post_init__()
-        # TODO once python3.9 no longer supported and kw_only is allowed in dataclasses, remove:
+        # TODO: Consider using `kw_only` instead
         if not self.upload_config:
             raise TypeError(
                 f"{self.__class__.__name__}.__init__() "

{unstructured_ingest-1.1.2 → unstructured_ingest-1.2.0}/unstructured_ingest/utils/filesystem.py

@@ -11,12 +11,12 @@ from pathlib import Path
 def mkdir_concurrent_safe(path: Path) -> None:
     """
     Create directory safely in concurrent environments, handling race conditions.
-    
+
     This addresses the issue where Path.mkdir(parents=True, exist_ok=True) can still
-    raise FileExistsError when multiple processes attempt to create overlapping 
+    raise FileExistsError when multiple processes attempt to create overlapping
     directory structures simultaneously. In this codebase, this occurs when multiple
     files are being downloaded in parallel and archive extraction is happening in parallel.
-    
+
     Related: https://github.com/python/cpython/pull/112966/files
     Python core team used the same approach to fix zipfile race conditions.
     """
@@ -24,4 +24,4 @@ def mkdir_concurrent_safe(path: Path) -> None:
         path.mkdir(parents=True, exist_ok=True)
     except FileExistsError:
         if not (path.exists() and path.is_dir()):
-            raise
+            raise

unstructured_ingest-1.1.2/unstructured_ingest/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.1.2"  # pragma: no cover