unstructured-ingest 1.0.51__py3-none-any.whl → 1.0.53__py3-none-any.whl

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.51"  # pragma: no cover
+__version__ = "1.0.53"  # pragma: no cover

unstructured_ingest/processes/connectors/fsspec/fsspec.py

@@ -30,6 +30,7 @@ from unstructured_ingest.interfaces import (
 )
 from unstructured_ingest.logger import logger
 from unstructured_ingest.processes.connectors.fsspec.utils import sterilize_dict
+from unstructured_ingest.utils.filesystem import mkdir_concurrent_safe
 
 if TYPE_CHECKING:
     from fsspec import AbstractFileSystem
@@ -270,7 +271,7 @@ class FsspecDownloader(Downloader):
 
     def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
         download_path = self.get_download_path(file_data=file_data)
-        download_path.parent.mkdir(parents=True, exist_ok=True)
+        mkdir_concurrent_safe(download_path.parent)
         try:
             rpath = file_data.additional_metadata["original_file_path"]
             with self.connection_config.get_client(protocol=self.protocol) as client:
@@ -282,7 +283,7 @@ class FsspecDownloader(Downloader):
 
     async def async_run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
         download_path = self.get_download_path(file_data=file_data)
-        download_path.parent.mkdir(parents=True, exist_ok=True)
+        mkdir_concurrent_safe(download_path.parent)
         try:
             rpath = file_data.additional_metadata["original_file_path"]
             with self.connection_config.get_client(protocol=self.protocol) as client:

unstructured_ingest/processes/connectors/sharepoint.py

@@ -106,8 +106,11 @@ class SharepointIndexer(OnedriveIndexer):
         try:
             client_site = client.sites.get_by_url(self.connection_config.site).get().execute_query()
             site_drive_item = self.connection_config._get_drive_item(client_site)
-        except ClientRequestException:
-            logger.info("Site not found")
+        except ClientRequestException as e:
+            logger.error(f"Failed to access SharePoint site: {self.connection_config.site}")
+            raise SourceConnectionError(
+                f"Unable to access SharePoint site at {self.connection_config.site}: {str(e)}"
+            )
 
         path = self.index_config.path
         # Deprecated sharepoint sdk needed a default path. Microsoft Graph SDK does not.

unstructured_ingest/utils/__init__.py

@@ -0,0 +1,5 @@
+"""Utility functions for unstructured-ingest."""
+
+from unstructured_ingest.utils.filesystem import mkdir_concurrent_safe
+
+__all__ = ["mkdir_concurrent_safe"]

unstructured_ingest/utils/compression.py

@@ -6,6 +6,7 @@ from pathlib import Path
 from typing import Optional
 
 from unstructured_ingest.logger import logger
+from unstructured_ingest.utils.filesystem import mkdir_concurrent_safe
 
 ZIP_FILE_EXT = [".zip"]
 TAR_FILE_EXT = [".tar", ".tar.gz", ".tgz"]
@@ -17,7 +18,7 @@ def uncompress_file(filename: str, path: Optional[str] = None) -> str:
     """
     # Create path if it doesn't already exist
     if path:
-        Path(path).mkdir(parents=True, exist_ok=True)
+        mkdir_concurrent_safe(Path(path))
 
     if any(filename.endswith(ext) for ext in ZIP_FILE_EXT):
         return uncompress_zip_file(zip_filename=filename, path=path)

unstructured_ingest/utils/filesystem.py

@@ -0,0 +1,27 @@
+"""
+Filesystem utilities for concurrent operations.
+
+This module provides race-condition-safe filesystem operations that are needed
+when multiple processes operate on the same directory structures simultaneously.
+"""
+
+from pathlib import Path
+
+
+def mkdir_concurrent_safe(path: Path) -> None:
+    """
+    Create directory safely in concurrent environments, handling race conditions.
+    
+    This addresses the issue where Path.mkdir(parents=True, exist_ok=True) can still
+    raise FileExistsError when multiple processes attempt to create overlapping 
+    directory structures simultaneously. In this codebase, this occurs when multiple
+    files are being downloaded in parallel and archive extraction is happening in parallel.
+    
+    Related: https://github.com/python/cpython/pull/112966/files
+    Python core team used the same approach to fix zipfile race conditions.
+    """
+    try:
+        path.mkdir(parents=True, exist_ok=True)
+    except FileExistsError:
+        if not (path.exists() and path.is_dir()):
+            raise

{unstructured_ingest-1.0.51.dist-info → unstructured_ingest-1.0.53.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.51
+Version: 1.0.53
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

{unstructured_ingest-1.0.51.dist-info → unstructured_ingest-1.0.53.dist-info}/RECORD

@@ -1,5 +1,5 @@
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=jkcvVBVHKL5jaZGZh7CF4yFZfuGwbHAHHUGV-bTIVBs,43
+unstructured_ingest/__version__.py,sha256=R6_J5XcJv3o4se9ZaxG3ld-O1m1WwU2CyxpqODyEhmg,43
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/errors_v2.py,sha256=9RuRCi7lbDxCguDz07y5RiHoQiFIOWwOD7xqzJ2B3Yw,436
 unstructured_ingest/logger.py,sha256=7e_7UeK6hVOd5BQ6i9NzRUAPCS_DF839Y8TjUDywraY,1428
@@ -84,7 +84,7 @@ unstructured_ingest/processes/connectors/outlook.py,sha256=6HHubZI_zttEfYp0XNd4Y
 unstructured_ingest/processes/connectors/pinecone.py,sha256=jCabAqKQyBFzaGjphxLMr57y7P0Z15Jd9Jj-JM40YnU,15090
 unstructured_ingest/processes/connectors/redisdb.py,sha256=rTihbfv0Mlk1eo5Izn-JXRu5Ad5C-KD58nSqeKsaZJ8,8024
 unstructured_ingest/processes/connectors/salesforce.py,sha256=N_UoebrhzXZNWw-X7lg8_qAziXx5L_d8XHnHWKNNYR8,11767
-unstructured_ingest/processes/connectors/sharepoint.py,sha256=oGBZ8czM3XwcfhYwqnkb9A9YVAGWTmCvD5vE3Q_vfUs,6303
+unstructured_ingest/processes/connectors/sharepoint.py,sha256=Wgv9Pih9S9FmQJud1bg7kj_qqi55d7QZ48LqlTU_mk0,6509
 unstructured_ingest/processes/connectors/slack.py,sha256=oboIfX7ayBMK0te5Nv50iyL3FQJFXJbRxZSQaCMp3kM,9318
 unstructured_ingest/processes/connectors/utils.py,sha256=TAd0hb1f291N-q7-TUe6JKSCGkhqDyo7Ij8zmliBZUc,2071
 unstructured_ingest/processes/connectors/vectara.py,sha256=xrC6jkgW8BII4UjdzUelDu122xT484cpfMTK2wl-sko,12292
@@ -109,7 +109,7 @@ unstructured_ingest/processes/connectors/fsspec/__init__.py,sha256=3HTdw4L4mdN4W
 unstructured_ingest/processes/connectors/fsspec/azure.py,sha256=31VNiG5YnXfhrFX7QJ2O1ubeWHxbe1sYVIztefbscAQ,7148
 unstructured_ingest/processes/connectors/fsspec/box.py,sha256=1gLS7xR2vbjgKBrQ4ZpI1fKTsJuIDfXuAzx_a4FzxG4,5873
 unstructured_ingest/processes/connectors/fsspec/dropbox.py,sha256=HwwKjQmjM7yFk9Esh_F20xDisRPXGUkFduzaasByRDE,8355
-unstructured_ingest/processes/connectors/fsspec/fsspec.py,sha256=viPp5NABSycN1RjAOyAYcHlYsd__Xc9owtvshLXFN4U,14477
+unstructured_ingest/processes/connectors/fsspec/fsspec.py,sha256=S1siX888TfHAByEXuvOqkTbcNAzx-m5UNqhKjiEKR5s,14524
 unstructured_ingest/processes/connectors/fsspec/gcs.py,sha256=ouxISCKpZTAj3T6pWGYbASu93wytJjl5WSICvQcrgfE,7172
 unstructured_ingest/processes/connectors/fsspec/s3.py,sha256=2ZV6b2E2pIsf_ab1Lty74FwpMnJZhpQUdamPgpwcKsQ,7141
 unstructured_ingest/processes/connectors/fsspec/sftp.py,sha256=pR_a2SgLjt8ffNkariHrPB1E0HVSTj5h3pt7KxTU3TI,6371
@@ -220,19 +220,20 @@ unstructured_ingest/processes/connectors/zendesk/client.py,sha256=GvPIpx4aYdD58-
 unstructured_ingest/processes/connectors/zendesk/zendesk.py,sha256=doS6d7ZhXBgJN8aPVf7vnQr8BciQbzX8-4yl4_hDZ7w,9253
 unstructured_ingest/processes/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 unstructured_ingest/processes/utils/blob_storage.py,sha256=apMUmm9loxdbTRkkLH4VhG9kUVyiw9PFUJheSDxSxPk,1023
-unstructured_ingest/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+unstructured_ingest/utils/__init__.py,sha256=URnsQu-y3Vmc7vn6GVL2sYuXxlSJ3naR3c9o6oKSm3w,157
 unstructured_ingest/utils/chunking.py,sha256=9b3sXMA6L8RW5xAkKQbwdtVudGLAcj_sgT6Grh5tyYM,1870
-unstructured_ingest/utils/compression.py,sha256=_BkFREoa0fkJ6z-1lY76HCmy8mLymbPCg55iMUQTd5c,2653
+unstructured_ingest/utils/compression.py,sha256=PPC-ys3qEAtELf6-irhp8v8M634pFFCJEvA6o7PXaLI,2712
 unstructured_ingest/utils/constants.py,sha256=pDspTYz-nEojHBqrZNfssGEiujmVa02pIWL63PQP9sU,103
 unstructured_ingest/utils/data_prep.py,sha256=yqrv7x_nlj0y3uaN0m0Bnsekb7VIQnwABWPa24KU5QI,7426
 unstructured_ingest/utils/dep_check.py,sha256=SXXcUna2H0RtxA6j1S2NGkvQa9JP2DujWhmyBa7776Y,2400
+unstructured_ingest/utils/filesystem.py,sha256=nWxpQd8ogTgmXb7ZouupX6sE5v_qFXNzPl4DtZSStwE,1036
 unstructured_ingest/utils/html.py,sha256=78ou1vVZ0SJ3c6-Nmxg2iR5MoqubJTvwiuTNMtSFDh4,6816
 unstructured_ingest/utils/ndjson.py,sha256=nz8VUOPEgAFdhaDOpuveknvCU4x82fVwqE01qAbElH0,1201
 unstructured_ingest/utils/pydantic_models.py,sha256=BT_j15e4rX40wQbt8LUXbqfPhA3rJn1PHTI_G_A_EHY,1720
 unstructured_ingest/utils/string_and_date_utils.py,sha256=oXOI6rxXq-8ncbk7EoJK0WCcTXWj75EzKl8pfQMID3U,2522
 unstructured_ingest/utils/table.py,sha256=WZechczgVFvlodUWFcsnCGvBNh1xRm6hr0VbJTPxKAc,3669
-unstructured_ingest-1.0.51.dist-info/METADATA,sha256=-YIqizM0jt0c3B2u2Kh9kzTlQ0_FK3JDr0dkuLpbELU,8842
-unstructured_ingest-1.0.51.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-unstructured_ingest-1.0.51.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-1.0.51.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-1.0.51.dist-info/RECORD,,
+unstructured_ingest-1.0.53.dist-info/METADATA,sha256=FTZfBOPcfl-pbxoBAIK-d0MwQKfN5nbJXQh92H1QwMs,8842
+unstructured_ingest-1.0.53.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+unstructured_ingest-1.0.53.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-1.0.53.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-1.0.53.dist-info/RECORD,,