unstructured-ingest 1.0.49__tar.gz → 1.0.52__tar.gz

{unstructured_ingest-1.0.49 → unstructured_ingest-1.0.52}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.49
+Version: 1.0.52
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

unstructured_ingest-1.0.52/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.0.52"  # pragma: no cover

{unstructured_ingest-1.0.49 → unstructured_ingest-1.0.52}/unstructured_ingest/cli/README.md

@@ -7,7 +7,7 @@ source and destination connectors.
 
 To manually run the cli:
 ```shell
-PYTHONPATH=. python unstructured_ingest/v2/main.py --help
+PYTHONPATH=. python unstructured_ingest/main.py --help
 ```
 
 The `main.py` file simply wraps the generated Click command created in `cli.py`.

{unstructured_ingest-1.0.49 → unstructured_ingest-1.0.52}/unstructured_ingest/processes/connectors/fsspec/fsspec.py

@@ -30,6 +30,7 @@ from unstructured_ingest.interfaces import (
 )
 from unstructured_ingest.logger import logger
 from unstructured_ingest.processes.connectors.fsspec.utils import sterilize_dict
+from unstructured_ingest.utils.filesystem import mkdir_concurrent_safe
 
 if TYPE_CHECKING:
     from fsspec import AbstractFileSystem
@@ -270,7 +271,7 @@ class FsspecDownloader(Downloader):
 
     def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
         download_path = self.get_download_path(file_data=file_data)
-        download_path.parent.mkdir(parents=True, exist_ok=True)
+        mkdir_concurrent_safe(download_path.parent)
         try:
             rpath = file_data.additional_metadata["original_file_path"]
             with self.connection_config.get_client(protocol=self.protocol) as client:
@@ -282,7 +283,7 @@ class FsspecDownloader(Downloader):
 
     async def async_run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
         download_path = self.get_download_path(file_data=file_data)
-        download_path.parent.mkdir(parents=True, exist_ok=True)
+        mkdir_concurrent_safe(download_path.parent)
         try:
             rpath = file_data.additional_metadata["original_file_path"]
             with self.connection_config.get_client(protocol=self.protocol) as client:

{unstructured_ingest-1.0.49 → unstructured_ingest-1.0.52}/unstructured_ingest/processes/connectors/sharepoint.py

@@ -150,6 +150,7 @@ class SharepointDownloader(OnedriveDownloader):
             site_drive_item = self.connection_config._get_drive_item(client_site)
         except ClientRequestException:
             logger.info("Site not found")
+            raise SourceConnectionError(f"Site not found: {self.connection_config.site}")
         file = site_drive_item.get_by_path(server_relative_path).get().execute_query()
 
         if not file:

unstructured_ingest-1.0.52/unstructured_ingest/utils/__init__.py

@@ -0,0 +1,5 @@
+"""Utility functions for unstructured-ingest."""
+
+from unstructured_ingest.utils.filesystem import mkdir_concurrent_safe
+
+__all__ = ["mkdir_concurrent_safe"]

{unstructured_ingest-1.0.49 → unstructured_ingest-1.0.52}/unstructured_ingest/utils/compression.py

@@ -6,6 +6,7 @@ from pathlib import Path
 from typing import Optional
 
 from unstructured_ingest.logger import logger
+from unstructured_ingest.utils.filesystem import mkdir_concurrent_safe
 
 ZIP_FILE_EXT = [".zip"]
 TAR_FILE_EXT = [".tar", ".tar.gz", ".tgz"]
@@ -17,7 +18,7 @@ def uncompress_file(filename: str, path: Optional[str] = None) -> str:
     """
     # Create path if it doesn't already exist
     if path:
-        Path(path).mkdir(parents=True, exist_ok=True)
+        mkdir_concurrent_safe(Path(path))
 
     if any(filename.endswith(ext) for ext in ZIP_FILE_EXT):
         return uncompress_zip_file(zip_filename=filename, path=path)

unstructured_ingest-1.0.52/unstructured_ingest/utils/filesystem.py

@@ -0,0 +1,27 @@
+"""
+Filesystem utilities for concurrent operations.
+
+This module provides race-condition-safe filesystem operations that are needed
+when multiple processes operate on the same directory structures simultaneously.
+"""
+
+from pathlib import Path
+
+
+def mkdir_concurrent_safe(path: Path) -> None:
+    """
+    Create directory safely in concurrent environments, handling race conditions.
+    
+    This addresses the issue where Path.mkdir(parents=True, exist_ok=True) can still
+    raise FileExistsError when multiple processes attempt to create overlapping 
+    directory structures simultaneously. In this codebase, this occurs when multiple
+    files are being downloaded in parallel and archive extraction is happening in parallel.
+    
+    Related: https://github.com/python/cpython/pull/112966/files
+    Python core team used the same approach to fix zipfile race conditions.
+    """
+    try:
+        path.mkdir(parents=True, exist_ok=True)
+    except FileExistsError:
+        if not (path.exists() and path.is_dir()):
+            raise

unstructured_ingest-1.0.49/unstructured_ingest/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.0.49"  # pragma: no cover