data-sourcerer 0.7.0__tar.gz → 0.7.2__tar.gz

{data_sourcerer-0.7.0 → data_sourcerer-0.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-sourcerer
-Version: 0.7.0
+Version: 0.7.2
 Summary: Sourcerer is a terminal cloud storage navigator.
 Author-email: Bohdana Kuzmenko <bohdana.kuzmenko.dev@gmail.com>
 License: MIT

{data_sourcerer-0.7.0 → data_sourcerer-0.7.2}/pyproject.toml

@@ -76,9 +76,6 @@ source = [
     "src/sourcerer/infrastructure",
 ]
 
-[tool.coverage.report]
-fail_under = 75
-
 [tool.ruff]
 line-length = 120
 target-version = "py312"

{data_sourcerer-0.7.0 → data_sourcerer-0.7.2}/sourcerer/__init__.py

@@ -14,4 +14,4 @@ The application is structured using a clean architecture approach with:
 
 name = "sourcerer"
 package_name = "data_sourcerer"
-__version__ = "0.7.0"
+__version__ = "0.7.2"

{data_sourcerer-0.7.0 → data_sourcerer-0.7.2}/sourcerer/domain/storage_provider/services.py

@@ -4,7 +4,7 @@ Base storage provider service interface.
 This module defines the abstract base class for storage provider services,
 providing a common interface for cloud storage operations.
 """
-
+import threading
 from abc import ABC, abstractmethod
 from collections.abc import Callable
 from pathlib import Path
@@ -93,20 +93,28 @@ class BaseStorageProviderService(ABC):
         storage_path: str,
         source_path: Path,
         dest_path: str | None = None,
+        cancel_event: threading.Event | None = None,
+        progress_callback: Callable | None = None,
     ) -> None:
         """
-        Upload a file to the specified storage path.
+        upload a file to the specified storage path.
 
         Args:
             storage (str): The storage identifier
             storage_path (str): The path within the storage to upload
             source_path (Path): Local file path to upload
             dest_path (str, optional): Destination path in storage. Defaults to None.
+            cancel_event (threading.Event, optional): Event to signal upload cancellation. Defaults to None.
+            progress_callback (callable, optional): Callback function for progress updates. Defaults to None.
         """
 
     @abstractmethod
     def download_storage_item(
-        self, storage: str, key: str, progress_callback: Callable | None = None
+        self,
+        storage: str,
+        key: str,
+        progress_callback: Callable | None = None,
+        cancel_event: threading.Event | None = None,
     ) -> str:
         """
         Download a file from storage to local filesystem.
@@ -115,7 +123,7 @@ class BaseStorageProviderService(ABC):
             storage (str): The storage identifier
             key (str): The key/path of the item to download
             progress_callback (callable, optional): Callback function for progress updates. Defaults to None.
-
+            cancel_event (threading.Event, optional): Event to signal download cancellation. Defaults to None.
         Returns:
             str: Path to the downloaded file
         """

{data_sourcerer-0.7.0 → data_sourcerer-0.7.2}/sourcerer/infrastructure/storage_provider/services/azure.py

@@ -4,15 +4,19 @@ Implementation of Azure storage provider services.
 This module provides concrete implementations of the BaseStorageProviderService
 interface for various cloud storage providers.
 """
-
+import asyncio
+import base64
 import os.path
+import shutil
+import tempfile
 import threading
 from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 from typing import Any
 
 from azure.mgmt.storage import StorageManagementClient
-from azure.storage.blob import BlobServiceClient
+from azure.storage.blob import BlobBlock, BlobServiceClient
 from cachetools import LRUCache
 from platformdirs import user_downloads_dir
 
@@ -35,6 +39,7 @@ from sourcerer.infrastructure.storage_provider.exceptions import (
 )
 from sourcerer.infrastructure.storage_provider.registry import storage_provider
 from sourcerer.infrastructure.utils import generate_uuid, is_text_file
+from sourcerer.settings import DOWNLOAD_BLOCK_SIZE, MULTIPART_UPLOAD_BLOCK_SIZE
 
 
 @storage_provider(StorageProvider.AzureStorage)
@@ -141,13 +146,16 @@ class AzureStorageProviderService(BaseStorageProviderService):
                 folders.update([i.name for i in containers_client.list_containers()])
             else:
                 path_parts = path.split("/", 1)
-                if len(path_parts) > 1:
-                    path, prefix = path_parts[0], path_parts[1] + "/" + prefix
-                blobs_client = containers_client.get_container_client(path)
+
+                container = path_parts[0]
+                base_path = "" if len(path_parts) == 1 else path_parts[1] + "/"
+
+                blobs_client = containers_client.get_container_client(container)
+
                 for blob in blobs_client.walk_blobs(
-                    name_starts_with=prefix, delimiter="/"
+                    name_starts_with=base_path + prefix, delimiter="/"
                 ):
-                    remaining_path = blob.name[len(prefix) :]
+                    remaining_path = blob.name[len(base_path) :]
                     if "/" in remaining_path:
                         folder_name = remaining_path.split("/")[0]
                         if folder_name not in folders:
@@ -158,8 +166,8 @@ class AzureStorageProviderService(BaseStorageProviderService):
                         File(
                             generate_uuid(),
                             remaining_path,
-                            size=blob.size,
-                            date_modified=blob.last_modified,
+                            size=blob.size,  # type: ignore
+                            date_modified=blob.last_modified,  # type: ignore
                             is_text=is_text_file(blob.name),
                         )
                     )
@@ -208,6 +216,8 @@ class AzureStorageProviderService(BaseStorageProviderService):
         storage_path: str,
         source_path: Path,
         dest_path: str | None = None,
+        cancel_event: threading.Event | None = None,
+        progress_callback: Callable | None = None,
     ) -> None:
         """
         Upload a file to the specified Azure container path.
@@ -216,6 +226,8 @@ class AzureStorageProviderService(BaseStorageProviderService):
             storage_path (str): The path within the container to upload
             source_path (Path): Local file path to upload
             dest_path (str, optional): Destination path in storage. Defaults to None.
+            cancel_event (threading.Event, optional): Event to signal upload cancellation. Defaults to None.
+            progress_callback (Callable, optional): Callback function for progress updates. Defaults to None.
         """
         try:
             if not storage_path:
@@ -232,16 +244,38 @@ class AzureStorageProviderService(BaseStorageProviderService):
             storage_path = storage_path_parts[1] if len(storage_path_parts) > 1 else ""
             blob_name = os.path.join(storage_path, dest_path or source_path.name)
 
-            blob_client = containers_client.get_container_client(container)
-            with open(source_path, "rb") as file_handle:
-                blob_client.upload_blob(
-                    blob_name or source_path.name, file_handle, overwrite=True
-                )
+            if source_path.stat().st_size <= MULTIPART_UPLOAD_BLOCK_SIZE:
+                blob_client = containers_client.get_container_client(container)
+                with open(source_path, "rb") as file_handle:
+                    blob_client.upload_blob(
+                        blob_name or source_path.name, file_handle, overwrite=True
+                    )
+                if progress_callback:
+                    progress_callback(source_path.stat().st_size)
+            else:
+                try:
+                    run_async_sync_safe(
+                        self.upload_multipart(
+                            containers_client,
+                            container,
+                            source_path,
+                            blob_name,
+                            MULTIPART_UPLOAD_BLOCK_SIZE,
+                            cancel_event,
+                            progress_callback,
+                        )
+                    )
+                except Exception:
+                    raise
         except Exception as ex:
             raise UploadStorageItemsError(str(ex)) from ex
 
     def download_storage_item(
-        self, storage: str, key: str, progress_callback: Callable | None = None
+        self,
+        storage: str,
+        key: str,
+        progress_callback: Callable | None = None,
+        cancel_event: threading.Event | None = None,
     ) -> str:
         """
         Download a file from Azure to the local filesystem.
@@ -250,19 +284,52 @@ class AzureStorageProviderService(BaseStorageProviderService):
             storage (str): The container name
             key (str): The key/path of the item to download
             progress_callback (Callable, optional): Callback function for progress updates. Defaults to None.
+            cancel_event (threading.Event, optional): Event to signal download cancellation. Defaults to None.
         """
+        download_path = None
+        download_tmp_path = None
         try:
             download_path = Path(user_downloads_dir()) / Path(key).name
+            suffix = Path(key).suffix
+            download_tmp_path = (
+                Path(user_downloads_dir())
+                / f"{next(tempfile._get_candidate_names())}{suffix}"  # type: ignore
+            )
 
             containers_client = self.get_containers_client(storage)
             path_parts = key.split("/", 1)
             container, blob_name = path_parts
             blob_client = containers_client.get_container_client(container)
-            with open(download_path, "wb") as file:
-                download_stream = blob_client.download_blob(blob_name)
-                file.write(download_stream.readall())
+            blob_stream = blob_client.download_blob(blob_name)
+            total_bytes = blob_stream.properties.size
+
+            with open(download_tmp_path, "wb") as file:
+                if total_bytes <= DOWNLOAD_BLOCK_SIZE:
+                    file.write(blob_stream.readall())
+                else:
+                    downloaded = 0
+                    while downloaded < total_bytes:
+                        if cancel_event and cancel_event.is_set():
+                            raise Exception("Download cancelled")
+
+                        chunk = blob_stream.read(DOWNLOAD_BLOCK_SIZE)
+                        if not chunk:
+                            break
+
+                        file.write(chunk)
+
+                        chunk_size = len(chunk)
+                        downloaded += chunk_size
+
+                        if progress_callback:
+                            progress_callback(chunk_size)
+            shutil.move(download_tmp_path, download_path)
             return str(download_path)
         except Exception as ex:
+            if download_path and download_path.exists():
+                download_path.unlink()
+            if download_tmp_path and download_tmp_path.exists():
+                download_tmp_path.unlink()
             raise ReadStorageItemsError(str(ex)) from ex
 
     def get_file_size(self, storage: str, key: str) -> int:
@@ -282,3 +349,53 @@ class AzureStorageProviderService(BaseStorageProviderService):
             return props.size
         except Exception as ex:
             raise ReadStorageItemsError(str(ex)) from ex
+
+    async def upload_multipart(
+        self,
+        client,
+        container: str,
+        source_path: Path,
+        blob_name: str,
+        block_size: int,
+        cancel_event=None,
+        progress_callback=None,
+    ):
+        max_workers = 8
+
+        blob_client = client.get_blob_client(container, blob_name)
+        semaphore = asyncio.Semaphore(max_workers)
+
+        async def upload_block(offset, data):
+            async with semaphore:
+                block_id = f"{offset:08d}"
+                encoded_block_id = base64.b64encode(block_id.encode()).decode()
+                blob_client.stage_block(block_id=encoded_block_id, data=data)
+                if progress_callback:
+                    progress_callback(len(data))
+                if cancel_event and cancel_event.is_set():
+                    raise Exception("Upload cancelled")
+                return BlobBlock(block_id=encoded_block_id)
+
+        async def read_and_upload():
+            tasks = []
+            with open(source_path, "rb") as f:
+                offset = 0
+                while chunk := f.read(block_size):
+                    tasks.append(upload_block(offset, chunk))
+                    offset += len(chunk)
+                    if cancel_event and cancel_event.is_set():
+                        raise Exception("Upload cancelled")
+            return await asyncio.gather(*tasks)
+
+        block_ids = await read_and_upload()
+        blob_client.commit_block_list(block_ids)
+
+
+# Todo: tmp solution, we need to move to async
+def run_async_sync_safe(coro):
+    def runner():
+        return asyncio.run(coro)
+
+    with ThreadPoolExecutor(1) as executor:
+        future = executor.submit(runner)
+        return future.result()

{data_sourcerer-0.7.0 → data_sourcerer-0.7.2}/sourcerer/infrastructure/storage_provider/services/gcp.py

@@ -4,7 +4,9 @@ Implementation of GCP storage provider services.
 This module provides concrete implementations of the BaseStorageProviderService
 interface for various cloud storage providers.
 """
-
+import shutil
+import tempfile
+import threading
 from collections.abc import Callable
 from pathlib import Path
 from typing import Any
@@ -31,7 +33,12 @@ from sourcerer.infrastructure.storage_provider.exceptions import (
 )
 from sourcerer.infrastructure.storage_provider.registry import storage_provider
 from sourcerer.infrastructure.utils import generate_uuid, is_text_file
-from sourcerer.settings import PAGE_SIZE, PATH_DELIMITER
+from sourcerer.settings import (
+    DOWNLOAD_BLOCK_SIZE,
+    MULTIPART_UPLOAD_BLOCK_SIZE,
+    PAGE_SIZE,
+    PATH_DELIMITER,
+)
 
 
 @storage_provider(StorageProvider.GoogleCloudStorage)
@@ -201,6 +208,8 @@ class GCPStorageProviderService(BaseStorageProviderService):
         storage_path: str,
         source_path: Path,
         dest_path: str | None = None,
+        cancel_event: threading.Event | None = None,
+        progress_callback: Callable | None = None,
     ) -> None:
         """
         Upload a file to the specified GCP bucket path.
@@ -210,6 +219,8 @@ class GCPStorageProviderService(BaseStorageProviderService):
             storage_path (str): The path within the bucket
             source_path (Path): Local file path to upload
             dest_path (str, optional): Destination path in GCP. Defaults to None.
+            cancel_event (threading.Event, optional): Event to signal upload cancellation. Defaults to None.
+            progress_callback (Callable, optional): Callback function for progress updates. Defaults to None.
 
         Raises:
             UploadStorageItemsError: If an error occurs while uploading the item
@@ -219,12 +230,22 @@ class GCPStorageProviderService(BaseStorageProviderService):
             storage_path = str(
                 Path(storage_path or "") / (dest_path or source_path.name)
             )
-            bucket.blob(storage_path).upload_from_filename(source_path)
+            blob = bucket.blob(storage_path)
+            if source_path.stat().st_size <= MULTIPART_UPLOAD_BLOCK_SIZE:
+                blob.upload_from_filename(source_path)
+            else:
+                self._upload_storage_item_multipart(
+                    blob, source_path, cancel_event, progress_callback=progress_callback
+                )
         except Exception as ex:
             raise UploadStorageItemsError(str(ex)) from ex
 
     def download_storage_item(
-        self, storage: str, key: str, progress_callback: Callable | None = None
+        self,
+        storage: str,
+        key: str,
+        progress_callback: Callable | None = None,
+        cancel_event: threading.Event | None = None,
     ) -> str:
         """
         Download a file from GCP to the local filesystem.
@@ -233,22 +254,53 @@ class GCPStorageProviderService(BaseStorageProviderService):
             storage (str): The bucket name
             key (str): The key/path of the item to download
             progress_callback (Callable, optional): Callback function for progress updates. Defaults to None.
-
+            cancel_event (threading.Event, optional): Event to signal download cancellation. Defaults to None.
         Returns:
             str: Path to the downloaded file
 
         Raises:
             ReadStorageItemsError: If an error occurs while downloading the item
         """
+        download_path = None
         try:
             bucket = self.client.bucket(storage)
             blob = bucket.get_blob(key)
             if not blob:
                 raise BlobNotFoundError(key)
+
             download_path = Path(user_downloads_dir()) / Path(key).name
-            blob.download_to_filename(str(download_path))
+
+            suffix = Path(key).suffix
+            download_tmp_path = (
+                Path(user_downloads_dir())
+                / f"{next(tempfile._get_candidate_names())}{suffix}"  # type: ignore
+            )
+
+            downloaded = 0
+
+            with open(download_tmp_path, "wb") as file:
+                reader = blob.open("rb")  # streaming mode
+                while True:
+                    if cancel_event and cancel_event.is_set():
+                        raise Exception("Download cancelled")
+
+                    chunk = reader.read(DOWNLOAD_BLOCK_SIZE)
+                    if not chunk:
+                        break
+
+                    file.write(chunk)
+                    chunk_size = len(chunk)
+                    downloaded += chunk_size
+
+                    if progress_callback:
+                        progress_callback(chunk_size)
+
+            shutil.move(download_tmp_path, download_path)
             return str(download_path)
+
         except Exception as ex:
+            if download_path and Path(download_path).exists():
+                Path(download_path).unlink()
             raise ReadStorageItemsError(str(ex)) from ex
 
     def get_file_size(self, storage: str, key: str) -> int:
@@ -273,3 +325,78 @@ class GCPStorageProviderService(BaseStorageProviderService):
             return blob.size
         except Exception as ex:
             raise ReadStorageItemsError(str(ex)) from ex
+
+    def _upload_storage_item_multipart(
+        self,
+        blob,
+        source_path,
+        cancel_event: threading.Event | None = None,
+        progress_callback: Callable | None = None,
+    ):
+        """
+        Upload a file to the specified GCP bucket path using multipart upload.
+
+        This method is not implemented in the current version.
+        """
+        blob.chunk_size = MULTIPART_UPLOAD_BLOCK_SIZE
+
+        with CancelableFileReader(
+            source_path,
+            cancel_event,
+            chunk_size=MULTIPART_UPLOAD_BLOCK_SIZE,
+            progress_callback=progress_callback,
+        ) as stream:
+            blob.upload_from_file(
+                stream,
+                rewind=True,  # allow re-seek to beginning if needed
+                content_type="application/octet-stream",
+            )
+
+
+class CancelableFileReader:
+    def __init__(
+        self,
+        file_path,
+        cancel_event: threading.Event | None,
+        chunk_size,
+        progress_callback: Callable | None = None,
+    ):
+        self.file_path = file_path
+        self.file = None
+        self.cancel_event = cancel_event
+        self.chunk_size = chunk_size
+        self.progress_callback = progress_callback
+
+    def read(self, size=None):
+        if self.cancel_event and self.cancel_event.is_set():
+            raise RuntimeError("Upload cancelled")
+
+        if self.file is None:
+            raise RuntimeError("File is not opened")
+        chunk_size = size or self.chunk_size
+        data = self.file.read(chunk_size)
+        if data and self.progress_callback:
+            self.progress_callback(len(data))
+        return data
+
+    def seek(self, offset, whence=0):
+        if self.file is None:
+            raise RuntimeError("File is not opened")
+        return self.file.seek(offset, whence)
+
+    def tell(self):
+        if self.file is None:
+            raise RuntimeError("File is not opened")
+        return self.file.tell()
+
+    def close(self):
+        if self.file is None:
+            return None
+        return self.file.close()
+
+    def __enter__(self):
+        self.file = open(self.file_path, "rb")
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()

{data_sourcerer-0.7.0 → data_sourcerer-0.7.2}/sourcerer/infrastructure/storage_provider/services/s3.py

@@ -4,7 +4,9 @@ Implementation of S3 compatible storage provider services.
 This module provides concrete implementations of the BaseStorageProviderService
 interface for various cloud storage providers.
 """
-
+import shutil
+import tempfile
+import threading
 from collections.abc import Callable
 from itertools import groupby
 from pathlib import Path
@@ -32,7 +34,7 @@ from sourcerer.infrastructure.storage_provider.exceptions import (
 )
 from sourcerer.infrastructure.storage_provider.registry import storage_provider
 from sourcerer.infrastructure.utils import generate_uuid, is_text_file
-from sourcerer.settings import PAGE_SIZE, PATH_DELIMITER
+from sourcerer.settings import MULTIPART_UPLOAD_BLOCK_SIZE, PAGE_SIZE, PATH_DELIMITER
 
 
 @storage_provider(StorageProvider.S3)
@@ -223,6 +225,8 @@ class S3ProviderService(BaseStorageProviderService):
         storage_path: str,
         source_path: Path,
         dest_path: str | None = None,
+        cancel_event: threading.Event | None = None,
+        progress_callback: Callable | None = None,
     ) -> None:
         """
         Upload a file to the specified S3 bucket path.
@@ -232,18 +236,34 @@ class S3ProviderService(BaseStorageProviderService):
             storage_path (str): The path within the bucket
             source_path (Path): Local file path to upload
             dest_path (str, optional): Destination path in S3. Defaults to None.
+            cancel_event (threading.Event, optional): Event to signal upload cancellation. Defaults to None.
+            progress_callback (Callable, optional): Callback function for upload progress. Defaults to None.
 
         Raises:
             UploadStorageItemsError: If an error occurs while uploading the item
         """
         try:
             dest_path = str(Path(storage_path or "") / (dest_path or source_path.name))
-            self.client.upload_file(source_path, storage, dest_path)
+            if source_path.stat().st_size <= MULTIPART_UPLOAD_BLOCK_SIZE:
+                self.client.upload_file(source_path, storage, dest_path)
+            else:
+                self._upload_storage_item_multipart(
+                    source_path,
+                    storage,
+                    dest_path,
+                    MULTIPART_UPLOAD_BLOCK_SIZE,
+                    cancel_event,
+                    progress_callback,
+                )
         except Exception as ex:
             raise UploadStorageItemsError(str(ex)) from ex
 
     def download_storage_item(
-        self, storage: str, key: str, progress_callback: Callable | None = None
+        self,
+        storage: str,
+        key: str,
+        progress_callback: Callable | None = None,
+        cancel_event: threading.Event | None = None,
     ) -> str:
         """
         Download a file from S3 to local filesystem.
@@ -259,11 +279,25 @@ class S3ProviderService(BaseStorageProviderService):
         Raises:
             ReadStorageItemsError: If an error occurs while downloading the item
         """
+
+        def callback(size):
+            if progress_callback:
+                progress_callback(size)
+            if cancel_event and cancel_event.is_set():
+                raise ReadStorageItemsError("Download cancelled")
+
         try:
             download_path = Path(user_downloads_dir()) / Path(key).name
+            suffix = Path(key).suffix
+            download_tmp_path = (
+                Path(user_downloads_dir())
+                / f"{next(tempfile._get_candidate_names())}{suffix}"  # type: ignore
+            )
+
             self.client.download_file(
-                storage, key, download_path, Callback=progress_callback
+                storage, key, download_tmp_path, Callback=callback
             )
+            shutil.move(download_tmp_path, download_path)
             return str(download_path)
         except Exception as ex:
             raise ReadStorageItemsError(str(ex)) from ex
@@ -287,3 +321,68 @@ class S3ProviderService(BaseStorageProviderService):
             return metadata.get("ContentLength")
         except Exception as ex:
             raise ReadStorageItemsError(str(ex)) from ex
+
+    def _upload_storage_item_multipart(
+        self,
+        source_path,
+        storage: str,
+        dest_path: str,
+        block_size: int,
+        cancel_event: threading.Event | None = None,
+        progress_callback: Callable | None = None,
+    ):
+        upload_id = None
+        try:
+            parts = []
+
+            with open(source_path, "rb") as file_handle:
+                # Initiate multipart upload
+                response = self.client.create_multipart_upload(
+                    Bucket=storage, Key=dest_path
+                )
+                upload_id = response["UploadId"]
+
+                part_number = 1
+
+                while chunk := file_handle.read(block_size):
+                    if cancel_event and cancel_event.is_set():
+                        raise UploadStorageItemsError("Upload cancelled")
+
+                    part_response = self.client.upload_part(
+                        Bucket=storage,
+                        Key=dest_path,
+                        PartNumber=part_number,
+                        UploadId=upload_id,
+                        Body=chunk,
+                    )
+
+                    parts.append(
+                        {
+                            "PartNumber": part_number,
+                            "ETag": part_response["ETag"],
+                        }
+                    )
+
+                    if progress_callback:
+                        progress_callback(len(chunk))
+
+                    part_number += 1
+
+                # Finalize upload
+                if cancel_event and cancel_event.is_set():
+                    raise Exception("Upload canceled before completion")
+
+                self.client.complete_multipart_upload(
+                    Bucket=storage,
+                    Key=dest_path,
+                    UploadId=upload_id,
+                    MultipartUpload={"Parts": parts},
+                )
+
+        except Exception:
+            # Abort multipart if error or cancel
+            if upload_id:
+                self.client.abort_multipart_upload(
+                    Bucket=storage, Key=dest_path, UploadId=upload_id
+                )
+            raise