data-sourcerer 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

sourcerer/infrastructure/storage_provider/services/s3.py

@@ -0,0 +1,290 @@
+"""
+Implementation of S3 compatible storage provider services.
+
+This module provides concrete implementations of the BaseStorageProviderService
+interface for various cloud storage providers.
+"""
+
+from collections.abc import Callable
+from itertools import groupby
+from pathlib import Path
+from typing import Any
+
+import humanize
+from platformdirs import user_downloads_dir
+
+from sourcerer.domain.shared.entities import StorageProvider
+from sourcerer.domain.storage_provider.entities import (
+    File,
+    Folder,
+    Storage,
+    StorageContent,
+    StoragePermissions,
+)
+from sourcerer.domain.storage_provider.services import BaseStorageProviderService
+from sourcerer.infrastructure.storage_provider.exceptions import (
+    CredentialsNotFoundError,
+    DeleteStorageItemsError,
+    ListStorageItemsError,
+    ListStoragesError,
+    ReadStorageItemsError,
+    StoragePermissionError,
+    UploadStorageItemsError,
+)
+from sourcerer.infrastructure.storage_provider.registry import storage_provider
+from sourcerer.infrastructure.utils import generate_uuid, is_text_file
+from sourcerer.settings import PAGE_SIZE, PATH_DELIMITER
+
+
+@storage_provider(StorageProvider.S3)
+class S3ProviderService(BaseStorageProviderService):
+    """
+    AWS S3 storage provider service implementation.
+
+    This class provides methods for interacting with AWS S3 storage,
+    implementing the BaseStorageProviderService interface.
+    """
+
+    def __init__(self, credentials: Any):
+        """
+        Initialize the service with AWS credentials.
+
+        Args:
+            credentials (Any): AWS session or credentials object
+        """
+        self.credentials = credentials
+
+    @property
+    def client(self):
+        """
+        Get the S3 client.
+
+        Returns:
+            boto3.client: S3 client object
+        """
+        if not self.credentials:
+            raise CredentialsNotFoundError()
+
+        session = self.credentials.session
+
+        client_args = {}
+        if self.credentials.endpoint_url:
+            client_args["endpoint_url"] = self.credentials.endpoint_url
+
+        return session.client("s3", **client_args)
+
+    @property
+    def resource(self):
+        """
+        Get the S3 resource.
+
+        Returns:
+            boto3.resource: S3 resource object
+        """
+        if not self.credentials:
+            raise CredentialsNotFoundError()
+
+        session = self.credentials.session
+
+        client_args = {}
+        if self.credentials.endpoint_url:
+            client_args["endpoint_url"] = self.credentials.endpoint_url
+        return session.resource("s3", **client_args)
+
+    def list_storages(self) -> list[Storage]:
+        """
+        Return a list of available S3 buckets.
+
+        Returns:
+            List[Storage]: List of storage objects representing S3 buckets
+
+        Raises:
+            ListStoragesError: If an error occurs while listing buckets
+        """
+        try:
+            response = self.client.list_buckets()
+        except Exception as ex:
+            raise ListStoragesError(str(ex)) from ex
+        return [
+            Storage(StorageProvider.S3, i.get("Name"), i.get("CreationDate"))
+            for i in response.get("Buckets")
+        ]
+
+    def get_storage_permissions(self, storage: str) -> list[StoragePermissions]:
+        """
+        Return the permissions for the specified S3 bucket.
+
+        Args:
+            storage (str): The bucket name
+
+        Returns:
+            List[StoragePermissions]: List of permission objects for the bucket
+
+        Raises:
+            StoragePermissionError: If an error occurs while getting permissions
+        """
+        try:
+            permissions = self.client.get_bucket_acl(Bucket=storage)
+        except Exception as ex:
+            raise StoragePermissionError(str(ex)) from ex
+        return [
+            StoragePermissions(name, [i["Permission"] for i in items])
+            for name, items in groupby(
+                permissions["Grants"],
+                key=lambda x: x["Grantee"]["DisplayName"] or x["Grantee"]["ID"],
+            )
+        ]
+
+    def list_storage_items(
+        self, storage: str, path: str = "", prefix: str = ""
+    ) -> StorageContent:
+        """
+        List items in the specified S3 bucket path with the given prefix.
+
+        Args:
+            storage (str): The bucket name
+            path (str, optional): The path within the bucket. Defaults to ''.
+            prefix (str, optional): Filter items by this prefix. Defaults to ''.
+
+        Returns:
+            StorageContent: Object containing files and folders at the specified location
+
+        Raises:
+            ListStorageItemsError: If an error occurs while listing items
+        """
+        if path and not path.endswith("/"):
+            path += "/"
+        try:
+            result = self.client.list_objects_v2(
+                Bucket=storage,
+                Prefix=path + prefix,
+                Delimiter=PATH_DELIMITER,
+                MaxKeys=PAGE_SIZE,
+            )
+        except Exception as ex:
+            raise ListStorageItemsError(str(ex)) from ex
+
+        folders = [
+            Folder(i.get("Prefix").replace(path, ""))
+            for i in result.get("CommonPrefixes", [])
+            if i.get("Prefix")
+        ]
+        files = [
+            File(
+                generate_uuid(),
+                i.get("Key").replace(path, ""),
+                humanize.naturalsize(i.get("Size")),
+                is_text_file(i.get("Key")),
+                i.get("LastModified"),
+            )
+            for i in result.get("Contents", [])
+            if i.get("Key").replace(path, "")
+        ]
+        return StorageContent(files=files, folders=folders)
+
+    def read_storage_item(self, storage: str, key: str) -> str:
+        """
+        Read and return the content of the specified S3 object.
+
+        Args:
+            storage (str): The bucket name
+            key (str): The key/path of the item to read
+
+        Returns:
+            bytes: The content of the S3 object
+
+        Raises:
+            ReadStorageItemsError: If an error occurs while reading the item
+        """
+        try:
+            content_object = self.resource.Object(storage, key)
+            return content_object.get()["Body"].read().decode("utf-8")
+        except Exception as ex:
+            raise ReadStorageItemsError(str(ex)) from ex
+
+    def delete_storage_item(self, storage: str, key: str) -> None:
+        """
+        Delete the specified S3 object.
+
+        Args:
+            storage (str): The bucket name
+            key (str): The key/path of the item to delete
+
+        Raises:
+            DeleteStorageItemsError: If an error occurs while deleting the item
+        """
+        try:
+            return self.resource.Object(storage, key).delete()
+        except Exception as ex:
+            raise DeleteStorageItemsError(str(ex)) from ex
+
+    def upload_storage_item(
+        self,
+        storage: str,
+        storage_path: str,
+        source_path: Path,
+        dest_path: str | None = None,
+    ) -> None:
+        """
+        Upload a file to the specified S3 bucket path.
+
+        Args:
+            storage (str): The bucket name
+            storage_path (str): The path within the bucket
+            source_path (Path): Local file path to upload
+            dest_path (str, optional): Destination path in S3. Defaults to None.
+
+        Raises:
+            UploadStorageItemsError: If an error occurs while uploading the item
+        """
+        try:
+            dest_path = str(Path(storage_path or "") / (dest_path or source_path.name))
+            self.client.upload_file(source_path, storage, dest_path)
+        except Exception as ex:
+            raise UploadStorageItemsError(str(ex)) from ex
+
+    def download_storage_item(
+        self, storage: str, key: str, progress_callback: Callable | None = None
+    ) -> str:
+        """
+        Download a file from S3 to local filesystem.
+
+        Args:
+            storage (str): The bucket name
+            key (str): The key/path of the item to download
+            progress_callback (Callable, optional): Callback function for progress updates. Defaults to None.
+
+        Returns:
+            str: Path to the downloaded file
+
+        Raises:
+            ReadStorageItemsError: If an error occurs while downloading the item
+        """
+        try:
+            download_path = Path(user_downloads_dir()) / Path(key).name
+            self.client.download_file(
+                storage, key, download_path, Callback=progress_callback
+            )
+            return str(download_path)
+        except Exception as ex:
+            raise ReadStorageItemsError(str(ex)) from ex
+
+    def get_file_size(self, storage: str, key: str) -> int:
+        """
+        Get metadata for an S3 object without downloading content.
+
+        Args:
+            storage (str): The bucket name
+            key (str): The key/path of the item
+
+        Returns:
+            int: Size of the storage item in bytes
+
+        Raises:
+            ReadStorageItemsError: If an error occurs while getting metadata
+        """
+        try:
+            metadata = self.client.head_object(Bucket=storage, Key=key)
+            return metadata.get("ContentLength")
+        except Exception as ex:
+            raise ReadStorageItemsError(str(ex)) from ex

sourcerer/infrastructure/utils.py

@@ -60,9 +60,7 @@ def is_text_file(file_name):
     ext = Path(file_name).suffix.lower()
     if ext in TEXT_EXTENSIONS:
         return True
-    if is_text_mime(file_name):
-        return True
-    return False
+    return bool(is_text_mime(file_name))
 
 
 def custom_sort_key(s: str | Path):
@@ -102,5 +100,5 @@ class Singleton(type):
             object: The singleton instance of the class.
         """
         if cls not in cls._instances:
-            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
+            cls._instances[cls] = super().__call__(*args, **kwargs)
         return cls._instances[cls]

sourcerer/presentation/screens/critical_error/main.py

@@ -4,10 +4,10 @@ import webbrowser
 
 from textual import on
 from textual.app import ComposeResult
-from textual.containers import Horizontal, Container
+from textual.containers import Container, Horizontal
+from textual.css.query import NoMatches
 from textual.screen import ModalScreen
 from textual.widgets import Label, RichLog
-from textual.css.query import NoMatches
 
 from sourcerer import __version__
 from sourcerer.presentation.screens.shared.widgets.button import Button
@@ -30,7 +30,6 @@ class CriticalErrorScreen(ModalScreen[bool]):
             with Horizontal():
                 yield Button("Report", name="report")
                 yield Button("Dismiss", name="dismiss")
-                
 
     def on_mount(self) -> None:
         """
@@ -40,7 +39,7 @@ class CriticalErrorScreen(ModalScreen[bool]):
         try:
             text_log = self.query_one(RichLog)
         except NoMatches:
-            return 
+            return
 
         text_log.write(self.traceback)
 

sourcerer/presentation/screens/file_system_finder/main.py

@@ -1,12 +1,12 @@
+from collections.abc import Callable
 from dataclasses import dataclass
 from pathlib import Path
-from typing import List, Callable
 
 from dependency_injector.wiring import Provide
 from textual import on
 from textual.app import ComposeResult
 from textual.binding import Binding
-from textual.containers import Horizontal, Container
+from textual.containers import Container, Horizontal
 from textual.css.query import NoMatches
 from textual.reactive import reactive
 from textual.screen import ModalScreen
@@ -38,11 +38,11 @@ class FileSystemNavigationModal(ModalScreen):
 
     def __init__(
         self,
+        *args,
         file_system_service: FileSystemService = Provide[
             DiContainer.file_system_service
         ],
-        validation_rules: List[FileSystemSelectionValidationRule] | None = None,
-        *args,
+        validation_rules: list[FileSystemSelectionValidationRule] | None = None,
         **kwargs,
     ):
         """

sourcerer/presentation/screens/file_system_finder/widgets/file_system_navigator.py

@@ -1,8 +1,9 @@
+from collections.abc import Callable, Sequence
 from dataclasses import dataclass
 from enum import Enum
 from pathlib import Path
 from time import time
-from typing import ClassVar, Sequence, Literal, List, Callable, Tuple, Type
+from typing import ClassVar, Literal
 
 from textual import events, on
 from textual.app import ComposeResult
@@ -11,17 +12,16 @@ from textual.css.query import NoMatches
 from textual.message import Message
 from textual.reactive import reactive
 from textual.widget import Widget
-from textual.widgets import Rule, Label
+from textual.widgets import Label, Rule
 
-from sourcerer.infrastructure.file_system.exceptions import ListDirException
+from sourcerer.infrastructure.file_system.exceptions import ListDirError
 from sourcerer.infrastructure.file_system.services import FileSystemService
+from sourcerer.infrastructure.utils import generate_uuid
 from sourcerer.presentation.screens.shared.containers import (
     ScrollHorizontalContainerWithNoBindings,
     ScrollVerticalContainerWithNoBindings,
 )
-
-from sourcerer.infrastructure.utils import generate_uuid
-from sourcerer.settings import DOUBLE_CLICK_THRESHOLD, FILE_ICON, DIRECTORY_ICON
+from sourcerer.settings import DIRECTORY_ICON, DOUBLE_CLICK_THRESHOLD, FILE_ICON
 
 
 class PathListingContainer(ScrollVerticalContainerWithNoBindings):
@@ -95,7 +95,7 @@ class FileSystemWidget(Widget):
         """
         self.entity_name = entity_name
         self.icon = icon
-        self.last_file_click: Tuple[float, Path | None] = (
+        self.last_file_click: tuple[float, Path | None] = (
             time() - 2,
             None,
         )
@@ -231,7 +231,7 @@ class FileSystemNavigator(ScrollHorizontalContainerWithNoBindings):
     """
 
     # Consolidate key binding data
-    BINDINGS: ClassVar[List[BindingType]] = [
+    BINDINGS: ClassVar[list[BindingType]] = [
         Binding("enter", "select_cursor", "Select", show=False),
         Binding("up", "cursor_up", "Cursor up", show=False),
         Binding("down", "cursor_down", "Cursor down", show=False),
@@ -489,15 +489,15 @@ class FileSystemNavigator(ScrollHorizontalContainerWithNoBindings):
             path (Path): The directory path to list contents for
 
         Returns:
-            PathListingContainer | None: A container with folder and file widgets, or None if directory is empty or
+            - PathListingContainer | None: A container with folder and file widgets, or None if directory is empty or
             listing fails
 
         Raises:
-            Notifies user via self.notify() if directory listing encounters an error
+            - Notifies user via self.notify() if directory listing encounters an error
         """
         try:
             dir_list = self.file_system_service.list_dir(path, relative_paths=False)
-        except ListDirException as e:
+        except ListDirError as e:
             self.notify(str(e), markup=False)
             return
 
@@ -686,7 +686,7 @@ class FileSystemNavigator(ScrollHorizontalContainerWithNoBindings):
         elements: Sequence[Widget],
         direction: Literal["up", "down"],
         selector: Callable,
-        of_type: Type | None = None,
+        of_type: type | None = None,
     ) -> Widget | None:
         """
         Determine the next element in a sequence based on navigation direction and selection criteria.

sourcerer/presentation/screens/main/main.py

@@ -1,7 +1,9 @@
+import time
 import traceback
+from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 
-from textual import work, on
+from textual import on, work
 from textual.app import App, ComposeResult
 from textual.binding import Binding
 from textual.containers import Horizontal
@@ -10,7 +12,7 @@ from textual.widgets import Footer
 
 from sourcerer.infrastructure.access_credentials.services import CredentialsService
 from sourcerer.infrastructure.storage_provider.exceptions import (
-    ListStorageItemsException,
+    ListStorageItemsError,
 )
 from sourcerer.infrastructure.utils import generate_uuid
 from sourcerer.presentation.screens.critical_error.main import CriticalErrorScreen
@@ -44,16 +46,17 @@ from sourcerer.presentation.screens.provider_creds_list.main import (
     ProviderCredsListScreen,
 )
 from sourcerer.presentation.screens.storage_action_progress.main import (
+    DeleteKey,
+    DownloadKey,
     StorageActionProgressScreen,
     UploadKey,
-    DownloadKey,
-    DeleteKey,
 )
 from sourcerer.presentation.themes.github_dark import github_dark_theme
 from sourcerer.presentation.utils import (
-    get_provider_service_by_access_uuid,
     get_provider_service_by_access_credentials,
+    get_provider_service_by_access_uuid,
 )
+from sourcerer.settings import MAX_PARALLEL_STORAGE_LIST_OPERATIONS
 
 
 class Sourcerer(App, ResizeContainersWatcherMixin):
@@ -168,25 +171,11 @@ class Sourcerer(App, ResizeContainersWatcherMixin):
         if not access_credentials:
             return
 
-        for credentials in access_credentials:
-            provider_service = get_provider_service_by_access_credentials(credentials)
-            if not provider_service:
-                self.notify(
-                    f"Could not get storages list for {credentials.name}!",
-                    severity="error",
-                )
-                continue
-            try:
-                storages = provider_service.list_storages()
-                self.storage_list_sidebar.storages = {
-                    credentials.uuid: storages,
-                    **self.storage_list_sidebar.storages,
-                }
-            except Exception:
-                self.notify(
-                    f"Could not get storages list for {credentials.name}!",
-                    severity="error",
-                )
+        with ThreadPoolExecutor(
+            max_workers=MAX_PARALLEL_STORAGE_LIST_OPERATIONS
+        ) as executor:
+            for credentials in access_credentials:
+                executor.submit(self._load_storages, credentials)
 
     @on(SelectStorageItem)
     def on_select_storage_item(self, event: SelectStorageItem):
@@ -386,23 +375,22 @@ class Sourcerer(App, ResizeContainersWatcherMixin):
         self.storage_content.storage = storage_name
         self.storage_content.access_credentials_uuid = access_credentials_uuid
         self.storage_content.search_prefix = prefix or ""
+        self.storage_content.storage_content = None
+        self.storage_content.selected_files = set()
+        self.storage_content.selected_files_n = 0
 
         provider_service = get_provider_service_by_access_uuid(
             access_credentials_uuid, self.credentials_service
         )
 
         if not provider_service:
-            self.notify("Could not extract storage content", severity="error")
+            self.notify_error("Could not extract storage content")
             return
         params = {"storage": storage_name, "path": path or "", "prefix": prefix or ""}
         try:
             self.storage_content.storage_content = provider_service.list_storage_items(**params)  # type: ignore
-        except ListStorageItemsException as e:
-            self.notify(
-                f"""Could not extract storage content 
-{str(e)}""",
-                severity="error",
-            )
+        except ListStorageItemsError as e:
+            self.notify_error(f"""Could not extract storage content \n{str(e)}""")
 
     def _upload_file(
         self,
@@ -435,7 +423,7 @@ class Sourcerer(App, ResizeContainersWatcherMixin):
         """
         # Validate input parameters
         if not source_path:
-            self.notify("No file selected for upload", severity="error")
+            self.notify_error("No file selected for upload")
             return
 
         # Get the provider service
@@ -443,7 +431,7 @@ class Sourcerer(App, ResizeContainersWatcherMixin):
             access_credentials_uuid, self.credentials_service
         )
         if not provider_service:
-            self.notify("Could not get provider service for upload", severity="error")
+            self.notify_error("Could not get provider service for upload")
             return
 
         # Create upload key
@@ -467,3 +455,39 @@ class Sourcerer(App, ResizeContainersWatcherMixin):
                 access_credentials_uuid, storage_name, path
             ),
         )
+
+    def _load_storages(self, credentials):
+        """
+        Loads the list of storages for the given access credentials.
+
+        This method retrieves the list of storages from the provider service
+        associated with the provided access credentials and updates the
+        storage list sidebar with the retrieved storages.
+
+        Args:
+            credentials (Credentials): The access credentials for which to load storages.
+
+        Note:
+            If an error occurs while retrieving the storages, a notification is shown
+            to the user.
+        """
+        provider_service = get_provider_service_by_access_credentials(credentials)
+        if not provider_service:
+            self.notify_error(f"Could not get storages list for {credentials.name}!")
+            return
+
+        try:
+            storages = provider_service.list_storages()
+            self.storage_list_sidebar.storages[credentials.uuid] = storages
+            self.storage_list_sidebar.last_update_timestamp = time.time()
+        except Exception:
+            self.notify_error(f"Could not get storages list for {credentials.name}!")
+
+    def notify_error(self, message):
+        """
+        Displays an error notification to the user.
+
+        Args:
+            message (str): The error message to display.
+        """
+        self.notify(message, severity="error")

sourcerer/presentation/screens/main/messages/delete_request.py

@@ -1,5 +1,4 @@
 from dataclasses import dataclass
-from typing import List
 
 from textual.message import Message
 
@@ -9,4 +8,4 @@ class DeleteRequest(Message):
     storage_name: str
     access_credentials_uuid: str
     path: str
-    keys: List[str]
+    keys: list[str]

sourcerer/presentation/screens/main/messages/download_request.py

@@ -1,5 +1,4 @@
 from dataclasses import dataclass
-from typing import List
 
 from textual.message import Message
 
@@ -9,4 +8,4 @@ class DownloadRequest(Message):
     storage_name: str
     access_credentials_uuid: str
     path: str
-    keys: List[str]
+    keys: list[str]

sourcerer/presentation/screens/main/mixins/resize_containers_watcher_mixin.py

@@ -3,13 +3,13 @@ import time
 from textual.events import MouseMove, MouseUp
 
 from sourcerer.presentation.screens.main.messages.resizing_rule import (
-    ResizingRuleSelect,
-    ResizingRuleRelease,
     ResizingRuleMove,
+    ResizingRuleRelease,
+    ResizingRuleSelect,
 )
 from sourcerer.presentation.screens.main.widgets.resizing_rule import (
-    ResizingRule,
     MoveEvent,
+    ResizingRule,
 )
 from sourcerer.presentation.settings import MIN_SECTION_DIMENSION
 

sourcerer/presentation/screens/main/widgets/gradient.py

@@ -1,5 +1,5 @@
-from textual.widgets import Static
 from rich.text import Text
+from textual.widgets import Static
 
 
 def hex_to_rgb(hex_color: str):
@@ -20,10 +20,7 @@ def generate_gradient_text(text: str, start_color: str, end_color: str) -> Text:
     gradient_text = Text()
 
     for i, char in enumerate(text):
-        if len(text) > 1:
-            blend = i / (len(text) - 1)
-        else:
-            blend = 0
+        blend = i / (len(text) - 1) if len(text) > 1 else 0
 
         r = int(start_rgb[0] + (end_rgb[0] - start_rgb[0]) * blend)
         g = int(start_rgb[1] + (end_rgb[1] - start_rgb[1]) * blend)

sourcerer/presentation/screens/main/widgets/resizing_rule.py

@@ -6,8 +6,8 @@ from textual.widgets import Rule
 
 from sourcerer.presentation.screens.main.messages.resizing_rule import (
     ResizingRuleMove,
-    ResizingRuleSelect,
     ResizingRuleRelease,
+    ResizingRuleSelect,
 )