digitalkin 0.2.13__py3-none-any.whl → 0.2.15__py3-none-any.whl

digitalkin/services/filesystem/filesystem_strategy.py

@@ -1,7 +1,8 @@
 """This module contains the abstract base class for filesystem strategies."""
 
 from abc import ABC, abstractmethod
-from enum import Enum, auto
+from datetime import datetime
+from typing import Any
 
 from pydantic import BaseModel, Field
 
@@ -9,63 +10,197 @@ from digitalkin.services.base_strategy import BaseStrategy
 
 
 class FilesystemServiceError(Exception):
-    """Base exception for Setup service errors."""
+    """Base exception for Filesystem service errors."""
 
 
-class FileType(Enum):
-    """Enum defining the types of data that can be stored."""
-
-    DOCUMENT = auto()
-    IMAGE = auto()
-    VIDEO = auto()
-    AUDIO = auto()
-    ARCHIVE = auto()
-    OTHER = auto()
-
-
-class FilesystemData(BaseModel):
+class FilesystemRecord(BaseModel):
     """Data model for filesystem operations."""
 
-    kin_context: str = Field(description="The context of the file in the filesystem")
+    id: str = Field(description="Unique identifier for the file (UUID)")
+    context: str = Field(description="The context of the file in the filesystem")
     name: str = Field(description="The name of the file")
-    file_type: FileType = Field(default=FileType.DOCUMENT, description="The type of data stored")
-    url: str = Field(description="The URL of the file in the filesystem")
+    file_type: str = Field(default="UNSPECIFIED", description="The type of data stored")
+    content_type: str = Field(default="application/octet-stream", description="The MIME type of the file")
+    size_bytes: int = Field(default=0, description="Size of the file in bytes")
+    checksum: str = Field(default="", description="SHA-256 checksum of the file content")
+    metadata: dict[str, Any] | None = Field(default=None, description="Additional metadata for the file")
+    storage_url: str = Field(description="Internal URL for accessing the file content")
+    status: str = Field(default="UNSPECIFIED", description="Current status of the file")
+    content: bytes | None = Field(default=None, description="The content of the file")
+
+
+class FileFilter(BaseModel):
+    """Filter criteria for querying files."""
+
+    names: list[str] | None = Field(default=None, description="Filter by file names (exact matches)")
+    file_ids: list[str] | None = Field(default=None, description="Filter by file IDs")
+    file_types: list[str] | None = Field(default=None, description="Filter by file types")
+    created_after: datetime | None = Field(default=None, description="Filter files created after this timestamp")
+    created_before: datetime | None = Field(default=None, description="Filter files created before this timestamp")
+    updated_after: datetime | None = Field(default=None, description="Filter files updated after this timestamp")
+    updated_before: datetime | None = Field(default=None, description="Filter files updated before this timestamp")
+    status: str | None = Field(default=None, description="Filter by file status")
+    content_type_prefix: str | None = Field(default=None, description="Filter by content type prefix (e.g., 'image/')")
+    min_size_bytes: int | None = Field(default=None, description="Filter files with minimum size")
+    max_size_bytes: int | None = Field(default=None, description="Filter files with maximum size")
+    prefix: str | None = Field(default=None, description="Filter by path prefix (e.g., 'folder1/')")
+    content_type: str | None = Field(default=None, description="Filter by content type")
+
+
+class UploadFileData(BaseModel):
+    """Data model for uploading a file."""
+
+    content: bytes = Field(description="The content of the file")
+    name: str = Field(description="The name of the file")
+    file_type: str = Field(description="The type of the file")
+    content_type: str | None = Field(default=None, description="The content type of the file")
+    metadata: dict[str, Any] | None = Field(default=None, description="The metadata of the file")
+    replace_if_exists: bool = Field(default=False, description="Whether to replace the file if it already exists")
 
 
 class FilesystemStrategy(BaseStrategy, ABC):
-    """Abstract base class for filesystem strategies."""
+    """Abstract base class for filesystem strategies.
+
+    This strategy provides comprehensive file management capabilities including
+    upload, retrieval, update, and deletion operations with rich metadata support,
+    filtering, and pagination.
+    """
 
-    def __init__(self, mission_id: str, setup_version_id: str, config: dict[str, str]) -> None:
+    def __init__(self, mission_id: str, setup_version_id: str, config: dict[str, Any] | None = None) -> None:
         """Initialize the strategy.
 
         Args:
             mission_id: The ID of the mission this strategy is associated with
             setup_version_id: The ID of the setup version this strategy is associated with
-            config: configuration dictionary for the filesystem strategy
+            config: Configuration for the filesystem strategy
         """
         super().__init__(mission_id, setup_version_id)
-        self.config: dict[str, str] = config
+        self.config = config
 
     @abstractmethod
-    def upload(self, content: bytes, name: str, file_type: FileType) -> FilesystemData:
-        """Create a new file in the filesystem."""
+    def upload_files(
+        self,
+        files: list[UploadFileData],
+    ) -> tuple[list[FilesystemRecord], int, int]:
+        """Upload multiple files to the system.
 
-    @abstractmethod
-    def get(self, name: str) -> FilesystemData:
-        """Get file from the filesystem."""
+        This method allows batch uploading of files with validation and
+        error handling for each individual file. Files are processed
+        atomically - if one fails, others may still succeed.
+
+        Args:
+            files: List of tuples containing (content, name, file_type, content_type, metadata, replace_if_exists)
+
+        Returns:
+            tuple[list[FilesystemRecord], int, int]: List of uploaded files, total uploaded count, total failed count
+        """
 
     @abstractmethod
-    def get_batch(self, names: list[str]) -> dict[str, FilesystemData | None]:
-        """Get files from the filesystem."""
+    def get_file(
+        self,
+        file_id: str,
+        *,
+        include_content: bool = False,
+    ) -> tuple[FilesystemRecord, bytes | None]:
+        """Get a specific file by ID or name.
+
+        This method fetches detailed information about a single file,
+        with optional content inclusion. Supports lookup by either
+        unique ID or name within a context.
+
+        Args:
+            file_id: The ID of the file to be retrieved
+            include_content: Whether to include file content in response
+
+        Returns:
+            tuple[FilesystemRecord, bytes | None]: Metadata about the retrieved file and optional content
+        """
 
     @abstractmethod
-    def get_all(self) -> list[FilesystemData]:
-        """Get all files from the filesystem."""
+    def get_files(
+        self,
+        filters: FileFilter,
+        *,
+        list_size: int = 100,
+        offset: int = 0,
+        order: str | None = None,
+        include_content: bool = False,
+    ) -> tuple[list[FilesystemRecord], int]:
+        """Get multiple files by various criteria.
+
+        This method provides efficient retrieval of multiple files using:
+        - File IDs
+        - File names
+        - Path prefix
+        With support for:
+        - Pagination for large result sets
+        - Optional content inclusion
+        - Total count of matching files
+
+        Args:
+            filters: Filter criteria for the files
+            list_size: Number of files to return per page
+            offset: Offset to start listing files from
+            order: Field to order results by
+            include_content: Whether to include file content in response
+
+        Returns:
+            tuple[list[FilesystemRecord], int]: List of files and total count
+        """
 
     @abstractmethod
-    def update(self, name: str, content: bytes, file_type: FileType) -> FilesystemData:
-        """Update files in the filesystem."""
+    def update_file(
+        self,
+        file_id: str,
+        content: bytes | None = None,
+        file_type: str | None = None,
+        content_type: str | None = None,
+        metadata: dict[str, Any] | None = None,
+        new_name: str | None = None,
+        status: str | None = None,
+    ) -> FilesystemRecord:
+        """Update file metadata, content, or both.
+
+        This method allows updating various aspects of a file:
+        - Rename files
+        - Update content and content type
+        - Modify metadata
+        - Create new versions
+
+        Args:
+            file_id: The ID of the file to be updated
+            content: Optional new content of the file
+            file_type: Optional new type of data
+            content_type: Optional new MIME type
+            metadata: Optional new metadata (will merge with existing)
+            new_name: Optional new name for the file
+            status: Optional new status for the file
+
+        Returns:
+            FilesystemRecord: Metadata about the updated file
+        """
 
     @abstractmethod
-    def delete(self, name: str) -> bool:
-        """Delete file from the filesystem."""
+    def delete_files(
+        self,
+        filters: FileFilter,
+        *,
+        permanent: bool = False,
+        force: bool = False,
+    ) -> tuple[dict[str, bool], int, int]:
+        """Delete multiple files.
+
+        This method supports batch deletion of files with options for:
+        - Soft deletion (marking as deleted)
+        - Permanent deletion
+        - Force deletion of files in use
+        - Individual error reporting per file
+
+        Args:
+            filters: Filter criteria for the files
+            permanent: Whether to permanently delete the files
+            force: Whether to force delete even if files are in use
+
+        Returns:
+            tuple[dict[str, bool], int, int]: Results per file, total deleted count, total failed count
+        """

digitalkin/services/filesystem/grpc_filesystem.py

@@ -1,26 +1,23 @@
-"""Grpc filesystem."""
+"""gRPC filesystem implementation."""
 
 from collections.abc import Generator
 from contextlib import contextmanager
 from typing import Any
 
-from digitalkin_proto.digitalkin.filesystem.v2 import (
-    filesystem_pb2,
-    filesystem_service_pb2_grpc,
-)
-from digitalkin_proto.digitalkin.filesystem.v2.filesystem_pb2 import (
-    FileType as FileTypeProto,
-)
+from digitalkin_proto.digitalkin.filesystem.v1 import filesystem_pb2, filesystem_service_pb2_grpc
+from google.protobuf import struct_pb2
+from google.protobuf.json_format import MessageToDict
 
 from digitalkin.grpc_servers.utils.exceptions import ServerError
 from digitalkin.grpc_servers.utils.grpc_client_wrapper import GrpcClientWrapper
 from digitalkin.grpc_servers.utils.models import ClientConfig
 from digitalkin.logger import logger
 from digitalkin.services.filesystem.filesystem_strategy import (
-    FilesystemData,
+    FileFilter,
+    FilesystemRecord,
     FilesystemServiceError,
     FilesystemStrategy,
-    FileType,
+    UploadFileData,
 )
 
 
@@ -54,161 +51,272 @@ class GrpcFilesystem(FilesystemStrategy, GrpcClientWrapper):
             logger.exception(msg)
             raise FilesystemServiceError(msg) from e
 
+    @staticmethod
+    def _file_type_to_enum(file_type: str) -> filesystem_pb2.FileType:
+        """Convert a file type string to a FileType enum.
+
+        Args:
+            file_type: The file type string to convert
+
+        Returns:
+            filesystem_pb2.FileType: The converted file type enum
+        """
+        if not file_type.upper().startswith("FILE_TYPE_"):
+            file_type = f"FILE_TYPE_{file_type.upper()}"
+        try:
+            return getattr(filesystem_pb2.FileType, file_type.upper())
+        except AttributeError:
+            return filesystem_pb2.FileType.FILE_TYPE_UNSPECIFIED
+
+    @staticmethod
+    def _file_status_to_enum(file_status: str) -> filesystem_pb2.FileStatus:
+        """Convert a file status string to a FileStatus enum.
+
+        Args:
+            file_status: The file status string to convert
+
+        Returns:
+            filesystem_pb2.FileStatus: The converted file status enum
+        """
+        if not file_status.upper().startswith("FILE_STATUS_"):
+            file_status = f"FILE_STATUS_{file_status.upper()}"
+        try:
+            return getattr(filesystem_pb2.FileStatus, file_status.upper())
+        except AttributeError:
+            return filesystem_pb2.FileStatus.FILE_STATUS_UNSPECIFIED
+
+    def _filter_to_proto(self, filters: FileFilter) -> filesystem_pb2.FileFilter:
+        """Convert a FileFilter to a FileFilter proto message.
+
+        Args:
+            filters: The FileFilter to convert
+
+        Returns:
+            filesystem_pb2.FileFilter: The converted FileFilter proto message
+        """
+        return filesystem_pb2.FileFilter(
+            context=self.mission_id,
+            **filters.model_dump(exclude={"file_types", "status"}),
+            file_types=[self._file_type_to_enum(file_type) for file_type in filters.file_types]
+            if filters.file_types
+            else None,
+            status=self._file_status_to_enum(filters.status) if filters.status else None,
+        )
+
+    def _file_proto_to_data(self, file: filesystem_pb2.File, content: bytes | None = None) -> FilesystemRecord:
+        """Convert a File proto message to FilesystemRecord.
+
+        Args:
+            file: The File proto message to convert
+            content: The content of the file
+
+        Returns:
+            FilesystemRecord: The converted data
+        """
+        return FilesystemRecord(
+            id=file.file_id,
+            context=self.mission_id,
+            name=file.name,
+            file_type=filesystem_pb2.FileType.Name(file.file_type),
+            content_type=file.content_type,
+            size_bytes=file.size_bytes,
+            checksum=file.checksum,
+            metadata=MessageToDict(file.metadata),
+            storage_url=file.storage_url,
+            status=filesystem_pb2.FileStatus.Name(file.status),
+            content=content,
+        )
+
     def __init__(
         self,
         mission_id: str,
         setup_version_id: str,
-        config: dict[str, str],
         client_config: ClientConfig,
-        **kwargs,  # noqa: ANN003, ARG002
+        config: dict[str, Any] | None = None,
     ) -> None:
-        """Initialize the default filesystem strategy.
+        """Initialize the gRPC filesystem strategy.
 
         Args:
             mission_id: The ID of the mission this strategy is associated with
             setup_version_id: The ID of the setup version this strategy is associated with
-            config: A dictionary mapping names to Pydantic model classes
-            client_config: The server configuration object
-            kwargs: other optional arguments to pass to the parent class constructor
+            client_config: Configuration for the gRPC client connection
+            config: Configuration for the filesystem strategy
         """
         super().__init__(mission_id, setup_version_id, config)
+        self.service_name = "FilesystemService"
         channel = self._init_channel(client_config)
         self.stub = filesystem_service_pb2_grpc.FilesystemServiceStub(channel)
-        logger.info("Channel client 'Filesystem' initialized succesfully")
+        logger.debug("Channel client 'Filesystem' initialized succesfully")
 
-    def upload(self, content: bytes, name: str, file_type: FileType) -> FilesystemData:
-        """Create a new file in the file system.
+    def upload_files(
+        self,
+        files: list[UploadFileData],
+    ) -> tuple[list[FilesystemRecord], int, int]:
+        """Upload multiple files to the filesystem.
 
         Args:
-            content: The content of the file to be uploaded
-            name: The name of the file to be created
-            file_type: The type of data being uploaded
+            files: List of tuples containing (content, name, file_type, content_type, metadata, replace_if_exists)
 
         Returns:
-            FilesystemData: Metadata about the uploaded file
-
-        Raises:
-            ValueError: If the file already exists
+            tuple[list[FilesystemRecord], int, int]: List of uploaded files, total uploaded count, total failed count
         """
-        with GrpcFilesystem._handle_grpc_errors("UploadFile"):
-            request = filesystem_pb2.UploadFileRequest(
-                kin_context=self.mission_id,
-                name=name,
-                file_type=file_type.name,
-                content=content,
-            )
-            response: filesystem_pb2.UploadFileResponse = self.exec_grpc_query("UploadFile", request)
-            return FilesystemData(
-                kin_context=response.file.kin_context,
-                name=response.file.name,
-                file_type=FileType[FileTypeProto.Name(response.file.file_type)],
-                url=response.file.url,
-            )
+        logger.debug("Uploading %d files", len(files))
+        with GrpcFilesystem._handle_grpc_errors("UploadFiles"):
+            upload_files: list[filesystem_pb2.UploadFileData] = []
+            for file in files:
+                metadata_struct: struct_pb2.Struct | None = None
+                if file.metadata:
+                    metadata_struct = struct_pb2.Struct()
+                    metadata_struct.update(file.metadata)
+                upload_files.append(
+                    filesystem_pb2.UploadFileData(
+                        context=self.mission_id,
+                        name=file.name,
+                        file_type=self._file_type_to_enum(file.file_type),
+                        content_type=file.content_type or "application/octet-stream",
+                        content=file.content,
+                        metadata=metadata_struct,
+                        status=filesystem_pb2.FileStatus.FILE_STATUS_UPLOADING,
+                        replace_if_exists=file.replace_if_exists,
+                    )
+                )
+            request = filesystem_pb2.UploadFilesRequest(files=upload_files)
+            response: filesystem_pb2.UploadFilesResponse = self.exec_grpc_query("UploadFiles", request)
+            results = [self._file_proto_to_data(result.file) for result in response.results if result.HasField("file")]
+            logger.debug("Uploaded files: %s", results)
+            return results, response.total_uploaded, response.total_failed
 
-    def get(self, name: str) -> FilesystemData:
-        """Get file from the filesystem.
+    def get_file(
+        self,
+        file_id: str,
+        *,
+        include_content: bool = False,
+    ) -> FilesystemRecord:
+        """Get a file from the filesystem.
 
         Args:
-            name: The name of the file to be retrieved
+            file_id: The ID of the file to be retrieved
+            include_content: Whether to include file content in response
 
         Returns:
-            FilesystemData: Metadata about the retrieved file
+            FilesystemRecord: Metadata about the retrieved file
+
+        Raises:
+            FilesystemServiceError: If there is an error retrieving the file
         """
-        with GrpcFilesystem._handle_grpc_errors("GetFileByName"):
-            request = filesystem_pb2.GetFileByNameRequest(name=name)
-            response: filesystem_pb2.GetFileByNameResponse = self.exec_grpc_query("GetFileByName", request)
-            return FilesystemData(
-                kin_context=response.file.kin_context,
-                name=response.file.name,
-                file_type=FileType[FileTypeProto.Name(response.file.file_type)],
-                url=response.file.url,
+        with GrpcFilesystem._handle_grpc_errors("GetFile"):
+            request = filesystem_pb2.GetFileRequest(
+                context=self.mission_id,
+                file_id=file_id,
+                include_content=include_content,
             )
 
-    def update(self, name: str, content: bytes, file_type: FileType) -> FilesystemData:
-        """Update files in the filesystem.
+            response: filesystem_pb2.GetFileResponse = self.exec_grpc_query("GetFile", request)
+
+            return self._file_proto_to_data(response.file, response.content)
+
+    def update_file(
+        self,
+        file_id: str,
+        content: bytes | None = None,
+        file_type: str | None = None,
+        content_type: str | None = None,
+        metadata: dict[str, Any] | None = None,
+        new_name: str | None = None,
+        status: str | None = None,
+    ) -> FilesystemRecord:
+        """Update a file in the filesystem.
 
         Args:
-            name: The name of the file to be updated
-            content: The new content of the file
-            file_type: The type of data being uploaded
+            file_id: The id of the file to be updated
+            content: Optional new content of the file
+            file_type: Optional new type of data
+            content_type: Optional new MIME type
+            metadata: Optional new metadata (will merge with existing)
+            new_name: Optional new name for the file
+            status: Optional new status for the file
 
         Returns:
-            FilesystemData: Metadata about the updated file
+            FilesystemRecord: Metadata about the updated file
+
+        Raises:
+            FilesystemServiceError: If there is an error during update
         """
         with GrpcFilesystem._handle_grpc_errors("UpdateFile"):
             request = filesystem_pb2.UpdateFileRequest(
-                kin_context=self.mission_id,
-                name=name,
-                file_type=file_type.name,
+                context=self.mission_id,
+                file_id=file_id,
                 content=content,
-            )
-            response: filesystem_pb2.UpdateFileResponse = self.exec_grpc_query("UpdateFile", request)
-            return FilesystemData(
-                kin_context=response.file.kin_context,
-                name=response.file.name,
-                file_type=FileType[FileTypeProto.Name(response.file.file_type)],
-                url=response.file.url,
+                file_type=self._file_type_to_enum(file_type) if file_type else None,
+                content_type=content_type,
+                new_name=new_name,
+                status=self._file_status_to_enum(status) if status else None,
             )
 
-    def delete(self, name: str) -> bool:
-        """Delete files from the filesystem.
+            if metadata:
+                request.metadata.update(metadata)
 
-        Args:
-            name: The name of the file to be deleted
+            response: filesystem_pb2.UpdateFileResponse = self.exec_grpc_query("UpdateFile", request)
+            return self._file_proto_to_data(response.result.file)
 
-        Returns:
-            int: 1 if the file was deleted successfully, 0 if it didn't exist, None on error
-        """
-        with GrpcFilesystem._handle_grpc_errors("DeleteFile"):
-            request = filesystem_pb2.DeleteFileRequest(name=name)
-            _: filesystem_pb2.DeleteFileResponse = self.exec_grpc_query("DeleteFile", request)
-            return True
+    def delete_files(
+        self,
+        filters: FileFilter,
+        *,
+        permanent: bool = False,
+        force: bool = False,
+    ) -> tuple[dict[str, bool], int, int]:
+        """Delete multiple files from the filesystem.
 
-    def get_all(self) -> list[FilesystemData]:
-        """Get all files from the filesystem.
+        Args:
+            filters: Filter criteria for the files
+            permanent: Whether to permanently delete the files
+            force: Whether to force delete even if files are in use
 
         Returns:
-            list[FilesystemData]: A list of all files in the filesystem
+            tuple[dict[str, bool], int, int]: Results per file, total deleted count, total failed count
         """
-        with GrpcFilesystem._handle_grpc_errors("GetFilesByKinContext"):
-            request = filesystem_pb2.GetFilesByKinContextRequest(kin_context=self.mission_id)
-            response: filesystem_pb2.GetFilesByKinContextResponse = self.exec_grpc_query(
-                "GetFilesByKinContext", request
+        with GrpcFilesystem._handle_grpc_errors("DeleteFiles"):
+            request = filesystem_pb2.DeleteFilesRequest(
+                context=self.mission_id,
+                filters=self._filter_to_proto(filters),
+                permanent=permanent,
+                force=force,
             )
-            return [
-                FilesystemData(
-                    kin_context=file.kin_context,
-                    name=file.name,
-                    file_type=FileType[FileTypeProto.Name(file.file_type)],
-                    url=file.url,
-                )
-                for file in response.files
-            ]
 
-    def get_batch(self, names: list[str]) -> dict[str, FilesystemData | None]:
-        """Get files from the filesystem.
+            response: filesystem_pb2.DeleteFilesResponse = self.exec_grpc_query("DeleteFiles", request)
+            return dict(response.results), response.total_deleted, response.total_failed
+
+    def get_files(
+        self,
+        filters: FileFilter,
+        *,
+        list_size: int = 100,
+        offset: int = 0,
+        order: str | None = None,
+        include_content: bool = False,
+    ) -> tuple[list[FilesystemRecord], int]:
+        """Get multiple files from the filesystem.
 
         Args:
-            names: The names of the files to be retrieved
+            filters: Filter criteria for the files
+            list_size: Number of files to return per page
+            offset: Offset to start from
+            order: Field to order results by
+            include_content: Whether to include file content in response
 
         Returns:
-            list[FilesystemData]: A list of metadata about the retrieved files
+            tuple[list[FilesystemRecord], int]: List of files and total count
         """
-        with GrpcFilesystem._handle_grpc_errors("GetFilesByNames"):
-            request = filesystem_pb2.GetFilesByNamesRequest(names=names)
-            response: filesystem_pb2.GetFilesByNamesResponse = self.exec_grpc_query("GetFilesByNames", request)
-            result: dict[str, FilesystemData | None] = {}
-            for name, file_result in response.files.items():
-                which_field = file_result.WhichOneof("result")
-                if which_field == "file":
-                    result[name] = FilesystemData(
-                        kin_context=file_result.file.kin_context,
-                        name=file_result.file.name,
-                        file_type=FileType[FileTypeProto.Name(file_result.file.file_type)],
-                        url=file_result.file.url,
-                    )
-                elif which_field == "error":
-                    # Handle error case
-                    result[name] = None
-                    logger.warning("Error retrieving file '%s': %s", name, file_result.error)
-            return result
+        with GrpcFilesystem._handle_grpc_errors("GetFiles"):
+            request = filesystem_pb2.GetFilesRequest(
+                context=self.mission_id,
+                filters=self._filter_to_proto(filters),
+                include_content=include_content,
+                list_size=list_size,
+                offset=offset,
+                order=order,
+            )
+            response: filesystem_pb2.GetFilesResponse = self.exec_grpc_query("GetFiles", request)
+
+            return [self._file_proto_to_data(file) for file in response.files], response.total_count

digitalkin/services/setup/grpc_setup.py

@@ -139,6 +139,7 @@ class GrpcSetup(SetupStrategy, GrpcClientWrapper):
                 current_setup_version = setup_pb2.SetupVersion(**valid_data.current_setup_version.model_dump())
 
             request = setup_pb2.UpdateSetupRequest(
+                setup_id=valid_data.id,
                 name=valid_data.name,
                 owner_id=valid_data.owner_id or "",
                 current_setup_version=current_setup_version,