digitalkin 0.1.1__py3-none-any.whl → 0.2.1__py3-none-any.whl

digitalkin/services/snapshot/snapshot_strategy.py

@@ -3,12 +3,11 @@
 from abc import ABC, abstractmethod
 from typing import Any
 
+from digitalkin.services.base_strategy import BaseStrategy
 
-class SnapshotStrategy(ABC):
-    """Abstract base class for snapshot strategies."""
 
-    def __init__(self) -> None:
-        """Initialize the snapshot strategy."""
+class SnapshotStrategy(BaseStrategy, ABC):
+    """Abstract base class for snapshot strategies."""
 
     @abstractmethod
     def create(self, data: dict[str, Any]) -> str:

digitalkin/services/storage/__init__.py

@@ -1,6 +1,7 @@
 """This module is responsible for handling the storage service."""
 
-from .default_storage import DefaultStorage
-from .storage_strategy import StorageStrategy
+from digitalkin.services.storage.default_storage import DefaultStorage
+from digitalkin.services.storage.grpc_storage import GrpcStorage
+from digitalkin.services.storage.storage_strategy import StorageStrategy
 
-__all__ = ["DefaultStorage", "StorageStrategy"]
+__all__ = ["DefaultStorage", "GrpcStorage", "StorageStrategy"]

digitalkin/services/storage/default_storage.py

@@ -1,91 +1,218 @@
 """This module implements the default storage strategy."""
 
+import datetime
+import json
 import logging
-from typing import Any
+import tempfile
+from pathlib import Path
 
-from .storage_strategy import StorageStrategy
+from pydantic import BaseModel
+
+from digitalkin.services.storage.storage_strategy import DataType, StorageRecord, StorageStrategy
 
 logger = logging.getLogger(__name__)
 
 
 class DefaultStorage(StorageStrategy):
-    """This class implements the default storage strategy."""
-
-    storage: dict[str, list[dict[str, Any]]]
+    """This class implements the default storage strategy with file persistence.
 
-    def __init__(self) -> None:
-        """Initialize the default storage strategy."""
-        super().__init__()
-        self.storage = {"setups": []}
+    The storage persists data in a local JSON file, enabling data retention
+    across multiple requests or module instances.
+    """
 
-    def connect(self) -> bool:  # noqa: PLR6301
-        """Establish connection to the database.
+    def _load_from_file(self) -> dict[str, StorageRecord]:
+        """Load storage data from the file.
 
         Returns:
-            bool: True if the connection is successful, False otherwise
+            A dictionary containing the loaded storage records
         """
-        return True
-
-    def disconnect(self) -> bool:  # noqa: PLR6301
-        """Close connection to the database.
+        file_path = Path(self.storage_file_path)
+        if not file_path.exists():
+            return {}
+
+        try:
+            records = {}
+            file_content = json.loads(file_path.read_text(encoding="utf-8"))
+
+            for key, record_dict in file_content.items():
+                # Get the stored record data
+                name = record_dict.get("name", "")
+                model_class = self.config.get(name)
+
+                if not model_class:
+                    logger.warning("No model found for record %s", name)
+                    continue
+
+                # Create a model instance from the stored data
+                data_dict = record_dict.get("data", {})
+                try:
+                    data_model = model_class.model_validate(data_dict)
+                except Exception:
+                    logger.exception("Failed to validate data for record %s", name)
+                    continue
+
+                # Create a StorageRecord object
+                record = StorageRecord(
+                    mission_id=record_dict.get("mission_id", ""),
+                    name=name,
+                    data=data_model,
+                    data_type=DataType[record_dict.get("data_type", "OUTPUT")],
+                )
+
+                # Set dates if they exist
+                if "creation_date" in record_dict:
+                    record.creation_date = datetime.datetime.fromisoformat(record_dict["creation_date"])
+                if "update_date" in record_dict:
+                    record.update_date = datetime.datetime.fromisoformat(record_dict["update_date"])
+
+                records[key] = record
+        except json.JSONDecodeError:
+            logger.exception("Error decoding JSON from file")
+            return {}
+        except FileNotFoundError:
+            logger.info("Storage file not found, starting with empty storage")
+            return {}
+        except Exception:
+            logger.exception("Unexpected error loading storage")
+            return {}
+        return records
+
+    def _save_to_file(self) -> None:
+        """Save storage data to the file using a safe write pattern."""
+        # Usage of pathlib for file operations
+        file_path = Path(self.storage_file_path)
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+
+        try:
+            # Convert storage to a serializable format
+            serializable_data = {}
+            for key, record in self.storage.items():
+                record_dict = {
+                    "mission_id": record.mission_id,
+                    "name": record.name,
+                    "data_type": record.data_type.name,  # Convert enum to string
+                    "data": record.data.model_dump(),  # Convert Pydantic model to dict
+                }
+
+                # Handle dates (convert to ISO format strings)
+                if record.creation_date:
+                    record_dict["creation_date"] = record.creation_date.isoformat()
+                if record.update_date:
+                    record_dict["update_date"] = record.update_date.isoformat()
+
+                serializable_data[key] = record_dict
+
+            # usage of NamedTemporaryFile for atomic writes
+            with tempfile.NamedTemporaryFile(
+                mode="w", encoding="utf-8", dir=str(file_path.parent), delete=False, suffix=".tmp"
+            ) as temp_file:
+                json.dump(serializable_data, temp_file, indent=2)
+                temp_path = temp_file.name
+
+            # Creation of a backup if the file already exists
+            if file_path.exists():
+                backup_path = f"{self.storage_file_path}.bak"
+                file_path.replace(backup_path)
+
+            # Remplacement du fichier (opération atomique) avec pathlib
+            Path(temp_path).replace(str(file_path))
+
+        except PermissionError:
+            logger.exception("Permission denied when saving to file")
+        except OSError:
+            logger.exception("OS error when saving to file")
+        except Exception:
+            logger.exception("Unexpected error saving storage")
+
+    def _store(self, record: StorageRecord) -> StorageRecord:
+        """Store a new record in the database and persist to file.
+
+        Args:
+            record: The record to store
 
         Returns:
-            bool: True if the connection is closed, False otherwise
+            str: The ID of the new record
+
+        Raises:
+            ValueError: If the record already exists
         """
-        return True
+        name = record.name
+        if name in self.storage:
+            msg = f"Record with name {name} already exists"
+            raise ValueError(msg)
+        self.storage[name] = record
+        self.storage[name].creation_date = datetime.datetime.now(datetime.timezone.utc)
+        self.storage[name].update_date = datetime.datetime.now(datetime.timezone.utc)
 
-    def create(self, table: str, data: dict[str, Any]) -> str:
-        """Create a new record in the database.
+        # Persist to file
+        self._save_to_file()
 
-        Returns:
-            str: The ID of the new record
-        """
-        if table not in self.storage:
-            self.storage[table] = []
-        self.storage[table].append(data["data"])
-        logger.info("CREATE %s:%s succesfull", table, data["data"])
-        return f"{len(self.storage[table]) - 1}"
+        logger.info("CREATE %s:%s successful", name, record)
+        return self.storage[name]
 
-    def get(self, table: str, data: dict[str, Any]) -> list[dict[str, Any]]:
+    def _read(self, name: str) -> StorageRecord | None:
         """Get records from the database.
 
+        Args:
+            name: The unique name to retrieve data for
+
         Returns:
-            list[dict[str, Any]]: The list of records
+            StorageRecord: The corresponding record
         """
-        logger.info("GET table = %s: keys = %s", table, data["keys"])
-        if table not in self.storage:
-            logger.info("GET table = %s: TABLE DOESN'T EXIST", table)
-            return []
-        return [self.storage[table][int(key)] for key in data["keys"]]
+        logger.info("GET record linked to the key = %s", name)
+        if name not in self.storage:
+            logger.info("GET key = %s: DOESN'T EXIST", name)
+            return None
+        return self.storage[name]
+
+    def _modify(self, name: str, data: BaseModel) -> StorageRecord | None:
+        """Update records in the database and persist to file.
 
-    def update(self, table: str, data: dict[str, Any]) -> int:
-        """Update records in the database.
+        Args:
+            name: The unique name to store the data under
+            data: The data to modify
 
         Returns:
-            int: The number of records updated
+            StorageRecord: The modified record
         """
-        if table not in self.storage:
-            logger.info("UPDATE table = %s: TABLE DOESN'T EXIST", table)
-            return 0
-        self.storage[table][data["update_id"]] = data["update_value"]
-        return 1
+        if name not in self.storage:
+            logger.info("UPDATE key = %s: DOESN'T EXIST", name)
+            return None
+        self.storage[name].data = data
+        self.storage[name].update_date = datetime.datetime.now(datetime.timezone.utc)
 
-    def delete(self, table: str, data: dict[str, Any]) -> int:
-        """Delete records from the database.
+        # Persist to file
+        self._save_to_file()
 
-        Returns:
-            int: The number of records deleted
-        """
-        if table not in self.storage:
-            logger.info("UPDATE table = %s: TABLE DOESN'T EXIST", table)
-            return 0
-        del self.storage[table][data["delete_id"]]
-        return 1
+        return self.storage[name]
+
+    def _remove(self, name: str) -> bool:
+        """Delete records from the database and update file.
 
-    def get_all(self) -> dict[str, list[dict[str, Any]]]:
-        """Get all records from the database.
+        Args:
+            name: The unique name to remove a record
 
         Returns:
-            dict[str, list[dict[str, Any]]]: table with respective list of records
+            bool: True if the record was removed, False otherwise
         """
-        return self.storage
+        if name not in self.storage:
+            logger.info("DELETE key = %s: DOESN'T EXIST", name)
+            return False
+        del self.storage[name]
+
+        # Persist to file
+        self._save_to_file()
+
+        return True
+
+    def __init__(
+        self,
+        mission_id: str,
+        config: dict[str, type[BaseModel]],
+        storage_file_path: str = "local_storage",
+        **kwargs,  # noqa: ANN003, ARG002
+    ) -> None:
+        """Initialize the storage."""
+        super().__init__(mission_id=mission_id, config=config)
+        self.storage_file_path = f"{self.mission_id}_{storage_file_path}.json"
+        self.storage = self._load_from_file()

digitalkin/services/storage/grpc_storage.py

@@ -1,207 +1,113 @@
 """This module implements the default storage strategy."""
 
-import datetime
 import logging
-from enum import Enum, auto
-from typing import Any
 
-import grpc
 from digitalkin_proto.digitalkin.storage.v2 import data_pb2, storage_service_pb2_grpc
 from google.protobuf import json_format
 from google.protobuf.struct_pb2 import Struct
 from pydantic import BaseModel
 
-from digitalkin.grpc.utils.exceptions import ServerError
-from digitalkin.grpc.utils.models import SecurityMode, ServerConfig
-from digitalkin.services.storage.storage_strategy import StorageStrategy
+from digitalkin.grpc_servers.utils.grpc_client_wrapper import GrpcClientWrapper
+from digitalkin.grpc_servers.utils.models import ServerConfig
+from digitalkin.services.storage.storage_strategy import StorageRecord, StorageServiceError, StorageStrategy
 
 logger = logging.getLogger(__name__)
 
 
-class DataType(Enum):
-    """."""
-
-    OUTPUT = auto()
-    VIEW = auto()
-
-
-class StorageData(BaseModel):
-    """."""
-
-    data: dict[str, Any]
-    mission_id: str
-    name: str
-    timestamp: datetime.datetime
-    type: DataType
-
-
-class GrpcStorage(StorageStrategy):
+class GrpcStorage(StorageStrategy, GrpcClientWrapper):
     """This class implements the default storage strategy."""
 
-    def _init_channel(self, config: ServerConfig) -> grpc.Channel:
-        """Create an appropriate channel to the registry server.
-
-        Returns:
-            A gRPC channel for communication with the registry.
-
-        Raises:
-            ValueError: If credentials are required but not provided.
-        """
-        if config.security == SecurityMode.SECURE and config.credentials:
-            # Secure channel
-            with open(config.credentials.server_cert_path, "rb") as cert_file:  # noqa: FURB101
-                certificate_chain = cert_file.read()
-
-            root_certificates = None
-            if config.credentials.root_cert_path:
-                with open(config.credentials.root_cert_path, "rb") as root_cert_file:  # noqa: FURB101
-                    root_certificates = root_cert_file.read()
-
-            # Create channel credentials
-            channel_credentials = grpc.ssl_channel_credentials(root_certificates=root_certificates or certificate_chain)
-
-            return grpc.secure_channel(f"{config.host}:{config.port}", channel_credentials)
-        # Insecure channel
-        return grpc.insecure_channel(f"{config.host}:{config.port}")
-
-    def __post_init__(self, config: ServerConfig) -> None:
-        """Init the channel from a config file.
-
-        Need to be call if the user register a gRPC channel.
-        """
-        channel = self._init_channel(config)
+    def __init__(
+        self,
+        mission_id: str,
+        config: dict[str, type[BaseModel]],
+        server_config: ServerConfig,
+        **kwargs,  # noqa: ANN003, ARG002
+    ) -> None:
+        """Initialize the storage."""
+        super().__init__(mission_id=mission_id, config=config)
+
+        channel = self._init_channel(server_config)
         self.stub = storage_service_pb2_grpc.StorageServiceStub(channel)
         logger.info("Channel client 'storage' initialized succesfully")
 
-    def exec_grpc_query(self, query_endpoint: str, request: Any) -> Any:
-        """."""
-        try:
-            # Call the register method
-            logger.warning("send request to %s", query_endpoint)
-            response = getattr(self.stub, query_endpoint)(request)
-            logger.warning("recive response from request to registry: %s", response)
-
-            if response.success:
-                logger.info("Module registered successfully")
-            else:
-                logger.error("Module registration failed")
-                return response
-        except grpc.RpcError:
-            logger.exception("RPC error during registration:")
-            raise ServerError
-
-    def connect(self) -> bool:  # noqa: PLR6301
-        """Establish connection to the database.
-
-        Returns:
-            bool: True if the connection is successful, False otherwise
-        """
-        return True
+    def _store(self, record: StorageRecord) -> StorageRecord:
+        """Create a new record in the database.
 
-    def disconnect(self) -> bool:  # noqa: PLR6301
-        """Close connection to the database.
+        Parameters:
+            record: The record to store
 
         Returns:
-            bool: True if the connection is closed, False otherwise
-        """
-        return True
-
-    def create(self, table: str, data: dict[str, Any]) -> str:
-        """Create a new record in the database.
+            StorageRecord: The corresponding record
 
-        Returns:
-            str: The ID of the new record
+        Raises:
+            StorageServiceError: If there is an error while storing the record
         """
-        # Create a Struct for the data
-        data_struct = Struct()
-        if data.get("data"):
-            data_struct.update(data["data"])
-
-        request = data_pb2.StoreDataRequest(
-            data=data_struct,
-            mission_id=data["mission_id"],
-            name=data["name"],
-            type=data_pb2.DataType.Name(1),
-        )
-        return self.exec_grpc_query("StoreData", request)
-
-    def get_data_by_mission(self, table: str, data: dict[str, Any]):
-        request = data_pb2.GetDataByMissionRequest(mission_id=data["mission_id"])
-        response = self.exec_grpc_query("GetDataByMission", request)
-        return [
-            StorageData(
-                data=data.data,
-                mission_id=data.mission_id,
-                name=data.name,
-                timestamp=data.timestamp,
-                type=data.type,
-            )
-            for data in response.data_items
-        ]
-
-    def get_data_by_name(self, table: str, data: dict[str, Any]):
-        request = data_pb2.GetDataByNameRequest(mission_id=data["mission_id"], name=data_pb2.DataType())
-        response = self.exec_grpc_query("GetDataByName", request)
-        return [
-            StorageData(
-                data=data.data,
-                mission_id=data.mission_id,
-                name=data.name,
-                timestamp=data.timestamp,
-                type=data.type,
-            )
-            for data in response.stored_data
-        ]
-
-    def get_data_by_type(self, table: str, data: dict[str, Any]):
-        request = data_pb2.GetDataByTypeRequest(
-            mission_id=data["mission_id"],
-            type=data_pb2.DataType.Name(data["type"]),
-        )
-        response = self.exec_grpc_query("GetDataByType", request)
-        return [
-            StorageData(
-                data=json_format.MessageToDict(data.data),
-                mission_id=data.mission_id,
-                name=data.name,
-                timestamp=data.timestamp,
-                type=data.type,
+        try:
+            # Create a Struct for the data
+            data_struct = Struct()
+            data_struct.update(record.data.model_dump())
+
+            request = data_pb2.StoreRecordRequest(
+                data=data_struct,
+                mission_id=record.mission_id,
+                name=record.name,
+                data_type=record.data_type.name,
             )
-            for data in response.stored_data
-        ]
-
-    def delete(self, table: str, data: dict[str, Any]) -> int:
-        """Delete records from the database.
+            return self.exec_grpc_query("StoreRecord", request)
+        except Exception:
+            msg = f"Error while storing record {record.name}"
+            logger.exception(msg)
+            raise StorageServiceError(msg)
 
-        Returns:
-            int: The number of records deleted
-        """
-        request = data_pb2.DeleteDataRequest(
-            mission_id=data["mission_id"],
-            name=data_pb2.DataType.Name(data["name"]),
-        )
-        return self.exec_grpc_query("DeleteData", request)
-
-    def get(self, table: str, data: dict[str, Any]) -> list[dict[str, Any]]:
+    def _read(self, name: str) -> StorageRecord | None:
         """Get records from the database.
 
         Returns:
-            list[dict[str, Any]]: The list of records
+            list[StorageData]: The list of records
         """
-        return []
-
-    def update(self, table: str, data: dict[str, Any]) -> int:
+        try:
+            request = data_pb2.ReadRecordRequest(mission_id=self.mission_id, name=name)
+            response: data_pb2.ReadRecordResponse = self.exec_grpc_query("ReadRecord", request)
+            return StorageRecord(**json_format.MessageToDict(response.stored_data))
+        except Exception:
+            msg = f"Error while reading record {name}"
+            logger.exception(msg)
+            return None
+
+    def _modify(self, name: str, data: BaseModel) -> StorageRecord | None:
         """Update records in the database.
 
         Returns:
             int: The number of records updated
         """
-        return 1
-
-    def get_all(self) -> dict[str, list[dict[str, Any]]]:
-        """Get all records from the database.
+        try:
+            # Create a Struct for the data
+            data_struct = Struct()
+            data_struct.update(data.model_dump())
+
+            request = data_pb2.ModifyRecordRequest(data=data_struct, mission_id=self.mission_id, name=name)
+            response: data_pb2.ModifyRecordResponse = self.exec_grpc_query("ModifyRecord", request)
+            return StorageRecord(**json_format.MessageToDict(response.stored_data))
+        except Exception:
+            msg = f"Error while modifing record {name}"
+            logger.exception(msg)
+            return None
+
+    def _remove(self, name: str) -> bool:
+        """Delete records from the database.
 
         Returns:
-            dict[str, list[dict[str, Any]]]: table with respective list of records
+            int: The number of records deleted
         """
-        return {}
+        try:
+            request = data_pb2.RemoveRecordRequest(
+                mission_id=self.mission_id,
+                name=name,
+            )
+            self.exec_grpc_query("RemoveRecord", request)
+        except Exception:
+            msg = f"Error while removin record {name}"
+            logger.exception(msg)
+            return False
+        return True