manta-node 0.5b0__py3-none-any.whl

manta_node/tasks.py

@@ -0,0 +1,273 @@
+from datetime import datetime
+from pathlib import Path
+from typing import Callable, Iterable, Iterator, List, Tuple
+
+from manta_common.build.common.tasks import MqttTask, TaskStatus, TaskUpdate
+from manta_common.conversions import ID, TupleID
+from .domain.task_lifecycle import TaskLifecycleService, TaskTransitionRule
+
+
+class TaskProgression:
+    """
+    Task progression is a data structure which holds :
+
+    * a task update
+    * a module file
+    * a start time
+    """
+
+    __slots__ = ["task_update", "module_file", "start_time", "last_log_collected"]
+
+    def __init__(
+        self, task_update: TaskUpdate, module_file: Path, start_time: datetime
+    ):
+        self.task_update = task_update
+        self.module_file = module_file
+        self.start_time = start_time
+        self.last_log_collected = start_time
+
+    @property
+    def task_id(self) -> ID:
+        """
+        Get Task ID
+
+        Returns
+        -------
+        ID
+            Task ID
+        """
+        return ID(self.task_update.task_id)
+
+    @property
+    def swarm_id(self) -> ID:
+        """
+        Get Swarm ID
+
+        Returns
+        -------
+        ID
+            Swarm ID
+        """
+        return ID(self.task_update.swarm_id)
+
+    @property
+    def task_status(self) -> TaskStatus:
+        """
+        Get Task Status
+
+        Returns
+        -------
+        TaskStatus
+            Task Status
+        """
+        return self.task_update.task_status
+
+    @task_status.setter
+    def task_status(self, task_status: TaskStatus):
+        """
+        Set a new task status
+
+        Parameters
+        ----------
+        task_status : TaskStatus
+            Task Status to set
+        """
+        self.task_update.task_status = task_status
+
+    def unlink(self, missing_ok: bool = True):
+        """
+        Unlink module file
+
+        Parameters
+        ----------
+        missing_ok : bool
+            Extra parameter from :code:`Path.unlink`
+        """
+        self.module_file.unlink(missing_ok)
+
+    def is_active_task(self) -> bool:
+        """
+        Check if is an active task
+
+        Returns
+        -------
+        bool
+            Is active task
+        """
+        # Use domain service for business logic
+        return TaskLifecycleService.is_active(self.task_status)
+
+    def is_finished_task(self) -> bool:
+        """
+        Check if is a finished task
+
+        Returns
+        -------
+        bool
+            Is a finished task
+        """
+        # Use domain rule for terminal state check
+        return TaskTransitionRule.is_finished(self.task_status)
+
+    def compare_iteration(self, mqtt_task: MqttTask) -> bool:
+        """
+        Compare iteration between a task from MQTT and itself
+
+        Parameters
+        ----------
+        mqtt_task : MqttTask
+            Task from MQTT
+
+        Returns
+        -------
+        bool
+            Same iteration
+        """
+        return (
+            self.task_update.iteration == mqtt_task.iteration
+            and self.task_update.circular == mqtt_task.circular
+        )
+
+
+class TaskProgressions:
+    """
+    List of Task Progression used in Task Manager and World Servicer
+    """
+
+    __slots__ = ["_tasks", "_env_ids"]
+
+    def __init__(self):
+        self._tasks: List[TaskProgression] = []
+        self._env_ids: List[Tuple[TupleID, TupleID]] = []
+
+    def __iter__(self) -> Iterator[TaskProgression]:
+        """
+        Iterate over tasks
+
+        Returns
+        -------
+        Iterator[TaskProgression]
+            Iterator of task progressions
+        """
+        return iter(self._tasks)
+
+    def __getitem__(self, env_id: Tuple[TupleID, TupleID]) -> TaskProgression:
+        """
+        Return the task progression given the task ID
+
+        Parameters
+        ----------
+        env_id : str
+            Task and Swarms IDs in a tuple
+
+        Returns
+        -------
+        TaskProgression
+            Task Progression
+        """
+        if env_id not in self._env_ids:
+            raise KeyError(
+                f"Cannot get task due to {env_id} not found in tasks {self._env_ids}"
+            )
+        return self._tasks[self._env_ids.index(env_id)]
+
+    def __contains__(self, env_id: Tuple[TupleID, TupleID]) -> bool:
+        """
+        Check if the task ID exists in itself
+
+        Parameters
+        ----------
+        env_id : Tuple[TupleID, TupleID]
+            Task and Swarms IDs in a tuple
+
+        Returns
+        -------
+        bool
+            It contains the task ID
+        """
+        return env_id in self._env_ids
+
+    def append(self, task_progression: TaskProgression):
+        """
+        Append a task progression
+
+        Parameters
+        ----------
+        task_progression : TaskProgression
+            Task Progression
+        """
+        self._tasks.append(task_progression)
+        self._env_ids.append(
+            (
+                ID(task_progression.task_update.swarm_id).tid,
+                task_progression.task_id.tid,
+            )
+        )
+
+    def filter(
+        self, condition: Callable[[TaskProgression], bool]
+    ) -> Iterable[TaskProgression]:
+        """
+        Filter keys given the condition function
+
+        Parameters
+        ----------
+        condition : Callable[[TaskProgression], bool]
+            Condition function
+
+        Returns
+        -------
+        Iterable[TaskProgression]
+            Filtered keys
+        """
+        return filter(condition, self._tasks)
+
+    def pop(self, env_id: Tuple[TupleID, TupleID]) -> TaskProgression:
+        """
+        Pop a task progression given the task ID
+
+        Parameters
+        ----------
+        task_id : Tuple[TupleID, TupleID]
+            Task and Swarms IDs in a tuple
+
+        Returns
+        -------
+        TaskProgression
+            Task progression popped
+        """
+        if env_id not in self._env_ids:
+            raise KeyError(f"Cannot pop task due to {env_id} not found in tasks")
+        index = self._env_ids.index(env_id)
+        self._env_ids.pop(index)
+        return self._tasks.pop(index)
+
+    def active_tasks(self) -> Iterable[TaskProgression]:
+        """
+        Return an iterable of active tasks
+
+        Returns
+        -------
+        Iterable[TaskProgression]
+            Active task progressions
+        """
+        return (
+            task_progression
+            for task_progression in self._tasks
+            if task_progression.is_active_task()
+        )
+
+    def finished_tasks(self) -> Iterable[TaskProgression]:
+        """
+        Return an iterable of finished tasks
+
+        Returns
+        -------
+        Iterable[TaskProgression]
+            Finished task progressions
+        """
+        return (
+            task_progression
+            for task_progression in self._tasks
+            if task_progression.is_finished_task()
+        )

manta_node/utils.py

@@ -0,0 +1,52 @@
+from pathlib import Path
+from typing import List
+
+from manta_common.build.common.informations import Overview
+
+__all__ = ["collect_dataset_overviews"]
+
+DTYPES = {
+    ".txt": "text",
+    ".csv": "csv",
+    ".npx": "numpy",
+    ".npy": "numpy",
+    ".npz": "numpy",
+    ".np": "numpy",
+}
+
+
+def collect_dataset_overviews(data_folder: Path) -> List[Overview]:
+    """
+    Collect dataset overviews including :
+
+    - name of a dataset
+    - its description
+    - its dtype (numpy, pytorch ...)
+
+    Parameters
+    ----------
+    data_folder: Path
+        Data folder
+
+    Returns
+    -------
+    List[Overview]
+        List of dataset overviews
+    """
+    overviews = []
+    # Check if the file exists in the data_folder
+    if not data_folder.exists():
+        # Create the data folder
+        data_folder.mkdir(exist_ok=True)
+        return overviews
+
+    # Get the list of files in the data_folder
+    overviews = [
+        Overview(
+            name=file.name,
+            description="No description available",
+            dtype=DTYPES.get(file.suffix, "binary") if file.is_file() else "folder",
+        )
+        for file in data_folder.iterdir()
+    ]
+    return overviews