cars 1.0.0rc1__cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl

cars/orchestrator/orchestrator_constants.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+this module contains the orchestrator constants
+"""
+
+
+CARS_DATASET_KEY = "cars_ds_key"
+SAVING_INFO = "saving_info"
+CARS_DS_ROW = "cars_ds_row"
+CARS_DS_COL = "cars_ds_col"

cars/orchestrator/registry/__init__.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Cars registry module init file
+"""

cars/orchestrator/registry/abstract_registry.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+This module contains the abstract registry class
+"""
+
+import logging
+from abc import abstractmethod
+
+# CARS imports
+from cars.orchestrator.orchestrator_constants import (
+    CARS_DATASET_KEY,
+    CARS_DS_COL,
+    CARS_DS_ROW,
+    SAVING_INFO,
+)
+
+
+class AbstractCarsDatasetRegistry:
+    """
+    AbstractCarsDatasetRegistry
+    This is the abstract class of registries, managing delayed CarsDatasets
+
+    """
+
+    def __init__(self, id_generator):
+        """
+        Init function of AbstractCarsDatasetRegistry
+
+        :param id_generator: id generator
+        :type id_generator: IdGenerator
+
+        """
+
+        self.id_generator = id_generator
+        self.id_generator.add_registry(self)
+
+    @abstractmethod
+    def cars_dataset_in_registry(self, cars_ds):
+        """
+        Check if a CarsDataset is already registered, return id if exists
+
+        :param cars_ds: cars dataset
+        :type cars_ds: CarsDataset
+
+        :return : True if in registry, if of cars dataset
+        :rtype : Tuple(bool, int)
+        """
+
+    @abstractmethod
+    def get_cars_datasets_list(self):
+        """
+        Get a list of registered CarsDataset
+
+        :return list of CarsDataset
+        :rtype: list(CarsDataset)
+        """
+
+    @abstractmethod
+    def get_cars_ds(self, future_result):
+        """
+        Get a list of registered CarsDataset
+
+        :param future_result: object to get cars dataset from
+
+        :return corresponding CarsDataset
+        :rtype: CarsDataset
+        """
+
+    @staticmethod
+    def get_future_cars_dataset_id(future_result):
+        """
+        Get cars dataset id for current future result
+
+        :param future_result: future result:
+        :type future_result: xr.Dataset or pd.DataFrame
+
+        :return cars dataset id
+        :rtype : int
+        """
+
+        cars_ds_id = None
+
+        if isinstance(future_result, dict):
+            attributes_info_dict = future_result
+        else:
+            attributes_info_dict = future_result.attrs
+
+        if SAVING_INFO in attributes_info_dict:
+            if CARS_DATASET_KEY in attributes_info_dict[SAVING_INFO]:
+                cars_ds_id = attributes_info_dict[SAVING_INFO][CARS_DATASET_KEY]
+
+        return cars_ds_id
+
+    @staticmethod
+    def get_future_cars_dataset_position(future_result):
+        """
+        Get cars dataset positions for current future result
+
+        :param future_result: future result:
+        :type future_result: xr.Dataset or pd.DataFrame
+
+        :return cars dataset id
+        :rtype : tuple(int)
+        """
+
+        cars_ds_row = None
+        cars_ds_col = None
+
+        if isinstance(future_result, dict):
+            attributes_info_dict = future_result
+        else:
+            attributes_info_dict = future_result.attrs
+
+        if SAVING_INFO in attributes_info_dict:
+            if CARS_DS_ROW in attributes_info_dict[SAVING_INFO]:
+                cars_ds_row = attributes_info_dict[SAVING_INFO][CARS_DS_ROW]
+            else:
+                logging.debug("No row given in object")
+            if CARS_DS_COL in attributes_info_dict[SAVING_INFO]:
+                cars_ds_col = attributes_info_dict[SAVING_INFO][CARS_DS_COL]
+            else:
+                logging.debug("No col given in object")
+
+        return cars_ds_row, cars_ds_col

cars/orchestrator/registry/compute_registry.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+This module contains the replacer registry class
+"""
+
+# CARS imports
+from cars.orchestrator.registry.abstract_registry import (
+    AbstractCarsDatasetRegistry,
+)
+
+
+class CarsDatasetRegistryCompute(AbstractCarsDatasetRegistry):
+    """
+    CarsDatasetRegistryCompute
+    This registry manages the computation of arriving future results
+    into corresponding CarsDataset
+    """
+
+    def __init__(self, id_generator):
+        """
+        Init function of CarsDatasetRegistryCompute
+
+        :param id_generator: id generator
+        :type id_generator: IdGenerator
+
+        """
+        super().__init__(id_generator)
+        self.registered_cars_datasets = []
+        self.cars_ds_ids = []
+
+    def cars_dataset_in_registry(self, cars_ds):
+        """
+        Check if a CarsDataset is already registered, return id if exists
+
+        :param cars_ds: cars dataset
+        :type cars_ds: CarsDataset
+
+        :return : True if in registry, if of cars dataset
+        :rtype : Tuple(bool, int)
+        """
+        cars_ds_id = None
+        in_registry = False
+
+        if cars_ds in self.registered_cars_datasets:
+            in_registry = True
+            cars_ds_id = self.cars_ds_ids[
+                self.registered_cars_datasets.index(cars_ds)
+            ]
+
+        return in_registry, cars_ds_id
+
+    def get_cars_datasets_list(self):
+        """
+        Get a list of registered CarsDataset
+
+        :return list of CarsDataset
+        :rtype: list(CarsDataset)
+        """
+
+        return self.registered_cars_datasets
+
+    def add_cars_ds_to_compute(self, cars_ds):
+        """
+        Add cars dataset to registry
+
+        :param cars_ds: cars dataset
+        :type cars_ds: CarsDataset
+
+        """
+
+        # Generate_id
+        new_id = self.id_generator.get_new_id(cars_ds)
+        self.cars_ds_ids.append(new_id)
+        self.registered_cars_datasets.append(cars_ds)
+
+    def get_cars_ds(self, future_result):
+        """
+        Get a list of registered CarsDataset
+
+        :param future_result: object to get cars dataset from
+
+        :return corresponding CarsDataset
+        :rtype: CarsDataset
+        """
+        raise RuntimeError(
+            "get_cars_ds shoud not be used in CarsDatasetRegistryCompute"
+        )

cars/orchestrator/registry/id_generator.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+this module contains the abstract id generator class
+"""
+
+from cars.orchestrator.orchestrator_constants import CARS_DATASET_KEY
+
+# CARS imports
+from cars.orchestrator.registry.unseen_registry import CarsDatasetRegistryUnseen
+
+
+class IdGenerator:
+    """
+    IdGenerator
+    Creates
+    """
+
+    def __init__(self):
+        """
+        Init function of IdGenerator
+
+        """
+        self.current_id = 0
+        self.registries = []
+
+        # Create this registry if user gets saving infos
+        # before telling orchestrator
+        # he wants to save it or replace it
+        self.unseen_registry = CarsDatasetRegistryUnseen(self)
+        # self.unseen_registry is now in self.registries, seen Abstract init
+
+    def add_registry(self, registry):
+        """
+        Add registry to self
+
+        :param registry: registry
+        :type registry: AbstractCarsDatasetRegistry
+        """
+
+        self.registries.append(registry)
+
+    def get_new_id(self, cars_ds):
+        """
+        Generate new id
+
+        :param cars_ds: cars dataset
+        :type cars_ds: CarsDataset
+
+        :return id
+        :rtype: int
+        """
+
+        # Check if dataset already registered
+        registered = False
+        allready_registered_id = None
+        returned_id = None
+        for registry in self.registries:
+            (
+                is_registered,
+                allready_registered_id,
+            ) = registry.cars_dataset_in_registry(cars_ds)
+            if is_registered:
+                registered = True
+                returned_id = allready_registered_id
+
+        if not registered:
+            returned_id = self.current_id
+            self.current_id += 1
+
+        return returned_id
+
+    def get_saving_infos(self, cars_ds):
+        """
+        Get saving infos
+
+        :param cars_ds: cars dataset
+        :type cars_ds: CarsDataset
+
+        :return saving infos
+        :rtype: dict
+        """
+
+        obj_id = None
+        for registry in self.registries:
+            (
+                is_registered,
+                allready_registered_id,
+            ) = registry.cars_dataset_in_registry(cars_ds)
+            if is_registered:
+                obj_id = allready_registered_id
+
+        # add cars_ds to other register to create id
+        obj_id = self.unseen_registry.add_cars_ds_to_unseen(cars_ds)
+
+        infos = {CARS_DATASET_KEY: obj_id}
+
+        return infos

cars/orchestrator/registry/replacer_registry.py

@@ -0,0 +1,213 @@
+#!/usr/bin/env python
+# coding: utf8
+#
+# Copyright (c) 2020 Centre National d'Etudes Spatiales (CNES).
+#
+# This file is part of CARS
+# (see https://github.com/CNES/cars).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+This module contains the replacer registry class
+"""
+
+# CARS imports
+from cars.orchestrator.registry.abstract_registry import (
+    AbstractCarsDatasetRegistry,
+)
+
+
+class CarsDatasetRegistryReplacer(AbstractCarsDatasetRegistry):
+    """
+    CarsDatasetRegistryReplacer
+    This registry manages the replacement of arriving future results
+    into corresponding CarsDataset
+    """
+
+    def __init__(self, id_generator):
+        """
+        Init function of CarsDatasetRegistryReplacer
+
+        :param id_generator: id generator
+        :type id_generator: IdGenerator
+
+        """
+        super().__init__(id_generator)
+        self.registered_cars_datasets_replacers = []
+
+    def cars_dataset_in_registry(self, cars_ds):
+        """
+        Check if a CarsDataset is already registered, return id if exists
+
+        :param cars_ds: cars dataset
+        :type cars_ds: CarsDataset
+
+        :return : True if in registry, if of cars dataset
+        :rtype : Tuple(bool, int)
+        """
+
+        in_registry = False
+        registered_id = None
+        for obj in self.registered_cars_datasets_replacers:
+            if cars_ds == obj.cars_ds:
+                in_registry = True
+                registered_id = obj.obj_id
+                break
+
+        return in_registry, registered_id
+
+    def get_cars_datasets_list(self):
+        """
+        Get a list of registered CarsDataset
+
+        :return list of CarsDataset
+        :rtype: list(CarsDataset)
+        """
+
+        cars_ds_list = []
+
+        for cars_ds_replacer in self.registered_cars_datasets_replacers:
+            cars_ds_list.append(cars_ds_replacer.cars_ds)
+
+        return cars_ds_list
+
+    def get_cars_ds_replacer_corresponding_id(self, obj_id):
+        """
+        Get the SingleCarsDatasetReplacer corresponding to given id
+
+        :param obj_id: cars dataset id
+        :type obj_id: int
+
+        :return : single cars dataset replacer
+        :rtype : SingleCarsDatasetReplacer
+        """
+
+        cars_ds_replacer = None
+        for obj in self.registered_cars_datasets_replacers:
+            if obj_id == obj.obj_id:
+                cars_ds_replacer = obj
+                break
+
+        return cars_ds_replacer
+
+    def replace_lasting_jobs(self, delayed_type):
+        """
+        Replace last jobs by Nones
+        """
+        for cars_ds in self.get_cars_datasets_list():
+            for row in range(cars_ds.shape[0]):
+                for col in range(cars_ds.shape[1]):
+                    if isinstance(cars_ds[row, col], delayed_type):
+                        cars_ds[row, col] = None
+
+    def get_cars_ds(self, future_result):
+        """
+        Get a list of registered CarsDataset
+
+        :param future_result: object to get cars dataset from
+
+        :return corresponding CarsDataset
+        :rtype: CarsDataset
+        """
+
+        obj_id = self.get_future_cars_dataset_id(future_result)
+        cars_ds_replacer = self.get_cars_ds_replacer_corresponding_id(obj_id)
+
+        if cars_ds_replacer is None:
+            return None
+        return cars_ds_replacer.cars_ds
+
+    def add_cars_ds_to_replace(self, cars_ds):
+        """
+        Add cars dataset to registry
+
+        :param cars_ds: cars dataset
+        :type cars_ds: CarsDataset
+
+        """
+
+        # Generate_id
+        new_id = self.id_generator.get_new_id(cars_ds)
+        # create CarsDataset replacer
+        replacer = SingleCarsDatasetReplacer(cars_ds, new_id)
+        self.registered_cars_datasets_replacers.append(replacer)
+
+    def get_corresponding_replacer(self, future_result):
+        """
+        Get replacer corresponding to future result
+
+        :param future_result: future result
+        :type future_result: xr.Dataset or pandas.DataFrame
+
+        :return replacer
+        :rtype: SingleCarsDatasetReplacer
+        """
+
+        cars_ds_id = self.get_future_cars_dataset_id(future_result)
+
+        replacer = None
+
+        for cars_ds_replacer in self.registered_cars_datasets_replacers:
+            if cars_ds_replacer.obj_id == cars_ds_id:
+                replacer = cars_ds_replacer
+                break
+
+        return replacer
+
+    def replace(self, future_result):
+        """
+        Replace future result
+
+        :param future_result: xr.Dataset or pandas.DataFrame or Dict
+
+        """
+
+        replacer = self.get_corresponding_replacer(future_result)
+
+        if replacer is not None:
+            if not replacer.as_been_seen:
+                # reset all tiles to None
+                # for row in range(replacer.cars_ds.shape[0]):
+                #     for col in range(replacer.cars_ds.shape[1]):
+                #         replacer.cars_ds[row, col] = None
+
+                replacer.as_been_seen = True
+
+            # replace tile
+            row, col = self.get_future_cars_dataset_position(future_result)
+
+            replacer.cars_ds[row, col] = future_result
+
+
+class SingleCarsDatasetReplacer:  # pylint: disable=R0903
+    """
+    SingleCarsDatasetReplacer
+
+    Manages the replacement of a CarsDataset
+    """
+
+    def __init__(self, cars_ds, obj_id):
+        """
+        Init function of CarsDatasetRegistryReplacer
+
+        :param cars_ds: cars dataset
+        :type cars_ds: CarsDataset
+        :param obj_id: object id
+        :type obj_id: int
+        """
+
+        self.cars_ds = cars_ds
+        self.obj_id = obj_id
+
+        self.as_been_seen = False