idmtools-platform-general 0.0.0.dev0__py3-none-any.whl → 0.0.2__py3-none-any.whl

idmtools_platform_file/platform_operations/experiment_operations.py

@@ -0,0 +1,314 @@
+"""
+Here we implement the FilePlatform experiment operations.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import shutil
+from pathlib import Path
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, List, Type, Dict, Optional, Any
+from idmtools.assets import Asset, AssetCollection
+from idmtools.core import ItemType
+from idmtools.entities import Suite
+from idmtools.entities.experiment import Experiment
+from idmtools.entities.iplatform_ops.iplatform_experiment_operations import IPlatformExperimentOperations
+from idmtools_platform_file.platform_operations.utils import FileExperiment, FileSimulation, FileSuite
+from logging import getLogger
+
+logger = getLogger(__name__)
+user_logger = getLogger('user')
+
+if TYPE_CHECKING:
+    from idmtools_platform_file.file_platform import FilePlatform
+
+
+@dataclass
+class FilePlatformExperimentOperations(IPlatformExperimentOperations):
+    """
+    Experiment Operations for File Platform.
+    """
+    platform: 'FilePlatform'  # noqa: F821
+    platform_type: Type = field(default=FileExperiment)
+    RUN_SIMULATION_SCRIPT_PATH = Path(__file__).parent.parent.joinpath('assets/run_simulation.sh')
+
+    def get(self, experiment_id: str, **kwargs) -> FileExperiment:
+        """
+        Gets an experiment from the File platform.
+        Args:
+            experiment_id: experiment id
+            kwargs: keyword arguments used to expand functionality
+        Returns:
+            File Experiment object
+        """
+        metas = self.platform._metas.filter(item_type=ItemType.EXPERIMENT, property_filter={'id': str(experiment_id)})
+        if len(metas) > 0:
+            return FileExperiment(metas[0])
+        else:
+            raise RuntimeError(f"Not found Experiment with id '{experiment_id}'")
+
+    def platform_create(self, experiment: Experiment, **kwargs) -> FileExperiment:
+        """
+        Creates an experiment on File Platform.
+        Args:
+            experiment: idmtools experiment
+            kwargs: keyword arguments used to expand functionality
+        Returns:
+            File Experiment object created
+        """
+        self.platform.mk_directory(experiment, exist_ok=True)
+        meta = self.platform._metas.dump(experiment)
+        self.platform._assets.dump_assets(experiment)
+        self.platform.create_batch_file(experiment, **kwargs)
+
+        # Copy file run_simulation.sh
+        dest_script = Path(self.platform.get_directory(experiment)).joinpath('run_simulation.sh')
+        shutil.copy(str(self.RUN_SIMULATION_SCRIPT_PATH), str(dest_script))
+
+        # Make executable
+        self.platform.update_script_mode(dest_script)
+
+        # Return File Experiment
+        return FileExperiment(meta)
+
+    def get_children(self, experiment: FileExperiment, parent: Experiment = None, raw=True, **kwargs) -> List[Any]:
+        """
+        Fetch file experiment's children.
+        Args:
+            experiment: File experiment
+            raw: True/False
+            parent: the parent of the simulations
+            kwargs: keyword arguments used to expand functionality
+        Returns:
+            List of file simulations
+        """
+        sim_list = []
+        sim_meta_list = self.platform._metas.get_children(experiment)
+        for meta in sim_meta_list:
+            file_sim = FileSimulation(meta)
+            file_sim.status = self.platform.get_simulation_status(file_sim.id)
+            if raw:
+                sim_list.append(file_sim)
+            else:
+                sim = self.platform._simulations.to_entity(file_sim, parent=parent)
+                sim_list.append(sim)
+        return sim_list
+
+    def get_parent(self, experiment: FileExperiment, **kwargs) -> FileSuite:
+        """
+        Fetches the parent of an experiment.
+        Args:
+            experiment: File experiment
+            kwargs: keyword arguments used to expand functionality
+        Returns:
+            The Suite being the parent of this experiment.
+        """
+        if experiment.parent_id is None:
+            return None
+        else:
+            return self.platform._suites.get(experiment.parent_id, raw=True, **kwargs)
+
+    def platform_run_item(self, experiment: Experiment, **kwargs):
+        """
+        Run experiment.
+        Args:
+            experiment: idmtools Experiment
+            kwargs: keyword arguments used to expand functionality
+        Returns:
+            None
+        """
+        # Ensure parent
+        self.platform._metas.dump(experiment)
+        if experiment.parent:
+            user_logger.info(f'suite: {str(experiment.parent.id)}')
+
+        user_logger.info(f'job_directory: {Path(self.platform.job_directory).resolve()}')
+        user_logger.info(f'experiment: {experiment.id}')
+        user_logger.info(f"\nExperiment Directory: \n{self.platform.get_directory(experiment)}")
+
+    def post_run_item(self, experiment: Experiment, **kwargs):
+        """
+        Perform post-processing steps after an experiment run.
+        Args:
+            experiment: The experiment object that has just finished running
+            **kwargs: Additional keyword arguments
+
+        Returns:
+            None
+        """
+        super().post_run_item(experiment, **kwargs)
+        # Refresh platform object
+        experiment._platform_object = self.get(experiment.id, **kwargs)
+
+    def send_assets(self, experiment: Experiment, **kwargs):
+        """
+        Copy our experiment assets.
+        Replaced by self.platform._assets.dump_assets(experiment)
+        Args:
+            experiment: idmtools Experiment
+            kwargs: keyword arguments used to expand functionality
+        Returns:
+            None
+        """
+        pass
+
+    def list_assets(self, experiment: Experiment, **kwargs) -> List[Asset]:
+        """
+        List assets for an experiment.
+        Args:
+            experiment: Experiment to get assets for
+            kwargs:
+        Returns:
+            List[Asset]
+        """
+        assets = self.platform._assets.list_assets(experiment, **kwargs)
+        return assets
+
+    def get_assets_from_file_experiment(self, experiment: FileExperiment) -> AssetCollection:
+        """
+        Get assets for a comps experiment.
+        Args:
+            experiment: Experiment to get asset collection for.
+        Returns:
+            AssetCollection if configuration is set and configuration.asset_collection_id is set.
+        """
+        assets = AssetCollection()
+        assets_dir = Path(self.platform.get_directory_by_id(experiment.id, ItemType.EXPERIMENT), 'Assets')
+        if assets_dir.exists():
+            assets_list = AssetCollection.assets_from_directory(assets_dir, recursive=True)
+            for a in assets_list:
+                assets.add_asset(a)
+        return assets
+
+    def to_entity(self, file_exp: FileExperiment, parent: Optional[Suite] = None, children: bool = True,
+                  **kwargs) -> Experiment:
+        """
+        Convert a FileExperiment  to idmtools Experiment.
+        Args:
+            file_exp: simulation to convert
+            parent: optional experiment object
+            children: bool
+            kwargs:
+        Returns:
+            Experiment object
+        """
+        exp = Experiment()
+        exp.platform = self.platform
+        exp.uid = file_exp.uid
+        exp.name = file_exp.name
+        if parent:
+            exp.parent = parent
+        elif file_exp.suite_id:
+            exp.parent = self.platform.get_item(file_exp.suite_id, ItemType.SUITE, force=True)
+        exp.tags = file_exp.tags
+        exp._platform_object = file_exp
+        exp.simulations = []
+
+        exp.assets = self.get_assets_from_file_experiment(file_exp)
+        if exp.assets is None:
+            exp.assets = AssetCollection()
+
+        if children:
+            exp.simulations = self.get_children(file_exp, parent=exp, raw=False)
+
+        return exp
+
+    def refresh_status(self, experiment: Experiment, **kwargs):
+        """
+        Refresh status of experiment.
+        Args:
+            experiment: idmtools Experiment
+            kwargs: keyword arguments used to expand functionality
+        Returns:
+            Dict of simulation id as key and working dir as value
+        """
+        # Refresh status for each simulation
+        for sim in experiment.simulations:
+            sim.status = self.platform.get_simulation_status(sim.id, **kwargs)
+
+    def create_sim_directory_map(self, experiment_id: str) -> Dict:
+        """
+        Build simulation working directory mapping.
+        Args:
+            experiment_id: experiment id
+
+        Returns:
+            Dict of simulation id as key and working dir as value
+        """
+        exp = self.platform.get_item(experiment_id, ItemType.EXPERIMENT, raw=False)
+        sims = exp.simulations
+        return {sim.id: str(self.platform.get_directory(sim)) for sim in sims}
+
+    def platform_delete(self, experiment_id: str) -> None:
+        """
+        Delete platform experiment.
+        Args:
+            experiment_id: platform experiment id
+        Returns:
+            None
+        """
+        exp = self.platform.get_item(experiment_id, ItemType.EXPERIMENT, raw=False)
+        try:
+            shutil.rmtree(self.platform.get_directory(exp))
+        except RuntimeError:
+            logger.info("Could not delete the associated experiment...")
+            return
+
+    def platform_cancel(self, experiment_id: str, force: bool = True) -> Any:
+        """
+        Cancel platform experiment's file job.
+        Args:
+            experiment_id: experiment id
+            force: bool, True/False
+        Returns:
+            Any
+        """
+        pass
+
+    def get_assets(self, experiment: Experiment, files: List[str], **kwargs) -> Dict[str, bytearray]:
+        """
+        Fetch the files associated with an experiment.
+
+        Args:
+            experiment: Experiment (idmools Experiment or COMPSExperiment)
+            files: List of files to download
+            **kwargs:
+
+        Returns:
+            Dict[str, Dict[str, Dict[str, str]]]:
+                A nested dictionary structured as:
+                {
+                    experiment.id: {
+                        simulation.id {
+                            filename: file content as string,
+                            ...
+                        },
+                        ...
+                    }
+                }
+        """
+        ret = dict()
+        if isinstance(experiment, FileExperiment):
+            file_exp = experiment
+        else:
+            file_exp = experiment.get_platform_object()
+        simulations = self.platform.flatten_item(file_exp, raw=True)
+        for sim in simulations:
+            ret[sim.id] = self.platform._simulations.get_assets(sim, files, **kwargs)
+        return ret
+
+    def run_item(self, experiment: Experiment, **kwargs):
+        """
+        Called during commissioning of an item. This should create the remote resource.
+
+        Args:
+            experiment:Experiment
+            **kwargs: Keyword arguments to pass to pre_run_item, platform_run_item, post_run_item
+
+        Returns:
+            None
+        """
+        # Consider Suite
+        if experiment.parent:
+            experiment.parent.add_experiment(experiment)
+            self.platform._suites.platform_create(experiment.parent)
+        super().run_item(experiment, **kwargs)

idmtools_platform_file/platform_operations/json_metadata_operations.py

@@ -0,0 +1,320 @@
+"""
+Here we implement the JSON Metadata operations.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import os
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING, Dict, List, Type, Union
+from dataclasses import dataclass, field
+from idmtools.core import ItemType
+from idmtools.core.interfaces import imetadata_operations
+from idmtools.entities import Suite
+from idmtools.entities.experiment import Experiment
+from idmtools.entities.simulation import Simulation
+from idmtools.utils.json import IDMJSONEncoder
+from idmtools_platform_file.platform_operations.utils import FileSuite, FileExperiment
+
+if TYPE_CHECKING:
+    from idmtools_platform_file.file_platform import FilePlatform
+
+
+@dataclass
+class JSONMetadataOperations(imetadata_operations.IMetadataOperations):
+    """
+    JSON operations used in File Platform.
+    """
+    platform: 'FilePlatform'  # noqa: F821
+    platform_type: Type = field(default=None)
+    metadata_filename: str = field(default='metadata.json')
+
+    @staticmethod
+    def _read_from_file(filepath: Union[Path, str]) -> Dict:
+        """
+        Utility: read metadata from a file.
+        Args:
+            filepath: metadata file path
+        Returns:
+            JSON
+        """
+        filepath = Path(filepath)
+        with filepath.open(mode='r') as f:
+            metadata = json.load(f)
+        return metadata
+
+    @staticmethod
+    def _write_to_file(filepath: Union[Path, str], data: Dict, indent: int = None) -> None:
+        """
+        Utility: save metadata to a file.
+        Args:
+            filepath: metadata file path
+            data: metadata as dictionary
+            indent: indent level for pretty printing the JSON file. None for compact JSON.
+        Returns:
+            None
+        """
+        filepath = Path(filepath)
+        filepath.parent.mkdir(parents=True, exist_ok=True)
+        with filepath.open(mode='w') as f:
+            json.dump(data, f, indent=indent, cls=IDMJSONEncoder)
+
+    def get_metadata_filepath(self, item: Union[Suite, Experiment, Simulation]) -> Path:
+        """
+        Retrieve item's metadata file path.
+        Args:
+            item: idmtools entity (Suite, Experiment and Simulation)
+        Returns:
+            item's metadata file path
+        """
+        if not isinstance(item, (Suite, Experiment, Simulation)):
+            raise RuntimeError("get_metadata_filepath method supports Suite/Experiment/Simulation only.")
+        item_dir = self.platform.get_directory(item)
+        filepath = Path(item_dir, self.metadata_filename)
+        return filepath
+
+    def get_metadata_filepath_by_id(self, item_id: str, item_type: ItemType) -> Path:
+        """
+        Retrieve item's metadata file path.
+        Args:
+            item_id: item id
+            item_type: the type of metadata to search for matches (simulation, experiment, suite, etc.)
+        Returns:
+            item's metadata file path
+        """
+        if item_type not in (ItemType.SUITE, ItemType.EXPERIMENT, ItemType.SIMULATION):
+            raise RuntimeError("get_metadata_filepath method supports Suite/Experiment/Simulation only.")
+        item_dir = self.platform.get_directory_by_id(item_id, item_type)
+        filepath = Path(item_dir, self.metadata_filename)
+        return filepath
+
+    def get(self, item: Union[Suite, Experiment, Simulation]) -> Dict:
+        """
+        Obtain item's metadata.
+        Args:
+            item: idmtools entity (Suite, Experiment and Simulation)
+        Returns:
+             key/value dict of metadata from the given item
+        """
+        if not isinstance(item, (Suite, Experiment, Simulation)):
+            raise RuntimeError("Get method supports Suite/Experiment/Simulation only.")
+        data = item.to_dict()
+        if isinstance(item, Suite):
+            data.pop('experiments', None)
+        meta = json.loads(json.dumps(data, cls=IDMJSONEncoder))
+        meta['id'] = meta['_uid']
+        meta['uid'] = meta['_uid']
+        meta['status'] = 'CREATED'
+        meta['dir'] = os.path.abspath(self.platform.get_directory(item))
+
+        if isinstance(item, Suite):
+            meta['experiments'] = [experiment.id for experiment in item.experiments]
+        elif isinstance(item, Experiment):
+            meta['suite_id'] = meta["parent_id"]
+            meta['simulations'] = [simulation.id for simulation in item.simulations]
+        elif isinstance(item, Simulation):
+            meta['experiment_id'] = meta["parent_id"]
+        return meta
+
+    def dump(self, item: Union[Suite, Experiment, Simulation]) -> Dict:
+        """
+        Save item's metadata to a file and also save tags.json file.
+        Args:
+            item: idmtools entity (Suite, Experiment and Simulation)
+        Returns:
+            key/value dict of metadata from the given item
+        """
+        if not isinstance(item, (Suite, Experiment, Simulation)):
+            raise RuntimeError("Dump method supports Suite/Experiment/Simulation only.")
+        dest = self.get_metadata_filepath(item)
+        meta = self.get(item)
+        self._write_to_file(dest, meta)
+
+        # Also write tags.json file
+        keys_to_extract = ["id", "item_type", "tags"]
+        extracted = {key: meta[key] for key in keys_to_extract}
+
+        tags_path = dest.parent / "tags.json"
+        self._write_to_file(tags_path, extracted, indent=2)
+        return meta
+
+    def load(self, item: Union[Suite, Experiment, Simulation]) -> Dict:
+        """
+        Obtain item's metadata file.
+        Args:
+            item: idmtools entity (Suite, Experiment and Simulation)
+        Returns:
+             key/value dict of metadata from the given item
+        """
+        if not isinstance(item, (Suite, Experiment, Simulation)):
+            raise RuntimeError("Load method supports Suite/Experiment/Simulation only.")
+        meta_file = self.get_metadata_filepath(item)
+        meta = self._read_from_file(meta_file)
+        return meta
+
+    def load_from_file(self, metadata_filepath: Union[Path, str]) -> Dict:
+        """
+        Obtain the metadata for the given filepath.
+        Args:
+            metadata_filepath: str
+        Returns:
+             key/value dict of metadata from the given filepath
+        """
+        if not Path(metadata_filepath).exists():
+            raise RuntimeError(f"File not found: '{metadata_filepath}'.")
+        meta = self._read_from_file(metadata_filepath)
+        return meta
+
+    def update(self, item: Union[Suite, Experiment, Simulation], metadata: Dict = None, replace=True) -> None:
+        """
+        Update or replace item's metadata file.
+        Args:
+            item: idmtools entity (Suite, Experiment and Simulation.)
+            metadata: dict to be updated or replaced
+            replace: True/False
+        Returns:
+             None
+        """
+        if metadata is None:
+            metadata = {}
+        if not isinstance(item, (Suite, Experiment, Simulation)):
+            raise RuntimeError("Set method supports Suite/Experiment/Simulation only.")
+        meta = metadata
+        if not replace:
+            meta = self.load(item)
+            meta.update(metadata)
+        meta_file = self.get_metadata_filepath(item)
+        self._write_to_file(meta_file, meta)
+
+    def clear(self, item: Union[Suite, Experiment, Simulation]) -> None:
+        """
+        Clear the item's metadata file.
+        Args:
+            item: clear the item's metadata file
+        Returns:
+            None
+        """
+        if not isinstance(item, (Suite, Experiment, Simulation)):
+            raise RuntimeError("Clear method supports Suite/Experiment/Simulation only.")
+        self.update(item=item, metadata={}, replace=True)
+
+    def get_children(self, item: Union[Suite, Experiment, FileSuite, FileExperiment]) -> List[Dict]:
+        """
+        Fetch item's children.
+        Args:
+            item: idmtools entity (Suite, FileSuite, Experiment, FileExperiment)
+        Returns:
+            Lis of metadata
+        """
+        if not isinstance(item, (Suite, FileSuite, Experiment, FileExperiment)):
+            raise RuntimeError("Get children method supports [File]Suite and [File]Experiment only.")
+        item_list = []
+        if isinstance(item, (Suite, FileSuite)):
+            meta = self.load(item)
+            for exp_id in meta['experiments']:
+                meta_file = self.get_metadata_filepath_by_id(exp_id, ItemType.EXPERIMENT)
+                exp_meta = self._read_from_file(meta_file)
+                item_list.append(exp_meta)
+        else:
+            item_dir = self.platform.get_directory(item)
+            pattern = f'*/{self.metadata_filename}'
+            for meta_file in item_dir.glob(pattern=pattern):
+                meta = self.load_from_file(meta_file)
+                item_list.append(meta)
+        return item_list
+
+    def get_all(self, item_type: ItemType, item_id: str = '') -> List[Dict]:
+        """
+        Obtain all the metadata for a given item type.
+        Args:
+            item_type: the type of metadata to search for matches (simulation, experiment, suite, etc.)
+            item_id: item id
+        Returns:
+            list of metadata with given item type
+        """
+        root = Path(self.platform.job_directory)
+        item_list = []
+
+        if item_type is ItemType.SIMULATION:
+            # Match sim under experiment, under optional suite
+            patterns = [
+                f"s_*/e_*/*{item_id}/{self.metadata_filename}",  # suite/experiment/simulation
+                f"e_*/*{item_id}/{self.metadata_filename}",  # experiment/simulation (no suite)
+            ]
+        elif item_type is ItemType.EXPERIMENT:
+            patterns = [
+                f"s_*/e_*{item_id}/{self.metadata_filename}",  # suite/experiment
+                f"e_*{item_id}/{self.metadata_filename}",  # standalone experiment
+            ]
+        elif item_type is ItemType.SUITE:
+            patterns = [
+                f"s_*{item_id}/{self.metadata_filename}",  # suite only
+            ]
+        else:
+            raise RuntimeError(f"Unknown item type: {item_type}")
+
+        # Search each pattern
+        for pattern in patterns:
+            for meta_file in root.glob(pattern):
+                try:
+                    meta = self.load_from_file(meta_file)
+                    item_list.append(meta)
+                except Exception as e:
+                    print(f"Warning: Failed to load metadata from {meta_file}: {e}")
+
+        return item_list
+
+    @staticmethod
+    def _match_filter(item: Dict, metadata: Dict, ignore_none=True):
+        """
+        Utility: verify if item match metadata.
+        Note: compare key/value if value is not None else just check key exists
+        Args:
+            item: dict represents metadata of Suite/Experiment/Simulation
+            metadata: dict as a filter
+            ignore_none: True/False (ignore None value or not)
+        Returns:
+            list of Dict items
+        """
+        for k, v in metadata.items():
+            if ignore_none:
+                if v is None:
+                    is_match = k in item
+                else:
+                    is_match = k in item and item[k] == v
+            else:
+                if v is None:
+                    is_match = k in item and item[k] is None
+                else:
+                    is_match = k in item and item[k] == v
+            if not is_match:
+                return False
+        return True
+
+    def filter(self, item_type: ItemType, property_filter: Dict = None, tag_filter: Dict = None,
+               meta_items: List[Dict] = None, ignore_none=True) -> List[Dict]:
+        """
+        Obtain all items that match the given properties key/value pairs passed.
+        The two filters are applied on item with 'AND' logical checking.
+        Args:
+            item_type: the type of items to search for matches (simulation, experiment, suite, etc.)
+            property_filter: a dict of metadata key/value pairs for exact match searching
+            tag_filter: a dict of metadata key/value pairs for exact match searching
+            meta_items: list of metadata
+            ignore_none: True/False (ignore None value or not)
+        Returns:
+            a list of metadata matching the properties key/value with given item type
+        """
+        if meta_items is None:
+            item_id = property_filter["id"] if property_filter and "id" in property_filter else ''
+            meta_items = self.get_all(item_type, item_id=item_id)
+        item_list = []
+        for meta in meta_items:
+            is_match = True
+            if property_filter:
+                is_match = self._match_filter(meta, property_filter, ignore_none=ignore_none)
+            if tag_filter:
+                is_match = is_match and self._match_filter(meta['tags'], tag_filter, ignore_none=ignore_none)
+            if is_match:
+                item_list.append(meta)
+        return item_list