idmtools 0.0.0.dev0__py3-none-any.whl → 0.0.3__py3-none-any.whl

idmtools/utils/filter_simulations.py

@@ -0,0 +1,182 @@
+"""
+Filtering utility.
+
+Copyright 2025, Gates Foundation. All rights reserved.
+"""
+from idmtools.core import ItemType, EntityStatus
+from idmtools.core.interfaces.ientity import IEntity
+from idmtools.entities.experiment import Experiment
+from idmtools.entities.iplatform import IPlatform
+from idmtools.utils.general import parse_value_tags
+
+
+class FilterItem:
+    """
+    FilterItem provides a utility to filter items on a platform.
+    """
+
+    @staticmethod
+    def filter_item(platform: IPlatform, item: IEntity, tags=None, status: EntityStatus = None,
+                    entity_type: bool = False, max_simulations: int = None, skip_sims=None, **kwargs):
+        """
+        Filter simulations from an Experiment or Suite using tag and status criteria.
+
+        By default, this filters simulations that have a status of `EntityStatus.SUCCEEDED`.
+        Additional filtering can be applied by specifying tag values or tag-based conditions.
+
+        This method supports:
+            - Skipping specific simulations by ID
+            - Filtering based on simulation status (e.g., FAILED, SUCCEEDED)
+            - Tag-based filtering (both exact match and conditional/lambda-based)
+
+        Examples:
+            >>> filter_item(platform, experiment, status=EntityStatus.FAILED)
+            >>> filter_item(platform, experiment, tags={"Run_Number": "2"})
+            >>> filter_item(platform, experiment, tags={"Run_Number": lambda v: 2 <= v <= 10})
+            >>> filter_item(platform, experiment, tags={"Coverage": 0.8}, status=EntityStatus.SUCCEEDED)
+            >>> filter_item(platform, experiment, tags={"Coverage": 0.8}, max_simulations=10)
+
+        Args:
+            platform (IPlatform): The platform instance to query simulations from.
+            item (IEntity): An Experiment or Suite to filter simulations from.
+            tags (dict): A dictionary of tag key-value pairs to filter by for a simulation. Values may be:
+                * A fixed value (e.g., {"Run_Number": 2})
+                * A lambda or callable function for conditional logic
+                (e.g., {"Run_Number": lambda v: 2 <= v <= 10})
+            status (EntityStatus, Optional): The experiment's status.
+            entity_type (bool, optional): If True, return simulation entities instead of just their IDs.
+            skip_sims (list, optional): A list of simulation IDs (as strings) to exclude from the results.
+            max_simulations (int, optional): Maximum number of simulations to return. Returns all if not set.
+            **kwargs: Extra args.
+
+        Returns:
+            Union[List[Union[str, Simulation]], Dict[str, List[Union[str, Simulation]]]]:
+            If the item is an `Experiment`, returns a list of simulation IDs or simulation entities
+            (if `entity_type=True`).
+
+            If the item is a `Suite`, returns list ofdictionary where each key is an experiment ID and
+            each value is a list of simulation IDs or simulation entities (depending on the `entity_type` flag).
+
+        """
+        def match_tags(sim: IEntity, tags=None):
+            """
+            Check if simulation match tags.
+
+            Args:
+                sim: simulation
+                tags: search tags
+
+            Returns: bool True/False
+            """
+            # If no tags are provided, treat it as an empty filter (match all)
+            if tags is None:
+                return True
+            # Normalize simulation tag values and wrap them with TagValue for safe comparisons
+            # (e.g., allows "5" == 5 and supports operators like >, <, == in tag filters)
+            sim_tags = parse_value_tags(sim.tags, wrap_with_tagvalue=True)
+
+            # Iterate over each tag filter condition
+            for k, v in tags.items():
+                sim_val = sim_tags.get(k)
+                # If the simulation does not have the tag, skip it
+                if sim_val is None:
+                    return False
+                # If the filter value is a callable (e.g., lambda), evaluate the condition
+                if callable(v):
+                    if not v(sim_val):
+                        return False
+                # Otherwise, do a direct comparison between the simulation tag and expected value
+                elif sim_val != v:
+                    return False
+
+            return True
+
+        if item.item_type not in [ItemType.EXPERIMENT, ItemType.SUITE]:
+            raise ValueError("This method only supports Experiment and Suite types!")
+
+        if skip_sims is None:
+            skip_sims = []
+        # ------------------------------------------------------------------ #
+        # Base case  ─ Experiment
+        # ------------------------------------------------------------------ #
+        if isinstance(item, Experiment):
+            potential_sims = item.get_simulations()
+
+            # filter by status
+            sims_status_filtered = [sim for sim in potential_sims if sim.status == status] if status else potential_sims
+
+            # filter tags
+            sims_tags_filtered = [sim for sim in sims_status_filtered if match_tags(sim, tags)]
+
+            # filter sims
+            sims_id_filtered = [sim for sim in sims_tags_filtered if str(sim.uid) not in skip_sims]
+
+            # consider max_simulations for return
+            sims_final = sims_id_filtered[0:max_simulations if max_simulations else len(sims_id_filtered)]
+
+            if entity_type:
+                return sims_final
+            else:
+                return [s.id for s in sims_final]
+
+        # Suite case:
+        experiments = item.get_experiments()
+        result = {}
+
+        for exp in experiments:
+            filtered = FilterItem.filter_item(
+                platform,
+                item=exp,
+                tags=tags,
+                status=status,
+                entity_type=entity_type,
+                max_simulations=max_simulations,
+                skip_sims=skip_sims,
+                **kwargs,
+            )
+            result[exp.id] = filtered
+        return result
+
+    @classmethod
+    def filter_item_by_id(cls, platform: IPlatform, item_id: str, item_type: ItemType = ItemType.EXPERIMENT,
+                          tags=None, status=None, entity_type=False, skip_sims=None, max_simulations: int = None,
+                          **kwargs):
+        """
+        Retrieve and filter simulations from an Experiment or Suite by item ID.
+
+        This method looks up the specified item (Experiment or Suite) by ID on the given platform,
+        then filters its simulations using the class's `filter_item()` method.
+
+        Args:
+            platform (IPlatform): The platform instance used to fetch the item.
+            item_id (str): The unique identifier of the Experiment or Suite.
+            item_type (ItemType, optional): The type of item (Experiment or Suite). Defaults to Experiment.
+            tags (dict, optional): A simulation's tags to filter by.
+            status (EntityStatus, Optional): The experiment's status.
+            entity_type (bool, optional): If True, return simulation entities instead of just their IDs.
+            skip_sims (List[str], optional): List of simulation IDs to skip during filtering. Defaults to an empty list.
+            max_simulations (int, optional): Maximum number of simulations to return. Defaults to None (no limit).
+            **kwargs: Additional keyword arguments passed to `filter_item()`.
+
+        Returns:
+            Union[List[Union[str, Simulation]], Dict[str, List[Union[str, Simulation]]]]:
+            If the item is an `Experiment`, returns a list of simulation IDs or simulation entities
+            (if `entity_type=True`).
+
+            If the item is a `Suite`, returns a dictionary where each key is an experiment ID and
+            each value is a list of simulation IDs or simulation entities (depending on the `entity_type` flag).
+
+        Raises:
+            ValueError: If the provided `item_type` is not Experiment or Suite.
+        """
+        if skip_sims is None:
+            skip_sims = []
+        if item_type not in [ItemType.EXPERIMENT, ItemType.SUITE]:
+            raise ValueError("This method only supports Experiment and Suite types!")
+
+        # retrieve item by id and type
+        item = platform.get_item(item_id, item_type, raw=False, force=True)
+
+        # filter simulations
+        return cls.filter_item(platform, item=item, tags=tags, status=status, entity_type=entity_type,
+                               skip_sims=skip_sims, max_simulations=max_simulations, **kwargs)

idmtools/utils/filters/__init__.py

@@ -0,0 +1,5 @@
+"""
+defines filter utilities.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""

idmtools/utils/filters/asset_filters.py

@@ -0,0 +1,88 @@
+"""
+This module contains all the default filters for the assets.
+
+A filter function needs to take only one argument: an asset. It returns True/False indicating whether to add or filter
+out the asset.
+
+You can notice functions taking more than only an asset.
+To use those functions, use must create a partial before adding it to a filters list.
+For example::
+
+    python
+    fname = partial(file_name_is, filenames=["a.txt", "b.txt"])
+    AssetCollection.from_directory(... filters=[fname], ...)
+"""
+import os
+import typing
+
+if typing.TYPE_CHECKING:
+    from typing import List
+    from idmtools.core import TAsset
+
+
+def default_asset_file_filter(asset: 'TAsset') -> 'bool':
+    """
+    Default filter to leave out Python caching.
+
+    This filter is used in the creation of
+    :class:`~idmtools.assets.asset_collection.AssetCollection`, regardless of user filters.
+
+    Returns:
+        True if no files match default patterns of "__py_cache__" and ".pyc"
+    """
+    patterns = [
+        "__pycache__",
+        ".pyc"
+    ]
+    return not any([p in asset.absolute_path for p in patterns])
+
+
+def file_name_is(asset: 'TAsset', filenames: 'List[str]') -> 'bool':
+    """
+    Restrict filtering to assets with the indicated filenames.
+
+    Args:
+        asset: The asset to filter.
+        filenames: List of filenames to filter on.
+
+    Returns:
+        True if asset.filename in filenames
+    """
+    return asset.filename in filenames
+
+
+def file_extension_is(asset: 'TAsset', extensions: 'List[str]') -> 'bool':
+    """
+    Restrict filtering to assets with the indicated filetypes.
+
+    Args:
+        asset: The asset to filter.
+        extensions: List of extensions to filter on.
+
+    Returns:
+        True if extension in extensions
+    """
+    return asset.extension in extensions
+
+
+def asset_in_directory(asset: 'TAsset', directories: 'List[str]', base_path: str = None) -> 'bool':
+    """
+    Restrict filtering to assets within a given directory.
+
+    This filter is not strict and simply checks if the directory portion is present in the assets absolute path.
+
+    Args:
+        asset: The asset to filter.
+        directories: List of directory portions to include.
+        base_path: base_path
+    """
+    if base_path is None:
+        base_path = os.getcwd()
+    norm_base_path = os.path.abspath(base_path)
+    norm_dirs = [f"{os.sep}{d}{os.sep}" for d in directories]
+    norm_asset_absolute_path = os.path.abspath(asset.absolute_path)
+    norm_root = norm_asset_absolute_path.replace(norm_base_path, "")
+
+    if not norm_asset_absolute_path.startswith(norm_base_path):
+        return False
+    return any([d in norm_root for d in norm_dirs])

idmtools/utils/general.py

@@ -0,0 +1,286 @@
+"""
+Tag Parsing and Normalization Utilities.
+
+This module provides helper functions and classes for converting string-based metadata tags
+into native Python types (e.g., bool, int, float) to enable accurate filtering and comparison.
+
+It includes:
+- JSON serialization/deserialization of tags
+- Conversion of sets to lists for JSON compatibility
+- Coercion logic via TagValue for safe and flexible comparisons
+
+Typical use cases include tag-based filtering of simulations or experiments during analysis.
+
+Typical Use Case:
+-----------------
+Used during filtering of simulations or experiments in analysis pipelines where user-defined
+tags are compared with numeric thresholds or exact matches.
+
+Example:
+--------
+    sim.tags={"Run_Number": lambda v: 4 <= v <= 10, "Coverage": "0.8"}
+    parsed = parse_value_tags(sim.tags, wrap_with_tagvalue=True)
+    assert parsed["Run_Number"] >=4 and <=10
+    assert parsed["Coverage"] == 0.8
+
+Copyright 2025, Gates Foundation. All rights reserved.
+"""
+import json
+from typing import Dict
+
+from idmtools.core.interfaces.ientity import IEntity
+
+
+def parse_item_tags(item: IEntity):
+    """
+    Normalize and update an entity's tags in place.
+
+    This function parses the given item's tags using `parse_value_tags` and updates
+    the `item.tags` dictionary with the normalized values.
+
+    Args:
+        item: An entity object that contains a `.tags` dictionary.
+
+    Returns:
+        The same item, with its `.tags` dictionary updated in-place.
+    """
+    if item.tags is None:
+        return item
+    else:
+        item.tags = parse_value_tags(item.tags)
+        return item
+
+
+def parse_value_tags(tags: Dict, wrap_with_tagvalue: bool = False) -> Dict[str, any]:
+    """
+    Parse and normalize a tag dictionary into native Python types.
+
+    Converts string tag values such as:
+        - "true"/"false" → bool
+        - "5" → int
+        - "0.8" → float
+        - sets → lists
+
+    Optionally wraps values using `TagValue` for comparison safety.
+
+    Args:
+        tags (dict): The dictionary of raw tag values to parse.
+        wrap_with_tagvalue (bool): If True, wraps each value in a TagValue object
+                                   for smart comparison support.
+
+    Returns:
+        dict: A dictionary with normalized or wrapped tag values.
+    """
+    # convert tags value as set to list first
+    tags = {k: list(v) if isinstance(v, set) else v for k, v in tags.items()}
+    tags_json = json.dumps(tags, cls=SetEncoder)
+    # Then: parse back into a dict with properly typed values
+    converted_tags = json.loads(tags_json, cls=CustomDecoder)
+    # Update result.tags in-place
+    for k, v in converted_tags.items():
+        converted = v
+        if wrap_with_tagvalue:
+            converted = TagValue(v)
+        tags[k] = converted
+    return tags
+
+
+class SetEncoder(json.JSONEncoder):
+    """
+    Custom JSON encoder that converts Python sets into lists to ensure compatibility with JSON serialization.
+    """
+
+    def default(self, obj):
+        """
+        Override default encoding behavior.
+
+        Args:
+            obj: The object being encoded.
+
+        Returns:
+            JSON-compatible representation of the object.
+        """
+        if isinstance(obj, set):
+            return list(obj)
+        return super().default(obj)
+
+
+class CustomDecoder(json.JSONDecoder):
+    """
+    Custom JSON decoder that converts string values into appropriate Python types (bool, int, float, None).
+    """
+
+    def __init__(self, *args, **kwargs):  # noqa D415
+        # Optionally override object_hook here
+        super().__init__(object_hook=self.custom_object_hook, *args, **kwargs)
+
+    @staticmethod
+    def denormalize_tag_value(val):
+        """
+        Convert a raw string tag value into a Python-native type.
+
+        Supported conversions:
+        - "true"/"false" → bool
+        - "null"/"none" → None
+        - "5", "0.8" → int, float
+
+        Args:
+            val (Any): The value to convert.
+
+        Returns:
+            The normalized Python value.
+        """
+        if not isinstance(val, str):
+            return val  # Already a native type
+
+        val_lower = val.strip().lower()
+        if val_lower == "true":
+            return True
+        elif val_lower == "false":
+            return False
+        elif val_lower in ("null", "none"):
+            return None
+
+        # Try integer/float
+        try:
+            if "." in val:
+                return float(val)
+            return int(val)
+        except ValueError:
+            pass
+
+        # Return as-is if it's not a simple number/bool/null
+        return val
+
+    def custom_object_hook(self, obj: Dict):
+        """
+        Apply normalization to each value in a decoded JSON object.
+
+        Args:
+            obj (dict): Dictionary of tag values.
+
+        Returns:
+            dict: Normalized tag dictionary.
+        """
+        # Modify dicts as they're loaded
+        new_obj = {}
+        for k, v in obj.items():
+            new_obj[k] = self.denormalize_tag_value(v)
+        return new_obj
+
+
+class TagValue:
+    """
+    Wrapper for a tag value that supports smart comparisons.
+
+    Automatically converts strings like "5" or "0.8" to int/float
+    and enables comparison against numbers or other tag values.
+
+    Useful when users perform tag-based filtering with operators
+    like >, <, ==, etc., and tag values may be strings.
+
+    Attributes:
+        raw (Any): The original tag value before coercion.
+    """
+
+    def __init__(self, raw):  # noqa  D107
+        self.raw = raw
+
+    def _coerce(self, other):
+        """
+        Convert both self.raw and other to comparable types using the same logic as CustomDecoder.
+
+        Args:
+            other (Any): Value to compare to.
+
+        Returns:
+            Tuple[Any, Any]: Coerced values ready for comparison.
+        """
+        convert = CustomDecoder.denormalize_tag_value
+        return convert(self.raw), convert(other)
+
+    def __eq__(self, other): return self._coerce(other)[0] == self._coerce(other)[1]  # noqa E704
+
+    def __lt__(self, other): return self._coerce(other)[0] < self._coerce(other)[1]  # noqa E704
+
+    def __le__(self, other): return self._coerce(other)[0] <= self._coerce(other)[1]  # noqa E704
+
+    def __gt__(self, other): return self._coerce(other)[0] > self._coerce(other)[1]  # noqa E704
+
+    def __ge__(self, other): return self._coerce(other)[0] >= self._coerce(other)[1]  # noqa E704
+
+    def __repr__(self): return repr(self.raw)  # noqa E704
+
+    def __str__(self): return str(self.raw)  # noqa E704
+
+
+class FilterSafeItem:
+    """
+    A lightweight wrapper around an entity (e.g., Simulation or Experiment) to enable safe tag-based filtering during multiprocessing operations (e.g., within analyzers).
+
+    This wrapper:
+    - Normalizes and wraps tag values via `TagValue` to support flexible comparisons (e.g., >, ==).
+    - Supports safe pickling/unpickling by stripping unpickleable fields like `_platform`.
+    - Delegates attribute access to the original wrapped item via `__getattr__` (if implemented).
+
+    Typical usage:
+        safe_item = FilterSafeItem(simulation)
+        tags = safe_item.tags
+        # Now tags["Run_Number"] supports TagValue comparison logic
+
+    Attributes:
+        _item (IEntity): The original entity being wrapped.
+    """
+
+    def __init__(self, item):
+        """
+        Initialize the filter-safe wrapper with a given entity.
+
+        Args:
+            item (IEntity): The original simulation or experiment to wrap.
+        """
+        self._item = item
+
+    def __getstate__(self):
+        """
+        Prepare the object for pickling by removing unpickleable platform references.
+
+        Returns:
+            dict: The serializable state dictionary.
+        """
+        state = self.__dict__.copy()
+        for key in ['_platform', '_platform_object', '_parent']:
+            state.pop(key, None)
+        return state
+
+    def __getattr__(self, attr):
+        """
+        Delegate access to attributes not defined on the wrapper to the wrapped item.
+
+        This makes FilterSafeItem behave like the original simulation object for all
+        standard attributes (e.g., `id`, `status`, `experiment_id`, etc.).
+        This function makes simulation._item.id = simulation.id the same in filter function.
+
+        Returns:
+            The attribute value from the wrapped entity.
+        """
+        return getattr(self._item, attr)
+
+    def __setstate__(self, state):
+        """
+        Restore object state after unpickling.
+
+        Args:
+            state (dict): The state dictionary.
+        """
+        self.__dict__.update(state)
+
+    @property
+    def tags(self):
+        """
+        Get normalized tags from the wrapped item, with each value wrapped in TagValue for type-safe comparison.
+
+        Returns:
+            Dict[str, any]: Dictionary of tags.
+        """
+        return parse_value_tags(self._item.tags)