idmtools-test 0.0.0.dev0__py3-none-any.whl → 0.0.3__py3-none-any.whl

idmtools_test/utils/execute_operations/simulate_operations.py

@@ -0,0 +1,368 @@
+import hashlib
+import json
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+from dataclasses import dataclass
+from functools import partial
+from logging import getLogger
+from pathlib import Path
+from threading import Lock
+from typing import List, Dict, Any, Type, TYPE_CHECKING, Optional
+from idmtools.assets import Asset, AssetCollection
+from idmtools.core import EntityStatus
+from idmtools.core.task_factory import TaskFactory
+from idmtools.entities import CommandLine
+from idmtools.entities.command_task import CommandTask
+from idmtools.entities.experiment import Experiment
+from idmtools.entities.iplatform_ops.iplatform_simulation_operations import IPlatformSimulationOperations
+from idmtools.entities.simulation import Simulation
+from idmtools.utils.file import file_content_to_generator
+from idmtools.utils.json import IDMJSONEncoder
+
+if TYPE_CHECKING:  # pragma: no cover
+    from idmtools_test.utils.test_execute_platform import TestExecutePlatform
+
+current_directory = os.path.dirname(os.path.realpath(__file__))
+data_path = os.path.abspath(os.path.join(current_directory, "..", "..", "data"))
+
+logger = getLogger(__name__)
+SIMULATION_LOCK = Lock()
+
+
+class SimulationDict(dict):
+    pass
+
+
+def run_simulation(simulation_id: Simulation, command: str, parent_uid: str, execute_directory, shell: bool = False):
+    simulation_path = os.path.join(execute_directory, str(parent_uid), str(simulation_id))
+    os.makedirs(simulation_path, exist_ok=True)
+    with open(os.path.join(simulation_path, "StdOut.txt"), "w") as out, \
+            open(os.path.join(simulation_path, "StdErr.txt"), "w") as err:
+        try:
+            cmd = str(command)
+            print(cmd)
+            print(execute_directory)
+            if cmd.startswith(execute_directory):
+                cmd = cmd.replace(execute_directory, "")
+            logger.info('Executing %s from working directory %s', cmd, simulation_path)
+            err.write(f"{cmd}\n")
+
+            # Run our task
+            if sys.platform in ['win32', 'cygwin']:
+                cmd = shlex.split(cmd.replace("\\", "/"))
+                if os.path.exists(os.path.join(simulation_path, cmd[0])):
+                    cmd[0] = os.path.join(simulation_path, cmd[0])
+                else:
+                    cmd[0] = os.path.abspath(cmd[0])
+                cmd = subprocess.list2cmdline(cmd)
+            else:
+                cmd = shlex.split(cmd.replace("\\", "/"))
+                try:
+                    if not os.access(cmd[0], os.X_OK):
+                        os.chmod(cmd[0], 0o777)
+                except:
+                    pass
+            logger.info(cmd)
+            if cmd[0].endswith(".sh"):
+                cmd.insert(0, "/bin/bash")
+            p = subprocess.Popen(
+                cmd,
+                cwd=simulation_path,
+                env=os.environ,
+                shell=shell,
+                stdout=out,
+                stderr=err
+            )
+            # store the pid in case we want to cancel later
+            logger.info(f"Process id: {p.pid}")
+            # Log that we have started this particular simulation
+            p.wait()
+            if p.returncode == 0:
+                status = EntityStatus.SUCCEEDED
+            else:
+                status = EntityStatus.FAILED
+        except Exception as e:
+            logger.exception(e)
+            err.write(str(e))
+            status = EntityStatus.FAILED
+    return parent_uid, simulation_id, status
+
+
+@dataclass
+class TestExecutePlatformSimulationOperation(IPlatformSimulationOperations):
+    platform: 'TestExecutePlatform'
+    platform_type: Type = SimulationDict
+
+    def get(self, simulation_id: str, experiment_id: str = None, **kwargs) -> Any:
+        if simulation_id and experiment_id:
+            exp_path = os.path.join(self.platform.execute_directory, str(experiment_id))
+            sim_path = os.path.join(exp_path, str(simulation_id))
+            metadata_src = os.path.join(sim_path, "simulation_metadata.json")
+            if not os.path.exists(metadata_src):
+                logger.error("Cannot find the simulation at {metadata}")
+                raise ValueError(f"Cannot find the simulation at {metadata_src}")
+            with open(metadata_src, 'r') as metadata_in:
+                metadata = json.load(metadata_in)
+                return SimulationDict(metadata)
+
+    def platform_create(self, simulation: Simulation, **kwargs) -> Simulation:
+        simulation.platform = self
+        experiment_id = simulation.parent_id
+        self.save_metadata(simulation)
+        return simulation
+
+    def batch_create(self, sims: List[Simulation], **kwargs) -> List[Simulation]:
+
+        simulations = []
+        experiment_id = None
+        for simulation in sims:
+            if simulation.status is None:
+                self.pre_create(simulation, **kwargs)
+                experiment_id = simulation.parent_id
+                self.save_metadata(simulation)
+                self.post_create(simulation, **kwargs)
+                simulations.append(simulation)
+            else:
+                simulations.append(simulation)
+        return simulations
+
+    def save_metadata(self, simulation: Simulation, update_data: dict = None):
+        exp_path = os.path.join(self.platform.execute_directory, str(simulation.parent_id))
+        sim_path = os.path.join(exp_path, str(simulation.id))
+        metadata_file = os.path.join(sim_path, "simulation_metadata.json")
+        os.makedirs(sim_path, exist_ok=True)
+        if update_data and os.path.exists(metadata_file):
+            with open(metadata_file, 'r') as metadata_src:
+                metadata = json.load(metadata_src)
+                metadata.update(update_data)
+        else:
+            metadata = simulation.to_dict()
+
+        with open(metadata_file, 'w') as out:
+            out.write(json.dumps(metadata, cls=IDMJSONEncoder))
+
+    def get_parent(self, simulation: Any, **kwargs) -> Any:
+        return self.platform._experiments.experiments.get(simulation.parent_id)
+
+    def platform_run_item(self, simulation: Simulation, **kwargs):
+        """
+        Run the item on the test platform
+
+        This method executes the simulation in a thread pool
+
+        Args:
+            simulation: Simulation to run
+            **kwargs:
+
+        Returns:
+
+        """
+        logger.info(f"Running simulation {simulation.id}")
+        self.send_assets(simulation, **kwargs)
+        future = self.platform.pool.submit(
+            run_simulation,
+            simulation.id,
+            simulation.task.command,
+            simulation.parent.uid,
+            self.platform.execute_directory
+        )
+        self.platform.queue.append(future)
+
+    def send_assets(self, simulation: Any, **kwargs):
+        """
+        Send assets to the test platform. This method uses the execute directory and first tries to link files
+        If that fails, files are copied into the directory
+
+        Args:
+            simulation: Simulation assets to send
+            **kwargs:
+
+        Returns:
+
+        """
+        exp_path = self.platform._experiments.get_experiment_path(simulation.parent.uid)
+        sim_path = self.get_simulation_asset_path(simulation, experiment_path=exp_path)
+        logger.info(f"Creating {exp_path}")
+        os.makedirs(sim_path, exist_ok=True)
+        self.__copy_simulation_assets_to_simulation_directory(sim_path, simulation)
+        exp_path = os.path.join(exp_path, "Assets")
+        self._copy_or_link_parent_assets(exp_path, sim_path, simulation)
+
+    def get_simulation_asset_path(self, simulation: Simulation, experiment_path: str = None) -> str:
+        """
+        Get path to simulation assets
+
+        Args:
+            simulation: Simulation Assets to get path to
+
+        Returns:
+            Str path to assets
+        """
+        if experiment_path is None:
+            experiment_path = self.platform._experiments.get_experiment_path(simulation.parent.uid)
+        return os.path.join(experiment_path, str(simulation.id))
+
+    @staticmethod
+    def _copy_or_link_parent_assets(exp_path: str, sim_path: str, simulation: Simulation):
+        """
+        Link or Copy a simulation parent assets into its directory
+
+        Args:
+            exp_path: Path to experiment assets
+            sim_path: Simulation path
+            simulation: Simulation
+
+        Returns:
+
+        """
+        for asset in simulation.parent.assets:
+            remote_path = Path(exp_path).joinpath(asset.relative_path) if asset.relative_path else Path(exp_path)
+            sim_assets = Path(sim_path).joinpath("Assets")
+            src_path = remote_path.joinpath(asset.filename)
+            dest_path = sim_assets.joinpath(asset.filename)
+            sim_assets.mkdir(parents=True, exist_ok=True)
+            if sys.platform in ['win32']:
+                import win32file
+                link_worked = True
+                try:
+                    logger.info("Trying to link the files")
+                    if link_worked:
+                        win32file.CreateSymbolicLink(src_path, dest_path, 1)
+                    else:
+                        shutil.copy(src_path, dest_path)
+                except Exception:
+                    link_worked = False
+                    logger.info("Linking failed. Copying instread")
+                    shutil.copy(src_path, dest_path)
+            else:
+                os.symlink(src_path, dest_path)
+
+    @staticmethod
+    def __copy_simulation_assets_to_simulation_directory(sim_path:str, simulation: Simulation):
+        for asset in simulation.assets:
+            remote_path = os.path.join(sim_path, asset.relative_path) if asset.relative_path else sim_path
+            remote_path = os.path.join(remote_path, asset.filename)
+            if asset.absolute_path:
+                logger.info(f"Copying {asset.absolute_path} to {remote_path}")
+                shutil.copy(asset.absolute_path, remote_path)
+            else:
+                logger.info(f"Writing {asset.absolute_path} to {remote_path}")
+                with open(os.path.join(remote_path), 'wb') as out:
+                    out.write(asset.content.encode())
+
+    def refresh_status(self, simulation: Simulation, **kwargs):
+        pass
+
+    def get_assets(self, simulation: Simulation, files: List[str], **kwargs) -> Dict[str, bytearray]:
+        """
+        Get list of files
+        Args:
+            simulation:
+            files:
+            **kwargs:
+
+        Returns:
+
+        """
+        logger.info(f'Listing assets for {simulation.id}')
+        assets = {}
+        for root, dirs, actual_files in os.walk(self.get_simulation_asset_path(simulation)):
+            for file in actual_files:
+                if file in files:
+                    fp = os.path.abspath(os.path.join(root, file))
+                    with open(fp, 'rb') as i:
+                        assets[file] = i.read()
+        return assets
+
+    def list_assets(self, simulation: Simulation, **kwargs) -> List[Asset]:
+        """
+        List assets for an item
+
+        Args:
+            simulation: Simulation to list assets for
+            **kwargs:
+
+        Returns:
+
+        """
+        logger.info(f'Listing assets for {simulation.id}')
+        assets = []
+        for root, dirs, files in os.walk(self.get_simulation_asset_path(simulation)):
+            for file in files:
+                fp = os.path.abspath(os.path.join(root, file))
+                asset = Asset(absolute_path=fp, filename=file)
+                asset.download_generator_hook = partial(file_content_to_generator, fp)
+                cksum_hash = hashlib.md5()
+                with open(fp, 'rb') as fin:
+                    cksum_hash.update(fin.read())
+                    asset.checksum = cksum_hash.hexdigest()
+                assets.append(asset)
+        return assets
+
+    def to_entity(self, dict_sim: Dict, load_task: bool = False, parent: Optional[Experiment] = None,
+                  **kwargs) -> Simulation:
+        # convert the dictionary to simulation
+        sim: Simulation = Simulation(**{k: v for k, v in dict_sim.items() if k not in ['platform_id', 'item_type']})
+        try:
+            # load status from str
+            sim.status = EntityStatus[sim.status.upper()]
+        except:
+            pass
+        # set platform before we load assets
+        sim.platform = self.platform
+        # and our parent
+        if parent:
+            sim.parent = parent
+        # get path to our assets
+        sim_path = self.get_simulation_asset_path(sim)
+        # set task to a blank slate
+        sim.task = None
+        # load assets first
+        if dict_sim['assets']:
+            # load the assets
+            ac = AssetCollection()
+            for dict_asset in dict_sim['assets']:
+                if dict_asset['absolute_path'] is None:
+                    if dict_asset['relative_path']:
+                        dict_asset['absolute_path'] = os.path.join(sim_path, dict_asset['relative_path'], dict_asset['filename'])
+                    else:
+                        dict_asset['absolute_path'] = os.path.join(sim_path, dict_asset['filename'])
+                asset = Asset(**dict_asset)
+                asset.persisted = True
+                asset.download_generator_hook = partial(file_content_to_generator, asset.absolute_path)
+                ac.add_asset(asset)
+            sim.assets = ac
+        # should we fully load the task?
+        if load_task:
+            if parent.task_type:
+                try:
+                    sim.task = TaskFactory().create(parent.task_type, **dict_sim['task'])
+                except Exception as e:
+                    logger.exception(e)
+
+            cli = self._detect_command_line_from_simulation(dict_sim)
+            # if we could not find task, set it now, otherwise rebuild the cli
+            if sim.task is None:
+                sim.task = CommandTask(CommandLine.from_string(cli))
+            else:
+                sim.task.command = CommandLine.from_string(cli)
+            # call task load options(load configs from files, etc)
+            sim.task.reload_from_simulation(sim)
+        else:
+            cli = self._detect_command_line_from_simulation(dict_sim)
+            sim.task = CommandTask(cli)
+
+        # load assets
+        return sim
+
+    def _detect_command_line_from_simulation(self, dict_sim):
+        if 'task' in dict_sim and 'command' in dict_sim['task']:
+            return dict_sim['task']['command']
+        return ''
+
+
+
+

idmtools_test/utils/itest_with_persistence.py

@@ -0,0 +1,25 @@
+import os
+import shutil
+import unittest
+
+from idmtools.services.ipersistance_service import IPersistenceService
+
+
+class ITestWithPersistence(unittest.TestCase):
+    current_directory = os.path.dirname(os.path.realpath(__file__))
+
+    def setUp(self) -> None:
+        self.data_dir = os.path.join(self.current_directory, "data")
+        try:
+            os.makedirs(self.data_dir, exist_ok=True)
+        except FileExistsError:
+            # Very Very Very rarely, high thread counts, we hit this error
+            pass
+
+        IPersistenceService.cache_directory = self.data_dir
+
+    def tearDown(self) -> None:
+        try:
+            shutil.rmtree(self.data_dir)
+        except Exception:
+            pass

idmtools_test/utils/operations/experiment_operations.py

@@ -0,0 +1,64 @@
+import os
+from dataclasses import field, dataclass
+from logging import getLogger, DEBUG
+from threading import Lock
+from typing import List, Any, Type, Dict, Union, TYPE_CHECKING
+from idmtools.core import EntityStatus, UnknownItemException
+from idmtools.entities.experiment import Experiment
+from idmtools.entities.iplatform_ops.iplatform_experiment_operations import IPlatformExperimentOperations
+if TYPE_CHECKING:  # pragma: no cover
+    from idmtools_test.utils.test_platform import TestPlatform
+
+logger = getLogger(__name__)
+current_directory = os.path.dirname(os.path.realpath(__file__))
+data_path = os.path.abspath(os.path.join(current_directory, "..", "..", "data"))
+EXPERIMENTS_LOCK = Lock()
+
+
+@dataclass
+class TestPlatformExperimentOperation(IPlatformExperimentOperations):
+    platform: 'TestPlatform'
+    platform_type: Type = Experiment
+    experiments: Dict[str, Experiment] = field(default_factory=dict, compare=False, metadata={"pickle_ignore": True})
+
+    def get(self, experiment_id: str, **kwargs) -> Experiment:
+        e = self.experiments.get(experiment_id)
+        if e is None:
+            raise UnknownItemException(f"Cannot find the experiment with the ID of: {experiment_id}")
+        e.platform = self.platform
+        return e
+
+    def platform_create(self, experiment: Experiment, **kwargs) -> Experiment:
+        if logger.isEnabledFor(DEBUG):
+            logger.debug('Creating Experiment')
+        EXPERIMENTS_LOCK.acquire()
+        self.experiments[experiment.uid] = experiment
+        EXPERIMENTS_LOCK.release()
+        self.platform._simulations._save_simulations_to_cache(experiment.uid, list(), overwrite=True)
+        logger.debug(f"Created Experiment {experiment.uid}")
+        return experiment
+
+    def get_children(self, experiment: Experiment, **kwargs) -> List[Any]:
+        return self.platform._simulations.simulations.get(experiment.uid)
+
+    def get_parent(self, experiment: Any, **kwargs) -> Any:
+        return None
+
+    def platform_run_item(self, experiment: Experiment, **kwargs):
+        self.platform._simulations.set_simulation_status(experiment.uid, EntityStatus.RUNNING)
+
+    def send_assets(self, experiment: Any, **kwargs):
+        pass
+
+    def refresh_status(self, experiment: Experiment, **kwargs):
+        if logger.isEnabledFor(DEBUG):
+            logger.debug(f'Refreshing status for Experiment: {experiment.uid}')
+        for simulation in self.platform._simulations.simulations.get(experiment.uid):
+            for esim in experiment.simulations:
+                if esim == simulation:
+                    logger.debug(f'Setting {simulation.uid} Status to {simulation.status}')
+                    esim.status = simulation.status
+                    break
+
+    def list_assets(self, experiment: Experiment, **kwargs) -> List[str]:
+        pass

idmtools_test/utils/operations/simulation_operations.py

@@ -0,0 +1,114 @@
+import os
+from dataclasses import dataclass, field
+from logging import getLogger, DEBUG
+from threading import Lock
+from typing import List, Dict, Any, Type, TYPE_CHECKING
+from uuid import uuid4
+import numpy as np
+from idmtools.assets import Asset
+from idmtools.entities.iplatform_ops.iplatform_simulation_operations import IPlatformSimulationOperations
+from idmtools.entities.simulation import Simulation
+if TYPE_CHECKING:  # pragma: no cover
+    from idmtools_test.utils.test_platform import TestPlatform
+current_directory = os.path.dirname(os.path.realpath(__file__))
+data_path = os.path.abspath(os.path.join(current_directory, "..", "..", "data"))
+
+logger = getLogger(__name__)
+SIMULATION_LOCK = Lock()
+
+
+@dataclass
+class TestPlatformSimulationOperation(IPlatformSimulationOperations):
+    platform: 'TestPlatform'
+
+    def all_files(self, simulation: Simulation, **kwargs):
+        pass
+
+    platform_type: Type = Simulation
+    simulations: dict = field(default_factory=dict, compare=False, metadata={"pickle_ignore": True})
+
+    def get(self, simulation_id: str, **kwargs) -> Any:
+        obj = None
+        for eid in self.simulations:
+            sims = self.simulations.get(eid)
+            if sims:
+                for sim in self.simulations.get(eid):
+                    if sim.uid == simulation_id:
+                        obj = sim
+                        break
+            if obj:
+                break
+        obj.platform = self.platform
+        return obj
+
+    def platform_create(self, simulation: Simulation, **kwargs) -> Simulation:
+        experiment_id = simulation.parent_id
+        simulation.uid = str(uuid4())
+
+        self._save_simulations_to_cache(experiment_id, [simulation])
+        return simulation
+
+    def _save_simulations_to_cache(self, experiment_id, simulations: List[Simulation], overwrite: bool = False):
+        if logger.isEnabledFor(DEBUG):
+            logger.debug(f'Saving {len(simulations)} to Experiment {experiment_id}')
+        SIMULATION_LOCK.acquire()
+        existing_simulations = [] if overwrite else self.simulations.pop(experiment_id)
+        self.simulations[experiment_id] = existing_simulations + simulations
+        SIMULATION_LOCK.release()
+        logger.debug(f'Saved sims')
+
+    def batch_create(self, sims: List[Simulation], **kwargs) -> List[Simulation]:
+        simulations = []
+        experiment_id = None
+        for simulation in sims:
+            if simulation.status is None:
+                self.pre_create(simulation)
+                experiment_id = simulation.parent_id
+                self.post_create(simulation)
+                simulations.append(simulation)
+
+        if experiment_id:
+            self._save_simulations_to_cache(experiment_id, simulations)
+        return simulations
+
+    def get_parent(self, simulation: Any, **kwargs) -> Any:
+        return self.platform._experiments.experiments.get(simulation.parent_id)
+
+    def platform_run_item(self, simulation: Simulation, **kwargs):
+        pass
+
+    def send_assets(self, simulation: Any, **kwargs):
+        pass
+
+    def refresh_status(self, simulation: Simulation, **kwargs):
+        pass
+
+    def get_assets(self, simulation: Simulation, files: List[str], **kwargs) -> Dict[str, bytearray]:
+        return {}
+
+    def list_assets(self, simulation: Simulation, **kwargs) -> List[Asset]:
+        pass
+
+    def set_simulation_status(self, experiment_uid, status):
+        self.set_simulation_prob_status(experiment_uid, {status: 1})
+
+    def set_simulation_prob_status(self, experiment_uid, status):
+        if logger.isEnabledFor(DEBUG):
+            logger.debug(f'Setting status for sim s on exp {experiment_uid} to {status}')
+        simulations = self.simulations.get(experiment_uid)
+        for simulation in simulations:
+            new_status = np.random.choice(
+                a=list(status.keys()),
+                p=list(status.values())
+            )
+            simulation.status = new_status
+        self._save_simulations_to_cache(experiment_uid, simulations, True)
+
+    def set_simulation_num_status(self, experiment_uid, status, number):
+        simulations = self.simulations.get(experiment_uid)
+        for simulation in simulations:
+            simulation.status = status
+            number -= 1
+            if number <= 0:
+                break
+        self._save_simulations_to_cache(experiment_uid, simulations, True)

idmtools_test/utils/shared_functions.py

@@ -0,0 +1,25 @@
+from operator import itemgetter
+from COMPS.Data import Experiment as COMPSExperiment, QueryCriteria
+
+
+def validate_output(self, exp_id, expected_sim_count):
+    sim_count = 0
+    for simulation in COMPSExperiment.get(exp_id).get_simulations():
+        sim_count = sim_count + 1
+        result_file_string = simulation.retrieve_output_files(paths=['output/result.json'])
+        print(result_file_string)
+        config_string = simulation.retrieve_output_files(paths=['config.json'])
+        print(config_string)
+        self.assertEqual(result_file_string, config_string)
+
+    self.assertEqual(sim_count, expected_sim_count)
+
+
+def validate_sim_tags(self, exp_id, expected_tags):
+    tags = []
+    for simulation in COMPSExperiment.get(exp_id).get_simulations():
+        tags.append(simulation.get(simulation.id, QueryCriteria().select_children('tags')).tags)
+
+    sorted_tags = sorted(tags, key=itemgetter('a'))
+    sorted_expected_tags = sorted(expected_tags, key=itemgetter('a'))
+    self.assertEqual(sorted_tags, sorted_expected_tags)

idmtools_test/utils/test_asset.py

@@ -0,0 +1,89 @@
+import os
+from typing import TypeVar, Union, List, Callable, Any
+from idmtools.frozen.ifrozen import IFrozen
+
+
+class Asset(IFrozen):
+    """
+    A class representing an asset. An asset can either be related to a physical 
+    asset present on the computer or directly specified by a filename and content.
+    """
+
+    def __init__(self, absolute_path: 'str' = None, relative_path: 'str' = None, filename: 'str' = None,
+                 content: 'Any' = None, handler: 'Callable' = str, checksum: str = None):
+        """
+        A constructor.
+
+        Args:
+            absolute_path: The absolute path of the asset. Optional if **filename** and **content** are given.
+            relative_path:  The relative path (compared to the simulation root folder).
+            filename: Name of the file. Optional if **absolute_path** is given.
+            content: The content of the file. Optional if **absolute_path** is given.
+        """
+
+        super().__init__()
+        if not absolute_path and (not filename and not content):
+            raise ValueError("Impossible to create the asset without either absolute path or filename and content!")
+
+        self.absolute_path = absolute_path
+        self.relative_path = relative_path
+        self.filename = filename or os.path.basename(self.absolute_path)
+        self._content = content
+        self.persisted = False
+        self.handler = handler
+        self.checksum = checksum
+
+    def __repr__(self):
+        return f"<Asset: {os.path.join(self.relative_path, self.filename)} from {self.absolute_path}>"
+
+    @property
+    def relative_path(self):
+        return self._relative_path or ""
+
+    @relative_path.setter
+    def relative_path(self, relative_path):
+        self._relative_path = relative_path.strip(" \\/") if relative_path else None
+
+    @property
+    def bytes(self):
+        if isinstance(self.content, bytes):
+            return self.content
+        return str.encode(self.handler(self.content))
+
+    @property
+    def content(self):
+        """
+        Returns: 
+            The content of the file, either from the content attribute or by opening the absolute path.
+        """
+        if not self._content:
+            with open(self.absolute_path, "rb") as fp:
+                self._content = fp.read()
+
+        return self._content
+
+    # region Equality and Hashing
+    def __eq__(self, other):
+        return self.__key() == other.__key()
+
+    def __key(self):
+        if self.absolute_path:
+            return self.absolute_path
+
+        if self.filename and self.relative_path:
+            return self.filename, self.relative_path
+
+        return self._content, self.filename
+
+    def __hash__(self):
+        return hash(self.__key())
+    # endregion
+
+
+TAsset = TypeVar("TAsset", bound=Asset)
+# Assets types
+TAssetList = List[TAsset]
+
+# Filters types
+TAssetFilter = Union[Callable[[TAsset], bool], Callable]
+TAssetFilterList = List[TAssetFilter]