idmtools-platform-general 0.0.0.dev0__py3-none-any.whl → 0.0.2__py3-none-any.whl

idmtools_platform_file/__init__.py

@@ -0,0 +1,18 @@
+"""
+idmtools file platform.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+# flake8: noqa F821
+try:
+    from importlib.metadata import version, PackageNotFoundError
+except ImportError:
+    # Python < 3.8
+    from importlib_metadata import version, PackageNotFoundError
+
+try:
+    __version__ = version("idmtools-platform-files")  # Use your actual package name
+except PackageNotFoundError:
+    # Package not installed, use fallback
+    __version__ = "0.0.0+unknown"
+

idmtools_platform_file/assets/__init__.py

@@ -0,0 +1,77 @@
+"""
+Utility functions to generate batch scripts.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from pathlib import Path
+from jinja2 import Template
+from typing import TYPE_CHECKING, Optional
+from idmtools.entities.experiment import Experiment
+from idmtools.entities.simulation import Simulation
+
+if TYPE_CHECKING:
+    from idmtools_platform_file.file_platform import FilePlatform
+
+DEFAULT_TEMPLATE_FILE = Path(__file__).parent.joinpath("batch.sh.jinja2")
+DEFAULT_SIMULATION_TEMPLATE = Path(__file__).parent.parent.joinpath("assets/_run.sh.jinja2")
+
+
+def generate_script(platform: 'FilePlatform', experiment: Experiment, max_job: int = None, run_sequence: bool = None,
+                    **kwargs) -> None:
+    """
+    Generate batch file batch.sh.
+    Args:
+        platform: File Platform
+        experiment: idmtools Experiment
+        max_job: int
+        run_sequence: bool
+        kwargs: keyword arguments used to expand functionality
+    Returns:
+        None
+    """
+    output_target = platform.get_directory(experiment).joinpath("batch.sh")
+    with open(output_target, "w") as tout:
+        with open(DEFAULT_TEMPLATE_FILE) as tin:
+            t = Template(tin.read())
+            tvars = dict(
+                platform=platform,
+                max_job=max_job if max_job is not None else platform.max_job,
+                run_sequence=run_sequence if run_sequence is not None else platform.run_sequence
+            )
+            if platform.modules:
+                tvars['modules'] = platform.modules
+            if platform.extra_packages:
+                tvars['packages'] = platform.extra_packages
+            tout.write(t.render(tvars))
+
+    # Make executable
+    platform.update_script_mode(output_target)
+
+
+def generate_simulation_script(platform: 'FilePlatform', simulation: Simulation, retries: Optional[int] = None,
+                               **kwargs) -> None:
+    """
+    Generate batch file _run.sh.
+    Args:
+        platform: File Platform
+        simulation: idmtools Simulation
+        retries: int
+        extra_packages: List of extra packages to install
+        kwargs: keyword arguments used to expand functionality
+    Returns:
+        None
+    """
+    sim_script = platform.get_directory(simulation).joinpath("_run.sh")
+    with open(sim_script, "w") as tout:
+        with open(DEFAULT_SIMULATION_TEMPLATE) as tin:
+            t = Template(tin.read())
+            tvars = dict(
+                platform=platform,
+                simulation=simulation,
+                retries=retries if retries else platform.retries,
+                ntasks=platform.ntasks
+            )
+            tout.write(t.render(tvars))
+
+    # Make executable
+    platform.update_script_mode(sim_script)

idmtools_platform_file/assets/_run.sh.jinja2

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# define the handler function
+term_handler()
+{
+    # do whatever cleanup you want here
+    echo "-1" > job_status.txt
+    exit -1
+}
+
+# Register cleanup function to handle SIGINT and SIGTERM signals
+trap 'term_handler' SIGINT SIGTERM
+
+n=0
+{% set mpi_command = "mpirun -n " + ntasks|string if ntasks > 1 else "" %}
+
+until [ "$n" -ge {{retries}} ]
+do
+    echo "100" > job_status.txt
+    {% if simulation.task.sif_path is defined and simulation.task.sif_path %}
+        {% if simulation.task.command.cmd.startswith('singularity') %}
+            {{ mpi_command }} {{simulation.task.command.cmd}} &
+        {% else %}
+           singularity exec {{simulation.task.sif_path}} {{ mpi_command }} {{simulation.task.command.cmd}} &
+        {% endif %}
+    {% else %}
+        exec -a "SIMULATION:{{simulation.id}}" {{ mpi_command }} {{simulation.task.command.cmd}} &
+    {% endif %}
+
+   child_pid=$!
+   echo "Running simulation with PID: $child_pid"
+   # Wait for the child process to complete
+   wait $child_pid
+
+   RESULT=$?
+   if [ $RESULT -eq 0 ]; then
+      echo "0" > job_status.txt
+      exit 0
+   elif [ $RESULT -eq 255 ] || [ $RESULT -eq -1 ]; then   # Normalize -1 or 255 to 1 to avoid process abort
+      echo "-1" > job_status.txt
+      echo "_run.sh exiting with code: $RESULT" >> exit_code.log
+      exit 1
+   fi
+   n=$((n+1))
+done
+echo "-1" > job_status.txt
+exit $RESULT

idmtools_platform_file/assets/batch.sh.jinja2

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+chmod +x run_simulation.sh
+
+{% if packages is defined and packages is not none and packages|length > 0 %}
+echo "Installing custom packages"
+
+{% for package in packages %}
+    echo "Installing {{ package }}" 1>> stdout.txt 2>> stderr.txt
+    pip install {{ package }} --extra-index-url=https://packages.idmod.org/api/pypi/pypi-production/simple --upgrade 1>> stdout.txt 2>> stderr.txt
+{% endfor %}
+{% endif %}
+
+{% if modules is defined and modules is not none and modules|length > 0 %}
+{% for m in modules %}
+module load {{m}}
+{% endfor %}
+{% endif %}
+
+{% if run_sequence is defined and run_sequence %}
+    find $(pwd) -maxdepth 2 -name "_run.sh" -print0 | xargs -0 -I% dirname % | xargs -d "\n" -I% bash -c 'cd $(pwd) && $(pwd)/run_simulation.sh %  1>> stdout.txt 2>> stderr.txt'
+{% else %}
+    find $(pwd) -maxdepth 2 -name "_run.sh" -print0 | xargs -0 -I% dirname % | xargs -d "\n" -P {{ max_job }} -I% bash -c 'cd $(pwd) && $(pwd)/run_simulation.sh %  1>> stdout.txt 2>> stderr.txt'
+{% endif %}

idmtools_platform_file/assets/run_simulation.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+JOB_DIRECTORY=$1
+echo "enter directory: '$JOB_DIRECTORY'"
+cd $JOB_DIRECTORY
+sed -i 's/\r//g' _run.sh
+chmod +x _run.sh
+bash _run.sh 1> stdout.txt 2> stderr.txt

idmtools_platform_file/cli/__init__.py

@@ -0,0 +1,5 @@
+"""
+idmtools file cli module.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""

idmtools_platform_file/cli/file.py

@@ -0,0 +1,185 @@
+"""
+idmtools FilePlatform CLI commands.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import json
+import click
+from idmtools.core import ItemType
+from idmtools.core.platform_factory import Platform
+from idmtools_platform_file.tools.status_report.status_report import generate_status_report
+from idmtools_platform_file.tools.status_report.utils import get_latest_experiment, check_status, clear_history
+from logging import getLogger
+
+user_logger = getLogger('user')
+
+
+@click.group(short_help="File platform related commands.")
+@click.argument('job-directory')
+@click.pass_context
+def file(ctx: click.Context, job_directory):
+    """
+    Commands related to managing the File Platform.
+
+    job_directory: FilePlatform Working Directory
+    """
+    ctx.obj = dict(job_directory=job_directory)
+
+
+@file.command(help="Get simulation's report")
+@click.option('--suite-id', default=None, help="Idmtools Suite id")
+@click.option('--exp-id', default=None, help="Idmtools Experiment id")
+@click.option('--status-filter', type=click.Choice(['0', '-1', '100']), multiple=True, help="list of status")
+@click.option('--sim-filter', multiple=True, help="list of simulations")
+@click.option('--verbose/--no-verbose', default=True, help="Enable verbose output in results")
+@click.option('--display/--no-display', default=True, help="Display with working directory or not")
+@click.option('--display-count', default=20, help="Display Count")
+@click.pass_context
+def status_report(ctx: click.Context, suite_id, exp_id, status_filter, sim_filter, verbose, display,
+                  display_count):
+    """
+    Build status report.
+    Args:
+        ctx: click.Context
+        suite_id: suite id
+        exp_id: experiment id
+        status_filter: status filter
+        sim_filter: simulation filter
+        verbose: bool True/False
+        display: bool True/False
+        display_count: how many to display
+    Returns:
+        None
+    """
+    job_dir = ctx.obj['job_directory']
+    platform = Platform('FILE', job_directory=job_dir)
+
+    if suite_id is not None:
+        scope = (suite_id, ItemType.SUITE)
+    elif exp_id is not None:
+        scope = (exp_id, ItemType.EXPERIMENT)
+    else:
+        scope = None
+
+    generate_status_report(platform=platform, scope=scope,
+                           status_filter=status_filter if len(status_filter) > 0 else None,
+                           sim_filter=sim_filter if len(sim_filter) > 0 else None,
+                           verbose=verbose, display=display, display_count=display_count)
+
+
+@file.command(help="Get the latest experiment info")
+@click.pass_context
+def get_latest(ctx: click.Context):
+    """
+    Get the latest experiment directory.
+    Args:
+        ctx: click.Context
+    Returns:
+        None
+    """
+    job_dir = ctx.obj['job_directory']
+    platform = Platform('FILE', job_directory=job_dir)
+
+    result = get_latest_experiment(platform)
+    user_logger.info(json.dumps(result, indent=3))
+
+
+@file.command(help="Get Suite/Experiment/Simulation directory")
+@click.option('--sim-id', default=None, help="Idmtools Simulation id")
+@click.option('--exp-id', default=None, help="Idmtools Experiment id")
+@click.option('--suite-id', default=None, help="Idmtools Suite id")
+@click.pass_context
+def get_path(ctx: click.Context, sim_id, exp_id, suite_id):
+    """
+    Get entity directory.
+    Args:
+        ctx: click.Context
+        sim_id: simulation id
+        exp_id: experiment id
+        suite_id: suite id
+    Returns:
+        None
+    """
+    job_dir = ctx.obj['job_directory']
+    platform = Platform('FILE', job_directory=job_dir)
+
+    if sim_id is not None:
+        item_dir = platform.get_directory_by_id(sim_id, ItemType.SIMULATION)
+    elif exp_id is not None:
+        item_dir = platform.get_directory_by_id(exp_id, ItemType.EXPERIMENT)
+    elif suite_id is not None:
+        item_dir = platform.get_directory_by_id(suite_id, ItemType.SUITE)
+    else:
+        raise Exception('Must provide at least one: suite-id, exp-id or sim-id!')
+
+    user_logger.info(item_dir)
+
+
+@file.command(help="Get status of Experiment/Simulation")
+@click.option('--sim-id', default=None, help="Idmtools Simulation id")
+@click.option('--exp-id', default=None, help="Idmtools Experiment id")
+@click.pass_context
+def get_status(ctx: click.Context, sim_id, exp_id):
+    """
+    Retrieve status.
+    Args:
+        ctx: click.Context
+        sim_id: simulation id
+        exp_id: experiment id
+    Returns:
+        None
+    """
+    job_dir = ctx.obj['job_directory']
+    platform = Platform('FILE', job_directory=job_dir)
+
+    if sim_id is not None:
+        status = platform.get_simulation_status(sim_id)
+    elif exp_id is not None:
+        exp = platform.get_item(exp_id, ItemType.EXPERIMENT)
+        status = exp.status
+    else:
+        raise Exception('Must provide at least one: exp-id or sim-id!')
+
+    user_logger.info(status.name if status else None)
+
+
+@file.command(help="Get simulation's status")
+@click.option('--exp-id', default=None, help="Idmtools Experiment id")
+@click.option('--display/--no-display', default=False, help="Display with working directory or not")
+@click.pass_context
+def status(ctx: click.Context, exp_id, display):
+    """
+    Get job status.
+    Args:
+        ctx: click.Context
+        exp_id: experiment id
+        display: bool True/False
+    Returns:
+        None
+    """
+    job_dir = ctx.obj['job_directory']
+    platform = Platform('FILE', job_directory=job_dir)
+
+    check_status(platform=platform, exp_id=exp_id, display=display)
+
+
+@file.command(help="Clear generated files/folders")
+@click.option('--exp-id', default=None, help="Idmtools Experiment id")
+@click.option('--sim-id', multiple=True, help="Idmtools Simulation id")
+@click.option('--remove', multiple=True, help="list of files/folders to be removed from simulation")
+@click.pass_context
+def clear_files(ctx: click.Context, exp_id, sim_id, remove):
+    """
+    Clear running history.
+    Args:
+        ctx: click.Context
+        exp_id: experiment id
+        sim_id: simulation id
+        remove: list of files/folders
+    Returns:
+        None
+    """
+    job_dir = ctx.obj['job_directory']
+    platform = Platform('FILE', job_directory=job_dir)
+
+    clear_history(platform=platform, exp_id=exp_id, sim_id=sim_id, remove_list=remove)

idmtools_platform_file/file_operations/__init__.py

@@ -0,0 +1,4 @@
+"""idmtools comps utils.
+
+Copyright 2025, Gates Foundation. All rights reserved.
+"""

idmtools_platform_file/file_operations/file_operations.py

@@ -0,0 +1,298 @@
+"""
+Here we implement the operations_interface.
+
+Copyright 2025, Gates Foundation. All rights reserved.
+"""
+import os
+import shlex
+import shutil
+from dataclasses import dataclass
+from logging import getLogger
+from pathlib import Path
+from typing import Union
+from idmtools.core import ItemType, EntityStatus
+from idmtools.entities import Suite
+from idmtools.entities.experiment import Experiment
+from idmtools.entities.simulation import Simulation
+from idmtools_platform_file.assets import generate_script, generate_simulation_script
+from idmtools_platform_file.file_operations.operations_interface import IOperations
+from idmtools_platform_file.platform_operations.utils import FILE_MAPS, validate_file_path_length, \
+    clean_item_name, validate_folder_files_path_length, FileExperiment, FileSimulation, FileSuite
+from idmtools.utils.decorators import check_symlink_capabilities, cache_directory
+
+logger = getLogger(__name__)
+user_logger = getLogger('user')
+
+
+@dataclass
+class FileOperations(IOperations):
+    """
+    Implement operations_interface.
+    """
+
+    def entity_display_name(self, item: Union[Suite, Experiment, Simulation]) -> str:
+        """
+        Get display name for entity.
+        Args:
+            item: Suite, Experiment or Simulation
+        Returns:
+            str
+        """
+        use_name = getattr(self.platform, "name_directory", False)
+        use_sim_name = getattr(self.platform, "sim_name_directory", True)
+
+        # Determine if we should include name
+        if isinstance(item, Simulation) and not use_sim_name:
+            use_name = False
+
+        if use_name and getattr(item, "name", None):
+            safe_name = clean_item_name(item.name, maxlen=self.platform.maxlen)
+            return f"{safe_name}_{item.id}"
+        else:
+            return item.id
+
+    @cache_directory
+    def get_directory(self, item: Union[Suite, Experiment, Simulation]) -> Path:
+        """
+        Get item's path.
+        Args:
+            item: Suite, Experiment, Simulation
+        Returns:
+            item file directory
+        """
+        job_dir = Path(self.platform.job_directory)
+        if isinstance(item, (FileSimulation, FileExperiment, FileSuite)):
+            item_dir = item.get_directory()
+        elif isinstance(item, Suite):
+            return job_dir / f"s_{self.entity_display_name(item)}"
+        elif isinstance(item, Experiment):
+            parent = item.parent
+            if parent:  # Case 1: Parent suite object is available, build job_dir/suite/experiment path
+                suite_dir = job_dir / f"s_{self.entity_display_name(parent)}"
+                return suite_dir / f"e_{self.entity_display_name(item)}"
+            else:  # Case 2: No parent or suite_id — build job_dir/experiment path
+                return job_dir / f"e_{self.entity_display_name(item)}"
+        elif isinstance(item, Simulation):
+            exp = item.parent
+            if exp is None:
+                raise RuntimeError("Simulation missing parent!")
+            exp_dir = self.get_directory(exp)
+            return exp_dir / self.entity_display_name(item)
+        else:
+            raise RuntimeError(f"Get directory is not supported for {type(item)} object on FilePlatform")
+
+        return item_dir
+
+    def get_directory_by_id(self, item_id: str, item_type: ItemType) -> Path:
+        """
+        Get item's path by id.
+        Args:
+            item_id: entity id (Suite, Experiment, Simulation)
+            item_type: the type of items (Suite, Experiment, Simulation)
+        Returns:
+            item file directory
+        """
+        metas = self.platform._metas.filter(item_type=item_type, property_filter={'id': str(item_id)})
+        if len(metas) > 0:
+            return Path(metas[0]['dir'])
+        else:
+            raise RuntimeError(f"Not found path for item_id: {item_id} with type: {item_type}.")
+
+    def mk_directory(self, item: Union[Suite, Experiment, Simulation] = None, dest: Union[Path, str] = None,
+                     exist_ok: bool = True) -> None:
+        """
+        Make a new directory.
+        Args:
+            item: Suite/Experiment/Simulation
+            dest: the folder path
+            exist_ok: True/False
+        Returns:
+            None
+        """
+        if dest is not None:
+            target = Path(dest)
+        elif isinstance(item, (Suite, Experiment, Simulation)):
+            target = self.get_directory(item)
+        else:
+            raise RuntimeError('Only support Suite/Experiment/Simulation or not None dest.')
+
+        # Validate target path length
+        validate_file_path_length(self.platform.job_directory)
+        target.mkdir(parents=True, exist_ok=exist_ok)
+
+    @check_symlink_capabilities
+    def link_file(self, target: Union[Path, str], link: Union[Path, str]) -> None:
+        """
+        Link files.
+        Args:
+            target: the source file path
+            link: the file path
+        Returns:
+            None
+        """
+        target = Path(target).absolute()
+        link = Path(link).absolute()
+        if self.platform.sym_link:
+            # Ensure the source folder exists
+            if not target.exists():
+                raise FileNotFoundError(f"Source folder does not exist: {target}")
+
+            # Compute the relative path from the destination to the source
+            relative_source = os.path.relpath(target, link.parent)
+
+            # Remove existing symbolic link or file at destination if it exists
+            if link.exists() or link.is_symlink():
+                link.unlink()
+
+            # Create the symbolic link
+            try:
+                link.symlink_to(relative_source, target_is_directory=False)
+            except OSError as e:
+                user_logger.error(f"\n Failed to create symbolic link: {e}")
+                if self.platform.system() == 'Windows':
+                    user_logger.warning("\n/!\\ WARNING:  Please follow the instructions to enable Developer Mode for Windows: ")
+                    user_logger.warning("https://learn.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development. \n")
+                exit(-1)
+        else:
+            shutil.copyfile(target, link)
+
+    @check_symlink_capabilities
+    def link_dir(self, target: Union[Path, str], link: Union[Path, str]) -> None:
+        """
+        Link directory/files.
+        Args:
+            target: the source folder path
+            link: the folder path
+        Returns:
+            None
+        """
+        target = Path(target).absolute()
+        link = Path(link).absolute()
+
+        # Validate file path length
+        validate_folder_files_path_length(target, link)
+
+        if self.platform.sym_link:
+            # Ensure the source folder exists
+            if not target.exists():
+                raise FileNotFoundError(f"Source folder does not exist: {target}")
+
+            # Compute the relative path from the destination to the source
+            relative_source = os.path.relpath(target, link.parent)
+
+            # Remove existing symbolic link or folder at destination if it exists
+            if link.exists() or link.is_symlink():
+                if link.is_symlink():
+                    link.unlink()
+                else:
+                    shutil.rmtree(link)
+
+            # Create the symbolic link
+            try:
+                link.symlink_to(relative_source, target_is_directory=True)
+            except OSError as e:
+                user_logger.error(f"\n Failed to create symbolic link: {e}")
+                if self.platform.system() == 'Windows':
+                    user_logger.warning("\n/!\\ WARNING:  Please follow the instructions to enable Developer Mode for Windows: ")
+                    user_logger.warning("https://learn.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development. \n")
+                exit(-1)
+        else:
+            shutil.copytree(target, link, dirs_exist_ok=True)
+
+    @staticmethod
+    def update_script_mode(script_path: Union[Path, str], mode: int = 0o777) -> None:
+        """
+        Change file mode.
+        Args:
+            script_path: script path
+            mode: permission mode
+        Returns:
+            None
+        """
+        script_path = Path(script_path)
+        script_path.chmod(mode)
+
+    def make_command_executable(self, simulation: Simulation) -> None:
+        """
+        Make simulation command executable.
+        Args:
+            simulation: idmtools Simulation
+        Returns:
+            None
+        """
+        exe = simulation.task.command.executable
+        if exe == 'singularity':
+            # split the command
+            cmd = shlex.split(simulation.task.command.cmd.replace("\\", "/"))
+            # get real executable
+            exe = cmd[3]
+
+        sim_dir = self.get_directory(simulation)
+        exe_path = sim_dir.joinpath(exe)
+
+        # see if it is a file
+        if exe_path.exists():
+            exe = exe_path
+        elif shutil.which(exe) is not None:
+            exe = Path(shutil.which(exe))
+        else:
+            logger.debug(f"Failed to find executable: {exe}")
+            exe = None
+        try:
+            if exe and not os.access(exe, os.X_OK):
+                self.update_script_mode(exe)
+        except:
+            logger.debug(f"Failed to change file mode for executable: {exe}")
+
+    def get_simulation_status(self, sim_id: str, **kwargs) -> EntityStatus:
+        """
+        Retrieve simulation status.
+        Args:
+            sim_id: Simulation ID
+            kwargs: keyword arguments used to expand functionality
+        Returns:
+            EntityStatus
+        """
+        sim_dir = self.get_directory_by_id(sim_id, ItemType.SIMULATION)
+
+        # Check process status
+        job_status_path = sim_dir.joinpath('job_status.txt')
+        if job_status_path.exists():
+            status = open(job_status_path).read().strip()
+            if status in ['100', '0', '-1']:
+                status = FILE_MAPS[status]
+            else:
+                status = FILE_MAPS['100']  # To be safe
+        else:
+            status = FILE_MAPS['None']
+
+        return status
+
+    def create_file(self, file_path: str, content: str) -> None:
+        """
+        Create a file with given content and file path.
+
+        Args:
+            file_path: the full path of the file to be created
+            content: file content
+        Returns:
+            Nothing
+        """
+        with open(file_path, 'w') as f:
+            f.write(content)
+
+    def create_batch_file(self, item: Union[Experiment, Simulation], **kwargs) -> None:
+        """
+        Create batch file.
+        Args:
+            item: the item to build batch file for
+            kwargs: keyword arguments used to expand functionality.
+        Returns:
+            None
+        """
+        if isinstance(item, Experiment):
+            generate_script(self.platform, item, **kwargs)
+        elif isinstance(item, Simulation):
+            generate_simulation_script(self.platform, item, **kwargs)
+        else:
+            raise NotImplementedError(f"{item.__class__.__name__} is not supported for batch creation.")