cimsi 0.7.8.dev1__py3-none-any.whl

imsi/cli/core_cli.py

@@ -0,0 +1,307 @@
+
+import click
+from functools import wraps
+import os
+from pathlib import Path
+import sys
+import warnings
+
+
+class CommandWithPassthroughEOO(click.Command):
+    def format_usage(self, ctx, formatter):
+        formatter.write_usage(ctx.command.name, "[OPTIONS] -- [PASSTHROUGH_ARGS]")
+
+
+def passthrough_eoo_delimiter(ctx, param, value):
+    # enforce requirement that end of options delimiter '--'
+    # is entered before trailing args
+    if value:
+        try:
+            delim_index = sys.argv.index('--')
+            first_arg_index = sys.argv.index(value[0])
+            if delim_index > first_arg_index:
+                raise click.UsageError("Use '--' before passthrough arguments.")
+        except ValueError:
+            raise click.UsageError("Use '--' before the trailing arguments.")
+    return value
+
+
+def force_dirs(
+    path: Path = Path("src"),
+):
+    user_in_path = path.exists()
+    wrk_dir_env_set = os.getenv("WRK_DIR") is not None
+
+    if user_in_path and wrk_dir_env_set and Path.cwd().resolve() == Path(os.getenv("WRK_DIR")).resolve():
+        return
+
+    if wrk_dir_env_set and user_in_path:
+        warnings.warn(
+            f"\n\n**WARNING**: Both WRK_DIR and {path} directory found. Defaulting to CWD {path} at {Path('.').resolve()}\n"
+        )
+        return
+
+    if wrk_dir_env_set and not Path(os.getenv("WRK_DIR"), path).exists():
+        raise ValueError(
+            f"⚠️  $WRK_DIR = {os.getenv('WRK_DIR')} is not a valid imsi directory. Please check the path and try again."
+        )
+
+    if not any([user_in_path, wrk_dir_env_set]):
+        raise ValueError(
+            f"⚠️  {path} directory not found! Possibly because:\n"
+            "1. You are not currently in your setup directory or one hasn't been created.\n"
+            "    or \n"
+            "2. The environment variable WRK_DIR is not set to the correct directory."
+        )
+
+
+def log_cli(func=None, logger_name='cli'):
+    from imsi.utils.git_tools import is_repo_clean, git_add_commit
+    import imsi.cli.core_tracking as ct
+
+    # decorator for logging imsi cli click commands
+    def decorator_func(func):
+        # the actual decorator
+        @wraps(func)
+        def wrapper_func(*args, **kwargs):
+            # the actual function being wrapped
+
+            track = True
+
+            # hack - required to make sure that:
+            #  - this logging isn't possible for all imsi functions (eg setup)
+            #  - logs aren't written to files when imsi cli commands are invoked
+            #    from the wrong location
+            force_dirs()     # success -> pwd == work dir
+            path = Path.cwd()
+
+            if args:
+                if isinstance(args[0], click.core.Context):
+                    # get the cli func name from the context rather than
+                    # func.__name__ (because of how click invokes func names)
+                    ctx = args[0]
+                    func_name = ctx.info_name
+            else:
+                # FIXME TODO fallback
+                func_name = func.__name__
+
+            # init log
+            imsi_logger = ct.get_imsi_logger(logger_name, path)
+            ct.imsi_log_prelude(func_name, imsi_logger)
+
+            if track:
+                config_dir = path / 'config'
+                ct.imsi_state_snapshot(path, logger=imsi_logger)
+
+                # force a clean repo for /config
+                if logger_name == 'cli':
+                    clean_config, _ = is_repo_clean(config_dir)
+                    if not clean_config:
+                        # always force the config dir to be clean
+                        msg = f"IMSI pre-run commit cli:{func_name}"
+                        git_add_commit(msg=msg, path=config_dir)
+                        imsi_logger.info(msg)
+
+            # invoke the wrapped function
+            try:
+                result = func(*args, **kwargs)
+            except Exception as e:
+                imsi_logger.error(f'ERROR {func_name} {type(e).__name__}')
+                raise e
+            ct.imsi_log_postlude(func_name, imsi_logger)
+            return result
+        return wrapper_func
+    if func:
+        # hack to handle decorator without kwargs (style)
+        return decorator_func(func)
+    return decorator_func
+
+
+@click.command(short_help="Set up a run directory and obtain model source code")
+@click.option(
+    "--runid",
+    default=None,
+    required=True,
+    help='Unique short string, without "_" or special chars.',
+)
+@click.option(
+    "--repo",
+    default="git@gitlab.science.gc.ca:CanESM/CanESM5.git",
+    required=True,
+    help="Git repository URL or file path.",
+)
+@click.option("--ver", default=None, help="Version of the code to clone.")
+@click.option("--exp", default="cmip6-piControl", help="Experiment name.")
+@click.option("--model", default=None, help="Model name.")
+@click.option(
+    "--fetch_method",
+    default="clone", type=click.Choice(["clone", "clone-full", "link", "copy"]),
+    help="Fetch method for source code.",
+    show_default=True
+)
+@click.option("--seq", default=None, help='Sequencer to use, like "iss" or "maestro".')
+@click.option("--machine", default=None, help="Machine to use.")
+@click.option("--flow", default=None, help="Workflow to use.")
+@click.option("--postproc", default=None, help="Postprocessing profile to use.")
+@click.pass_context
+def setup(ctx, **kwargs):
+    """Create a run directory, obtain the model source code, and extract all required model configuration files.
+
+    https://imsi.readthedocs.io/en/latest/usage.html#setting-up-a-run
+    """
+    from imsi.user_interface.setup_manager import (
+        setup_run,
+        ValidatedSetupOptions,
+        InvalidSetupConfig,
+    )
+    import imsi.cli.core_tracking as ct
+
+    try:
+        setup_args = ValidatedSetupOptions(**kwargs)
+    except InvalidSetupConfig as e:
+        click.echo(e)
+        raise e
+
+    setup_run(setup_args, force=ctx.obj["FORCE"])
+
+    ct.log_setup(sys.argv, setup_args)
+
+
+@click.command(help="Log the imsi state.",
+               epilog="Note: this is nominally an imsi utility function.")
+@click.option('-m', '--msg', default=None, required=False,
+              help='Message to include in the log.')
+@click.option('-p', '--path', type=click.Path(exists=True), default='.',
+              required=True,
+              help='Path to run folder.')
+def log_state(msg, path):
+    import imsi.cli.core_tracking as ct
+
+    path = Path(path).resolve()
+    logger = ct.get_imsi_logger('runtime', path)
+    ct.imsi_log_prelude('log-state', logger)
+    ct.imsi_state_snapshot(path, logger=logger)
+    if msg is not None:
+        logger.info(f'MESSAGE: {msg}')
+    ct.imsi_log_postlude('log-state', logger)
+
+
+@click.command()
+@click.pass_context
+@log_cli
+def config(ctx):
+    """Configure a simulation with updated settings from on-disk repo."""
+    import imsi.user_interface.ui_manager as uim
+
+    force_dirs(Path("src"))
+    uim.validate_version_reqs()
+    uim.update_config_from_state(force=ctx.obj["FORCE"])
+    click.echo("IMSI Config")
+
+
+@click.command(
+    short_help="Reload the imsi configuration from the on-disk repo"
+)
+@click.pass_context
+@log_cli
+def reload(ctx):
+    """Reload the imsi configuration from the on-disk repo config files and update the simulation configuration"""
+    import imsi.user_interface.ui_manager as uim
+
+    force_dirs(Path("src"))
+    uim.validate_version_reqs()
+    uim.reload_config_from_source(force=ctx.obj["FORCE"])
+    click.echo("IMSI Update")
+
+
+@click.command(short_help="Set an imsi selection in the configuration")
+@click.option("-f", "--file", help="Name of a configuration file containing imsi selections")
+@click.option(
+    "-s",
+    "--selections",
+    metavar="KEY=VALUE",
+    multiple=True,
+    help="A series of KEY=VALUE selection pairs.",
+)
+@click.option(
+    "-o",
+    "--options",
+    metavar="KEY=VALUE",
+    multiple=True,
+    help="A series of KEY=VALUE option pairs.",
+)
+@click.pass_context
+@log_cli
+def set(ctx, file, selections, options):
+    import imsi.user_interface.ui_manager as uim
+
+    force_dirs(Path("src"))
+    uim.validate_version_reqs()
+    if any([file, selections, options]):
+        uim.set_selections(file, selections, options, force=ctx.obj["FORCE"])
+    else:
+        click.echo(
+            "Error: Must provide at least one of --file, --selections, or --options",
+            err=True,
+        )
+
+
+@click.command(
+    cls=CommandWithPassthroughEOO,
+    context_settings=dict(ignore_unknown_options=True,),
+    short_help="Compile model components.",
+    help="Compile model components.",
+    epilog="The script 'imsi-tmp-compile.sh' will be executed."
+)
+@click.option("--script-help", is_flag=True, help="Display the help message of the script.")
+@click.argument("args", nargs=-1, callback=passthrough_eoo_delimiter)
+@click.pass_context
+@log_cli
+def build(ctx, script_help, args):
+    import imsi.user_interface.ui_manager as uim
+
+    force_dirs(Path("src"))
+    uim.validate_version_reqs()
+    args = ['-h'] if script_help else args
+    click.echo("IMSI Build")
+    uim.compile_model_execs(args)
+
+
+@click.command(help="Submit a simulation to run")
+@log_cli
+def submit():
+    import imsi.user_interface.ui_manager as uim
+
+    force_dirs(Path("src"))
+    uim.validate_version_reqs()
+    click.echo("IMSI submit")
+    uim.submit_run()
+
+
+@click.command(
+    cls=CommandWithPassthroughEOO,
+    context_settings=dict(ignore_unknown_options=True,),
+    short_help="Save the model restart files.",
+    help="Save the model restart files.",
+    epilog="The script 'save_restart_files.sh' will be executed."
+)
+@click.option("--script-help", is_flag=True, help="Display the help message of the script.")
+@click.argument("args", nargs=-1, callback=passthrough_eoo_delimiter)
+@log_cli
+def save_restarts(script_help, args):
+    import imsi.user_interface.ui_manager as uim
+
+    force_dirs(Path("src"))
+    uim.validate_version_reqs()
+    args = ['-h'] if script_help else args
+    click.echo("IMSI save restarts")
+    uim.save_restarts(args)
+
+
+@click.command(help="Get sequencer status information.")
+def status():
+    import imsi.user_interface.ui_manager as uim
+
+    force_dirs(Path("src"))
+    uim.validate_version_reqs()
+    uim.get_sequencer_status()

imsi/cli/core_tracking.py

@@ -0,0 +1,169 @@
+from importlib.resources import files
+import logging
+from pathlib import Path
+import subprocess
+import sys
+
+import imsi
+from imsi.utils.general import get_active_venv
+from imsi.user_interface.setup_manager import ValidatedSetupOptions
+
+
+IMSI_STATEFUL_FOLDERS = ['src', 'config']
+
+_logger_config = {
+    'setup': {
+        'logger_name': 'imsi-setup',
+        'filename': '.imsi-setup.log',
+        'hidden': True,
+        'level': logging.DEBUG
+    },
+    'cli': {
+        'logger_name': 'imsi-cli',
+        'filename': '.imsi-cli.log',
+        'hidden': True,
+        'level': logging.INFO
+    },
+    'runtime': {
+        'logger_name': 'imsi-runtime',
+        'filename': '.imsi-cli.log',
+        'hidden': True,
+        'level': logging.INFO
+    }
+}
+
+
+def _get_logger(logger_name: str=None, path: Path=None, filename: str=None,
+                hidden=True, level=logging.INFO):
+    # Return a logging.Logger with the name `logger_name` set at logging
+    # level `level`. The log file will be written to the full path
+    # constructed from `path / filename`. The `filename` must end with
+    # extension `.log`. The `hidden=True` flag ensures that the filename
+    # begins with a period (`.`) (conversely, ensures it is not included
+    # if `hidden=False`).
+
+    path = Path(path)
+
+    if not path.exists():
+        raise FileNotFoundError(f"ERROR: can't write log file to path {path}")
+    if not filename.endswith('.log'):
+        raise ValueError('filename must end with .log')
+
+    logger = logging.getLogger(logger_name)
+    if logger.hasHandlers():
+        logger.handlers.clear()
+
+    if hidden:
+        basename = f".{filename}" if not filename.startswith('.') else filename
+    else:
+        basename = filename.lstrip('.')
+
+    formatter = logging.Formatter(
+        fmt='%(asctime)s - %(thread)d - %(name)s - %(levelname)s - %(message)s',
+        datefmt='%Y-%m-%dT%H:%M:%S%z'
+        )
+
+    # instantiate log config
+    fh = logging.FileHandler(str(path / basename))
+    fh.setFormatter(formatter)
+    logger.addHandler(fh)
+    logger.setLevel(level)
+
+    return logger
+
+
+def get_imsi_logger(name, path):
+    """Return an imsi logger of name `name` written to the log file `path`."""
+    logger_settings = _logger_config[name]
+    logger_settings['path'] = path
+    return _get_logger(**logger_settings)
+
+
+def log_setup(cli_args, setup_params: ValidatedSetupOptions, with_src_status=True):
+    """Log setup command and information to the setup log file."""
+    setup_path = Path(setup_params.runid).resolve()
+
+    logger = get_imsi_logger('setup', setup_path)
+    logger.info(f"🚀 IMSI setup for {setup_params.runid} 🚀")
+    logger.info('setup_params: {}'.format(' '.join(f"{k}={v}" for k, v in setup_params.model_dump().items())))
+
+    if with_src_status:
+        # capture the status of the git repo under /src
+        s = files("imsi").joinpath("utils/repo_query_status.sh")
+        src_path = setup_path / 'src'
+        try:
+            proc = subprocess.run([str(s)], capture_output=True, cwd=src_path)
+            proc.check_returncode()
+        except subprocess.CalledProcessError as e:
+            src_msg = f'Could not determine status under {src_path}'
+        else:
+            src_msg = f'src: {proc.stdout.decode().strip()}'
+
+        logger.info(src_msg)
+
+    # DEV: keep this as the last setup log entry (newline)
+    input_args = ' '.join(sys.argv)
+    logger.info(f"Setup command used:\n{input_args}")
+
+
+def imsi_log_prelude(func_name, logger):
+    """Log header information (imsi metadata) to the `logger` related to
+    running `func_name`. Logging is set to logging.INFO.
+    """
+    imsi_meta = f"imsi {imsi.__version__} {imsi.__path__[0]}"
+    imsi_venv = get_active_venv()
+    logger.info(f'INVOKING {func_name}')
+    logger.info(' '.join(sys.argv))
+    logger.info(imsi_meta)
+    logger.info(f'VIRTUAL_ENV {imsi_venv}')
+
+
+def imsi_log_postlude(func_name, logger):
+    """Log footer information related to running `func_name`.
+    Logging level is set to logging.INFO.
+    """
+    logger.info(f'COMPLETED {func_name}')
+
+
+def imsi_state_snapshot(path, folders=None, logger=None):
+    """Take a snapshot of the imsi state and return the state hash.
+
+    This runs the snapshot tool (cli_snapshot_state.sh) for the `folders`
+    specified under the `path`.
+
+    Parameters:
+        path : path to an imsi run folder
+        folders : folder names (relative to `path`) on which to take the state
+            snapshot. These folders must be git repos. Default list is
+            ['config', 'src'].
+        logger : instance of a Logger.
+
+    Returns:
+        the hash of the imsi state folder
+    """
+
+    if folders is None:
+        folders = IMSI_STATEFUL_FOLDERS
+    elif isinstance(folders, str):
+        folders = [folders]
+
+    tracker_script = Path(__file__).parent / 'cli_snapshot_state.sh'
+    folder_args = [l for s in [['-p', p] for p in folders] for l in s]
+    io_args = ['-i', path, '-o', path]
+
+    if not tracker_script.exists():
+        raise FileNotFoundError(f'ERROR missing snapshot tool: {tracker_script}')
+    try:
+        proc = subprocess.run([str(tracker_script)] + folder_args + io_args, capture_output=True)
+        proc.check_returncode()
+    except subprocess.CalledProcessError as e:
+        if logger:
+            logger.error(f'state snapshot failed')
+            logger.error(f'HALTING')
+        raise ChildProcessError("Failed call: {cmd}\n{err}".format(cmd=' '.join(proc.args), err=proc.stdout.decode())) from e
+
+    state_sha = proc.stdout.decode().strip()
+    if logger:
+        logger.info(f'IMSI state:{state_sha}')
+
+    return state_sha

imsi/cli/entry.py

@@ -0,0 +1,59 @@
+"""
+imsi CLI
+--------
+
+The entry-point console script that interfaces all users commands to imsi.
+
+imsi has several categories of sub-commands. As this module develops further,
+the sub-groups are implemented in the relevant downstream modules.
+"""
+
+import click
+from imsi.cli.sectioned_group import SectionedGroup
+
+
+CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
+
+
+@click.group(
+    cls=SectionedGroup,
+    invoke_without_command=True,
+    context_settings=CONTEXT_SETTINGS,
+)
+@click.version_option()
+@click.option("-f", "--force", is_flag=True, help="Force the operation")
+@click.pass_context
+def cli(ctx, force):
+    """IMSI CLI — manage configs, builds, runs, and tools. Add the -h or --help flag to any command for more information."""
+    ctx.ensure_object(dict)
+    ctx.obj["FORCE"] = force
+
+
+cli.add_lazy_command("imsi.cli.core_cli.setup")
+cli.add_lazy_command("imsi.cli.core_cli.config", short_help="Configure a simulation with updated settings from on-disk repo")
+cli.add_lazy_command(
+    "imsi.cli.core_cli.save_restarts",
+    name="save-restarts",
+    short_help="Save restarts into the local run database & RUNPATH. Add -h for more information.",
+    context_settings={"ignore_unknown_options": True},
+    add_help_option=False,  # Disable automatic help flag,
+    )
+cli.add_lazy_command(
+    "imsi.cli.core_cli.build",
+    short_help="Compile model components. Add -h for more information.",
+    context_settings={"ignore_unknown_options": True},
+    add_help_option=False,  # Disable automatic help flag
+)
+cli.add_lazy_command("imsi.cli.core_cli.submit")
+cli.add_lazy_command("imsi.cli.core_cli.status")
+cli.add_lazy_command("imsi.tools.disk_tools.disk_tools_cli.clean")
+cli.add_lazy_command("imsi.cli.core_cli.reload")
+cli.add_lazy_command("imsi.cli.core_cli.set")
+cli.add_lazy_command("imsi.tools.list.list_cli.list")
+cli.add_lazy_command("imsi.cli.core_cli.log_state", name="log-state")
+
+cli.add_lazy_command("imsi.tools.ensemble.ensemble_cli.ensemble")
+cli.add_lazy_command("imsi.tools.time_manager.timer_cli.chunk_manager", name="chunk-manager")
+cli.add_lazy_command("imsi.tools.simple_sequencer.iss_cli.iss")
+cli.add_lazy_command("imsi.tools.menu.menu_cli.setup_menu", name="setup-menu", short_help="Interactive menu to explore IMSI configurations.")
+cli.add_lazy_command("imsi.tools.validate.validate_cli.validate", name="validate", short_help="Validate imsi configuration files.")

imsi/cli/lazy.py

@@ -0,0 +1,104 @@
+import click
+import importlib
+
+
+class LazyCommand(click.Command):
+    def __init__(
+        self,
+        import_path,
+        name=None,
+        short_help="",
+        context_settings=None,
+        add_help_option=False,
+    ):
+        self._import_path = import_path
+        self._real_command = None
+        self._short_help = short_help
+        super().__init__(
+            name or import_path.split(".")[-1],
+            short_help=short_help,
+            context_settings=context_settings or {},
+            add_help_option=add_help_option,
+        )
+
+    def _load_command(self):
+        if self._real_command is None:
+            module_path, func_name = self._import_path.rsplit(".", 1)
+            module = importlib.import_module(module_path)
+            self._real_command = getattr(module, func_name)
+        return self._real_command
+
+    def invoke(self, ctx):
+        return self._load_command().invoke(ctx)
+
+    def get_help(self, ctx):
+        return self._load_command().get_help(ctx)
+
+    def get_params(self, ctx):
+        return self._load_command().get_params(ctx)
+
+    def format_help(self, ctx, formatter):
+        return self._load_command().format_help(ctx, formatter)
+
+    def format_usage(self, ctx, formatter):
+        return self._load_command().format_usage(ctx, formatter)
+
+    def get_short_help_str(self, limit=45):
+        return self._short_help or self._load_command().get_short_help_str(limit)
+
+
+class LazyGroup(click.Group):
+    def __init__(self, *args, **kwargs):
+        self._lazy_commands = {}
+        super().__init__(*args, **kwargs)
+
+    def add_lazy_command(
+        self,
+        import_path: str,
+        name: str | None = None,
+        short_help: str = "",
+        **lazy_kwargs,
+    ):
+        """
+        Register a lazy‑loaded command or group.
+
+        Parameters
+        ----------
+        import_path : str
+            Dotted import path to a click.Command or click.Group object.
+        name : str, optional
+            Override the command name shown in the CLI.
+        short_help : str, optional
+            One‑liner shown in the parent help without importing the module.
+        **lazy_kwargs
+            Extra keyword args forwarded to LazyCommand
+            (e.g. context_settings, add_help_option).
+        """
+        name = name or import_path.split(".")[-1]
+        self._lazy_commands[name] = (import_path, short_help, lazy_kwargs)
+
+    def get_command(self, ctx, cmd_name):
+        if cmd_name in self._lazy_commands:
+            import_path, short_help, extra = self._lazy_commands[cmd_name]
+            module_path, func_name = import_path.rsplit(".", 1)
+            module = importlib.import_module(module_path)
+            real_cmd = getattr(module, func_name)
+
+            # If it's a Group, return it as‑is (no wrapper needed)
+            if isinstance(real_cmd, click.Group):
+                return real_cmd
+
+            # Else wrap it, forwarding all extra kwargs
+            return LazyCommand(
+                import_path,
+                name=cmd_name,
+                short_help=short_help,
+                **extra,
+            )
+
+        return super().get_command(ctx, cmd_name)
+
+    def list_commands(self, ctx):
+        cmds = list(self.commands.keys())
+        lazies = [name for name in self._lazy_commands.keys() if name not in self.commands]
+        return cmds + lazies

imsi/cli/post_install.py

@@ -0,0 +1,16 @@
+import click
+from importlib.resources import files
+
+
+def add_site_repos_to_rc(path_to_site_repos):
+    path_to_rc = files("imsi").joinpath("imsi.site.rc")
+    with open(path_to_rc, "a") as f:
+        # append IMSI_DEFAULT_CONFIG_REPOS=path_to_site_repos
+        f.write(f"\nIMSI_DEFAULT_CONFIG_REPOS={path_to_site_repos}\n")
+    click.echo(f"Added IMSI_DEFAULT_CONFIG_REPOS={path_to_site_repos} to {path_to_rc}")
+
+
+@click.command()
+@click.option('--path-to-site-repos', help='Location on disk where site-repos are stored. Will add value to the imsi.site.rc file. Not meant for regular imsi users, only for system install.', required=True)
+def post_install(path_to_site_repos):
+    add_site_repos_to_rc(path_to_site_repos)

imsi/cli/sectioned_group.py

@@ -0,0 +1,26 @@
+import click
+from imsi.cli.lazy import LazyGroup
+
+
+class SectionedGroup(LazyGroup):
+    """Like LazyGroup, but prints commands and sub‑groups in separate sections in the help message."""
+
+    def format_commands(self, ctx, formatter):
+        commands, groups = [], []
+
+        for name in self.list_commands(ctx):
+            cmd = self.get_command(ctx, name)
+            if cmd is None or cmd.hidden:
+                continue
+            row = (name, cmd.get_short_help_str())
+            # check if in a group or a command
+            (groups if isinstance(cmd, click.Group) else commands).append(row)
+
+        if commands:
+            with formatter.section("Commands"):
+                formatter.write_dl(commands)
+
+        if groups:
+            # Blank line between sections is automatic
+            with formatter.section("Command Groups"):
+                formatter.write_dl(groups)

imsi/config_manager/.gitignore

@@ -0,0 +1,5 @@
+test-configuration.json
+compilation_template
+computational_environment
+shell_parameters
+flattened_config