hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a199__py3-none-any.whl

hpcflow/sdk/config/config.py

@@ -14,42 +14,45 @@ import os
 import socket
 import uuid
 from dataclasses import dataclass, field
-from hashlib import new
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Union
-import fsspec
+from typing import cast, overload, TYPE_CHECKING
+import fsspec  # type: ignore
 
 from rich.console import Console, Group
 from rich.table import Table
 from rich.pretty import Pretty
 from rich.panel import Panel
 from rich import print as rich_print
-from fsspec.registry import known_implementations as fsspec_protocols
-from fsspec.implementations.local import LocalFileSystem
-from platformdirs import user_data_dir
-from valida.schema import Schema
+from fsspec.registry import known_implementations as fsspec_protocols  # type: ignore
+from fsspec.implementations.local import LocalFileSystem  # type: ignore
 from hpcflow.sdk.core.utils import get_in_container, read_YAML_file, set_in_container
 
-from hpcflow.sdk.core.validation import get_schema
+from hpcflow.sdk.core.validation import get_schema, Schema
 from hpcflow.sdk.submission.shells import DEFAULT_SHELL_NAMES
 from hpcflow.sdk.typing import PathLike
 
-from .callbacks import (
+from hpcflow.sdk.config.callbacks import (
     callback_bool,
     callback_lowercase,
     callback_scheduler_set_up,
     callback_supported_schedulers,
     callback_supported_shells,
     callback_update_log_console_level,
+    callback_unset_log_console_level,
     callback_vars,
     callback_file_paths,
     exists_in_schedulers,
     set_callback_file_paths,
     check_load_data_files,
     set_scheduler_invocation_match,
+    callback_update_log_file_path,
+    callback_update_log_file_level,
+    callback_unset_log_file_level,
+    callback_unset_log_file_path,
+    callback_log_file_path,
 )
-from .config_file import ConfigFile
-from .errors import (
+from hpcflow.sdk.config.config_file import ConfigFile
+from hpcflow.sdk.config.errors import (
     ConfigChangeInvalidJSONError,
     ConfigChangePopIndexError,
     ConfigChangeTypeInvalidError,
@@ -57,16 +60,35 @@ from .errors import (
     ConfigItemAlreadyUnsetError,
     ConfigItemCallbackError,
     ConfigNonConfigurableError,
+    ConfigReadOnlyError,
     ConfigUnknownItemError,
     ConfigUnknownOverrideError,
     ConfigValidationError,
 )
 
+if TYPE_CHECKING:
+    from collections.abc import Callable, Iterator, Mapping, Sequence
+    from typing import Any, Literal
+    from .types import (
+        ConfigDescriptor,
+        ConfigMetadata,
+        DefaultConfiguration,
+        SchedulerConfigDescriptor,
+        ShellConfigDescriptor,
+        GetterCallback,
+        SetterCallback,
+        UnsetterCallback,
+        T,
+    )
+    from ..app import BaseApp
+    from ..core.types import AbstractFileSystem
+
+
 logger = logging.getLogger(__name__)
 
 _DEFAULT_SHELL = DEFAULT_SHELL_NAMES[os.name]
 #: The default configuration descriptor.
-DEFAULT_CONFIG = {
+DEFAULT_CONFIG: DefaultConfiguration = {
     "invocation": {"environment_setup": None, "match": {}},
     "config": {
         "machine": socket.gethostname(),
@@ -88,29 +110,29 @@ class ConfigOptions:
     """Application-level options for configuration"""
 
     #: The default directory.
-    default_directory: Union[Path, str]
+    default_directory: Path | str
     #: The environment variable containing the directory name.
     directory_env_var: str
     #: The default configuration.
-    default_config: Optional[Dict] = field(
+    default_config: DefaultConfiguration = field(
         default_factory=lambda: deepcopy(DEFAULT_CONFIG)
     )
     #: Any extra schemas to apply.
-    extra_schemas: Optional[List[Schema]] = field(default_factory=lambda: [])
+    extra_schemas: Sequence[Schema] = field(default_factory=list)
     #: Default directory of known configurations.
-    default_known_configs_dir: Optional[str] = None
+    default_known_configs_dir: str | None = None
+    _schemas: Sequence[Schema] = field(init=False)
+    _configurable_keys: Sequence[str] = field(init=False)
 
-    def __post_init__(self):
-        cfg_schemas, cfg_keys = self.init_schemas()
-        self._schemas = cfg_schemas
-        self._configurable_keys = cfg_keys
+    def __post_init__(self) -> None:
+        self._schemas, self._configurable_keys = self.init_schemas()
 
-    def init_schemas(self):
+    def init_schemas(self) -> tuple[Sequence[Schema], Sequence[str]]:
         """
         Get allowed configurable keys from config schemas.
         """
-        cfg_schemas = [get_schema("config_schema.yaml")] + self.extra_schemas
-        cfg_keys = []
+        cfg_schemas = [get_schema("config_schema.yaml"), *self.extra_schemas]
+        cfg_keys: list[str] = []
         for cfg_schema in cfg_schemas:
             for rule in cfg_schema.rules:
                 if not rule.path and rule.condition.callable.name == "allowed_keys":
@@ -118,7 +140,13 @@ class ConfigOptions:
 
         return (cfg_schemas, cfg_keys)
 
-    def validate(self, data, logger, metadata=None, raise_with_metadata=True):
+    def validate(
+        self,
+        data: T,
+        logger: logging.Logger,
+        metadata: ConfigMetadata | None = None,
+        raise_with_metadata: bool = True,
+    ) -> T:
         """Validate configuration items of the loaded invocation."""
 
         logger.debug("Validating configuration...")
@@ -140,7 +168,8 @@ class ConfigOptions:
 
 
 class Config:
-    """Application configuration as defined in one or more config files.
+    """
+    Application configuration as defined in one or more config files.
 
     This class supports indexing into the collection of properties via Python dot notation.
 
@@ -178,76 +207,25 @@ class Config:
 
     Attributes
     ----------
-    config_directory:
-        The directory containing the configuration file.
-    config_file_name:
-        The name of the configuration file.
-    config_file_path:
-        The full path to the configuration file.
-    config_file_contents:
-        The cached contents of the configuration file.
-    config_key:
-        The primary key to select the configuration within the configuration file.
-    config_schemas:
-        The schemas that apply to the configuration file.
-    host_user_id:
-        User ID as understood by the script.
-    host_user_id_file_path:
-        Where user ID information is stored.
-    invoking_user_id:
-        User ID that created the workflow.
-    machine:
-        Machine to submit to.
-        Mapped to a field in the configuration file.
-    user_name:
+    user_name: str
         User to submit as.
         Mapped to a field in the configuration file.
-    user_orcid:
+    user_orcid: str
         User's ORCID.
         Mapped to a field in the configuration file.
-    user_affiliation:
+    user_affiliation: str
         User's institutional affiliation.
         Mapped to a field in the configuration file.
-    linux_release_file:
+    linux_release_file: str
         Where to get the description of the Linux release version data.
         Mapped to a field in the configuration file.
-    log_file_path:
-        Where to log to.
-        Mapped to a field in the configuration file.
-    log_file_level:
+    log_file_level: str
         At what level to do logging to the file.
         Mapped to a field in the configuration file.
-    log_console_level:
+    log_console_level: str
         At what level to do logging to the console. Usually coarser than to a file.
         Mapped to a field in the configuration file.
-    task_schema_sources:
-        Where to get task schemas.
-        Mapped to a field in the configuration file.
-    parameter_sources:
-        Where to get parameter descriptors.
-        Mapped to a field in the configuration file.
-    command_file_sources:
-        Where to get command files.
-        Mapped to a field in the configuration file.
-    environment_sources:
-        Where to get execution environment descriptors.
-        Mapped to a field in the configuration file.
-    default_scheduler:
-        The name of the default scheduler.
-        Mapped to a field in the configuration file.
-    default_shell:
-        The name of the default shell.
-        Mapped to a field in the configuration file.
-    schedulers:
-        Settings for supported scheduler(s).
-        Mapped to a field in the configuration file.
-    shells:
-        Settings for supported shell(s).
-        Mapped to a field in the configuration file.
-    demo_data_dir:
-        Location of demo data.
-        Mapped to a field in the configuration file.
-    demo_data_manifest_file:
+    demo_data_manifest_file: str
         Where the manifest describing the demo data is.
         Mapped to a field in the configuration file.
     """
@@ -258,10 +236,10 @@ class Config:
         config_file: ConfigFile,
         options: ConfigOptions,
         logger: logging.Logger,
-        config_key: Optional[str],
-        uid=None,
-        callbacks=None,
-        variables=None,
+        config_key: str | None,
+        uid: str | None = None,
+        callbacks: dict[str, tuple[GetterCallback, ...]] | None = None,
+        variables: dict[str, str] | None = None,
         **overrides,
     ):
         self._app = app
@@ -281,12 +259,12 @@ class Config:
         )
 
         # Callbacks are run on get:
-        self._get_callbacks = {
+        self._get_callbacks: dict[str, tuple[GetterCallback, ...]] = {
             "task_schema_sources": (callback_file_paths,),
             "environment_sources": (callback_file_paths,),
             "parameter_sources": (callback_file_paths,),
             "command_file_sources": (callback_file_paths,),
-            "log_file_path": (callback_vars, callback_file_paths),
+            "log_file_path": (callback_vars, callback_log_file_path),
             "telemetry": (callback_bool,),
             "schedulers": (callback_lowercase, callback_supported_schedulers),
             "shells": (callback_lowercase,),
@@ -297,7 +275,7 @@ class Config:
         }
 
         # Set callbacks are run on set:
-        self._set_callbacks = {
+        self._set_callbacks: dict[str, tuple[SetterCallback, ...]] = {
             "task_schema_sources": (set_callback_file_paths, check_load_data_files),
             "environment_sources": (set_callback_file_paths, check_load_data_files),
             "parameter_sources": (set_callback_file_paths, check_load_data_files),
@@ -305,22 +283,28 @@ class Config:
             "default_scheduler": (exists_in_schedulers, set_scheduler_invocation_match),
             "default_shell": (callback_supported_shells,),
             "schedulers": (callback_supported_schedulers, callback_scheduler_set_up),
-            "log_file_path": (set_callback_file_paths,),
+            "log_file_path": (callback_update_log_file_path,),
+            "log_file_level": (callback_update_log_file_level,),
             "log_console_level": (callback_update_log_console_level,),
             "demo_data_manifest_file": (set_callback_file_paths,),
         }
 
+        self._unset_callbacks: dict[str, tuple[UnsetterCallback, ...]] = {
+            "log_console_level": (callback_unset_log_console_level,),
+            "log_file_level": (callback_unset_log_file_level,),
+            "log_file_path": (callback_unset_log_file_path,),
+        }
+
         self._configurable_keys = self._options._configurable_keys
-        self._modified_keys = {}
-        self._unset_keys = []
+        self._modified_keys: ConfigDescriptor = {}
+        self._unset_keys: set[str] = set()
 
-        for name in overrides:
-            if name not in self._configurable_keys:
-                raise ConfigUnknownOverrideError(name=name)
+        if any((unknown := name) not in self._configurable_keys for name in overrides):
+            raise ConfigUnknownOverrideError(name=unknown)
 
         host_uid, host_uid_file_path = self._get_user_id()
 
-        metadata = {
+        metadata: ConfigMetadata = {
             "config_directory": self._file.directory,
             "config_file_name": self._file.path.name,
             "config_file_path": self._file.path,
@@ -333,22 +317,222 @@ class Config:
         }
         self._meta_data = metadata
 
+        # used within context manager `cached_config`:
+        self._use_cache = False
+        self._config_cache: dict[tuple[str, bool, bool, bool], Any] = {}
+
+        # note: this must go at the end, after all instance attributes have been set!
         self._options.validate(
             data=self.get_all(include_overrides=True),
             logger=self._logger,
             metadata=metadata,
         )
 
-    def __dir__(self):
-        return super().__dir__() + self._all_keys
+    def __dir__(self) -> Iterator[str]:
+        yield from super().__dir__()
+        yield from self._all_keys
 
-    def __getattr__(self, name):
-        if not name.startswith("__"):
-            return self._get(name)
-        else:
+    @property
+    def config_directory(self) -> Path:
+        """
+        The directory containing the configuration file.
+        """
+        return self._get("config_directory")
+
+    @property
+    def config_file_name(self) -> str:
+        """
+        The name of the configuration file.
+        """
+        return self._get("config_file_name")
+
+    @property
+    def config_file_path(self) -> Path:
+        """
+        The full path to the configuration file.
+        """
+        return self._get("config_file_path")
+
+    @property
+    def config_file_contents(self) -> str:
+        """
+        The cached contents of the configuration file.
+        """
+        return self._get("config_file_contents")
+
+    @property
+    def config_key(self) -> str:
+        """
+        The primary key to select the configuration within the configuration file.
+        """
+        return self._get("config_key")
+
+    @property
+    def config_schemas(self) -> Sequence[Schema]:
+        """
+        The schemas that apply to the configuration file.
+        """
+        return self._get("config_schemas")
+
+    @property
+    def invoking_user_id(self) -> str:
+        """
+        User ID that created the workflow.
+        """
+        return self._get("invoking_user_id")
+
+    @property
+    def host_user_id(self) -> str:
+        """
+        User ID as understood by the script.
+        """
+        return self._get("host_user_id")
+
+    @property
+    def host_user_id_file_path(self) -> Path:
+        """
+        Where user ID information is stored.
+        """
+        return self._get("host_user_id_file_path")
+
+    @property
+    def machine(self) -> str:
+        """
+        Machine to submit to.
+        Mapped to a field in the configuration file.
+        """
+        return self._get("machine")
+
+    @machine.setter
+    def machine(self, value: str):
+        self._set("machine", value)
+
+    @property
+    def log_file_path(self) -> str:
+        """
+        Where to log to.
+        Mapped to a field in the configuration file.
+        """
+        return self._get("log_file_path")
+
+    @log_file_path.setter
+    def log_file_path(self, value: str):
+        self._set("log_file_path", value)
+
+    @property
+    def environment_sources(self) -> Sequence[Path]:
+        """
+        Where to get execution environment descriptors.
+        Mapped to a field in the configuration file.
+        """
+        return self._get("environment_sources")
+
+    @environment_sources.setter
+    def environment_sources(self, value: Sequence[Path]):
+        self._set("environment_sources", value)
+
+    @property
+    def task_schema_sources(self) -> Sequence[str]:
+        """
+        Where to get task schemas.
+        Mapped to a field in the configuration file.
+        """
+        return self._get("task_schema_sources")
+
+    @task_schema_sources.setter
+    def task_schema_sources(self, value: Sequence[str]):
+        self._set("task_schema_sources", value)
+
+    @property
+    def command_file_sources(self) -> Sequence[str]:
+        """
+        Where to get command files.
+        Mapped to a field in the configuration file.
+        """
+        return self._get("command_file_sources")
+
+    @command_file_sources.setter
+    def command_file_sources(self, value: Sequence[str]):
+        self._set("command_file_sources", value)
+
+    @property
+    def parameter_sources(self) -> Sequence[str]:
+        """
+        Where to get parameter descriptors.
+        Mapped to a field in the configuration file.
+        """
+        return self._get("parameter_sources")
+
+    @parameter_sources.setter
+    def parameter_sources(self, value: Sequence[str]):
+        self._set("parameter_sources", value)
+
+    @property
+    def default_scheduler(self) -> str:
+        """
+        The name of the default scheduler.
+        Mapped to a field in the configuration file.
+        """
+        return self._get("default_scheduler")
+
+    @default_scheduler.setter
+    def default_scheduler(self, value: str):
+        self._set("default_scheduler", value)
+
+    @property
+    def default_shell(self) -> str:
+        """
+        The name of the default shell.
+        Mapped to a field in the configuration file.
+        """
+        return self._get("default_shell")
+
+    @default_shell.setter
+    def default_shell(self, value: str):
+        self._set("default_shell", value)
+
+    @property
+    def schedulers(self) -> Mapping[str, SchedulerConfigDescriptor]:
+        """
+        Settings for supported scheduler(s).
+        Mapped to a field in the configuration file.
+        """
+        return self._get("schedulers")
+
+    @schedulers.setter
+    def schedulers(self, value: Mapping[str, SchedulerConfigDescriptor]):
+        self._set("schedulers", value)
+
+    @property
+    def shells(self) -> Mapping[str, ShellConfigDescriptor]:
+        """
+        Settings for supported shell(s).
+        Mapped to a field in the configuration file.
+        """
+        return self._get("shells")
+
+    @shells.setter
+    def shells(self, value: Mapping[str, ShellConfigDescriptor]):
+        self._set("shells", value)
+
+    @property
+    def demo_data_dir(self) -> str | None:
+        """
+        Location of demo data.
+        Mapped to a field in the configuration file.
+        """
+        return self._get("demo_data_dir")
+
+    @demo_data_dir.setter
+    def demo_data_dir(self, value: str | None):
+        self._set("demo_data_dir", value)
+
+    def __getattr__(self, name: str):
+        if name.startswith("__"):
             raise AttributeError(f"Attribute not known: {name!r}.")
+        return self._get(name)
 
-    def __setattr__(self, name, value):
+    def __setattr__(self, name: str, value):
         if (
             "_configurable_keys" in self.__dict__
             and name in self.__dict__["_configurable_keys"]
@@ -357,37 +541,53 @@ class Config:
         else:
             super().__setattr__(name, value)
 
-    def _disable_callbacks(self, callbacks) -> Tuple[Dict]:
-        """Disable named get and set callbacks.
+    def _disable_callbacks(
+        self, callbacks: Sequence[str]
+    ) -> tuple[
+        dict[str, tuple[GetterCallback, ...]],
+        dict[str, tuple[SetterCallback, ...]],
+        dict[str, tuple[UnsetterCallback, ...]],
+    ]:
+        """
+        Disable named get, set, and unset callbacks.
 
         Returns
         -------
         The original get and set callback dictionaries.
         """
         self._logger.info(f"disabling config callbacks: {callbacks!r}")
-        get_callbacks_tmp = {
-            k: tuple(i for i in v if i.__name__ not in callbacks)
+        get_callbacks_tmp: dict[str, tuple[GetterCallback, ...]] = {
+            k: tuple(cb for cb in v if cb.__name__ not in callbacks)
             for k, v in self._get_callbacks.items()
         }
-        set_callbacks_tmp = {
-            k: tuple(i for i in v if i.__name__ not in callbacks)
+        set_callbacks_tmp: dict[str, tuple[SetterCallback, ...]] = {
+            k: tuple(cb for cb in v if cb.__name__ not in callbacks)
             for k, v in self._set_callbacks.items()
         }
+        unset_callbacks_tmp = {
+            k: tuple(i for i in v if i.__name__ not in callbacks)
+            for k, v in self._unset_callbacks.items()
+        }
         get_callbacks = copy.deepcopy(self._get_callbacks)
         set_callbacks = copy.deepcopy(self._set_callbacks)
+        unset_callbacks = copy.deepcopy(self._unset_callbacks)
         self._get_callbacks = get_callbacks_tmp
         self._set_callbacks = set_callbacks_tmp
-        return (get_callbacks, set_callbacks)
+        self._unset_callbacks = unset_callbacks_tmp
+        return (get_callbacks, set_callbacks, unset_callbacks)
 
     @contextlib.contextmanager
-    def _without_callbacks(self, *callbacks):
-        """Context manager to temporarily exclude named get and set callbacks."""
-        get_callbacks, set_callbacks = self._disable_callbacks(*callbacks)
-        yield
-        self._get_callbacks = get_callbacks
-        self._set_callbacks = set_callbacks
-
-    def _validate(self):
+    def _without_callbacks(self, *callbacks: str) -> Iterator[None]:
+        """Context manager to temporarily exclude named get, set, and unset callbacks."""
+        get_cb, set_cb, unset_cb = self._disable_callbacks(callbacks)
+        try:
+            yield
+        finally:
+            self._get_callbacks = get_cb
+            self._set_callbacks = set_cb
+            self._unset_callbacks = unset_cb
+
+    def _validate(self) -> None:
         data = self.get_all(include_overrides=True)
         self._options.validate(
             data=data,
@@ -396,95 +596,117 @@ class Config:
             raise_with_metadata=True,
         )
 
-    def _resolve_path(self, path):
+    def _resolve_path(self, path: PathLike) -> PathLike:
         """Resolve a file path, but leave fsspec protocols alone."""
-        if not any(str(path).startswith(i + ":") for i in fsspec_protocols):
-            path = Path(path)
-            path = path.expanduser()
-            if not path.is_absolute():
-                path = self._meta_data["config_directory"].joinpath(path)
-        else:
+        if path is None:
+            return None
+        if any(str(path).startswith(i + ":") for i in fsspec_protocols):
             self._logger.debug(
                 f"Not resolving path {path!r} because it looks like an `fsspec` URL."
             )
+            return path
+        real_path = Path(path).expanduser()
+        if real_path.is_absolute():
+            return real_path
+        return self._meta_data["config_directory"].joinpath(real_path)
+
+    def register_config_get_callback(
+        self, name: str
+    ) -> Callable[[GetterCallback], GetterCallback]:
+        """
+        Decorator to register a function as a configuration callback for a specified
+        configuration item name, to be invoked on `get` of the item.
+        """
 
-        return path
-
-    def register_config_get_callback(self, name):
-        """Decorator to register a function as a configuration callback for a specified
-        configuration item name, to be invoked on `get` of the item."""
-
-        def decorator(func):
+        def decorator(func: GetterCallback) -> GetterCallback:
             if name in self._get_callbacks:
-                self._get_callbacks[name] = tuple(
-                    list(self._get_callbacks[name]) + [func]
-                )
+                self._get_callbacks[name] = self._get_callbacks[name] + (func,)
             else:
                 self._get_callbacks[name] = (func,)
 
             @functools.wraps(func)
-            def wrap(value):
-                return func(value)
+            def wrap(config: Config, value: T) -> T:
+                return func(config, value)
 
             return wrap
 
         return decorator
 
-    def register_config_set_callback(self, name):
-        """Decorator to register a function as a configuration callback for a specified
-        configuration item name, to be invoked on `set` of the item."""
+    def register_config_set_callback(
+        self, name: str
+    ) -> Callable[[SetterCallback], SetterCallback]:
+        """
+        Decorator to register a function as a configuration callback for a specified
+        configuration item name, to be invoked on `set` of the item.
+        """
 
-        def decorator(func):
+        def decorator(func: SetterCallback) -> SetterCallback:
             if name in self._set_callbacks:
-                self._set_callbacks[name] = tuple(
-                    list(self._set_callbacks[name]) + [func]
-                )
+                self._set_callbacks[name] = self._set_callbacks[name] + (func,)
             else:
                 self._set_callbacks[name] = (func,)
 
             @functools.wraps(func)
-            def wrap(value):
-                return func(value)
+            def wrap(config: Config, value: T) -> Any:
+                return func(config, value)
 
             return wrap
 
         return decorator
 
     @property
-    def _all_keys(self):
-        return self._configurable_keys + list(self._meta_data.keys())
-
-    def get_all(self, include_overrides=True, as_str=False):
+    def _all_keys(self) -> list[str]:
+        return [*self._configurable_keys, *self._meta_data]
+
+    @overload
+    def get_all(
+        self, *, include_overrides: bool = True, as_str: Literal[True]
+    ) -> Mapping[str, str]:
+        ...
+
+    @overload
+    def get_all(
+        self, *, include_overrides: bool = True, as_str: Literal[False] = False
+    ) -> Mapping[str, Any]:
+        ...
+
+    def get_all(
+        self, *, include_overrides: bool = True, as_str: bool = False
+    ) -> Mapping[str, Any]:
         """Get all configurable items."""
-        items = {}
+        items: dict[str, Any] = {}
         for key in self._configurable_keys:
             if key in self._unset_keys:
                 continue
-            else:
-                try:
-                    val = self._get(
+            try:
+                if as_str:
+                    items[key] = self._get(
                         name=key,
                         include_overrides=include_overrides,
                         raise_on_missing=True,
-                        as_str=as_str,
+                        as_str=True,
                     )
-                except ValueError:
-                    continue
-                items.update({key: val})
+                else:
+                    items[key] = self._get(
+                        name=key,
+                        include_overrides=include_overrides,
+                        raise_on_missing=True,
+                    )
+            except ValueError:
+                continue
         return items
 
-    def _show(self, config=True, metadata=False):
-        group_args = []
+    def _show(self, config: bool = True, metadata: bool = False):
+        group_args: list[Panel] = []
         if metadata:
-            tab_md = Table(show_header=False, box=None)
-            tab_md.add_column()
-            tab_md.add_column()
+            tab = Table(show_header=False, box=None)
+            tab.add_column()
+            tab.add_column()
             for k, v in self._meta_data.items():
                 if k == "config_file_contents":
                     continue
-                tab_md.add_row(k, Pretty(v))
-            panel_md = Panel(tab_md, title="Config metadata")
-            group_args.append(panel_md)
+                tab.add_row(k, Pretty(v))
+            group_args.append(Panel(tab, title="Config metadata"))
 
         if config:
             tab = Table(show_header=False, box=None)
@@ -492,15 +714,13 @@ class Config:
             tab.add_column()
             for k, v in self.get_all().items():
                 tab.add_row(k, Pretty(v))
-            panel = Panel(tab, title=f"Config {self._config_key!r}")
-            group_args.append(panel)
+            group_args.append(Panel(tab, title=f"Config {self._config_key!r}"))
 
-        group = Group(*group_args)
-        rich_print(group)
+        rich_print(Group(*group_args))
 
-    def _get_callback_value(self, name, value):
+    def _get_callback_value(self, name: str, value):
         if name in self._get_callbacks and value is not None:
-            for cb in self._get_callbacks.get(name, []):
+            for cb in self._get_callbacks.get(name, ()):
                 self._logger.debug(
                     f"Invoking `config.get` callback ({cb.__name__!r}) for item {name!r}={value!r}"
                 )
@@ -510,9 +730,36 @@ class Config:
                     raise ConfigItemCallbackError(name, cb, err) from None
         return value
 
+    @overload
+    def _get(
+        self,
+        name: str,
+        *,
+        include_overrides=True,
+        raise_on_missing=False,
+        as_str: Literal[False] = False,
+        callback=True,
+        default_value=None,
+    ) -> Any:
+        ...
+
+    @overload
     def _get(
         self,
-        name,
+        name: str,
+        *,
+        include_overrides=True,
+        raise_on_missing=False,
+        as_str: Literal[True],
+        callback=True,
+        default_value=None,
+    ) -> list[str] | str:
+        ...
+
+    def _get(
+        self,
+        name: str,
+        *,
         include_overrides=True,
         raise_on_missing=False,
         as_str=False,
@@ -521,11 +768,22 @@ class Config:
     ):
         """Get a configuration item."""
 
+        if self._use_cache:
+            # note: we default_value is not necessarily hashable, so we can't cache on it!
+            key = (
+                name,
+                include_overrides,
+                raise_on_missing,
+                as_str,
+            )
+            if key in self._config_cache:
+                return self._config_cache[key]
+
         if name not in self._all_keys:
             raise ConfigUnknownItemError(name=name)
 
         elif name in self._meta_data:
-            val = self._meta_data[name]
+            val = cast("dict", self._meta_data)[name]
 
         elif include_overrides and name in self._overrides:
             val = self._overrides[name]
@@ -538,7 +796,7 @@ class Config:
                 val = default_value
 
         elif name in self._modified_keys:
-            val = self._modified_keys[name]
+            val = cast("dict", self._modified_keys)[name]
 
         elif name in self._configurable_keys:
             val = self._file.get_config_item(
@@ -557,74 +815,121 @@ class Config:
             else:
                 val = str(val)
 
+        if self._use_cache:
+            self._config_cache[key] = val
+
         return val
 
-    def _parse_JSON(self, name, value):
+    def _parse_JSON(self, name: str, value: str) -> Any:
         try:
-            value = json.loads(value)
+            return json.loads(value)
         except json.decoder.JSONDecodeError as err:
             raise ConfigChangeInvalidJSONError(name=name, json_str=value, err=err)
-        return value
 
-    def _set(self, name, value, is_json=False, callback=True, quiet=False):
+    @overload
+    def _set(
+        self, name: str, value: str, *, is_json: Literal[True], callback=True, quiet=False
+    ) -> None:
+        ...
+
+    @overload
+    def _set(
+        self,
+        name: str,
+        value: Any,
+        *,
+        is_json: Literal[False] = False,
+        callback=True,
+        quiet=False,
+    ) -> None:
+        ...
+
+    def _set(
+        self, name: str, value, *, is_json=False, callback=True, quiet=False
+    ) -> None:
+        """
+        Set a configuration item.
+        """
+        if self._use_cache:
+            raise ConfigReadOnlyError()
+
         if name not in self._configurable_keys:
             raise ConfigNonConfigurableError(name=name)
-        else:
-            if is_json:
-                value = self._parse_JSON(name, value)
-            current_val = self._get(name)
-            callback_val = self._get_callback_value(name, value)
-            file_val_raw = self._file.get_config_item(self._config_key, name)
-            file_val = self._get_callback_value(name, file_val_raw)
-
-            if callback_val != current_val:
-                was_in_modified = False
-                was_in_unset = False
-                prev_modified_val = None
-                modified_updated = False
-
-                if name in self._modified_keys:
-                    was_in_modified = True
-                    prev_modified_val = self._modified_keys[name]
-
-                if name in self._unset_keys:
-                    was_in_unset = True
-                    idx = self._unset_keys.index(name)
-                    self._unset_keys.pop(idx)
-
-                if callback_val != file_val:
-                    self._modified_keys[name] = value
-                    modified_updated = True
+        if is_json:
+            value = self._parse_JSON(name, cast("str", value))
+        current_val = self._get(name)
+        callback_val = self._get_callback_value(name, value)
+        file_val = self._get_callback_value(
+            name, self._file.get_config_item(self._config_key, name)
+        )
 
-                try:
-                    self._validate()
-
-                    if callback:
-                        for cb in self._set_callbacks.get(name, []):
-                            self._logger.debug(
-                                f"Invoking `config.set` callback for item {name!r}: {cb.__name__!r}"
-                            )
-                            cb(self, callback_val)
-
-                except ConfigValidationError as err:
-                    # revert:
-                    if modified_updated:
-                        if was_in_modified:
-                            self._modified_keys[name] = prev_modified_val
-                        else:
-                            del self._modified_keys[name]
-                    if was_in_unset:
-                        self._unset_keys.append(name)
-
-                    raise ConfigChangeValidationError(name, validation_err=err) from None
+        if callback_val != current_val:
+            was_in_modified = False
+            was_in_unset = False
+            prev_modified_val = None
+            modified_updated = False
+            mk = cast("dict", self._modified_keys)
+
+            if name in self._modified_keys:
+                was_in_modified = True
+                prev_modified_val = mk[name]
+
+            if name in self._unset_keys:
+                was_in_unset = True
+                self._unset_keys.remove(name)
+
+            if callback_val != file_val:
+                mk[name] = value
+                modified_updated = True
+
+            try:
+                self._validate()
+
+                if callback:
+                    for cb in self._set_callbacks.get(name, ()):
+                        self._logger.debug(
+                            f"Invoking `config.set` callback for item {name!r}: {cb.__name__!r}"
+                        )
+                        cb(self, callback_val)
+
+            except ConfigValidationError as err:
+                # revert:
+                if modified_updated:
+                    if was_in_modified:
+                        mk[name] = prev_modified_val
+                    else:
+                        del mk[name]
+                if was_in_unset:
+                    self._unset_keys.add(name)
+
+                raise ConfigChangeValidationError(name, validation_err=err) from None
 
-                self._logger.debug(
-                    f"Successfully set config item {name!r} to {callback_val!r}."
-                )
-            elif not quiet:
-                print(f"value is already: {callback_val!r}")
+            self._logger.debug(
+                f"Successfully set config item {name!r} to {callback_val!r}."
+            )
+        elif not quiet:
+            print(f"value is already: {callback_val!r}")
 
-    def set(self, path: str, value, is_json=False, quiet=False):
+    @overload
+    def set(
+        self,
+        path: str,
+        value: Any,
+        *,
+        is_json: Literal[False] = False,
+        quiet: bool = False,
+    ) -> None:
+        ...
+
+    @overload
+    def set(
+        self, path: str, value: str, *, is_json: Literal[True], quiet: bool = False
+    ) -> None:
+        ...
+
+    def set(
+        self, path: str, value: Any, *, is_json: bool = False, quiet: bool = False
+    ) -> None:
         """
         Set the value of a configuration item.
 
@@ -640,16 +945,15 @@ class Config:
         if is_json:
             value = self._parse_JSON(path, value)
 
-        parts = path.split(".")
-        name = parts[0]
+        name, *path_suffix = path.split(".")
         root = deepcopy(self._get(name, callback=False))
-        if parts[1:]:
+        if path_suffix:
             if root is None:
                 root = {}
-                self.set(path=parts[0], value={}, quiet=True)
+                self.set(path=name, value={}, quiet=True)
             set_in_container(
                 root,
-                path=parts[1:],
+                path=path_suffix,
                 value=value,
                 ensure_path=True,
                 cast_indices=True,
@@ -658,13 +962,13 @@ class Config:
             root = value
         self._set(name, root, quiet=quiet)
 
-    def unset(self, name):
+    def unset(self, name: str, callback: bool = True) -> None:
         """
         Unset the value of a configuration item.
 
         Parameters
         ----------
-        name: str
+        name:
             The name of the configuration item.
 
         Notes
@@ -676,48 +980,74 @@ class Config:
         if name in self._unset_keys or not self._file.is_item_set(self._config_key, name):
             raise ConfigItemAlreadyUnsetError(name=name)
 
-        self._unset_keys.append(name)
+        self._unset_keys.add(name)
         try:
             self._validate()
+            if callback:
+                for cb in self._unset_callbacks.get(name, []):
+                    self._logger.debug(
+                        f"Invoking `config.unset` callback for item {name!r}: "
+                        f"{cb.__name__!r}."
+                    )
+                    cb(self)
         except ConfigValidationError as err:
-            self._unset_keys.pop()
+            self._unset_keys.remove(name)
             raise ConfigChangeValidationError(name, validation_err=err) from None
 
+    @overload
     def get(
         self,
-        path,
-        callback=True,
-        copy=False,
-        ret_root=False,
-        ret_parts=False,
-        default=None,
-    ):
+        path: str,
+        *,
+        callback: bool = True,
+        copy: bool = False,
+        ret_root_and_parts: Literal[False] = False,
+        default: Any | None = None,
+    ) -> Any:
+        ...
+
+    @overload
+    def get(
+        self,
+        path: str,
+        *,
+        callback: bool = True,
+        copy: bool = False,
+        ret_root_and_parts: Literal[True],
+        default: Any | None = None,
+    ) -> tuple[Any, Any, list[str]]:
+        ...
+
+    def get(
+        self,
+        path: str,
+        *,
+        callback: bool = True,
+        copy: bool = False,
+        ret_root_and_parts: bool = False,
+        default: Any | None = None,
+    ) -> Any:
         """
         Get the value of a configuration item.
 
         Parameters
         ----------
-        path: str
+        path:
             The name of or path to the configuration item.
         """
-        parts = path.split(".")
-        root = deepcopy(self._get(parts[0], callback=callback))
+        name, *suffix = parts = path.split(".")
+        root = deepcopy(self._get(name, callback=callback))
         try:
-            out = get_in_container(root, parts[1:], cast_indices=True)
+            out = get_in_container(root, suffix, cast_indices=True)
         except KeyError:
             out = default
         if copy:
             out = deepcopy(out)
-        if not (ret_root or ret_parts):
+        if not ret_root_and_parts:
             return out
-        ret = [out]
-        if ret_root:
-            ret += [root]
-        if ret_parts:
-            ret += [parts]
-        return tuple(ret)
+        return out, root, parts
 
-    def append(self, path, value, is_json=False):
+    def append(self, path: str, value, *, is_json: bool = False) -> None:
         """
         Append a value to a list-like configuration item.
 
@@ -733,8 +1063,7 @@ class Config:
 
         existing, root, parts = self.get(
             path,
-            ret_root=True,
-            ret_parts=True,
+            ret_root_and_parts=True,
             callback=False,
             default=[],
         )
@@ -756,7 +1085,7 @@ class Config:
             root = new
         self._set(parts[0], root)
 
-    def prepend(self, path, value, is_json=False):
+    def prepend(self, path: str, value, *, is_json: bool = False) -> None:
         """
         Prepend a value to a list-like configuration item.
 
@@ -771,7 +1100,7 @@ class Config:
             value = self._parse_JSON(path, value)
 
         existing, root, parts = self.get(
-            path, ret_root=True, ret_parts=True, callback=False, default=[]
+            path, ret_root_and_parts=True, callback=False, default=[]
         )
 
         try:
@@ -791,7 +1120,7 @@ class Config:
             root = new
         self._set(parts[0], root)
 
-    def pop(self, path, index):
+    def pop(self, path: str, index) -> None:
         """
         Remove a value from a specified index of a list-like configuration item.
 
@@ -802,11 +1131,9 @@ class Config:
         index: int
             Where to remove the value from. 0 for the first item, -1 for the last.
         """
-
         existing, root, parts = self.get(
             path,
-            ret_root=True,
-            ret_parts=True,
+            ret_root_and_parts=True,
             callback=False,
             default=[],
         )
@@ -832,8 +1159,9 @@ class Config:
             root = new
         self._set(parts[0], root)
 
-    def update(self, path: str, value, is_json=False):
-        """Update a map-like configuration item.
+    def update(self, path: str, value, *, is_json: bool = False) -> None:
+        """
+        Update a map-like configuration item.
 
         Parameters
         ----------
@@ -842,15 +1170,13 @@ class Config:
         value: dict
             A dictionary to merge in.
         """
-
         if is_json:
             value = self._parse_JSON(path, value)
 
         val_mod, root, parts = self.get(
             path,
             copy=True,
-            ret_root=True,
-            ret_parts=True,
+            ret_root_and_parts=True,
             callback=False,
             default={},
         )
@@ -872,20 +1198,22 @@ class Config:
             root = val_mod
         self._set(parts[0], root)
 
-    def save(self):
+    def save(self) -> None:
         """Save any modified/unset configuration items into the file."""
         if not self._modified_keys and not self._unset_keys:
             print("No modifications to save!")
         else:
             self._file.save()
 
-    def get_configurable(self):
+    def get_configurable(self) -> Sequence[str]:
         """Get a list of all configurable keys."""
         return self._configurable_keys
 
-    def _get_user_id(self):
-        """Retrieve (and set if non-existent) a unique user ID that is independent of the
-        config directory."""
+    def _get_user_id(self) -> tuple[str, Path]:
+        """
+        Retrieve (and set if non-existent) a unique user ID that is independent of the
+        config directory.
+        """
 
         uid_file_path = self._app.user_data_dir.joinpath("user_id.txt")
         if not uid_file_path.exists():
@@ -898,12 +1226,12 @@ class Config:
 
         return uid, uid_file_path
 
-    def reset(self):
+    def reset(self) -> None:
         """Reset to the default configuration."""
-        self._logger.info(f"Resetting config file to defaults.")
+        self._logger.info("Resetting config file to defaults.")
         self._app.reset_config()
 
-    def add_scheduler(self, scheduler, **defaults):
+    def add_scheduler(self, scheduler: str, **defaults) -> None:
         """
         Add a scheduler.
         """
@@ -912,7 +1240,7 @@ class Config:
             return
         self.update(f"schedulers.{scheduler}.defaults", defaults)
 
-    def add_shell(self, shell, **defaults):
+    def add_shell(self, shell: str, **defaults) -> None:
         """
         Add a shell.
         """
@@ -923,7 +1251,7 @@ class Config:
             self.add_scheduler("direct_posix")
         self.update(f"shells.{shell}.defaults", defaults)
 
-    def add_shell_WSL(self, **defaults):
+    def add_shell_WSL(self, **defaults) -> None:
         """
         Add shell with WSL prefix.
         """
@@ -931,8 +1259,11 @@ class Config:
             defaults["WSL_executable"] = "wsl.exe"
         self.add_shell("wsl", **defaults)
 
-    def import_from_file(self, file_path, rename=True, make_new=False):
-        """Import config items from a (remote or local) YAML file. Existing config items
+    def import_from_file(
+        self, file_path: Path | str, *, rename=True, make_new=False
+    ) -> None:
+        """
+        Import config items from a (remote or local) YAML file. Existing config items
         of the same names will be overwritten.
 
         Parameters
@@ -947,17 +1278,12 @@ class Config:
             If True, add the config items as a new config, rather than modifying the
             current config. The name of the new config will be the stem of the file
             specified in `file_path`.
-
         """
-
         self._logger.debug(f"import from file: {file_path!r}")
 
         console = Console()
-        status = console.status(f"Importing config from file {file_path!r}...")
-        status.start()
-
-        try:
-            file_dat = read_YAML_file(file_path)
+        with console.status(f"Importing config from file {file_path!r}...") as status:
+            file_dat: DefaultConfiguration = read_YAML_file(file_path)
             if rename or make_new:
                 file_stem = Path(file_path).stem
                 name = file_stem
@@ -992,72 +1318,102 @@ class Config:
                     )
 
             new_invoc = file_dat.get("invocation")
-            new_config = file_dat.get("config")
+            new_config = file_dat.get("config", {})
 
-            if new_invoc:
+            if new_invoc is not None:
                 status.update("Updating invocation details...")
                 config_key = file_stem if (make_new or rename) else self._config_key
                 obj._file.update_invocation(
                     config_key=config_key,
                     environment_setup=new_invoc.get("environment_setup"),
-                    match=new_invoc.get("match"),
+                    match=new_invoc.get("match", {}),
                 )
 
             # sort in reverse so "schedulers" and "shells" are set before
             # "default_scheduler" and "default_shell" which might reference the former:
-            new_config = dict(sorted(new_config.items(), reverse=True))
-            for k, v in new_config.items():
+            for k, v in sorted(new_config.items(), reverse=True):
                 status.update(f"Updating configurable item {k!r}")
                 obj.set(k, value=v, quiet=True)
 
             obj.save()
 
-        except Exception:
-            status.stop()
-            raise
-
-        status.stop()
         print(f"Config {name!r} updated.")
 
-    def init(self, known_name: str, path: Optional[str] = None):
+    def init(self, known_name: str, path: str | None = None) -> None:
         """Configure from a known importable config."""
         if not path:
-            path = self._options.default_known_configs_dir
-            if not path:
+            if not (path := self._options.default_known_configs_dir):
                 raise ValueError("Specify an `path` to search for known config files.")
         elif path == ".":
             path = str(Path(path).resolve())
 
         self._logger.debug(f"init with `path` = {path!r}")
 
-        fs = fsspec.open(path).fs
+        fs: AbstractFileSystem = fsspec.open(path).fs
         local_path = f"{path}/" if isinstance(fs, LocalFileSystem) else ""
         files = fs.glob(f"{local_path}*.yaml") + fs.glob(f"{local_path}*.yml")
         self._logger.debug(f"All YAML files found in file-system {fs!r}: {files}")
 
-        files = [i for i in files if Path(i).stem.startswith(known_name)]
-        if not files:
+        if not (files := [i for i in files if Path(i).stem.startswith(known_name)]):
             print(f"No configuration-import files found matching name {known_name!r}.")
             return
 
         print(f"Found configuration-import files: {files!r}")
-        for i in files:
-            self.import_from_file(file_path=i, make_new=True)
+        for file_i in files:
+            self.import_from_file(file_path=file_i, make_new=True)
 
-        print(f"imports complete")
+        print("imports complete")
         # if current config is named "default", rename machine to DEFAULT_CONFIG:
         if self._config_key == "default":
             self.set("machine", "DEFAULT_MACHINE")
             self.save()
 
-    def set_github_demo_data_dir(self, sha):
-        """Set the `demo_data_dir` item, to an fsspec Github URL.
+    def set_github_demo_data_dir(self, sha: str) -> None:
+        """
+        Set the `demo_data_dir` item, to an fsspec Github URL.
 
         We use this (via the CLI) when testing the frozen app on Github, because, by
         default, the SHA is set to the current version tag, which might not include recent
         changes to the demo data.
-
         """
-        path = "/".join(self._app.demo_data_dir.split("."))
-        url = self._app._get_github_url(sha=sha, path=path)
-        self.set("demo_data_dir", url)
+        assert self._app.demo_data_dir is not None
+        self.set(
+            "demo_data_dir",
+            self._app._get_github_url(
+                sha=sha, path=self._app.demo_data_dir.replace(".", "/")
+            ),
+        )
+
+    @contextlib.contextmanager
+    def cached_config(self) -> Iterator[None]:
+        try:
+            self._use_cache = True
+            yield
+        finally:
+            self._use_cache = False
+            self._config_cache = {}  # reset the cache
+
+    def _is_set(self, name: str) -> bool:
+        """Check if a (non-metadata) config item is set."""
+        if name in self._unset_keys:
+            return False
+        elif name in self._modified_keys:
+            return True
+        else:
+            return self._file.is_item_set(self._config_key, name)
+
+    @contextlib.contextmanager
+    def _with_updates(self, updates: dict[str, Any]) -> Iterator[None]:
+        # need to run callbacks for unsetting?
+        prev_unset = copy.deepcopy(self._unset_keys)
+        prev_modified = copy.deepcopy(self._modified_keys)
+        to_unset = []
+        try:
+            for k, v in updates.items():
+                if not self._is_set(k):
+                    to_unset.append(k)
+                self.set(k, v)
+            yield
+        finally:
+            self._unset_keys = prev_unset
+            self._modified_keys = prev_modified