laco 0.0.0__py3-none-any.whl

laco/__init__.py

@@ -0,0 +1,33 @@
+"""
+Lazy configuration system, inspired by and based on Detectron2 and Hydra.
+"""
+
+from ._lazy import *
+from ._loader import *
+from ._overrides import *
+from ._resolvers import *
+
+__lazy__ = ("env", "language", "builtins", "cli", "env", "handler", "keys", "utils", "handler")
+
+
+def __getattr__(name: str):
+    from importlib.metadata import PackageNotFoundError, version
+    from importlib import import_module
+
+    if name in __lazy__:
+        return import_module(name, package=__name__)
+    if name == "__version__":
+        try:
+            return version(__name__)
+        except PackageNotFoundError:
+            return "unknown"
+    msg = f"Module {__name__!r} has no attribute {name!r}"
+    raise AttributeError(msg)
+
+def __dir__() -> list[str]:
+    from ._lazy import __all__ as all_lazy
+    from ._loader import __all__ as all_loader
+    from ._overrides import __all__ as all_overrides
+    from ._resolvers import __all__ as all_resolvers
+    
+    return sorted(__lazy__ + ["__version__"] + all_lazy + all_loader + all_overrides + all_resolvers)

laco/__init__.pyi

@@ -0,0 +1,5 @@
+from ._lazy import *
+from ._loader import *
+from ._overrides import *
+from ._resolvers import *
+from . import env, language, builtins, cli, env, handler, keys, utils, handler

laco/_dump.py

@@ -0,0 +1,104 @@
+import dataclasses
+import pprint
+from contextlib import suppress
+from copy import deepcopy
+
+import iopathlib
+import yaml
+from omegaconf import DictConfig, OmegaConf, SCMode
+
+from . import keys, utils
+
+__all__ = ["dump_config", "save_config"]
+
+
+def dump_config(cfg) -> str:  # noqa: C901
+    if not isinstance(cfg, DictConfig):
+        cfg = utils.as_omegadict(
+            dataclasses.asdict(cfg) if dataclasses.is_dataclass(cfg) else cfg
+        )
+    try:
+        cfg = deepcopy(cfg)
+    except Exception:
+        pass
+    else:
+
+        def _replace_type_by_name(x):
+            if keys.LAZY_CALL in x and callable(x._target_):
+                with suppress(AttributeError):
+                    x._target_ = utils.generate_path(x._target_)
+
+        utils.apply_recursive(cfg, _replace_type_by_name)
+
+    try:
+        cfg_as_dict = OmegaConf.to_container(
+            cfg,
+            # Do not resolve interpolation when saving, i.e. do not turn ${a} into
+            # actual values when saving.
+            resolve=False,
+            # Save structures (dataclasses) in a format that can be instantiated later.
+            # Without this option, the type information of the dataclass will be erased.
+            structured_config_mode=SCMode.INSTANTIATE,
+        )
+    except Exception as err:
+        cfg_pretty = pprint.pformat(OmegaConf.to_container(cfg)).replace("\n", "\n\t")
+        msg = f"Config cannot be converted to a dict!\n\nConfig node:\n{cfg_pretty}"
+        raise ValueError(msg) from err
+
+    dump_kwargs = {"default_flow_style": None, "allow_unicode": True}
+
+    def _find_undumpable(cfg_as_dict, *, _key=()) -> tuple[str, ...] | None:
+        for key, value in cfg_as_dict.items():
+            if not isinstance(value, dict):
+                continue
+            try:
+                _ = yaml.dump(value, **dump_kwargs)
+                continue
+            except Exception:
+                pass
+            key_with_error = _find_undumpable(value, _key=_key + (key,))
+            if key_with_error:
+                return key_with_error
+            return _key + (key,)
+        return None
+
+    try:
+        dumped = yaml.dump(cfg_as_dict, **dump_kwargs)
+    except Exception as err:
+        cfg_pretty = pprint.pformat(cfg_as_dict).replace("\n", "\n\t")
+        problem_key = _find_undumpable(cfg_as_dict)
+        if problem_key:
+            problem_key = ".".join(problem_key)
+            msg = f"Config cannot be saved due to key {problem_key!r}"
+        else:
+            msg = "Config cannot be saved due to an unknown entry"
+        msg += f"\n\nConfig node:\n\t{cfg_pretty}"
+        raise SyntaxError(msg) from err
+
+    return dumped
+
+
+def save_config(cfg, path: str):
+    """
+    Save a config object to a yaml file.
+
+    Parameters
+    ----------
+    cfg
+        An omegaconf config object.
+    filename
+        The file name to save the config file.
+    """
+    local_path = iopathlib.get_local_path(path)  # type: ignore[arg-type]
+    if not local_path.endswith(".yaml"):
+        msg = f"Config file should be saved as a yaml file! Got: {path}"
+        raise ValueError(msg)
+
+    dumped = dump_config(cfg)
+    try:
+        with open(local_path, "w") as fh:  # noqa: PTH123
+            fh.write(dumped)
+        _ = yaml.unsafe_load(dumped)
+    except Exception as err:
+        msg = f"Config file cannot be saved at {local_path!r}"
+        raise SyntaxError(msg) from err

laco/_lazy.py

@@ -0,0 +1,139 @@
+"""
+Instantiation of configuration objects.
+"""
+
+import dataclasses
+import logging
+import pprint
+import types
+import typing
+
+import omegaconf
+from omegaconf import DictConfig, ListConfig
+
+
+__all__ = [
+    "instantiate",
+]
+
+if typing.TYPE_CHECKING:
+
+    class LazyObject[_L]:
+        def __getattr__(self, name: str, /) -> typing.Any: ...
+
+        @typing.override
+        def __setattr__(self, name: str, value: typing.Any, /) -> None: ...  # noqa: PYI063
+
+else:
+
+    class LazyObject(dict[str, typing.Any]):
+        def __class_getitem__(cls, item: typing.Any) -> dict[str, typing.Any]:
+            return types.GenericAlias(dict, (str, typing.Any))
+
+
+type AnyConfig = DictConfig | ListConfig
+
+
+def migrate_target(target: typing.Any) -> typing.Any:
+    return target
+
+
+_INST_SEQ_TYPEMAP: dict[type, type] = {
+    ListConfig: list,
+    list: list,
+    tuple: tuple,
+    set: set,
+    frozenset: frozenset,
+}
+
+
+def instantiate(cfg: typing.Any, /) -> object:  # noqa: C901, PLR0912
+    """
+    Recursively instantiate objects defined in dictionaries with keys:
+
+    - Special key ``keys.CONFIG_CALL``: defines the callable/objec to be instantiated.
+    - Special key ``"_args_"``: defines the positional arguments to be passed to the
+        callable.
+    - Other keys define the keyword arguments to be passed to the callable.
+    """
+    import laco.env
+    import laco.keys
+    import laco.utils
+
+    if cfg is None or isinstance(
+        cfg,
+        int
+        | float
+        | bool
+        | str
+        | set
+        | frozenset
+        | bytes
+        | type
+        | types.NoneType
+        | types.FunctionType,
+    ):
+        return cfg  # type: ignore[return-value]
+
+    if laco.env.fetch(bool, "LACO_TRACE", default=False):
+        logging.getLogger(__name__).info(
+            "Instantiating %s", pprint.pprint(omegaconf.OmegaConf.to_container(cfg))
+        )
+
+    if isinstance(cfg, typing.Sequence) and not isinstance(
+        cfg, typing.Mapping | str | bytes
+    ):
+        cls = type(cfg)
+        cls = _INST_SEQ_TYPEMAP.get(cls, cls)
+        return cls(instantiate(x) for x in cfg)
+
+    # If input is a DictConfig backed by dataclasses (structured config)
+    # instantiate it to the actual dataclass.
+    if isinstance(cfg, DictConfig) and dataclasses.is_dataclass(
+        cfg._metadata.object_type
+    ):
+        return omegaconf.OmegaConf.to_object(cfg)
+
+    if isinstance(cfg, typing.Mapping) and laco.keys.LAZY_CALL in cfg:
+        # conceptually equivalent to hydra.utils.instantiate(cfg) with _convert_=all,
+        # but faster: https://github.com/facebookresearch/hydra/issues/1200
+        cfg = {k: instantiate(v) for k, v in cfg.items()}
+        cls = cfg.pop(laco.keys.LAZY_CALL)
+        cls = migrate_target(cls)
+        cls = instantiate(cls)
+
+        if isinstance(cls, str):
+            cls_name = cls
+            cls = laco.utils.locate_object(cls_name)
+            assert cls is not None, cls_name
+        else:
+            try:
+                cls_name = cls.__module__ + "." + cls.__qualname__
+            except Exception:  # noqa: B902, PIE786
+                # target could be anything, so the above could fail
+                cls_name = str(cls)
+        if not callable(cls):
+            msg = f"Non-callable object found: {laco.keys.LAZY_CALL}={cls!r}!"
+            raise TypeError(msg)
+
+        cfg_args = cfg.pop(laco.keys.LAZY_ARGS, ())
+        if not isinstance(cfg_args, typing.Sequence):
+            msg = f"Expected sequence for {laco.keys.LAZY_ARGS}, but got {type(cfg_args)}!"
+            raise TypeError(msg)
+
+        try:
+            return cls(*cfg_args, **cfg)
+        except Exception as err:
+            msg = (
+                f"Error instantiating lazy object {cls_name}.\n\nConfig node:\n\t{cfg}!"
+            )
+            raise RuntimeError(msg) from err
+
+    if isinstance(cfg, dict | DictConfig):
+        return {k: instantiate(v) for k, v in cfg.items()}  # type: ignore[return-value]
+
+    if callable(cfg):
+        return cfg
+
+    err = f"Cannot instantiate {cfg}, type {type(cfg)}!"
+    raise ValueError(err)

laco/_loader.py

@@ -0,0 +1,194 @@
+import builtins
+import os
+import typing
+from contextlib import contextmanager
+from uuid import uuid4
+
+import iopathlib
+import yaml
+from omegaconf import DictConfig, ListConfig
+
+__all__ = ["load_config_remote", "load_config_local"]
+
+
+PATCH_PREFIX: typing.Final = "_laco_"
+
+
+def load_config_remote(path: str):
+    """
+    Load a configuration from a remote source. Currently accepted external configuration
+    sources are:
+
+    - `Weights & Biases <https://wandb.ai/>`_ runs: ``wandb-run://<run_id>``
+    """
+    from unipercept.engine.integrations.wandb_integration import WANDB_RUN_PREFIX
+    from unipercept.engine.integrations.wandb_integration import (
+        read_run as wandb_read_run,
+    )
+
+    if path.startswith(WANDB_RUN_PREFIX):
+        run = wandb_read_run(path)
+        cfg = DictConfig(run.config)
+    else:
+        raise FileNotFoundError(path)
+
+    return cfg
+
+
+@contextmanager
+def _patch_import():  # noqa: C901
+    import importlib.machinery
+    import importlib.util
+
+    import_default = builtins.__import__
+
+    def find_relative(original_file, relative_import_path, level):
+        # NOTE: "from . import x" is not handled. Because then it's unclear
+        # if such import should produce `x` as a python module or DictConfig.
+        # This can be discussed further if needed.
+        relative_import_err = (
+            "Relative import of directories is not allowed within config files. "
+            "Within a config file, relative import can only import other config files."
+        )
+        if not len(relative_import_path):
+            raise ImportError(relative_import_err)
+
+        cur_file = os.path.dirname(original_file)  # noqa: PTH120
+        for _ in range(level - 1):
+            cur_file = os.path.dirname(cur_file)  # noqa: PTH120
+        cur_name = relative_import_path.lstrip(".")
+        for part in cur_name.split("."):
+            cur_file = os.path.join(cur_file, part)  # noqa: PTH118
+        if not cur_file.endswith(".py"):
+            cur_file += ".py"
+        if not iopathlib.isfile(cur_file):
+            cur_file_no_suffix = cur_file[: -len(".py")]
+            if iopathlib.isdir(cur_file_no_suffix):
+                raise ImportError(
+                    f"Cannot import from {cur_file_no_suffix}." + relative_import_err
+                )
+            msg = (
+                f"Cannot import name {relative_import_path} from "
+                f"{original_file}: {cur_file} does not exist."
+            )
+            raise ImportError(msg)
+        return cur_file
+
+    def import_patched(name, globals=None, locals=None, fromlist=(), level=0):
+        if (
+            # Only deal with relative imports inside config files
+            level != 0
+            and globals is not None
+            and (globals.get("__package__", "") or "").startswith(PATCH_PREFIX)
+        ):
+            cur_file = find_relative(globals["__file__"], name, level)
+            laco.utils.check_syntax(cur_file)
+            spec = importlib.machinery.ModuleSpec(
+                _generate_packagename(cur_file), None, origin=cur_file
+            )
+            module = importlib.util.module_from_spec(spec)
+            module.__file__ = cur_file
+            with iopathlib.open(cur_file) as f:
+                content = f.read()
+            exec(compile(content, cur_file, "exec"), module.__dict__)
+            for name in fromlist:  # noqa: PLR1704
+                val = laco.utils.as_omegadict(module.__dict__[name])
+                module.__dict__[name] = val
+            return module
+        return import_default(name, globals, locals, fromlist=fromlist, level=level)
+
+    builtins.__import__ = import_patched
+    yield import_patched
+    builtins.__import__ = import_default
+
+
+def load_config_local(path: str):
+    """
+    Loads a configuration from a local source.
+
+    Users should prefer to load configurations via the unified API with
+    :func:`unipercept.read_config` instead of calling this method directly.
+    """
+    import laco
+    import laco.keys
+    import laco.utils
+
+    ext = os.path.splitext(path)[1]  # noqa: PTH122
+    match ext.lower():
+        case ".py":
+            laco.utils.check_syntax(path)
+
+            with _patch_import():
+                # Record the filename
+                nsp = {
+                    "__file__": path,
+                    "__package__": _generate_packagename(path),
+                }
+                with iopathlib.open(path) as f:
+                    content = f.read()
+                # Compile first with filename to:
+                # 1. make filename appears in stacktrace
+                # 2. make load_rel able to find its parent's (possibly remote) location
+                exec(compile(content, iopathlib.get_local_path(path), "exec"), nsp)
+
+            export = nsp.get(
+                "__all__",
+                (
+                    k
+                    for k, v in nsp.items()
+                    if not k.startswith("_")
+                    and (
+                        isinstance(
+                            v,
+                            dict
+                            | list
+                            | DictConfig
+                            | ListConfig
+                            | int
+                            | float
+                            | str
+                            | bool,
+                        )
+                        or v is None
+                    )
+                ),
+            )
+            obj: dict[str, typing.Any] = {k: v for k, v in nsp.items() if k in export}
+            obj.setdefault(laco.keys.CONFIG_NAME, _filepath_to_name(path))
+            obj.setdefault(laco.keys.CONFIG_VERSION, laco.__version__)
+
+        case ".yaml":
+            with iopathlib.open(path) as f:
+                obj = yaml.unsafe_load(f)
+            obj.setdefault(laco.keys.CONFIG_NAME, "unknown")
+            obj.setdefault(laco.keys.CONFIG_VERSION, "unknown")
+        case _:
+            msg = "Unsupported file extension %s!"
+            raise ValueError(msg, ext)
+
+    return laco.utils.as_omegadict(obj)
+
+
+def _filepath_to_name(path: str | iopathlib.Path) -> str | None:
+    """
+    Convert a file path to a module name.
+    """
+
+    configs_root = iopathlib.Path("./configs").resolve()
+    path = iopathlib.Path(path).resolve()
+    try:
+        name = path.relative_to(configs_root).parent.as_posix() + "/" + path.stem
+    except Exception:
+        name = "/".join([path.parent.stem, path.stem])
+
+    name = name.replace("./", "")
+    name = name.replace("//", "/")
+
+    if name in {"__init__", "defaults", "unknown", "config", "configs"}:
+        return None
+    return name.removesuffix(".py")
+
+
+def _generate_packagename(path: str):
+    # generate a random package name when loading config files
+    return PATCH_PREFIX + str(uuid4())[:4] + "." + iopathlib.Path(path).name

laco/_overrides.py

@@ -0,0 +1,53 @@
+
+from omegaconf import OmegaConf
+
+__all__ = ["apply_overrides"]
+
+def apply_overrides(cfg, overrides: list[str]):
+    """
+    In-place override contents of cfg.
+
+    Parameters
+    ----------
+    cfg
+        An omegaconf config object
+    overrides
+        List of strings in the format of "a=b" to override configs.
+        See: https://hydra.cc/docs/next/advanced/override_grammar/basic/
+
+    Returns
+    -------
+    DictConfig
+        Lazy configuration object
+    """
+
+    try:
+        from hydra.core.override_parser.overrides_parser import OverridesParser
+    except ImportError as err:
+        msg = "Hydra is not installed. Please install Hydra to use this function."
+        raise ImportError(msg) from err
+
+    def safe_update(cfg, key, value):
+        parts = key.split(".")
+        for idx in range(1, len(parts)):
+            prefix = ".".join(parts[:idx])
+            v = OmegaConf.select(cfg, prefix, default=None)
+            if v is None:
+                break
+            if not OmegaConf.is_config(v):
+                msg = (
+                    f"Trying to update key {key}, but {prefix} "
+                    f"is not a config, but has type {type(v)}."
+                )
+                raise KeyError(msg)
+        OmegaConf.update(cfg, key, value, merge=True)
+
+    for o in OverridesParser.create().parse_overrides(overrides):
+        key = o.key_or_group
+        value = o.value()
+        if o.is_delete():
+            msg = "deletion is not yet a supported override"
+            raise NotImplementedError(msg)
+        safe_update(cfg, key, value)
+
+    return cfg

laco/_resolvers.py

@@ -0,0 +1,17 @@
+import math
+
+from omegaconf import OmegaConf
+
+__all__ = []
+
+OmegaConf.register_new_resolver("sum", lambda *numbers: sum(numbers))
+OmegaConf.register_new_resolver("min", lambda *numbers: min(numbers))
+OmegaConf.register_new_resolver("max", lambda *numbers: max(numbers))
+OmegaConf.register_new_resolver("div", lambda a, b: a / b)
+OmegaConf.register_new_resolver("pow", lambda a, b: a**b)
+OmegaConf.register_new_resolver("mod", lambda a, b: a % b)
+OmegaConf.register_new_resolver("neg", lambda a: -a)
+OmegaConf.register_new_resolver("reciprocal", lambda a: 1 / a)
+OmegaConf.register_new_resolver("abs", lambda a: abs(a))
+OmegaConf.register_new_resolver("round", lambda a, b: round(a, b))
+OmegaConf.register_new_resolver("math", lambda name, *args: getattr(math, name)(args))

laco/builtins.py

@@ -0,0 +1,20 @@
+import functools
+import typing
+from dataclasses import is_dataclass
+
+from omegaconf import DictConfig
+
+import laco.utils 
+import laco.keys
+
+
+def partial(
+    **kwargs: typing.Any,
+) -> typing.Callable[..., typing.Any]:
+    cb = kwargs.get(laco.keys.LAZY_PART, None)
+    if isinstance(cb, str):
+        cb = laco.utils.locate_object(cb)
+    if not callable(cb):
+        msg = f"Expected a callable object or location (str), got {cb} (type {type(cb)}"
+        raise TypeError(msg)
+    return functools.partial(cb, **kwargs)

laco/cli.py

@@ -0,0 +1,2 @@
+def main():
+    pass

laco/env.py

@@ -0,0 +1,111 @@
+r"""
+Working with environment variables.
+"""
+
+import enum
+import functools
+import os
+import typing
+
+
+__all__ = ["fetch", "EnvFilter"]
+
+type EnvVarCompatible = int | str | bool
+
+
+class EnvFilter(enum.StrEnum):
+    STRING = enum.auto()
+    TRUTHY = enum.auto()
+    FALSY = enum.auto()
+    POSITIVE = enum.auto()
+    NEGATIVE = enum.auto()
+    NONNEGATIVE = enum.auto()
+    NONPOSITIVE = enum.auto()
+
+    @staticmethod
+    def apply(f: "EnvFilter | str | None", v: typing.Any, /) -> bool:  # noqa: PLR0911
+        if f is None:
+            return True
+        if v is None:
+            return False
+        match EnvFilter(f):
+            case EnvFilter.STRING:
+                assert isinstance(v, str)
+                v = v.lower()
+                return v != ""
+            case EnvFilter.TRUTHY:
+                return bool(v)
+            case EnvFilter.FALSY:
+                return not bool(v)
+            case EnvFilter.POSITIVE:
+                return v > 0
+            case EnvFilter.NEGATIVE:
+                return v < 0
+            case EnvFilter.NONNEGATIVE:
+                return v >= 0
+            case EnvFilter.NONPOSITIVE:
+                return v <= 0
+            case _:
+                msg = f"Invalid filter: {f!r}"
+                raise ValueError(msg)
+
+
+@typing.overload
+def fetch[_T: EnvVarCompatible](
+    __type: type[_T],
+    /,
+    *keys: str,
+    default: _T,
+    filter: EnvFilter | None = None,
+) -> _T: ...
+
+
+@typing.overload
+def fetch[_T: EnvVarCompatible](
+    __type: type[_T],
+    /,
+    *keys: str,
+    default: _T | None = None,
+    filter: EnvFilter | None = None,
+) -> _T | None: ...
+
+
+@functools.cache
+def fetch[_T: EnvVarCompatible](
+    __type: type[_T],
+    /,
+    *keys: str,
+    default: _T | None = None,
+    filter: EnvFilter | None = None,
+) -> _T | None:
+    """
+    Read an environment variable. If the variable is not set, return the default value.
+
+    If no default is given, an error is raised if the variable is not set.
+    """
+    keys_read = []
+    for k in keys:
+        keys_read.append(k)
+        v = os.getenv(k)
+        if v is None:
+            continue
+        v = strtobool(v) if issubclass(__type, bool) else __type(v)
+        if not EnvFilter.apply(filter, v):
+            continue
+        break
+    else:
+        v = default
+    return typing.cast(_T, v)
+
+
+def strtobool(v: str) -> bool:
+    """
+    Convert a string to a boolean value.
+    """
+    v = v.lower()
+    if v in {"true", "yes", "on", "1"}:
+        return True
+    if v in {"false", "no", "off", "0"}:
+        return False
+    msg = f"Invalid boolean value: {v!r}"
+    raise ValueError(msg)