mlbricks-core 0.1.1__tar.gz → 0.2.0__tar.gz

{mlbricks_core-0.1.1 → mlbricks_core-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlbricks-core
-Version: 0.1.1
+Version: 0.2.0
 Summary: Core ML tooling, including config resolution, registry, and storage.
 Author: Shivam Verma
 Author-email: Shivam Verma <shivam.vvm.verma@gmail.com>
@@ -65,7 +65,7 @@ seed: 0
 import torch.nn as nn
 from pydantic import PositiveInt
 import yaml
-from mlbricks import Configurable, Registry, resolve
+from mlbricks import Configurable, Registry, resolve, to_dict
 
 REGISTRY = Registry()
 MODEL_REGISTRY = REGISTRY.subgroup("models")
@@ -80,6 +80,7 @@ class MLP(nn.Module, Configurable):
 
     def __init__(self, cfg: "MLP.Config") -> None:
         super().__init__()
+        self.hparams = to_dict(cfg, registry=REGISTRY)
         self.layers = nn.Sequential(
             nn.Linear(cfg.in_dim, cfg.hidden_dim),
             nn.ReLU(),
@@ -93,6 +94,8 @@ class MLP(nn.Module, Configurable):
 raw = yaml.safe_load(open("config.yaml"))  # or OmegaConf.to_object(OmegaConf.load(...))
 cfg = resolve(raw, registry=REGISTRY)
 model = cfg["model"].build()
+print(model.hparams)
+# {'_registry_': 'models.mlp', 'in_dim': 784, 'hidden_dim': 128, 'out_dim': 10}
 ```
 
 ## Installation

{mlbricks_core-0.1.1 → mlbricks_core-0.2.0}/README.md

@@ -52,7 +52,7 @@ seed: 0
 import torch.nn as nn
 from pydantic import PositiveInt
 import yaml
-from mlbricks import Configurable, Registry, resolve
+from mlbricks import Configurable, Registry, resolve, to_dict
 
 REGISTRY = Registry()
 MODEL_REGISTRY = REGISTRY.subgroup("models")
@@ -67,6 +67,7 @@ class MLP(nn.Module, Configurable):
 
     def __init__(self, cfg: "MLP.Config") -> None:
         super().__init__()
+        self.hparams = to_dict(cfg, registry=REGISTRY)
         self.layers = nn.Sequential(
             nn.Linear(cfg.in_dim, cfg.hidden_dim),
             nn.ReLU(),
@@ -80,6 +81,8 @@ class MLP(nn.Module, Configurable):
 raw = yaml.safe_load(open("config.yaml"))  # or OmegaConf.to_object(OmegaConf.load(...))
 cfg = resolve(raw, registry=REGISTRY)
 model = cfg["model"].build()
+print(model.hparams)
+# {'_registry_': 'models.mlp', 'in_dim': 784, 'hidden_dim': 128, 'out_dim': 10}
 ```
 
 ## Installation

{mlbricks_core-0.1.1 → mlbricks_core-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mlbricks-core"
-version = "0.1.1"
+version = "0.2.0"
 description = "Core ML tooling, including config resolution, registry, and storage."
 readme = "README.md"
 license = "MIT"

{mlbricks_core-0.1.1 → mlbricks_core-0.2.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "mlbricks-core"
-version = "0.1.1"
+version = "0.2.0"
 description = "Core ML tooling, including config resolution, registry, and storage."
 readme = "README.md"
 license = "MIT"

{mlbricks_core-0.1.1 → mlbricks_core-0.2.0}/src/mlbricks/__init__.py

@@ -1,5 +1,7 @@
 from mlbricks.cache import StorageCache
 from mlbricks.configurable import (
+    UNSET,
+    BuildTimeField,
     ConfigFieldNameClashError,
     Configurable,
     ConfigurableBuildError,
@@ -19,6 +21,11 @@ from mlbricks.featurization import (
     TransformHooks,
     TransformPipeline,
 )
+from mlbricks.magic_registry import (
+    MagicRegistryError,
+    MagicRegistryImportError,
+    MagicRegistryUnknownFieldError,
+)
 from mlbricks.registry import (
     DuplicateRegistrationError,
     InvalidRegistryNameError,
@@ -36,6 +43,8 @@ from mlbricks.storage import StorageDriver
 from mlbricks.utils import path_slug, uri_slug
 
 __all__ = [
+    "UNSET",
+    "BuildTimeField",
     "ConfigFieldNameClashError",
     "ConfigInstantiationError",
     "ConfigSerializationError",
@@ -47,6 +56,9 @@ __all__ = [
     "Context",
     "DuplicateRegistrationError",
     "InvalidRegistryNameError",
+    "MagicRegistryError",
+    "MagicRegistryImportError",
+    "MagicRegistryUnknownFieldError",
     "MissingCfgParameterError",
     "MissingConfigError",
     "NamedTransformStep",

{mlbricks_core-0.1.1 → mlbricks_core-0.2.0}/src/mlbricks/configurable.py

@@ -1,11 +1,14 @@
 import inspect
 import types
-from typing import Any, ClassVar
+from typing import Any, ClassVar, Self
 
 import pydantic
 from pydantic import BaseModel, ConfigDict, ValidationInfo, model_validator
+from pydantic.fields import FieldInfo
 
 __all__ = [
+    "UNSET",
+    "BuildTimeField",
     "ConfigFieldNameClashError",
     "ConfigValidationError",
     "Configurable",
@@ -86,6 +89,55 @@ def build_validator(func: types.FunctionType) -> Any:
     return model_validator(mode="after")(wrapper)
 
 
+class _Unset:
+    """Sentinel for a BuildTimeField that has not been resolved yet.
+
+    Distinct from None so a BuildTimeField[T] can hold a T that is itself
+    None. Never constructed outside this module.
+    """
+
+    __slots__ = ()
+
+    def __repr__(self) -> str:
+        return "UNSET"
+
+
+UNSET = _Unset()
+
+
+# Config field annotation for build-time-only, non-serializable values.
+#
+# Never required at Config construction, never emitted by
+# resolve_config.to_dict(), and required at build() time unless the field is
+# given its own default (same precedence as any pydantic field). Declare the
+# "= UNSET" default explicitly -- it is not implicit.
+#
+# Example:
+#   class Trainer(Configurable):
+#       class Config(Configurable.Config["Trainer"]):
+#           dataloader: BuildTimeField[DataLoader] = UNSET
+#
+#       def __init__(self, cfg: "Trainer.Config") -> None:
+#           self.loader = cfg.dataloader  # plain DataLoader, no unwrap
+#
+#   cfg = Trainer.Config()
+#   trainer = cfg.build(dataloader=my_loader)
+type BuildTimeField[T] = T | _Unset
+
+
+def is_build_time_field(field: FieldInfo) -> bool:
+    """Return True if field's class-declared default is the BuildTimeField sentinel.
+
+    Checks the declared default (``FieldInfo.default``), not the current
+    instance value, so this stays correct after build() resolves a real value
+    on a validated Config instance.
+
+    Used by build()'s automatic unresolved-field check and by
+    resolve_config.to_dict() to skip these fields during serialization.
+    """
+    return field.default is UNSET
+
+
 class Configurable:
     """Mixin for classes that are constructed from a nested Config pydantic model.
 
@@ -159,6 +211,17 @@ class Configurable:
             except pydantic.ValidationError as exc:
                 raise ConfigValidationError(exc) from exc
 
+        @build_validator
+        def _check_build_time_fields(self) -> Self:
+            unresolved = [
+                name
+                for name, field in type(self).model_fields.items()
+                if is_build_time_field(field) and getattr(self, name) is UNSET
+            ]
+            if unresolved:
+                raise ValueError(f"BuildTimeField(s) required at build(): {unresolved}")
+            return self
+
         def build(self, **kwargs: Any) -> T:
             """Construct the owning class from this config.
 

mlbricks_core-0.2.0/src/mlbricks/magic_registry.py

@@ -0,0 +1,204 @@
+import importlib
+import inspect
+import typing
+from collections.abc import Iterable
+from typing import Any, cast
+
+from pydantic import ConfigDict, create_model
+
+from mlbricks.configurable import Configurable
+
+__all__ = [
+    "MagicRegistryError",
+    "MagicRegistryImportError",
+    "MagicRegistryUnknownFieldError",
+    "magic_wrap",
+]
+
+
+class MagicRegistryError(ValueError):
+    """Base for magic-registry resolution errors."""
+
+
+class MagicRegistryImportError(MagicRegistryError):
+    """Raised when a dotted path cannot be imported or resolved to a class."""
+
+
+class MagicRegistryUnknownFieldError(MagicRegistryError):
+    """Raised when a field key has no matching __init__ parameter on the
+    target class and the target has no **kwargs catch-all."""
+
+
+def _import_dotted(path: str) -> type:
+    """Import a dotted path to a class, trying the longest module prefix first.
+
+    Args:
+        path: A dotted path, e.g. "torch.optim.Adam".
+
+    Returns:
+        The resolved class.
+
+    Raises:
+        MagicRegistryImportError: If no prefix of `path` imports as a module,
+            an attribute in the remaining chain is missing, or the resolved
+            object is not a class.
+    """
+    parts = path.split(".")
+
+    for split in range(len(parts) - 1, 0, -1):
+        module_path = ".".join(parts[:split])
+        try:
+            obj: Any = importlib.import_module(module_path)
+        except ImportError:
+            continue
+
+        try:
+            for attr in parts[split:]:
+                obj = getattr(obj, attr)
+        except AttributeError as exc:
+            raise MagicRegistryImportError(
+                f"'{path}' does not resolve -- '{module_path}' has no"
+                f" attribute chain {'.'.join(parts[split:])!r}"
+            ) from exc
+
+        if not isinstance(obj, type):
+            raise MagicRegistryImportError(f"'{path}' does not resolve to a class")
+
+        return obj
+
+    raise MagicRegistryImportError(f"cannot import any module prefix of '{path}'")
+
+
+class _MagicConfigBase(Configurable.Config):
+    model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid")
+
+
+def magic_wrap(path: str, field_keys: Iterable[str]) -> type[Configurable]:
+    """Build a Configurable wrapper subclassing the class at `path`.
+
+    The wrapper's Config declares exactly `field_keys` as fields, typed from
+    the target's own __init__ signature where possible. Anything not in
+    field_keys is never a Config field -- pass it at build() time instead,
+    same as any other Configurable's owner kwargs (e.g. build(params=...)).
+
+    Stamped with `_magic_path_` (kept separate from Registry's
+    `_registry_path_` -- curated and magic entries are distinct namespaces,
+    see docs/registry.md) so resolve_config.to_dict() can round-trip it as
+    `_magic_registry_`.
+
+    No caching: each call builds a fresh class. importlib already memoizes
+    the underlying import; a small pydantic model + dynamic subclass is
+    cheap enough to build per config node.
+
+    Args:
+        path: Dotted import path to the target class, e.g. "torch.optim.Adam".
+        field_keys: Names of the config node's sibling keys -- the fields
+            this Config should declare.
+
+    Returns:
+        A new Configurable subclass of the target class.
+
+    Raises:
+        MagicRegistryImportError: If `path` can't be imported / isn't a class.
+        MagicRegistryUnknownFieldError: If a field key has no matching
+            __init__ parameter and the target has no **kwargs catch-all.
+    """
+    target_cls = _import_dotted(path)
+    field_keys = list(field_keys)
+    field_set = set(field_keys)
+    fields = field_specs(target_cls, field_keys)
+
+    config_cls = create_model(
+        f"Magic{target_cls.__name__}Config",
+        __base__=_MagicConfigBase,
+        **fields,
+    )  # ty: ignore[no-matching-overload]
+
+    def __init__(self: Any, cfg: Any, **kwargs: Any) -> None:
+        field_values = {name: getattr(cfg, name) for name in type(cfg).model_fields}
+        target_cls.__init__(self, **field_values, **kwargs)
+
+    # build() infers legal extra ("owner") kwargs from __init__'s *signature*,
+    # not from the **kwargs catch-all above -- expose target_cls's remaining
+    # real params (everything not already a Config field) so e.g.
+    # build(params=...) is recognized instead of rejected as unexpected.
+    try:
+        target_sig = inspect.signature(target_cls.__init__)
+        remaining = [
+            p
+            for name, p in target_sig.parameters.items()
+            if name not in field_set and name != "self"
+        ]
+        __init__.__signature__ = inspect.Signature(  # ty: ignore[unresolved-attribute]
+            [
+                inspect.Parameter("self", inspect.Parameter.POSITIONAL_OR_KEYWORD),
+                inspect.Parameter("cfg", inspect.Parameter.POSITIONAL_OR_KEYWORD),
+                *remaining,
+            ]
+        )
+    except (TypeError, ValueError):
+        pass  # unintrospectable __init__ -- fall back to bare **kwargs signature
+
+    wrapper_cls = type(
+        f"Magic{target_cls.__name__}",
+        (target_cls, Configurable),
+        {"Config": config_cls, "__init__": __init__, "_magic_path_": path},
+    )
+    return cast(type[Configurable], wrapper_cls)
+
+
+def field_specs(
+    target_cls: type, field_keys: Iterable[str]
+) -> dict[str, tuple[Any, Any]]:
+    """Build create_model field specs for exactly the given keys.
+
+    Types come from target_cls.__init__'s own annotations when resolvable;
+    falls back to Any per-field, or wholesale if the signature itself can't
+    be introspected (some C-extension __init__s raise on inspect.signature).
+
+    Args:
+        target_cls: The class whose __init__ is introspected.
+        field_keys: Field names to build specs for -- exactly the keys
+            present in the config node, not the full signature.
+
+    Returns:
+        Dict mapping field name to a (type, default) tuple for
+        pydantic.create_model, all required (default is `...`).
+
+    Raises:
+        MagicRegistryUnknownFieldError: If a key has no matching parameter
+            on target_cls.__init__ and there's no **kwargs catch-all.
+    """
+    try:
+        sig = inspect.signature(target_cls.__init__)
+    except (TypeError, ValueError):
+        return dict.fromkeys(field_keys, (Any, ...))
+
+    try:
+        hints = typing.get_type_hints(target_cls.__init__)
+    except Exception:
+        hints = {}
+
+    has_var_keyword = any(
+        p.kind is inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values()
+    )
+
+    specs: dict[str, tuple[Any, Any]] = {}
+    for key in field_keys:
+        param = sig.parameters.get(key)
+
+        if param is None:
+            if has_var_keyword:
+                specs[key] = (Any, ...)
+                continue
+            raise MagicRegistryUnknownFieldError(
+                f"{target_cls.__module__}.{target_cls.__qualname__} has no"
+                f" '{key}' parameter"
+            )
+
+        annotation = hints.get(key, param.annotation)
+        if annotation is inspect.Parameter.empty or isinstance(annotation, str):
+            annotation = Any
+        specs[key] = (annotation, ...)
+
+    return specs

{mlbricks_core-0.1.1 → mlbricks_core-0.2.0}/src/mlbricks/registry.py

@@ -39,8 +39,12 @@ class Registry[T]:
         True
     """
 
-    def __init__(self) -> None:
+    def __init__(
+        self, *, prefix: str = "", root: "Registry[Any] | None" = None
+    ) -> None:
         self._entries: dict[str, type[Any] | Registry[Any]] = {}
+        self._prefix = prefix
+        self._root = root if root is not None else self
 
     def subgroup[S](self, name: str) -> "Registry[S]":
         """Create a named child registry and store it under ``name``.
@@ -61,7 +65,8 @@ class Registry[T]:
         if name in self._entries:
             raise DuplicateRegistrationError(f"'{name}' is already registered")
 
-        child: Registry[S] = Registry()
+        full_prefix = f"{self._prefix}.{name}" if self._prefix else name
+        child: Registry[S] = Registry(prefix=full_prefix, root=self._root)
         self._entries[name] = child
 
         return child
@@ -73,20 +78,44 @@ class Registry[T]:
             name: Single-segment name. Must not contain a period.
 
         Returns:
-            A decorator that stores the class and returns it unchanged.
+            A decorator that stamps the class's dotted registry path onto
+            ``cls._registry_path_``, stores it, and returns it unchanged.
 
         Raises:
             InvalidRegistryNameError: If ``name`` contains a period.
             DuplicateRegistrationError: If ``name`` is already registered in
                 this registry (raised at factory-call time, before the
-                decorated class is defined).
+                decorated class is defined), or if ``cls`` is already
+                reachable at another path within this registry's tree
+                (raised at decoration time). A stamp left over from an
+                unrelated, independently-constructed ``Registry()`` --
+                e.g. a fresh registry built per test -- does not count,
+                since it no longer resolves to anything.
         """
         _validate_name(name)
 
         if name in self._entries:
             raise DuplicateRegistrationError(f"'{name}' is already registered")
 
+        full_path = f"{self._prefix}.{name}" if self._prefix else name
+
         def decorator(cls: type[S]) -> type[S]:
+            existing_path = getattr(cls, "_registry_path_", None)
+            if existing_path is not None:
+                # existing_path may belong to an unrelated Registry() tree
+                # (e.g. a fresh registry built per test) that simply hasn't
+                # reached this path yet -- that's not a real duplicate.
+                try:
+                    still_live = self._root.get(existing_path) is cls
+                except RegistryKeyError:
+                    still_live = False
+                if still_live:
+                    raise DuplicateRegistrationError(
+                        f"{cls.__name__} is already registered at"
+                        f" '{existing_path}' -- cannot also register it at"
+                        f" '{full_path}'"
+                    )
+            cast(Any, cls)._registry_path_ = full_path
             self._entries[name] = cls
             return cls
 
@@ -183,8 +212,9 @@ class Registry[T]:
     def path_of(self, cls: type) -> str | None:
         """Return the dotted registry path for ``cls``, or ``None`` if not registered.
 
-        Performs a depth-first tree-walk. Useful for serializing a registered
-        class back to its declarative ``_registry_`` key.
+        Reads the path stamped on ``cls`` by ``register()`` and confirms it
+        resolves back to ``cls`` within this registry -- O(path depth), not
+        O(registry size).
 
         Args:
             cls: The class to look up.
@@ -201,32 +231,14 @@ class Registry[T]:
             >>> reg.path_of(MLP)
             'models.mlp'
         """
-        return self._find_path(cls, prefix="")
-
-    def _find_path(self, cls: type, prefix: str) -> str | None:
-        """Recursively search for a class in the registry tree.
-
-        Args:
-            cls: The class to look up.
-            prefix: Dot-separated path prefix accumulated by parent calls.
+        path = getattr(cls, "_registry_path_", None)
+        if path is None:
+            return None
 
-        Returns:
-            Dotted path string if found, or ``None`` if not found.
-        """
-        for name, entry in self._entries.items():
-            full_name = f"{prefix}.{name}" if prefix else name
-
-            if entry is cls:
-                # found answer
-                return full_name
-
-            if isinstance(entry, Registry):
-                result = entry._find_path(cls, full_name)
-                if result is not None:
-                    # pass solution up
-                    return result
-
-        return None
+        try:
+            return path if self.get(path) is cls else None
+        except RegistryKeyError:
+            return None
 
 
 def _validate_name(name: str) -> None:

{mlbricks_core-0.1.1 → mlbricks_core-0.2.0}/src/mlbricks/resolve_config.py

@@ -4,7 +4,8 @@ from typing import Any
 
 from pydantic import BaseModel
 
-from mlbricks.configurable import Configurable
+from mlbricks.configurable import Configurable, is_build_time_field
+from mlbricks.magic_registry import magic_wrap
 from mlbricks.registry import Registry
 
 __all__ = [
@@ -61,8 +62,11 @@ def parse(
         A top-level _registry_ dict is itself replaced by a ParsedNode.
 
     Raises:
-        RegistryParseError: If a _registry_ value is not a string.
+        RegistryParseError: If a _registry_ value is not a string, or a node
+            has both _registry_ and _magic_registry_ keys.
         RegistryKeyError: If a _registry_ key is not found in the registry.
+        MagicRegistryError: If a _magic_registry_ path can't be imported or a
+            field doesn't match the target's __init__.
 
     Example:
         >>> result = parse({"_registry_": "models.mlp", "dim": 64}, registry=reg)
@@ -80,10 +84,24 @@ def _parse_dict(
     node: dict[str, Any],
     registry: Registry[Any],
 ) -> dict[str, Any] | ParsedNode:
-    # plain dict -- recurse into values without creating a ParsedNode
-    if "_registry_" not in node:
-        return {k: parse(v, registry) for k, v in node.items()}
-
+    match "_registry_" in node, "_magic_registry_" in node:
+        case True, True:
+            raise RegistryParseError(
+                "node cannot have both '_registry_' and '_magic_registry_' keys"
+            )
+        case False, False:
+            # plain dict -- recurse into values without creating a ParsedNode
+            return {k: parse(v, registry) for k, v in node.items()}
+        case False, True:
+            return _parse_magic_dict(node, registry)
+        case True, False:
+            return _parse_registry_dict(node, registry)
+
+
+def _parse_registry_dict(
+    node: dict[str, Any],
+    registry: Registry[Any],
+) -> ParsedNode:
     key = node["_registry_"]
     if not isinstance(key, str):
         raise RegistryParseError(
@@ -99,6 +117,24 @@ def _parse_dict(
     return ParsedNode(key=key, cls=cls, kwargs=kwargs)
 
 
+def _parse_magic_dict(
+    node: dict[str, Any],
+    registry: Registry[Any],
+) -> ParsedNode:
+    path = node["_magic_registry_"]
+    if not isinstance(path, str):
+        raise RegistryParseError(
+            f"_magic_registry_ value must be a string, got {type(path).__name__!r}"
+        )
+
+    kwargs = {k: parse(v, registry) for k, v in node.items() if k != "_magic_registry_"}
+
+    # MagicRegistryError propagates as-is
+    cls = magic_wrap(path, kwargs.keys())
+
+    return ParsedNode(key=path, cls=cls, kwargs=kwargs)
+
+
 def instantiate(parsed: Any) -> Any:
     """Construct all ParsedNodes in a parsed value into Python objects.
 
@@ -232,6 +268,10 @@ def _config_to_dict(
             "Config has no owner -- define it inside a Configurable subclass"
         )
 
+    magic_path = getattr(owner, "_magic_path_", None)
+    if magic_path is not None:
+        return _dump_fields(cfg, registry, key="_magic_registry_", path=magic_path)
+
     path = registry.path_of(owner)
     if path is None:
         raise ConfigSerializationError(
@@ -239,11 +279,17 @@ def _config_to_dict(
             " -- cannot serialize"
         )
 
-    result: dict[str, Any] = {"_registry_": path}
+    return _dump_fields(cfg, registry, key="_registry_", path=path)
 
-    for name in type(cfg).model_fields:
-        result[name] = to_dict(getattr(cfg, name), registry)
 
+def _dump_fields(
+    cfg: Configurable.Config, registry: Registry[Any], *, key: str, path: str
+) -> dict[str, Any]:
+    result: dict[str, Any] = {key: path}
+    for name, field in type(cfg).model_fields.items():
+        if is_build_time_field(field):
+            continue
+        result[name] = to_dict(getattr(cfg, name), registry)
     return result