netloader 3.11.0__py3-none-any.whl

netloader/utils/__init__.py

@@ -0,0 +1,46 @@
+"""
+Utility functions for PyTorch-Network-Loader
+"""
+from netloader.utils.utils import (
+    Shapes,
+    ascii_plot,
+    check_params,
+    compare_versions,
+    deep_merge,
+    dict_list_append,
+    dict_list_convert,
+    get_device,
+    label_change,
+    list_dict_convert,
+    progress_bar,
+    safe_globals,
+    save_name,
+    suppress_logger_warnings,
+)
+from netloader.utils.types import ArrayT
+from netloader.utils import configs, types, utils
+from netloader.utils.configs import BaseConfig, Config, NetConfig
+
+__all__ = [
+    'ArrayT',
+    'BaseConfig',
+    'Config',
+    'NetConfig',
+    'Shapes',
+    'ascii_plot',
+    'check_params',
+    'compare_versions',
+    'deep_merge',
+    'dict_list_append',
+    'dict_list_convert',
+    'get_device',
+    'label_change',
+    'list_dict_convert',
+    'progress_bar',
+    'safe_globals',
+    'save_name',
+    'suppress_logger_warnings',
+    'configs',
+    'types',
+    'utils',
+]

netloader/utils/configs.py

@@ -0,0 +1,241 @@
+"""
+Configuration dataclasses for the network and layers.
+"""
+import logging as log
+from dataclasses import dataclass, field
+from typing import TypeVar, Any, Literal, get_type_hints
+
+from netloader import layers
+
+
+ConfigT = TypeVar('ConfigT', bound='BaseConfig')
+
+
+@dataclass
+class BaseConfig:
+    """
+    Base dataclass to hold configuration parameters.
+    """
+    _changed_fields: set[str] = field(default_factory=set, init=False, repr=False)
+
+    def __getattr__(self, item: str) -> Any:
+        """
+        Gets an attribute of the dataclass.
+
+        Parameters
+        ----------
+        item : str
+            Key of the attribute
+
+        Returns
+        -------
+        Any
+            Value of the attribute
+        """
+        item = item.replace('-', '_')
+        return super().__getattribute__(item)
+
+    def __setattr__(self, key: str, value: Any) -> None:
+        """
+        Sets an attribute of the dataclass.
+
+        Parameters
+        ----------
+        key : str
+            Key of the attribute
+        value : Any
+            Value to set
+        """
+        if key.startswith('_'):
+            return super().__setattr__(key, value)
+
+        key = key.replace('-', '_')
+        hint: type = get_type_hints(type(self))[key]
+
+        if getattr(hint, '__origin__', None) is Literal:
+            self._check_literal(key, value, getattr(hint, '__args__', ()))
+
+        if value != self.__dataclass_fields__[key].default:
+            self._changed_fields.add(key)
+        return super().__setattr__(key, value)
+
+    @staticmethod
+    def _check_literal(key: str, value: Any, options: tuple[Any, ...]) -> None:
+        """
+        Checks if a value is in a list of options.
+
+        Parameters
+        ----------
+        key : str
+            Key of the value
+        value : Any
+            Value to check
+        options : list[Any]
+            List of options
+        """
+        if value not in options:
+            raise ValueError(f'Invalid value for {key} ({value}), must be one of {options}')
+
+    @classmethod
+    def from_dict(cls: type[ConfigT], config: dict[str, Any], new_fields: bool = False) -> ConfigT:
+        """
+        Creates a BaseConfig object from a dictionary.
+
+        Parameters
+        ----------
+        config : dict[str, Any]
+            Configuration dictionary
+        new_fields : bool, Optional
+            Whether to allow new fields in the dictionary, default = False
+
+        Returns
+        -------
+        ConfigT
+            Configuration dataclass
+        """
+        instance: ConfigT = type.__call__(cls)
+        instance.merge_dict(config, new_fields=new_fields)
+        return instance
+
+    def get_changed_fields(self) -> set[str]:
+        """
+        Gets the set of changed fields in the dataclass.
+
+        Returns
+        -------
+        set[str]
+            Set of changed fields
+        """
+        return self._changed_fields
+
+    def merge(self: ConfigT, other: ConfigT, new_fields: bool = False) -> None:
+        """
+        Merges another dataclass into this one.
+
+        Parameters
+        ----------
+        other : ConfigT
+            Configuration dataclass to merge
+        new_fields : bool, Optional
+            Whether to allow new fields in the other dataclass, default = False
+        """
+        key: str
+
+        for key in other.get_changed_fields():
+            if not hasattr(self, key) and not new_fields:
+                raise AttributeError(f'Unknown config key ({key}) and new_fields is False')
+
+            if hasattr(self, key) and isinstance(getattr(self, key), BaseConfig):
+                getattr(self, key).merge(getattr(other, key), new_fields=new_fields)
+            else:
+                setattr(self, key, getattr(other, key))
+
+    def merge_dict(self, other: dict[str, Any], new_fields: bool = False) -> None:
+        """
+        Merges a dictionary into this dataclass.
+
+        Parameters
+        ----------
+        other : dict[str, Any]
+            Configuration dictionary to merge
+        new_fields : bool, Optional
+            Whether to allow new fields in the dictionary, default = False
+        """
+        key: str
+        value: Any
+
+        for key, value in other.items():
+            key.replace('-', '_')
+
+            if not hasattr(self, key) and not new_fields:
+                raise AttributeError(f'Unknown config key ({key}) and new_fields is False')
+
+            if hasattr(self, key) and isinstance(getattr(self, key), BaseConfig):
+                getattr(self, key).merge_dict(value, new_fields=new_fields)
+            else:
+                setattr(self, key, value)
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Converts the dataclass to a dictionary.
+
+        Returns
+        -------
+        dict[str, Any]
+            Configuration dictionary
+        """
+        key: str
+        result: dict[str, Any] = {}
+        value: Any
+
+        for key, value in self.__dict__.items():
+            if key.startswith('_'):
+                continue
+            if isinstance(value, BaseConfig):
+                result[key] = value.to_dict()
+            else:
+                result[key] = value
+        return result
+
+
+@dataclass
+class NetConfig(BaseConfig):
+    """
+    Dataclass to hold global network configuration parameters.
+
+    Attributes
+    ----------
+    checkpoints : bool
+        Whether to capture intermediate outputs as checkpoints
+    paper : str
+        Reference to the paper describing the network
+    github : str
+        Link to the GitHub repository for the network
+    description : str
+        Description of the network
+    layers : dict[str, Any]
+        Dictionary of default layer configurations
+    """
+    checkpoints: bool = False
+    paper: str = ''
+    github: str = ''
+    description: str = ''
+    layers: dict[str, Any] = field(default_factory=dict)
+
+    def merge_dict(self, other: dict[str, Any], new_fields: bool = False) -> None:
+        key: str
+        value: Any
+
+        if 'layers' in other:
+            self.layers.update(other.pop('layers'))
+
+        for key, value in other.items():
+            if hasattr(layers, key) or key == 'Composite':
+                self.layers[key] = value
+            elif not hasattr(self, key) and not new_fields:
+                raise AttributeError(f'Unknown net config key ({key}) and new_fields is False')
+            elif hasattr(NetConfig, key) and isinstance(getattr(self, key), BaseConfig):
+                getattr(self, key).merge_dict(value, new_fields=new_fields)
+            elif hasattr(NetConfig, key):
+                setattr(self, key, value)
+            else:
+                log.getLogger(__name__).warning(f'Unknown net config key ({key})')
+
+
+@dataclass
+class Config(BaseConfig):
+    """
+    Dataclass to hold global network and layer configuration parameters.
+
+    Attributes
+    ----------
+    layers : list[dict[str, Any]]
+        List of layers
+    net : NetConfig
+        Network configuration
+    """
+    layers: list[dict[str, Any]] = field(default_factory=list)
+    net: NetConfig = field(default_factory=NetConfig)
+
+
+__all__ = ['BaseConfig', 'NetConfig', 'Config', 'ConfigT']

netloader/utils/transforms.py

@@ -0,0 +1,22 @@
+"""
+Deprecated module for backwards compatibility
+"""
+from warnings import warn
+
+from netloader.transforms import (
+    BaseTransform,
+    Log,
+    MinClamp,
+    MultiTransform,
+    Normalise,
+    NumpyTensor,
+)
+
+
+warn(
+    'Importing transforms from utils is deprecated, please use netloader.transforms',
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+__all__ = ['BaseTransform', 'Log', 'MinClamp', 'MultiTransform', 'Normalise', 'NumpyTensor']

netloader/utils/types.py

@@ -0,0 +1,130 @@
+"""
+Type definitions.
+"""
+from warnings import warn
+from typing import Union, TypeVar, Any, Protocol, TypeAlias, ParamSpec, TYPE_CHECKING
+
+from torch import Tensor
+from numpy import ndarray
+
+if TYPE_CHECKING:
+    from netloader.data import Data, DataList
+
+
+ShapeLike: TypeAlias = list[int] | list[list[int]]
+Param: TypeAlias = dict[str, Union[Tensor, 'Param']]
+DataLike: TypeAlias = Union[ndarray, Tensor, 'Data[ArrayCT]']
+ArrayLike: TypeAlias = ndarray | Tensor
+TensorLike: TypeAlias = Union[Tensor, 'Data[Tensor]', 'DataList[Tensor | Data[Tensor]]']
+NDArrayLike: TypeAlias = Union[ndarray, 'Data[ndarray]', 'DataList[ndarray | Data[ndarray]]']
+DataListLike: TypeAlias = Union[DataLike, 'DataList[DataLike]']
+TensorListLike: TypeAlias = Union[Tensor, 'DataList[Tensor]']
+NDArrayListLike: TypeAlias = Union[ndarray, 'DataList[ndarray]']
+
+T = TypeVar('T')
+P = ParamSpec('P')
+DataT = TypeVar('DataT', bound=DataLike)
+ArrayT = TypeVar('ArrayT', bound=ArrayLike)
+TensorT = TypeVar('TensorT', bound=TensorLike)
+NDArrayT = TypeVar('NDArrayT', bound=NDArrayLike)
+DataListT = TypeVar('DataListT', bound=DataListLike)
+TensorListT = TypeVar('TensorListT', bound=TensorListLike)
+
+LossCT = TypeVar('LossCT', float, dict[str, float])
+DataCT = TypeVar('DataCT', ndarray, Tensor, 'Data[ndarray]', 'Data[Tensor]')
+ArrayCT = TypeVar('ArrayCT', ndarray, Tensor)
+TensorLossCT = TypeVar('TensorLossCT', Tensor, dict[str, Tensor])
+
+
+class DatasetProtocol(Protocol[DataT]):
+    """
+    Protocol for datasets used in NetLoader
+
+    Attributes
+    ----------
+    extra : list[Any] | ndarray | None
+        Additional data for each sample in the dataset of length N with shape (N,...) and type Any
+    idxs : ndarray
+        Index for each sample in the dataset with shape (N) and type int
+    low_dim : ndarray | Tensor | None
+        Low dimensional data for each sample in the dataset with shape (N)
+    high_dim : ndarray | Tensor | object
+        High dimensional data for each sample in the dataset with shape (N), this is required
+    """
+    extra: list[Any] | ndarray | Tensor | None
+    idxs: ndarray
+    low_dim: DataT | None
+    high_dim: DataT | None
+
+    def __len__(self) -> int:
+        """
+        Returns the number of samples in the dataset
+
+        Returns
+        -------
+        int
+            Number of samples in the dataset
+        """
+
+    def __getitem__(self, idx: int) -> tuple[int, DataListT, DataListT, Any]:
+        """
+        Parameters
+        ----------
+        idx : int
+            Sample index
+
+        Returns
+        -------
+        tuple[int, DataListT, DataListT, Any]
+            Sample index, low dimensional data, high dimensional data, and extra data
+        """
+
+    def step(self, epoch: float) -> None:
+        """
+        Step the dataset for each training iteration.
+
+        Parameters
+        ----------
+        epoch : float
+            Current epoch number
+        """
+
+
+DatasetT = TypeVar('DatasetT', bound=DatasetProtocol)
+
+
+__all__ = [
+    'DatasetProtocol',
+    'Param',
+    'DataLike',
+    'ShapeLike',
+    'ArrayLike',
+    'TensorLike',
+    'NDArrayLike',
+    'DataListLike',
+    'TensorListLike',
+    'NDArrayListLike',
+    'T',
+    'P',
+    'DataT',
+    'ArrayT',
+    'TensorT',
+    'NDArrayT',
+    'DatasetT',
+    'DataListT',
+    'TensorListT',
+    'LossCT',
+    'DataCT',
+    'ArrayCT',
+    'TensorLossCT',
+]
+
+def __getattr__(name: str) -> Any:
+    if name == 'ArrayTC':
+        warn(
+            'ArrayTC is deprecated, use ArrayCT instead',
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return ArrayCT
+    raise AttributeError(f"module {__name__} has no attribute {name}")