nshtrainer 0.10.9__py3-none-any.whl → 0.10.11__py3-none-any.whl

nshtrainer/_checkpoint/metadata.py

@@ -1,6 +1,8 @@
 import copy
 import datetime
 import logging
+import shutil
+from collections.abc import Callable
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, cast
 
@@ -100,3 +102,74 @@ def _write_checkpoint_metadata(
         log.warning(f"Failed to write hparams to {checkpoint_path}: {e}")
     else:
         log.info(f"Checkpoint metadata written to {checkpoint_path}")
+
+
+def _remove_checkpoint_metadata(checkpoint_path: Path):
+    for path in (
+        checkpoint_path.with_suffix(METADATA_PATH_SUFFIX),
+        checkpoint_path.with_suffix(HPARAMS_PATH_SUFFIX),
+    ):
+        try:
+            path.unlink(missing_ok=True)
+        except Exception as e:
+            log.warning(f"Failed to remove {path}: {e}")
+        else:
+            log.info(f"Removed {path}")
+
+
+def _link_checkpoint_metadata(checkpoint_path: Path, linked_checkpoint_path: Path):
+    # First, remove any existing metadata files
+    _remove_checkpoint_metadata(linked_checkpoint_path)
+
+    # Link the metadata files to the new checkpoint
+    for path in (
+        checkpoint_path.with_suffix(METADATA_PATH_SUFFIX),
+        checkpoint_path.with_suffix(HPARAMS_PATH_SUFFIX),
+    ):
+        linked_path = linked_checkpoint_path.with_suffix(path.suffix)
+        try:
+            try:
+                linked_path.symlink_to(path)
+            except OSError:
+                # on Windows, special permissions are required to create symbolic links as a regular user
+                # fall back to copying the file
+                shutil.copy(path, linked_path)
+        except Exception as e:
+            log.warning(f"Failed to link {path} to {linked_path}: {e}")
+        else:
+            log.info(f"Linked {path} to {linked_path}")
+
+
+def _checkpoint_sort_key_fn(key: Callable[[CheckpointMetadata, Path], Any]):
+    def sort_key_fn(checkpoint_path: Path):
+        if not (p := checkpoint_path.with_suffix(METADATA_PATH_SUFFIX)).exists():
+            raise FileNotFoundError(f"Metadata file not found: {p}")
+
+        nonlocal key
+        return key(CheckpointMetadata.from_file(p), p)
+
+    return sort_key_fn
+
+
+def _sort_ckpts_by_metadata(
+    checkpoint_paths: list[Path],
+    key: Callable[[CheckpointMetadata, Path], Any],
+    fallback_key: Callable[[Path], Any],
+):
+    # First, let's make sure all the metadata files exist.
+    # If not, use the fallback function to sort the checkpoints.
+    no_metadata_paths: list[Path] = []
+    for path in checkpoint_paths:
+        if (path.with_suffix(METADATA_PATH_SUFFIX)).exists():
+            continue
+
+        no_metadata_paths.append(path)
+
+    if no_metadata_paths:
+        log.warning(
+            f"Metadata file not found on {len(no_metadata_paths)} checkpoints: {no_metadata_paths}\n"
+            "Falling back to sorting by last modified time."
+        )
+        return sorted(checkpoint_paths, key=fallback_key)
+
+    return sorted(checkpoint_paths, key=_checkpoint_sort_key_fn(key))

nshtrainer/_checkpoint/saver.py

@@ -0,0 +1,52 @@
+import os
+import shutil
+from pathlib import Path
+
+from lightning.pytorch import Trainer
+
+from .metadata import _link_checkpoint_metadata, _remove_checkpoint_metadata
+
+
+def _link_checkpoint(
+    trainer: Trainer,
+    filepath: str | Path | os.PathLike,
+    linkpath: str | Path | os.PathLike,
+    *,
+    barrier: bool,
+    metadata: bool,
+):
+    if not isinstance(filepath, Path):
+        filepath = Path(filepath)
+    if not isinstance(linkpath, Path):
+        linkpath = Path(linkpath)
+
+    if trainer.is_global_zero:
+        if linkpath.exists():
+            if linkpath.is_symlink() or linkpath.is_file():
+                linkpath.unlink()
+            elif linkpath.is_dir():
+                shutil.rmtree(linkpath)
+            _remove_checkpoint_metadata(linkpath)
+
+        try:
+            target_path = filepath.relative_to(linkpath.parent)
+            linkpath.symlink_to(target_path)
+        except OSError:
+            # on Windows, special permissions are required to create symbolic links as a regular user
+            # fall back to copying the file
+            shutil.copy(filepath, linkpath)
+
+        _link_checkpoint_metadata(filepath, linkpath)
+    if barrier:
+        trainer.strategy.barrier()
+
+
+def _remove_checkpoint(
+    trainer: Trainer,
+    filepath: str | Path | os.PathLike,
+    remove_metadata: bool = True,
+):
+    if not isinstance(filepath, Path):
+        filepath = Path(filepath)
+    trainer.strategy.remove_checkpoint(filepath)
+    _remove_checkpoint_metadata(filepath)

nshtrainer/callbacks/early_stopping.py

@@ -1,5 +1,5 @@
+import logging
 import math
-from logging import getLogger
 
 from lightning.fabric.utilities.rank_zero import _get_rank
 from lightning.pytorch import Trainer
@@ -7,7 +7,7 @@ from lightning.pytorch.callbacks import EarlyStopping as _EarlyStopping
 from lightning.pytorch.utilities.rank_zero import rank_prefixed_message
 from typing_extensions import override
 
-log = getLogger(__name__)
+log = logging.getLogger(__name__)
 
 
 class EarlyStopping(_EarlyStopping):

nshtrainer/callbacks/finite_checks.py

@@ -1,4 +1,4 @@
-from logging import getLogger
+import logging
 from typing import Literal
 
 import torch
@@ -7,7 +7,7 @@ from typing_extensions import override
 
 from .base import CallbackConfigBase
 
-log = getLogger(__name__)
+log = logging.getLogger(__name__)
 
 
 def finite_checks(

nshtrainer/callbacks/latest_epoch_checkpoint.py

@@ -6,6 +6,8 @@ from lightning.pytorch import LightningModule, Trainer
 from lightning.pytorch.callbacks import Checkpoint
 from typing_extensions import override
 
+from .._checkpoint.metadata import _sort_ckpts_by_metadata
+from .._checkpoint.saver import _link_checkpoint, _remove_checkpoint
 from .base import CallbackConfigBase
 
 log = logging.getLogger(__name__)
@@ -17,15 +19,18 @@ class LatestEpochCheckpointCallbackConfig(CallbackConfigBase):
     dirpath: str | Path | None = None
     """Directory path to save the checkpoint file."""
 
-    filename: str = "latest_epoch{epoch:02d}_step{step:04d}.ckpt"
+    filename: str = "epoch{epoch:02d}_step{step:04d}"
     """Checkpoint filename. This must not include the extension."""
 
     save_weights_only: bool = False
     """Whether to save only the model's weights or the entire model object."""
 
-    latest_symlink_filename: str | None = "latest.ckpt"
+    latest_symlink_filename: str | None = "latest"
     """Filename for the latest symlink. If None, no symlink will be created."""
 
+    latest_k: int | Literal["all"] = 1
+    """Number of latest checkpoints to keep. If "all", all checkpoints are kept."""
+
     @override
     def create_callbacks(self, root_config):
         dirpath = self.dirpath or root_config.directory.resolve_subdirectory(
@@ -37,38 +42,73 @@ class LatestEpochCheckpointCallbackConfig(CallbackConfigBase):
 
 
 class LatestEpochCheckpoint(Checkpoint):
+    PREFIX = "latest_"
+    EXTENSION = ".ckpt"
+
     def __init__(self, config: LatestEpochCheckpointCallbackConfig, dirpath: Path):
         super().__init__()
 
         self.config = config
         self.dirpath = dirpath
 
+    @override
+    def on_train_epoch_end(self, trainer: Trainer, pl_module: LightningModule):
+        self._save_new_checkpoint(trainer)
+
+    def _latest_symlink_filename(self):
+        if (filename := self.config.latest_symlink_filename) is None:
+            return None
+        return f"{filename}{self.EXTENSION}"
+
     def _ckpt_path(self, trainer: Trainer):
-        return self.dirpath / self.config.filename.format(
+        filename = self.config.filename.format(
             epoch=trainer.current_epoch, step=trainer.global_step
         )
+        filename = f"{self.PREFIX}{filename}.{self.EXTENSION}"
+        return self.dirpath / filename
+
+    def _remove_checkpoints(self, trainer: Trainer, ckpt_paths: list[Path]):
+        for ckpt_path in ckpt_paths:
+            _remove_checkpoint(trainer, ckpt_path, remove_metadata=True)
+
+    def _remove_old_checkpoints(self, trainer: Trainer):
+        if (latest_k := self.config.latest_k) == "all":
+            return
+
+        # Get all configs, ignoring the latest symlink
+        ckpt_paths = list(self.dirpath.glob(f"{self.PREFIX}*{self.EXTENSION}"))
+        # Ignore the latest symlink
+        if (latest_symlink_filename := self._latest_symlink_filename()) is not None:
+            ckpt_paths = [p for p in ckpt_paths if p.name != latest_symlink_filename]
+
+        # Sort by epoch, then step, then last modified
+        ckpt_paths = _sort_ckpts_by_metadata(
+            ckpt_paths,
+            key=lambda meta, p: (meta.epoch, meta.global_step, p.stat().st_mtime),
+            fallback_key=lambda p: p.stat().st_mtime,
+            # ^ Called if metadata is not found on all checkpoints
+        )
+
+        # Remove all but the latest k checkpoints
+        ckpts_to_remove = ckpt_paths[:-latest_k]
+        self._remove_checkpoints(trainer, ckpts_to_remove)
+
+    def _save_new_checkpoint(self, trainer: Trainer):
+        # Remove old checkpoints
+        self._remove_old_checkpoints(trainer)
 
-    @override
-    def on_train_epoch_end(self, trainer: Trainer, pl_module: LightningModule):
         # Save the new checkpoint
         filepath = self._ckpt_path(trainer)
         trainer.save_checkpoint(filepath, self.config.save_weights_only)
 
         # Create the latest symlink
-        if (
-            trainer.is_global_zero
-            and (symlink_filename := self.config.latest_symlink_filename) is not None
-        ):
+        if (symlink_filename := self._latest_symlink_filename()) is not None:
             symlink_path = self.dirpath / symlink_filename
-            symlink_path.unlink(missing_ok=True)
-            symlink_path.symlink_to(filepath.name)
+            _link_checkpoint(
+                trainer,
+                filepath,
+                symlink_path,
+                barrier=True,
+                metadata=True,
+            )
             log.info(f"Created latest symlink: {symlink_path}")
-
-    def latest_checkpoint(self):
-        if (symlink_filename := self.config.latest_symlink_filename) is None:
-            return None
-
-        if not (symlink_path := self.dirpath / symlink_filename).exists():
-            return None
-
-        return symlink_path

nshtrainer/callbacks/model_checkpoint.py

@@ -1,21 +1,23 @@
+import logging
 import re
 from datetime import timedelta
-from logging import getLogger
 from pathlib import Path
 from typing import TYPE_CHECKING, Literal
 
+from lightning.pytorch import Trainer
 from lightning.pytorch.callbacks.model_checkpoint import (
     ModelCheckpoint as _ModelCheckpoint,
 )
 from typing_extensions import override
 
+from .._checkpoint.saver import _link_checkpoint, _remove_checkpoint
 from ..metrics import MetricConfig
 from .base import CallbackConfigBase
 
 if TYPE_CHECKING:
     from ..model.config import BaseConfig
 
-log = getLogger(__name__)
+log = logging.getLogger(__name__)
 
 
 def _convert_string(input_string: str):
@@ -158,6 +160,8 @@ class ModelCheckpointCallbackConfig(CallbackConfigBase):
 
 
 class ModelCheckpoint(_ModelCheckpoint):
+    CHECKPOINT_NAME_LAST = "best"
+
     @override
     def __init__(
         self,
@@ -185,3 +189,17 @@ class ModelCheckpoint(_ModelCheckpoint):
             save_on_train_epoch_end=self.config.save_on_train_epoch_end,
             enable_version_counter=self.config.enable_version_counter,
         )
+
+    @override
+    def _link_checkpoint(self, trainer: Trainer, filepath: str, linkpath: str):  # pyright: ignore[reportIncompatibleMethodOverride]
+        return _link_checkpoint(
+            trainer,
+            filepath,
+            linkpath,
+            barrier=True,
+            metadata=True,
+        )
+
+    @override
+    def _remove_checkpoint(self, trainer: Trainer, filepath: str):
+        return _remove_checkpoint(trainer, filepath, remove_metadata=True)

nshtrainer/callbacks/norm_logging.py

@@ -1,4 +1,4 @@
-from logging import getLogger
+import logging
 from typing import Literal, cast
 
 import torch
@@ -9,7 +9,7 @@ from typing_extensions import override
 
 from .base import CallbackConfigBase
 
-log = getLogger(__name__)
+log = logging.getLogger(__name__)
 
 
 def grad_norm(

nshtrainer/callbacks/throughput_monitor.py

@@ -1,4 +1,4 @@
-from logging import getLogger
+import logging
 from typing import Any, Literal, Protocol, TypedDict, cast, runtime_checkable
 
 from typing_extensions import NotRequired, override
@@ -6,7 +6,7 @@ from typing_extensions import NotRequired, override
 from ._throughput_monitor_callback import ThroughputMonitor as _ThroughputMonitor
 from .base import CallbackConfigBase
 
-log = getLogger(__name__)
+log = logging.getLogger(__name__)
 
 
 class ThroughputMonitorBatchStats(TypedDict):

nshtrainer/callbacks/wandb_watch.py

@@ -1,4 +1,4 @@
-from logging import getLogger
+import logging
 from typing import Literal, Protocol, cast, runtime_checkable
 
 import torch.nn as nn
@@ -9,7 +9,7 @@ from typing_extensions import override
 
 from .base import CallbackConfigBase
 
-log = getLogger(__name__)
+log = logging.getLogger(__name__)
 
 
 @runtime_checkable

nshtrainer/data/balanced_batch_sampler.py

@@ -1,6 +1,6 @@
 import heapq
+import logging
 from functools import cached_property
-from logging import getLogger
 from typing import Any, Protocol, runtime_checkable
 
 import numpy as np
@@ -10,7 +10,7 @@ from lightning_fabric.utilities.distributed import _DatasetSamplerWrapper
 from torch.utils.data import BatchSampler, Dataset, DistributedSampler
 from typing_extensions import override
 
-log = getLogger(__name__)
+log = logging.getLogger(__name__)
 
 
 def _all_gather(tensor: torch.Tensor, device: torch.device | None = None):

nshtrainer/model/base.py

@@ -1,7 +1,7 @@
 import inspect
+import logging
 from abc import ABC, abstractmethod
 from collections.abc import MutableMapping
-from logging import getLogger
 from typing import IO, TYPE_CHECKING, Any, Generic, cast
 
 import torch
@@ -21,7 +21,7 @@ from .modules.profiler import ProfilerMixin
 from .modules.rlp_sanity_checks import RLPSanityCheckModuleMixin
 from .modules.shared_parameters import SharedParametersModuleMixin
 
-log = getLogger(__name__)
+log = logging.getLogger(__name__)
 
 THparams = TypeVar("THparams", bound=BaseConfig, infer_variance=True)
 

nshtrainer/model/config.py

@@ -1,4 +1,5 @@
 import copy
+import logging
 import os
 import string
 import time
@@ -6,7 +7,6 @@ import warnings
 from abc import ABC, abstractmethod
 from collections.abc import Iterable, Sequence
 from datetime import timedelta
-from logging import getLogger
 from pathlib import Path
 from typing import (
     Annotated,
@@ -46,7 +46,7 @@ from ..callbacks.base import CallbackConfigBase
 from ..metrics import MetricConfig
 from ._environment import EnvironmentConfig
 
-log = getLogger(__name__)
+log = logging.getLogger(__name__)
 
 
 class IdSeedWarning(Warning):

nshtrainer/model/modules/callback.py

@@ -1,6 +1,6 @@
+import logging
 from collections import abc
 from collections.abc import Callable, Iterable
-from logging import getLogger
 from typing import Any, TypeAlias, cast, final
 
 from lightning.pytorch import Callback, LightningModule
@@ -9,7 +9,7 @@ from typing_extensions import override
 
 from ...util.typing_utils import mixin_base_type
 
-log = getLogger(__name__)
+log = logging.getLogger(__name__)
 
 CallbackFn: TypeAlias = Callable[[], Callback | Iterable[Callback] | None]
 

nshtrainer/model/modules/debug.py

@@ -1,9 +1,9 @@
-from logging import getLogger
+import logging
 
 import torch
 import torch.distributed
 
-log = getLogger(__name__)
+log = logging.getLogger(__name__)
 
 
 class DebugModuleMixin:

nshtrainer/model/modules/rlp_sanity_checks.py

@@ -1,5 +1,5 @@
+import logging
 from collections.abc import Mapping
-from logging import getLogger
 from typing import cast
 
 import torch
@@ -14,7 +14,7 @@ from ...util.typing_utils import mixin_base_type
 from ..config import BaseConfig
 from .callback import CallbackModuleMixin
 
-log = getLogger(__name__)
+log = logging.getLogger(__name__)
 
 
 def _on_train_start_callback(trainer: Trainer, pl_module: LightningModule):

nshtrainer/model/modules/shared_parameters.py

@@ -1,5 +1,5 @@
+import logging
 from collections.abc import Sequence
-from logging import getLogger
 from typing import cast
 
 import torch.nn as nn
@@ -10,7 +10,7 @@ from ...util.typing_utils import mixin_base_type
 from ..config import BaseConfig
 from .callback import CallbackRegistrarModuleMixin
 
-log = getLogger(__name__)
+log = logging.getLogger(__name__)
 
 
 def _parameters_to_names(parameters: Sequence[nn.Parameter], model: nn.Module):

nshtrainer/util/environment.py

@@ -1,8 +1,8 @@
+import logging
 import os
 from contextlib import contextmanager
-from logging import getLogger
 
-log = getLogger(__name__)
+log = logging.getLogger(__name__)
 
 
 @contextmanager

nshtrainer/util/seed.py

@@ -1,8 +1,8 @@
-from logging import getLogger
+import logging
 
 import lightning.fabric.utilities.seed as LS
 
-log = getLogger(__name__)
+log = logging.getLogger(__name__)
 
 
 def seed_everything(seed: int | None, *, workers: bool = False):

{nshtrainer-0.10.9.dist-info → nshtrainer-0.10.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nshtrainer
-Version: 0.10.9
+Version: 0.10.11
 Summary: 
 Author: Nima Shoghi
 Author-email: nimashoghi@gmail.com

{nshtrainer-0.10.9.dist-info → nshtrainer-0.10.11.dist-info}/RECORD

@@ -1,6 +1,7 @@
 nshtrainer/__init__.py,sha256=39loiLLXbaGiozEsAn8mPHopxaPsek8JsgR9DD2gxtY,583
 nshtrainer/_checkpoint/loader.py,sha256=48flPr1XgQHOgIPaCrRqOEvRuG0SZuV3cQ1vgHLqFqI,11025
-nshtrainer/_checkpoint/metadata.py,sha256=C7je_soYyEbZjiq7p2_pSVFkgcXnz2J2H5sMy8oskx0,3051
+nshtrainer/_checkpoint/metadata.py,sha256=B6kPmWsq2TQh0gTzBx-1pLIwTVEs_Qw5v0nHEeTBdO4,5636
+nshtrainer/_checkpoint/saver.py,sha256=KZp9ITUVHwj2Ttu81zXKdlS_h-fKkHearspwuAijDpM,1501
 nshtrainer/_experimental/__init__.py,sha256=2tQIcrWT8U8no_AeBTYnozaTmxN40kuAJdGQ4b-PoWM,120
 nshtrainer/_experimental/flops/__init__.py,sha256=edo9Ez3LlrnxkNRX9W6YBhPkRPKYGLpkpnl5gx7sEX8,1550
 nshtrainer/_experimental/flops/flop_counter.py,sha256=-sL0Fy6poXa__hyzUMdZScjPULp4coQELQpPU6p6dXU,25736
@@ -9,22 +10,22 @@ nshtrainer/callbacks/__init__.py,sha256=ifXQRwtccznl4lMKwKLSuuAQC4bKFBgfzQ4rx9gO
 nshtrainer/callbacks/_throughput_monitor_callback.py,sha256=aJo_11rc4lo0IYOd-kHmPDtzdC4ctgXyRudkRJqH4m4,23184
 nshtrainer/callbacks/actsave.py,sha256=aY6T_NAzaFAVU8WMHOXnWL5wd2bi8eVxeU2S0iAs70c,4446
 nshtrainer/callbacks/base.py,sha256=UnlYZAqSb8UwBJR-N5-XunxFx2yZjZ4lyGqUfhbCRlI,3555
-nshtrainer/callbacks/early_stopping.py,sha256=jriSU761wf_qTJ9Bos0D3h5aDvZHYpRqK62Ne8aWp5I,3768
+nshtrainer/callbacks/early_stopping.py,sha256=LGn3rdbvkFfUo9kwMzK4eMGlPAqD9uFdowDx6VdfozQ,3761
 nshtrainer/callbacks/ema.py,sha256=8-WHmKFP3VfnzMviJaIFmVD9xHPqIPmq9NRF5xdu3c8,12131
-nshtrainer/callbacks/finite_checks.py,sha256=AO5fa51uANAjAkeJfTquOjK6W_4RSU5Kky3f5jmAPlQ,2084
+nshtrainer/callbacks/finite_checks.py,sha256=gJC_RUr3ais3FJI0uB6wUZnDdE3WRwCix3ppA3PwQXA,2077
 nshtrainer/callbacks/gradient_skipping.py,sha256=pqu5AELx4ctJxR2Y7YSSiGd5oGauVCTZFCEIIS6s88w,3665
 nshtrainer/callbacks/interval.py,sha256=smz5Zl8cN6X6yHKVsMRS2e3SEkzRCP3LvwE1ONvLfaw,8080
-nshtrainer/callbacks/latest_epoch_checkpoint.py,sha256=zCRAUsqW-2PaoIwVKlXOqdh2uF_B_YUUTmQO1wSomR8,2489
+nshtrainer/callbacks/latest_epoch_checkpoint.py,sha256=t4vWa4PvJDO3rKXKZbuegm7iLl7xCEd17wNif0Bp-BA,4138
 nshtrainer/callbacks/log_epoch.py,sha256=fTa_K_Y8A7g09630cG4YkDE6AzSMPkjb9bpPm4gtqos,1120
-nshtrainer/callbacks/model_checkpoint.py,sha256=N0raLsHlCVSbO3QU5eNFUXUDqxxW3C73oQwceMnFE_k,5955
-nshtrainer/callbacks/norm_logging.py,sha256=EWyrfkp8iHjQi9iAAXHxb0xStw2RwkdpKG2_gLarQRA,6281
+nshtrainer/callbacks/model_checkpoint.py,sha256=MaDkD8Ismcj8u6l2flCFlqJR3-k1Tc4xzhxNWNux4n0,6556
+nshtrainer/callbacks/norm_logging.py,sha256=T2psu8mYsw9iahPKT6aUPjkGrZ4TIzm6_UUUmE09GJs,6274
 nshtrainer/callbacks/on_exception_checkpoint.py,sha256=x42BYZ2ejf2rhqPLCmT5nyWKhA9qBEosiV8ZNhhZ6lI,3355
 nshtrainer/callbacks/print_table.py,sha256=_FdAHhqylWGk4Z0c2FrLFeiMA4jhfA_beZRK_BHpzmE,2837
-nshtrainer/callbacks/throughput_monitor.py,sha256=4EF3b79HdHiRgBGIFDyD4O1oywb5h1tV8nml7NuuDjU,1845
+nshtrainer/callbacks/throughput_monitor.py,sha256=H_ocXErZxUO3dxFk8Tx_VQdpI9E_Ztvqof5WtFevLyQ,1838
 nshtrainer/callbacks/timer.py,sha256=quS79oYClDUvQxJkNWmDMe0hwRUkkREgTgqzVrnom50,4607
-nshtrainer/callbacks/wandb_watch.py,sha256=0JKDEPSDmiKverFm00lFfufOvtvr49akFXScvdUQnqc,2930
+nshtrainer/callbacks/wandb_watch.py,sha256=EJ93mtJlph4BZsXh8HJPNiw2VNSm2N6TOwpCwqRAeKI,2923
 nshtrainer/data/__init__.py,sha256=7mk1tr7SWUZ7ySbsf0y0ZPszk7u4QznPhQ-7wnpH9ec,149
-nshtrainer/data/balanced_batch_sampler.py,sha256=bcJBcQjh1hB1yKF_xSlT9AtEWv0BJjYc1CuH2BF-ea8,4392
+nshtrainer/data/balanced_batch_sampler.py,sha256=dGBTDDtlBU6c-ZlVQOCnTW7SjTB5hczWsOWEdUWjvkA,4385
 nshtrainer/data/transform.py,sha256=6SNs3_TpNpfhcwTwvPKyEJ3opM1OT7LmMEYQNHKgRl8,2227
 nshtrainer/ll/__init__.py,sha256=dD0ISxHJ2lg1HLSM0b3db7TBlsPpQCtChnuYO-c2oqI,2635
 nshtrainer/ll/_experimental.py,sha256=oBQCKOEVYoxuUU9eLb-Fg2B2mzZD7SA0zfAO6lmWZ88,53
@@ -51,15 +52,15 @@ nshtrainer/metrics/__init__.py,sha256=ObLIELGguIEcUpRsUkqh1ltrvZii6vglTpJGrPvoy0
 nshtrainer/metrics/_config.py,sha256=hWWS4IXENRyH3RmJ7z1Wx1n3Lt1sNMlGOrcU6PW15o0,1104
 nshtrainer/model/__init__.py,sha256=TbexTxiE20WHYg5q3L88Hysk4LlHeKk_isv33aSBREA,1918
 nshtrainer/model/_environment.py,sha256=oTtecQeF5oY2RV7UkkSLnzDy3clz4AUkf9oocD6-e54,23115
-nshtrainer/model/base.py,sha256=Bmw-t70TydDbE9P0ee-lTibGoUhrCx5Qke-upa7FGVM,17512
-nshtrainer/model/config.py,sha256=FRjn2pVbCzitBeIFcKXXPQ0TCmqvAwSylAvfedf1NHg,53356
-nshtrainer/model/modules/callback.py,sha256=JF59U9-CjJsAIspEhTJbVaGN0wGctZG7UquE3IS7R8A,6408
-nshtrainer/model/modules/debug.py,sha256=DTVty8cKnzj1GCULRyGx_sWTTsq9NLi30dzqjRTnuCU,1127
+nshtrainer/model/base.py,sha256=WtCj0-nLWeW04Tu2TWVjIq0D-jW_kMN2hg--4VWVnvE,17505
+nshtrainer/model/config.py,sha256=orGBrp8TXnHksfAzXxNJVDdo0X_iIn_nda6BZDS9N70,53349
+nshtrainer/model/modules/callback.py,sha256=K0-cyEtBcQhI7Q2e-AGTE8T-GghUPY9DYmneU6ULV6g,6401
+nshtrainer/model/modules/debug.py,sha256=Yy7XEdPou9BkCsD5hJchwJGmCVGrfUru5g9VjPM4uAw,1120
 nshtrainer/model/modules/distributed.py,sha256=ABpR9d-3uBS_fivfy_WYW-dExW6vp5BPaoPQnOudHng,1725
 nshtrainer/model/modules/logger.py,sha256=YYhehQysqTjuVFcd_EREYDh57CIlezidFBS2Ohp_xKo,5661
 nshtrainer/model/modules/profiler.py,sha256=rQ_jRMcM1Z2AIROZlRnBRHM5rkTpq67afZPD6CIRfXs,825
-nshtrainer/model/modules/rlp_sanity_checks.py,sha256=o6gUceFwsuDHmL8eLOYuT3JGXFzq_qc4awl2RWaBygU,8900
-nshtrainer/model/modules/shared_parameters.py,sha256=mD5wrlBE3c025vzVdTpnSyC8yxzuI-aUWMmPhqPT0a0,2694
+nshtrainer/model/modules/rlp_sanity_checks.py,sha256=I_ralr2ThQ-D_FkVQTwbdXLLlgHJEr7-s01I5wSDjps,8893
+nshtrainer/model/modules/shared_parameters.py,sha256=ZiRKkZXr6RwdwLCdZCJPl3dXe7bnT8Z9yTeRK5bXBGk,2687
 nshtrainer/nn/__init__.py,sha256=0QPFl02a71WZQjLMGOlFNMmsYP5aa1q3eABHmnWH58Q,1427
 nshtrainer/nn/mlp.py,sha256=V0FrScpIUdg_IgIO8GMtIsGEtmHjwF14i2IWxmZrsqg,5952
 nshtrainer/nn/module_dict.py,sha256=NOY0B6WDTnktyWH4GthsprMQo0bpehC-hCq9SfD8paE,2329
@@ -73,11 +74,11 @@ nshtrainer/trainer/_runtime_callback.py,sha256=sd2cUdRJG-UCdQr9ruZvEYpNGNF1t2W2f
 nshtrainer/trainer/checkpoint_connector.py,sha256=xoqI2dcPnlNFPPLVIU6dBOvRPC9PtfX5qu__xV1lx0Y,2124
 nshtrainer/trainer/signal_connector.py,sha256=llwc8pdKAWxREFpjdi14Bpy8rGVMEJsmJx_s2p4gI8E,10689
 nshtrainer/trainer/trainer.py,sha256=tFyzIsF8c-FABTH6wwDOR9y8kydVJqeVO7PDNFMvhSU,16950
-nshtrainer/util/environment.py,sha256=_SEtiQ_s5bL5pllUlf96AOUv15kNvCPvocVC13S7mIk,4166
-nshtrainer/util/seed.py,sha256=HEXgVs-wldByahOysKwq7506OHxdYTEgmP-tDQVAEkQ,287
+nshtrainer/util/environment.py,sha256=AeW_kLl-N70wmb6L_JLz1wRj0kA70xs6RCmc9iUqczE,4159
+nshtrainer/util/seed.py,sha256=Or2wMPsnQxfnZ2xfBiyMcHFIUt3tGTNeMMyOEanCkqs,280
 nshtrainer/util/slurm.py,sha256=rofIU26z3SdL79SF45tNez6juou1cyDLz07oXEZb9Hg,1566
 nshtrainer/util/typed.py,sha256=NGuDkDzFlc1fAoaXjOFZVbmj0mRFjsQi1E_hPa7Bn5U,128
 nshtrainer/util/typing_utils.py,sha256=8ptjSSLZxlmy4FY6lzzkoGoF5fGNClo8-B_c0XHQaNU,385
-nshtrainer-0.10.9.dist-info/METADATA,sha256=zpMnR1Jwc9kX7BhnHT6NA3nsPUwQzMAraBPOGsUSD1w,695
-nshtrainer-0.10.9.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-nshtrainer-0.10.9.dist-info/RECORD,,
+nshtrainer-0.10.11.dist-info/METADATA,sha256=9WAsp25_csjDcchr5X22g7ocQpQ-d-ewB3gS9EAZSE8,696
+nshtrainer-0.10.11.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+nshtrainer-0.10.11.dist-info/RECORD,,