nshtrainer 1.0.0b45__py3-none-any.whl → 1.0.0b46__py3-none-any.whl

nshtrainer/_checkpoint/metadata.py

@@ -139,15 +139,30 @@ def remove_checkpoint_metadata(checkpoint_path: Path):
         log.debug(f"Removed {path}")
 
 
+def remove_checkpoint_metadata_link(ckpt_link_path: Path):
+    path = _metadata_path(ckpt_link_path)
+    # If the metadata does not exist, we can safely ignore this
+    if not path.exists(follow_symlinks=False):
+        # This is EXTREMELY important here
+        # Otherwise, we've already deleted the file that the symlink
+        # used to point to, so this always returns False
+        log.debug(f"Metadata file does not exist: {path}")
+        return
+
+    # If the metadata exists, we can remove it
+    try:
+        path.unlink(missing_ok=True)
+    except Exception:
+        log.warning(f"Failed to remove {path}", exc_info=True)
+    else:
+        log.debug(f"Removed {path}")
+
+
 def link_checkpoint_metadata(checkpoint_path: Path, linked_checkpoint_path: Path):
     # First, remove any existing metadata files
-    remove_checkpoint_metadata(linked_checkpoint_path)
+    remove_checkpoint_metadata_link(linked_checkpoint_path)
 
     # Link the metadata files to the new checkpoint
     path = _metadata_path(checkpoint_path)
     linked_path = _metadata_path(linked_checkpoint_path)
-
-    if not path.exists():
-        raise FileNotFoundError(f"Checkpoint path does not exist: {checkpoint_path}")
-
     try_symlink_or_copy(path, linked_path)

nshtrainer/_checkpoint/saver.py

@@ -8,7 +8,11 @@ from pathlib import Path
 from lightning.pytorch import Trainer
 
 from ..util.path import try_symlink_or_copy
-from .metadata import link_checkpoint_metadata, remove_checkpoint_metadata
+from .metadata import (
+    link_checkpoint_metadata,
+    remove_checkpoint_metadata,
+    remove_checkpoint_metadata_link,
+)
 
 log = logging.getLogger(__name__)
 
@@ -39,7 +43,7 @@ def link_checkpoint(
             log.debug(f"Removed {linkpath=}")
 
         if metadata:
-            remove_checkpoint_metadata(linkpath)
+            remove_checkpoint_metadata_link(linkpath)
 
     try_symlink_or_copy(filepath, linkpath)
     if metadata:

nshtrainer/callbacks/checkpoint/_base.py

@@ -160,7 +160,7 @@ class CheckpointBase(Checkpoint, ABC, Generic[TConfig]):
         filepath = self.resolve_checkpoint_path(self.current_metrics(trainer))
         trainer.save_checkpoint(filepath, self.config.save_weights_only)
 
-        if trainer.is_global_zero:
+        if trainer.hparams.save_checkpoint_metadata and trainer.is_global_zero:
             # Remove old checkpoints
             self.remove_old_checkpoints(trainer)
 

nshtrainer/util/path.py

@@ -120,7 +120,8 @@ def try_symlink_or_copy(
                 shutil.copy(file_path, link_path)
         else:
             link_path.symlink_to(
-                symlink_target, target_is_directory=target_is_directory
+                symlink_target,
+                target_is_directory=target_is_directory,
             )
     except Exception:
         log.warning(

{nshtrainer-1.0.0b45.dist-info → nshtrainer-1.0.0b46.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: nshtrainer
-Version: 1.0.0b45
+Version: 1.0.0b46
 Summary: 
 Author: Nima Shoghi
 Author-email: nimashoghi@gmail.com

{nshtrainer-1.0.0b45.dist-info → nshtrainer-1.0.0b46.dist-info}/RECORD

@@ -1,8 +1,8 @@
 nshtrainer/.nshconfig.generated.json,sha256=yZd6cn1RhvNNJUgiUTRYut8ofZYvbulnpPG-rZIRhi4,106
 nshtrainer/__init__.py,sha256=g_moPnfQxSxFZX5NB9ILQQOJrt4RTRuiFt9N0STIpxM,874
 nshtrainer/_callback.py,sha256=tXQCDzS6CvMTuTY5lQSH5qZs1pXUi-gt9bQdpXMVdEs,12715
-nshtrainer/_checkpoint/metadata.py,sha256=LQZ8g50rKxQQx-FqiW3n8EWmal9qSWRouOpIIn6NJJY,4758
-nshtrainer/_checkpoint/saver.py,sha256=rWl4d2lCTMU4_wt8yZFL2pFQaP9hj5sPgqHMPQ4zuyI,1584
+nshtrainer/_checkpoint/metadata.py,sha256=XoKqY3eR95CYuc_Kk9ck-p4iM2Q1OXU3vSXNrzohHz0,5332
+nshtrainer/_checkpoint/saver.py,sha256=65UDrz3KuhkgVfco-RkWuoa1wzTZoXxunlC769yJaMc,1639
 nshtrainer/_directory.py,sha256=TJR9ccyuzRlAVfVjGyeQ3E2AFAcz-XbBCxWfiXo2SlY,3191
 nshtrainer/_experimental/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
 nshtrainer/_hf_hub.py,sha256=4OsCbIITnZk_YLyoMrVyZ0SIN04FBxlC0ig2Et8UAdo,14287
@@ -10,7 +10,7 @@ nshtrainer/callbacks/__init__.py,sha256=w80d6PGNu3wjUj9NiRGMqCX9NnXD5ZlvbY-DIK4z
 nshtrainer/callbacks/actsave.py,sha256=NSXIIu62MNYe5gz479SMW33bdoKYoYtWtd_iTWFpKpc,3881
 nshtrainer/callbacks/base.py,sha256=Alaou1IHAIlMEM7g58d_02ozY2xWlshBN7fsw5Ee21s,3683
 nshtrainer/callbacks/checkpoint/__init__.py,sha256=l8tkHc83_mLiU0-wT09SWdRzwpm2ulbkLzcuCmuTwzE,620
-nshtrainer/callbacks/checkpoint/_base.py,sha256=wCJBRI0pQYZc3GBu0b-aUBlBDhd39AdL82VvFgKmv3k,6300
+nshtrainer/callbacks/checkpoint/_base.py,sha256=f7lpk8W4xqxk3PolBEU3AWt9VTIpoLW7wMUhC5DNm3c,6345
 nshtrainer/callbacks/checkpoint/best_checkpoint.py,sha256=2CQuhPJ3Fi7lDw7z-J8kXXXuDU8-4HcU48oZxR49apk,2667
 nshtrainer/callbacks/checkpoint/last_checkpoint.py,sha256=vn-as3ex7kaTRcKsIurVtM6kUSHYNwHJeYG82j2dMcc,3554
 nshtrainer/callbacks/checkpoint/on_exception_checkpoint.py,sha256=nljzETqkHwA-4g8mxaeFK5HxA8My0dlIPzIUscSMWyk,3525
@@ -149,11 +149,11 @@ nshtrainer/util/config/__init__.py,sha256=Z39JJufSb61Lhn2GfVcv3eFW_eorOrN9-9llDW
 nshtrainer/util/config/dtype.py,sha256=Fn_MhhQoHPyFAnFPSwvcvLiGR3yWFIszMba02CJiC4g,2213
 nshtrainer/util/config/duration.py,sha256=mM-UfU_HvhXwW33TYEDg0x58n80tnle2e6VaWtxZTjk,764
 nshtrainer/util/environment.py,sha256=s-B5nY0cKYXdFMdNYumvC_xxacMATiI4DvV2gUDu20k,4195
-nshtrainer/util/path.py,sha256=9fIjE3S78pPL6wjAgEJUYfIJQAPdKOQqIYvTS9lWTUk,3959
+nshtrainer/util/path.py,sha256=wADa_qFiekiX0PENf9jI-9_abICB0vJFXav8pNLR7lw,3976
 nshtrainer/util/seed.py,sha256=diMV8iwBKN7Xxt5pELmui-gyqyT80_CZzomrWhNss0k,316
 nshtrainer/util/slurm.py,sha256=HflkP5iI_r4UHMyPjw9R4dD5AHsJUpcfJw5PLvGYBRM,1603
 nshtrainer/util/typed.py,sha256=Xt5fUU6zwLKSTLUdenovnKK0N8qUq89Kddz2_XeykVQ,164
 nshtrainer/util/typing_utils.py,sha256=MjY-CUX9R5Tzat-BlFnQjwl1PQ_W2yZQoXhkYHlJ_VA,442
-nshtrainer-1.0.0b45.dist-info/METADATA,sha256=_RPpe6F7DXpsQSmBF1GTc-E5VUfaC69fIYfoFhsip2s,988
-nshtrainer-1.0.0b45.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-nshtrainer-1.0.0b45.dist-info/RECORD,,
+nshtrainer-1.0.0b46.dist-info/METADATA,sha256=L6-5RyLlIcoFyURkoCuHsAgItT0gSl6Ip0l4iDKvs4o,988
+nshtrainer-1.0.0b46.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+nshtrainer-1.0.0b46.dist-info/RECORD,,