nshtrainer 0.28.0__py3-none-any.whl → 0.29.1__py3-none-any.whl

nshtrainer/_checkpoint/metadata.py

@@ -117,7 +117,7 @@ def _write_checkpoint_metadata(
     try:
         metadata_path.write_text(metadata.model_dump_json(indent=4), encoding="utf-8")
     except Exception:
-        log.exception(f"Failed to write metadata to {metadata_path}")
+        log.warning(f"Failed to write metadata to {metadata_path}", exc_info=True)
         return None
 
     log.debug(f"Checkpoint metadata written to {metadata_path}")
@@ -129,7 +129,7 @@ def _remove_checkpoint_metadata(checkpoint_path: Path):
     try:
         path.unlink(missing_ok=True)
     except Exception:
-        log.exception(f"Failed to remove {path}")
+        log.warning(f"Failed to remove {path}", exc_info=True)
     else:
         log.debug(f"Removed {path}")
 

nshtrainer/_checkpoint/saver.py

@@ -25,11 +25,11 @@ def _link_checkpoint(
         try:
             if linkpath.exists():
                 if linkpath.is_dir():
-                    shutil.rmtree(linkpath, ignore_errors=True)
+                    shutil.rmtree(linkpath)
                 else:
                     linkpath.unlink(missing_ok=True)
         except Exception:
-            log.exception(f"Failed to remove {linkpath}")
+            log.warning(f"Failed to remove {linkpath}", exc_info=True)
 
         if metadata:
             _remove_checkpoint_metadata(linkpath)

nshtrainer/_hf_hub.py

@@ -179,7 +179,9 @@ class HFHubCallback(NTCallbackBase):
         try:
             yield
         except Exception:
-            log.exception(f"Failed to {opeartion}, repo_id={self._repo_id}")
+            log.warning(
+                f"Failed to {opeartion}, repo_id={self._repo_id}", exc_info=True
+            )
         else:
             log.debug(f"Successfully {opeartion}, repo_id={self._repo_id}")
 
@@ -261,9 +263,13 @@ class HFHubCallback(NTCallbackBase):
                     )
                     log.info(f"Created new repository '{self.repo_id}'.")
                 except Exception:
-                    log.exception(f"Failed to create repository '{self.repo_id}'")
+                    log.warning(
+                        f"Failed to create repository '{self.repo_id}'", exc_info=True
+                    )
             except Exception:
-                log.exception(f"Error checking repository '{self.repo_id}'")
+                log.warning(
+                    f"Error checking repository '{self.repo_id}'", exc_info=True
+                )
 
     def _save_config(self, root_config: "BaseConfig"):
         with self._with_error_handling("upload config"):
@@ -300,9 +306,15 @@ class HFHubCallback(NTCallbackBase):
 
     def _save_file(self, p: _Upload):
         with self._with_error_handling("save file"):
+            # First, read the file into memory.
+            # We do this to avoid issues with
+            # the file being moved or deleted.
+            with p.local_path.open("rb") as f:
+                data = f.read()
+
             # Upload the checkpoint files to the repository
             self.api.upload_file(
-                path_or_fileobj=p.local_path,
+                path_or_fileobj=data,
                 path_in_repo=str(p.path_in_repo),
                 repo_id=self.repo_id,
                 repo_type="model",

nshtrainer/callbacks/checkpoint/_base.py

@@ -9,7 +9,7 @@ from lightning.pytorch import Trainer
 from lightning.pytorch.callbacks import Checkpoint
 from typing_extensions import TypeVar, override
 
-from ..._checkpoint.metadata import CheckpointMetadata, _metadata_path
+from ..._checkpoint.metadata import CheckpointMetadata
 from ..._checkpoint.saver import _link_checkpoint, _remove_checkpoint
 from ..base import CallbackConfigBase
 
@@ -155,29 +155,19 @@ class CheckpointBase(Checkpoint, ABC, Generic[TConfig]):
         trainer.save_checkpoint(filepath, self.config.save_weights_only)
 
         if trainer.is_global_zero:
+            # Remove old checkpoints
+            self.remove_old_checkpoints(trainer)
+
             # Create the latest symlink
             if (symlink_filename := self.symlink_path()) is not None:
                 symlink_path = self.dirpath / symlink_filename
                 _link_checkpoint(filepath, symlink_path, metadata=True)
                 log.debug(f"Created latest symlink: {symlink_path}")
 
-            # Remove old checkpoints
-            self.remove_old_checkpoints(trainer)
-
         # Barrier to ensure all processes have saved the checkpoint,
         # deleted the old checkpoints, and created the symlink before continuing
         trainer.strategy.barrier()
 
-        # Call the on save checkpoint callback for the symlink (if it exists)
-        if (symlink_filename := self.symlink_path()) is not None:
-            from ... import _callback
-
-            symlink_path = self.dirpath / symlink_filename
-            symlink_metadata_path = _metadata_path(symlink_path)
-            _callback._call_on_checkpoint_saved(
-                trainer, symlink_path, symlink_metadata_path
-            )
-
     def _should_skip_saving_checkpoint(self, trainer: Trainer) -> bool:
         from lightning.pytorch.trainer.states import TrainerFn
 

nshtrainer/util/_environment_info.py

@@ -434,7 +434,7 @@ class EnvironmentPackageConfig(C.Config):
                         requires=requires,
                     )
                 except Exception:
-                    log.exception(f"Error processing package {dist.name}")
+                    log.warning(f"Error processing package {dist.name}", exc_info=True)
 
         except ImportError:
             log.warning(
@@ -673,7 +673,7 @@ class GitRepositoryConfig(C.Config):
         except git.InvalidGitRepositoryError:
             draft.is_git_repo = False
         except Exception:
-            log.exception("Failed to get Git repository information")
+            log.warning("Failed to get Git repository information", exc_info=True)
             draft.is_git_repo = None
 
         return draft.finalize()

nshtrainer/util/path.py

@@ -97,7 +97,10 @@ def try_symlink_or_copy(
                 symlink_target, target_is_directory=target_is_directory
             )
     except Exception:
-        log.exception(f"Failed to create symlink or copy {file_path} to {link_path}")
+        log.warning(
+            f"Failed to create symlink or copy {file_path} to {link_path}",
+            exc_info=True,
+        )
         return False
     else:
         log.debug(f"Created symlink or copied {file_path} to {link_path}")

{nshtrainer-0.28.0.dist-info → nshtrainer-0.29.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nshtrainer
-Version: 0.28.0
+Version: 0.29.1
 Summary: 
 Author: Nima Shoghi
 Author-email: nimashoghi@gmail.com

{nshtrainer-0.28.0.dist-info → nshtrainer-0.29.1.dist-info}/RECORD

@@ -1,16 +1,16 @@
 nshtrainer/__init__.py,sha256=39loiLLXbaGiozEsAn8mPHopxaPsek8JsgR9DD2gxtY,583
 nshtrainer/_callback.py,sha256=A1zLsTy4b_wOYnInLLXGSRdHzT2yNa6mPEql-ozm0u0,1013
 nshtrainer/_checkpoint/loader.py,sha256=5vjg-OFChXJjgiOVv8vnV8nwTscfdDtEdxQRz6uPfDE,14158
-nshtrainer/_checkpoint/metadata.py,sha256=hxZwwsUKVbBtt4wjqcKZbObx0PuO-qCdF3BTdnyqaQo,4711
-nshtrainer/_checkpoint/saver.py,sha256=1loCDYDy_Cay37uKs_wvxnkwvr41WMmga85qefct80Q,1271
+nshtrainer/_checkpoint/metadata.py,sha256=5D4PgKodzhLsmQvuF3xxkH49epKaegxi4wh_ImDTtns,4737
+nshtrainer/_checkpoint/saver.py,sha256=MbX_WjkDtHHAf9Ms-KXDlknkjiPXVoGIe2ciO28AdZ0,1264
 nshtrainer/_experimental/__init__.py,sha256=pEXPyI184UuDHvfh4p9Kg9nQZQZI41e4_HvNd4BK-yg,81
-nshtrainer/_hf_hub.py,sha256=42cR6viOiUInbBW4R7n9-AMnAn_ovN6YoLU3jD14PtI,12608
+nshtrainer/_hf_hub.py,sha256=0bkXkqhve5D1onMW-fCfuvVKlTn0i6jv_6uMNgZ7OHQ,12974
 nshtrainer/callbacks/__init__.py,sha256=4qocBDzQbLLhhbIEfvbA3SQB_Dy9ZJH7keMwPay-ZS8,2359
 nshtrainer/callbacks/_throughput_monitor_callback.py,sha256=aJo_11rc4lo0IYOd-kHmPDtzdC4ctgXyRudkRJqH4m4,23184
 nshtrainer/callbacks/actsave.py,sha256=qbnaKts4_dvjPeAaPtv7Ds12_vEWzaHUfg_--49NB9I,4041
 nshtrainer/callbacks/base.py,sha256=NpjeKmonJ1Kaz5_39XSn3LlDwvbGjk6WV8BpHSNCvI4,3508
 nshtrainer/callbacks/checkpoint/__init__.py,sha256=g-3zIthupERKqWZQw-A_busQPaPRkto6iHBV-M7nK1Y,527
-nshtrainer/callbacks/checkpoint/_base.py,sha256=dXCfyFyf2TjH9Pnc4FCqFUGrmy_n25MSCn8jdf3L-_8,6605
+nshtrainer/callbacks/checkpoint/_base.py,sha256=vvlwuD-20NozYVIolGGShmUdkkNYeuwN6xCoFnK4GiU,6157
 nshtrainer/callbacks/checkpoint/best_checkpoint.py,sha256=8BHgLAd3Tuzf5sup0guEAKF1jJiAwYsjdKBFYZw98ac,2171
 nshtrainer/callbacks/checkpoint/last_checkpoint.py,sha256=CWWv0cSwQ1VAX26N7hAyMxbNCk26Keh39oQguBEK5To,1102
 nshtrainer/callbacks/checkpoint/on_exception_checkpoint.py,sha256=ctT88EGT22_t_6tr5r7Sfo43cuve6XeroBnBYRMPOus,3372
@@ -79,14 +79,14 @@ nshtrainer/trainer/_runtime_callback.py,sha256=sd2cUdRJG-UCdQr9ruZvEYpNGNF1t2W2f
 nshtrainer/trainer/checkpoint_connector.py,sha256=r0ir4xYSdf_jebM0x09qaO6nJsvsiRQDyM0fs80ppOQ,2347
 nshtrainer/trainer/signal_connector.py,sha256=2EzkVktlasl8PgWAKNLDZRUMY__gRlDy1HdinAU-tfU,10740
 nshtrainer/trainer/trainer.py,sha256=L4nYXq6Gts2sS9CQGenwEcvMET4L5vO5c60KM5Hm8Do,17544
-nshtrainer/util/_environment_info.py,sha256=gIdq9TJgzGCdcVzZxjHcwYasJ_HmEGVHbvE-KJVVtWs,24187
+nshtrainer/util/_environment_info.py,sha256=CFUUZYjXhBLWGc0jtPNOaZgYMueUDEHpEaWFA1f3GoY,24213
 nshtrainer/util/_useful_types.py,sha256=dwZokFkIe7M5i2GR3nQ9A1lhGw06DMAFfH5atyquqSA,8000
 nshtrainer/util/environment.py,sha256=AeW_kLl-N70wmb6L_JLz1wRj0kA70xs6RCmc9iUqczE,4159
-nshtrainer/util/path.py,sha256=jAEjF1qp8Aii32L5lWG4UFgVyQAFkHOMYEc_TC2hDx8,2947
+nshtrainer/util/path.py,sha256=VkpuhR4GaZtSFBVqbGAvfjcrU-PR8xwiGzzwFNOWP9c,2995
 nshtrainer/util/seed.py,sha256=Or2wMPsnQxfnZ2xfBiyMcHFIUt3tGTNeMMyOEanCkqs,280
 nshtrainer/util/slurm.py,sha256=rofIU26z3SdL79SF45tNez6juou1cyDLz07oXEZb9Hg,1566
 nshtrainer/util/typed.py,sha256=NGuDkDzFlc1fAoaXjOFZVbmj0mRFjsQi1E_hPa7Bn5U,128
 nshtrainer/util/typing_utils.py,sha256=8ptjSSLZxlmy4FY6lzzkoGoF5fGNClo8-B_c0XHQaNU,385
-nshtrainer-0.28.0.dist-info/METADATA,sha256=-4iVbXySZ9NGbwhqDWfBRdZmMNOEUx5ir1FKGrpdASg,916
-nshtrainer-0.28.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-nshtrainer-0.28.0.dist-info/RECORD,,
+nshtrainer-0.29.1.dist-info/METADATA,sha256=Qck1QY1pNnjQH9zLMyAMKVVvYMovEeIyP5zV7VlZios,916
+nshtrainer-0.29.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+nshtrainer-0.29.1.dist-info/RECORD,,