kostyl-toolkit 0.1.17__py3-none-any.whl → 0.1.19__py3-none-any.whl

kostyl/ml/clearml/pulling_utils.py

@@ -46,7 +46,7 @@ def get_tokenizer_from_clearml(
     tokenizer = AutoTokenizer.from_pretrained(
         clearml_tokenizer.get_local_copy(raise_on_error=True)
     )
-    return tokenizer
+    return tokenizer, clearml_tokenizer
 
 
 def get_model_from_clearml[

kostyl/ml/dist_utils.py

@@ -96,4 +96,12 @@ def _get_rank() -> int:
 
 def is_main_process() -> bool:
     """Checks if the current process is the main process (rank 0) in a distributed setting."""
-    return _get_rank() == 0
+    if dist.is_initialized():
+        return dist.get_rank() == 0
+    if "RANK" in os.environ:
+        return int(os.environ["RANK"]) == 0
+    if "SLURM_PROCID" in os.environ:
+        return int(os.environ["SLURM_PROCID"]) == 0
+    if "LOCAL_RANK" in os.environ:
+        return int(os.environ["LOCAL_RANK"]) == 0
+    return True

kostyl/ml/lightning/loggers/tb_logger.py

@@ -7,7 +7,7 @@ from kostyl.ml.dist_utils import is_main_process
 from kostyl.utils.logging import setup_logger
 
 
-logger = setup_logger("callbacks/tb_logger.py", add_rank=True)
+logger = setup_logger()
 
 
 def setup_tb_logger(

{kostyl_toolkit-0.1.17.dist-info → kostyl_toolkit-0.1.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: kostyl-toolkit
-Version: 0.1.17
+Version: 0.1.19
 Summary: Kickass Orchestration System for Training, Yielding & Logging 
 Requires-Dist: case-converter>=1.2.0
 Requires-Dist: loguru>=0.7.3

{kostyl_toolkit-0.1.17.dist-info → kostyl_toolkit-0.1.19.dist-info}/RECORD

@@ -3,12 +3,12 @@ kostyl/ml/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kostyl/ml/clearml/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kostyl/ml/clearml/dataset_utils.py,sha256=eij_sr2KDhm8GxEbVbK8aBjPsuVvLl9-PIGGaKVgXLA,1729
 kostyl/ml/clearml/logging_utils.py,sha256=GBjIIZbH_itd5sj7XpvxjkyZwxxGOpEcQ3BiWaJTyq8,1210
-kostyl/ml/clearml/pulling_utils.py,sha256=I6B8YF3gv9QXsggMRD2RSA2bcEcPUUhsnQjZf-uI0es,4058
+kostyl/ml/clearml/pulling_utils.py,sha256=07bb7ZYlZy-qoZLn7uWZCtz02eX2idgk3JA-PPooS9E,4077
 kostyl/ml/configs/__init__.py,sha256=IetcivbqYGutowLqxdKp7QR4tkXKBr4m8t4Zkk9jHZU,911
 kostyl/ml/configs/base_model.py,sha256=Eofn14J9RsjpVx_J4rp6C19pDDCANU4hr3JtX-d0FpQ,4820
 kostyl/ml/configs/hyperparams.py,sha256=CaVNEvpW4LvlHhLsbe2FockIGI1mJufCqjH298nYgKE,2971
 kostyl/ml/configs/training_settings.py,sha256=0cyKF6EuTv6KgXC1g0Oy9zbwnMkDP4uXTJJO1TRQ0aY,2556
-kostyl/ml/dist_utils.py,sha256=G8atjzkRbXZZiZh9rdEYBmeXqX26rJdDDovft2n6xiU,3201
+kostyl/ml/dist_utils.py,sha256=XKshB_9j4G8BLN_fU2sZtvLk4jsgjPb8z_XOhrEGAaI,3502
 kostyl/ml/lightning/__init__.py,sha256=-F3JAyq8KU1d-nACWryGu8d1CbvWbQ1rXFdeRwfE2X8,175
 kostyl/ml/lightning/callbacks/__init__.py,sha256=Vd-rozY4T9Prr3IMqbliXxj6sC6y9XsovHQqRwzc2HI,297
 kostyl/ml/lightning/callbacks/checkpoint.py,sha256=FooGeeUz6TtoXQglpcK16NWAmSX3fbu6wntRtK3a_Io,1936
@@ -18,7 +18,7 @@ kostyl/ml/lightning/extenstions/__init__.py,sha256=OY6QGv1agYgqqKf1xJBrxgp_i8Fun
 kostyl/ml/lightning/extenstions/custom_module.py,sha256=nB5jW7cqRD1tyh-q5LD2EtiFQwFkLXpnS9Yu6c5xMRg,5987
 kostyl/ml/lightning/extenstions/pretrained_model.py,sha256=ZOKtrVl095cwvI43wAz-Xdzu4l0v0lHH2mfh4WXwxKQ,5059
 kostyl/ml/lightning/loggers/__init__.py,sha256=e51dszaoJbuzwBkbdugmuDsPldoSO4yaRgmZUg1Bdy0,71
-kostyl/ml/lightning/loggers/tb_logger.py,sha256=I0WQV3SVVIds9kHhAYnN3SXodX64jfTk7_5gH-j3uzA,932
+kostyl/ml/lightning/loggers/tb_logger.py,sha256=j02HK5ue8yzXXV8FWKmmXyHkFlIxgHx-ahHWk_rFCZs,893
 kostyl/ml/lightning/steps_estimation.py,sha256=fTZ0IrUEZV3H6VYlx4GYn56oco56mMiB7FO9F0Z7qc4,1511
 kostyl/ml/metrics_formatting.py,sha256=w0rTz61z0Um_d2pomYLvcQFcZX_C-KolZcIPRsa1efE,1421
 kostyl/ml/params_groups.py,sha256=nUyw5d06Pvy9QPiYtZzLYR87xwXqJLxbHthgQH8oSCM,3583
@@ -30,6 +30,6 @@ kostyl/utils/__init__.py,sha256=hkpmB6c5pr4Ti5BshOROebb7cvjDZfNCw83qZ_FFKMM,240
 kostyl/utils/dict_manipulations.py,sha256=e3vBicID74nYP8lHkVTQc4-IQwoJimrbFELy5uSF6Gk,1073
 kostyl/utils/fs.py,sha256=gAQNIU4R_2DhwjgzOS8BOMe0gZymtY1eZwmdgOdDgqo,510
 kostyl/utils/logging.py,sha256=3MvfDPArZhwakHu5nMlp_LpOsWg0E0SP26y41clsBtA,5232
-kostyl_toolkit-0.1.17.dist-info/WHEEL,sha256=3id4o64OvRm9dUknh3mMJNcfoTRK08ua5cU6DFyVy-4,79
-kostyl_toolkit-0.1.17.dist-info/METADATA,sha256=T547zPqZpJcgiXR2SAloz2W7_MCNhhhxlTsgzdWkXlo,4269
-kostyl_toolkit-0.1.17.dist-info/RECORD,,
+kostyl_toolkit-0.1.19.dist-info/WHEEL,sha256=3id4o64OvRm9dUknh3mMJNcfoTRK08ua5cU6DFyVy-4,79
+kostyl_toolkit-0.1.19.dist-info/METADATA,sha256=zAq1MkJ8Wt88R_Zlv5O_pTNNib80qVuKC4rZjtCFhO8,4269
+kostyl_toolkit-0.1.19.dist-info/RECORD,,