kostyl-toolkit 0.1.3__py3-none-any.whl → 0.1.6__py3-none-any.whl

kostyl/ml_core/{configs/config_base.py → clearml/config_mixin.py}

@@ -2,80 +2,17 @@ from pathlib import Path
 from typing import TypeVar
 
 import clearml
-import yaml
 from caseconverter import pascalcase
 from caseconverter import snakecase
 from pydantic import BaseModel as PydanticBaseModel
 
-from kostyl.utils import convert_to_flat_dict
-from kostyl.utils import flattened_dict_to_nested
+from kostyl.ml_core.configs.base import ConfigLoadingMixin
+from kostyl.utils.dict_manipulations import convert_to_flat_dict
+from kostyl.utils.dict_manipulations import flattened_dict_to_nested
+from kostyl.utils.fs import load_config
 
 
-def load_config(path: Path | str) -> dict:
-    """Load a configuration from file."""
-    if isinstance(path, str):
-        path = Path(path)
-
-    if not path.is_file():
-        raise ValueError(f"Config file {path} does not exist or is not a file.")
-
-    match path.suffix:
-        case ".yaml" | ".yml":
-            config = yaml.safe_load(path.open("r"))
-        case _:
-            raise ValueError(f"Unsupported config file format: {path.suffix}")
-    return config
-
-
-TConfig = TypeVar("TConfig", bound=PydanticBaseModel)
-
-
-class ConfigLoadingMixin:
-    """Pydantic mixin class providing basic configuration loading functionality."""
-
-    @classmethod
-    def from_file(
-        cls: type[TConfig],  # pyright: ignore
-        path: str | Path,
-    ) -> TConfig:
-        """
-        Create an instance of the class from a configuration file.
-
-        Args:
-            cls_: The class type to instantiate.
-            path (str | Path): Path to the configuration file.
-
-        Returns:
-            An instance of the class created from the configuration file.
-
-        """
-        config = load_config(path)
-        instance = cls.model_validate(config)
-        return instance
-
-    @classmethod
-    def from_dict(
-        cls: type[TConfig],  # pyright: ignore
-        state_dict: dict,
-    ) -> TConfig:
-        """
-        Creates an instance from a dictionary.
-
-        Args:
-            cls_: The class type to instantiate.
-            state_dict (dict): A dictionary representing the state of the
-                class that must be validated and used for initialization.
-
-        Returns:
-            An initialized instance of the class based on the
-                provided state dictionary.
-
-        """
-        instance = cls.model_validate(state_dict)
-        return instance
-
-
-TModel = TypeVar("TModel", bound="ClearMLBaseModel")
+TModel = TypeVar("TModel", bound="_ClearMLBaseModel")
 
 
 class ClearMLConfigMixin(ConfigLoadingMixin):
@@ -155,7 +92,7 @@ class ClearMLConfigMixin(ConfigLoadingMixin):
         return model
 
 
-class ClearMLBaseModel(PydanticBaseModel, ClearMLConfigMixin):
+class _ClearMLBaseModel(PydanticBaseModel, ClearMLConfigMixin):
     """A Pydantic model class with ClearML configuration loading and syncing functionality."""
 
     pass

kostyl/ml_core/clearml/dataset_utils.py

@@ -0,0 +1,61 @@
+from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import as_completed
+from pathlib import Path
+
+from clearml import Dataset as ClearMLDataset
+
+
+def collect_clearml_datasets(
+    datasets_mapping: dict[str, str],
+) -> dict[str, ClearMLDataset]:
+    """
+    Collect ClearML datasets by dataset ID.
+
+    Args:
+        datasets_mapping: Mapping where keys are human-readable names and values
+            are ClearML dataset IDs.
+
+    Returns:
+        A mapping of dataset names to fetched `ClearMLDataset` instances.
+
+    """
+    datasets_list = {}
+    for name, dataset_id in datasets_mapping.items():
+        clearml_dataset = ClearMLDataset.get(dataset_id, alias=name)
+        datasets_list[name] = clearml_dataset
+    return datasets_list
+
+
+def download_clearml_datasets(datasets_mapping: dict[str, ClearMLDataset]) -> None:
+    """
+    Download all ClearML datasets in parallel.
+
+    Args:
+        datasets_mapping: Mapping of dataset names to initialized
+            `ClearMLDataset` instances whose contents must be downloaded
+            locally.
+
+    """
+    with ThreadPoolExecutor() as executor:
+        futures = [
+            executor.submit(ds.get_local_copy) for ds in datasets_mapping.values()
+        ]
+        for future in as_completed(futures):
+            future.result()
+    return
+
+
+def get_datasets_paths(datasets_mapping: dict[str, ClearMLDataset]) -> dict[str, Path]:
+    """
+    Return local filesystem paths for ClearML datasets.
+
+    Args:
+        datasets_mapping: Mapping of dataset names to initialized
+            `ClearMLDataset` instances.
+
+    Returns:
+        Mapping of dataset names to local `Path` objects pointing to the
+        downloaded dataset copies.
+
+    """
+    return {name: Path(ds.get_local_copy()) for name, ds in datasets_mapping.items()}

kostyl/ml_core/configs/__init__.py

@@ -1,4 +1,4 @@
-from .config_base import ConfigLoadingMixin
+from .base import ConfigLoadingMixin
 from .hyperparams import HyperparamsConfig
 from .hyperparams import Lr
 from .hyperparams import Optimizer

kostyl/ml_core/configs/base.py

@@ -0,0 +1,54 @@
+from pathlib import Path
+from typing import TypeVar
+
+from pydantic import BaseModel as PydanticBaseModel
+
+from kostyl.utils.fs import load_config
+
+
+TConfig = TypeVar("TConfig", bound=PydanticBaseModel)
+
+
+class ConfigLoadingMixin:
+    """Pydantic mixin class providing basic configuration loading functionality."""
+
+    @classmethod
+    def from_file(
+        cls: type[TConfig],  # pyright: ignore
+        path: str | Path,
+    ) -> TConfig:
+        """
+        Create an instance of the class from a configuration file.
+
+        Args:
+            cls_: The class type to instantiate.
+            path (str | Path): Path to the configuration file.
+
+        Returns:
+            An instance of the class created from the configuration file.
+
+        """
+        config = load_config(path)
+        instance = cls.model_validate(config)
+        return instance
+
+    @classmethod
+    def from_dict(
+        cls: type[TConfig],  # pyright: ignore
+        state_dict: dict,
+    ) -> TConfig:
+        """
+        Creates an instance from a dictionary.
+
+        Args:
+            cls_: The class type to instantiate.
+            state_dict (dict): A dictionary representing the state of the
+                class that must be validated and used for initialization.
+
+        Returns:
+            An initialized instance of the class based on the
+                provided state dictionary.
+
+        """
+        instance = cls.model_validate(state_dict)
+        return instance

kostyl/ml_core/configs/hyperparams.py

@@ -2,10 +2,9 @@ from pydantic import BaseModel
 from pydantic import Field
 from pydantic import model_validator
 
+from kostyl.ml_core.clearml.config_mixin import ClearMLConfigMixin
 from kostyl.utils.logging import setup_logger
 
-from .config_base import ClearMLBaseModel
-
 
 logger = setup_logger(fmt="only_message")
 
@@ -75,7 +74,7 @@ class WeightDecay(BaseModel):
         return self
 
 
-class HyperparamsConfig(ClearMLBaseModel):
+class HyperparamsConfig(BaseModel, ClearMLConfigMixin):
     """Model training hyperparameters configuration."""
 
     grad_clip_val: float | None = Field(default=None, gt=0, validate_default=False)

kostyl/ml_core/configs/training_params.py

@@ -3,10 +3,10 @@ from typing import Literal
 from pydantic import BaseModel
 from pydantic import Field
 
+from kostyl.ml_core.clearml.config_mixin import ClearMLConfigMixin
 from kostyl.utils.logging import setup_logger
 
-from .config_base import ClearMLBaseModel
-from .config_base import ConfigLoadingMixin
+from .base import ConfigLoadingMixin
 
 
 logger = setup_logger(fmt="only_message")
@@ -101,8 +101,8 @@ class TrainingParams(BaseModel, ConfigLoadingMixin):
 
 
 class ClearMLTrainingParameters(
+    ClearMLConfigMixin,
     TrainingParams,
-    ClearMLBaseModel,
 ):
     """Training parameters configuration with ClearML features support (config syncing, model identifiers tracking and etc)."""
 

kostyl/ml_core/lightning/extenstions/pretrained_model.py

@@ -27,7 +27,7 @@ class LightningCheckpointLoaderMixin(PreTrainedModel):
         checkpoint_path: str | Path,
         config_key: str = "config",
         weights_prefix: str = "model.",
-        log_incompatible: bool = True,
+        should_log_incompatible_keys: bool = True,
     ) -> TModelInstance:
         """
         Load a model from a Lightning checkpoint file.
@@ -47,7 +47,7 @@ class LightningCheckpointLoaderMixin(PreTrainedModel):
                 Defaults to "config".
             weights_prefix (str, optional): Prefix to strip from state dict keys. Defaults to "model.".
                 If not empty and doesn't end with ".", a "." is appended.
-            log_incompatible (bool, optional): Whether to log incompatible keys. Defaults to True.
+            should_log_incompatible_keys (bool, optional): Whether to log incompatible keys. Defaults to True.
 
         Returns:
             TModelInstance: The loaded model instance.
@@ -110,6 +110,6 @@ class LightningCheckpointLoaderMixin(PreTrainedModel):
         )
         incompatible_keys["missing_keys"] = missing_keys
         incompatible_keys["unexpected_keys"] = unexpected_keys
-        if log_incompatible:
+        if should_log_incompatible_keys:
             log_incompatible_keys(incompatible_keys=incompatible_keys, logger=logger)
         return model

kostyl/utils/fs.py

@@ -0,0 +1,19 @@
+from pathlib import Path
+
+import yaml
+
+
+def load_config(path: Path | str) -> dict:
+    """Load a configuration from file."""
+    if isinstance(path, str):
+        path = Path(path)
+
+    if not path.is_file():
+        raise ValueError(f"Config file {path} does not exist or is not a file.")
+
+    match path.suffix:
+        case ".yaml" | ".yml":
+            config = yaml.safe_load(path.open("r"))
+        case _:
+            raise ValueError(f"Unsupported config file format: {path.suffix}")
+    return config

{kostyl_toolkit-0.1.3.dist-info → kostyl_toolkit-0.1.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: kostyl-toolkit
-Version: 0.1.3
+Version: 0.1.6
 Summary: Kickass Orchestration System for Training, Yielding & Logging 
 Requires-Dist: case-converter>=1.2.0
 Requires-Dist: loguru>=0.7.3

{kostyl_toolkit-0.1.3.dist-info → kostyl_toolkit-0.1.6.dist-info}/RECORD

@@ -1,12 +1,14 @@
 kostyl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kostyl/ml_core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kostyl/ml_core/clearml/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+kostyl/ml_core/clearml/config_mixin.py,sha256=a3YO8kocbiJMSRKcnEIYZlHe_-jvozIh0t_CFcBd1MU,3610
+kostyl/ml_core/clearml/dataset_utils.py,sha256=mkuXqLXjuxR2UUeVwV_NdToILOObOse88ItdfIyyx04,1812
 kostyl/ml_core/clearml/logging_utils.py,sha256=GBjIIZbH_itd5sj7XpvxjkyZwxxGOpEcQ3BiWaJTyq8,1210
 kostyl/ml_core/clearml/pulling_utils.py,sha256=Yf70ux8dS0_ENdvfbNQkXOrDxwd4ed2GnRCmOR2ppEk,3252
-kostyl/ml_core/configs/__init__.py,sha256=RKSHp5J8eksqMxFu5xkpSxyswSpgKhrHLjltLS3yZXc,896
-kostyl/ml_core/configs/config_base.py,sha256=ctjedEKZbwByUr5HA-Ic0dVCPWPAIPL9kK8T0S-BOvk,5276
-kostyl/ml_core/configs/hyperparams.py,sha256=iKzuFOAL3xSVGjXlvRX_mbSBt0pqh6RQAxyHPmN-Bik,2974
-kostyl/ml_core/configs/training_params.py,sha256=a8ewftu_xDatlbJ6qk_87WkuRpdThBGYQA2fHbjb9RU,2598
+kostyl/ml_core/configs/__init__.py,sha256=AcLsl_aGnUuhEtNxfZWYbrKMfJrCAf47laPNeOKPfT4,889
+kostyl/ml_core/configs/base.py,sha256=4trY_fO_VnguNrwT0BgSSoxmnG8BS_z9mEsgX6DmaNU,1467
+kostyl/ml_core/configs/hyperparams.py,sha256=u-7FIM-cD3nz9Sycuvg7r0Vdiu4pduafiCoAxq8JK0s,3011
+kostyl/ml_core/configs/training_params.py,sha256=vYZUcOPNQNGLoMD9_-RCO9t72D8z7AwUU5JumkncYBk,2618
 kostyl/ml_core/dist_utils.py,sha256=G8atjzkRbXZZiZh9rdEYBmeXqX26rJdDDovft2n6xiU,3201
 kostyl/ml_core/lightning/__init__.py,sha256=-F3JAyq8KU1d-nACWryGu8d1CbvWbQ1rXFdeRwfE2X8,175
 kostyl/ml_core/lightning/callbacks/__init__.py,sha256=Vd-rozY4T9Prr3IMqbliXxj6sC6y9XsovHQqRwzc2HI,297
@@ -15,7 +17,7 @@ kostyl/ml_core/lightning/callbacks/early_stopping.py,sha256=nEj3OkMNJkpQzR6pt0Z0
 kostyl/ml_core/lightning/callbacks/registry_uploading.py,sha256=1aqT38FVOMQo4JphXcyjyK3ZY6A6HF1JBOsKqYNXar8,4706
 kostyl/ml_core/lightning/extenstions/__init__.py,sha256=OY6QGv1agYgqqKf1xJBrxgp_i8FunVfPzYezfaRrGXU,182
 kostyl/ml_core/lightning/extenstions/custom_module.py,sha256=8Z9iT7kdfLkzBQ8nLaCIxolXytiQqFVRLDSaNACjfK8,6146
-kostyl/ml_core/lightning/extenstions/pretrained_model.py,sha256=XvTU4joUbzmUUPOLXVhNDSRs0bVkpswdeE7YDJrnQZk,4635
+kostyl/ml_core/lightning/extenstions/pretrained_model.py,sha256=67ihzuO9J5AHpuE4f4qzL8XGJnA6c4LYaDyhXAGYnaw,4671
 kostyl/ml_core/lightning/loggers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kostyl/ml_core/lightning/loggers/tb_logger.py,sha256=Zh9n-lLu-bXMld-FIUO3lJfCyDf0IQFhS3JVShDJmvg,937
 kostyl/ml_core/lightning/steps_estimation.py,sha256=fTZ0IrUEZV3H6VYlx4GYn56oco56mMiB7FO9F0Z7qc4,1511
@@ -27,7 +29,8 @@ kostyl/ml_core/schedulers/composite.py,sha256=ee4xlMDMMtjKPkbTF2ue9GTr9DuGCGjZWf
 kostyl/ml_core/schedulers/cosine.py,sha256=jufULVHn_L_ZZEc3ZTG3QCY_pc0jlAMH5Aw496T31jo,8203
 kostyl/utils/__init__.py,sha256=hkpmB6c5pr4Ti5BshOROebb7cvjDZfNCw83qZ_FFKMM,240
 kostyl/utils/dict_manipulations.py,sha256=e3vBicID74nYP8lHkVTQc4-IQwoJimrbFELy5uSF6Gk,1073
+kostyl/utils/fs.py,sha256=gAQNIU4R_2DhwjgzOS8BOMe0gZymtY1eZwmdgOdDgqo,510
 kostyl/utils/logging.py,sha256=3MvfDPArZhwakHu5nMlp_LpOsWg0E0SP26y41clsBtA,5232
-kostyl_toolkit-0.1.3.dist-info/WHEEL,sha256=YUH1mBqsx8Dh2cQG2rlcuRYUhJddG9iClegy4IgnHik,79
-kostyl_toolkit-0.1.3.dist-info/METADATA,sha256=ey3tBiOwKHe8qqlcd6n_yx0p-bmkfIw09OAXcSevOmI,4268
-kostyl_toolkit-0.1.3.dist-info/RECORD,,
+kostyl_toolkit-0.1.6.dist-info/WHEEL,sha256=AaqHSNJgTyoT6I9ETCXrbV_7cVSjA_q07lkDGeNjGdQ,79
+kostyl_toolkit-0.1.6.dist-info/METADATA,sha256=S3AJNbTWSplf75-tGR-nGqNFubXyxJb4BLVYNvvE5Rk,4268
+kostyl_toolkit-0.1.6.dist-info/RECORD,,

{kostyl_toolkit-0.1.3.dist-info → kostyl_toolkit-0.1.6.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: uv 0.9.11
+Generator: uv 0.9.12
 Root-Is-Purelib: true
 Tag: py3-none-any