kostyl-toolkit 0.1.4__tar.gz → 0.1.7__tar.gz

{kostyl_toolkit-0.1.4 → kostyl_toolkit-0.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: kostyl-toolkit
-Version: 0.1.4
+Version: 0.1.7
 Summary: Kickass Orchestration System for Training, Yielding & Logging 
 Requires-Dist: case-converter>=1.2.0
 Requires-Dist: loguru>=0.7.3

kostyl_toolkit-0.1.7/kostyl/ml_core/clearml/dataset_utils.py

@@ -0,0 +1,58 @@
+from collections.abc import Collection
+from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import as_completed
+from pathlib import Path
+
+from clearml import Dataset as ClearMLDataset
+
+
+def collect_clearml_datasets(
+    datasets_mapping: dict[str, str],
+) -> dict[str, ClearMLDataset]:
+    """
+    Collect ClearML datasets by dataset ID.
+
+    Args:
+        datasets_mapping: Mapping where keys are human-readable names and values
+            are ClearML dataset IDs.
+
+    Returns:
+        A mapping of dataset names to fetched `ClearMLDataset` instances.
+
+    """
+    datasets_list = {}
+    for name, dataset_id in datasets_mapping.items():
+        clearml_dataset = ClearMLDataset.get(dataset_id, alias=name)
+        datasets_list[name] = clearml_dataset
+    return datasets_list
+
+
+def download_clearml_datasets(datasets: Collection[ClearMLDataset]) -> None:
+    """
+    Download all ClearML datasets in parallel.
+
+    Args:
+        datasets: Collection of initialized `ClearMLDataset` instances to download.
+
+    """
+    with ThreadPoolExecutor() as executor:
+        futures = [executor.submit(ds.get_local_copy) for ds in datasets]
+        for future in as_completed(futures):
+            future.result()
+    return
+
+
+def get_datasets_paths(datasets_mapping: dict[str, ClearMLDataset]) -> dict[str, Path]:
+    """
+    Return local filesystem paths for ClearML datasets.
+
+    Args:
+        datasets_mapping: Mapping of dataset names to initialized
+            `ClearMLDataset` instances.
+
+    Returns:
+        Mapping of dataset names to local `Path` objects pointing to the
+        downloaded dataset copies.
+
+    """
+    return {name: Path(ds.get_local_copy()) for name, ds in datasets_mapping.items()}

kostyl_toolkit-0.1.7/kostyl/ml_core/configs/__init__.py

@@ -0,0 +1,30 @@
+from .base import ConfigLoadingMixin
+from .hyperparams import HyperparamsConfig
+from .hyperparams import Lr
+from .hyperparams import Optimizer
+from .hyperparams import WeightDecay
+from .training_settings import CheckpointConfig
+from .training_settings import ClearMLTrainingSettings
+from .training_settings import DataConfig
+from .training_settings import DDPStrategyConfig
+from .training_settings import EarlyStoppingConfig
+from .training_settings import FSDP1StrategyConfig
+from .training_settings import SingleDeviceStrategyConfig
+from .training_settings import TrainingSettings
+
+
+__all__ = [
+    "CheckpointConfig",
+    "ClearMLTrainingSettings",
+    "ConfigLoadingMixin",
+    "DDPStrategyConfig",
+    "DataConfig",
+    "EarlyStoppingConfig",
+    "FSDP1StrategyConfig",
+    "HyperparamsConfig",
+    "Lr",
+    "Optimizer",
+    "SingleDeviceStrategyConfig",
+    "TrainingSettings",
+    "WeightDecay",
+]

{kostyl_toolkit-0.1.4 → kostyl_toolkit-0.1.7}/kostyl/ml_core/configs/hyperparams.py

@@ -2,10 +2,9 @@ from pydantic import BaseModel
 from pydantic import Field
 from pydantic import model_validator
 
+from kostyl.ml_core.clearml.config_mixin import ClearMLConfigMixin
 from kostyl.utils.logging import setup_logger
 
-from .base import ClearMLBaseModel
-
 
 logger = setup_logger(fmt="only_message")
 
@@ -75,7 +74,7 @@ class WeightDecay(BaseModel):
         return self
 
 
-class HyperparamsConfig(ClearMLBaseModel):
+class HyperparamsConfig(BaseModel, ClearMLConfigMixin):
     """Model training hyperparameters configuration."""
 
     grad_clip_val: float | None = Field(default=None, gt=0, validate_default=False)

kostyl_toolkit-0.1.4/kostyl/ml_core/configs/training_params.py → kostyl_toolkit-0.1.7/kostyl/ml_core/configs/training_settings.py

@@ -49,6 +49,7 @@ class DDPStrategyConfig(BaseModel):
     """Distributed Data Parallel (DDP) strategy configuration."""
 
     type: Literal["ddp"]
+    find_unused_parameters: bool = False
 
 
 class LightningTrainerParameters(BaseModel):
@@ -91,7 +92,7 @@ class DataConfig(BaseModel):
     data_columns: list[str]
 
 
-class TrainingParams(BaseModel, ConfigLoadingMixin):
+class TrainingSettings(BaseModel, ConfigLoadingMixin):
     """Training parameters configuration."""
 
     trainer: LightningTrainerParameters
@@ -100,9 +101,9 @@ class TrainingParams(BaseModel, ConfigLoadingMixin):
     data: DataConfig
 
 
-class ClearMLTrainingParameters(
+class ClearMLTrainingSettings(
     ClearMLConfigMixin,
-    TrainingParams,
+    TrainingSettings,
 ):
     """Training parameters configuration with ClearML features support (config syncing, model identifiers tracking and etc)."""
 

{kostyl_toolkit-0.1.4 → kostyl_toolkit-0.1.7}/kostyl/ml_core/lightning/extenstions/pretrained_model.py

@@ -9,7 +9,7 @@ from transformers import PreTrainedModel
 try:
     from peft import PeftConfig
 except ImportError:
-    PeftConfig = None
+    PeftConfig = None  # ty: ignore
 
 from kostyl.utils.logging import log_incompatible_keys
 from kostyl.utils.logging import setup_logger
@@ -27,7 +27,7 @@ class LightningCheckpointLoaderMixin(PreTrainedModel):
         checkpoint_path: str | Path,
         config_key: str = "config",
         weights_prefix: str = "model.",
-        log_incompatible: bool = True,
+        should_log_incompatible_keys: bool = True,
     ) -> TModelInstance:
         """
         Load a model from a Lightning checkpoint file.
@@ -47,7 +47,7 @@ class LightningCheckpointLoaderMixin(PreTrainedModel):
                 Defaults to "config".
             weights_prefix (str, optional): Prefix to strip from state dict keys. Defaults to "model.".
                 If not empty and doesn't end with ".", a "." is appended.
-            log_incompatible (bool, optional): Whether to log incompatible keys. Defaults to True.
+            should_log_incompatible_keys (bool, optional): Whether to log incompatible keys. Defaults to True.
 
         Returns:
             TModelInstance: The loaded model instance.
@@ -110,6 +110,6 @@ class LightningCheckpointLoaderMixin(PreTrainedModel):
         )
         incompatible_keys["missing_keys"] = missing_keys
         incompatible_keys["unexpected_keys"] = unexpected_keys
-        if log_incompatible:
+        if should_log_incompatible_keys:
             log_incompatible_keys(incompatible_keys=incompatible_keys, logger=logger)
         return model

{kostyl_toolkit-0.1.4 → kostyl_toolkit-0.1.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "kostyl-toolkit"
-version = "0.1.4"
+version = "0.1.7"
 description = "Kickass Orchestration System for Training, Yielding & Logging "
 readme = "README.md"
 requires-python = ">=3.12"

kostyl_toolkit-0.1.4/kostyl/ml_core/configs/__init__.py

@@ -1,30 +0,0 @@
-from .base import ConfigLoadingMixin
-from .hyperparams import HyperparamsConfig
-from .hyperparams import Lr
-from .hyperparams import Optimizer
-from .hyperparams import WeightDecay
-from .training_params import CheckpointConfig
-from .training_params import ClearMLTrainingParameters
-from .training_params import DataConfig
-from .training_params import DDPStrategyConfig
-from .training_params import EarlyStoppingConfig
-from .training_params import FSDP1StrategyConfig
-from .training_params import SingleDeviceStrategyConfig
-from .training_params import TrainingParams
-
-
-__all__ = [
-    "CheckpointConfig",
-    "ClearMLTrainingParameters",
-    "ConfigLoadingMixin",
-    "DDPStrategyConfig",
-    "DataConfig",
-    "EarlyStoppingConfig",
-    "FSDP1StrategyConfig",
-    "HyperparamsConfig",
-    "Lr",
-    "Optimizer",
-    "SingleDeviceStrategyConfig",
-    "TrainingParams",
-    "WeightDecay",
-]