mlrun 1.6.4rc8__py3-none-any.whl → 1.7.0__py3-none-any.whl

mlrun/frameworks/parallel_coordinates.py

@@ -14,12 +14,11 @@
 #
 import datetime
 import os
-from typing import List, Union
+from typing import Union
 
 import numpy as np
 import pandas as pd
-from IPython.core.display import HTML
-from IPython.display import display
+from IPython.display import HTML, display
 from pandas.api.types import is_numeric_dtype, is_string_dtype
 
 import mlrun
@@ -216,7 +215,7 @@ def _show_and_export_html(html: str, show=None, filename=None, runs_list=None):
                 fp.write("</body></html>")
             else:
                 fp.write(html)
-    if show or (show is None and mlrun.utils.is_ipython):
+    if show or (show is None and mlrun.utils.is_jupyter):
         display(HTML(html))
         if runs_list and len(runs_list) <= max_table_rows:
             display(HTML(html_table))
@@ -239,7 +238,7 @@ def _runs_list_to_df(runs_list, extend_iterations=False):
 
 @filter_warnings("ignore", FutureWarning)
 def compare_run_objects(
-    runs_list: Union[mlrun.model.RunObject, List[mlrun.model.RunObject]],
+    runs_list: Union[mlrun.model.RunObject, list[mlrun.model.RunObject]],
     hide_identical: bool = True,
     exclude: list = None,
     show: bool = None,
@@ -295,7 +294,7 @@ def compare_db_runs(
     iter=False,
     start_time_from: datetime = None,
     hide_identical: bool = True,
-    exclude: list = [],
+    exclude: list = None,
     show=None,
     colorscale: str = "Blues",
     filename=None,
@@ -332,6 +331,7 @@ def compare_db_runs(
         **query_args,
     )
 
+    exclude = exclude or []
     runs_df = _runs_list_to_df(runs_list)
     plot_as_html = gen_pcp_plot(
         runs_df,

mlrun/frameworks/pytorch/__init__.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 # flake8: noqa  - this is until we take care of the F401 violations with respect to __all__ & sphinx
-from typing import Any, Dict, List, Tuple, Union
+from typing import Any, Union
 
 from torch.nn import Module
 from torch.optim import Optimizer
@@ -35,23 +35,23 @@ def train(
     loss_function: Module,
     optimizer: Optimizer,
     validation_set: DataLoader = None,
-    metric_functions: List[PyTorchTypes.MetricFunctionType] = None,
+    metric_functions: list[PyTorchTypes.MetricFunctionType] = None,
     scheduler=None,
     scheduler_step_frequency: Union[int, float, str] = "epoch",
     epochs: int = 1,
     training_iterations: int = None,
     validation_iterations: int = None,
-    callbacks_list: List[Callback] = None,
+    callbacks_list: list[Callback] = None,
     use_cuda: bool = True,
     use_horovod: bool = None,
     auto_log: bool = True,
     model_name: str = None,
-    modules_map: Union[Dict[str, Union[None, str, List[str]]], str] = None,
-    custom_objects_map: Union[Dict[str, Union[str, List[str]]], str] = None,
+    modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
+    custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
     custom_objects_directory: str = None,
     tensorboard_directory: str = None,
-    mlrun_callback_kwargs: Dict[str, Any] = None,
-    tensorboard_callback_kwargs: Dict[str, Any] = None,
+    mlrun_callback_kwargs: dict[str, Any] = None,
+    tensorboard_callback_kwargs: dict[str, Any] = None,
     context: mlrun.MLClientCtx = None,
 ) -> PyTorchModelHandler:
     """
@@ -112,7 +112,7 @@ def train(
 
                                             {
                                                 "/.../custom_optimizer.py": "optimizer",
-                                                "/.../custom_layers.py": ["layer1", "layer2"]
+                                                "/.../custom_layers.py": ["layer1", "layer2"],
                                             }
 
                                         All the paths will be accessed from the given 'custom_objects_directory',
@@ -205,19 +205,19 @@ def evaluate(
     dataset: DataLoader,
     model: Module = None,
     loss_function: Module = None,
-    metric_functions: List[PyTorchTypes.MetricFunctionType] = None,
+    metric_functions: list[PyTorchTypes.MetricFunctionType] = None,
     iterations: int = None,
-    callbacks_list: List[Callback] = None,
+    callbacks_list: list[Callback] = None,
     use_cuda: bool = True,
     use_horovod: bool = False,
     auto_log: bool = True,
     model_name: str = None,
-    modules_map: Union[Dict[str, Union[None, str, List[str]]], str] = None,
-    custom_objects_map: Union[Dict[str, Union[str, List[str]]], str] = None,
+    modules_map: Union[dict[str, Union[None, str, list[str]]], str] = None,
+    custom_objects_map: Union[dict[str, Union[str, list[str]]], str] = None,
     custom_objects_directory: str = None,
-    mlrun_callback_kwargs: Dict[str, Any] = None,
+    mlrun_callback_kwargs: dict[str, Any] = None,
     context: mlrun.MLClientCtx = None,
-) -> Tuple[PyTorchModelHandler, List[PyTorchTypes.MetricValueType]]:
+) -> tuple[PyTorchModelHandler, list[PyTorchTypes.MetricValueType]]:
     """
     Use MLRun's PyTorch interface to evaluate the model with the given parameters. For more information and further
     options regarding the auto logging, see 'PyTorchMLRunInterface' documentation. Notice for auto-logging: In order to
@@ -264,7 +264,7 @@ def evaluate(
 
                                          {
                                              "/.../custom_optimizer.py": "optimizer",
-                                             "/.../custom_layers.py": ["layer1", "layer2"]
+                                             "/.../custom_layers.py": ["layer1", "layer2"],
                                          }
 
                                      All the paths will be accessed from the given 'custom_objects_directory', meaning
@@ -343,9 +343,9 @@ def evaluate(
 def _parse_callbacks_kwargs(
     handler: PyTorchModelHandler,
     tensorboard_directory: Union[str, None],
-    mlrun_callback_kwargs: Union[Dict[str, Any], None],
-    tensorboard_callback_kwargs: Union[Dict[str, Any], None],
-) -> Tuple[dict, dict]:
+    mlrun_callback_kwargs: Union[dict[str, Any], None],
+    tensorboard_callback_kwargs: Union[dict[str, Any], None],
+) -> tuple[dict, dict]:
     """
     Parse the given parameters into the MLRun and Tensorboard callbacks kwargs.
 

mlrun/frameworks/pytorch/callbacks/callback.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 #
 from abc import ABC, abstractmethod
-from typing import List
 
 from torch import Tensor
 from torch.nn import Module
@@ -68,7 +67,7 @@ class Callback(ABC):
         validation_set: DataLoader = None,
         loss_function: Module = None,
         optimizer: Optimizer = None,
-        metric_functions: List[PyTorchTypes.MetricFunctionType] = None,
+        metric_functions: list[PyTorchTypes.MetricFunctionType] = None,
         scheduler=None,
     ):
         """
@@ -141,7 +140,7 @@ class Callback(ABC):
         pass
 
     def on_validation_end(
-        self, loss_value: PyTorchTypes.MetricValueType, metric_values: List[float]
+        self, loss_value: PyTorchTypes.MetricValueType, metric_values: list[float]
     ) -> bool:
         """
         Before the validation (in a training case it will be per epoch) ends, this method will be called.
@@ -258,7 +257,7 @@ class Callback(ABC):
         """
         pass
 
-    def on_train_metrics_end(self, metric_values: List[PyTorchTypes.MetricValueType]):
+    def on_train_metrics_end(self, metric_values: list[PyTorchTypes.MetricValueType]):
         """
         After the training calculation of the metrics, this method will be called.
 
@@ -273,7 +272,7 @@ class Callback(ABC):
         pass
 
     def on_validation_metrics_end(
-        self, metric_values: List[PyTorchTypes.MetricValueType]
+        self, metric_values: list[PyTorchTypes.MetricValueType]
     ):
         """
         After the validating calculation of the metrics, this method will be called.

mlrun/frameworks/pytorch/callbacks/logging_callback.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from typing import Callable, Dict, List, Tuple, Union
+from typing import Callable, Union
 
 import numpy as np
 from torch import Tensor
@@ -59,15 +59,15 @@ class LoggingCallback(Callback):
     def __init__(
         self,
         context: mlrun.MLClientCtx = None,
-        dynamic_hyperparameters: Dict[
+        dynamic_hyperparameters: dict[
             str,
-            Tuple[
+            tuple[
                 str,
-                Union[List[Union[str, int]], Callable[[], PyTorchTypes.TrackableType]],
+                Union[list[Union[str, int]], Callable[[], PyTorchTypes.TrackableType]],
             ],
         ] = None,
-        static_hyperparameters: Dict[
-            str, Union[PyTorchTypes.TrackableType, Tuple[str, List[Union[str, int]]]]
+        static_hyperparameters: dict[
+            str, Union[PyTorchTypes.TrackableType, tuple[str, list[Union[str, int]]]]
         ] = None,
         auto_log: bool = False,
     ):
@@ -100,7 +100,7 @@ class LoggingCallback(Callback):
         :param auto_log:                Whether or not to enable auto logging, trying to track common static and dynamic
                                         hyperparameters.
         """
-        super(LoggingCallback, self).__init__()
+        super().__init__()
 
         # Store the configurations:
         self._dynamic_hyperparameters_keys = (
@@ -117,7 +117,7 @@ class LoggingCallback(Callback):
         self._is_training = None  # type: bool
         self._auto_log = auto_log
 
-    def get_training_results(self) -> Dict[str, List[List[float]]]:
+    def get_training_results(self) -> dict[str, list[list[float]]]:
         """
         Get the training results logged. The results will be stored in a dictionary where each key is the metric name
         and the value is a list of lists of values. The first list is by epoch and the second list is by iteration
@@ -127,7 +127,7 @@ class LoggingCallback(Callback):
         """
         return self._logger.training_results
 
-    def get_validation_results(self) -> Dict[str, List[List[float]]]:
+    def get_validation_results(self) -> dict[str, list[list[float]]]:
         """
         Get the validation results logged. The results will be stored in a dictionary where each key is the metric name
         and the value is a list of lists of values. The first list is by epoch and the second list is by iteration
@@ -137,7 +137,7 @@ class LoggingCallback(Callback):
         """
         return self._logger.validation_results
 
-    def get_static_hyperparameters(self) -> Dict[str, PyTorchTypes.TrackableType]:
+    def get_static_hyperparameters(self) -> dict[str, PyTorchTypes.TrackableType]:
         """
         Get the static hyperparameters logged. The hyperparameters will be stored in a dictionary where each key is the
         hyperparameter name and the value is his logged value.
@@ -148,7 +148,7 @@ class LoggingCallback(Callback):
 
     def get_dynamic_hyperparameters(
         self,
-    ) -> Dict[str, List[PyTorchTypes.TrackableType]]:
+    ) -> dict[str, list[PyTorchTypes.TrackableType]]:
         """
         Get the dynamic hyperparameters logged. The hyperparameters will be stored in a dictionary where each key is the
         hyperparameter name and the value is a list of his logged values per epoch.
@@ -157,7 +157,7 @@ class LoggingCallback(Callback):
         """
         return self._logger.dynamic_hyperparameters
 
-    def get_summaries(self) -> Dict[str, List[float]]:
+    def get_summaries(self) -> dict[str, list[float]]:
         """
         Get the validation summaries of the metrics results. The summaries will be stored in a dictionary where each key
         is the metric names and the value is a list of all the summary values per epoch.
@@ -210,7 +210,7 @@ class LoggingCallback(Callback):
             self._add_auto_hyperparameters()
         # # Static hyperparameters:
         for name, value in self._static_hyperparameters_keys.items():
-            if isinstance(value, Tuple):
+            if isinstance(value, tuple):
                 # Its a parameter that needed to be extracted via key chain.
                 self._logger.log_static_hyperparameter(
                     parameter_name=name,
@@ -294,7 +294,7 @@ class LoggingCallback(Callback):
             self._logger.set_mode(mode=LoggingMode.EVALUATION)
 
     def on_validation_end(
-        self, loss_value: PyTorchTypes.MetricValueType, metric_values: List[float]
+        self, loss_value: PyTorchTypes.MetricValueType, metric_values: list[float]
     ):
         """
         Before the validation (in a training case it will be per epoch) ends, this method will be called to log the
@@ -372,7 +372,7 @@ class LoggingCallback(Callback):
             result=float(loss_value),
         )
 
-    def on_train_metrics_end(self, metric_values: List[PyTorchTypes.MetricValueType]):
+    def on_train_metrics_end(self, metric_values: list[PyTorchTypes.MetricValueType]):
         """
         After the training calculation of the metrics, this method will be called to log the metrics values.
 
@@ -389,7 +389,7 @@ class LoggingCallback(Callback):
             )
 
     def on_validation_metrics_end(
-        self, metric_values: List[PyTorchTypes.MetricValueType]
+        self, metric_values: list[PyTorchTypes.MetricValueType]
     ):
         """
         After the validating calculation of the metrics, this method will be called to log the metrics values.
@@ -456,7 +456,7 @@ class LoggingCallback(Callback):
         self,
         source: str,
         key_chain: Union[
-            List[Union[str, int]], Callable[[], PyTorchTypes.TrackableType]
+            list[Union[str, int]], Callable[[], PyTorchTypes.TrackableType]
         ],
     ) -> PyTorchTypes.TrackableType:
         """

mlrun/frameworks/pytorch/callbacks/mlrun_logging_callback.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from typing import Callable, Dict, List, Tuple, Union
+from typing import Callable, Union
 
 import torch
 from torch import Tensor
@@ -53,20 +53,20 @@ class MLRunLoggingCallback(LoggingCallback):
         context: mlrun.MLClientCtx,
         model_handler: PyTorchModelHandler,
         log_model_tag: str = "",
-        log_model_labels: Dict[str, PyTorchTypes.TrackableType] = None,
-        log_model_parameters: Dict[str, PyTorchTypes.TrackableType] = None,
-        log_model_extra_data: Dict[
+        log_model_labels: dict[str, PyTorchTypes.TrackableType] = None,
+        log_model_parameters: dict[str, PyTorchTypes.TrackableType] = None,
+        log_model_extra_data: dict[
             str, Union[PyTorchTypes.TrackableType, Artifact]
         ] = None,
-        dynamic_hyperparameters: Dict[
+        dynamic_hyperparameters: dict[
             str,
-            Tuple[
+            tuple[
                 str,
-                Union[List[Union[str, int]], Callable[[], PyTorchTypes.TrackableType]],
+                Union[list[Union[str, int]], Callable[[], PyTorchTypes.TrackableType]],
             ],
         ] = None,
-        static_hyperparameters: Dict[
-            str, Union[PyTorchTypes.TrackableType, Tuple[str, List[Union[str, int]]]]
+        static_hyperparameters: dict[
+            str, Union[PyTorchTypes.TrackableType, tuple[str, list[Union[str, int]]]]
         ] = None,
         auto_log: bool = False,
     ):
@@ -107,7 +107,7 @@ class MLRunLoggingCallback(LoggingCallback):
         :param auto_log:                 Whether or not to enable auto logging for logging the context parameters and
                                          trying to track common static and dynamic hyperparameters.
         """
-        super(MLRunLoggingCallback, self).__init__(
+        super().__init__(
             dynamic_hyperparameters=dynamic_hyperparameters,
             static_hyperparameters=static_hyperparameters,
             auto_log=auto_log,
@@ -160,7 +160,7 @@ class MLRunLoggingCallback(LoggingCallback):
 
         :param epoch: The epoch that has just ended.
         """
-        super(MLRunLoggingCallback, self).on_epoch_end(epoch=epoch)
+        super().on_epoch_end(epoch=epoch)
 
         # Create child context to hold the current epoch's results:
         self._logger.log_epoch_to_context(epoch=epoch)

mlrun/frameworks/pytorch/callbacks/tensorboard_logging_callback.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 from datetime import datetime
-from typing import Callable, Dict, List, Tuple, Union
+from typing import Callable, Union
 
 import torch
 from torch import Tensor
@@ -63,7 +63,7 @@ class _PyTorchTensorboardLogger(TensorboardLogger):
 
     def __init__(
         self,
-        statistics_functions: List[
+        statistics_functions: list[
             Callable[[Union[Parameter]], Union[float, Parameter]]
         ],
         context: mlrun.MLClientCtx = None,
@@ -94,7 +94,7 @@ class _PyTorchTensorboardLogger(TensorboardLogger):
                                       update. Notice that writing to tensorboard too frequently may cause the training
                                       to be slower. Default: 'epoch'.
         """
-        super(_PyTorchTensorboardLogger, self).__init__(
+        super().__init__(
             statistics_functions=statistics_functions,
             context=context,
             tensorboard_directory=tensorboard_directory,
@@ -249,19 +249,19 @@ class TensorboardLoggingCallback(LoggingCallback):
         context: mlrun.MLClientCtx = None,
         tensorboard_directory: str = None,
         run_name: str = None,
-        weights: Union[bool, List[str]] = False,
-        statistics_functions: List[
+        weights: Union[bool, list[str]] = False,
+        statistics_functions: list[
             Callable[[Union[Parameter, Tensor]], Union[float, Tensor]]
         ] = None,
-        dynamic_hyperparameters: Dict[
+        dynamic_hyperparameters: dict[
             str,
-            Tuple[
+            tuple[
                 str,
-                Union[List[Union[str, int]], Callable[[], PyTorchTypes.TrackableType]],
+                Union[list[Union[str, int]], Callable[[], PyTorchTypes.TrackableType]],
             ],
         ] = None,
-        static_hyperparameters: Dict[
-            str, Union[PyTorchTypes.TrackableType, Tuple[str, List[Union[str, int]]]]
+        static_hyperparameters: dict[
+            str, Union[PyTorchTypes.TrackableType, tuple[str, list[Union[str, int]]]]
         ] = None,
         update_frequency: Union[int, str] = "epoch",
         auto_log: bool = False,
@@ -322,7 +322,7 @@ class TensorboardLoggingCallback(LoggingCallback):
         :raise MLRunInvalidArgumentError: In case both 'context' and 'tensorboard_directory' parameters were not given
                                           or the 'update_frequency' was incorrect.
         """
-        super(TensorboardLoggingCallback, self).__init__(
+        super().__init__(
             dynamic_hyperparameters=dynamic_hyperparameters,
             static_hyperparameters=static_hyperparameters,
             auto_log=auto_log,
@@ -345,7 +345,7 @@ class TensorboardLoggingCallback(LoggingCallback):
         # Save the configurations:
         self._tracked_weights = weights
 
-    def get_weights(self) -> Dict[str, Parameter]:
+    def get_weights(self) -> dict[str, Parameter]:
         """
         Get the weights tensors tracked. The weights will be stored in a dictionary where each key is the weight's name
         and the value is the weight's parameter (tensor).
@@ -354,7 +354,7 @@ class TensorboardLoggingCallback(LoggingCallback):
         """
         return self._logger.weights
 
-    def get_weights_statistics(self) -> Dict[str, Dict[str, List[float]]]:
+    def get_weights_statistics(self) -> dict[str, dict[str, list[float]]]:
         """
         Get the weights mean results logged. The results will be stored in a dictionary where each key is the weight's
         name and the value is a list of mean values per epoch.
@@ -365,7 +365,7 @@ class TensorboardLoggingCallback(LoggingCallback):
 
     @staticmethod
     def get_default_weight_statistics_list() -> (
-        List[Callable[[Union[Parameter, Tensor]], Union[float, Tensor]]]
+        list[Callable[[Union[Parameter, Tensor]], Union[float, Tensor]]]
     ):
         """
         Get the default list of statistics functions being applied on the tracked weights each epoch.
@@ -381,7 +381,7 @@ class TensorboardLoggingCallback(LoggingCallback):
         validation_set: DataLoader = None,
         loss_function: Module = None,
         optimizer: Optimizer = None,
-        metric_functions: List[PyTorchTypes.MetricFunctionType] = None,
+        metric_functions: list[PyTorchTypes.MetricFunctionType] = None,
         scheduler=None,
     ):
         """
@@ -396,7 +396,7 @@ class TensorboardLoggingCallback(LoggingCallback):
         :param metric_functions: The metric functions to be stored in this callback.
         :param scheduler:        The scheduler to be stored in this callback.
         """
-        super(TensorboardLoggingCallback, self).on_setup(
+        super().on_setup(
             model=model,
             training_set=training_set,
             validation_set=validation_set,
@@ -439,7 +439,7 @@ class TensorboardLoggingCallback(LoggingCallback):
         for logging. Epoch 0 (pre-run state) will be logged here.
         """
         # Setup all the results and hyperparameters dictionaries:
-        super(TensorboardLoggingCallback, self).on_run_begin()
+        super().on_run_begin()
 
         # Log the initial summary of the run:
         self._logger.write_initial_summary_text()
@@ -470,10 +470,10 @@ class TensorboardLoggingCallback(LoggingCallback):
         # Write the final summary of the run:
         self._logger.write_final_summary_text()
 
-        super(TensorboardLoggingCallback, self).on_run_end()
+        super().on_run_end()
 
     def on_validation_end(
-        self, loss_value: PyTorchTypes.MetricValueType, metric_values: List[float]
+        self, loss_value: PyTorchTypes.MetricValueType, metric_values: list[float]
     ):
         """
         Before the validation (in a training case it will be per epoch) ends, this method will be called to log the
@@ -482,9 +482,7 @@ class TensorboardLoggingCallback(LoggingCallback):
         :param loss_value:    The loss summary of this validation.
         :param metric_values: The metrics summaries of this validation.
         """
-        super(TensorboardLoggingCallback, self).on_validation_end(
-            loss_value=loss_value, metric_values=metric_values
-        )
+        super().on_validation_end(loss_value=loss_value, metric_values=metric_values)
 
         # Check if this run was part of an evaluation:
         if not self._is_training:
@@ -503,7 +501,7 @@ class TensorboardLoggingCallback(LoggingCallback):
 
         :param epoch: The epoch that has just ended.
         """
-        super(TensorboardLoggingCallback, self).on_epoch_end(epoch=epoch)
+        super().on_epoch_end(epoch=epoch)
 
         # Log the weights statistics:
         self._logger.log_weights_statistics()
@@ -540,9 +538,7 @@ class TensorboardLoggingCallback(LoggingCallback):
         :param y_true: The true value part of the current batch.
         :param y_pred: The prediction (output) of the model for this batch's input ('x').
         """
-        super(TensorboardLoggingCallback, self).on_train_batch_end(
-            batch=batch, x=x, y_true=y_true, y_pred=y_pred
-        )
+        super().on_train_batch_end(batch=batch, x=x, y_true=y_true, y_pred=y_pred)
 
         # Write the batch loss and metrics results to their graphs:
         self._logger.write_training_results()
@@ -559,9 +555,7 @@ class TensorboardLoggingCallback(LoggingCallback):
         :param y_true: The true value part of the current batch.
         :param y_pred: The prediction (output) of the model for this batch's input ('x').
         """
-        super(TensorboardLoggingCallback, self).on_validation_batch_end(
-            batch=batch, x=x, y_true=y_true, y_pred=y_pred
-        )
+        super().on_validation_batch_end(batch=batch, x=x, y_true=y_true, y_pred=y_pred)
 
         # Write the batch loss and metrics results to their graphs:
         self._logger.write_validation_results()