libinephany 0.18.1__py3-none-any.whl → 0.19.0__py3-none-any.whl

libinephany/observations/statistic_trackers.py

@@ -11,6 +11,7 @@ from typing import Any, Callable, final
 import torch
 import torch.distributed as dist
 import torch.nn as nn
+import torch.nn.functional as F
 import torch.optim as optim
 from torch.distributed import ReduceOp
 
@@ -597,6 +598,58 @@ class ActivationStatistics(Statistic):
         return None
 
 
+class InnerStepParameterUpdateStatistics(Statistic):
+
+    def __init__(
+        self,
+        *,
+        skip_statistics: list[str] | None = None,
+        **kwargs,
+    ) -> None:
+        """
+        :param skip_statistics: If the observation uses the TensorStatistic model to return observations, names of the
+        fields in the model to not include in returned observations.
+        :param kwargs: Other observation keyword arguments.
+        """
+
+        super().__init__(**kwargs)
+
+        self.skip_statistics = skip_statistics
+
+    def _get_storage_format(self) -> StatisticStorageTypes:
+        """
+        :return: Storage format this observation stores data in. Must be one of the enum attributes in the
+        StatisticStorageTypes enumeration class.
+        """
+
+        return StatisticStorageTypes.TENSOR_STATISTICS
+
+    def _gather(
+        self,
+        *,
+        optimizer: optim.Optimizer,
+        model: nn.Module,
+        parameters: list[torch.Tensor],
+        parameter_group: dict[str, Any],
+    ) -> torch.Tensor | TensorStatistics | float | None:
+        """
+        :param optimizer: Optimizer the given parameters and parameter group came from.
+        :param model: Inner model to gather statistics from.
+        :param parameters: List of parameters to gather statistics from.
+        :param parameter_group: Parameter group the parameters originate from.
+        :return: None, TensorStatistics model or a float.
+        """
+
+        update_tensor = observation_utils.form_update_tensor(
+            optimizer=optimizer, parameters=parameters, parameter_group=parameter_group
+        )
+
+        if update_tensor is None:
+            update_tensor = torch.cat([torch.zeros(p.view(-1).shape, device=p.device) for p in parameters])
+
+        return update_tensor
+
+
 class ParameterUpdateStatistics(Statistic):
 
     def __init__(
@@ -694,6 +747,51 @@ class ParameterStatistics(Statistic):
         return torch.cat([p.data.view(-1) for p in parameters if observation_utils.tensor_on_local_rank(p)])
 
 
+class InnerStepParameterStatistics(Statistic):
+
+    def __init__(
+        self,
+        *,
+        skip_statistics: list[str] | None = None,
+        **kwargs,
+    ) -> None:
+        """
+        :param skip_statistics: If the observation uses the TensorStatistic model to return observations, names of the
+        fields in the model to not include in returned observations.
+        :param kwargs: Other observation keyword arguments.
+        """
+
+        super().__init__(**kwargs)
+
+        self.skip_statistics = skip_statistics
+
+    def _get_storage_format(self) -> StatisticStorageTypes:
+        """
+        :return: Storage format this observation stores data in. Must be one of the enum attributes in the
+        StatisticStorageTypes enumeration class.
+        """
+
+        return StatisticStorageTypes.TENSOR_STATISTICS
+
+    def _gather(
+        self,
+        *,
+        optimizer: optim.Optimizer,
+        model: nn.Module,
+        parameters: list[torch.Tensor],
+        parameter_group: dict[str, Any],
+    ) -> torch.Tensor | TensorStatistics | float | None:
+        """
+        :param optimizer: Optimizer the given parameters and parameter group came from.
+        :param model: Inner model to gather statistics from.
+        :param parameters: List of parameters to gather statistics from.
+        :param parameter_group: Parameter group the parameters originate from.
+        :return: None, TensorStatistics model or a float.
+        """
+
+        return torch.cat([p.data.view(-1) for p in parameters if observation_utils.tensor_on_local_rank(p)])
+
+
 class LAMBTrustRatioStatistics(Statistic):
 
     def __init__(
@@ -759,6 +857,71 @@ class LAMBTrustRatioStatistics(Statistic):
         return lamb_trust_ratio
 
 
+class LHOPTLAMBTrustRatioStatistics(Statistic):
+
+    def __init__(
+        self,
+        *,
+        use_log_transform: bool = False,
+        **kwargs,
+    ) -> None:
+        """
+        :param use_log_transform: Whether to transform the LAMB trust ratio by taking ln(1 + R).
+        :param kwargs: Other observation keyword arguments.
+        """
+
+        super().__init__(**kwargs)
+
+        self.use_log_transform = use_log_transform
+
+    def _get_storage_format(self) -> StatisticStorageTypes:
+        """
+        :return: Storage format this observation stores data in. Must be one of the enum attributes in the
+        StatisticStorageTypes enumeration class.
+        """
+
+        return StatisticStorageTypes.FLOAT
+
+    def _gather(
+        self,
+        *,
+        optimizer: optim.Optimizer,
+        model: nn.Module,
+        parameters: list[torch.Tensor],
+        parameter_group: dict[str, Any],
+    ) -> torch.Tensor | TensorStatistics | float | None:
+        """
+        :param optimizer: Optimizer the given parameters and parameter group came from.
+        :param model: Inner model to gather statistics from.
+        :param parameters: List of parameters to gather statistics from.
+        :param parameter_group: Parameter group the parameters originate from.
+        :return: None, TensorStatistics model or a float.
+        """
+
+        weights_list = [p.data.view(-1) for p in parameters if observation_utils.tensor_on_local_rank(p)]
+        if weights_list:
+            weights = torch.cat(weights_list)
+
+        else:
+            weights = None
+
+        updates = observation_utils.form_update_tensor(
+            optimizer=optimizer, parameters=parameters, parameter_group=parameter_group
+        )
+
+        update_norm = torch.norm(updates, p=2).item() if updates is not None else 0
+        weight_norm = torch.norm(weights, p=2).item() if weights is not None else 0
+
+        lamb_trust_ratio = 0.0
+        if update_norm > 0:
+            lamb_trust_ratio = weight_norm / update_norm
+
+            if self.use_log_transform:
+                lamb_trust_ratio = math.log(1 + lamb_trust_ratio)
+
+        return lamb_trust_ratio
+
+
 class NumberOfParameters(Statistic):
 
     def __init__(
@@ -958,3 +1121,435 @@ class GradientVarianceFraction(Statistic):
             return 0.0
 
         return variance_parameters / total_parameters
+
+
+class AverageParameterUpdateMagnitudeStatistics(Statistic):
+
+    def __init__(
+        self,
+        *,
+        skip_statistics: list[str] | None = None,
+        **kwargs,
+    ) -> None:
+        """
+        :param skip_statistics: If the observation uses the TensorStatistic model to return observations, names of the
+        fields in the model to not include in returned observations.
+        :param kwargs: Other observation keyword arguments.
+        """
+
+        super().__init__(**kwargs)
+
+        self.skip_statistics = skip_statistics
+
+    def _get_storage_format(self) -> StatisticStorageTypes:
+        """
+        :return: Storage format this observation stores data in. Must be one of the enum attributes in the
+        StatisticStorageTypes enumeration class.
+        """
+
+        return StatisticStorageTypes.FLOAT
+
+    def _gather(
+        self,
+        *,
+        optimizer: optim.Optimizer,
+        model: nn.Module,
+        parameters: list[torch.Tensor],
+        parameter_group: dict[str, Any],
+    ) -> torch.Tensor | TensorStatistics | float | None:
+        """
+        :param optimizer: Optimizer the given parameters and parameter group came from.
+        :param model: Inner model to gather statistics from.
+        :param parameters: List of parameters to gather statistics from.
+        :param parameter_group: Parameter group the parameters originate from.
+        :return: None or a float.
+        """
+
+        update_tensor = observation_utils.form_update_tensor(
+            optimizer=optimizer, parameters=parameters, parameter_group=parameter_group
+        )
+
+        # when update tensor is none, return 0.0
+        if update_tensor is None:
+            return 0.0
+
+        update_tensor = update_tensor.view(-1)
+        update_tensor = update_tensor.abs()
+
+        average_update_magnitude = update_tensor.mean().item()
+
+        return average_update_magnitude
+
+
+class MomentumGradientRatioStatistics(Statistic):
+
+    def __init__(
+        self,
+        *,
+        skip_statistics: list[str] | None = None,
+        **kwargs,
+    ) -> None:
+        """
+        :param skip_statistics: If the observation uses the TensorStatistic model to return observations, names of the
+        fields in the model to not include in returned observations.
+        :param kwargs: Other observation keyword arguments.
+        """
+
+        super().__init__(**kwargs)
+
+        self.skip_statistics = skip_statistics
+
+    def _get_storage_format(self) -> StatisticStorageTypes:
+        """
+        :return: Storage format this observation stores data in. Must be one of the enum attributes in the
+        StatisticStorageTypes enumeration class.
+        """
+
+        return StatisticStorageTypes.FLOAT
+
+    def _gather(
+        self,
+        *,
+        optimizer: optim.Optimizer,
+        model: nn.Module,
+        parameters: list[torch.Tensor],
+        parameter_group: dict[str, Any],
+    ) -> torch.Tensor | TensorStatistics | float | None:
+        """
+        :param optimizer: Optimizer the given parameters and parameter group came from.
+        :param model: Inner model to gather statistics from.
+        :param parameters: List of parameters to gather statistics from.
+        :param parameter_group: Parameter group the parameters originate from.
+        :return: None, TensorStatistics model or a float.
+        """
+
+        momentum = observation_utils.form_momentum_tensor(
+            optimizer=optimizer, parameters=parameters, parameter_group=parameter_group
+        )
+        if momentum is None:
+            return None
+
+        gradients_list = [
+            p.grad.view(-1) for p in parameters if observation_utils.tensor_on_local_rank(p) and p.grad is not None
+        ]
+
+        # Handle empty gradients list
+        if not gradients_list:
+            return 0.0
+
+        gradients = torch.cat(gradients_list).view(-1)
+
+        # momentum_gradient_ratio r^t=\frac{\|g^t\|_2}{\|\nabla f(w^t)\|_2}
+        gradients_norm = gradients.norm(p=2)
+        momentum_norm = momentum.norm(p=2)
+
+        if momentum_norm == 0:
+            momentum_gradient_ratio = 0.0
+        else:
+            momentum_gradient_ratio = (gradients_norm / momentum_norm).item()
+
+        return momentum_gradient_ratio
+
+
+class LogOfNoiseScaleStatistics(Statistic):
+    """
+    Statistics for the log of noise scale in training.
+
+    Tracks the log of noise scale B_{noise} using the formula:
+    B_{noise} = tr(ΣH) / (G^T H G) ≈ (B/ε) * tr(HΣ) / tr(H^3 Σ)
+    where:
+    - H is the Hessian matrix
+    - G is the gradient vector
+    - Σ is the noise covariance matrix
+    - B is the batch size
+    - ε is the learning rate
+
+
+    """
+
+    def __init__(
+        self,
+        *,
+        skip_statistics: list[str] | None = None,
+        **kwargs,
+    ) -> None:
+        """
+        :param skip_statistics: If the observation uses the TensorStatistic model to return observations, names of the
+        fields in the model to not include in returned observations.
+        :param kwargs: Other observation keyword arguments.
+        """
+
+        super().__init__(**kwargs)
+
+        self.skip_statistics = skip_statistics
+
+    @property
+    def requires_gradient_graphs(self) -> bool:
+        """
+        :return: Whether the statistic requires gradient graphs to be retained.
+        """
+
+        return False
+
+    @staticmethod
+    def compute_hessian_diagonals(parameters: list[torch.Tensor]) -> torch.Tensor:
+        """
+        :param parameters: Parameters to compute the hessian diagonal matrices for.
+        :return: Tensor containing the hessian diagonal matrices for all given parameters.
+        """
+
+        hessian_diagonals = []
+
+        for parameter in parameters:
+            if parameter.grad is not None:
+                so_gradient = torch.autograd.grad(
+                    outputs=parameter.grad.clone(),
+                    inputs=parameter,
+                    grad_outputs=torch.ones_like(parameter.grad, requires_grad=True),
+                    only_inputs=True,
+                    retain_graph=True,
+                    create_graph=True,
+                    allow_unused=True,
+                )[0]
+
+                if so_gradient is not None:
+                    hessian_diagonals.append(so_gradient.view(-1))
+                else:
+                    hessian_diagonals.append(torch.zeros_like(parameter.view(-1)))
+
+        return torch.cat(hessian_diagonals)
+
+    def _get_storage_format(self) -> StatisticStorageTypes:
+        """
+        :return: Storage format this observation stores data in. Must be one of the enum attributes in the
+        StatisticStorageTypes enumeration class.
+        """
+
+        return StatisticStorageTypes.FLOAT
+
+    def _gather(
+        self,
+        *,
+        optimizer: optim.Optimizer,
+        model: nn.Module,
+        parameters: list[torch.Tensor],
+        parameter_group: dict[str, Any],
+    ) -> torch.Tensor | TensorStatistics | float | None:
+        """
+        :param optimizer: Optimizer the given parameters and parameter group came from.
+        :param model: Inner model to gather statistics from.
+        :param parameters: List of parameters to gather statistics from.
+        :param parameter_group: Parameter group the parameters originate from.
+        :return: None, TensorStatistics model or a float.
+
+        Computes the log of noise scale using the approximate formula:
+        log(B_{noise}) ≈ log(B/ε) (move to observer) + log(tr(HΣ)) - log(tr(H^3 Σ))
+        where:
+        - H is the Hessian matrix
+        - Σ is the noise covariance matrix
+        - B is the batch size
+        - ε is the learning rate
+
+        """
+
+        # Compute Hessian diagonals as in SecondOrderGradients Observation
+        # hessian_diagonals = self.compute_hessian_diagonals(parameters)
+        # use squared first order gradients as approximations
+        fo_gradients = [
+            p.grad.view(-1) for p in parameters if observation_utils.tensor_on_local_rank(p) and p.grad is not None
+        ]
+        if not fo_gradients:
+            return None
+
+        hessian_diagonals = torch.cat(fo_gradients) ** 2
+
+        if hessian_diagonals.numel() == 0:  # No gradients
+            return None
+
+        # For noise covariance matrix Σ, we'll use the identity matrix as an approximation
+        # This is a common assumption when the exact noise structure is unknown
+        noise_covariance = torch.ones_like(hessian_diagonals)
+
+        # Compute tr(HΣ)
+        trace_hessian_noise_covariance = torch.sum(hessian_diagonals * noise_covariance)
+
+        # Avoid division by zero and log of zero
+        if trace_hessian_noise_covariance <= 0:
+            return None
+
+        log_trace_hessian_noise_covariance = torch.log(trace_hessian_noise_covariance).item()
+
+        # Compute tr(H^3 Σ)
+        trace_hessian_cubed_noise_covariance = torch.sum(hessian_diagonals**3 * noise_covariance)
+        if trace_hessian_cubed_noise_covariance <= 0:
+            return None
+
+        log_trace_hessian_cubed_noise_covariance = torch.log(trace_hessian_cubed_noise_covariance).item()
+
+        # Compute final result: log(B_{noise}) ≈ log(tr(HΣ)) - log(tr(H^3 Σ))
+        # Note: log(B/ε) term is handled in the observer layer
+        log_noise_scale_without_log_b_over_epsilon = (
+            log_trace_hessian_noise_covariance - log_trace_hessian_cubed_noise_covariance
+        )
+
+        return log_noise_scale_without_log_b_over_epsilon
+
+
+class CosineSimilarityObserverOfGradientAndMomentumStatistics(Statistic):
+    """
+    Statistics for the cosine similarity of gradient and momentum.
+    """
+
+    def _get_storage_format(self) -> StatisticStorageTypes:
+        """
+        :return: Storage format this observation stores data in. Must be one of the enum attributes in the
+        StatisticStorageTypes enumeration class.
+        """
+
+        return StatisticStorageTypes.FLOAT
+
+    def _gather(
+        self,
+        *,
+        optimizer: optim.Optimizer,
+        model: nn.Module,
+        parameters: list[torch.Tensor],
+        parameter_group: dict[str, Any],
+    ) -> torch.Tensor | TensorStatistics | float | None:
+        """
+        :param optimizer: Optimizer the given parameters and parameter group came from.
+        :param model: Inner model to gather statistics from.
+        :param parameters: List of parameters to gather statistics from.
+        :param parameter_group: Parameter group the parameters originate from.
+        :return: None, TensorStatistics model or a float.
+        """
+        parameters_with_grads = [
+            p for p in parameters if observation_utils.tensor_on_local_rank(p) and p.grad is not None
+        ]
+
+        if not parameters_with_grads:
+            return None
+
+        gradients_list = [p.grad.view(-1) for p in parameters_with_grads]
+        gradients = torch.cat(gradients_list).view(-1)
+
+        momentum = observation_utils.form_momentum_tensor(
+            optimizer=optimizer, parameters=parameters_with_grads, parameter_group=parameter_group
+        )
+        if momentum is None:
+            return None
+
+        gradients_2d = gradients.unsqueeze(0)
+        momentum_2d = momentum.unsqueeze(0)
+
+        cosine_similarity = F.cosine_similarity(gradients_2d, momentum_2d, dim=1).item()
+
+        return cosine_similarity
+
+
+class CosineSimilarityObserverOfGradientAndUpdateStatistics(Statistic):
+    """
+    Statistics for the cosine similarity of gradient and update.
+    """
+
+    def _get_storage_format(self) -> StatisticStorageTypes:
+        """
+        :return: Storage format this observation stores data in. Must be one of the enum attributes in the
+        StatisticStorageTypes enumeration class.
+        """
+
+        return StatisticStorageTypes.FLOAT
+
+    def _gather(
+        self,
+        *,
+        optimizer: optim.Optimizer,
+        model: nn.Module,
+        parameters: list[torch.Tensor],
+        parameter_group: dict[str, Any],
+    ) -> torch.Tensor | TensorStatistics | float | None:
+        """
+        :param optimizer: Optimizer the given parameters and parameter group came from.
+        :param model: Inner model to gather statistics from.
+        :param parameters: List of parameters to gather statistics from.
+        :param parameter_group: Parameter group the parameters originate from.
+        :return: None, TensorStatistics model or a float.
+        """
+        # Filter parameters that have gradients to ensure consistent tensor sizes
+        parameters_with_grads = [
+            p for p in parameters if observation_utils.tensor_on_local_rank(p) and p.grad is not None
+        ]
+
+        if not parameters_with_grads:
+            return None
+
+        gradients_list = [p.grad.view(-1) for p in parameters_with_grads]
+        gradients = torch.cat(gradients_list).view(-1)
+
+        update_tensor = observation_utils.form_update_tensor(
+            optimizer=optimizer, parameters=parameters_with_grads, parameter_group=parameter_group
+        )
+
+        if update_tensor is None:
+            return None
+
+        gradients_2d = gradients.unsqueeze(0)
+        update_tensor_2d = update_tensor.unsqueeze(0)
+
+        cosine_similarity = F.cosine_similarity(gradients_2d, update_tensor_2d, dim=1).item()
+
+        return cosine_similarity
+
+
+class CosineSimilarityOfGradientAndParameterStatistics(Statistic):
+    """
+    Statistics for the cosine similarity of gradient and parameter.
+    """
+
+    def _get_storage_format(self) -> StatisticStorageTypes:
+        """
+        :return: Storage format this observation stores data in. Must be one of the enum attributes in the
+        StatisticStorageTypes enumeration class.
+        """
+
+        return StatisticStorageTypes.FLOAT
+
+    def _gather(
+        self,
+        *,
+        optimizer: optim.Optimizer,
+        model: nn.Module,
+        parameters: list[torch.Tensor],
+        parameter_group: dict[str, Any],
+    ) -> torch.Tensor | TensorStatistics | float | None:
+        """
+        :param optimizer: Optimizer the given parameters and parameter group came from.
+        :param model: Inner model to gather statistics from.
+        :param parameters: List of parameters to gather statistics from.
+        :param parameter_group: Parameter group the parameters originate from.
+        :return: None, TensorStatistics model or a float.
+        """
+        # Filter parameters that have gradients to ensure consistent tensor sizes
+        parameters_with_grads = [
+            p for p in parameters if observation_utils.tensor_on_local_rank(p) and p.grad is not None
+        ]
+
+        if not parameters_with_grads:
+            return None
+
+        gradients_list = [p.grad.view(-1) for p in parameters_with_grads]
+        gradients = torch.cat(gradients_list).view(-1)
+
+        parameters_list = [p.view(-1) for p in parameters_with_grads]
+
+        if not parameters_list:
+            return None
+
+        parameters_tensor = torch.cat(parameters_list).view(-1)
+
+        gradients_2d = gradients.unsqueeze(0)
+        parameters_tensor_2d = parameters_tensor.unsqueeze(0)
+
+        cosine_similarity = F.cosine_similarity(gradients_2d, parameters_tensor_2d, dim=1).item()
+
+        return cosine_similarity

{libinephany-0.18.1.dist-info → libinephany-0.19.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: libinephany
-Version: 0.18.1
+Version: 0.19.0
 Summary: Inephany library containing code commonly used by multiple subpackages.
 Author-email: Inephany <info@inephany.com>
 License: Apache 2.0

{libinephany-0.18.1.dist-info → libinephany-0.19.0.dist-info}/RECORD

@@ -2,22 +2,22 @@ libinephany/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 libinephany/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 libinephany/aws/s3_functions.py,sha256=W8u85A6tDloo4FlJvydJbVHCUq_m9i8KDGdnKzy-Xpg,1745
 libinephany/observations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-libinephany/observations/observation_utils.py,sha256=ejQ-hzKq_MX7A0304KypScmzloReNrH8dmcgonoRl4Q,16266
+libinephany/observations/observation_utils.py,sha256=JSNJYEi2d-VQ0ZovfHrn28RDv41u-a6M-W4ZT8UUyhI,17279
 libinephany/observations/observer_pipeline.py,sha256=_xA4vrijhG8-9MCtGXnKAEmpd6q0nKVpJgY_qSbypIA,12979
 libinephany/observations/pipeline_coordinator.py,sha256=mLfaHhkXVhMp9w5jWIAL3jPyauCM-795qOzyqwGOSdw,7932
 libinephany/observations/statistic_manager.py,sha256=LLg1zSxnJr2oQQepYla3qoUuRy10rsthr9jta4wEbnc,8956
-libinephany/observations/statistic_trackers.py,sha256=06NjCy1MI865oU0KB5f-wQE3b2RvYawOOWxNJx4rFpw,32939
+libinephany/observations/statistic_trackers.py,sha256=Z0JCJouVaPRUdR9o8GbD4Vm72aRREuxUIqGY8zgFUWg,53622
 libinephany/observations/observers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 libinephany/observations/observers/base_observers.py,sha256=Tkk2AvQ6mRY6fdhhxHQWohA0xSHGdPqxul1C7C7Frj4,15924
-libinephany/observations/observers/local_observers.py,sha256=7azTW227-rG_QJR5a0StfE0O4Ca1boMV9nrCphnhWik,40344
+libinephany/observations/observers/local_observers.py,sha256=9zWPizg6LuuembSibX67azIF12_Mr07EIzx_6d9ANVU,45290
 libinephany/observations/observers/observer_containers.py,sha256=VNyqGgxYJ4r49Msp_kk-POgicb-_5w54twuT1qfNdxw,9562
-libinephany/observations/observers/global_observers/__init__.py,sha256=lQO-nq7hILu4F3ddFXcCR-ghfv0dzEA9nAYxZta7rxk,2306
+libinephany/observations/observers/global_observers/__init__.py,sha256=87WHRPYmL0tVsaTKUd91pwEpCZtHPSKRQoba2VQjswA,3018
 libinephany/observations/observers/global_observers/base_classes.py,sha256=CCkRx86Lll3gFzfqervP0jKdzNFKkKU7tEBh8ic1Yrc,8249
-libinephany/observations/observers/global_observers/constants.py,sha256=olXYxh353Th6hyhfk85kHxAWPeaDpuPkcz_awFvEz6c,1054
-libinephany/observations/observers/global_observers/gradient_observers.py,sha256=z2ow6zmTr7ujeaYyh5qGRE1woIt_yc2uxAaie7zgxqc,8398
+libinephany/observations/observers/global_observers/constants.py,sha256=C_PwYhKxatJxNe5Jzb1tpoiRXAxxPrGkcdQBMQD8msY,1139
+libinephany/observations/observers/global_observers/gradient_observers.py,sha256=6L9Hw6K--vTRe0Zp4dQ9ByOLwU6p9Z8vhMYcTQVA4IU,21042
 libinephany/observations/observers/global_observers/hyperparameter_observers.py,sha256=soGWoYpO5rqUQU0p4pr6QVjBvVg2odK71mVkxV38Ras,14838
 libinephany/observations/observers/global_observers/loss_observers.py,sha256=FlSuJqAJIXcAS_ypdZna6xxz89glI23A6D00sDn7ZLU,18508
-libinephany/observations/observers/global_observers/model_observers.py,sha256=bJIEdq5wWkLrw-MNllCe36FE4aMLbC4RoR1-wOOHMxc,19537
+libinephany/observations/observers/global_observers/model_observers.py,sha256=xNqNWz2YUATakm_BuHXCGtGCeYodpyeupa8yxLo-TAA,28433
 libinephany/observations/observers/global_observers/progress_observers.py,sha256=m62jUiwPaOUzYG1h7Vg6znj_jK9699lDhg4AhK212s8,5615
 libinephany/observations/post_processors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 libinephany/observations/post_processors/postprocessors.py,sha256=43_e5UaDPr2KbAvqc_w3wLqnlm7bgRjqgCtyQ95-8cM,5913
@@ -57,8 +57,8 @@ libinephany/utils/typing.py,sha256=rGbaPO3MaUndsWiC_wHzReD_TOLYqb43i01pKN-j7Xs,6
 libinephany/web_apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 libinephany/web_apps/error_logger.py,sha256=gAQIaqerqP4ornXZwFF1cghjnd2mMZEt3aVrTuUCr34,16653
 libinephany/web_apps/web_app_utils.py,sha256=qiq_lasPipgN1RgRudPJc342kYci8O_4RqppxmIX8NY,4095
-libinephany-0.18.1.dist-info/licenses/LICENSE,sha256=pogfDoMBP07ehIOvWymuWIar8pg2YLUhqOHsJQU3wdc,9250
-libinephany-0.18.1.dist-info/METADATA,sha256=d2rH-yA7F1cVk2OnTyvdaQ6_CFey1uiY4ATbqsxH9Pg,8390
-libinephany-0.18.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-libinephany-0.18.1.dist-info/top_level.txt,sha256=bYAOXQdJgIoLkO2Ui0kxe7pSYegS_e38u0dMscd7COQ,12
-libinephany-0.18.1.dist-info/RECORD,,
+libinephany-0.19.0.dist-info/licenses/LICENSE,sha256=pogfDoMBP07ehIOvWymuWIar8pg2YLUhqOHsJQU3wdc,9250
+libinephany-0.19.0.dist-info/METADATA,sha256=iojcQg83wJVyPzuFssPtW_Maz0wO2JGFXCLcolBsnfQ,8390
+libinephany-0.19.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+libinephany-0.19.0.dist-info/top_level.txt,sha256=bYAOXQdJgIoLkO2Ui0kxe7pSYegS_e38u0dMscd7COQ,12
+libinephany-0.19.0.dist-info/RECORD,,