libinephany 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

libinephany/observations/observers/base_observers.py

@@ -44,6 +44,7 @@ class Observer(ABC):
         observer_config: ObserverConfig,
         should_standardize: bool = True,
         include_statistics: list[str] | None = None,
+        include_hparams: list[str] | None = None,
         **kwargs,
     ) -> None:
         """
@@ -52,6 +53,8 @@ class Observer(ABC):
         :param should_standardize: Whether standardization should be applied to returned values.
         :param include_statistics: If the observation uses the TensorStatistic model to return observations, names of the
         fields in the model to include in returned observations.
+        :param include_hparams: If the observation uses the HyperparameterStates model to return observations, names of the
+        hyperparameters to include in returned observations.
         :param kwargs: Miscellaneous keyword arguments.
         """
 
@@ -64,10 +67,17 @@ class Observer(ABC):
         self.should_standardize = should_standardize and self.can_standardize
 
         self.include_statistics: list[str] | None = None
+        self.include_hparams = include_hparams
 
         if include_statistics is not None:
             self.include_statistics = TensorStatistics.filter_include_statistics(include_statistics=include_statistics)
 
+        if self.requires_include_statistics and not self.include_statistics:
+            raise ValueError(f"{self.__class__.__name__} must be provided with include_statistics.")
+
+        if self.requires_include_hparams and not self.include_hparams:
+            raise ValueError(f"{self.__class__.__name__} must be provided with include_hparams.")
+
     @final
     @property
     def in_training_mode(self) -> bool:
@@ -143,6 +153,22 @@ class Observer(ABC):
 
         return True
 
+    @property
+    def requires_include_statistics(self) -> bool:
+        """
+        :return: Whether the observation requires include_statistics to be provided.
+        """
+
+        return False
+
+    @property
+    def requires_include_hparams(self) -> bool:
+        """
+        :return: Whether the observation requires include_hparams to be provided.
+        """
+
+        return False
+
     @property
     @abstractmethod
     def standardizer_key_infix(self) -> str:

libinephany/observations/observers/global_observers/base_classes.py

@@ -1,6 +1,6 @@
 # ======================================================================================================================
 #
-# BASE CLASSES
+# IMPORTS
 #
 # ======================================================================================================================
 
@@ -15,6 +15,12 @@ from libinephany.pydantic_models.schemas.observation_models import ObservationIn
 from libinephany.pydantic_models.schemas.tensor_statistics import TensorStatistics
 from libinephany.pydantic_models.states.hyperparameter_states import HyperparameterStates
 
+# ======================================================================================================================
+#
+# CLASSES
+#
+# ======================================================================================================================
+
 
 class LHOPTBaseObserver(GlobalObserver, ABC):
     """
@@ -33,13 +39,14 @@ class LHOPTBaseObserver(GlobalObserver, ABC):
         :param kwargs: Other observation keyword arguments.
         """
         super().__init__(**kwargs)
-        self.decay_factor = max(0.0, decay_factor)
-        self.time_window = max(1, time_window)
 
         # Store time series data for CDF calculation
         self._time_series: list[tuple[float, float]] = []  # (time, value) pairs
         self._current_time: float = 0.0
 
+        self.decay_factor = max(0.0, decay_factor)
+        self.time_window = max(1, time_window)
+
     @property
     def can_standardize(self) -> bool:
         """
@@ -48,6 +55,28 @@ class LHOPTBaseObserver(GlobalObserver, ABC):
         """
         return False
 
+    @staticmethod
+    def _compute_log_ratio(numerator: float, denominator: float) -> float:
+        """
+        Compute the log ratio.
+
+        :param numerator: Numerator value
+        :param denominator: Denominator value
+        :return: Log ratio value
+        """
+        # Calculate the ratio of numerator to denominator
+        invalid_denominator = math.isinf(denominator) or math.isnan(denominator)
+
+        if denominator <= LHOPT_CONSTANTS["ZERO_DIVISION_TOLERANCE"] or invalid_denominator:
+            return 0.0
+
+        ratio = numerator / denominator
+
+        if ratio <= 0:
+            return 0.0
+
+        return math.log(ratio)
+
     def _get_observation_format(self) -> StatisticStorageTypes:
         """
         :return: Format the observation returns data in. Must be one of the StatisticStorageTypes
@@ -68,18 +97,6 @@ class LHOPTBaseObserver(GlobalObserver, ABC):
         """Update the current time counter."""
         self._current_time += 1.0
 
-    def get_required_trackers(self) -> dict[str, dict[str, Any] | None]:
-        """
-        :return: Dictionary mapping statistic tracker class names to kwargs for the class or None if no kwargs are
-        needed.
-        """
-        return {}
-
-    def reset(self) -> None:
-        """Reset the observer by clearing the time series."""
-        self._time_series = []
-        self._current_time = 0.0
-
     @abstractmethod
     def _observe(
         self,
@@ -94,27 +111,13 @@ class LHOPTBaseObserver(GlobalObserver, ABC):
         :param tracked_statistics: Dictionary mapping statistic tracker class names to dictionaries mapping module
         :param action_taken: Action taken by the agent this class instance is assigned to.
         """
-        raise NotImplementedError
-
-    def _compute_log_ratio(self, numerator: float, denominator: float) -> float:
-        """
-        Compute the log ratio.
-
-        :param numerator: Numerator value
-        :param denominator: Denominator value
-        :return: Log ratio value
-        """
-        # Calculate the ratio of numerator to denominator
 
-        if denominator <= LHOPT_CONSTANTS["ZERO_DIVISION_TOLERANCE"]:
-            return 0.0
+        ...
 
-        ratio = numerator / denominator
-
-        if ratio <= 0:
-            return 0.0
-
-        return math.log(ratio)
+    def reset(self) -> None:
+        """Reset the observer by clearing the time series."""
+        self._time_series = []
+        self._current_time = 0.0
 
 
 class LHOPTCheckpointBaseObserver(GlobalObserver, ABC):
@@ -128,8 +131,10 @@ class LHOPTCheckpointBaseObserver(GlobalObserver, ABC):
         :param kwargs: Miscellaneous keyword arguments.
         """
         super().__init__(**kwargs)
-        self.checkpoint_interval = checkpoint_interval
+
         self._history: list[float] = []
+
+        self.checkpoint_interval = checkpoint_interval
         self.last_value: float | None = None
 
     @property
@@ -175,18 +180,6 @@ class LHOPTCheckpointBaseObserver(GlobalObserver, ABC):
         if self.last_value is None:
             self.last_value = value
 
-    def get_required_trackers(self) -> dict[str, dict[str, Any] | None]:
-        """
-        :return: Dictionary mapping statistic tracker class names to kwargs for the class or None if no kwargs are
-        needed.
-        """
-        return {}
-
-    def reset(self) -> None:
-        """Reset the observer by clearing history."""
-        self._history = []
-        self.last_value = None
-
     @abstractmethod
     def _observe(
         self,
@@ -201,24 +194,17 @@ class LHOPTCheckpointBaseObserver(GlobalObserver, ABC):
         :param tracked_statistics: Dictionary mapping statistic tracker class names to dictionaries mapping module
         :param action_taken: Action taken by the agent this class instance is assigned to.
         """
-        raise NotImplementedError
 
-    def _compute_log_ratio(self, numerator: float, denominator: float) -> float:
-        """
-        Compute the log ratio.
+        ...
 
-        :param numerator: Numerator value
-        :param denominator: Denominator value
-        :return: Log ratio value
+    def get_required_trackers(self) -> dict[str, dict[str, Any] | None]:
         """
-        # Calculate the ratio of numerator to denominator
-
-        if denominator <= LHOPT_CONSTANTS["ZERO_DIVISION_TOLERANCE"]:
-            return 0.0
-
-        ratio = numerator / denominator
-
-        if ratio <= 0:
-            return 0.0
+        :return: Dictionary mapping statistic tracker class names to kwargs for the class or None if no kwargs are
+        needed.
+        """
+        return {}
 
-        return math.log(ratio)
+    def reset(self) -> None:
+        """Reset the observer by clearing history."""
+        self._history = []
+        self.last_value = None

libinephany/observations/observers/global_observers/constants.py

@@ -1,11 +1,17 @@
 # ======================================================================================================================
 #
-# CONSTANTS
+# IMPORTS
 #
 # ======================================================================================================================
 
 from typing import TypedDict
 
+# ======================================================================================================================
+#
+# CLASSES
+#
+# ======================================================================================================================
+
 
 class LHOPTConstants(TypedDict):
     IS_NAN: float
@@ -23,6 +29,13 @@ class LHOPTConstants(TypedDict):
     DEFAULT_ENV_STEP_SAMPLE_FREQUENCY: int
 
 
+# ======================================================================================================================
+#
+# CONSTANTS
+#
+# ======================================================================================================================
+
+
 # Create the constants instance
 LHOPT_CONSTANTS: LHOPTConstants = LHOPTConstants(
     IS_NAN=1.0,

libinephany/observations/observers/global_observers/gradient_observers.py

@@ -1,6 +1,6 @@
 # ======================================================================================================================
 #
-# GRADIENT OBSERVERS
+# IMPORTS
 #
 # ======================================================================================================================
 
@@ -16,9 +16,23 @@ from libinephany.pydantic_models.schemas.observation_models import ObservationIn
 from libinephany.pydantic_models.schemas.tensor_statistics import TensorStatistics
 from libinephany.pydantic_models.states.hyperparameter_states import HyperparameterStates
 
+# ======================================================================================================================
+#
+# CLASSES
+#
+# ======================================================================================================================
+
 
 class GlobalFirstOrderGradients(GlobalObserver):
 
+    @property
+    def requires_include_statistics(self) -> bool:
+        """
+        :return: Whether the observation requires include_statistics to be provided.
+        """
+
+        return True
+
     def _get_observation_format(self) -> StatisticStorageTypes:
         """
         :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
@@ -74,6 +88,14 @@ class GlobalSecondOrderGradients(GlobalObserver):
 
         self.compute_hessian_diagonal = compute_hessian_diagonal
 
+    @property
+    def requires_include_statistics(self) -> bool:
+        """
+        :return: Whether the observation requires include_statistics to be provided.
+        """
+
+        return True
+
     def _get_observation_format(self) -> StatisticStorageTypes:
         """
         :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
@@ -136,21 +158,6 @@ class LHOPTGradientVarianceFraction(LHOPTBaseObserver):
         super().__init__(**kwargs)
         self.variance_threshold = variance_threshold
 
-    @property
-    def can_standardize(self) -> bool:
-        """
-        This observer has its own CDF calculation, no need to standardize.
-        :return: Whether the observation can be standardized.
-        """
-        return False
-
-    def _get_observation_format(self) -> StatisticStorageTypes:
-        """
-        :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
-        enumeration class.
-        """
-        return StatisticStorageTypes.VECTOR
-
     @property
     def vector_length(self) -> int:
         """
@@ -173,8 +180,6 @@ class LHOPTGradientVarianceFraction(LHOPTBaseObserver):
         :param action_taken: Action taken by the agent this class instance is assigned to.
         :return: Single float/int, list of floats/ints or TensorStatistics model to add to the observation vector.
         """
-        if statistic_trackers.GradientVarianceFraction.__name__ not in tracked_statistics:
-            return [0.0, 0.0]
 
         raw_value = list(tracked_statistics[statistic_trackers.GradientVarianceFraction.__name__].values())[0]  # type: ignore[list-item]
 
@@ -204,14 +209,6 @@ class LHOPTMomentumGradientRatio(LHOPTBaseObserver):
     It returns two-dimensional observations: [raw_value, cdf_feature] for momentum gradient ratio values.
     """
 
-    def _get_observation_format(self) -> StatisticStorageTypes:
-        """
-        :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
-        enumeration class.
-        """
-
-        return StatisticStorageTypes.VECTOR
-
     @property
     def vector_length(self) -> int:
         """
@@ -219,6 +216,14 @@ class LHOPTMomentumGradientRatio(LHOPTBaseObserver):
         """
         return 2  # [raw_value, cdf_feature]
 
+    @property
+    def requires_include_statistics(self) -> bool:
+        """
+        :return: Whether the observation requires include_statistics to be provided.
+        """
+
+        return True
+
     def _observe(
         self,
         observation_inputs: ObservationInputs,
@@ -266,29 +271,6 @@ class CosineSimilarityObserverOfGradientAndMomentum(LHOPTBaseObserver):
     It returns two-dimensional observations: [raw_value, cdf_feature] for cosine similarity of gradient and momentum values.
     """
 
-    def __init__(
-        self,
-        *,
-        include_statistics: list[str] | None = None,
-        **kwargs,
-    ) -> None:
-        """
-        :param include_statistics: List of statistics to include.
-        :param kwargs: Miscellaneous keyword arguments.
-        """
-
-        super().__init__(**kwargs)
-
-        self.include_statistics = include_statistics
-
-    def _get_observation_format(self) -> StatisticStorageTypes:
-        """
-        :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
-        enumeration class.
-        """
-
-        return StatisticStorageTypes.VECTOR
-
     @property
     def vector_length(self) -> int:
         """
@@ -296,6 +278,14 @@ class CosineSimilarityObserverOfGradientAndMomentum(LHOPTBaseObserver):
         """
         return 3  # [raw_value, cdf_feature, logit_of_cdf_feature]
 
+    @property
+    def requires_include_statistics(self) -> bool:
+        """
+        :return: Whether the observation requires include_statistics to be provided.
+        """
+
+        return True
+
     def _observe(
         self,
         observation_inputs: ObservationInputs,
@@ -351,29 +341,6 @@ class CosineSimilarityObserverOfGradientAndUpdate(LHOPTBaseObserver):
     It returns two-dimensional observations: [raw_value, cdf_feature] for cosine similarity of gradient and update values.
     """
 
-    def __init__(
-        self,
-        *,
-        include_statistics: list[str] | None = None,
-        **kwargs,
-    ) -> None:
-        """
-        :param include_statistics: List of statistics to include.
-        :param kwargs: Miscellaneous keyword arguments.
-        """
-
-        super().__init__(**kwargs)
-
-        self.include_statistics = include_statistics
-
-    def _get_observation_format(self) -> StatisticStorageTypes:
-        """
-        :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
-        enumeration class.
-        """
-
-        return StatisticStorageTypes.VECTOR
-
     @property
     def vector_length(self) -> int:
         """
@@ -381,6 +348,14 @@ class CosineSimilarityObserverOfGradientAndUpdate(LHOPTBaseObserver):
         """
         return 3  # [raw_value, cdf_feature, logit_of_cdf_feature]
 
+    @property
+    def requires_include_statistics(self) -> bool:
+        """
+        :return: Whether the observation requires include_statistics to be provided.
+        """
+
+        return True
+
     def _observe(
         self,
         observation_inputs: ObservationInputs,
@@ -436,28 +411,6 @@ class CosineSimilarityOfGradientAndParameter(LHOPTBaseObserver):
     It returns two-dimensional observations: [raw_value, cdf_feature] for cosine similarity of gradient and parameter values.
     """
 
-    def __init__(
-        self,
-        *,
-        include_statistics: list[str] | None = None,
-        **kwargs,
-    ) -> None:
-        """
-        :param include_statistics: List of statistics to include.
-        :param kwargs: Miscellaneous keyword arguments.
-        """
-        super().__init__(**kwargs)
-
-        self.include_statistics = include_statistics
-
-    def _get_observation_format(self) -> StatisticStorageTypes:
-        """
-        :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
-        enumeration class.
-        """
-
-        return StatisticStorageTypes.VECTOR
-
     @property
     def vector_length(self) -> int:
         """
@@ -465,6 +418,14 @@ class CosineSimilarityOfGradientAndParameter(LHOPTBaseObserver):
         """
         return 3  # [raw_value, cdf_feature, logit_of_cdf_feature]
 
+    @property
+    def requires_include_statistics(self) -> bool:
+        """
+        :return: Whether the observation requires include_statistics to be provided.
+        """
+
+        return True
+
     def _observe(
         self,
         observation_inputs: ObservationInputs,

libinephany/observations/observers/global_observers/hyperparameter_observers.py

@@ -1,6 +1,6 @@
 # ======================================================================================================================
 #
-# HYPERPARAMETER OBSERVERS
+# IMPORTS
 #
 # ======================================================================================================================
 
@@ -12,25 +12,28 @@ from torch.optim import SGD, Adam, AdamW
 from libinephany.observations import observation_utils
 from libinephany.observations.observation_utils import StatisticStorageTypes
 from libinephany.observations.observers.base_observers import GlobalObserver
-from libinephany.observations.observers.global_observers.base_classes import LHOPT_CONSTANTS
+from libinephany.observations.observers.global_observers.constants import LHOPT_CONSTANTS
 from libinephany.pydantic_models.schemas.observation_models import ObservationInputs
 from libinephany.pydantic_models.schemas.tensor_statistics import TensorStatistics
 from libinephany.pydantic_models.states.hyperparameter_states import HyperparameterStates
 from libinephany.utils.enums import ModelFamilies
 
+# ======================================================================================================================
+#
+# CLASSES
+#
+# ======================================================================================================================
+
 
 class InitialHyperparameters(GlobalObserver):
 
-    def __init__(self, include_hparams: list[str] | None = None, pad_with: float = 0.0, **kwargs) -> None:
+    def __init__(self, pad_with: float = 0.0, **kwargs) -> None:
         """
-        :param include_hparams: Names of the hyperparameters to include in the initial values vector returned by
-        this observation.
         :param kwargs: Miscellaneous keyword arguments.
         """
 
         super().__init__(**kwargs)
 
-        self.include_hparams = include_hparams
         self.pad_with = pad_with
 
     @property
@@ -62,6 +65,14 @@ class InitialHyperparameters(GlobalObserver):
 
         return False
 
+    @property
+    def requires_include_hparams(self) -> bool:
+        """
+        :return: Whether the observation requires include_hparams to be provided.
+        """
+
+        return True
+
     def _get_observation_format(self) -> StatisticStorageTypes:
         """
         :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
@@ -298,7 +309,7 @@ class LHOPTHyperparameterRatio(GlobalObserver):
     providing insights into how much hyperparameters have changed from their starting values.
     """
 
-    def __init__(self, include_hparams: list[str] | None = None, pad_with: float = 0.0, **kwargs) -> None:
+    def __init__(self, pad_with: float = 0.0, **kwargs) -> None:
         """
         :param include_hparams: Names of the hyperparameters to include in the initial values vector returned by
         this observation.
@@ -307,7 +318,6 @@ class LHOPTHyperparameterRatio(GlobalObserver):
 
         super().__init__(**kwargs)
 
-        self.include_hparams = include_hparams
         self.pad_with = pad_with
 
     @property
@@ -339,6 +349,14 @@ class LHOPTHyperparameterRatio(GlobalObserver):
 
         return False
 
+    @property
+    def requires_include_hparams(self) -> bool:
+        """
+        :return: Whether the observation requires include_hparams to be provided.
+        """
+
+        return True
+
     def _get_observation_format(self) -> StatisticStorageTypes:
         """
         :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes

libinephany/observations/observers/global_observers/loss_observers.py

@@ -239,6 +239,13 @@ class LHOPTTrainingLoss(LHOPTBaseObserver):
     This observer use the CDF calculation from the paper and applies CDF transformation using the CDF mean and std.
     """
 
+    @property
+    def vector_length(self) -> int:
+        """
+        :return: Length of the vector returned by this observation if it returns a vector.
+        """
+        return 3  # [is_nan, is_inf, cdf_feature]
+
     def _observe(
         self,
         observation_inputs: ObservationInputs,
@@ -271,18 +278,6 @@ class LHOPTTrainingLoss(LHOPTBaseObserver):
 
         return {}
 
-    def reset(self) -> None:
-        """Reset the observer by clearing the time series."""
-        self._time_series: list[tuple[float, float]] = []
-        self._current_time: float = 0.0
-
-    @property
-    def vector_length(self) -> int:
-        """
-        :return: Length of the vector returned by this observation if it returns a vector.
-        """
-        return 3  # [is_nan, is_inf, cdf_feature]
-
 
 class LHOPTValidationLoss(LHOPTBaseObserver):
     """
@@ -294,6 +289,13 @@ class LHOPTValidationLoss(LHOPTBaseObserver):
     This observer use the CDF calculation from the paper and applies CDF transformation using the CDF mean and std.
     """
 
+    @property
+    def vector_length(self) -> int:
+        """
+        :return: Length of the vector returned by this observation if it returns a vector.
+        """
+        return 3  # [is_nan, is_inf, cdf_feature]
+
     def _observe(
         self,
         observation_inputs: ObservationInputs,
@@ -326,18 +328,6 @@ class LHOPTValidationLoss(LHOPTBaseObserver):
 
         return {}
 
-    def reset(self) -> None:
-        """Reset the observer by clearing the time series."""
-        self._time_series: list[tuple[float, float]] = []
-        self._current_time: float = 0.0
-
-    @property
-    def vector_length(self) -> int:
-        """
-        :return: Length of the vector returned by this observation if it returns a vector.
-        """
-        return 3  # [is_nan, is_inf, cdf_feature]
-
 
 class LHOPTLossRatio(LHOPTBaseObserver):
     """
@@ -353,6 +343,13 @@ class LHOPTLossRatio(LHOPTBaseObserver):
     3. cdf_feature - CDF transformed feature using CDF mean and std
     """
 
+    @property
+    def vector_length(self) -> int:
+        """
+        :return: Length of the vector returned by this observation if it returns a vector.
+        """
+        return 3  # [is_nan, tanh, cdf_feature]
+
     def _observe(
         self,
         observation_inputs: ObservationInputs,
@@ -370,7 +367,7 @@ class LHOPTLossRatio(LHOPTBaseObserver):
         """
 
         log_ratio = self._compute_log_ratio(
-            training_score=observation_inputs.training_score, validation_score=observation_inputs.validation_score
+            numerator=observation_inputs.training_loss, denominator=observation_inputs.validation_loss
         )
 
         tanh_feature = math.tanh(max(-LHOPT_CONSTANTS["TANH_BOUND"], min(LHOPT_CONSTANTS["TANH_BOUND"], log_ratio)))
@@ -381,31 +378,13 @@ class LHOPTLossRatio(LHOPTBaseObserver):
 
         return [int(math.isnan(log_ratio)), tanh_feature, cdf_feature]
 
-    def _compute_log_ratio(self, training_score: float, validation_score: float) -> float:
+    def get_required_trackers(self) -> dict[str, dict[str, Any] | None]:
         """
-        Compute the log ratio of validation_score to training_score.
-
-        :param training_score: Training score value
-        :param validation_score: Validation score value
-        :return: Log ratio value
+        :return: Dictionary mapping statistic tracker class names to kwargs for the class or None if no kwargs are
+        needed.
         """
-        if training_score <= 0:
-            return 0.0
-
-        if validation_score <= 0:
-            return 0.0
 
-        # Calculate the ratio of validation_score to training_score
-        score_ratio = validation_score / training_score
-
-        return math.log(score_ratio)
-
-    @property
-    def vector_length(self) -> int:
-        """
-        :return: Length of the vector returned by this observation if it returns a vector.
-        """
-        return 3  # [is_nan, tanh, cdf_feature]
+        return {}
 
 
 class PercentileOfLossAtEachCheckpoint(LHOPTCheckpointBaseObserver):

libinephany/observations/observers/global_observers/model_observers.py

@@ -25,6 +25,14 @@ from libinephany.pydantic_models.states.hyperparameter_states import Hyperparame
 
 class GlobalActivations(GlobalObserver):
 
+    @property
+    def requires_include_statistics(self) -> bool:
+        """
+        :return: Whether the observation requires include_statistics to be provided.
+        """
+
+        return True
+
     def _get_observation_format(self) -> StatisticStorageTypes:
         """
         :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
@@ -64,6 +72,14 @@ class GlobalActivations(GlobalObserver):
 
 class GlobalParameterUpdates(GlobalObserver):
 
+    @property
+    def requires_include_statistics(self) -> bool:
+        """
+        :return: Whether the observation requires include_statistics to be provided.
+        """
+
+        return True
+
     def _get_observation_format(self) -> StatisticStorageTypes:
         """
         :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
@@ -103,6 +119,14 @@ class GlobalParameterUpdates(GlobalObserver):
 
 class GlobalParameters(GlobalObserver):
 
+    @property
+    def requires_include_statistics(self) -> bool:
+        """
+        :return: Whether the observation requires include_statistics to be provided.
+        """
+
+        return True
+
     def _get_observation_format(self) -> StatisticStorageTypes:
         """
         :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
@@ -342,6 +366,14 @@ class LogRatioOfPreviousAndCurrentParamNormEnvStepObserver(LHOPTBaseObserver):
         """
         return 2  # [tanh_feature, cdf_feature]
 
+    @property
+    def requires_include_statistics(self) -> bool:
+        """
+        :return: Whether the observation requires include_statistics to be provided.
+        """
+
+        return True
+
     def _observe(
         self,
         observation_inputs: ObservationInputs,
@@ -410,6 +442,14 @@ class LogRatioOfUpdateAndPreviousParamNormEnvStepObserver(LHOPTBaseObserver):
         """
         return 2  # [tanh_feature, cdf_feature]
 
+    @property
+    def requires_include_statistics(self) -> bool:
+        """
+        :return: Whether the observation requires include_statistics to be provided.
+        """
+
+        return True
+
     def _observe(
         self,
         observation_inputs: ObservationInputs,
@@ -473,28 +513,20 @@ class LogRatioOfUpdateAndPreviousParamNormEnvStepObserver(LHOPTBaseObserver):
 
 class LHOPTAverageParameterUpdateMagnitudeObserver(LHOPTBaseObserver):
 
-    def _get_observation_format(self) -> StatisticStorageTypes:
-        """
-        :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
-        enumeration class.
-        """
-
-        return StatisticStorageTypes.VECTOR
-
     @property
-    def can_standardize(self) -> bool:
+    def vector_length(self) -> int:
         """
-        :return: Whether the observation can be standardized.
+        :return: Length of the vector returned by this observation if it returns a vector.
         """
-
-        return False
+        return 2  # [raw_feature, cdf_feature]
 
     @property
-    def vector_length(self) -> int:
+    def requires_include_statistics(self) -> bool:
         """
-        :return: Length of the vector returned by this observation if it returns a vector.
+        :return: Whether the observation requires include_statistics to be provided.
         """
-        return 2  # [raw_feature, cdf_feature]
+
+        return True
 
     def _observe(
         self,
@@ -537,8 +569,8 @@ class LHOPTAverageParameterUpdateMagnitudeObserver(LHOPTBaseObserver):
 class LogRatioOfUpdateAndPreviousParamNormInnerStepObserver(LHOPTBaseObserver):
     def __init__(self, **kwargs):
         """
-        This observer is used to compute the log ratio of the update and previous parameter norm for the inner step. The sample frequency of the statistics needs to be set to 4 (according to the OpenAI paper).
-
+        This observer is used to compute the log ratio of the update and previous parameter norm for the inner step.
+        The sample frequency of the statistics needs to be set to 4 (according to the OpenAI paper).
         """
         super().__init__(**kwargs)
         self._previous_param_norm = None
@@ -550,6 +582,14 @@ class LogRatioOfUpdateAndPreviousParamNormInnerStepObserver(LHOPTBaseObserver):
         """
         return 2  # [tanh_feature, cdf_feature]
 
+    @property
+    def requires_include_statistics(self) -> bool:
+        """
+        :return: Whether the observation requires include_statistics to be provided.
+        """
+
+        return True
+
     def _observe(
         self,
         observation_inputs: ObservationInputs,
@@ -630,14 +670,6 @@ class LHOPTGlobalLAMBTrustRatio(LHOPTBaseObserver):
 
         self.use_log_transform = use_log_transform
 
-    def _get_observation_format(self) -> StatisticStorageTypes:
-        """
-        :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
-        enumeration class.
-        """
-
-        return StatisticStorageTypes.VECTOR
-
     @property
     def vector_length(self) -> int:
         """
@@ -645,6 +677,14 @@ class LHOPTGlobalLAMBTrustRatio(LHOPTBaseObserver):
         """
         return 2  # [raw_value, cdf_feature]
 
+    @property
+    def requires_include_statistics(self) -> bool:
+        """
+        :return: Whether the observation requires include_statistics to be provided.
+        """
+
+        return True
+
     def _observe(
         self,
         observation_inputs: ObservationInputs,

libinephany/observations/observers/local_observers.py

@@ -27,6 +27,14 @@ from libinephany.utils.transforms import HyperparameterTransformType
 
 class FirstOrderGradients(LocalObserver):
 
+    @property
+    def requires_include_statistics(self) -> bool:
+        """
+        :return: Whether the observation requires include_statistics to be provided.
+        """
+
+        return True
+
     def _get_observation_format(self) -> StatisticStorageTypes:
         """
         :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
@@ -89,6 +97,14 @@ class SecondOrderGradients(LocalObserver):
 
         self.compute_hessian_diagonal = compute_hessian_diagonal
 
+    @property
+    def requires_include_statistics(self) -> bool:
+        """
+        :return: Whether the observation requires include_statistics to be provided.
+        """
+
+        return True
+
     def _get_observation_format(self) -> StatisticStorageTypes:
         """
         :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
@@ -139,6 +155,14 @@ class SecondOrderGradients(LocalObserver):
 
 class Activations(LocalObserver):
 
+    @property
+    def requires_include_statistics(self) -> bool:
+        """
+        :return: Whether the observation requires include_statistics to be provided.
+        """
+
+        return True
+
     def _get_observation_format(self) -> StatisticStorageTypes:
         """
         :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
@@ -185,6 +209,14 @@ class Activations(LocalObserver):
 
 class ParameterUpdates(LocalObserver):
 
+    @property
+    def requires_include_statistics(self) -> bool:
+        """
+        :return: Whether the observation requires include_statistics to be provided.
+        """
+
+        return True
+
     def _get_observation_format(self) -> StatisticStorageTypes:
         """
         :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
@@ -231,6 +263,14 @@ class ParameterUpdates(LocalObserver):
 
 class Parameters(LocalObserver):
 
+    @property
+    def requires_include_statistics(self) -> bool:
+        """
+        :return: Whether the observation requires include_statistics to be provided.
+        """
+
+        return True
+
     def _get_observation_format(self) -> StatisticStorageTypes:
         """
         :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
@@ -692,17 +732,6 @@ class ModuleTypeOneHot(LocalObserver):
 
 class CurrentHyperparameters(LocalObserver):
 
-    def __init__(self, include_hparams: list[str] | None = None, **kwargs) -> None:
-        """
-        :param include_hparams: Names of the hyperparameters to include in the initial values vector returned by
-        this observation.
-        :param kwargs: Miscellaneous keyword arguments.
-        """
-
-        super().__init__(**kwargs)
-
-        self.include_hparams = include_hparams
-
     @property
     def can_standardize(self) -> bool:
         """
@@ -732,6 +761,14 @@ class CurrentHyperparameters(LocalObserver):
 
         return len([hparam for hparam in available_hparams if hparam in self.include_hparams])
 
+    @property
+    def requires_include_hparams(self) -> bool:
+        """
+        :return: Whether the observation requires include_hparams to be provided.
+        """
+
+        return True
+
     def _get_observation_format(self) -> StatisticStorageTypes:
         """
         :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
@@ -777,17 +814,6 @@ class CurrentHyperparameters(LocalObserver):
 
 class CurrentHyperparameterDeltas(LocalObserver):
 
-    def __init__(self, include_hparams: list[str] | None = None, **kwargs) -> None:
-        """
-        :param include_hparams: Names of the hyperparameters to include in the initial deltas vector returned by
-        this observation.
-        :param kwargs: Miscellaneous keyword arguments.
-        """
-
-        super().__init__(**kwargs)
-
-        self.include_hparams = include_hparams
-
     @property
     def can_standardize(self) -> bool:
         """
@@ -817,6 +843,14 @@ class CurrentHyperparameterDeltas(LocalObserver):
 
         return len([hparam for hparam in available_hparams if hparam in self.include_hparams])
 
+    @property
+    def requires_include_hparams(self) -> bool:
+        """
+        :return: Whether the observation requires include_hparams to be provided.
+        """
+
+        return True
+
     def _get_observation_format(self) -> StatisticStorageTypes:
         """
         :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
@@ -865,17 +899,6 @@ class HyperparameterTransformTypes(LocalObserver):
 
     TRANSFORM_TYPE_TO_IDX = dict(((s, i) for i, s in enumerate(HyperparameterTransformType)))
 
-    def __init__(self, include_hparams: list[str] | None = None, **kwargs) -> None:
-        """
-        :param include_hparams: Names of the hyperparameters to include in the transforms vector returned by
-        this observation.
-        :param kwargs: Miscellaneous keyword arguments.
-        """
-
-        super().__init__(**kwargs)
-
-        self.include_hparams = include_hparams
-
     @property
     def can_standardize(self) -> bool:
         """
@@ -907,6 +930,14 @@ class HyperparameterTransformTypes(LocalObserver):
             [hparam for hparam in available_hparams if hparam in self.include_hparams]
         )
 
+    @property
+    def requires_include_hparams(self) -> bool:
+        """
+        :return: Whether the observation requires include_hparams to be provided.
+        """
+
+        return True
+
     def _get_observation_format(self) -> StatisticStorageTypes:
         """
         :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
@@ -1087,7 +1118,6 @@ class LogOfNoiseScaleObserver(LocalObserver):
         *,
         decay_factor: float = LHOPT_CONSTANTS["DEFAULT_DECAY_FACTOR"],
         time_window: int = LHOPT_CONSTANTS["DEFAULT_TIME_WINDOW"],
-        include_statistics: list[str] | None = None,
         **kwargs,
     ) -> None:
         """
@@ -1100,7 +1130,6 @@ class LogOfNoiseScaleObserver(LocalObserver):
 
         super().__init__(**kwargs)
 
-        self.include_statistics = include_statistics
         self.decay_factor = max(0.0, decay_factor)
         self.time_window = max(1, time_window)
 
@@ -1108,14 +1137,6 @@ class LogOfNoiseScaleObserver(LocalObserver):
         self._time_series: list[tuple[float, float]] = []  # (time, value) pairs
         self._current_time: float = 0.0
 
-    def _get_observation_format(self) -> StatisticStorageTypes:
-        """
-        :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
-        enumeration class.
-        """
-
-        return StatisticStorageTypes.VECTOR
-
     @property
     def can_standardize(self) -> bool:
         """
@@ -1132,6 +1153,29 @@ class LogOfNoiseScaleObserver(LocalObserver):
 
         return False
 
+    @property
+    def vector_length(self) -> int:
+        """
+        :return: Length of the vector returned by this observation if it returns a vector.
+        """
+        return 2  # [log_noise_scale, cdf_feature]
+
+    @property
+    def requires_include_statistics(self) -> bool:
+        """
+        :return: Whether the observation requires include_statistics to be provided.
+        """
+
+        return True
+
+    def _get_observation_format(self) -> StatisticStorageTypes:
+        """
+        :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
+        enumeration class.
+        """
+
+        return StatisticStorageTypes.VECTOR
+
     def _update_time(self) -> None:
         """Update the current time counter."""
         self._current_time += 1.0
@@ -1145,13 +1189,6 @@ class LogOfNoiseScaleObserver(LocalObserver):
         """
         return compute_cdf_feature(value, self._time_series, self.decay_factor, self._current_time, self.time_window)
 
-    @property
-    def vector_length(self) -> int:
-        """
-        :return: Length of the vector returned by this observation if it returns a vector.
-        """
-        return 2  # [log_noise_scale, cdf_feature]
-
     def _observe(
         self,
         observation_inputs: ObservationInputs,
@@ -1169,16 +1206,16 @@ class LogOfNoiseScaleObserver(LocalObserver):
         """
 
         statistics = tracked_statistics[statistic_trackers.LogOfNoiseScaleStatistics.__name__]
-
         raw_value = list(statistics.values())[0]  # type: ignore[list-item]
+
         assert isinstance(raw_value, float), f"Expected float, got {type(raw_value)}"  # to avoid type errors with mypy
+
         batch_size = hyperparameter_states.global_hparams.batch_size.external_value
         learning_rate = hyperparameter_states.parameter_group_hparams[
             self.parameter_group_name
         ].learning_rate.external_value
 
         log_b_over_epsilon = math.log(batch_size / learning_rate)
-
         log_noise_scale = raw_value + log_b_over_epsilon
 
         cdf_feature = self._compute_cdf_feature(log_noise_scale)  # type: ignore[arg-type]

libinephany/observations/statistic_trackers.py

@@ -1187,19 +1187,17 @@ class LogOfNoiseScaleStatistics(Statistic):
         # This is a common assumption when the exact noise structure is unknown
         noise_covariance = torch.ones_like(hessian_diagonals)
 
-        # Compute tr(HΣ)
-        trace_hessian_noise_covariance = torch.sum(hessian_diagonals * noise_covariance)
-
-        # Avoid division by zero and log of zero
-        if trace_hessian_noise_covariance <= 0:
-            return None
+        # Compute tr(HΣ), add zero division tolerance to avoid log of zero when gradient is too small
+        trace_hessian_noise_covariance = (
+            torch.sum(hessian_diagonals * noise_covariance) + LHOPT_CONSTANTS["ZERO_DIVISION_TOLERANCE"]
+        )
 
         log_trace_hessian_noise_covariance = torch.log(trace_hessian_noise_covariance).item()
 
-        # Compute tr(H^3 Σ)
-        trace_hessian_cubed_noise_covariance = torch.sum(hessian_diagonals**3 * noise_covariance)
-        if trace_hessian_cubed_noise_covariance <= 0:
-            return None
+        # Compute tr(H^3 Σ), add zero division tolerance to avoid log of zero when gradient is too small
+        trace_hessian_cubed_noise_covariance = (
+            torch.sum(hessian_diagonals**3 * noise_covariance) + LHOPT_CONSTANTS["ZERO_DIVISION_TOLERANCE"]
+        )
 
         log_trace_hessian_cubed_noise_covariance = torch.log(trace_hessian_cubed_noise_covariance).item()
 

libinephany/utils/enums.py

@@ -110,3 +110,9 @@ class ModuleTypes(EnumWithIndices):
     Attention = "attention"
     Linear = "linear"
     Embedding = "embedding"
+
+
+class ToyTaskName(EnumWithIndices):
+
+    XOR = "XOR"
+    NQM = "NQM"

{libinephany-1.0.0.dist-info → libinephany-1.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: libinephany
-Version: 1.0.0
+Version: 1.0.2
 Summary: Inephany library containing code commonly used by multiple subpackages.
 Author-email: Inephany <info@inephany.com>
 License: Apache 2.0

{libinephany-1.0.0.dist-info → libinephany-1.0.2.dist-info}/RECORD

@@ -6,18 +6,18 @@ libinephany/observations/observation_utils.py,sha256=JSNJYEi2d-VQ0ZovfHrn28RDv41
 libinephany/observations/observer_pipeline.py,sha256=_xA4vrijhG8-9MCtGXnKAEmpd6q0nKVpJgY_qSbypIA,12979
 libinephany/observations/pipeline_coordinator.py,sha256=mLfaHhkXVhMp9w5jWIAL3jPyauCM-795qOzyqwGOSdw,7932
 libinephany/observations/statistic_manager.py,sha256=LLg1zSxnJr2oQQepYla3qoUuRy10rsthr9jta4wEbnc,8956
-libinephany/observations/statistic_trackers.py,sha256=3LHvBXQ977-9ZW-KE9UohDeOayZdqQ5UQMTt0kuea40,47574
+libinephany/observations/statistic_trackers.py,sha256=F98V-H2Ljx0v2YnppYCCJLJojL6pzYBdbBh8Lb4lasA,47666
 libinephany/observations/observers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-libinephany/observations/observers/base_observers.py,sha256=9YI_jkivoCjyPtNSn3VnPADF1VqwGdHPkgi1kDzed3Y,16516
-libinephany/observations/observers/local_observers.py,sha256=yBXmuCaDZotJPmBZKdrPfGYtVO-CCvT5ZS0KvROOcE4,45657
+libinephany/observations/observers/base_observers.py,sha256=V8PIysq2wT6K-w_CqeM5benyif-xK1hPT3M6a4ic1So,17535
+libinephany/observations/observers/local_observers.py,sha256=PJDsZ-DO2rptW87wrZsclrx3GKuSMdFfxT_FF7ov0Is,46137
 libinephany/observations/observers/observer_containers.py,sha256=VNyqGgxYJ4r49Msp_kk-POgicb-_5w54twuT1qfNdxw,9562
 libinephany/observations/observers/global_observers/__init__.py,sha256=87WHRPYmL0tVsaTKUd91pwEpCZtHPSKRQoba2VQjswA,3018
-libinephany/observations/observers/global_observers/base_classes.py,sha256=CCkRx86Lll3gFzfqervP0jKdzNFKkKU7tEBh8ic1Yrc,8249
-libinephany/observations/observers/global_observers/constants.py,sha256=C_PwYhKxatJxNe5Jzb1tpoiRXAxxPrGkcdQBMQD8msY,1139
-libinephany/observations/observers/global_observers/gradient_observers.py,sha256=ZeujBhKeq8adw_J13omurjZnfloiadMYkiPuXYUZ8BU,20972
-libinephany/observations/observers/global_observers/hyperparameter_observers.py,sha256=o035-nSfjj7dy7Pz1IxpAqvU3tYQraxQd8Pttknxa6A,15034
-libinephany/observations/observers/global_observers/loss_observers.py,sha256=FlSuJqAJIXcAS_ypdZna6xxz89glI23A6D00sDn7ZLU,18508
-libinephany/observations/observers/global_observers/model_observers.py,sha256=HVNHnqk2uuXkmP8y3SL-IQ0AYArfXc7b-wckv9X7qbM,28457
+libinephany/observations/observers/global_observers/base_classes.py,sha256=Q7OblhmKscypTs9JBepSQwo6ljjOdPKTU9kbpuhq_W4,7800
+libinephany/observations/observers/global_observers/constants.py,sha256=TDQM_sGU8Swze794oB4TBaXFjSddt0OBhYPVhrXQ9Ko,1654
+libinephany/observations/observers/global_observers/gradient_observers.py,sha256=j9uX7043ic06W6vb6vDt_PH2a5WtLFCdKQZ1JQGT24Q,19531
+libinephany/observations/observers/global_observers/hyperparameter_observers.py,sha256=5Av8FgwWBJtcn4gpDzPdOTlKOOYy2lVEHS_gt5Sz7xo,15334
+libinephany/observations/observers/global_observers/loss_observers.py,sha256=Kf943FiuYWuWvjmhgmp3TGyIQoZ27ZKJcxfTBwXS-gA,17761
+libinephany/observations/observers/global_observers/model_observers.py,sha256=SGWXrmTdgp0kHvEvDSF7d3v1FEcK1sQDMPLQ8Wy3qv4,29306
 libinephany/observations/observers/global_observers/progress_observers.py,sha256=ypLk1_POAjA8V8rAaQ0B6Qh8m_04s9PAoXsw1KxVrLg,5872
 libinephany/observations/post_processors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 libinephany/observations/post_processors/postprocessors.py,sha256=43_e5UaDPr2KbAvqc_w3wLqnlm7bgRjqgCtyQ95-8cM,5913
@@ -42,7 +42,7 @@ libinephany/utils/backend_statuses.py,sha256=ZbpBPbz0qKmeqxyGGN_ePTrQ7Wrxh7KM6W2
 libinephany/utils/constants.py,sha256=XAOuPowvM4FDSbfvNsubKTAqSB84AANX4CoHb7LwgEI,2330
 libinephany/utils/directory_utils.py,sha256=408unVeE_5_Hm-ZYZuxc9sdvfuU0CgYELX7EzPlPieo,1217
 libinephany/utils/dropout_utils.py,sha256=X43yCW7Dh1cC5sNnivgS5j1fn871K_RCvxCBTT0YHKg,3392
-libinephany/utils/enums.py,sha256=6_6k_1I2BwYTIfquUOsoaQT5fkhMXUWtwCxLoTYuFyU,2906
+libinephany/utils/enums.py,sha256=6fTgUd4EiFh4TzNXjvWX-zx1UKb90emgDaGB5gyAbdo,2977
 libinephany/utils/error_severities.py,sha256=B9oidqOVaYOe0W6P6GwjpmuDsrkyTX30v1xdiUStCFk,1427
 libinephany/utils/exceptions.py,sha256=kgwLpHOgy3kciUz_I18xnYsWRtzdonfadUtwG2uDYk8,1823
 libinephany/utils/import_utils.py,sha256=WzC6V6UIa0nCiU2MekROwG82fWBh9RuVzichtby5EvM,1495
@@ -57,8 +57,8 @@ libinephany/utils/typing.py,sha256=rGbaPO3MaUndsWiC_wHzReD_TOLYqb43i01pKN-j7Xs,6
 libinephany/web_apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 libinephany/web_apps/error_logger.py,sha256=gAQIaqerqP4ornXZwFF1cghjnd2mMZEt3aVrTuUCr34,16653
 libinephany/web_apps/web_app_utils.py,sha256=qiq_lasPipgN1RgRudPJc342kYci8O_4RqppxmIX8NY,4095
-libinephany-1.0.0.dist-info/licenses/LICENSE,sha256=pogfDoMBP07ehIOvWymuWIar8pg2YLUhqOHsJQU3wdc,9250
-libinephany-1.0.0.dist-info/METADATA,sha256=N_0lNQBtTOt8bzKEyQhmiA_l4AkWKtMoq6GDL3FGXKI,8389
-libinephany-1.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-libinephany-1.0.0.dist-info/top_level.txt,sha256=bYAOXQdJgIoLkO2Ui0kxe7pSYegS_e38u0dMscd7COQ,12
-libinephany-1.0.0.dist-info/RECORD,,
+libinephany-1.0.2.dist-info/licenses/LICENSE,sha256=pogfDoMBP07ehIOvWymuWIar8pg2YLUhqOHsJQU3wdc,9250
+libinephany-1.0.2.dist-info/METADATA,sha256=pATrPbN9k--PfUM5JDsn6jPj-77eUnHD94rJrVUs8JI,8389
+libinephany-1.0.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+libinephany-1.0.2.dist-info/top_level.txt,sha256=bYAOXQdJgIoLkO2Ui0kxe7pSYegS_e38u0dMscd7COQ,12
+libinephany-1.0.2.dist-info/RECORD,,