libinephany 0.16.4__py3-none-any.whl → 0.17.0__py3-none-any.whl

libinephany/observations/observation_utils.py

@@ -64,8 +64,10 @@ def get_exponential_weighted_average(values: list[int | float]) -> float:
     :param values: List of values to average via EWA.
     :return: EWA of the given values.
     """
+
     exp_weighted_average = pd.Series(values).ewm(alpha=0.1).mean().iloc[-1]
     assert isinstance(exp_weighted_average, float)
+
     return exp_weighted_average
 
 

libinephany/observations/observer_pipeline.py

@@ -226,8 +226,10 @@ class ObserverPipeline:
         names to floats or TensorStatistic models.
         :param actions_taken: Dictionary mapping agent IDs to actions taken by that agent.
         :param return_dict: Whether to return a dictionary of observations as well as the normal vector.
-        :return: Tuple of a dictionary mapping agent ID to that agent's completed observation vector, a boolean
-        indicating whether an observation clip occurred and a dictionary of observations.
+        :return: Tuple of:
+         - A dictionary mapping agent ID to that agent's completed observation vector,
+         - A boolean indicating whether an observation clip occurred,
+         - A dictionary mapping agent ID to a dictionary mapping observer name to that observer's observation vector.
         """
 
         global_obs, global_obs_dict = self.global_observers.observe(

libinephany/observations/observers/global_observers/__init__.py

@@ -8,8 +8,13 @@
 # ======================================================================================================================
 
 
-from .gradient_observers import GlobalFirstOrderGradients, GlobalSecondOrderGradients
-from .hyperparameter_observers import InitialHyperparameters, ModelFamilyOneHot, OptimizerTypeOneHot
+from .gradient_observers import GlobalFirstOrderGradients, GlobalSecondOrderGradients, LHOPTGradientVarianceFraction
+from .hyperparameter_observers import (
+    InitialHyperparameters,
+    LHOPTHyperparameterRatio,
+    ModelFamilyOneHot,
+    OptimizerTypeOneHot,
+)
 from .loss_observers import (
     LHOPTLossRatio,
     LHOPTTrainingLoss,
@@ -26,6 +31,8 @@ from .model_observers import (
     GlobalLAMBTrustRatio,
     GlobalParameters,
     GlobalParameterUpdates,
+    LogRatioOfPreviousAndCurrentParamNormEnvStepObserver,
+    LogRatioOfUpdateAndPreviousParamNormEnvStepObserver,
     NumberOfLayers,
     NumberOfParameters,
 )
@@ -33,6 +40,7 @@ from .progress_observers import EpochsCompleted, ProgressAtEachCheckpoint, Train
 
 __all__ = [
     InitialHyperparameters.__name__,
+    LHOPTHyperparameterRatio.__name__,
     OptimizerTypeOneHot.__name__,
     ModelFamilyOneHot.__name__,
     TrainingLoss.__name__,
@@ -42,12 +50,15 @@ __all__ = [
     ValidationScore.__name__,
     GlobalFirstOrderGradients.__name__,
     GlobalSecondOrderGradients.__name__,
+    LHOPTGradientVarianceFraction.__name__,
     GlobalActivations.__name__,
     GlobalParameterUpdates.__name__,
     GlobalParameters.__name__,
     GlobalLAMBTrustRatio.__name__,
     NumberOfParameters.__name__,
     NumberOfLayers.__name__,
+    LogRatioOfPreviousAndCurrentParamNormEnvStepObserver.__name__,
+    LogRatioOfUpdateAndPreviousParamNormEnvStepObserver.__name__,
     TrainingProgress.__name__,
     EpochsCompleted.__name__,
     ProgressAtEachCheckpoint.__name__,

libinephany/observations/observers/global_observers/base_classes.py

@@ -4,6 +4,7 @@
 #
 # ======================================================================================================================
 
+import math
 from abc import ABC, abstractmethod
 from typing import Any
 
@@ -15,7 +16,7 @@ from libinephany.pydantic_models.schemas.tensor_statistics import TensorStatisti
 from libinephany.pydantic_models.states.hyperparameter_states import HyperparameterStates
 
 
-class LHOPTOuterStepBaseObserver(GlobalObserver, ABC):
+class LHOPTBaseObserver(GlobalObserver, ABC):
     """
     Base class for LHOPT outer step observers to eliminate duplicate code.
     """
@@ -95,6 +96,26 @@ class LHOPTOuterStepBaseObserver(GlobalObserver, ABC):
         """
         raise NotImplementedError
 
+    def _compute_log_ratio(self, numerator: float, denominator: float) -> float:
+        """
+        Compute the log ratio.
+
+        :param numerator: Numerator value
+        :param denominator: Denominator value
+        :return: Log ratio value
+        """
+        # Calculate the ratio of numerator to denominator
+
+        if denominator <= LHOPT_CONSTANTS["ZERO_DIVISION_TOLERANCE"]:
+            return 0.0
+
+        ratio = numerator / denominator
+
+        if ratio <= 0:
+            return 0.0
+
+        return math.log(ratio)
+
 
 class LHOPTCheckpointBaseObserver(GlobalObserver, ABC):
     """
@@ -181,3 +202,23 @@ class LHOPTCheckpointBaseObserver(GlobalObserver, ABC):
         :param action_taken: Action taken by the agent this class instance is assigned to.
         """
         raise NotImplementedError
+
+    def _compute_log_ratio(self, numerator: float, denominator: float) -> float:
+        """
+        Compute the log ratio.
+
+        :param numerator: Numerator value
+        :param denominator: Denominator value
+        :return: Log ratio value
+        """
+        # Calculate the ratio of numerator to denominator
+
+        if denominator <= LHOPT_CONSTANTS["ZERO_DIVISION_TOLERANCE"]:
+            return 0.0
+
+        ratio = numerator / denominator
+
+        if ratio <= 0:
+            return 0.0
+
+        return math.log(ratio)

libinephany/observations/observers/global_observers/constants.py

@@ -17,6 +17,9 @@ class LHOPTConstants(TypedDict):
     DEFAULT_TIME_WINDOW: int
     DEFAULT_CHECKPOINT_INTERVAL: int
     DEFAULT_PERCENTILE: float
+    ZERO_DIVISION_TOLERANCE: float
+    DEFAULT_SAMPLE_FREQUENCY: int
+    DEFAULT_VARIANCE_THRESHOLD: float
 
 
 # Create the constants instance
@@ -30,4 +33,7 @@ LHOPT_CONSTANTS: LHOPTConstants = LHOPTConstants(
     DEFAULT_TIME_WINDOW=32,
     DEFAULT_CHECKPOINT_INTERVAL=100,
     DEFAULT_PERCENTILE=0.6,
+    ZERO_DIVISION_TOLERANCE=1e-8,
+    DEFAULT_SAMPLE_FREQUENCY=4,
+    DEFAULT_VARIANCE_THRESHOLD=1e-6,
 )

libinephany/observations/observers/global_observers/gradient_observers.py

@@ -9,6 +9,8 @@ from typing import Any
 from libinephany.observations import observation_utils, statistic_trackers
 from libinephany.observations.observation_utils import StatisticStorageTypes
 from libinephany.observations.observers.base_observers import GlobalObserver
+from libinephany.observations.observers.global_observers.base_classes import LHOPTBaseObserver
+from libinephany.observations.observers.global_observers.constants import LHOPT_CONSTANTS
 from libinephany.pydantic_models.schemas.observation_models import ObservationInputs
 from libinephany.pydantic_models.schemas.tensor_statistics import TensorStatistics
 from libinephany.pydantic_models.states.hyperparameter_states import HyperparameterStates
@@ -110,3 +112,82 @@ class GlobalSecondOrderGradients(GlobalObserver):
                 skip_statistics=self.skip_statistics, compute_hessian_diagonal=self.compute_hessian_diagonal
             )
         }
+
+
+class LHOPTGradientVarianceFraction(LHOPTBaseObserver):
+    """
+    This is a global observer from the OpenAI paper "Learning to Optimize with Reinforcement Learning"
+    https://arxiv.org/abs/2305.18291.
+
+    It returns two-dimensional observations: [raw_value, cdf_feature] for gradient variance fraction values.
+    """
+
+    def __init__(
+        self,
+        *,
+        variance_threshold: float = LHOPT_CONSTANTS["DEFAULT_VARIANCE_THRESHOLD"],
+        **kwargs,
+    ) -> None:
+        """
+        :param variance_threshold: Threshold for variance comparison in gradient variance fraction calculation
+        :param kwargs: Other observation keyword arguments.
+        """
+        super().__init__(**kwargs)
+        self.variance_threshold = variance_threshold
+
+    @property
+    def can_standardize(self) -> bool:
+        """
+        This observer has its own CDF calculation, no need to standardize.
+        :return: Whether the observation can be standardized.
+        """
+        return False
+
+    def _get_observation_format(self) -> StatisticStorageTypes:
+        """
+        :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
+        enumeration class.
+        """
+        return StatisticStorageTypes.VECTOR
+
+    @property
+    def vector_length(self) -> int:
+        """
+        :return: Length of the vector returned by this observation if it returns a vector.
+        """
+        return 2  # [raw_value, cdf_feature]
+
+    def _observe(
+        self,
+        observation_inputs: ObservationInputs,
+        hyperparameter_states: HyperparameterStates,
+        tracked_statistics: dict[str, dict[str, float | TensorStatistics]],
+        action_taken: float | int | None,
+    ) -> float | int | list[int | float] | TensorStatistics:
+        """
+        :param observation_inputs: Observation input metrics not calculated with statistic trackers.
+        :param hyperparameter_states: HyperparameterStates that manages the hyperparameters.
+        :param tracked_statistics: Dictionary mapping statistic tracker class names to dictionaries mapping module
+        names to floats or TensorStatistic models.
+        :param action_taken: Action taken by the agent this class instance is assigned to.
+        :return: Single float/int, list of floats/ints or TensorStatistics model to add to the observation vector.
+        """
+        if statistic_trackers.GradientVarianceFraction.__name__ not in tracked_statistics:
+            return [0.0, 0.0]
+
+        raw_value = list(tracked_statistics[statistic_trackers.GradientVarianceFraction.__name__].values())[0]  # type: ignore[list-item]
+
+        cdf_feature = self._compute_cdf_feature(raw_value)  # type: ignore[arg-type]
+        self._update_time()
+
+        return [raw_value, cdf_feature]  # type: ignore[list-item]
+
+    def get_required_trackers(self) -> dict[str, dict[str, Any] | None]:
+        """
+        :return: Dictionary mapping statistic tracker class names to kwargs for the class or None if no kwargs are
+        needed.
+        """
+
+        return {
+            statistic_trackers.GradientVarianceFraction.__name__: dict(variance_threshold=self.variance_threshold),
+        }

libinephany/observations/observers/global_observers/hyperparameter_observers.py

@@ -12,6 +12,7 @@ from torch.optim import SGD, Adam, AdamW
 from libinephany.observations import observation_utils
 from libinephany.observations.observation_utils import StatisticStorageTypes
 from libinephany.observations.observers.base_observers import GlobalObserver
+from libinephany.observations.observers.global_observers.base_classes import LHOPT_CONSTANTS
 from libinephany.pydantic_models.schemas.observation_models import ObservationInputs
 from libinephany.pydantic_models.schemas.tensor_statistics import TensorStatistics
 from libinephany.pydantic_models.states.hyperparameter_states import HyperparameterStates
@@ -42,9 +43,7 @@ class InitialHyperparameters(GlobalObserver):
 
         available_hparams = HyperparameterStates.get_all_hyperparameters()
 
-        return len(
-            [hparam for hparam in available_hparams if not any(skipped in hparam for skipped in self.skip_hparams)]
-        )
+        return len([hparam for hparam in available_hparams if hparam not in self.skip_hparams])
 
     @property
     def can_standardize(self) -> bool:
@@ -284,3 +283,115 @@ class ModelFamilyOneHot(GlobalObserver):
         """
 
         self._sample_zero_vector()
+
+
+class LHOPTHyperparameterRatio(GlobalObserver):
+    """
+    LHOPT-specific hyperparameter ratio observer that returns the ratio of current value to initial value
+    for all hyperparameter actions.
+
+    This observer computes: current_value / initial_value for each hyperparameter,
+    providing insights into how much hyperparameters have changed from their starting values.
+    """
+
+    def __init__(self, skip_hparams: list[str] | None = None, pad_with: float = 0.0, **kwargs) -> None:
+        """
+        :param skip_hparams: Names of the hyperparameters to not include in the initial values vector returned by
+        this observation.
+        :param kwargs: Miscellaneous keyword arguments.
+        """
+
+        super().__init__(**kwargs)
+
+        force_skip = ["samples", "gradient_accumulation"]
+        self.skip_hparams = force_skip if skip_hparams is None else skip_hparams + force_skip
+        self.pad_with = pad_with
+
+    @property
+    def vector_length(self) -> int:
+        """
+        :return: Length of the vector returned by this observation if it returns a vector.
+        """
+
+        available_hparams = HyperparameterStates.get_all_hyperparameters()
+
+        return len([hparam for hparam in available_hparams if hparam not in self.skip_hparams])
+
+    @property
+    def can_standardize(self) -> bool:
+        """
+        :return: Whether the observation can be standardized.
+        """
+
+        return False
+
+    @property
+    def can_inform(self) -> bool:
+        """
+        :return: Whether observations from the observer can be used in the agent info dictionary.
+        """
+
+        return False
+
+    def _get_observation_format(self) -> StatisticStorageTypes:
+        """
+        :return: Format the observation returns data in. Must be one of the enum attributes in the StatisticStorageTypes
+        enumeration class.
+        """
+
+        return StatisticStorageTypes.VECTOR
+
+    def _observe(
+        self,
+        observation_inputs: ObservationInputs,
+        hyperparameter_states: HyperparameterStates,
+        tracked_statistics: dict[str, dict[str, float | TensorStatistics]],
+        action_taken: float | int | None,
+    ) -> float | int | list[int | float] | TensorStatistics:
+        """
+        :param observation_inputs: Observation input metrics not calculated with statistic trackers.
+        :param hyperparameter_states: HyperparameterStates that manages the hyperparameters.
+        :param tracked_statistics: Dictionary mapping statistic tracker class names to dictionaries mapping module
+        names to floats or TensorStatistic models.
+        :param action_taken: Action taken by the agent this class instance is assigned to.
+        :return: Single float/int, list of floats/ints or TensorStatistics model to add to the observation vector.
+        """
+
+        # Get initial and current hyperparameter values
+        initial_values = hyperparameter_states.get_initial_internal_values(self.skip_hparams)
+        initial_values = {
+            hparam_name: self.pad_with if initial_value is None else initial_value
+            for hparam_name, initial_value in initial_values.items()
+            if hparam_name not in self.skip_hparams
+        }
+        current_values = hyperparameter_states.get_current_internal_values(self.skip_hparams)
+        current_values = {
+            hparam_name: self.pad_with if current_value is None else current_value
+            for hparam_name, current_value in current_values.items()
+            if hparam_name not in self.skip_hparams
+        }
+
+        ratios = []
+
+        for hparam_name in initial_values.keys():
+            initial_value = initial_values[hparam_name]
+            current_value = current_values[hparam_name]
+
+            if initial_value is None or current_value is None:
+                ratios.append(0.0)
+                continue
+
+            if abs(initial_value) < LHOPT_CONSTANTS["ZERO_DIVISION_TOLERANCE"]:
+                ratios.append(0.0)
+            else:
+                ratios.append(current_value / initial_value)
+
+        return ratios
+
+    def get_required_trackers(self) -> dict[str, dict[str, Any] | None]:
+        """
+        :return: Dictionary mapping statistic tracker class names to kwargs for the class or None if no kwargs are
+        needed.
+        """
+
+        return {}

libinephany/observations/observers/global_observers/loss_observers.py

@@ -10,8 +10,8 @@ from typing import Any
 from libinephany.observations.observation_utils import StatisticStorageTypes
 from libinephany.observations.observers.base_observers import GlobalObserver
 from libinephany.observations.observers.global_observers.base_classes import (
+    LHOPTBaseObserver,
     LHOPTCheckpointBaseObserver,
-    LHOPTOuterStepBaseObserver,
 )
 from libinephany.observations.observers.global_observers.constants import LHOPT_CONSTANTS
 from libinephany.pydantic_models.schemas.observation_models import ObservationInputs
@@ -229,7 +229,7 @@ class ValidationScore(GlobalObserver):
         return {}
 
 
-class LHOPTTrainingLoss(LHOPTOuterStepBaseObserver):
+class LHOPTTrainingLoss(LHOPTBaseObserver):
     """
     This is a global observer from the OpenAI paper "Learning to Optimize with Reinforcement Learning"
     https://arxiv.org/abs/2305.18291.
@@ -284,7 +284,7 @@ class LHOPTTrainingLoss(LHOPTOuterStepBaseObserver):
         return 3  # [is_nan, is_inf, cdf_feature]
 
 
-class LHOPTValidationLoss(LHOPTOuterStepBaseObserver):
+class LHOPTValidationLoss(LHOPTBaseObserver):
     """
     This is a global observer from the OpenAI paper "Learning to Optimize with Reinforcement Learning"
     https://arxiv.org/abs/2305.18291.
@@ -339,7 +339,7 @@ class LHOPTValidationLoss(LHOPTOuterStepBaseObserver):
         return 3  # [is_nan, is_inf, cdf_feature]
 
 
-class LHOPTLossRatio(LHOPTOuterStepBaseObserver):
+class LHOPTLossRatio(LHOPTBaseObserver):
     """
     This is a global observer from the OpenAI paper "Learning to Optimize with Reinforcement Learning"
     https://arxiv.org/abs/2305.18291.

libinephany/observations/observers/global_observers/model_observers.py

@@ -10,6 +10,8 @@ from typing import Any
 from libinephany.observations import observation_utils, statistic_trackers
 from libinephany.observations.observation_utils import StatisticStorageTypes
 from libinephany.observations.observers.base_observers import GlobalObserver
+from libinephany.observations.observers.global_observers.base_classes import LHOPTBaseObserver
+from libinephany.observations.observers.global_observers.constants import LHOPT_CONSTANTS
 from libinephany.pydantic_models.schemas.observation_models import ObservationInputs
 from libinephany.pydantic_models.schemas.tensor_statistics import TensorStatistics
 from libinephany.pydantic_models.states.hyperparameter_states import HyperparameterStates
@@ -325,3 +327,143 @@ class NumberOfLayers(GlobalObserver):
         """
 
         return {statistic_trackers.NumberOfLayers.__name__: dict(trainable_only=self.trainable_only)}
+
+
+class LogRatioOfPreviousAndCurrentParamNormEnvStepObserver(LHOPTBaseObserver):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._previous_param_norm = None
+
+    @property
+    def vector_length(self) -> int:
+        """
+        :return: Length of the vector returned by this observation if it returns a vector.
+        """
+        return 2  # [tanh_feature, cdf_feature]
+
+    def _observe(
+        self,
+        observation_inputs: ObservationInputs,
+        hyperparameter_states: HyperparameterStates,
+        tracked_statistics: dict[str, dict[str, float | TensorStatistics]],
+        action_taken: float | int | None,
+    ) -> float | int | list[int | float] | TensorStatistics:
+        """
+        :param observation_inputs: Observation input metrics not calculated with statistic trackers.
+        :param hyperparameter_states: HyperparameterStates that manages the hyperparameters.
+        :param tracked_statistics: Dictionary mapping statistic tracker class names to dictionaries mapping module
+        names to floats or TensorStatistic models.
+        :param action_taken: Action taken by the agent this class instance is assigned to.
+        :return: Single float/int, list of floats/ints or TensorStatistics model to add to the observation vector.
+        """
+
+        statistics = tracked_statistics[statistic_trackers.ParameterStatistics.__name__]
+
+        current_param_norm = observation_utils.average_tensor_statistics(
+            tensor_statistics=[stats for stats in statistics.values() if isinstance(stats, TensorStatistics)]
+        ).norm_
+
+        if self._previous_param_norm is None:
+            self._previous_param_norm = current_param_norm
+            self._compute_cdf_feature(0.0)  # default value since we can't compute log ratio yet
+            self._update_time()
+            return [0.0, 0.0]
+
+        log_ratio = self._compute_log_ratio(current_param_norm, self._previous_param_norm)
+        tanh_feature = math.tanh(max(-LHOPT_CONSTANTS["TANH_BOUND"], min(LHOPT_CONSTANTS["TANH_BOUND"], log_ratio)))
+        cdf_feature = self._compute_cdf_feature(log_ratio)
+        self._update_time()
+        self._previous_param_norm = current_param_norm
+
+        return [tanh_feature, cdf_feature]
+
+    def get_required_trackers(self) -> dict[str, dict[str, Any] | None]:
+        """
+        :return: Dictionary mapping statistic tracker class names to kwargs for the class or None if no kwargs are
+        needed.
+        """
+
+        return {
+            statistic_trackers.ParameterStatistics.__name__: dict(skip_statistics=self.skip_statistics),
+        }
+
+    def reset(self) -> None:
+        """
+        Reset the observer by clearing the previous parameter norm and time series.
+        """
+
+        super().reset()
+        self._previous_param_norm = None
+
+
+class LogRatioOfUpdateAndPreviousParamNormEnvStepObserver(LHOPTBaseObserver):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._previous_param_norm = None
+
+    @property
+    def vector_length(self) -> int:
+        """
+        :return: Length of the vector returned by this observation if it returns a vector.
+        """
+        return 2  # [tanh_feature, cdf_feature]
+
+    def _observe(
+        self,
+        observation_inputs: ObservationInputs,
+        hyperparameter_states: HyperparameterStates,
+        tracked_statistics: dict[str, dict[str, float | TensorStatistics]],
+        action_taken: float | int | None,
+    ) -> float | int | list[int | float] | TensorStatistics:
+        """
+        :param observation_inputs: Observation input metrics not calculated with statistic trackers.
+        :param hyperparameter_states: HyperparameterStates that manages the hyperparameters.
+        :param tracked_statistics: Dictionary mapping statistic tracker class names to dictionaries mapping module
+        names to floats or TensorStatistics models.
+        :param action_taken: Action taken by the agent this class instance is assigned to.
+        :return: List containing [raw_log_ratio, cdf_feature].
+        """
+
+        update_statistics = tracked_statistics[statistic_trackers.ParameterUpdateStatistics.__name__]
+        param_statistics = tracked_statistics[statistic_trackers.ParameterStatistics.__name__]
+        update_norm = observation_utils.average_tensor_statistics(
+            tensor_statistics=[stats for stats in update_statistics.values() if isinstance(stats, TensorStatistics)]
+        ).norm_
+
+        current_param_norm = observation_utils.average_tensor_statistics(
+            tensor_statistics=[stats for stats in param_statistics.values() if isinstance(stats, TensorStatistics)]
+        ).norm_
+
+        if self._previous_param_norm is None:
+            self._previous_param_norm = current_param_norm
+            self._compute_cdf_feature(0.0)  # default value since we can't compute log ratio yet
+            self._update_time()
+            return [0.0, 0.0]
+        log_ratio = self._compute_log_ratio(update_norm, self._previous_param_norm)
+        tanh_feature = math.tanh(max(-LHOPT_CONSTANTS["TANH_BOUND"], min(LHOPT_CONSTANTS["TANH_BOUND"], log_ratio)))
+        cdf_feature = self._compute_cdf_feature(log_ratio)
+        self._update_time()
+        self._previous_param_norm = current_param_norm
+
+        return [tanh_feature, cdf_feature]
+
+    def get_required_trackers(self) -> dict[str, dict[str, Any] | None]:
+        """
+        :return: Dictionary mapping statistic tracker class names to kwargs for the class or None if no kwargs are
+        needed.
+        """
+
+        return {
+            statistic_trackers.ParameterUpdateStatistics.__name__: dict(skip_statistics=self.skip_statistics),
+            statistic_trackers.ParameterStatistics.__name__: dict(skip_statistics=self.skip_statistics),
+        }
+
+    def reset(self) -> None:
+        """
+        Reset the observer by clearing the previous parameter norm and time series.
+        """
+
+        super().reset()
+        self._previous_param_norm = None

libinephany/observations/observers/local_observers.py

@@ -469,6 +469,94 @@ class ActionSchemeOneHot(LocalObserver):
         return {}
 
 
+class PreviousActionRepresentation(LocalObserver):
+    """
+    Observer that returns the representation of the previous action taken by the agent.
+
+    This observer tracks the previous action and returns it in an appropriate format:
+    - For discrete actions: returns one-hot encoding of the previous action
+    - For continuous actions: returns the previous action value directly
+    """
+
+    DISCRETE_INDEX = 0
+
+    def __init__(self, **kwargs) -> None:
+        super().__init__(**kwargs)
+        self._previous_action: float | int | None = None
+
+    @property
+    def vector_length(self) -> int:
+        """
+        :return: Length of the vector returned by this observation if it returns a vector.
+        """
+        return self.number_of_discrete_actions if self.is_discrete else 1
+
+    @property
+    def is_discrete(self) -> bool:
+        """
+        :return: Whether the agent is using discrete actions.
+        """
+
+        valid_actions = self.number_of_discrete_actions is not None and self.number_of_discrete_actions > 0
+        return self.action_scheme_index == self.DISCRETE_INDEX and valid_actions
+
+    @property
+    def can_inform(self) -> bool:
+        """
+        :return: Whether observations from the observer can be used in the agent info dictionary.
+        """
+        return False
+
+    def _get_observation_format(self) -> StatisticStorageTypes:
+        """
+        :return: Format the observation returns data in.
+        """
+        return StatisticStorageTypes.VECTOR
+
+    def _observe(
+        self,
+        observation_inputs: ObservationInputs,
+        hyperparameter_states: HyperparameterStates,
+        tracked_statistics: dict[str, dict[str, float | TensorStatistics]],
+        action_taken: float | int | None,
+    ) -> float | int | list[int | float] | TensorStatistics:
+        """
+        Returns the representation of the previous action.
+
+        :param observation_inputs: Observation input metrics not calculated with statistic trackers.
+        :param hyperparameter_states: HyperparameterStates that manages the hyperparameters.
+        :param tracked_statistics: Dictionary mapping statistic tracker class names to dictionaries mapping module
+        names to floats or TensorStatistic models.
+        :param action_taken: Action taken by the agent this class instance is assigned to.
+        :return: Previous action representation (one-hot vector for discrete, float for continuous).
+        """
+
+        if self._previous_action is None:
+            result = [0.0] * self.vector_length
+        else:
+            if self.is_discrete:
+                result = observation_utils.create_one_hot_observation(
+                    vector_length=self.vector_length, one_hot_index=int(self._previous_action)
+                )
+            else:
+                result = [float(self._previous_action)]
+
+        self._previous_action = action_taken
+
+        return result
+
+    def get_required_trackers(self) -> dict[str, dict[str, Any] | None]:
+        """
+        :return: Dictionary mapping statistic tracker class names to kwargs for the class or None if no kwargs are
+        needed.
+        """
+        return {}
+
+    def reset(self) -> None:
+        """Resets the observer by clearing the previous action."""
+        self._previous_action = None
+
+
 class DepthOneHot(LocalObserver):
 
     @property

libinephany/observations/statistic_trackers.py

@@ -16,6 +16,7 @@ from torch.distributed import ReduceOp
 
 from libinephany.observations import observation_utils
 from libinephany.observations.observation_utils import StatisticStorageTypes
+from libinephany.observations.observers.global_observers.constants import LHOPT_CONSTANTS
 from libinephany.pydantic_models.schemas.tensor_statistics import TensorStatistics
 from libinephany.utils import torch_distributed_utils
 from libinephany.utils.constants import PARAMS, SCHEDULER_GROUP_NAME
@@ -883,3 +884,77 @@ class NumberOfLayers(Statistic):
         self._count = count
 
         return count
+
+
+class GradientVarianceFraction(Statistic):
+
+    def __init__(
+        self,
+        *,
+        variance_threshold: float = LHOPT_CONSTANTS["DEFAULT_VARIANCE_THRESHOLD"],
+        **kwargs,
+    ) -> None:
+        """
+        :param variance_threshold: Threshold for variance comparison in gradient variance fraction calculation.
+        :param kwargs: Other observation keyword arguments.
+        """
+
+        super().__init__(**kwargs)
+        self.variance_threshold = variance_threshold
+
+    def _get_storage_format(self) -> StatisticStorageTypes:
+        """
+        :return: Storage format this observation stores data in. Must be one of the enum attributes in the
+        StatisticStorageTypes enumeration class.
+        """
+
+        return StatisticStorageTypes.FLOAT
+
+    def _gather(
+        self,
+        *,
+        optimizer: optim.Optimizer,
+        model: nn.Module,
+        parameters: list[torch.Tensor],
+        parameter_group: dict[str, Any],
+    ) -> torch.Tensor | TensorStatistics | float | None:
+        """
+        :param optimizer: Optimizer the given parameters and parameter group came from.
+        :param model: Inner model to gather statistics from.
+        :param parameters: List of parameters to gather statistics from.
+        :param parameter_group: Parameter group the parameters originate from.
+        :return: TensorStatistics model or a float.
+        """
+
+        gradients = [p.grad for p in parameters if observation_utils.tensor_on_local_rank(p) and p.grad is not None]
+
+        if not gradients:
+            return 0.0  # Return 0.0 instead of None for no gradients
+
+        # Calculate variance fraction
+        return self._calculate_variance_fraction(gradients)
+
+    def _calculate_variance_fraction(self, gradients: list[torch.Tensor]) -> float:
+        """
+        Calculate the fraction of parameters with variance above threshold.
+        fraction = sqrt(variance) >= threshold
+
+        :param gradients: List of gradient tensors
+        :return: Fraction of parameters with high variance (0.0 to 1.0)
+        """
+        total_parameters = 0
+        variance_parameters = 0
+
+        for grad in gradients:
+            parameter_count = grad.numel()
+            total_parameters += parameter_count
+
+            variance = grad.var().item()
+
+            if math.sqrt(variance) >= self.variance_threshold:
+                variance_parameters += parameter_count
+
+        if total_parameters == 0:
+            return 0.0
+
+        return variance_parameters / total_parameters

libinephany/pydantic_models/states/hyperparameter_states.py

@@ -855,3 +855,20 @@ class HyperparameterStates(BaseModel):
         }
 
         return initial_internal_values
+
+    def get_current_internal_values(self, skip_hparams: list[str] | None = None) -> dict[str, float | int | None]:
+        """
+        :param skip_hparams: Hyperparameters to ignore while retrieving current values.
+        :return: Dictionary mapping hyperparameter names to their current values during training.
+        """
+
+        current_internal_values = {
+            **self.global_hparams.get_current_internal_values(skip_hparams),
+            **next(iter(self.parameter_group_hparams.values())).get_current_internal_values(skip_hparams),
+        }
+        current_internal_values = {
+            hparam_name: current_internal_values.get(hparam_name, None)
+            for hparam_name in self.initial_hyperparameter_internal_values
+        }
+
+        return current_internal_values

libinephany/utils/constants.py

@@ -8,6 +8,7 @@ KEY_HEADER_CASE = "X-API-Key"
 KEY_HEADER_NO_CASE = KEY_HEADER_CASE.lower()
 
 TIMESTAMP_FORMAT = "%Y-%m-%d-%H-%M-%S"
+TIMESTAMP_FORMAT_WITH_MS = "%Y-%m-%d-%H-%M-%S-%f"
 
 RLLIB_TRUNC_EPISODES = "truncate_episodes"
 RLLIB_COMP_EPISODES = "complete_episodes"

{libinephany-0.16.4.dist-info → libinephany-0.17.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: libinephany
-Version: 0.16.4
+Version: 0.17.0
 Summary: Inephany library containing code commonly used by multiple subpackages.
 Author-email: Inephany <info@inephany.com>
 License: Apache 2.0

{libinephany-0.16.4.dist-info → libinephany-0.17.0.dist-info}/RECORD

@@ -2,22 +2,22 @@ libinephany/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 libinephany/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 libinephany/aws/s3_functions.py,sha256=W8u85A6tDloo4FlJvydJbVHCUq_m9i8KDGdnKzy-Xpg,1745
 libinephany/observations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-libinephany/observations/observation_utils.py,sha256=CXep6CVvzgh8I_SSmZvyupQLiwDzKDEiVLYqidhnL-A,16264
-libinephany/observations/observer_pipeline.py,sha256=RvMH-TTDTu1Nk4S_KSHDkII1YuIRMSOXkPhn6g4B9ow,12815
+libinephany/observations/observation_utils.py,sha256=ejQ-hzKq_MX7A0304KypScmzloReNrH8dmcgonoRl4Q,16266
+libinephany/observations/observer_pipeline.py,sha256=ySIXueu_SRONewygsye4EYxWoJvjhLx7M-MKGZpDOgo,12915
 libinephany/observations/pipeline_coordinator.py,sha256=mw3c5jy_BWvNigUKNjIWMpReOjxFDblzOcWtsIkcls4,7907
 libinephany/observations/statistic_manager.py,sha256=LLg1zSxnJr2oQQepYla3qoUuRy10rsthr9jta4wEbnc,8956
-libinephany/observations/statistic_trackers.py,sha256=PUBqGgMRi51SmiNh5HAH5kpYxsaflRepmM-uKyMiQZg,30326
+libinephany/observations/statistic_trackers.py,sha256=06NjCy1MI865oU0KB5f-wQE3b2RvYawOOWxNJx4rFpw,32939
 libinephany/observations/observers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 libinephany/observations/observers/base_observers.py,sha256=RkG5SW0b6Ooy0_oscRHxyB_YFNP7k8fxu37jBZElxIM,15418
-libinephany/observations/observers/local_observers.py,sha256=EdivrylOcmxRsu4xiMwZqwmPX8Ru9-IRwoPk6En7qvw,37050
+libinephany/observations/observers/local_observers.py,sha256=7azTW227-rG_QJR5a0StfE0O4Ca1boMV9nrCphnhWik,40344
 libinephany/observations/observers/observer_containers.py,sha256=Inz4vkqaXsyj09EJYaWWx1eOIuzmOjbiul5wRL5Gm5Y,9274
-libinephany/observations/observers/global_observers/__init__.py,sha256=hhJIOCGUKR7ai3pkHm9p42ZnyTWGOayg0jyLo7yQsZ0,1897
-libinephany/observations/observers/global_observers/base_classes.py,sha256=WFaadwQHqN1G5-repyzvG3gq1Yii-NmjXmEN5g1eDK8,7184
-libinephany/observations/observers/global_observers/constants.py,sha256=3Mc5rXtg9kBLjRn-uOjp-shgUNZAbd_K1S1ut-2DoBc,844
-libinephany/observations/observers/global_observers/gradient_observers.py,sha256=KzaFR0jCPnJwVr1Gr7vK5tDlou0dU7iZ0qTvY19xzqg,5000
-libinephany/observations/observers/global_observers/hyperparameter_observers.py,sha256=G3ic1Y1A-UGvXiXPeLrxn2efZrvOe4QY4DhIDbaOV-0,10483
-libinephany/observations/observers/global_observers/loss_observers.py,sha256=q0fn_WyeqrB28n231dZMqoDp3x6WUPcvvq9gWKJS0wM,18544
-libinephany/observations/observers/global_observers/model_observers.py,sha256=Fa0jQEBtiKmq0jk5x91ha_k-tURKHJo_wRhOhuEtxVk,13337
+libinephany/observations/observers/global_observers/__init__.py,sha256=lQO-nq7hILu4F3ddFXcCR-ghfv0dzEA9nAYxZta7rxk,2306
+libinephany/observations/observers/global_observers/base_classes.py,sha256=CCkRx86Lll3gFzfqervP0jKdzNFKkKU7tEBh8ic1Yrc,8249
+libinephany/observations/observers/global_observers/constants.py,sha256=olXYxh353Th6hyhfk85kHxAWPeaDpuPkcz_awFvEz6c,1054
+libinephany/observations/observers/global_observers/gradient_observers.py,sha256=z2ow6zmTr7ujeaYyh5qGRE1woIt_yc2uxAaie7zgxqc,8398
+libinephany/observations/observers/global_observers/hyperparameter_observers.py,sha256=soGWoYpO5rqUQU0p4pr6QVjBvVg2odK71mVkxV38Ras,14838
+libinephany/observations/observers/global_observers/loss_observers.py,sha256=FlSuJqAJIXcAS_ypdZna6xxz89glI23A6D00sDn7ZLU,18508
+libinephany/observations/observers/global_observers/model_observers.py,sha256=bJIEdq5wWkLrw-MNllCe36FE4aMLbC4RoR1-wOOHMxc,19537
 libinephany/observations/observers/global_observers/progress_observers.py,sha256=m62jUiwPaOUzYG1h7Vg6znj_jK9699lDhg4AhK212s8,5615
 libinephany/observations/post_processors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 libinephany/observations/post_processors/postprocessors.py,sha256=43_e5UaDPr2KbAvqc_w3wLqnlm7bgRjqgCtyQ95-8cM,5913
@@ -34,12 +34,12 @@ libinephany/pydantic_models/schemas/request_schemas.py,sha256=VED8eAUvBofxeAx9gW
 libinephany/pydantic_models/schemas/response_schemas.py,sha256=SKFuasdjX5aH_I0vT3SwnpwhyMf9cNPB1ZpDeAGgoO8,2158
 libinephany/pydantic_models/schemas/tensor_statistics.py,sha256=Z-x-Fi_Dm0pLoHI88DnJO1krY671o0zbGRzx-gXPtVY,7534
 libinephany/pydantic_models/states/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-libinephany/pydantic_models/states/hyperparameter_states.py,sha256=fwqUmRbT5WxcnMPK8DmRXkBQOtCs9n6V24BeCyFTFL8,32688
+libinephany/pydantic_models/states/hyperparameter_states.py,sha256=qQqnYyKR5ucENb_Zyk0x71_zpgUtG_rOzbOcE0kTrkQ,33478
 libinephany/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 libinephany/utils/agent_utils.py,sha256=_2w1AY5Y4mQ5hes_Rq014VhZXOtIOn-W92mZgeixv3g,2658
 libinephany/utils/asyncio_worker.py,sha256=Ew23zKIbG1zwyCudcyiObMrw4G0f3p2QXzZfM4mePqI,2751
 libinephany/utils/backend_statuses.py,sha256=ZbpBPbz0qKmeqxyGGN_ePTrQ7Wrxh7KM6W26UDbPXtQ,644
-libinephany/utils/constants.py,sha256=Qh8iz5o1R4UDVVCB69jOQPX2SLWRCncpb_2yTHpFSbY,2259
+libinephany/utils/constants.py,sha256=q4R_DjFEvMhV4TKN0pswEr9YIpFTGzZeNGB-fTNttdM,2309
 libinephany/utils/directory_utils.py,sha256=408unVeE_5_Hm-ZYZuxc9sdvfuU0CgYELX7EzPlPieo,1217
 libinephany/utils/dropout_utils.py,sha256=X43yCW7Dh1cC5sNnivgS5j1fn871K_RCvxCBTT0YHKg,3392
 libinephany/utils/enums.py,sha256=6_6k_1I2BwYTIfquUOsoaQT5fkhMXUWtwCxLoTYuFyU,2906
@@ -57,8 +57,8 @@ libinephany/utils/typing.py,sha256=rGbaPO3MaUndsWiC_wHzReD_TOLYqb43i01pKN-j7Xs,6
 libinephany/web_apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 libinephany/web_apps/error_logger.py,sha256=gAQIaqerqP4ornXZwFF1cghjnd2mMZEt3aVrTuUCr34,16653
 libinephany/web_apps/web_app_utils.py,sha256=qiq_lasPipgN1RgRudPJc342kYci8O_4RqppxmIX8NY,4095
-libinephany-0.16.4.dist-info/licenses/LICENSE,sha256=pogfDoMBP07ehIOvWymuWIar8pg2YLUhqOHsJQU3wdc,9250
-libinephany-0.16.4.dist-info/METADATA,sha256=HsVgYNptrATUp_vFZziVcK60xJM9whyMbbf0ZZPnj4s,8390
-libinephany-0.16.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-libinephany-0.16.4.dist-info/top_level.txt,sha256=bYAOXQdJgIoLkO2Ui0kxe7pSYegS_e38u0dMscd7COQ,12
-libinephany-0.16.4.dist-info/RECORD,,
+libinephany-0.17.0.dist-info/licenses/LICENSE,sha256=pogfDoMBP07ehIOvWymuWIar8pg2YLUhqOHsJQU3wdc,9250
+libinephany-0.17.0.dist-info/METADATA,sha256=OJryt-xjeAYOYaDzTyv4UXSeTRGxBFipvjlDPNOLZvI,8390
+libinephany-0.17.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+libinephany-0.17.0.dist-info/top_level.txt,sha256=bYAOXQdJgIoLkO2Ui0kxe7pSYegS_e38u0dMscd7COQ,12
+libinephany-0.17.0.dist-info/RECORD,,