libinephany 0.18.1__tar.gz → 1.0.0__tar.gz

libinephany-1.0.0/CODE_VERSION.cfg

@@ -0,0 +1 @@
+1.0.0

{libinephany-0.18.1/libinephany.egg-info → libinephany-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: libinephany
-Version: 0.18.1
+Version: 1.0.0
 Summary: Inephany library containing code commonly used by multiple subpackages.
 Author-email: Inephany <info@inephany.com>
 License: Apache 2.0

{libinephany-0.18.1 → libinephany-1.0.0}/libinephany/observations/observation_utils.py

@@ -25,6 +25,7 @@ from libinephany.utils import optim_utils
 # ======================================================================================================================
 
 EXP_AVERAGE = "exp_avg"
+MOMENTUM_BUFFER = "momentum_buffer"
 MIN_DECAY_FACTOR = 1e-10
 
 MIN_TOTAL_WEIGHT = 1e-15  # Minimum total weight threshold for numerical stability
@@ -64,10 +65,8 @@ def get_exponential_weighted_average(values: list[int | float]) -> float:
     :param values: List of values to average via EWA.
     :return: EWA of the given values.
     """
-
     exp_weighted_average = pd.Series(values).ewm(alpha=0.1).mean().iloc[-1]
     assert isinstance(exp_weighted_average, float)
-
     return exp_weighted_average
 
 
@@ -232,6 +231,24 @@ def form_update_tensor(
         raise NotImplementedError(f"Optimizer {type(optimizer).__name__} is not supported!")
 
 
+def form_momentum_tensor(
+    optimizer: optim.Optimizer, parameters: list[torch.Tensor], parameter_group: dict[str, Any]
+) -> None | torch.Tensor:
+    """
+    :param optimizer: Optimizer to form the momentum tensor from.
+    :param parameters: Parameters to create the momentum tensor from.
+    :param parameter_group: Parameter group within the optimizer the given parameters came from.
+    """
+    if type(optimizer) in optim_utils.ADAM_OPTIMISERS:
+        momentum_list = [optimizer.state[p][EXP_AVERAGE].view(-1) for p in parameters if tensor_on_local_rank(p)]
+        return torch.cat(momentum_list) if momentum_list else None
+    elif type(optimizer) in optim_utils.SGD_OPTIMISERS:
+        momentum_list = [optimizer.state[p][MOMENTUM_BUFFER].view(-1) for p in parameters if tensor_on_local_rank(p)]
+        return torch.cat(momentum_list) if momentum_list else None
+    else:
+        raise NotImplementedError(f"Optimizer {type(optimizer).__name__} is not supported!")
+
+
 def null_standardizer(value_to_standardize: float, **kwargs) -> float:
     """
     :param value_to_standardize: Value to mock the standardization of.

{libinephany-0.18.1 → libinephany-1.0.0}/libinephany/observations/observers/base_observers.py

@@ -43,15 +43,15 @@ class Observer(ABC):
         standardizer: Standardizer | None,
         observer_config: ObserverConfig,
         should_standardize: bool = True,
-        skip_statistics: list[str] | None = None,
+        include_statistics: list[str] | None = None,
         **kwargs,
     ) -> None:
         """
         :param standardizer: None or the standardizer to apply to the returned observations.
         :param global_config: ObserverConfig that can be used to inform various observation calculations.
         :param should_standardize: Whether standardization should be applied to returned values.
-        :param skip_statistics: If the observation uses the TensorStatistic model to return observations, names of the
-        fields in the model to not include in returned observations.
+        :param include_statistics: If the observation uses the TensorStatistic model to return observations, names of the
+        fields in the model to include in returned observations.
         :param kwargs: Miscellaneous keyword arguments.
         """
 
@@ -63,7 +63,10 @@ class Observer(ABC):
         self.standardize = standardizer if standardizer is not None else observation_utils.null_standardizer
         self.should_standardize = should_standardize and self.can_standardize
 
-        self.skip_statistics = TensorStatistics.filter_skip_statistics(skip_statistics=skip_statistics)
+        self.include_statistics: list[str] | None = None
+
+        if include_statistics is not None:
+            self.include_statistics = TensorStatistics.filter_include_statistics(include_statistics=include_statistics)
 
     @final
     @property
@@ -102,7 +105,10 @@ class Observer(ABC):
         observation_format = self.observation_format
 
         if observation_format is StatisticStorageTypes.TENSOR_STATISTICS:
-            return len([field for field in TensorStatistics.model_fields.keys() if field not in self.skip_statistics])
+            if self.include_statistics is None:
+                raise ValueError(f"{self.__class__.__name__} must be provided with include_statistics.")
+
+            return len([field for field in TensorStatistics.model_fields.keys() if field in self.include_statistics])
 
         elif observation_format is StatisticStorageTypes.FLOAT:
             return 1
@@ -231,10 +237,13 @@ class Observer(ABC):
             self._cached_observation = deepcopy(observations)
 
         if self.observation_format is StatisticStorageTypes.TENSOR_STATISTICS:
+            if self.include_statistics is None:
+                raise ValueError(f"{self.__class__.__name__} must be provided with include_statistics.")
+
             if return_dict:
                 observations_dict = observations.as_observation_dict()  # type: ignore
 
-            observations = observations.to_list(skip_statistics=self.skip_statistics)  # type: ignore
+            observations = observations.to_list(include_statistics=self.include_statistics)  # type: ignore
 
         observations = [observations] if not isinstance(observations, list) else observations  # type: ignore
 
@@ -256,7 +265,7 @@ class Observer(ABC):
     def inform(self) -> float | int | dict[str, float] | None:
         """
         :return: The cached observation. If the observation format is TensorStatistics then it is converted to a
-        dictionary with the statistics specified in skip_statistics excluded.
+        dictionary with the statistics specified in include_statistics included.
         """
 
         if not self.can_inform:
@@ -269,7 +278,10 @@ class Observer(ABC):
             )
 
         if self.observation_format is StatisticStorageTypes.TENSOR_STATISTICS:
-            observation = self._cached_observation.model_dump(exclude=set(self.skip_statistics))  # type: ignore
+            if self.include_statistics is None:
+                raise ValueError(f"{self.__class__.__name__} must be provided with include_statistics.")
+
+            observation = self._cached_observation.model_dump(include=set(self.include_statistics))  # type: ignore
 
         else:
             observation = self._cached_observation

{libinephany-0.18.1 → libinephany-1.0.0}/libinephany/observations/observers/global_observers/__init__.py

@@ -8,7 +8,15 @@
 # ======================================================================================================================
 
 
-from .gradient_observers import GlobalFirstOrderGradients, GlobalSecondOrderGradients, LHOPTGradientVarianceFraction
+from .gradient_observers import (
+    CosineSimilarityObserverOfGradientAndMomentum,
+    CosineSimilarityObserverOfGradientAndUpdate,
+    CosineSimilarityOfGradientAndParameter,
+    GlobalFirstOrderGradients,
+    GlobalSecondOrderGradients,
+    LHOPTGradientVarianceFraction,
+    LHOPTMomentumGradientRatio,
+)
 from .hyperparameter_observers import (
     InitialHyperparameters,
     LHOPTHyperparameterRatio,
@@ -31,8 +39,11 @@ from .model_observers import (
     GlobalLAMBTrustRatio,
     GlobalParameters,
     GlobalParameterUpdates,
+    LHOPTAverageParameterUpdateMagnitudeObserver,
+    LHOPTGlobalLAMBTrustRatio,
     LogRatioOfPreviousAndCurrentParamNormEnvStepObserver,
     LogRatioOfUpdateAndPreviousParamNormEnvStepObserver,
+    LogRatioOfUpdateAndPreviousParamNormInnerStepObserver,
     NumberOfLayers,
     NumberOfParameters,
 )
@@ -51,14 +62,17 @@ __all__ = [
     GlobalFirstOrderGradients.__name__,
     GlobalSecondOrderGradients.__name__,
     LHOPTGradientVarianceFraction.__name__,
+    LHOPTMomentumGradientRatio.__name__,
     GlobalActivations.__name__,
     GlobalParameterUpdates.__name__,
     GlobalParameters.__name__,
     GlobalLAMBTrustRatio.__name__,
     NumberOfParameters.__name__,
     NumberOfLayers.__name__,
+    LHOPTAverageParameterUpdateMagnitudeObserver.__name__,
     LogRatioOfPreviousAndCurrentParamNormEnvStepObserver.__name__,
     LogRatioOfUpdateAndPreviousParamNormEnvStepObserver.__name__,
+    LogRatioOfUpdateAndPreviousParamNormInnerStepObserver.__name__,
     TrainingProgress.__name__,
     EpochsCompleted.__name__,
     ProgressAtEachCheckpoint.__name__,
@@ -66,4 +80,8 @@ __all__ = [
     LHOPTValidationLoss.__name__,
     LHOPTLossRatio.__name__,
     PercentileOfLossAtEachCheckpoint.__name__,
+    LHOPTGlobalLAMBTrustRatio.__name__,
+    CosineSimilarityObserverOfGradientAndMomentum.__name__,
+    CosineSimilarityObserverOfGradientAndUpdate.__name__,
+    CosineSimilarityOfGradientAndParameter.__name__,
 ]

{libinephany-0.18.1 → libinephany-1.0.0}/libinephany/observations/observers/global_observers/constants.py

@@ -20,6 +20,7 @@ class LHOPTConstants(TypedDict):
     ZERO_DIVISION_TOLERANCE: float
     DEFAULT_SAMPLE_FREQUENCY: int
     DEFAULT_VARIANCE_THRESHOLD: float
+    DEFAULT_ENV_STEP_SAMPLE_FREQUENCY: int
 
 
 # Create the constants instance
@@ -36,4 +37,5 @@ LHOPT_CONSTANTS: LHOPTConstants = LHOPTConstants(
     ZERO_DIVISION_TOLERANCE=1e-8,
     DEFAULT_SAMPLE_FREQUENCY=4,
     DEFAULT_VARIANCE_THRESHOLD=1e-6,
+    DEFAULT_ENV_STEP_SAMPLE_FREQUENCY=10,
 )