hyperbolix 0.2.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/PKG-INFO +1 -5
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/distributions/uniform_poincare.py +41 -13
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/nn_layers/__init__.py +10 -5
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/nn_layers/hyperboloid_conv.py +193 -1
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/nn_layers/hyperboloid_core.py +69 -3
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/nn_layers/hyperboloid_linear.py +171 -2
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/nn_layers/hyperboloid_regression.py +107 -1
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/nn_layers/hyperboloid_regularization.py +96 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/pyproject.toml +1 -5
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/README.md +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/__init__.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/distributions/__init__.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/distributions/_common.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/distributions/_wrapped_normal_base.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/distributions/wrapped_normal_hyperboloid.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/distributions/wrapped_normal_poincare.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/manifolds/__init__.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/manifolds/_base.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/manifolds/euclidean.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/manifolds/hyperboloid.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/manifolds/isometry_mappings.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/manifolds/poincare.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/manifolds/protocol.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/nn_layers/_helpers.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/nn_layers/hyperboloid_activations.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/nn_layers/hyperboloid_attention.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/nn_layers/hyperboloid_positional.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/nn_layers/poincare_activations.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/nn_layers/poincare_conv.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/nn_layers/poincare_linear.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/nn_layers/poincare_regression.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/nn_layers/poincare_rl.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/optim/__init__.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/optim/_riemannian_base.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/optim/manifold_metadata.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/optim/riemannian_adam.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/optim/riemannian_sgd.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/utils/__init__.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/utils/helpers.py +0 -0
- {hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/utils/math_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: hyperbolix
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Hyperbolic Deep Learning in JAX
|
|
5
5
|
Keywords: hyperbolic,deep-learning,jax,flax,machine-learning,riemannian,poincare,hyperboloid,neural-networks,geometric-deep-learning
|
|
6
6
|
Author: Timo Klein, Thomas Lang
|
|
@@ -19,11 +19,7 @@ Classifier: Typing :: Typed
|
|
|
19
19
|
Requires-Dist: chex>=0.1.91
|
|
20
20
|
Requires-Dist: flax>=0.12.0
|
|
21
21
|
Requires-Dist: jaxtyping>=0.3.2
|
|
22
|
-
Requires-Dist: numpy>=2.3.3
|
|
23
22
|
Requires-Dist: optax>=0.2.6
|
|
24
|
-
Requires-Dist: scikit-learn>=1.7.2
|
|
25
|
-
Requires-Dist: scipy>=1.16.2
|
|
26
|
-
Requires-Dist: tqdm>=4.67.1
|
|
27
23
|
Requires-Python: >=3.12
|
|
28
24
|
Project-URL: Homepage, https://github.com/timoklein/hyperbolix
|
|
29
25
|
Project-URL: Documentation, https://timoklein.github.io/hyperbolix/
|
|
@@ -23,21 +23,49 @@ from jaxtyping import Array, Float, PRNGKeyArray
|
|
|
23
23
|
|
|
24
24
|
from hyperbolix.manifolds import Manifold
|
|
25
25
|
|
|
26
|
-
|
|
27
26
|
# ---------------------------------------------------------------------------
|
|
28
|
-
# Gauss-Legendre quadrature
|
|
29
|
-
# Precomputed
|
|
27
|
+
# 64-point Gauss-Legendre quadrature nodes/weights on [-1, 1].
|
|
28
|
+
# Precomputed with numpy.polynomial.legendre.leggauss(64).
|
|
29
|
+
# fmt: off
|
|
30
30
|
# ---------------------------------------------------------------------------
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
31
|
+
_GL_NODES = jnp.array([
|
|
32
|
+
-9.993050417357721704e-01, -9.963401167719552198e-01, -9.910133714767442870e-01, -9.833362538846259771e-01,
|
|
33
|
+
-9.733268277899109755e-01, -9.610087996520537690e-01, -9.464113748584027652e-01, -9.295691721319395695e-01,
|
|
34
|
+
-9.105221370785028245e-01, -8.893154459951141400e-01, -8.659993981540927699e-01, -8.406292962525803159e-01,
|
|
35
|
+
-8.132653151227975385e-01, -7.839723589433413853e-01, -7.528199072605319397e-01, -7.198818501716108820e-01,
|
|
36
|
+
-6.852363130542332703e-01, -6.489654712546573112e-01, -6.111553551723932776e-01, -5.718956462026340004e-01,
|
|
37
|
+
-5.312794640198945650e-01, -4.894031457070529556e-01, -4.463660172534640869e-01, -4.022701579639916258e-01,
|
|
38
|
+
-3.572201583376681255e-01, -3.113228719902109698e-01, -2.646871622087674236e-01, -2.174236437400070832e-01,
|
|
39
|
+
-1.696444204239928311e-01, -1.214628192961205444e-01, -7.299312178779904237e-02, -2.435029266342442905e-02,
|
|
40
|
+
2.435029266342442905e-02, 7.299312178779904237e-02, 1.214628192961205444e-01, 1.696444204239928311e-01,
|
|
41
|
+
2.174236437400070832e-01, 2.646871622087674236e-01, 3.113228719902109698e-01, 3.572201583376681255e-01,
|
|
42
|
+
4.022701579639916258e-01, 4.463660172534640869e-01, 4.894031457070529556e-01, 5.312794640198945650e-01,
|
|
43
|
+
5.718956462026340004e-01, 6.111553551723932776e-01, 6.489654712546573112e-01, 6.852363130542332703e-01,
|
|
44
|
+
7.198818501716108820e-01, 7.528199072605319397e-01, 7.839723589433413853e-01, 8.132653151227975385e-01,
|
|
45
|
+
8.406292962525803159e-01, 8.659993981540927699e-01, 8.893154459951141400e-01, 9.105221370785028245e-01,
|
|
46
|
+
9.295691721319395695e-01, 9.464113748584027652e-01, 9.610087996520537690e-01, 9.733268277899109755e-01,
|
|
47
|
+
9.833362538846259771e-01, 9.910133714767442870e-01, 9.963401167719552198e-01, 9.993050417357721704e-01,
|
|
48
|
+
], dtype=jnp.float64)
|
|
49
|
+
|
|
50
|
+
_GL_WEIGHTS = jnp.array([
|
|
51
|
+
1.783280721694139931e-03, 4.147033260564499287e-03, 6.504457968978502418e-03, 8.846759826363397028e-03,
|
|
52
|
+
1.116813946013102757e-02, 1.346304789671786024e-02, 1.572603047602503037e-02, 1.795171577569728422e-02,
|
|
53
|
+
2.013482315353008756e-02, 2.227017380838296895e-02, 2.435270256871120004e-02, 2.637746971505491173e-02,
|
|
54
|
+
2.833967261425953538e-02, 3.023465707240255429e-02, 3.205792835485149483e-02, 3.380516183714179362e-02,
|
|
55
|
+
3.547221325688230259e-02, 3.705512854024008845e-02, 3.855015317861563984e-02, 3.995374113272054384e-02,
|
|
56
|
+
4.126256324262357611e-02, 4.247351512365361154e-02, 4.358372452932354757e-02, 4.459055816375656622e-02,
|
|
57
|
+
4.549162792741818367e-02, 4.628479658131447183e-02, 4.696818281621007590e-02, 4.754016571483041936e-02,
|
|
58
|
+
4.799938859645842132e-02, 4.834476223480299595e-02, 4.857546744150351148e-02, 4.869095700913981389e-02,
|
|
59
|
+
4.869095700913981389e-02, 4.857546744150351148e-02, 4.834476223480299595e-02, 4.799938859645842132e-02,
|
|
60
|
+
4.754016571483041936e-02, 4.696818281621007590e-02, 4.628479658131447183e-02, 4.549162792741818367e-02,
|
|
61
|
+
4.459055816375656622e-02, 4.358372452932354757e-02, 4.247351512365361154e-02, 4.126256324262357611e-02,
|
|
62
|
+
3.995374113272054384e-02, 3.855015317861563984e-02, 3.705512854024008845e-02, 3.547221325688230259e-02,
|
|
63
|
+
3.380516183714179362e-02, 3.205792835485149483e-02, 3.023465707240255429e-02, 2.833967261425953538e-02,
|
|
64
|
+
2.637746971505491173e-02, 2.435270256871120004e-02, 2.227017380838296895e-02, 2.013482315353008756e-02,
|
|
65
|
+
1.795171577569728422e-02, 1.572603047602503037e-02, 1.346304789671786024e-02, 1.116813946013102757e-02,
|
|
66
|
+
8.846759826363397028e-03, 6.504457968978502418e-03, 4.147033260564499287e-03, 1.783280721694139931e-03,
|
|
67
|
+
], dtype=jnp.float64)
|
|
68
|
+
# fmt: on
|
|
41
69
|
|
|
42
70
|
|
|
43
71
|
# ---------------------------------------------------------------------------
|
|
@@ -18,12 +18,12 @@ from .hyperboloid_attention import (
|
|
|
18
18
|
HyperbolicSoftmaxAttention,
|
|
19
19
|
focus_transform,
|
|
20
20
|
)
|
|
21
|
-
from .hyperboloid_conv import HypConv2DHyperboloid, HypConv3DHyperboloid, LorentzConv2D
|
|
22
|
-
from .hyperboloid_core import hrc, htc, lorentz_midpoint, lorentz_residual, spatial_to_hyperboloid
|
|
23
|
-
from .hyperboloid_linear import HTCLinear, HypLinearHyperboloidFHCNN
|
|
21
|
+
from .hyperboloid_conv import FGGConv2D, HypConv2DHyperboloid, HypConv3DHyperboloid, LorentzConv2D
|
|
22
|
+
from .hyperboloid_core import build_spacelike_V, hrc, htc, lorentz_midpoint, lorentz_residual, spatial_to_hyperboloid
|
|
23
|
+
from .hyperboloid_linear import FGGLinear, HTCLinear, HypLinearHyperboloidFHCNN
|
|
24
24
|
from .hyperboloid_positional import HyperbolicRoPE, HypformerPositionalEncoding, hope
|
|
25
|
-
from .hyperboloid_regression import HypRegressionHyperboloid
|
|
26
|
-
from .hyperboloid_regularization import HRCBatchNorm, HRCDropout, HRCLayerNorm, HRCRMSNorm
|
|
25
|
+
from .hyperboloid_regression import FGGLorentzMLR, HypRegressionHyperboloid
|
|
26
|
+
from .hyperboloid_regularization import FGGMeanOnlyBatchNorm, HRCBatchNorm, HRCDropout, HRCLayerNorm, HRCRMSNorm
|
|
27
27
|
from .poincare_activations import poincare_leaky_relu, poincare_relu, poincare_tanh
|
|
28
28
|
from .poincare_conv import HypConv2DPoincare
|
|
29
29
|
from .poincare_linear import HypLinearPoincare, HypLinearPoincarePP
|
|
@@ -31,6 +31,10 @@ from .poincare_regression import HypRegressionPoincare, HypRegressionPoincarePP
|
|
|
31
31
|
from .poincare_rl import HypRegressionPoincareHDRL
|
|
32
32
|
|
|
33
33
|
__all__ = [
|
|
34
|
+
"FGGConv2D",
|
|
35
|
+
"FGGLinear",
|
|
36
|
+
"FGGLorentzMLR",
|
|
37
|
+
"FGGMeanOnlyBatchNorm",
|
|
34
38
|
"HRCBatchNorm",
|
|
35
39
|
"HRCDropout",
|
|
36
40
|
"HRCLayerNorm",
|
|
@@ -52,6 +56,7 @@ __all__ = [
|
|
|
52
56
|
"HyperbolicSoftmaxAttention",
|
|
53
57
|
"HypformerPositionalEncoding",
|
|
54
58
|
"LorentzConv2D",
|
|
59
|
+
"build_spacelike_V",
|
|
55
60
|
"focus_transform",
|
|
56
61
|
"hope",
|
|
57
62
|
"hrc",
|
|
@@ -7,6 +7,8 @@ Dimension key:
|
|
|
7
7
|
A: ambient dimension (in_channels or hcat output dim)
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
|
+
from collections.abc import Callable
|
|
11
|
+
|
|
10
12
|
import jax
|
|
11
13
|
import jax.numpy as jnp
|
|
12
14
|
from flax import nnx
|
|
@@ -16,7 +18,7 @@ from hyperbolix.manifolds.hyperboloid import Hyperboloid
|
|
|
16
18
|
|
|
17
19
|
from ._helpers import validate_hyperboloid_manifold
|
|
18
20
|
from .hyperboloid_core import hrc
|
|
19
|
-
from .hyperboloid_linear import HypLinearHyperboloidFHCNN
|
|
21
|
+
from .hyperboloid_linear import FGGLinear, HypLinearHyperboloidFHCNN
|
|
20
22
|
|
|
21
23
|
|
|
22
24
|
class LorentzConv2D(nnx.Module):
|
|
@@ -530,3 +532,193 @@ class HypConv3DHyperboloid(nnx.Module):
|
|
|
530
532
|
output_BZHWC = linear_out_NC.reshape(batch, out_d, out_h, out_w, self.out_channels)
|
|
531
533
|
|
|
532
534
|
return output_BZHWC
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
class FGGConv2D(nnx.Module):
|
|
538
|
+
"""Fast and Geometrically Grounded Lorentz 2D convolutional layer.
|
|
539
|
+
|
|
540
|
+
Uses HCat (Lorentz direct concatenation) to combine receptive field points,
|
|
541
|
+
then applies FGGLinear for the channel mixing. This matches the reference
|
|
542
|
+
implementation pattern from Klis et al. 2026.
|
|
543
|
+
|
|
544
|
+
Computation steps:
|
|
545
|
+
1) Extract receptive field patches, pad with manifold origin if needed
|
|
546
|
+
2) Apply HCat (Lorentz direct concatenation) to combine patch points
|
|
547
|
+
3) Pass through FGGLinear for channel transformation
|
|
548
|
+
|
|
549
|
+
Parameters
|
|
550
|
+
----------
|
|
551
|
+
manifold_module : Hyperboloid
|
|
552
|
+
Class-based Hyperboloid manifold instance.
|
|
553
|
+
in_channels : int
|
|
554
|
+
Input ambient channels (D_in + 1), including time component.
|
|
555
|
+
out_channels : int
|
|
556
|
+
Output ambient channels (D_out + 1), including time component.
|
|
557
|
+
kernel_size : int or tuple[int, int]
|
|
558
|
+
Size of the convolutional kernel.
|
|
559
|
+
rngs : nnx.Rngs
|
|
560
|
+
Random number generators for parameter initialization.
|
|
561
|
+
stride : int or tuple[int, int], optional
|
|
562
|
+
Stride of the convolution (default: 1).
|
|
563
|
+
padding : str, optional
|
|
564
|
+
Padding mode: ``"SAME"`` or ``"VALID"`` (default: ``"SAME"``).
|
|
565
|
+
pad_mode : str, optional
|
|
566
|
+
How to fill padding pixels: ``"origin"`` fills with the manifold
|
|
567
|
+
origin ``(sqrt(1/c), 0, ..., 0)`` (matching reference), ``"edge"``
|
|
568
|
+
replicates border values (default: ``"origin"``).
|
|
569
|
+
activation : Callable or None, optional
|
|
570
|
+
Euclidean activation for the FGGLinear (default: None).
|
|
571
|
+
reset_params : str, optional
|
|
572
|
+
Weight init for FGGLinear: ``"eye"``, ``"xavier"``, ``"kaiming"``,
|
|
573
|
+
``"lorentz_kaiming"``, or ``"mlr"`` (default: ``"kaiming"``).
|
|
574
|
+
use_weight_norm : bool, optional
|
|
575
|
+
Weight normalization in FGGLinear (default: False).
|
|
576
|
+
init_bias : float, optional
|
|
577
|
+
Initial bias for FGGLinear (default: 0.5).
|
|
578
|
+
eps : float, optional
|
|
579
|
+
Numerical stability floor (default: 1e-7).
|
|
580
|
+
|
|
581
|
+
References
|
|
582
|
+
----------
|
|
583
|
+
Klis et al. "Fast and Geometrically Grounded Lorentz Neural Networks" (2026).
|
|
584
|
+
"""
|
|
585
|
+
|
|
586
|
+
def __init__(
|
|
587
|
+
self,
|
|
588
|
+
manifold_module: Hyperboloid,
|
|
589
|
+
in_channels: int,
|
|
590
|
+
out_channels: int,
|
|
591
|
+
kernel_size: int | tuple[int, int],
|
|
592
|
+
*,
|
|
593
|
+
rngs: nnx.Rngs,
|
|
594
|
+
stride: int | tuple[int, int] = 1,
|
|
595
|
+
padding: str = "SAME",
|
|
596
|
+
pad_mode: str = "origin",
|
|
597
|
+
activation: Callable | None = None,
|
|
598
|
+
reset_params: str = "kaiming",
|
|
599
|
+
use_weight_norm: bool = False,
|
|
600
|
+
init_bias: float = 0.5,
|
|
601
|
+
eps: float = 1e-7,
|
|
602
|
+
):
|
|
603
|
+
if padding not in ("SAME", "VALID"):
|
|
604
|
+
raise ValueError(f"padding must be 'SAME' or 'VALID', got '{padding}'")
|
|
605
|
+
if pad_mode not in ("origin", "edge"):
|
|
606
|
+
raise ValueError(f"pad_mode must be 'origin' or 'edge', got '{pad_mode}'")
|
|
607
|
+
|
|
608
|
+
validate_hyperboloid_manifold(manifold_module, required_methods=("hcat",))
|
|
609
|
+
self.manifold = manifold_module
|
|
610
|
+
self.in_channels = in_channels
|
|
611
|
+
self.out_channels = out_channels
|
|
612
|
+
self.padding = padding
|
|
613
|
+
self.pad_mode = pad_mode
|
|
614
|
+
self.eps = eps
|
|
615
|
+
|
|
616
|
+
if isinstance(kernel_size, int):
|
|
617
|
+
self.kernel_size = (kernel_size, kernel_size)
|
|
618
|
+
else:
|
|
619
|
+
self.kernel_size = kernel_size
|
|
620
|
+
|
|
621
|
+
if isinstance(stride, int):
|
|
622
|
+
self.stride = (stride, stride)
|
|
623
|
+
else:
|
|
624
|
+
self.stride = stride
|
|
625
|
+
|
|
626
|
+
# HCat output ambient dim: (in_channels - 1) * kh * kw + 1
|
|
627
|
+
kh, kw = self.kernel_size
|
|
628
|
+
hcat_out_ambient = (in_channels - 1) * kh * kw + 1
|
|
629
|
+
|
|
630
|
+
# FGGLinear for channel transformation
|
|
631
|
+
self.fgg_linear = FGGLinear(
|
|
632
|
+
hcat_out_ambient,
|
|
633
|
+
out_channels,
|
|
634
|
+
rngs=rngs,
|
|
635
|
+
activation=activation,
|
|
636
|
+
reset_params=reset_params,
|
|
637
|
+
use_weight_norm=use_weight_norm,
|
|
638
|
+
init_bias=init_bias,
|
|
639
|
+
eps=eps,
|
|
640
|
+
)
|
|
641
|
+
|
|
642
|
+
def _extract_patches(
|
|
643
|
+
self,
|
|
644
|
+
x: Float[Array, "batch height width in_channels"],
|
|
645
|
+
c: float,
|
|
646
|
+
) -> Float[Array, "batch out_height out_width kernel_h kernel_w in_channels"]:
|
|
647
|
+
"""Extract patches, padding with manifold origin or edge replication for SAME mode."""
|
|
648
|
+
batch, height, width, in_channels = x.shape
|
|
649
|
+
kh, kw = self.kernel_size
|
|
650
|
+
stride_h, stride_w = self.stride
|
|
651
|
+
|
|
652
|
+
if self.padding == "SAME":
|
|
653
|
+
out_height = (height + stride_h - 1) // stride_h
|
|
654
|
+
out_width = (width + stride_w - 1) // stride_w
|
|
655
|
+
pad_h = max((out_height - 1) * stride_h + kh - height, 0)
|
|
656
|
+
pad_w = max((out_width - 1) * stride_w + kw - width, 0)
|
|
657
|
+
pad_top = pad_h // 2
|
|
658
|
+
pad_bottom = pad_h - pad_top
|
|
659
|
+
pad_left = pad_w // 2
|
|
660
|
+
pad_right = pad_w - pad_left
|
|
661
|
+
|
|
662
|
+
if self.pad_mode == "origin":
|
|
663
|
+
# Pad with manifold origin: (√(1/c), 0, ..., 0)
|
|
664
|
+
padded_h = height + pad_h
|
|
665
|
+
padded_w = width + pad_w
|
|
666
|
+
padded = jnp.zeros((batch, padded_h, padded_w, in_channels))
|
|
667
|
+
padded = padded.at[..., 0].set(jnp.sqrt(1.0 / c))
|
|
668
|
+
x = padded.at[:, pad_top : pad_top + height, pad_left : pad_left + width, :].set(x)
|
|
669
|
+
else: # edge
|
|
670
|
+
x = jnp.pad(
|
|
671
|
+
x,
|
|
672
|
+
((0, 0), (pad_top, pad_bottom), (pad_left, pad_right), (0, 0)),
|
|
673
|
+
mode="edge",
|
|
674
|
+
)
|
|
675
|
+
|
|
676
|
+
# Extract patches: (B, H, W, C*kh*kw)
|
|
677
|
+
patches_flat = jax.lax.conv_general_dilated_patches(
|
|
678
|
+
lhs=x,
|
|
679
|
+
filter_shape=(kh, kw),
|
|
680
|
+
window_strides=(stride_h, stride_w),
|
|
681
|
+
padding="VALID",
|
|
682
|
+
dimension_numbers=("NHWC", "OIHW", "NHWC"),
|
|
683
|
+
)
|
|
684
|
+
|
|
685
|
+
out_h, out_w = patches_flat.shape[1], patches_flat.shape[2]
|
|
686
|
+
patches_BHWCkhkw = patches_flat.reshape(batch, out_h, out_w, in_channels, kh, kw)
|
|
687
|
+
patches_BHWkhkwC = patches_BHWCkhkw.transpose(0, 1, 2, 4, 5, 3)
|
|
688
|
+
|
|
689
|
+
return patches_BHWkhkwC
|
|
690
|
+
|
|
691
|
+
def __call__(
|
|
692
|
+
self,
|
|
693
|
+
x: Float[Array, "batch height width in_channels"],
|
|
694
|
+
c: float = 1.0,
|
|
695
|
+
) -> Float[Array, "batch out_height out_width out_channels"]:
|
|
696
|
+
"""Forward pass through the FGG convolutional layer.
|
|
697
|
+
|
|
698
|
+
Parameters
|
|
699
|
+
----------
|
|
700
|
+
x : Array, shape (B, H, W, in_channels)
|
|
701
|
+
Input feature map on the hyperboloid.
|
|
702
|
+
c : float, optional
|
|
703
|
+
Curvature parameter (default: 1.0).
|
|
704
|
+
|
|
705
|
+
Returns
|
|
706
|
+
-------
|
|
707
|
+
out : Array, shape (B, H', W', out_channels)
|
|
708
|
+
Output feature map on the hyperboloid.
|
|
709
|
+
"""
|
|
710
|
+
# Extract patches: (B, H', W', kh, kw, C)
|
|
711
|
+
patches = self._extract_patches(x, c)
|
|
712
|
+
batch, out_h, out_w, kh, kw, in_c = patches.shape
|
|
713
|
+
|
|
714
|
+
# Flatten batch+spatial: (B*H'*W', K, C) where K = kh*kw
|
|
715
|
+
patches_flat_NKC = patches.reshape(-1, kh * kw, in_c)
|
|
716
|
+
|
|
717
|
+
# HCat: (K, C) -> (hcat_dim,) per patch
|
|
718
|
+
hcat_out_NA = jax.vmap(self.manifold.hcat, in_axes=(0, None))(patches_flat_NKC, c)
|
|
719
|
+
|
|
720
|
+
# FGGLinear: (hcat_dim,) -> (out_channels,)
|
|
721
|
+
linear_out_NC = self.fgg_linear(hcat_out_NA, c)
|
|
722
|
+
|
|
723
|
+
# Reshape back to spatial
|
|
724
|
+
return linear_out_NC.reshape(batch, out_h, out_w, self.out_channels)
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
"""Core Hypformer operations for hyperboloid manifolds.
|
|
2
2
|
|
|
3
3
|
This module contains the foundational HRC (Hyperbolic Regularization Component) and
|
|
4
|
-
HTC (Hyperbolic Transformation Component) operations from the Hypformer paper
|
|
4
|
+
HTC (Hyperbolic Transformation Component) operations from the Hypformer paper, as well
|
|
5
|
+
as the spacelike V-matrix construction from FGG-LNN (Klis et al. 2026). These
|
|
5
6
|
are the building blocks used throughout the library for creating hyperbolic neural
|
|
6
7
|
network layers with curvature-change support.
|
|
7
8
|
|
|
@@ -9,13 +10,15 @@ Key Components
|
|
|
9
10
|
--------------
|
|
10
11
|
- **hrc**: Wraps Euclidean operations on spatial components only
|
|
11
12
|
- **htc**: Wraps Euclidean operations on full hyperboloid points
|
|
13
|
+
- **build_spacelike_V**: Constructs spacelike weight vectors for FGG layers
|
|
12
14
|
|
|
13
|
-
Both functions enable curvature transformations (c_in → c_out) and avoid
|
|
14
|
-
exp/log maps by using constraint-based time reconstruction.
|
|
15
|
+
Both HRC/HTC functions enable curvature transformations (c_in → c_out) and avoid
|
|
16
|
+
expensive exp/log maps by using constraint-based time reconstruction.
|
|
15
17
|
|
|
16
18
|
References
|
|
17
19
|
----------
|
|
18
20
|
Hypformer paper (citation to be added)
|
|
21
|
+
Klis et al. "Fast and Geometrically Grounded Lorentz Neural Networks" (2026)
|
|
19
22
|
"""
|
|
20
23
|
|
|
21
24
|
from collections.abc import Callable
|
|
@@ -23,6 +26,69 @@ from collections.abc import Callable
|
|
|
23
26
|
import jax.numpy as jnp
|
|
24
27
|
from jaxtyping import Array, Float
|
|
25
28
|
|
|
29
|
+
from hyperbolix.utils.math_utils import cosh as safe_cosh
|
|
30
|
+
from hyperbolix.utils.math_utils import sinh as safe_sinh
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def build_spacelike_V(
|
|
34
|
+
U_IO: Float[Array, "I O"],
|
|
35
|
+
b_O: Float[Array, "O"],
|
|
36
|
+
c: float,
|
|
37
|
+
eps: float = 1e-7,
|
|
38
|
+
) -> Float[Array, "Ai O"]:
|
|
39
|
+
"""Build spacelike V matrix with Minkowski metric absorbed.
|
|
40
|
+
|
|
41
|
+
Constructs the spacelike weight vectors from Euclidean weights U and bias b
|
|
42
|
+
via parallel transport (Eq. 12, Klis et al. 2026). The Minkowski metric
|
|
43
|
+
signature diag(-1, +1, ..., +1) is absorbed into the time row so that the
|
|
44
|
+
forward pass reduces to a single ``x @ V`` matmul.
|
|
45
|
+
|
|
46
|
+
Parameters
|
|
47
|
+
----------
|
|
48
|
+
U_IO : Array, shape (I, O)
|
|
49
|
+
Euclidean weight matrix. I = in_spatial, O = out_spatial.
|
|
50
|
+
b_O : Array, shape (O,)
|
|
51
|
+
Bias per output neuron.
|
|
52
|
+
c : float
|
|
53
|
+
Curvature parameter (positive).
|
|
54
|
+
eps : float, optional
|
|
55
|
+
Numerical stability floor for column norms (default: 1e-7).
|
|
56
|
+
|
|
57
|
+
Returns
|
|
58
|
+
-------
|
|
59
|
+
V_AiO : Array, shape (I+1, O)
|
|
60
|
+
Spacelike V matrix with negated time row (Minkowski metric absorbed).
|
|
61
|
+
Columns are v^(i) with ``v_time_mink = -||w|| * sinh(arg)``.
|
|
62
|
+
|
|
63
|
+
References
|
|
64
|
+
----------
|
|
65
|
+
Klis et al. "Fast and Geometrically Grounded Lorentz Neural Networks" (2026), Eq. 12.
|
|
66
|
+
"""
|
|
67
|
+
# Dimension key: I=in_spatial, O=out_spatial, Ai=in_ambient (I+1)
|
|
68
|
+
|
|
69
|
+
# Column norms of U: ||w^(i)||_E
|
|
70
|
+
norm_sq_O = jnp.sum(U_IO**2, axis=0) # (O,)
|
|
71
|
+
norm_O = jnp.sqrt(norm_sq_O + eps) # (O,) gradient-safe (never 0)
|
|
72
|
+
|
|
73
|
+
# Smooth gate: 0 for zero-norm columns, ~1 for normal columns.
|
|
74
|
+
# Ensures arg→0 smoothly when U column→0 (no weight → no bias transport).
|
|
75
|
+
gate_O = norm_sq_O / (norm_sq_O + eps) # (O,)
|
|
76
|
+
|
|
77
|
+
# Argument for sinh/cosh: -sqrt(c) * b / ||w||, gated for zero columns
|
|
78
|
+
arg_O = jnp.clip(-jnp.sqrt(c) * b_O * gate_O / norm_O, -100.0, 100.0) # (O,)
|
|
79
|
+
|
|
80
|
+
# Time row: negate to absorb Minkowski metric I_{1,D}
|
|
81
|
+
# v_time = ||w|| * sinh(arg), v_time_mink = -v_time
|
|
82
|
+
v_time_mink_O = -norm_O * safe_sinh(arg_O) # (O,)
|
|
83
|
+
|
|
84
|
+
# Spatial rows: w^(i) * cosh(arg)
|
|
85
|
+
v_space_IO = U_IO * safe_cosh(arg_O)[None, :] # (I, O)
|
|
86
|
+
|
|
87
|
+
# Stack: [v_time_mink; v_space] -> (I+1, O) = (Ai, O)
|
|
88
|
+
V_AiO = jnp.concatenate([v_time_mink_O[None, :], v_space_IO], axis=0) # (Ai, O)
|
|
89
|
+
|
|
90
|
+
return V_AiO
|
|
91
|
+
|
|
26
92
|
|
|
27
93
|
def spatial_to_hyperboloid(
|
|
28
94
|
spatial: Float[Array, "... D"],
|
|
@@ -1,9 +1,15 @@
|
|
|
1
1
|
"""Hyperboloid linear layers for JAX/Flax NNX.
|
|
2
2
|
|
|
3
3
|
This module contains linear transformation layers for the hyperboloid manifold,
|
|
4
|
-
including the Hyperbolic Transformation Component (HTC) module from the Hypformer paper
|
|
4
|
+
including the Hyperbolic Transformation Component (HTC) module from the Hypformer paper
|
|
5
|
+
and the FGG linear layer from Klis et al. 2026.
|
|
5
6
|
|
|
6
7
|
For the core HTC/HRC functions, see hyperboloid_core module.
|
|
8
|
+
|
|
9
|
+
Dimension key:
|
|
10
|
+
B: batch size
|
|
11
|
+
I: in_spatial (in_features - 1) O: out_spatial (out_features - 1)
|
|
12
|
+
Ai: in_ambient (in_features) Ao: out_ambient (out_features)
|
|
7
13
|
"""
|
|
8
14
|
|
|
9
15
|
from collections.abc import Callable
|
|
@@ -16,7 +22,7 @@ from jaxtyping import Array, Float
|
|
|
16
22
|
from hyperbolix.manifolds import Manifold
|
|
17
23
|
|
|
18
24
|
from ._helpers import validate_hyperboloid_manifold
|
|
19
|
-
from .hyperboloid_core import htc
|
|
25
|
+
from .hyperboloid_core import build_spacelike_V, htc
|
|
20
26
|
|
|
21
27
|
|
|
22
28
|
class HypLinearHyperboloidFHCNN(nnx.Module):
|
|
@@ -301,3 +307,166 @@ class HTCLinear(nnx.Module):
|
|
|
301
307
|
return out
|
|
302
308
|
|
|
303
309
|
return htc(x, linear_fn, c_in, c_out, self.eps)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
class FGGLinear(nnx.Module):
|
|
313
|
+
"""Fast and Geometrically Grounded Lorentz linear layer.
|
|
314
|
+
|
|
315
|
+
Implements the FGG linear layer from Klis et al. 2026. The key insight is that
|
|
316
|
+
the sinh/arcsinh cancellation in the Lorentzian activation chain simplifies the
|
|
317
|
+
forward pass to: matmul with spacelike V matrix -> Euclidean activation ->
|
|
318
|
+
time reconstruction. This achieves linear growth of hyperbolic distance (vs
|
|
319
|
+
logarithmic for Chen et al. 2022) and ~3x faster training/inference.
|
|
320
|
+
|
|
321
|
+
Forward pass:
|
|
322
|
+
1. Build spacelike V matrix from (U, b) with Minkowski metric absorbed
|
|
323
|
+
2. z = x @ V (Minkowski inner products via a single matmul)
|
|
324
|
+
3. z = h(z) (Euclidean activation, e.g. ReLU)
|
|
325
|
+
4. y_0 = sqrt(||z||^2 + 1/c) (time reconstruction)
|
|
326
|
+
5. y = [y_0, z] (on hyperboloid)
|
|
327
|
+
|
|
328
|
+
Parameters
|
|
329
|
+
----------
|
|
330
|
+
in_features : int
|
|
331
|
+
Input ambient dimension (D_in + 1), including time component.
|
|
332
|
+
out_features : int
|
|
333
|
+
Output ambient dimension (D_out + 1), including time component.
|
|
334
|
+
rngs : nnx.Rngs
|
|
335
|
+
Random number generators for parameter initialization.
|
|
336
|
+
activation : Callable or None, optional
|
|
337
|
+
Euclidean activation function applied after matmul (default: None).
|
|
338
|
+
reset_params : str, optional
|
|
339
|
+
Weight initialization scheme: ``"eye"``, ``"xavier"``, ``"kaiming"``,
|
|
340
|
+
``"lorentz_kaiming"``, or ``"mlr"`` (default: ``"eye"``).
|
|
341
|
+
use_weight_norm : bool, optional
|
|
342
|
+
If True, reparameterize U as ``g * v / ||v||`` for weight normalization
|
|
343
|
+
(default: False).
|
|
344
|
+
init_bias : float, optional
|
|
345
|
+
Initial value for bias entries (default: 0.5).
|
|
346
|
+
eps : float, optional
|
|
347
|
+
Numerical stability floor (default: 1e-7).
|
|
348
|
+
|
|
349
|
+
References
|
|
350
|
+
----------
|
|
351
|
+
Klis et al. "Fast and Geometrically Grounded Lorentz Neural Networks" (2026).
|
|
352
|
+
|
|
353
|
+
Examples
|
|
354
|
+
--------
|
|
355
|
+
>>> from flax import nnx
|
|
356
|
+
>>> from hyperbolix.nn_layers import FGGLinear
|
|
357
|
+
>>> import jax.numpy as jnp
|
|
358
|
+
>>>
|
|
359
|
+
>>> layer = FGGLinear(33, 65, rngs=nnx.Rngs(0), activation=jax.nn.relu)
|
|
360
|
+
>>> x = jnp.ones((8, 33))
|
|
361
|
+
>>> # project to hyperboloid
|
|
362
|
+
>>> x = x.at[:, 0].set(jnp.sqrt(jnp.sum(x[:, 1:]**2, axis=-1) + 1.0))
|
|
363
|
+
>>> y = layer(x, c=1.0)
|
|
364
|
+
>>> y.shape
|
|
365
|
+
(8, 65)
|
|
366
|
+
"""
|
|
367
|
+
|
|
368
|
+
def __init__(
|
|
369
|
+
self,
|
|
370
|
+
in_features: int,
|
|
371
|
+
out_features: int,
|
|
372
|
+
*,
|
|
373
|
+
rngs: nnx.Rngs,
|
|
374
|
+
activation: Callable[[jax.Array], jax.Array] | None = None,
|
|
375
|
+
reset_params: str = "eye",
|
|
376
|
+
use_weight_norm: bool = False,
|
|
377
|
+
init_bias: float = 0.5,
|
|
378
|
+
eps: float = 1e-7,
|
|
379
|
+
):
|
|
380
|
+
if reset_params not in ("eye", "xavier", "kaiming", "lorentz_kaiming", "mlr"):
|
|
381
|
+
raise ValueError(
|
|
382
|
+
f"reset_params must be 'eye', 'xavier', 'kaiming', 'lorentz_kaiming', or 'mlr', got '{reset_params}'"
|
|
383
|
+
)
|
|
384
|
+
|
|
385
|
+
in_spatial = in_features - 1 # I
|
|
386
|
+
out_spatial = out_features - 1 # O
|
|
387
|
+
|
|
388
|
+
self.in_features = in_features
|
|
389
|
+
self.out_features = out_features
|
|
390
|
+
self.activation = activation
|
|
391
|
+
self.use_weight_norm = use_weight_norm
|
|
392
|
+
self.eps = eps
|
|
393
|
+
|
|
394
|
+
# Initialize Euclidean weight U: (I, O)
|
|
395
|
+
key = rngs.params()
|
|
396
|
+
if reset_params == "eye":
|
|
397
|
+
U_init = 0.5 * jnp.eye(in_spatial, out_spatial)
|
|
398
|
+
elif reset_params == "xavier":
|
|
399
|
+
std = jnp.sqrt(1.0 / (in_spatial + out_spatial))
|
|
400
|
+
U_init = jax.random.normal(key, (in_spatial, out_spatial)) * std
|
|
401
|
+
elif reset_params == "kaiming":
|
|
402
|
+
std = jnp.sqrt(2.0 / in_spatial)
|
|
403
|
+
U_init = jax.random.normal(key, (in_spatial, out_spatial)) * std
|
|
404
|
+
elif reset_params == "lorentz_kaiming":
|
|
405
|
+
std = jnp.sqrt(1.0 / in_spatial)
|
|
406
|
+
U_init = jax.random.normal(key, (in_spatial, out_spatial)) * std
|
|
407
|
+
else: # mlr
|
|
408
|
+
std = jnp.sqrt(5.0 / in_spatial)
|
|
409
|
+
U_init = jax.random.normal(key, (in_spatial, out_spatial)) * std
|
|
410
|
+
|
|
411
|
+
# Weight normalization: decompose U = softplus(g) * v / ||v||
|
|
412
|
+
if use_weight_norm:
|
|
413
|
+
# Reference: v from reset_params (normalized in forward), g fixed magnitude
|
|
414
|
+
self.v = nnx.Param(U_init) # (I, O) direction
|
|
415
|
+
g_init_val = jnp.sqrt(1.0 / (in_spatial + out_spatial))
|
|
416
|
+
self.g = nnx.Param(jnp.full((out_spatial,), g_init_val)) # (O,)
|
|
417
|
+
else:
|
|
418
|
+
self.U = nnx.Param(U_init) # (I, O)
|
|
419
|
+
|
|
420
|
+
# Bias: init to init_bias
|
|
421
|
+
self.b = nnx.Param(jnp.full((out_spatial,), init_bias)) # (O,)
|
|
422
|
+
|
|
423
|
+
def _get_U(self) -> jax.Array:
|
|
424
|
+
"""Return the effective weight matrix, handling weight normalization."""
|
|
425
|
+
if self.use_weight_norm:
|
|
426
|
+
v_IO = self.v[...] # (I, O)
|
|
427
|
+
g_O = self.g[...] # (O,)
|
|
428
|
+
g_pos_O = jax.nn.softplus(g_O) # (O,) force positive magnitudes
|
|
429
|
+
v_norm_O = jnp.sqrt(jnp.sum(v_IO**2, axis=0) + self.eps) # (O,)
|
|
430
|
+
return g_pos_O[None, :] * v_IO / v_norm_O[None, :] # (I, O)
|
|
431
|
+
return self.U[...] # (I, O)
|
|
432
|
+
|
|
433
|
+
def __call__(
|
|
434
|
+
self,
|
|
435
|
+
x_BAi: Float[Array, "batch in_features"],
|
|
436
|
+
c: float = 1.0,
|
|
437
|
+
) -> Float[Array, "batch out_features"]:
|
|
438
|
+
"""Forward pass through the FGG linear layer.
|
|
439
|
+
|
|
440
|
+
Parameters
|
|
441
|
+
----------
|
|
442
|
+
x_BAi : Array, shape (B, Ai)
|
|
443
|
+
Input points on the hyperboloid with curvature ``c``.
|
|
444
|
+
Ai = in_features (ambient dimension).
|
|
445
|
+
c : float, optional
|
|
446
|
+
Curvature parameter (default: 1.0).
|
|
447
|
+
|
|
448
|
+
Returns
|
|
449
|
+
-------
|
|
450
|
+
y_BAo : Array, shape (B, Ao)
|
|
451
|
+
Output points on the hyperboloid with curvature ``c``.
|
|
452
|
+
Ao = out_features (ambient dimension).
|
|
453
|
+
"""
|
|
454
|
+
# 1. Get effective U (handle weight norm)
|
|
455
|
+
U_IO = self._get_U() # (I, O)
|
|
456
|
+
|
|
457
|
+
# 2. Build V_mink from (U, b) — Minkowski metric absorbed
|
|
458
|
+
V_AiO = build_spacelike_V(U_IO, self.b[...], c, self.eps) # (Ai, O)
|
|
459
|
+
# Cast V to match input dtype (avoids float32/float64 scatter warnings)
|
|
460
|
+
V_AiO = V_AiO.astype(x_BAi.dtype)
|
|
461
|
+
|
|
462
|
+
# 3. Minkowski inner products via matmul (metric in V)
|
|
463
|
+
z_BO = x_BAi @ V_AiO # (B, O)
|
|
464
|
+
|
|
465
|
+
# 4. Apply Euclidean activation (Lorentzian wrapping implicit via cancellation)
|
|
466
|
+
if self.activation is not None:
|
|
467
|
+
z_BO = self.activation(z_BO)
|
|
468
|
+
|
|
469
|
+
# 5. Reconstruct hyperboloid point: spatial = z, time from constraint
|
|
470
|
+
y_0_B1 = jnp.sqrt(jnp.sum(z_BO**2, axis=-1, keepdims=True) + 1.0 / c) # (B, 1)
|
|
471
|
+
|
|
472
|
+
return jnp.concatenate([y_0_B1, z_BO], axis=-1) # (B, Ao)
|
|
@@ -1,12 +1,21 @@
|
|
|
1
|
-
"""Hyperboloid regression layers for JAX/Flax NNX.
|
|
1
|
+
"""Hyperboloid regression layers for JAX/Flax NNX.
|
|
2
|
+
|
|
3
|
+
Dimension key:
|
|
4
|
+
B: batch size
|
|
5
|
+
I: in_spatial (in_features - 1) K: num_classes
|
|
6
|
+
Ai: in_ambient (in_features)
|
|
7
|
+
"""
|
|
2
8
|
|
|
3
9
|
import jax
|
|
10
|
+
import jax.numpy as jnp
|
|
4
11
|
from flax import nnx
|
|
5
12
|
from jaxtyping import Array, Float
|
|
6
13
|
|
|
7
14
|
from hyperbolix.manifolds.hyperboloid import Hyperboloid
|
|
15
|
+
from hyperbolix.utils.math_utils import asinh as safe_asinh
|
|
8
16
|
|
|
9
17
|
from ._helpers import validate_hyperboloid_manifold
|
|
18
|
+
from .hyperboloid_core import build_spacelike_V
|
|
10
19
|
|
|
11
20
|
|
|
12
21
|
class HypRegressionHyperboloid(nnx.Module):
|
|
@@ -110,3 +119,100 @@ class HypRegressionHyperboloid(nnx.Module):
|
|
|
110
119
|
)
|
|
111
120
|
|
|
112
121
|
return res
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class FGGLorentzMLR(nnx.Module):
|
|
125
|
+
"""Fast and Geometrically Grounded Lorentz multinomial logistic regression.
|
|
126
|
+
|
|
127
|
+
Outputs Euclidean logits (signed scaled distances to hyperplanes) using the
|
|
128
|
+
FGG spacelike V construction. Unlike ``HypRegressionHyperboloid``, this layer
|
|
129
|
+
uses the distance-to-hyperplane formulation matching the reference fc_mlr
|
|
130
|
+
(``signed_dist2hyperplanes_scaled_angle``).
|
|
131
|
+
|
|
132
|
+
Forward pass (matching reference fc_mlr):
|
|
133
|
+
1. Build V_mink from (z, a)
|
|
134
|
+
2. mink = x @ V_mink (Minkowski inner products)
|
|
135
|
+
3. logits = asinh(sqrt(c) * mink) / sqrt(c) (signed scaled distances)
|
|
136
|
+
|
|
137
|
+
Parameters
|
|
138
|
+
----------
|
|
139
|
+
in_features : int
|
|
140
|
+
Input ambient dimension (D_in + 1), including time component.
|
|
141
|
+
num_classes : int
|
|
142
|
+
Number of output classes.
|
|
143
|
+
rngs : nnx.Rngs
|
|
144
|
+
Random number generators for parameter initialization.
|
|
145
|
+
reset_params : str, optional
|
|
146
|
+
Weight initialization scheme for hyperplane normals: ``"mlr"``
|
|
147
|
+
(normal, std=sqrt(5/I)) or ``"default"`` (uniform) (default: ``"mlr"``).
|
|
148
|
+
init_bias : float, optional
|
|
149
|
+
Initial value for bias entries (default: 0.5).
|
|
150
|
+
eps : float, optional
|
|
151
|
+
Numerical stability floor (default: 1e-7).
|
|
152
|
+
|
|
153
|
+
References
|
|
154
|
+
----------
|
|
155
|
+
Klis et al. "Fast and Geometrically Grounded Lorentz Neural Networks" (2026), Eq. 23.
|
|
156
|
+
"""
|
|
157
|
+
|
|
158
|
+
def __init__(
|
|
159
|
+
self,
|
|
160
|
+
in_features: int,
|
|
161
|
+
num_classes: int,
|
|
162
|
+
*,
|
|
163
|
+
rngs: nnx.Rngs,
|
|
164
|
+
reset_params: str = "mlr",
|
|
165
|
+
init_bias: float = 0.5,
|
|
166
|
+
eps: float = 1e-7,
|
|
167
|
+
):
|
|
168
|
+
if reset_params not in ("default", "mlr"):
|
|
169
|
+
raise ValueError(f"reset_params must be 'default' or 'mlr', got '{reset_params}'")
|
|
170
|
+
|
|
171
|
+
in_spatial = in_features - 1 # I
|
|
172
|
+
self.in_features = in_features
|
|
173
|
+
self.num_classes = num_classes
|
|
174
|
+
self.eps = eps
|
|
175
|
+
|
|
176
|
+
# Hyperplane normals (spatial) and bias offsets
|
|
177
|
+
key = rngs.params()
|
|
178
|
+
if reset_params == "mlr":
|
|
179
|
+
# Reference: N(0, sqrt(5/I)) for spatial weights
|
|
180
|
+
std = jnp.sqrt(5.0 / in_spatial)
|
|
181
|
+
self.z = nnx.Param(jax.random.normal(key, (in_spatial, num_classes)) * std)
|
|
182
|
+
else: # default
|
|
183
|
+
stdv = 1.0 / jnp.sqrt(jnp.array(in_spatial, dtype=jnp.float32))
|
|
184
|
+
self.z = nnx.Param(jax.random.uniform(key, (in_spatial, num_classes), minval=-stdv, maxval=stdv))
|
|
185
|
+
self.a = nnx.Param(jnp.full((num_classes,), init_bias))
|
|
186
|
+
|
|
187
|
+
def __call__(
|
|
188
|
+
self,
|
|
189
|
+
x_BAi: Float[Array, "batch in_features"],
|
|
190
|
+
c: float = 1.0,
|
|
191
|
+
) -> Float[Array, "batch num_classes"]:
|
|
192
|
+
"""Forward pass returning Euclidean logits.
|
|
193
|
+
|
|
194
|
+
Parameters
|
|
195
|
+
----------
|
|
196
|
+
x_BAi : Array, shape (B, Ai)
|
|
197
|
+
Input points on the hyperboloid with curvature ``c``.
|
|
198
|
+
c : float, optional
|
|
199
|
+
Curvature parameter (default: 1.0).
|
|
200
|
+
|
|
201
|
+
Returns
|
|
202
|
+
-------
|
|
203
|
+
logits_BK : Array, shape (B, K)
|
|
204
|
+
Euclidean logits (signed scaled distances to hyperplanes).
|
|
205
|
+
"""
|
|
206
|
+
# 1. Build V_mink from (z, a)
|
|
207
|
+
V_AiK = build_spacelike_V(self.z[...], self.a[...], c, self.eps) # (Ai, K)
|
|
208
|
+
# Cast V to match input dtype (avoids float32/float64 scatter warnings)
|
|
209
|
+
V_AiK = V_AiK.astype(x_BAi.dtype)
|
|
210
|
+
|
|
211
|
+
# 2. Minkowski inner products
|
|
212
|
+
mink_BK = x_BAi @ V_AiK # (B, K)
|
|
213
|
+
|
|
214
|
+
# 3. Signed scaled distances (matching reference fc_mlr: no norm scaling)
|
|
215
|
+
sqrt_c = jnp.sqrt(c)
|
|
216
|
+
logits_BK = safe_asinh(sqrt_c * mink_BK) / sqrt_c # (B, K)
|
|
217
|
+
|
|
218
|
+
return logits_BK
|
|
@@ -12,12 +12,16 @@ Key components:
|
|
|
12
12
|
- HRCLayerNorm: Hyperbolic layer normalization with curvature change
|
|
13
13
|
- HRCRMSNorm: Hyperbolic RMS normalization with curvature change
|
|
14
14
|
- HRCBatchNorm: Hyperbolic batch normalization with curvature change
|
|
15
|
+
- FGGMeanOnlyBatchNorm: Mean-only batch normalization for FGG-LNN (no variance division)
|
|
15
16
|
|
|
16
17
|
References
|
|
17
18
|
----------
|
|
18
19
|
Hypformer paper (citation to be added)
|
|
20
|
+
Klis et al. "Fast and Geometrically Grounded Lorentz Neural Networks" (2026), Section 4.4.
|
|
21
|
+
Salimans & Kingma "Weight Normalization" (2016).
|
|
19
22
|
"""
|
|
20
23
|
|
|
24
|
+
import jax.numpy as jnp
|
|
21
25
|
from flax import nnx
|
|
22
26
|
from jaxtyping import Array, Float
|
|
23
27
|
|
|
@@ -338,3 +342,95 @@ class HRCBatchNorm(nnx.Module):
|
|
|
338
342
|
return self.bn(z, use_running_average=use_running_average)
|
|
339
343
|
|
|
340
344
|
return hrc(x, bn_fn, c_in, c_out, self.eps)
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
class FGGMeanOnlyBatchNorm(nnx.Module):
|
|
348
|
+
"""Mean-only batch normalization for FGG-LNN layers.
|
|
349
|
+
|
|
350
|
+
Subtracts the batch mean from spatial components without dividing by variance,
|
|
351
|
+
then adds a learnable bias. This avoids the instability that standard BatchNorm
|
|
352
|
+
causes in hyperbolic space (exponentially large/small variance denominators)
|
|
353
|
+
while still centering activations for stable training.
|
|
354
|
+
|
|
355
|
+
Designed to pair with weight normalization (``FGGLinear(use_weight_norm=True)``),
|
|
356
|
+
which handles magnitude control. Together they replace standard BatchNorm in
|
|
357
|
+
fully hyperbolic FGG-LNN networks.
|
|
358
|
+
|
|
359
|
+
Formula (applied to spatial components via HRC):
|
|
360
|
+
``y = x - E[x] + bias``
|
|
361
|
+
|
|
362
|
+
Parameters
|
|
363
|
+
----------
|
|
364
|
+
num_features : int
|
|
365
|
+
Number of spatial features (D, not D+1).
|
|
366
|
+
momentum : float, optional
|
|
367
|
+
Exponential moving average momentum for running mean (default: 0.99).
|
|
368
|
+
Update: ``running_mean = momentum * running_mean + (1 - momentum) * batch_mean``.
|
|
369
|
+
eps : float, optional
|
|
370
|
+
Numerical stability floor for HRC time reconstruction (default: 1e-7).
|
|
371
|
+
|
|
372
|
+
References
|
|
373
|
+
----------
|
|
374
|
+
Klis et al. "Fast and Geometrically Grounded Lorentz Neural Networks" (2026), §4.4.
|
|
375
|
+
Salimans & Kingma "Weight Normalization" (2016).
|
|
376
|
+
"""
|
|
377
|
+
|
|
378
|
+
def __init__(
|
|
379
|
+
self,
|
|
380
|
+
num_features: int,
|
|
381
|
+
*,
|
|
382
|
+
momentum: float = 0.99,
|
|
383
|
+
eps: float = 1e-7,
|
|
384
|
+
):
|
|
385
|
+
self.num_features = num_features
|
|
386
|
+
self.momentum = momentum
|
|
387
|
+
self.eps = eps
|
|
388
|
+
|
|
389
|
+
# Learnable bias (shift after centering)
|
|
390
|
+
self.bias = nnx.Param(jnp.zeros((num_features,)))
|
|
391
|
+
# Running mean for eval mode
|
|
392
|
+
self.running_mean = nnx.BatchStat(jnp.zeros((num_features,)))
|
|
393
|
+
|
|
394
|
+
def __call__(
|
|
395
|
+
self,
|
|
396
|
+
x: Float[Array, "batch dim_plus_1"],
|
|
397
|
+
c_in: float = 1.0,
|
|
398
|
+
c_out: float = 1.0,
|
|
399
|
+
use_running_average: bool = False,
|
|
400
|
+
) -> Float[Array, "batch dim_plus_1"]:
|
|
401
|
+
"""Apply mean-only batch normalization via HRC.
|
|
402
|
+
|
|
403
|
+
Parameters
|
|
404
|
+
----------
|
|
405
|
+
x : Array, shape (B, D+1) or (B, ..., D+1)
|
|
406
|
+
Input points on hyperboloid with curvature ``c_in``.
|
|
407
|
+
c_in : float, optional
|
|
408
|
+
Input curvature (default: 1.0).
|
|
409
|
+
c_out : float, optional
|
|
410
|
+
Output curvature (default: 1.0).
|
|
411
|
+
use_running_average : bool, optional
|
|
412
|
+
If True, use running mean (eval mode). If False, compute from
|
|
413
|
+
batch and update running mean (train mode). Default: False.
|
|
414
|
+
|
|
415
|
+
Returns
|
|
416
|
+
-------
|
|
417
|
+
Array, same shape as input
|
|
418
|
+
Points on hyperboloid with curvature ``c_out``.
|
|
419
|
+
"""
|
|
420
|
+
|
|
421
|
+
def mean_only_bn(z):
|
|
422
|
+
# z: spatial components, shape (..., D)
|
|
423
|
+
if use_running_average:
|
|
424
|
+
mean = self.running_mean[...]
|
|
425
|
+
else:
|
|
426
|
+
# Compute mean over all dims except the last (feature) dim
|
|
427
|
+
# Flatten leading dims for mean computation
|
|
428
|
+
z_flat = z.reshape(-1, z.shape[-1]) # (N, D)
|
|
429
|
+
mean = jnp.mean(z_flat, axis=0) # (D,)
|
|
430
|
+
|
|
431
|
+
# Update running mean (EMA)
|
|
432
|
+
self.running_mean[...] = self.momentum * self.running_mean[...] + (1.0 - self.momentum) * mean
|
|
433
|
+
|
|
434
|
+
return z - mean + self.bias[...]
|
|
435
|
+
|
|
436
|
+
return hrc(x, mean_only_bn, c_in, c_out, self.eps)
|
|
@@ -6,7 +6,7 @@ build-backend = "uv_build"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "hyperbolix"
|
|
9
|
-
version = "0.
|
|
9
|
+
version = "0.3.0"
|
|
10
10
|
description = "Hyperbolic Deep Learning in JAX"
|
|
11
11
|
authors = [
|
|
12
12
|
{ name="Timo Klein", email="timo.klein@univie.ac.at" },
|
|
@@ -43,11 +43,7 @@ dependencies = [
|
|
|
43
43
|
"chex>=0.1.91",
|
|
44
44
|
"flax>=0.12.0",
|
|
45
45
|
"jaxtyping>=0.3.2",
|
|
46
|
-
"numpy>=2.3.3",
|
|
47
46
|
"optax>=0.2.6",
|
|
48
|
-
"scikit-learn>=1.7.2",
|
|
49
|
-
"scipy>=1.16.2",
|
|
50
|
-
"tqdm>=4.67.1",
|
|
51
47
|
]
|
|
52
48
|
|
|
53
49
|
[project.urls]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{hyperbolix-0.2.1 → hyperbolix-0.3.0}/hyperbolix/distributions/wrapped_normal_hyperboloid.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|