robusta-hmf 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,11 @@
1
+ # __init__.py
2
+
3
+ from .convergence import ConvergenceTester
4
+ from .initialisation import Initialiser
5
+ from .main import Robusta
6
+
7
+ __all__ = [
8
+ "ConvergenceTester",
9
+ "Initialiser",
10
+ "Robusta",
11
+ ]
@@ -0,0 +1,34 @@
1
+ # file generated by setuptools-scm
2
+ # don't change, don't track in version control
3
+
4
+ __all__ = [
5
+ "__version__",
6
+ "__version_tuple__",
7
+ "version",
8
+ "version_tuple",
9
+ "__commit_id__",
10
+ "commit_id",
11
+ ]
12
+
13
+ TYPE_CHECKING = False
14
+ if TYPE_CHECKING:
15
+ from typing import Tuple
16
+ from typing import Union
17
+
18
+ VERSION_TUPLE = Tuple[Union[int, str], ...]
19
+ COMMIT_ID = Union[str, None]
20
+ else:
21
+ VERSION_TUPLE = object
22
+ COMMIT_ID = object
23
+
24
+ version: str
25
+ __version__: str
26
+ __version_tuple__: VERSION_TUPLE
27
+ version_tuple: VERSION_TUPLE
28
+ commit_id: COMMIT_ID
29
+ __commit_id__: COMMIT_ID
30
+
31
+ __version__ = version = '0.0.1'
32
+ __version_tuple__ = version_tuple = (0, 0, 1)
33
+
34
+ __commit_id__ = commit_id = None
robusta_hmf/als.py ADDED
@@ -0,0 +1,45 @@
1
+ # als.py
2
+
3
+ import equinox as eqx
4
+ import jax
5
+ import jax.numpy as jnp
6
+
7
+ from .state import RHMFState, update_state
8
+
9
+
10
+ class WeightedAStep(eqx.Module):
11
+ ridge: float | None = eqx.field(static=True, default=None)
12
+
13
+ def __call__(self, Y, W, state: RHMFState):
14
+ G = state.G
15
+
16
+ def solve_row(y_i, w_i):
17
+ s = jnp.sqrt(w_i)
18
+ Gw = G * s[:, None]
19
+ M = Gw.T @ Gw
20
+ if self.ridge is not None:
21
+ M = M + self.ridge * jnp.eye(G.shape[1], dtype=G.dtype)
22
+ b = Gw.T @ (s * y_i)
23
+ return jnp.linalg.solve(M, b)
24
+
25
+ A_new = jax.vmap(solve_row)(Y, W) # [N, K]
26
+ return update_state(state, A=A_new)
27
+
28
+
29
+ class WeightedGStep(eqx.Module):
30
+ ridge: float | None = eqx.field(static=True, default=None)
31
+
32
+ def __call__(self, Y, W, state: RHMFState):
33
+ A = state.A
34
+
35
+ def solve_col(y_j, w_j):
36
+ s = jnp.sqrt(w_j)
37
+ Aw = A * s[:, None]
38
+ M = Aw.T @ Aw
39
+ if self.ridge is not None:
40
+ M = M + self.ridge * jnp.eye(A.shape[1], dtype=A.dtype)
41
+ b = Aw.T @ (s * y_j)
42
+ return jnp.linalg.solve(M, b)
43
+
44
+ G_new = jax.vmap(solve_col)(Y.T, W.T) # [D, K]
45
+ return update_state(state, G=G_new)
@@ -0,0 +1,64 @@
1
+ # convergence.py
2
+
3
+ from dataclasses import dataclass, field
4
+ from typing import Literal, get_args
5
+
6
+ import jax
7
+ import jax.numpy as jnp
8
+ from jaxtyping import Array
9
+
10
+ from .state import RHMFState
11
+
12
+ jax.config.update("jax_enable_x64", True)
13
+
14
+ ConvStrategy = Literal["max_frac_G", "max_frac_A", "rel_frac_loss", "none"]
15
+
16
+ DEFAULT_STRATEGY = "max_frac_G"
17
+ DEFAULT_TOL = 1e-6
18
+
19
+
20
+ def max_frac_mat(old_mat: Array, new_mat: Array, tol: float) -> bool:
21
+ d_mat = old_mat - new_mat
22
+ if jnp.max(d_mat * d_mat) / jnp.mean(new_mat * new_mat) < tol:
23
+ return True
24
+ else:
25
+ return False
26
+
27
+
28
+ def rel_frac_loss(old_loss: float, new_loss: float, tol: float) -> bool:
29
+ if jnp.abs(old_loss - new_loss) / jnp.abs(new_loss) < tol:
30
+ return True
31
+ else:
32
+ return False
33
+
34
+
35
+ def never_converged(*args, **kwargs) -> bool:
36
+ return False
37
+
38
+
39
+ @dataclass(frozen=True)
40
+ class ConvergenceTester:
41
+ strategy: ConvStrategy = field(default=DEFAULT_STRATEGY)
42
+ tol: float = field(default=DEFAULT_TOL)
43
+
44
+ def __post_init__(self):
45
+ if self.strategy not in get_args(ConvStrategy):
46
+ raise ValueError(
47
+ f"Invalid strategy: {self.strategy}. Must be one of {get_args(ConvStrategy)}."
48
+ )
49
+
50
+ def is_converged(
51
+ self,
52
+ old_state: RHMFState,
53
+ new_state: RHMFState,
54
+ old_loss: float,
55
+ new_loss: float,
56
+ ) -> bool:
57
+ if self.strategy == "max_frac_G":
58
+ return max_frac_mat(old_state.G, new_state.G, tol=self.tol)
59
+ elif self.strategy == "max_frac_A":
60
+ return max_frac_mat(old_state.A, new_state.A, tol=self.tol)
61
+ elif self.strategy == "rel_frac_loss":
62
+ return rel_frac_loss(old_loss, new_loss, tol=self.tol)
63
+ elif self.strategy == "none":
64
+ return never_converged()
robusta_hmf/frame.py ADDED
@@ -0,0 +1,59 @@
1
+ # frame.py
2
+
3
+ from copy import deepcopy
4
+ from dataclasses import dataclass
5
+
6
+ import jax.numpy as jnp
7
+ from jaxtyping import Array
8
+
9
+ from .convergence import ConvergenceTester
10
+ from .hmf import HMF
11
+ from .state import RHMFState
12
+
13
+
14
+ @dataclass
15
+ class OptFrame:
16
+ method: HMF
17
+ conv_tester: ConvergenceTester
18
+
19
+ def run(
20
+ self,
21
+ Y: Array,
22
+ W: Array,
23
+ init_state: RHMFState,
24
+ rotation_cadence: int,
25
+ conv_check_cadence: int,
26
+ max_iter: int,
27
+ ):
28
+ # Get the step function from the method
29
+ step_fn = self.method.get_stepper()
30
+
31
+ # Always rotate for als
32
+ if self.method.opt_method == "als":
33
+ rotation_cadence = 1
34
+
35
+ # To store the loss history
36
+ loss_history = []
37
+
38
+ # Initialise states and loss
39
+ state = init_state
40
+ prev_state = deepcopy(init_state)
41
+ prev_loss = jnp.inf
42
+
43
+ # Training loop
44
+ for i in range(max_iter):
45
+ # Do we rotate this iteration?
46
+ rot = True if (i % rotation_cadence == 0 and i != 0) else False
47
+ # Take an optimization step and record the loss
48
+ state, loss = step_fn(Y=Y, W_data=W, state=state, rotate=rot)
49
+ loss_history.append(loss)
50
+ # Check convergence and print loss every conv_check_cadence iterations
51
+ if i % conv_check_cadence == 0 and i != 0:
52
+ if self.conv_tester.is_converged(prev_state, state, prev_loss, loss):
53
+ print(f"Converged at iteration {i}")
54
+ break
55
+ prev_state = deepcopy(state)
56
+ prev_loss = loss
57
+ print(f"iter {state.it:03d} | loss {loss:.4f}", flush=True)
58
+
59
+ return state, jnp.array(loss_history)
robusta_hmf/hmf.py ADDED
@@ -0,0 +1,173 @@
1
+ # hmf.py
2
+
3
+ from typing import Literal, TypeAlias
4
+
5
+ import equinox as eqx
6
+ import optax
7
+ from jaxtyping import Array
8
+
9
+ from .als import WeightedAStep, WeightedGStep
10
+ from .likelihoods import GaussianLikelihood, Likelihood, StudentTLikelihood
11
+ from .rotations import Rotation, RotationMethod, get_rotation_cls
12
+ from .state import RHMFState, refresh_opt_state, update_state
13
+
14
+ OptMethod: TypeAlias = Literal["sgd", "als"]
15
+
16
+
17
+ class HMF(eqx.Module):
18
+ # Common fields
19
+ likelihood: Likelihood = eqx.field(static=True)
20
+ rotation: Rotation = eqx.field(static=True)
21
+ opt_method: OptMethod = eqx.field(static=True)
22
+
23
+ # ALS-specific fields (None if using SGD)
24
+ a_step: WeightedAStep | None
25
+ g_step: WeightedGStep | None
26
+
27
+ # SGD-specific fields (None if using ALS)
28
+ opt: optax.GradientTransformation | None = eqx.field(static=True, default=None)
29
+
30
+ def __init__(
31
+ self,
32
+ method: OptMethod = "als",
33
+ robust: bool = True,
34
+ robust_scale: float = 1.0,
35
+ robust_nu: float = 1.0,
36
+ als_ridge: float | None = None,
37
+ learning_rate: float = 1e-3,
38
+ custom_opt: optax.GradientTransformation | None = None,
39
+ rotation: RotationMethod = "fast",
40
+ **rotation_kwargs,
41
+ ):
42
+ """
43
+ Unified Heteroscedastic Matrix Factorization class. Not really meant for users but live your best life I guess.
44
+
45
+ Args:
46
+ method: Optimization method, either "als" or "sgd"
47
+ robust: If True, use Student-t likelihood; if False, use Gaussian
48
+ robust_scale: Scale parameter for Student-t likelihood (only used if robust=True)
49
+ robust_nu: Degrees of freedom for Student-t likelihood (only used if robust=True); set to 1 for Cauchy distribution aka Hogg's version
50
+ als_ridge: Ridge parameter for ALS updates (only used if method="als")
51
+ learning_rate: Learning rate for SGD (only used if method="sgd" and custom_opt is None)
52
+ rotation: Rotation method to use
53
+ custom_opt: Custom optimizer for SGD (only used if method="sgd")
54
+ **rotation_kwargs: Additional arguments for rotation method
55
+ """
56
+ # Configure likelihood based on robust flag
57
+ if robust:
58
+ self.likelihood = StudentTLikelihood(nu=robust_nu, scale=robust_scale)
59
+ else:
60
+ self.likelihood = GaussianLikelihood()
61
+
62
+ # Set optimization method
63
+ self.opt_method = method
64
+
65
+ # Configure rotation
66
+ self.rotation = get_rotation_cls(method=rotation)(**rotation_kwargs)
67
+
68
+ # Configure method-specific components
69
+ if method == "als":
70
+ self.a_step = WeightedAStep(ridge=als_ridge)
71
+ self.g_step = WeightedGStep(ridge=als_ridge)
72
+ self.opt = None
73
+ elif method == "sgd":
74
+ self.a_step = None
75
+ self.g_step = None
76
+ if custom_opt is not None:
77
+ self.opt = custom_opt
78
+ else:
79
+ self.opt = optax.adafactor(
80
+ factored=True,
81
+ decay_rate=0.9,
82
+ learning_rate=learning_rate,
83
+ )
84
+ else:
85
+ raise ValueError(f"Unknown method: {method}. Must be 'als' or 'sgd'.")
86
+
87
+ def __check_init__(self):
88
+ """Validate parameters."""
89
+ if isinstance(self.likelihood, StudentTLikelihood):
90
+ if self.likelihood.scale <= 0:
91
+ raise ValueError(
92
+ f"robust_scale must be positive, got {self.likelihood.scale}"
93
+ )
94
+ if self.likelihood.nu <= 0:
95
+ raise ValueError(
96
+ f"robust_nu must be positive, got {self.likelihood.nu}"
97
+ )
98
+
99
+ if self.opt_method == "als":
100
+ if self.a_step.ridge is not None and self.a_step.ridge < 0:
101
+ raise ValueError(
102
+ f"als_ridge must be non-negative or None, got {self.a_step.ridge}"
103
+ )
104
+
105
+ @eqx.filter_jit
106
+ def step_als(
107
+ self,
108
+ Y: Array,
109
+ W_data: Array,
110
+ state: RHMFState,
111
+ rotate: bool = True,
112
+ ) -> tuple[RHMFState, float]:
113
+ """Perform one ALS optimization step. Not intended to be called by user."""
114
+ # W step
115
+ W = self.likelihood.weights_total(Y, W_data, state.A, state.G)
116
+ # ALS steps
117
+ state = self.a_step(Y, W, state)
118
+ state = self.g_step(Y, W, state)
119
+ # Optional rotation step (Actually should never be skipped for ALS, but this is handled in OptFrame, and kept here for interface consistency)
120
+ if rotate:
121
+ state = self.rotation(state)
122
+ # Compute loss, update states
123
+ loss = self.likelihood.loss(Y, W_data, state.A, state.G)
124
+ state = eqx.tree_at(lambda s: s.it, state, state.it + 1)
125
+ return state, loss
126
+
127
+ @eqx.filter_jit
128
+ def step_sgd(
129
+ self,
130
+ Y: Array,
131
+ W_data: Array,
132
+ state: RHMFState,
133
+ rotate: bool = True,
134
+ ) -> tuple[RHMFState, float]:
135
+ """Perform one SGD optimization step. Not intended to be called by user."""
136
+
137
+ # Define loss function
138
+ def loss_fn(params, Y):
139
+ A, G = params
140
+ return self.likelihood.loss(Y, W_data, A, G)
141
+
142
+ # Perform SGD step
143
+ params = (state.A, state.G)
144
+ loss, grads = eqx.filter_value_and_grad(loss_fn)(params, Y)
145
+ updates, opt_state = self.opt.update(grads, state.opt_state, params)
146
+ A_new, G_new = optax.apply_updates(params, updates)
147
+
148
+ # Apply updates and optionally rotate which also re-initialises optimiser state
149
+ if rotate:
150
+ state = update_state(state, A=A_new, G=G_new)
151
+ state = self.rotation(state) # rotates A/G
152
+ state = refresh_opt_state(state, self.opt) # refresh
153
+ # Recalculate loss after rotation
154
+ loss = self.likelihood.loss(Y, W_data, state.A, state.G)
155
+ else:
156
+ state = update_state(
157
+ state,
158
+ A=A_new,
159
+ G=G_new,
160
+ opt_state=opt_state,
161
+ )
162
+
163
+ state = update_state(state, it=state.it + 1)
164
+ return state, loss
165
+
166
+ def get_stepper(self):
167
+ """Return the appropriate step function based on opt_method. Not intended to be called by user."""
168
+ if self.opt_method == "als":
169
+ return self.step_als
170
+ elif self.opt_method == "sgd":
171
+ return self.step_sgd
172
+ else:
173
+ raise ValueError(f"Unknown opt_method: {self.opt_method}")
@@ -0,0 +1,91 @@
1
+ # initialisation.py
2
+
3
+ from dataclasses import dataclass
4
+ from typing import Literal, TypeAlias, get_args
5
+
6
+ import jax
7
+ import jax.numpy as jnp
8
+ import optax
9
+ from jaxtyping import Array
10
+
11
+ from .state import RHMFState
12
+
13
+ InitStrategy: TypeAlias = Literal["random", "svd", "custom"]
14
+
15
+
16
+ def random_init(seed: int, N: int, M: int, K: int) -> tuple[Array]:
17
+ if seed is None:
18
+ raise ValueError("Must provide seed for random initialisation.")
19
+ key = jax.random.key(seed)
20
+ k1, k2 = jax.random.split(key)
21
+ A = jax.random.normal(k1, (N, K))
22
+ G = jax.random.normal(k2, (M, K))
23
+ return A, G
24
+
25
+
26
+ def svd_init(Y: Array, N: int, M: int, K: int) -> tuple[Array]:
27
+ if Y is None:
28
+ raise ValueError("Must provide Y for svd initialisation.")
29
+ if Y.shape != (N, M):
30
+ raise ValueError(
31
+ f"Shape of provided Y: {Y.shape} does not match (N, M) = ({N}, {M})"
32
+ )
33
+ if K > min(N, M):
34
+ raise ValueError(f"K={K} exceeds min(N, M)={min(N, M)} in SVD initialisation.")
35
+ U, s, Vh = jnp.linalg.svd(Y, full_matrices=False)
36
+ s = jnp.sqrt(s[:K])
37
+ return U[:, :K] * s, Vh[:K, :].T * s
38
+
39
+
40
+ def custom_init(A: Array, G: Array, N: int, M: int, K: int) -> tuple[Array]:
41
+ if A is None or G is None:
42
+ raise ValueError("Must provide A and G for custom initialisation.")
43
+ if A.shape[0] != N or A.shape[1] != K:
44
+ raise ValueError(
45
+ f"Shape of provided A: {A.shape} does not match (N, K) = ({N}, {K})"
46
+ )
47
+ if G.shape[0] != M or G.shape[1] != K:
48
+ raise ValueError(
49
+ f"Shape of provided G: {G.shape} does not match (M, K) = ({M}, {K})"
50
+ )
51
+ return A, G
52
+
53
+
54
+ @dataclass(frozen=True)
55
+ class Initialiser:
56
+ N: int
57
+ M: int
58
+ K: int
59
+ strategy: InitStrategy
60
+
61
+ def __post_init__(self):
62
+ if self.strategy not in get_args(InitStrategy):
63
+ raise ValueError(
64
+ f"Invalid strategy: {self.strategy}. Must be one of {get_args(InitStrategy)}."
65
+ )
66
+
67
+ def execute(
68
+ self,
69
+ seed: int | None = None,
70
+ A: Array | None = None,
71
+ G: Array | None = None,
72
+ Y: Array | None = None,
73
+ opt: optax.GradientTransformation
74
+ | tuple[optax.GradientTransformation, optax.GradientTransformation]
75
+ | None = None,
76
+ ) -> RHMFState:
77
+ # Initialise A and G according to strategy
78
+ if self.strategy == "random":
79
+ A, G = random_init(seed, self.N, self.M, self.K)
80
+ elif self.strategy == "svd":
81
+ A, G = svd_init(Y, self.N, self.M, self.K)
82
+ elif self.strategy == "custom":
83
+ A, G = custom_init(A, G, self.N, self.M, self.K)
84
+
85
+ # Initialise the optax state
86
+ if opt is None:
87
+ opt_state = None
88
+ else:
89
+ opt_state = opt.init((A, G))
90
+
91
+ return RHMFState(A=A, G=G, it=0, opt_state=opt_state)
@@ -0,0 +1,63 @@
1
+ # likelihoods.py
2
+
3
+ import abc
4
+
5
+ import equinox as eqx
6
+ import jax.numpy as jnp
7
+
8
+
9
+ class Likelihood(eqx.Module):
10
+ """IRLS interface:
11
+ - weights_irls(Y,A,G,W_data): latent robust factor (same shape as Y)
12
+ - weights_total(Y,A,G,W_data): W_data * weights_irls
13
+ - loss(Y,A,G,W_data): robust loss (scalar)
14
+ """
15
+
16
+ @abc.abstractmethod
17
+ def weights_irls(self, Y, W_data, A, G): ...
18
+
19
+ def weights_total(self, Y, W_data, A, G):
20
+ return W_data * self.weights_irls(Y, W_data, A, G)
21
+
22
+ @abc.abstractmethod
23
+ def loss(self, Y, W_data, A, G): ...
24
+
25
+
26
+ class GaussianLikelihood(Likelihood):
27
+ def weights_irls(self, Y, W_data, A, G):
28
+ return jnp.ones_like(Y)
29
+
30
+ def loss(self, Y, W_data, A, G):
31
+ r2 = (Y - A @ G.T) ** 2
32
+ return jnp.sum(W_data * r2)
33
+
34
+
35
+ class CauchyLikelihood(Likelihood):
36
+ scale: float = 1.0
37
+
38
+ def weights_irls(self, Y, W_data, A, G):
39
+ r2 = (Y - A @ G.T) ** 2
40
+ s2 = jnp.asarray(self.scale, dtype=Y.dtype) ** 2
41
+ return s2 / (s2 + W_data * r2)
42
+
43
+ def loss(self, Y, W_data, A, G):
44
+ r2 = (Y - A @ G.T) ** 2
45
+ s2 = jnp.asarray(self.scale, dtype=Y.dtype) ** 2
46
+ return s2 * jnp.sum(jnp.log1p((W_data * r2) / s2))
47
+
48
+
49
+ class StudentTLikelihood(Likelihood):
50
+ nu: float = 3.0
51
+ scale: float = 1.0
52
+
53
+ def weights_irls(self, Y, W_data, A, G):
54
+ r2 = (Y - A @ G.T) ** 2
55
+ s2 = jnp.asarray(self.scale, dtype=Y.dtype) ** 2
56
+ nu = jnp.asarray(self.nu, dtype=Y.dtype)
57
+ return (nu * s2) / (nu * s2 + W_data * r2)
58
+
59
+ def loss(self, Y, W_data, A, G):
60
+ r2 = (Y - A @ G.T) ** 2
61
+ s2 = jnp.asarray(self.scale, dtype=Y.dtype) ** 2
62
+ nu = jnp.asarray(self.nu, dtype=Y.dtype)
63
+ return (nu * s2) * jnp.sum(jnp.log1p((W_data * r2) / (nu * s2)))