rbms 0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rbms/__init__.py ADDED
File without changes
@@ -0,0 +1,3 @@
1
+ # ruff: noqa
2
+ from rbms.bernoulli_bernoulli.classes import BBRBM
3
+ from rbms.bernoulli_bernoulli.functional import *
@@ -0,0 +1,230 @@
1
+ from typing import List, Optional, Self
2
+
3
+ import numpy as np
4
+ import torch
5
+ from torch import Tensor
6
+
7
+ from rbms.bernoulli_bernoulli.implement import (
8
+ _compute_energy,
9
+ _compute_energy_hiddens,
10
+ _compute_energy_visibles,
11
+ _compute_gradient,
12
+ _init_chains,
13
+ _init_parameters,
14
+ _sample_hiddens,
15
+ _sample_visibles,
16
+ )
17
+ from rbms.classes import RBM
18
+
19
+
20
+ class BBRBM(RBM):
21
+ """Parameters of the Bernoulli-Bernoulli RBM"""
22
+
23
+ def __init__(
24
+ self,
25
+ weight_matrix: Tensor,
26
+ vbias: Tensor,
27
+ hbias: Tensor,
28
+ device: Optional[torch.device] = None,
29
+ dtype: Optional[torch.dtype] = None,
30
+ ):
31
+ """Initialize the parameters of the Bernoulli-Bernoulli RBM.
32
+
33
+ Args:
34
+ weight_matrix (Tensor): The weight matrix of the RBM.
35
+ vbias (Tensor): The visible bias of the RBM.
36
+ hbias (Tensor): The hidden bias of the RBM.
37
+ device (Optional[torch.device], optional): The device for the parameters.
38
+ Defaults to the device of `weight_matrix`.
39
+ dtype (Optional[torch.dtype], optional): The data type for the parameters.
40
+ Defaults to the data type of `weight_matrix`.
41
+ """
42
+ if device is None:
43
+ device = weight_matrix.device
44
+ if dtype is None:
45
+ dtype = weight_matrix.dtype
46
+ self.device = device
47
+ self.dtype = dtype
48
+ self.weight_matrix = weight_matrix.to(device=self.device, dtype=self.dtype)
49
+ self.vbias = vbias.to(device=self.device, dtype=self.dtype)
50
+ self.hbias = hbias.to(device=self.device, dtype=self.dtype)
51
+ self.name = "BBRBM"
52
+
53
+ def __add__(self, other):
54
+ return BBRBM(
55
+ weight_matrix=self.weight_matrix + other.weight_matrix,
56
+ vbias=self.vbias + other.vbias,
57
+ hbias=self.hbias + other.hbias,
58
+ )
59
+
60
+ def __mul__(self, other):
61
+ return BBRBM(
62
+ weight_matrix=self.weight_matrix * other,
63
+ vbias=self.vbias * other,
64
+ hbias=self.hbias * other,
65
+ )
66
+
67
+ def clone(
68
+ self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None
69
+ ):
70
+ if device is None:
71
+ device = self.device
72
+ if dtype is None:
73
+ dtype = self.dtype
74
+ return BBRBM(
75
+ weight_matrix=self.weight_matrix.clone(),
76
+ vbias=self.vbias.clone(),
77
+ hbias=self.hbias.clone(),
78
+ device=device,
79
+ dtype=dtype,
80
+ )
81
+
82
+ def compute_energy(self, v: Tensor, h: Tensor) -> Tensor:
83
+ return _compute_energy(
84
+ v=v,
85
+ h=h,
86
+ vbias=self.vbias,
87
+ hbias=self.hbias,
88
+ weight_matrix=self.weight_matrix,
89
+ )
90
+
91
+ def compute_energy_hiddens(self, h: Tensor) -> Tensor:
92
+ return _compute_energy_hiddens(
93
+ h=h,
94
+ vbias=self.vbias,
95
+ hbias=self.hbias,
96
+ weight_matrix=self.weight_matrix,
97
+ )
98
+
99
+ def compute_energy_visibles(self, v: Tensor) -> Tensor:
100
+ return _compute_energy_visibles(
101
+ v=v,
102
+ vbias=self.vbias,
103
+ hbias=self.hbias,
104
+ weight_matrix=self.weight_matrix,
105
+ )
106
+
107
+ def compute_gradient(self, data, chains, centered=True):
108
+ _compute_gradient(
109
+ v_data=data["visible"],
110
+ mh_data=data["hidden_mag"],
111
+ w_data=data["weights"],
112
+ v_chain=chains["visible"],
113
+ h_chain=chains["hidden"],
114
+ w_chain=chains["weights"],
115
+ vbias=self.vbias,
116
+ hbias=self.hbias,
117
+ weight_matrix=self.weight_matrix,
118
+ centered=centered,
119
+ )
120
+
121
+ def independent_model(self):
122
+ return BBRBM(
123
+ weight_matrix=torch.zeros_like(self.weight_matrix),
124
+ vbias=self.vbias,
125
+ hbias=torch.zeros_like(self.hbias),
126
+ )
127
+
128
+ def init_chains(self, num_samples, weights=None, start_v=None):
129
+ visible, hidden, mean_visible, mean_hidden = _init_chains(
130
+ num_samples=num_samples,
131
+ weight_matrix=self.weight_matrix,
132
+ hbias=self.hbias,
133
+ start_v=start_v,
134
+ )
135
+ if weights is None:
136
+ weights = torch.ones(
137
+ visible.shape[0], device=visible.device, dtype=visible.dtype
138
+ )
139
+ return dict(
140
+ visible=visible,
141
+ hidden=hidden,
142
+ visible_mag=mean_visible,
143
+ hidden_mag=mean_hidden,
144
+ weights=weights,
145
+ )
146
+
147
+ @staticmethod
148
+ def init_parameters(num_hiddens, dataset, device, dtype, var_init=0.0001):
149
+ data = dataset.data
150
+ # Convert to torch Tensor if necessary
151
+ if isinstance(data, np.ndarray):
152
+ data = torch.from_numpy(dataset.data).to(device=device, dtype=dtype)
153
+ vbias, hbias, weight_matrix = _init_parameters(
154
+ num_hiddens=num_hiddens,
155
+ data=data,
156
+ device=device,
157
+ dtype=dtype,
158
+ var_init=var_init,
159
+ )
160
+ return BBRBM(weight_matrix=weight_matrix, vbias=vbias, hbias=hbias)
161
+
162
+ def named_parameters(self):
163
+ return {
164
+ "weight_matrix": self.weight_matrix,
165
+ "vbias": self.vbias,
166
+ "hbias": self.hbias,
167
+ }
168
+
169
+ def num_hiddens(self):
170
+ return self.hbias.shape[0]
171
+
172
+ def num_visibles(self):
173
+ return self.vbias.shape[0]
174
+
175
+ def parameters(self) -> List[Tensor]:
176
+ return [self.weight_matrix, self.vbias, self.hbias]
177
+
178
+ def ref_log_z(self):
179
+ return (
180
+ torch.log1p(torch.exp(self.vbias)).sum() + self.num_hiddens() * np.log(2)
181
+ ).item()
182
+
183
+ def sample_hiddens(self, chains: dict[str, Tensor], beta=1) -> dict[str, Tensor]:
184
+ chains["hidden"], chains["hidden_mag"] = _sample_hiddens(
185
+ v=chains["visible"],
186
+ weight_matrix=self.weight_matrix,
187
+ hbias=self.hbias,
188
+ beta=beta,
189
+ )
190
+ return chains
191
+
192
+ def sample_visibles(self, chains: dict[str, Tensor], beta=1) -> dict[str, Tensor]:
193
+ chains["visible"], chains["visible_mag"] = _sample_visibles(
194
+ h=chains["hidden"],
195
+ weight_matrix=self.weight_matrix,
196
+ vbias=self.vbias,
197
+ beta=beta,
198
+ )
199
+ return chains
200
+
201
+ @staticmethod
202
+ def set_named_parameters(named_params: dict[str, Tensor]) -> Self:
203
+ names = ["vbias", "hbias", "weight_matrix"]
204
+ for k in names:
205
+ if k not in named_params.keys():
206
+ raise ValueError(
207
+ f"""Dictionary params missing key '{k}'\n Provided keys : {named_params.keys()}\n Expected keys: {names}"""
208
+ )
209
+ params = BBRBM(
210
+ weight_matrix=named_params.pop("weight_matrix"),
211
+ vbias=named_params.pop("vbias"),
212
+ hbias=named_params.pop("hbias"),
213
+ )
214
+ if len(named_params.keys()) > 0:
215
+ raise ValueError(
216
+ f"Too many keys in params dictionary. Remaining keys: {named_params.keys()}"
217
+ )
218
+ return params
219
+
220
+ def to(
221
+ self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None
222
+ ):
223
+ if device is not None:
224
+ self.device = device
225
+ if dtype is not None:
226
+ self.dtype = dtype
227
+ self.weight_matrix = self.weight_matrix.to(device=self.device, dtype=self.dtype)
228
+ self.vbias = self.vbias.to(device=self.device, dtype=self.dtype)
229
+ self.hbias = self.hbias.to(device=self.device, dtype=self.dtype)
230
+ return self
@@ -0,0 +1,215 @@
1
+ from typing import Optional
2
+
3
+ import numpy as np
4
+ import torch
5
+ from torch import Tensor
6
+
7
+ from rbms.bernoulli_bernoulli.classes import BBRBM
8
+ from rbms.bernoulli_bernoulli.implement import (
9
+ _compute_energy,
10
+ _compute_energy_hiddens,
11
+ _compute_energy_visibles,
12
+ _compute_gradient,
13
+ _init_chains,
14
+ _init_parameters,
15
+ _sample_hiddens,
16
+ _sample_visibles,
17
+ )
18
+ from rbms.dataset.dataset_class import RBMDataset
19
+
20
+
21
+ def sample_hiddens(
22
+ chains: dict[str, Tensor], params: BBRBM, beta: float = 1.0
23
+ ) -> dict[str, Tensor]:
24
+ """Sample the hidden layer conditionally to the visible one.
25
+
26
+ Args:
27
+ chains (dict[str, Tensor]): The parallel chains used for sampling.
28
+ params (BBRBM): The parameters of the RBM.
29
+ beta (float, optional): The inverse temperature. Defaults to 1.0.
30
+
31
+ Returns:
32
+ dict[str, Tensor]: The updated chains with sampled hidden states.
33
+ """
34
+ chains["hidden"], chains["hidden_mag"] = _sample_hiddens(
35
+ v=chains["visible"],
36
+ weight_matrix=params.weight_matrix,
37
+ hbias=params.hbias,
38
+ beta=beta,
39
+ )
40
+ return chains
41
+
42
+
43
+ def sample_visibles(
44
+ chains: dict[str, Tensor], params: BBRBM, beta: float = 1.0
45
+ ) -> dict[str, Tensor]:
46
+ """Sample the visible layer conditionally to the hidden one.
47
+
48
+ Args:
49
+ chains (dict[str, Tensor]): The parallel chains used for sampling.
50
+ params (BBRBM): The parameters of the RBM.
51
+ beta (float, optional): The inverse temperature. Defaults to 1.0.
52
+
53
+ Returns:
54
+ dict[str, Tensor]: The updated chains with sampled visible states.
55
+ """
56
+ chains["visible"], chains["visible_mag"] = _sample_visibles(
57
+ h=chains["hidden"],
58
+ weight_matrix=params.weight_matrix,
59
+ vbias=params.vbias,
60
+ beta=beta,
61
+ )
62
+ return chains
63
+
64
+
65
+ def compute_energy(
66
+ v: Tensor,
67
+ h: Tensor,
68
+ params: BBRBM,
69
+ ) -> Tensor:
70
+ """Compute the energy of the RBM on the visible and hidden variables.
71
+
72
+ Args:
73
+ v (Tensor): Visible configurations.
74
+ h (Tensor): Hidden configurations.
75
+ params (BBRBM): Parameters of the RBM.
76
+
77
+ Returns:
78
+ Tensor: The computed energy.
79
+ """
80
+ return _compute_energy(
81
+ v=v,
82
+ h=h,
83
+ vbias=params.vbias,
84
+ hbias=params.hbias,
85
+ weight_matrix=params.weight_matrix,
86
+ )
87
+
88
+
89
+ def compute_energy_visibles(v: Tensor, params: BBRBM) -> Tensor:
90
+ """Returns the marginalized energy of the model computed on the visible configurations
91
+
92
+ Args:
93
+ v (Tensor): Visible configurations
94
+ params (BBRBM): Parameters of the RBM
95
+
96
+ Returns:
97
+ Tensor: The computed energy.
98
+ """
99
+ return _compute_energy_visibles(
100
+ v=v, vbias=params.vbias, hbias=params.hbias, weight_matrix=params.weight_matrix
101
+ )
102
+
103
+
104
+ def compute_energy_hiddens(h: Tensor, params: BBRBM) -> Tensor:
105
+ """Returns the marginalized energy of the model computed on hidden configurations
106
+
107
+ Args:
108
+ h (Tensor): Hidden configurations.
109
+ params (BBRBM): Parameters of the RBM.
110
+ """
111
+ return _compute_energy_hiddens(
112
+ h=h, vbias=params.vbias, hbias=params.hbias, weight_matrix=params.weight_matrix
113
+ )
114
+
115
+
116
+ def compute_gradient(
117
+ data: dict[str, Tensor],
118
+ chains: dict[str, Tensor],
119
+ params: BBRBM,
120
+ centered: bool = True,
121
+ ) -> None:
122
+ """Compute the gradient for each of the parameters and attach it.
123
+
124
+ Args:
125
+ data (dict[str, Tensor]): The data state.
126
+ chains (dict[str, Tensor]): The parallel chains used for gradient computation.
127
+ params (BBRBM): The parameters of the RBM.
128
+ centered (bool, optional): Whether to use centered gradients. Defaults to True.
129
+ """
130
+ _compute_gradient(
131
+ v_data=data["visible"],
132
+ mh_data=data["hidden_mag"],
133
+ w_data=data["weights"],
134
+ v_chain=chains["visible"],
135
+ h_chain=chains["hidden"],
136
+ w_chain=chains["weights"],
137
+ vbias=params.vbias,
138
+ hbias=params.hbias,
139
+ weight_matrix=params.weight_matrix,
140
+ centered=centered,
141
+ )
142
+
143
+
144
+ def init_chains(
145
+ num_samples: int,
146
+ params: BBRBM,
147
+ weights: Optional[Tensor] = None,
148
+ start_v: Optional[Tensor] = None,
149
+ ) -> dict[str, Tensor]:
150
+ """Initialize a Markov chain for the RBM by sampling a uniform distribution on the visible layer
151
+ and sampling the hidden layer according to the visible one.
152
+
153
+ Args:
154
+ num_samples (int): The number of samples to initialize.
155
+ params (BBRBM): The parameters of the RBM.
156
+ weights (Tensor, optional): The weights of each configuration. Defaults to None.
157
+ start_v (Optional[Tensor], optional): The initial visible states. Defaults to None.
158
+
159
+ Returns:
160
+ dict[str, Tensor]: The initialized Markov chain.
161
+
162
+ Notes:
163
+ - If start_v is specified, its number of samples will override the num_samples argument.
164
+ """
165
+ visible, hidden, mean_visible, mean_hidden = _init_chains(
166
+ num_samples=num_samples,
167
+ weight_matrix=params.weight_matrix,
168
+ hbias=params.hbias,
169
+ start_v=start_v,
170
+ )
171
+ if weights is None:
172
+ weights = torch.ones(visible.shape[0], device=visible.device, dtype=visible.dtype)
173
+ return dict(
174
+ visible=visible,
175
+ hidden=hidden,
176
+ visible_mag=mean_visible,
177
+ hidden_mag=mean_hidden,
178
+ weights=weights,
179
+ )
180
+
181
+
182
+ def init_parameters(
183
+ num_hiddens: int,
184
+ dataset: RBMDataset,
185
+ device: torch.device,
186
+ dtype: torch.dtype,
187
+ var_init: float = 1e-4,
188
+ ) -> BBRBM:
189
+ """Initialize the parameters of the RBM.
190
+
191
+ Args:
192
+ num_hiddens (int): Number of hidden units.
193
+ dataset (RBMDataset): Training dataset.
194
+ device (torch.device): PyTorch device for the parameters.
195
+ dtype (torch.dtype): PyTorch dtype for the parameters.
196
+ var_init (float, optional): Variance of the weight matrix. Defaults to 1e-4.
197
+
198
+ Notes:
199
+ - The number of visible units is induced from the dataset provided.
200
+ - Hidden biases are set to 0.
201
+ - Visible biases are set to the frequencies of the dataset.
202
+ - The weight matrix is initialized with a Gaussian distribution of variance `var_init`.
203
+ """
204
+ data = dataset.data
205
+ # Convert to torch Tensor if necessary
206
+ if isinstance(data, np.ndarray):
207
+ data = torch.from_numpy(dataset.data).to(device=device, dtype=dtype)
208
+ vbias, hbias, weight_matrix = _init_parameters(
209
+ num_hiddens=num_hiddens,
210
+ data=data,
211
+ device=device,
212
+ dtype=dtype,
213
+ var_init=var_init,
214
+ )
215
+ return BBRBM(weight_matrix=weight_matrix, vbias=vbias, hbias=hbias)
@@ -0,0 +1,175 @@
1
+ from typing import Optional, Tuple
2
+
3
+ import torch
4
+ from torch import Tensor
5
+ from torch.nn.functional import softmax
6
+
7
+
8
+ @torch.jit.script
9
+ def _sample_hiddens(
10
+ v: Tensor, weight_matrix: Tensor, hbias: Tensor, beta: float = 1.0
11
+ ) -> Tuple[Tensor, Tensor]:
12
+ mh = torch.sigmoid(beta * (hbias + (v @ weight_matrix)))
13
+ h = torch.bernoulli(mh)
14
+ return h, mh
15
+
16
+
17
+ @torch.jit.script
18
+ def _sample_visibles(
19
+ h: Tensor, weight_matrix: Tensor, vbias: Tensor, beta: float = 1.0
20
+ ) -> Tuple[Tensor, Tensor]:
21
+ mv = torch.sigmoid(beta * (vbias + (h @ weight_matrix.T)))
22
+ v = torch.bernoulli(mv)
23
+ return v, mv
24
+
25
+
26
+ @torch.jit.script
27
+ def _compute_energy(
28
+ v: Tensor,
29
+ h: Tensor,
30
+ vbias: Tensor,
31
+ hbias: Tensor,
32
+ weight_matrix: Tensor,
33
+ ) -> Tensor:
34
+ fields = torch.tensordot(vbias, v, dims=[[0], [1]]) + torch.tensordot(
35
+ hbias, h, dims=[[0], [1]]
36
+ )
37
+ interaction = torch.multiply(
38
+ v, torch.tensordot(h, weight_matrix, dims=[[1], [1]])
39
+ ).sum(1)
40
+
41
+ return -fields - interaction
42
+
43
+
44
+ @torch.jit.script
45
+ def _compute_energy_visibles(
46
+ v: Tensor, vbias: Tensor, hbias: Tensor, weight_matrix: Tensor
47
+ ) -> Tensor:
48
+ field = v @ vbias
49
+ exponent = hbias + (v @ weight_matrix)
50
+ log_term = torch.where(exponent < 10, torch.log(1.0 + torch.exp(exponent)), exponent)
51
+ return -field - log_term.sum(1)
52
+
53
+
54
+ @torch.jit.script
55
+ def _compute_energy_hiddens(
56
+ h: Tensor, vbias: Tensor, hbias: Tensor, weight_matrix: Tensor
57
+ ) -> Tensor:
58
+ field = h @ hbias
59
+ exponent = vbias + (h @ weight_matrix.T)
60
+ log_term = torch.where(exponent < 10, torch.log(1.0 + torch.exp(exponent)), exponent)
61
+ return -field - log_term.sum(1)
62
+
63
+
64
+ @torch.jit.script
65
+ def _compute_gradient(
66
+ v_data: Tensor,
67
+ mh_data: Tensor,
68
+ w_data: Tensor,
69
+ v_chain: Tensor,
70
+ h_chain: Tensor,
71
+ w_chain: Tensor,
72
+ vbias: Tensor,
73
+ hbias: Tensor,
74
+ weight_matrix: Tensor,
75
+ centered: bool = True,
76
+ ) -> None:
77
+ w_data = w_data.view(-1, 1)
78
+ w_chain = w_chain.view(-1, 1)
79
+ # Turn the weights of the chains into normalized weights
80
+ chain_weights = softmax(-w_chain, dim=0)
81
+ w_data_norm = w_data.sum()
82
+
83
+ # Averages over data and generated samples
84
+ v_data_mean = (v_data * w_data).sum(0) / w_data_norm
85
+ torch.clamp_(v_data_mean, min=1e-7, max=(1.0 - 1e-7))
86
+ h_data_mean = (mh_data * w_data).sum(0) / w_data_norm
87
+ v_gen_mean = (v_chain * chain_weights).sum(0)
88
+ torch.clamp_(v_gen_mean, min=1e-7, max=(1.0 - 1e-7))
89
+ h_gen_mean = (h_chain * chain_weights).sum(0)
90
+
91
+ if centered:
92
+ # Centered variables
93
+ v_data_centered = v_data - v_data_mean
94
+ h_data_centered = mh_data - h_data_mean
95
+ v_gen_centered = v_chain - v_data_mean
96
+ h_gen_centered = h_chain - h_data_mean
97
+
98
+ # Gradient
99
+ grad_weight_matrix = (
100
+ (v_data_centered * w_data).T @ h_data_centered
101
+ ) / w_data_norm - ((v_gen_centered * chain_weights).T @ h_gen_centered)
102
+ grad_vbias = v_data_mean - v_gen_mean - (grad_weight_matrix @ h_data_mean)
103
+ grad_hbias = h_data_mean - h_gen_mean - (v_data_mean @ grad_weight_matrix)
104
+ else:
105
+ v_data_centered = v_data
106
+ h_data_centered = mh_data
107
+ v_gen_centered = v_chain
108
+ h_gen_centered = h_chain
109
+
110
+ # Gradient
111
+ grad_weight_matrix = ((v_data * w_data).T @ mh_data) / w_data_norm - (
112
+ (v_chain * chain_weights).T @ h_chain
113
+ )
114
+ grad_vbias = v_data_mean - v_gen_mean
115
+ grad_hbias = h_data_mean - h_gen_mean
116
+
117
+ # Attach to the parameters
118
+
119
+ weight_matrix.grad.set_(grad_weight_matrix)
120
+ vbias.grad.set_(grad_vbias)
121
+ hbias.grad.set_(grad_hbias)
122
+
123
+
124
+ @torch.jit.script
125
+ def _init_chains(
126
+ num_samples: int,
127
+ weight_matrix: Tensor,
128
+ hbias: Tensor,
129
+ start_v: Optional[Tensor] = None,
130
+ ):
131
+ num_visibles, _ = weight_matrix.shape
132
+ device = weight_matrix.device
133
+ dtype = weight_matrix.dtype
134
+ # Handle negative number of samples
135
+ if num_samples <= 0:
136
+ if start_v is not None:
137
+ num_samples = start_v.shape[0]
138
+ else:
139
+ raise ValueError(f"Got negative num_samples arg: {num_samples}")
140
+
141
+ if start_v is None:
142
+ # Dummy mean visible
143
+ mv = torch.ones(size=(num_samples, num_visibles), device=device, dtype=dtype) / 2
144
+ v = torch.bernoulli(mv)
145
+ else:
146
+ # Dummy mean visible
147
+ mv = torch.ones_like(start_v, device=device, dtype=dtype) / 2
148
+ v = start_v.to(device=device, dtype=dtype)
149
+
150
+ # Initialize chains
151
+
152
+ h, mh = _sample_hiddens(v=v, weight_matrix=weight_matrix, hbias=hbias)
153
+ return v, h, mv, mh
154
+
155
+
156
+ def _init_parameters(
157
+ num_hiddens: int,
158
+ data: Tensor,
159
+ device: torch.device,
160
+ dtype: torch.dtype,
161
+ var_init: float = 1e-4,
162
+ ) -> Tuple[Tensor, Tensor, Tensor]:
163
+ _, num_visibles = data.shape
164
+ eps = 1e-4
165
+ weight_matrix = (
166
+ torch.randn(size=(num_visibles, num_hiddens), device=device, dtype=dtype)
167
+ * var_init
168
+ )
169
+ frequencies = data.mean(0)
170
+ frequencies = torch.clamp(frequencies, min=eps, max=(1.0 - eps))
171
+ vbias = (torch.log(frequencies) - torch.log(1.0 - frequencies)).to(
172
+ device=device, dtype=dtype
173
+ )
174
+ hbias = torch.zeros(num_hiddens, device=device, dtype=dtype)
175
+ return vbias, hbias, weight_matrix