rbms 0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rbms/__init__.py +0 -0
- rbms/bernoulli_bernoulli/__init__.py +3 -0
- rbms/bernoulli_bernoulli/classes.py +230 -0
- rbms/bernoulli_bernoulli/functional.py +215 -0
- rbms/bernoulli_bernoulli/implement.py +175 -0
- rbms/classes.py +216 -0
- rbms/const.py +4 -0
- rbms/custom_fn.py +41 -0
- rbms/dataset/__init__.py +99 -0
- rbms/dataset/dataset_class.py +124 -0
- rbms/dataset/fasta_utils.py +171 -0
- rbms/dataset/load_fasta.py +63 -0
- rbms/dataset/load_h5.py +53 -0
- rbms/dataset/parser.py +54 -0
- rbms/dataset/utils.py +18 -0
- rbms/io.py +134 -0
- rbms/map_model.py +5 -0
- rbms/metrics/__init__.py +0 -0
- rbms/metrics/aats.py +59 -0
- rbms/parser.py +147 -0
- rbms/partition_function/ais.py +84 -0
- rbms/partition_function/exact.py +27 -0
- rbms/plot.py +185 -0
- rbms/potts_bernoulli/__init__.py +0 -0
- rbms/potts_bernoulli/classes.py +230 -0
- rbms/potts_bernoulli/functional.py +215 -0
- rbms/potts_bernoulli/implement.py +219 -0
- rbms/potts_bernoulli/tools.py +55 -0
- rbms/potts_bernoulli/utils.py +10 -0
- rbms/sampling/gibbs.py +23 -0
- rbms/sampling/pt.py +186 -0
- rbms/scripts/__init__.py +0 -0
- rbms/scripts/entrypoint.py +36 -0
- rbms/scripts/pt_sampling.py +112 -0
- rbms/scripts/train_rbm.py +73 -0
- rbms/training/pcd.py +149 -0
- rbms/training/utils.py +159 -0
- rbms/utils.py +300 -0
- rbms-0.1.dist-info/METADATA +88 -0
- rbms-0.1.dist-info/RECORD +43 -0
- rbms-0.1.dist-info/WHEEL +4 -0
- rbms-0.1.dist-info/entry_points.txt +2 -0
- rbms-0.1.dist-info/licenses/LICENSE +21 -0
rbms/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
from typing import List, Optional, Self
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import torch
|
|
5
|
+
from torch import Tensor
|
|
6
|
+
|
|
7
|
+
from rbms.bernoulli_bernoulli.implement import (
|
|
8
|
+
_compute_energy,
|
|
9
|
+
_compute_energy_hiddens,
|
|
10
|
+
_compute_energy_visibles,
|
|
11
|
+
_compute_gradient,
|
|
12
|
+
_init_chains,
|
|
13
|
+
_init_parameters,
|
|
14
|
+
_sample_hiddens,
|
|
15
|
+
_sample_visibles,
|
|
16
|
+
)
|
|
17
|
+
from rbms.classes import RBM
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class BBRBM(RBM):
|
|
21
|
+
"""Parameters of the Bernoulli-Bernoulli RBM"""
|
|
22
|
+
|
|
23
|
+
def __init__(
|
|
24
|
+
self,
|
|
25
|
+
weight_matrix: Tensor,
|
|
26
|
+
vbias: Tensor,
|
|
27
|
+
hbias: Tensor,
|
|
28
|
+
device: Optional[torch.device] = None,
|
|
29
|
+
dtype: Optional[torch.dtype] = None,
|
|
30
|
+
):
|
|
31
|
+
"""Initialize the parameters of the Bernoulli-Bernoulli RBM.
|
|
32
|
+
|
|
33
|
+
Args:
|
|
34
|
+
weight_matrix (Tensor): The weight matrix of the RBM.
|
|
35
|
+
vbias (Tensor): The visible bias of the RBM.
|
|
36
|
+
hbias (Tensor): The hidden bias of the RBM.
|
|
37
|
+
device (Optional[torch.device], optional): The device for the parameters.
|
|
38
|
+
Defaults to the device of `weight_matrix`.
|
|
39
|
+
dtype (Optional[torch.dtype], optional): The data type for the parameters.
|
|
40
|
+
Defaults to the data type of `weight_matrix`.
|
|
41
|
+
"""
|
|
42
|
+
if device is None:
|
|
43
|
+
device = weight_matrix.device
|
|
44
|
+
if dtype is None:
|
|
45
|
+
dtype = weight_matrix.dtype
|
|
46
|
+
self.device = device
|
|
47
|
+
self.dtype = dtype
|
|
48
|
+
self.weight_matrix = weight_matrix.to(device=self.device, dtype=self.dtype)
|
|
49
|
+
self.vbias = vbias.to(device=self.device, dtype=self.dtype)
|
|
50
|
+
self.hbias = hbias.to(device=self.device, dtype=self.dtype)
|
|
51
|
+
self.name = "BBRBM"
|
|
52
|
+
|
|
53
|
+
def __add__(self, other):
|
|
54
|
+
return BBRBM(
|
|
55
|
+
weight_matrix=self.weight_matrix + other.weight_matrix,
|
|
56
|
+
vbias=self.vbias + other.vbias,
|
|
57
|
+
hbias=self.hbias + other.hbias,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
def __mul__(self, other):
|
|
61
|
+
return BBRBM(
|
|
62
|
+
weight_matrix=self.weight_matrix * other,
|
|
63
|
+
vbias=self.vbias * other,
|
|
64
|
+
hbias=self.hbias * other,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
def clone(
|
|
68
|
+
self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None
|
|
69
|
+
):
|
|
70
|
+
if device is None:
|
|
71
|
+
device = self.device
|
|
72
|
+
if dtype is None:
|
|
73
|
+
dtype = self.dtype
|
|
74
|
+
return BBRBM(
|
|
75
|
+
weight_matrix=self.weight_matrix.clone(),
|
|
76
|
+
vbias=self.vbias.clone(),
|
|
77
|
+
hbias=self.hbias.clone(),
|
|
78
|
+
device=device,
|
|
79
|
+
dtype=dtype,
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
def compute_energy(self, v: Tensor, h: Tensor) -> Tensor:
|
|
83
|
+
return _compute_energy(
|
|
84
|
+
v=v,
|
|
85
|
+
h=h,
|
|
86
|
+
vbias=self.vbias,
|
|
87
|
+
hbias=self.hbias,
|
|
88
|
+
weight_matrix=self.weight_matrix,
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
def compute_energy_hiddens(self, h: Tensor) -> Tensor:
|
|
92
|
+
return _compute_energy_hiddens(
|
|
93
|
+
h=h,
|
|
94
|
+
vbias=self.vbias,
|
|
95
|
+
hbias=self.hbias,
|
|
96
|
+
weight_matrix=self.weight_matrix,
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
def compute_energy_visibles(self, v: Tensor) -> Tensor:
|
|
100
|
+
return _compute_energy_visibles(
|
|
101
|
+
v=v,
|
|
102
|
+
vbias=self.vbias,
|
|
103
|
+
hbias=self.hbias,
|
|
104
|
+
weight_matrix=self.weight_matrix,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
def compute_gradient(self, data, chains, centered=True):
|
|
108
|
+
_compute_gradient(
|
|
109
|
+
v_data=data["visible"],
|
|
110
|
+
mh_data=data["hidden_mag"],
|
|
111
|
+
w_data=data["weights"],
|
|
112
|
+
v_chain=chains["visible"],
|
|
113
|
+
h_chain=chains["hidden"],
|
|
114
|
+
w_chain=chains["weights"],
|
|
115
|
+
vbias=self.vbias,
|
|
116
|
+
hbias=self.hbias,
|
|
117
|
+
weight_matrix=self.weight_matrix,
|
|
118
|
+
centered=centered,
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
def independent_model(self):
|
|
122
|
+
return BBRBM(
|
|
123
|
+
weight_matrix=torch.zeros_like(self.weight_matrix),
|
|
124
|
+
vbias=self.vbias,
|
|
125
|
+
hbias=torch.zeros_like(self.hbias),
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
def init_chains(self, num_samples, weights=None, start_v=None):
|
|
129
|
+
visible, hidden, mean_visible, mean_hidden = _init_chains(
|
|
130
|
+
num_samples=num_samples,
|
|
131
|
+
weight_matrix=self.weight_matrix,
|
|
132
|
+
hbias=self.hbias,
|
|
133
|
+
start_v=start_v,
|
|
134
|
+
)
|
|
135
|
+
if weights is None:
|
|
136
|
+
weights = torch.ones(
|
|
137
|
+
visible.shape[0], device=visible.device, dtype=visible.dtype
|
|
138
|
+
)
|
|
139
|
+
return dict(
|
|
140
|
+
visible=visible,
|
|
141
|
+
hidden=hidden,
|
|
142
|
+
visible_mag=mean_visible,
|
|
143
|
+
hidden_mag=mean_hidden,
|
|
144
|
+
weights=weights,
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
@staticmethod
|
|
148
|
+
def init_parameters(num_hiddens, dataset, device, dtype, var_init=0.0001):
|
|
149
|
+
data = dataset.data
|
|
150
|
+
# Convert to torch Tensor if necessary
|
|
151
|
+
if isinstance(data, np.ndarray):
|
|
152
|
+
data = torch.from_numpy(dataset.data).to(device=device, dtype=dtype)
|
|
153
|
+
vbias, hbias, weight_matrix = _init_parameters(
|
|
154
|
+
num_hiddens=num_hiddens,
|
|
155
|
+
data=data,
|
|
156
|
+
device=device,
|
|
157
|
+
dtype=dtype,
|
|
158
|
+
var_init=var_init,
|
|
159
|
+
)
|
|
160
|
+
return BBRBM(weight_matrix=weight_matrix, vbias=vbias, hbias=hbias)
|
|
161
|
+
|
|
162
|
+
def named_parameters(self):
|
|
163
|
+
return {
|
|
164
|
+
"weight_matrix": self.weight_matrix,
|
|
165
|
+
"vbias": self.vbias,
|
|
166
|
+
"hbias": self.hbias,
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
def num_hiddens(self):
|
|
170
|
+
return self.hbias.shape[0]
|
|
171
|
+
|
|
172
|
+
def num_visibles(self):
|
|
173
|
+
return self.vbias.shape[0]
|
|
174
|
+
|
|
175
|
+
def parameters(self) -> List[Tensor]:
|
|
176
|
+
return [self.weight_matrix, self.vbias, self.hbias]
|
|
177
|
+
|
|
178
|
+
def ref_log_z(self):
|
|
179
|
+
return (
|
|
180
|
+
torch.log1p(torch.exp(self.vbias)).sum() + self.num_hiddens() * np.log(2)
|
|
181
|
+
).item()
|
|
182
|
+
|
|
183
|
+
def sample_hiddens(self, chains: dict[str, Tensor], beta=1) -> dict[str, Tensor]:
|
|
184
|
+
chains["hidden"], chains["hidden_mag"] = _sample_hiddens(
|
|
185
|
+
v=chains["visible"],
|
|
186
|
+
weight_matrix=self.weight_matrix,
|
|
187
|
+
hbias=self.hbias,
|
|
188
|
+
beta=beta,
|
|
189
|
+
)
|
|
190
|
+
return chains
|
|
191
|
+
|
|
192
|
+
def sample_visibles(self, chains: dict[str, Tensor], beta=1) -> dict[str, Tensor]:
|
|
193
|
+
chains["visible"], chains["visible_mag"] = _sample_visibles(
|
|
194
|
+
h=chains["hidden"],
|
|
195
|
+
weight_matrix=self.weight_matrix,
|
|
196
|
+
vbias=self.vbias,
|
|
197
|
+
beta=beta,
|
|
198
|
+
)
|
|
199
|
+
return chains
|
|
200
|
+
|
|
201
|
+
@staticmethod
|
|
202
|
+
def set_named_parameters(named_params: dict[str, Tensor]) -> Self:
|
|
203
|
+
names = ["vbias", "hbias", "weight_matrix"]
|
|
204
|
+
for k in names:
|
|
205
|
+
if k not in named_params.keys():
|
|
206
|
+
raise ValueError(
|
|
207
|
+
f"""Dictionary params missing key '{k}'\n Provided keys : {named_params.keys()}\n Expected keys: {names}"""
|
|
208
|
+
)
|
|
209
|
+
params = BBRBM(
|
|
210
|
+
weight_matrix=named_params.pop("weight_matrix"),
|
|
211
|
+
vbias=named_params.pop("vbias"),
|
|
212
|
+
hbias=named_params.pop("hbias"),
|
|
213
|
+
)
|
|
214
|
+
if len(named_params.keys()) > 0:
|
|
215
|
+
raise ValueError(
|
|
216
|
+
f"Too many keys in params dictionary. Remaining keys: {named_params.keys()}"
|
|
217
|
+
)
|
|
218
|
+
return params
|
|
219
|
+
|
|
220
|
+
def to(
|
|
221
|
+
self, device: Optional[torch.device] = None, dtype: Optional[torch.dtype] = None
|
|
222
|
+
):
|
|
223
|
+
if device is not None:
|
|
224
|
+
self.device = device
|
|
225
|
+
if dtype is not None:
|
|
226
|
+
self.dtype = dtype
|
|
227
|
+
self.weight_matrix = self.weight_matrix.to(device=self.device, dtype=self.dtype)
|
|
228
|
+
self.vbias = self.vbias.to(device=self.device, dtype=self.dtype)
|
|
229
|
+
self.hbias = self.hbias.to(device=self.device, dtype=self.dtype)
|
|
230
|
+
return self
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
from typing import Optional
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import torch
|
|
5
|
+
from torch import Tensor
|
|
6
|
+
|
|
7
|
+
from rbms.bernoulli_bernoulli.classes import BBRBM
|
|
8
|
+
from rbms.bernoulli_bernoulli.implement import (
|
|
9
|
+
_compute_energy,
|
|
10
|
+
_compute_energy_hiddens,
|
|
11
|
+
_compute_energy_visibles,
|
|
12
|
+
_compute_gradient,
|
|
13
|
+
_init_chains,
|
|
14
|
+
_init_parameters,
|
|
15
|
+
_sample_hiddens,
|
|
16
|
+
_sample_visibles,
|
|
17
|
+
)
|
|
18
|
+
from rbms.dataset.dataset_class import RBMDataset
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def sample_hiddens(
|
|
22
|
+
chains: dict[str, Tensor], params: BBRBM, beta: float = 1.0
|
|
23
|
+
) -> dict[str, Tensor]:
|
|
24
|
+
"""Sample the hidden layer conditionally to the visible one.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
chains (dict[str, Tensor]): The parallel chains used for sampling.
|
|
28
|
+
params (BBRBM): The parameters of the RBM.
|
|
29
|
+
beta (float, optional): The inverse temperature. Defaults to 1.0.
|
|
30
|
+
|
|
31
|
+
Returns:
|
|
32
|
+
dict[str, Tensor]: The updated chains with sampled hidden states.
|
|
33
|
+
"""
|
|
34
|
+
chains["hidden"], chains["hidden_mag"] = _sample_hiddens(
|
|
35
|
+
v=chains["visible"],
|
|
36
|
+
weight_matrix=params.weight_matrix,
|
|
37
|
+
hbias=params.hbias,
|
|
38
|
+
beta=beta,
|
|
39
|
+
)
|
|
40
|
+
return chains
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def sample_visibles(
|
|
44
|
+
chains: dict[str, Tensor], params: BBRBM, beta: float = 1.0
|
|
45
|
+
) -> dict[str, Tensor]:
|
|
46
|
+
"""Sample the visible layer conditionally to the hidden one.
|
|
47
|
+
|
|
48
|
+
Args:
|
|
49
|
+
chains (dict[str, Tensor]): The parallel chains used for sampling.
|
|
50
|
+
params (BBRBM): The parameters of the RBM.
|
|
51
|
+
beta (float, optional): The inverse temperature. Defaults to 1.0.
|
|
52
|
+
|
|
53
|
+
Returns:
|
|
54
|
+
dict[str, Tensor]: The updated chains with sampled visible states.
|
|
55
|
+
"""
|
|
56
|
+
chains["visible"], chains["visible_mag"] = _sample_visibles(
|
|
57
|
+
h=chains["hidden"],
|
|
58
|
+
weight_matrix=params.weight_matrix,
|
|
59
|
+
vbias=params.vbias,
|
|
60
|
+
beta=beta,
|
|
61
|
+
)
|
|
62
|
+
return chains
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def compute_energy(
|
|
66
|
+
v: Tensor,
|
|
67
|
+
h: Tensor,
|
|
68
|
+
params: BBRBM,
|
|
69
|
+
) -> Tensor:
|
|
70
|
+
"""Compute the energy of the RBM on the visible and hidden variables.
|
|
71
|
+
|
|
72
|
+
Args:
|
|
73
|
+
v (Tensor): Visible configurations.
|
|
74
|
+
h (Tensor): Hidden configurations.
|
|
75
|
+
params (BBRBM): Parameters of the RBM.
|
|
76
|
+
|
|
77
|
+
Returns:
|
|
78
|
+
Tensor: The computed energy.
|
|
79
|
+
"""
|
|
80
|
+
return _compute_energy(
|
|
81
|
+
v=v,
|
|
82
|
+
h=h,
|
|
83
|
+
vbias=params.vbias,
|
|
84
|
+
hbias=params.hbias,
|
|
85
|
+
weight_matrix=params.weight_matrix,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def compute_energy_visibles(v: Tensor, params: BBRBM) -> Tensor:
|
|
90
|
+
"""Returns the marginalized energy of the model computed on the visible configurations
|
|
91
|
+
|
|
92
|
+
Args:
|
|
93
|
+
v (Tensor): Visible configurations
|
|
94
|
+
params (BBRBM): Parameters of the RBM
|
|
95
|
+
|
|
96
|
+
Returns:
|
|
97
|
+
Tensor: The computed energy.
|
|
98
|
+
"""
|
|
99
|
+
return _compute_energy_visibles(
|
|
100
|
+
v=v, vbias=params.vbias, hbias=params.hbias, weight_matrix=params.weight_matrix
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def compute_energy_hiddens(h: Tensor, params: BBRBM) -> Tensor:
|
|
105
|
+
"""Returns the marginalized energy of the model computed on hidden configurations
|
|
106
|
+
|
|
107
|
+
Args:
|
|
108
|
+
h (Tensor): Hidden configurations.
|
|
109
|
+
params (BBRBM): Parameters of the RBM.
|
|
110
|
+
"""
|
|
111
|
+
return _compute_energy_hiddens(
|
|
112
|
+
h=h, vbias=params.vbias, hbias=params.hbias, weight_matrix=params.weight_matrix
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def compute_gradient(
|
|
117
|
+
data: dict[str, Tensor],
|
|
118
|
+
chains: dict[str, Tensor],
|
|
119
|
+
params: BBRBM,
|
|
120
|
+
centered: bool = True,
|
|
121
|
+
) -> None:
|
|
122
|
+
"""Compute the gradient for each of the parameters and attach it.
|
|
123
|
+
|
|
124
|
+
Args:
|
|
125
|
+
data (dict[str, Tensor]): The data state.
|
|
126
|
+
chains (dict[str, Tensor]): The parallel chains used for gradient computation.
|
|
127
|
+
params (BBRBM): The parameters of the RBM.
|
|
128
|
+
centered (bool, optional): Whether to use centered gradients. Defaults to True.
|
|
129
|
+
"""
|
|
130
|
+
_compute_gradient(
|
|
131
|
+
v_data=data["visible"],
|
|
132
|
+
mh_data=data["hidden_mag"],
|
|
133
|
+
w_data=data["weights"],
|
|
134
|
+
v_chain=chains["visible"],
|
|
135
|
+
h_chain=chains["hidden"],
|
|
136
|
+
w_chain=chains["weights"],
|
|
137
|
+
vbias=params.vbias,
|
|
138
|
+
hbias=params.hbias,
|
|
139
|
+
weight_matrix=params.weight_matrix,
|
|
140
|
+
centered=centered,
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def init_chains(
|
|
145
|
+
num_samples: int,
|
|
146
|
+
params: BBRBM,
|
|
147
|
+
weights: Optional[Tensor] = None,
|
|
148
|
+
start_v: Optional[Tensor] = None,
|
|
149
|
+
) -> dict[str, Tensor]:
|
|
150
|
+
"""Initialize a Markov chain for the RBM by sampling a uniform distribution on the visible layer
|
|
151
|
+
and sampling the hidden layer according to the visible one.
|
|
152
|
+
|
|
153
|
+
Args:
|
|
154
|
+
num_samples (int): The number of samples to initialize.
|
|
155
|
+
params (BBRBM): The parameters of the RBM.
|
|
156
|
+
weights (Tensor, optional): The weights of each configuration. Defaults to None.
|
|
157
|
+
start_v (Optional[Tensor], optional): The initial visible states. Defaults to None.
|
|
158
|
+
|
|
159
|
+
Returns:
|
|
160
|
+
dict[str, Tensor]: The initialized Markov chain.
|
|
161
|
+
|
|
162
|
+
Notes:
|
|
163
|
+
- If start_v is specified, its number of samples will override the num_samples argument.
|
|
164
|
+
"""
|
|
165
|
+
visible, hidden, mean_visible, mean_hidden = _init_chains(
|
|
166
|
+
num_samples=num_samples,
|
|
167
|
+
weight_matrix=params.weight_matrix,
|
|
168
|
+
hbias=params.hbias,
|
|
169
|
+
start_v=start_v,
|
|
170
|
+
)
|
|
171
|
+
if weights is None:
|
|
172
|
+
weights = torch.ones(visible.shape[0], device=visible.device, dtype=visible.dtype)
|
|
173
|
+
return dict(
|
|
174
|
+
visible=visible,
|
|
175
|
+
hidden=hidden,
|
|
176
|
+
visible_mag=mean_visible,
|
|
177
|
+
hidden_mag=mean_hidden,
|
|
178
|
+
weights=weights,
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def init_parameters(
|
|
183
|
+
num_hiddens: int,
|
|
184
|
+
dataset: RBMDataset,
|
|
185
|
+
device: torch.device,
|
|
186
|
+
dtype: torch.dtype,
|
|
187
|
+
var_init: float = 1e-4,
|
|
188
|
+
) -> BBRBM:
|
|
189
|
+
"""Initialize the parameters of the RBM.
|
|
190
|
+
|
|
191
|
+
Args:
|
|
192
|
+
num_hiddens (int): Number of hidden units.
|
|
193
|
+
dataset (RBMDataset): Training dataset.
|
|
194
|
+
device (torch.device): PyTorch device for the parameters.
|
|
195
|
+
dtype (torch.dtype): PyTorch dtype for the parameters.
|
|
196
|
+
var_init (float, optional): Variance of the weight matrix. Defaults to 1e-4.
|
|
197
|
+
|
|
198
|
+
Notes:
|
|
199
|
+
- The number of visible units is induced from the dataset provided.
|
|
200
|
+
- Hidden biases are set to 0.
|
|
201
|
+
- Visible biases are set to the frequencies of the dataset.
|
|
202
|
+
- The weight matrix is initialized with a Gaussian distribution of variance `var_init`.
|
|
203
|
+
"""
|
|
204
|
+
data = dataset.data
|
|
205
|
+
# Convert to torch Tensor if necessary
|
|
206
|
+
if isinstance(data, np.ndarray):
|
|
207
|
+
data = torch.from_numpy(dataset.data).to(device=device, dtype=dtype)
|
|
208
|
+
vbias, hbias, weight_matrix = _init_parameters(
|
|
209
|
+
num_hiddens=num_hiddens,
|
|
210
|
+
data=data,
|
|
211
|
+
device=device,
|
|
212
|
+
dtype=dtype,
|
|
213
|
+
var_init=var_init,
|
|
214
|
+
)
|
|
215
|
+
return BBRBM(weight_matrix=weight_matrix, vbias=vbias, hbias=hbias)
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
from typing import Optional, Tuple
|
|
2
|
+
|
|
3
|
+
import torch
|
|
4
|
+
from torch import Tensor
|
|
5
|
+
from torch.nn.functional import softmax
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@torch.jit.script
|
|
9
|
+
def _sample_hiddens(
|
|
10
|
+
v: Tensor, weight_matrix: Tensor, hbias: Tensor, beta: float = 1.0
|
|
11
|
+
) -> Tuple[Tensor, Tensor]:
|
|
12
|
+
mh = torch.sigmoid(beta * (hbias + (v @ weight_matrix)))
|
|
13
|
+
h = torch.bernoulli(mh)
|
|
14
|
+
return h, mh
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@torch.jit.script
|
|
18
|
+
def _sample_visibles(
|
|
19
|
+
h: Tensor, weight_matrix: Tensor, vbias: Tensor, beta: float = 1.0
|
|
20
|
+
) -> Tuple[Tensor, Tensor]:
|
|
21
|
+
mv = torch.sigmoid(beta * (vbias + (h @ weight_matrix.T)))
|
|
22
|
+
v = torch.bernoulli(mv)
|
|
23
|
+
return v, mv
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@torch.jit.script
|
|
27
|
+
def _compute_energy(
|
|
28
|
+
v: Tensor,
|
|
29
|
+
h: Tensor,
|
|
30
|
+
vbias: Tensor,
|
|
31
|
+
hbias: Tensor,
|
|
32
|
+
weight_matrix: Tensor,
|
|
33
|
+
) -> Tensor:
|
|
34
|
+
fields = torch.tensordot(vbias, v, dims=[[0], [1]]) + torch.tensordot(
|
|
35
|
+
hbias, h, dims=[[0], [1]]
|
|
36
|
+
)
|
|
37
|
+
interaction = torch.multiply(
|
|
38
|
+
v, torch.tensordot(h, weight_matrix, dims=[[1], [1]])
|
|
39
|
+
).sum(1)
|
|
40
|
+
|
|
41
|
+
return -fields - interaction
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@torch.jit.script
|
|
45
|
+
def _compute_energy_visibles(
|
|
46
|
+
v: Tensor, vbias: Tensor, hbias: Tensor, weight_matrix: Tensor
|
|
47
|
+
) -> Tensor:
|
|
48
|
+
field = v @ vbias
|
|
49
|
+
exponent = hbias + (v @ weight_matrix)
|
|
50
|
+
log_term = torch.where(exponent < 10, torch.log(1.0 + torch.exp(exponent)), exponent)
|
|
51
|
+
return -field - log_term.sum(1)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@torch.jit.script
|
|
55
|
+
def _compute_energy_hiddens(
|
|
56
|
+
h: Tensor, vbias: Tensor, hbias: Tensor, weight_matrix: Tensor
|
|
57
|
+
) -> Tensor:
|
|
58
|
+
field = h @ hbias
|
|
59
|
+
exponent = vbias + (h @ weight_matrix.T)
|
|
60
|
+
log_term = torch.where(exponent < 10, torch.log(1.0 + torch.exp(exponent)), exponent)
|
|
61
|
+
return -field - log_term.sum(1)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@torch.jit.script
|
|
65
|
+
def _compute_gradient(
|
|
66
|
+
v_data: Tensor,
|
|
67
|
+
mh_data: Tensor,
|
|
68
|
+
w_data: Tensor,
|
|
69
|
+
v_chain: Tensor,
|
|
70
|
+
h_chain: Tensor,
|
|
71
|
+
w_chain: Tensor,
|
|
72
|
+
vbias: Tensor,
|
|
73
|
+
hbias: Tensor,
|
|
74
|
+
weight_matrix: Tensor,
|
|
75
|
+
centered: bool = True,
|
|
76
|
+
) -> None:
|
|
77
|
+
w_data = w_data.view(-1, 1)
|
|
78
|
+
w_chain = w_chain.view(-1, 1)
|
|
79
|
+
# Turn the weights of the chains into normalized weights
|
|
80
|
+
chain_weights = softmax(-w_chain, dim=0)
|
|
81
|
+
w_data_norm = w_data.sum()
|
|
82
|
+
|
|
83
|
+
# Averages over data and generated samples
|
|
84
|
+
v_data_mean = (v_data * w_data).sum(0) / w_data_norm
|
|
85
|
+
torch.clamp_(v_data_mean, min=1e-7, max=(1.0 - 1e-7))
|
|
86
|
+
h_data_mean = (mh_data * w_data).sum(0) / w_data_norm
|
|
87
|
+
v_gen_mean = (v_chain * chain_weights).sum(0)
|
|
88
|
+
torch.clamp_(v_gen_mean, min=1e-7, max=(1.0 - 1e-7))
|
|
89
|
+
h_gen_mean = (h_chain * chain_weights).sum(0)
|
|
90
|
+
|
|
91
|
+
if centered:
|
|
92
|
+
# Centered variables
|
|
93
|
+
v_data_centered = v_data - v_data_mean
|
|
94
|
+
h_data_centered = mh_data - h_data_mean
|
|
95
|
+
v_gen_centered = v_chain - v_data_mean
|
|
96
|
+
h_gen_centered = h_chain - h_data_mean
|
|
97
|
+
|
|
98
|
+
# Gradient
|
|
99
|
+
grad_weight_matrix = (
|
|
100
|
+
(v_data_centered * w_data).T @ h_data_centered
|
|
101
|
+
) / w_data_norm - ((v_gen_centered * chain_weights).T @ h_gen_centered)
|
|
102
|
+
grad_vbias = v_data_mean - v_gen_mean - (grad_weight_matrix @ h_data_mean)
|
|
103
|
+
grad_hbias = h_data_mean - h_gen_mean - (v_data_mean @ grad_weight_matrix)
|
|
104
|
+
else:
|
|
105
|
+
v_data_centered = v_data
|
|
106
|
+
h_data_centered = mh_data
|
|
107
|
+
v_gen_centered = v_chain
|
|
108
|
+
h_gen_centered = h_chain
|
|
109
|
+
|
|
110
|
+
# Gradient
|
|
111
|
+
grad_weight_matrix = ((v_data * w_data).T @ mh_data) / w_data_norm - (
|
|
112
|
+
(v_chain * chain_weights).T @ h_chain
|
|
113
|
+
)
|
|
114
|
+
grad_vbias = v_data_mean - v_gen_mean
|
|
115
|
+
grad_hbias = h_data_mean - h_gen_mean
|
|
116
|
+
|
|
117
|
+
# Attach to the parameters
|
|
118
|
+
|
|
119
|
+
weight_matrix.grad.set_(grad_weight_matrix)
|
|
120
|
+
vbias.grad.set_(grad_vbias)
|
|
121
|
+
hbias.grad.set_(grad_hbias)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@torch.jit.script
|
|
125
|
+
def _init_chains(
|
|
126
|
+
num_samples: int,
|
|
127
|
+
weight_matrix: Tensor,
|
|
128
|
+
hbias: Tensor,
|
|
129
|
+
start_v: Optional[Tensor] = None,
|
|
130
|
+
):
|
|
131
|
+
num_visibles, _ = weight_matrix.shape
|
|
132
|
+
device = weight_matrix.device
|
|
133
|
+
dtype = weight_matrix.dtype
|
|
134
|
+
# Handle negative number of samples
|
|
135
|
+
if num_samples <= 0:
|
|
136
|
+
if start_v is not None:
|
|
137
|
+
num_samples = start_v.shape[0]
|
|
138
|
+
else:
|
|
139
|
+
raise ValueError(f"Got negative num_samples arg: {num_samples}")
|
|
140
|
+
|
|
141
|
+
if start_v is None:
|
|
142
|
+
# Dummy mean visible
|
|
143
|
+
mv = torch.ones(size=(num_samples, num_visibles), device=device, dtype=dtype) / 2
|
|
144
|
+
v = torch.bernoulli(mv)
|
|
145
|
+
else:
|
|
146
|
+
# Dummy mean visible
|
|
147
|
+
mv = torch.ones_like(start_v, device=device, dtype=dtype) / 2
|
|
148
|
+
v = start_v.to(device=device, dtype=dtype)
|
|
149
|
+
|
|
150
|
+
# Initialize chains
|
|
151
|
+
|
|
152
|
+
h, mh = _sample_hiddens(v=v, weight_matrix=weight_matrix, hbias=hbias)
|
|
153
|
+
return v, h, mv, mh
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _init_parameters(
|
|
157
|
+
num_hiddens: int,
|
|
158
|
+
data: Tensor,
|
|
159
|
+
device: torch.device,
|
|
160
|
+
dtype: torch.dtype,
|
|
161
|
+
var_init: float = 1e-4,
|
|
162
|
+
) -> Tuple[Tensor, Tensor, Tensor]:
|
|
163
|
+
_, num_visibles = data.shape
|
|
164
|
+
eps = 1e-4
|
|
165
|
+
weight_matrix = (
|
|
166
|
+
torch.randn(size=(num_visibles, num_hiddens), device=device, dtype=dtype)
|
|
167
|
+
* var_init
|
|
168
|
+
)
|
|
169
|
+
frequencies = data.mean(0)
|
|
170
|
+
frequencies = torch.clamp(frequencies, min=eps, max=(1.0 - eps))
|
|
171
|
+
vbias = (torch.log(frequencies) - torch.log(1.0 - frequencies)).to(
|
|
172
|
+
device=device, dtype=dtype
|
|
173
|
+
)
|
|
174
|
+
hbias = torch.zeros(num_hiddens, device=device, dtype=dtype)
|
|
175
|
+
return vbias, hbias, weight_matrix
|