rbms 0.3.2__tar.gz → 0.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. {rbms-0.3.2 → rbms-0.4.1}/PKG-INFO +1 -1
  2. {rbms-0.3.2 → rbms-0.4.1}/pyproject.toml +1 -1
  3. {rbms-0.3.2 → rbms-0.4.1}/rbms/bernoulli_bernoulli/classes.py +5 -1
  4. {rbms-0.3.2 → rbms-0.4.1}/rbms/bernoulli_bernoulli/functional.py +9 -1
  5. {rbms-0.3.2 → rbms-0.4.1}/rbms/bernoulli_bernoulli/implement.py +12 -0
  6. {rbms-0.3.2 → rbms-0.4.1}/rbms/classes.py +8 -1
  7. rbms-0.4.1/rbms/dataset/__init__.py +83 -0
  8. {rbms-0.3.2 → rbms-0.4.1}/rbms/dataset/parser.py +9 -2
  9. {rbms-0.3.2 → rbms-0.4.1}/rbms/io.py +2 -0
  10. {rbms-0.3.2 → rbms-0.4.1}/rbms/parser.py +26 -0
  11. {rbms-0.3.2 → rbms-0.4.1}/rbms/potts_bernoulli/classes.py +5 -1
  12. {rbms-0.3.2 → rbms-0.4.1}/rbms/potts_bernoulli/functional.py +9 -1
  13. {rbms-0.3.2 → rbms-0.4.1}/rbms/potts_bernoulli/implement.py +12 -0
  14. {rbms-0.3.2 → rbms-0.4.1}/rbms/scripts/train_rbm.py +11 -8
  15. {rbms-0.3.2 → rbms-0.4.1}/rbms/training/pcd.py +19 -35
  16. {rbms-0.3.2 → rbms-0.4.1}/rbms/training/utils.py +61 -9
  17. {rbms-0.3.2 → rbms-0.4.1}/tests/conftest.py +3 -0
  18. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/bernoulli_bernoulli/test_bernoulli_utils_bbrbm.py +4 -0
  19. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/potts_bernoulli/test_utils_pbrbm.py +4 -0
  20. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/training/test_pcd.py +3 -0
  21. {rbms-0.3.2 → rbms-0.4.1}/tests/use_cases/test_bbrbm.py +3 -0
  22. {rbms-0.3.2 → rbms-0.4.1}/tests/use_cases/test_pbrbm.py +6 -0
  23. rbms-0.3.2/rbms/dataset/__init__.py +0 -100
  24. {rbms-0.3.2 → rbms-0.4.1}/.github/workflows/codecov.yaml +0 -0
  25. {rbms-0.3.2 → rbms-0.4.1}/.github/workflows/test.yaml +0 -0
  26. {rbms-0.3.2 → rbms-0.4.1}/.github/workflows/whats_new.yaml +0 -0
  27. {rbms-0.3.2 → rbms-0.4.1}/.gitignore +0 -0
  28. {rbms-0.3.2 → rbms-0.4.1}/LICENSE +0 -0
  29. {rbms-0.3.2 → rbms-0.4.1}/MANIFEST.in +0 -0
  30. {rbms-0.3.2 → rbms-0.4.1}/README.md +0 -0
  31. {rbms-0.3.2 → rbms-0.4.1}/rbms/__init__.py +0 -0
  32. {rbms-0.3.2 → rbms-0.4.1}/rbms/bernoulli_bernoulli/__init__.py +0 -0
  33. {rbms-0.3.2 → rbms-0.4.1}/rbms/const.py +0 -0
  34. {rbms-0.3.2 → rbms-0.4.1}/rbms/custom_fn.py +0 -0
  35. {rbms-0.3.2 → rbms-0.4.1}/rbms/dataset/dataset_class.py +0 -0
  36. {rbms-0.3.2 → rbms-0.4.1}/rbms/dataset/fasta_utils.py +0 -0
  37. {rbms-0.3.2 → rbms-0.4.1}/rbms/dataset/load_fasta.py +0 -0
  38. {rbms-0.3.2 → rbms-0.4.1}/rbms/dataset/load_h5.py +0 -0
  39. {rbms-0.3.2 → rbms-0.4.1}/rbms/dataset/utils.py +0 -0
  40. {rbms-0.3.2 → rbms-0.4.1}/rbms/map_model.py +0 -0
  41. {rbms-0.3.2 → rbms-0.4.1}/rbms/metrics/__init__.py +0 -0
  42. {rbms-0.3.2 → rbms-0.4.1}/rbms/metrics/aats.py +0 -0
  43. {rbms-0.3.2 → rbms-0.4.1}/rbms/partition_function/ais.py +0 -0
  44. {rbms-0.3.2 → rbms-0.4.1}/rbms/partition_function/exact.py +0 -0
  45. {rbms-0.3.2 → rbms-0.4.1}/rbms/plot.py +0 -0
  46. {rbms-0.3.2 → rbms-0.4.1}/rbms/potts_bernoulli/__init__.py +0 -0
  47. {rbms-0.3.2 → rbms-0.4.1}/rbms/potts_bernoulli/tools.py +0 -0
  48. {rbms-0.3.2 → rbms-0.4.1}/rbms/potts_bernoulli/utils.py +0 -0
  49. {rbms-0.3.2 → rbms-0.4.1}/rbms/sampling/gibbs.py +0 -0
  50. {rbms-0.3.2 → rbms-0.4.1}/rbms/sampling/pt.py +0 -0
  51. {rbms-0.3.2 → rbms-0.4.1}/rbms/scripts/__init__.py +0 -0
  52. {rbms-0.3.2 → rbms-0.4.1}/rbms/scripts/entrypoint.py +0 -0
  53. {rbms-0.3.2 → rbms-0.4.1}/rbms/scripts/pt_sampling.py +0 -0
  54. {rbms-0.3.2 → rbms-0.4.1}/rbms/utils.py +0 -0
  55. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/bernoulli_bernoulli/implement/test_compute_energy_bbrbm.py +0 -0
  56. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/bernoulli_bernoulli/implement/test_compute_energy_hiddens_bbrbm.py +0 -0
  57. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/bernoulli_bernoulli/implement/test_compute_energy_visibles_bbrbm.py +0 -0
  58. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/bernoulli_bernoulli/implement/test_compute_gradient_bbrbm.py +0 -0
  59. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/bernoulli_bernoulli/implement/test_init_chains_bbrbm.py +0 -0
  60. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/bernoulli_bernoulli/implement/test_init_parameters_bbrbm.py +0 -0
  61. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/bernoulli_bernoulli/implement/test_sample_hiddens_bbrbm.py +0 -0
  62. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/bernoulli_bernoulli/implement/test_sample_visibles_bbrbm.py +0 -0
  63. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/bernoulli_bernoulli/test_functional_bbrbm.py +0 -0
  64. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/bernoulli_bernoulli/test_methods_bbrbm.py +0 -0
  65. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/metrics/test_aats.py +0 -0
  66. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/potts_bernoulli/implement/test_compute_energy_hiddens_pbrbm.py +0 -0
  67. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/potts_bernoulli/implement/test_compute_energy_pbrbm.py +0 -0
  68. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/potts_bernoulli/implement/test_compute_energy_visibles_pbrbm.py +0 -0
  69. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/potts_bernoulli/implement/test_compute_gradient_pbrbm.py +0 -0
  70. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/potts_bernoulli/implement/test_init_chains_pbrbm.py +0 -0
  71. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/potts_bernoulli/implement/test_init_parameters_pbrbm.py +0 -0
  72. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/potts_bernoulli/implement/test_sample_hiddens_pbrbm.py +0 -0
  73. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/potts_bernoulli/implement/test_sample_visibles_pbrbm.py +0 -0
  74. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/potts_bernoulli/test_functional_pbrbm.py +0 -0
  75. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/potts_bernoulli/test_methods_pbrbm.py +0 -0
  76. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/test_custom_fn.py +0 -0
  77. {rbms-0.3.2 → rbms-0.4.1}/tests/unit_test/test_utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rbms
3
- Version: 0.3.2
3
+ Version: 0.4.1
4
4
  Summary: Training and analyzing Restricted Boltzmann Machines in PyTorch
5
5
  Project-URL: Homepage, https://github.com/DsysDML/rbms
6
6
  Project-URL: Issues, https://github.com/DsysDML/rbms/issues
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "rbms"
7
- version = "0.3.2"
7
+ version = "0.4.1"
8
8
  authors = [
9
9
  {name="Nicolas Béreux", email="nicolas.bereux@gmail.com"},
10
10
  {name="Aurélien Decelle"},
@@ -104,7 +104,9 @@ class BBRBM(RBM):
104
104
  weight_matrix=self.weight_matrix,
105
105
  )
106
106
 
107
- def compute_gradient(self, data, chains, centered=True):
107
+ def compute_gradient(
108
+ self, data, chains, centered=True, lambda_l1=0.0, lambda_l2=0.0
109
+ ):
108
110
  _compute_gradient(
109
111
  v_data=data["visible"],
110
112
  mh_data=data["hidden_mag"],
@@ -116,6 +118,8 @@ class BBRBM(RBM):
116
118
  hbias=self.hbias,
117
119
  weight_matrix=self.weight_matrix,
118
120
  centered=centered,
121
+ lambda_l1=lambda_l1,
122
+ lambda_l2=lambda_l2,
119
123
  )
120
124
 
121
125
  def independent_model(self):
@@ -118,6 +118,8 @@ def compute_gradient(
118
118
  chains: dict[str, Tensor],
119
119
  params: BBRBM,
120
120
  centered: bool = True,
121
+ lambda_l1: float = 0.0,
122
+ lambda_l2: float = 0.0,
121
123
  ) -> None:
122
124
  """Compute the gradient for each of the parameters and attach it.
123
125
 
@@ -126,6 +128,8 @@ def compute_gradient(
126
128
  chains (dict[str, Tensor]): The parallel chains used for gradient computation.
127
129
  params (BBRBM): The parameters of the RBM.
128
130
  centered (bool, optional): Whether to use centered gradients. Defaults to True.
131
+ lambda_l1 (float, optional): factor for the L1 regularization. Defaults to 0.
132
+ lambda_l2 (float, optional): factor for the L2 regularization. Defaults to 0.
129
133
  """
130
134
  _compute_gradient(
131
135
  v_data=data["visible"],
@@ -138,6 +142,8 @@ def compute_gradient(
138
142
  hbias=params.hbias,
139
143
  weight_matrix=params.weight_matrix,
140
144
  centered=centered,
145
+ lambda_l1=lambda_l1,
146
+ lambda_l2=lambda_l2,
141
147
  )
142
148
 
143
149
 
@@ -169,7 +175,9 @@ def init_chains(
169
175
  start_v=start_v,
170
176
  )
171
177
  if weights is None:
172
- weights = torch.ones(visible.shape[0], device=visible.device, dtype=visible.dtype)
178
+ weights = torch.ones(
179
+ visible.shape[0], device=visible.device, dtype=visible.dtype
180
+ )
173
181
  return dict(
174
182
  visible=visible,
175
183
  hidden=hidden,
@@ -77,6 +77,8 @@ def _compute_gradient(
77
77
  hbias: Tensor,
78
78
  weight_matrix: Tensor,
79
79
  centered: bool = True,
80
+ lambda_l1: float = 0.0,
81
+ lambda_l2: float = 0.0,
80
82
  ) -> None:
81
83
  w_data = w_data.view(-1, 1)
82
84
  w_chain = w_chain.view(-1, 1)
@@ -118,6 +120,16 @@ def _compute_gradient(
118
120
  grad_vbias = v_data_mean - v_gen_mean
119
121
  grad_hbias = h_data_mean - h_gen_mean
120
122
 
123
+ if lambda_l1 > 0:
124
+ grad_weight_matrix -= 2 * lambda_l2 * torch.sign(weight_matrix)
125
+ grad_vbias -= 2 * lambda_l2 * torch.sign(vbias)
126
+ grad_hbias -= 2 * lambda_l2 * torch.sign(hbias)
127
+
128
+ if lambda_l2 > 0:
129
+ grad_weight_matrix -= 2 * lambda_l2 * weight_matrix
130
+ grad_vbias -= 2 * lambda_l2 * vbias
131
+ grad_hbias -= 2 * lambda_l2 * hbias
132
+
121
133
  # Attach to the parameters
122
134
 
123
135
  weight_matrix.grad.set_(grad_weight_matrix)
@@ -76,7 +76,12 @@ class EBM(ABC):
76
76
 
77
77
  @abstractmethod
78
78
  def compute_gradient(
79
- self, data: dict[str, Tensor], chains: dict[str, Tensor], centered: bool = True
79
+ self,
80
+ data: dict[str, Tensor],
81
+ chains: dict[str, Tensor],
82
+ centered: bool = True,
83
+ lambda_l1: float = 0.0,
84
+ lambda_l2: float = 0.0,
80
85
  ) -> None:
81
86
  """Compute the gradient for each of the parameters and attach it.
82
87
 
@@ -84,6 +89,8 @@ class EBM(ABC):
84
89
  data (dict[str, Tensor]): The data state.
85
90
  chains (dict[str, Tensor]): The parallel chains used for gradient computation.
86
91
  centered (bool, optional): Whether to use centered gradients. Defaults to True.
92
+ lambda_l1 (float, optional): factor for the L1 regularization. Defaults to 0.
93
+ lambda_l2 (float, optional): factor for the L2 regularization. Defaults to 0.
87
94
  """
88
95
  ...
89
96
 
@@ -0,0 +1,83 @@
1
+ from pathlib import Path
2
+ from typing import List, Optional, Tuple
3
+
4
+ import numpy as np
5
+ import torch
6
+
7
+ from rbms.dataset.dataset_class import RBMDataset
8
+ from rbms.dataset.load_fasta import load_FASTA
9
+ from rbms.dataset.load_h5 import load_HDF5
10
+ from rbms.dataset.utils import get_subset_labels
11
+
12
+
13
+ def load_dataset(
14
+ dataset_name: str,
15
+ test_dataset_name: Optional[str] = None,
16
+ subset_labels: Optional[List[int]] = None,
17
+ use_weights: bool = False,
18
+ binarize: bool = False,
19
+ alphabet="protein",
20
+ device: str = "cpu",
21
+ dtype: torch.dtype = torch.float32,
22
+ ) -> Tuple[RBMDataset, RBMDataset | None]:
23
+ data = None
24
+ weights = None
25
+ names = None
26
+ labels = None
27
+ is_binary = True
28
+
29
+ return_datasets = []
30
+ for dset_name in [dataset_name, test_dataset_name]:
31
+
32
+ if dset_name is not None:
33
+ dataset_name = Path(dataset_name)
34
+
35
+ match dataset_name.suffix:
36
+ case ".h5":
37
+ data, labels = load_HDF5(filename=dataset_name, binarize=binarize)
38
+ case ".fasta":
39
+ data, weights, names = load_FASTA(
40
+ filename=dataset_name,
41
+ binarize=binarize,
42
+ use_weights=use_weights,
43
+ alphabet=alphabet,
44
+ device=device,
45
+ )
46
+ if not binarize:
47
+ is_binary = False
48
+ case _:
49
+ raise ValueError(
50
+ """
51
+ Dataset could not be loaded as the type is not recognized.
52
+ It should be either:
53
+ - '.h5',
54
+ - '.fasta'
55
+ """
56
+ )
57
+
58
+ # Select subset of dataset w.r.t. labels
59
+ if subset_labels is not None and labels is not None:
60
+ data, labels = get_subset_labels(data, labels, subset_labels)
61
+
62
+ if weights is None:
63
+ weights = np.ones(data.shape[0])
64
+ if names is None:
65
+ names = np.arange(data.shape[0])
66
+ if labels is None:
67
+ labels = -np.ones(data.shape[0])
68
+
69
+ return_datasets.append(
70
+ RBMDataset(
71
+ data=data,
72
+ labels=labels,
73
+ weights=weights,
74
+ names=names,
75
+ dataset_name=dataset_name,
76
+ is_binary=is_binary,
77
+ device=device,
78
+ dtype=dtype,
79
+ )
80
+ )
81
+ else:
82
+ return_datasets.append(None)
83
+ return tuple(return_datasets)
@@ -8,10 +8,17 @@ def add_args_dataset(parser: argparse.ArgumentParser) -> argparse.ArgumentParser
8
8
  dataset_args = parser.add_argument_group("Dataset")
9
9
  dataset_args.add_argument(
10
10
  "-d",
11
- "--data",
11
+ "--dataset",
12
12
  type=str,
13
13
  required=True,
14
- help="Name of the dataset ('HGD', 'MNIST', 'BKACE', 'PF00072', 'PF13354'), or path to a data file (type should be .h5 or .fasta)",
14
+ help="Path to a data file (type should be .h5 or .fasta)",
15
+ )
16
+ dataset_args.add_argument(
17
+ "--test_dataset",
18
+ type=str,
19
+ required=False,
20
+ default=None,
21
+ help="Path to test dataset file (type should be .h5 or .fasta)",
15
22
  )
16
23
  dataset_args.add_argument(
17
24
  "--subset_labels",
@@ -130,6 +130,8 @@ def load_model(
130
130
  hyperparameters["learning_rate"] = float(
131
131
  f["hyperparameters"]["learning_rate"][()]
132
132
  )
133
+ hyperparameters["L1"] = float(f["hyperparameters"]["L1"][()])
134
+ hyperparameters["L2"] = float(f["hyperparameters"]["L2"][()])
133
135
  if "seed" in f["hyperparameters"].keys():
134
136
  hyperparameters["seed"] = int(f["hyperparameters"]["seed"][()])
135
137
  if "train_size" in f["hyperparameters"].keys():
@@ -134,6 +134,29 @@ def add_args_rbm(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
134
134
  action="store_true",
135
135
  help="(Defaults to False). Restore the training",
136
136
  )
137
+ rbm_args.add_argument(
138
+ "--no_center",
139
+ default=False,
140
+ action="store_true",
141
+ help="(Defaults to False). Use the non-centered gradient.",
142
+ )
143
+ return parser
144
+
145
+
146
+ def add_args_regularization(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
147
+ reg_args = parser.add_argument_group("Regularization")
148
+ reg_args.add_argument(
149
+ "--L1",
150
+ default=None,
151
+ type=float,
152
+ help="(Defaults to 0.0). Lambda parameter for the L1 regularization.",
153
+ )
154
+ reg_args.add_argument(
155
+ "--L2",
156
+ default=None,
157
+ type=float,
158
+ help="(Defaults to 0.0). Lambda parameter for the L2 regularization.",
159
+ )
137
160
  return parser
138
161
 
139
162
 
@@ -186,6 +209,9 @@ default_args: dict[str, Any] = {
186
209
  "beta": 1.0,
187
210
  "restore": False,
188
211
  "seed": np.random.randint(0, 1000000000000),
212
+ "no_center": False,
213
+ "L1": 0.0,
214
+ "L2": 0.0,
189
215
  }
190
216
 
191
217
 
@@ -105,7 +105,9 @@ class PBRBM(RBM):
105
105
  weight_matrix=self.weight_matrix,
106
106
  )
107
107
 
108
- def compute_gradient(self, data, chains, centered=True):
108
+ def compute_gradient(
109
+ self, data, chains, centered=True, lambda_l1=0.0, lambda_l2=0.0
110
+ ):
109
111
  _compute_gradient(
110
112
  v_data=data["visible"],
111
113
  mh_data=data["hidden_mag"],
@@ -117,6 +119,8 @@ class PBRBM(RBM):
117
119
  hbias=self.hbias,
118
120
  weight_matrix=self.weight_matrix,
119
121
  centered=centered,
122
+ lambda_l1=0.0,
123
+ lambda_l2=lambda_l2,
120
124
  )
121
125
 
122
126
  def independent_model(self):
@@ -118,6 +118,8 @@ def compute_gradient(
118
118
  chains: dict[str, Tensor],
119
119
  params: PBRBM,
120
120
  centered: bool = True,
121
+ lambda_l1: float = 0.0,
122
+ lambda_l2: float = 0.0,
121
123
  ) -> None:
122
124
  """Compute the gradient for each of the parameters and attach it.
123
125
 
@@ -126,6 +128,8 @@ def compute_gradient(
126
128
  chains (dict[str, Tensor]): The parallel chains used for gradient computation.
127
129
  params (PBRBM): The parameters of the RBM.
128
130
  centered (bool, optional): Whether to use centered gradients. Defaults to True.
131
+ lambda_l1 (float, optional): factor for the L1 regularization. Defaults to 0.
132
+ lambda_l2 (float, optional): factor for the L2 regularization. Defaults to 0.
129
133
  """
130
134
  _compute_gradient(
131
135
  v_data=data["visible"],
@@ -138,6 +142,8 @@ def compute_gradient(
138
142
  hbias=params.hbias,
139
143
  weight_matrix=params.weight_matrix,
140
144
  centered=centered,
145
+ lambda_l1=lambda_l1,
146
+ lambda_l2=lambda_l2,
141
147
  )
142
148
 
143
149
 
@@ -169,7 +175,9 @@ def init_chains(
169
175
  start_v=start_v,
170
176
  )
171
177
  if weights is None:
172
- weights = torch.ones(visible.shape[0], device=visible.device, dtype=visible.dtype)
178
+ weights = torch.ones(
179
+ visible.shape[0], device=visible.device, dtype=visible.dtype
180
+ )
173
181
  return dict(
174
182
  visible=visible,
175
183
  hidden=hidden,
@@ -97,6 +97,8 @@ def _compute_gradient(
97
97
  hbias: Tensor,
98
98
  weight_matrix: Tensor,
99
99
  centered: bool = True,
100
+ lambda_l1: float = 0.0,
101
+ lambda_l2: float = 0.0,
100
102
  ):
101
103
  w_data = w_data.view(-1, 1, 1)
102
104
  w_chain = w_chain.view(-1, 1, 1)
@@ -175,6 +177,16 @@ def _compute_gradient(
175
177
 
176
178
  grad_vbias = v_data_mean - v_gen_mean
177
179
  grad_hbias = h_data_mean - h_gen_mean
180
+
181
+ if lambda_l1 > 0:
182
+ grad_weight_matrix -= 2 * lambda_l2 * torch.sign(weight_matrix)
183
+ grad_vbias -= 2 * lambda_l2 * torch.sign(vbias)
184
+ grad_hbias -= 2 * lambda_l2 * torch.sign(hbias)
185
+
186
+ if lambda_l2 != 0:
187
+ grad_weight_matrix -= 2 * lambda_l2 * weight_matrix
188
+ grad_vbias -= 2 * lambda_l2 * vbias
189
+ grad_hbias -= 2 * lambda_l2 * hbias
178
190
  weight_matrix.grad.set_(grad_weight_matrix)
179
191
  vbias.grad.set_(grad_vbias)
180
192
  hbias.grad.set_(grad_hbias)
@@ -9,6 +9,7 @@ from rbms.parser import (
9
9
  add_args_pytorch,
10
10
  add_args_rbm,
11
11
  add_args_saves,
12
+ add_args_regularization,
12
13
  match_args_dtype,
13
14
  remove_argument,
14
15
  default_args,
@@ -21,6 +22,7 @@ def create_parser():
21
22
  parser = argparse.ArgumentParser(description="Train a Restricted Boltzmann Machine")
22
23
  parser = add_args_dataset(parser)
23
24
  parser = add_args_rbm(parser)
25
+ parser = add_args_regularization(parser)
24
26
  parser = add_args_saves(parser)
25
27
  parser = add_args_pytorch(parser)
26
28
  remove_argument(parser, "use_torch")
@@ -28,28 +30,29 @@ def create_parser():
28
30
 
29
31
 
30
32
  def train_rbm(args: dict):
33
+ if args["num_updates"] is None:
34
+ args["num_updates"] = default_args["num_updates"]
31
35
  checkpoints = get_checkpoints(
32
36
  num_updates=args["num_updates"], n_save=args["n_save"], spacing=args["spacing"]
33
37
  )
34
- dataset, _ = load_dataset(
35
- dataset_name=args["data"],
38
+ train_dataset, test_dataset = load_dataset(
39
+ dataset_name=args["dataset"],
40
+ test_dataset_name=args["test_dataset"],
36
41
  subset_labels=args["subset_labels"],
37
42
  use_weights=args["use_weights"],
38
43
  alphabet=args["alphabet"],
39
44
  binarize=args["binarize"],
40
- train_size=1.0,
41
- test_size=None,
42
45
  device=args["device"],
43
46
  dtype=args["dtype"],
44
- seed=args["seed"],
45
47
  )
46
- print(dataset)
47
- if dataset.is_binary:
48
+ print(train_dataset)
49
+ if train_dataset.is_binary:
48
50
  model_type = "BBRBM"
49
51
  else:
50
52
  model_type = "PBRBM"
51
53
  train(
52
- dataset=dataset,
54
+ train_dataset=train_dataset,
55
+ test_dataset=test_dataset,
53
56
  model_type=model_type,
54
57
  args=args,
55
58
  dtype=args["dtype"],
@@ -1,5 +1,5 @@
1
1
  import time
2
- from typing import Tuple
2
+ from typing import Tuple, Optional
3
3
 
4
4
  import numpy as np
5
5
  import torch
@@ -12,7 +12,7 @@ from rbms.io import save_model
12
12
  from rbms.map_model import map_model
13
13
  from rbms.potts_bernoulli.classes import PBRBM
14
14
  from rbms.potts_bernoulli.utils import ensure_zero_sum_gauge
15
- from rbms.training.utils import create_machine, setup_training
15
+ from rbms.training.utils import setup_training, initialize_model_archive
16
16
  from rbms.utils import check_file_existence, log_to_csv
17
17
  from rbms.parser import default_args, set_args_default
18
18
 
@@ -54,7 +54,8 @@ def fit_batch_pcd(
54
54
 
55
55
 
56
56
  def train(
57
- dataset: RBMDataset,
57
+ train_dataset: RBMDataset,
58
+ test_dataset: Optional[RBMDataset],
58
59
  model_type: str,
59
60
  args: dict,
60
61
  dtype: torch.dtype,
@@ -73,41 +74,18 @@ def train(
73
74
  checkpoints (np.ndarray): An array of checkpoints for saving model states.
74
75
  """
75
76
 
76
- filename = args["filename"]
77
77
  if not (args["overwrite"]):
78
- check_file_existence(filename)
78
+ check_file_existence(args["filename"])
79
79
 
80
- num_visibles = dataset.get_num_visibles()
81
80
  # Create a first archive with the initialized model
82
81
  if not (args["restore"]):
83
- args = set_args_default(args=args, default_args=default_args)
84
- rng = np.random.default_rng(args["seed"])
85
- train_dataset, test_dataset = dataset.split_train_test(
86
- rng, args["train_size"], args["test_size"]
87
- )
88
- params = map_model[model_type].init_parameters(
89
- num_hiddens=args["num_hiddens"],
90
- dataset=train_dataset,
91
- device=args["device"],
82
+ initialize_model_archive(
83
+ args=args,
84
+ model_type=model_type,
85
+ train_dataset=train_dataset,
86
+ test_dataset=test_dataset,
92
87
  dtype=dtype,
93
88
  )
94
-
95
- if isinstance(params, PBRBM):
96
- ensure_zero_sum_gauge(params)
97
- create_machine(
98
- filename=filename,
99
- params=params,
100
- num_visibles=num_visibles,
101
- num_hiddens=args["num_hiddens"],
102
- num_chains=args["num_chains"],
103
- batch_size=args["batch_size"],
104
- gibbs_steps=args["gibbs_steps"],
105
- learning_rate=args["learning_rate"],
106
- train_size=args["train_size"],
107
- log=args["log"],
108
- flags=["checkpoint"],
109
- seed=args["seed"],
110
- )
111
89
  (
112
90
  params,
113
91
  parallel_chains,
@@ -119,7 +97,12 @@ def train(
119
97
  pbar,
120
98
  train_dataset,
121
99
  test_dataset,
122
- ) = setup_training(args, map_model=map_model, dataset=dataset)
100
+ ) = setup_training(
101
+ args,
102
+ map_model=map_model,
103
+ train_dataset=train_dataset,
104
+ test_dataset=test_dataset,
105
+ )
123
106
  args = set_args_default(args=args, default_args=default_args)
124
107
  optimizer = SGD(params.parameters(), lr=args["learning_rate"], maximize=True)
125
108
 
@@ -129,8 +112,8 @@ def train(
129
112
  # Continue the training
130
113
  with torch.no_grad():
131
114
  for idx in range(num_updates + 1, args["num_updates"] + 1):
132
- rand_idx = torch.randperm(len(dataset))[: args["batch_size"]]
133
- batch = (dataset.data[rand_idx], dataset.weights[rand_idx])
115
+ rand_idx = torch.randperm(len(train_dataset))[: args["batch_size"]]
116
+ batch = (train_dataset.data[rand_idx], train_dataset.weights[rand_idx])
134
117
 
135
118
  optimizer.zero_grad(set_to_none=False)
136
119
  parallel_chains, logs = fit_batch_pcd(
@@ -139,6 +122,7 @@ def train(
139
122
  params=params,
140
123
  gibbs_steps=args["gibbs_steps"],
141
124
  beta=args["beta"],
125
+ centered=not (args["no_center"]),
142
126
  )
143
127
  optimizer.step()
144
128
  if isinstance(params, PBRBM):
@@ -1,6 +1,6 @@
1
1
  import pathlib
2
2
  import time
3
- from typing import Any, List, Tuple
3
+ from typing import Any, List, Optional, Tuple
4
4
 
5
5
  import h5py
6
6
  import numpy as np
@@ -10,15 +10,19 @@ from tqdm import tqdm
10
10
 
11
11
  from rbms.classes import EBM
12
12
  from rbms.const import LOG_FILE_HEADER
13
+ from rbms.dataset.dataset_class import RBMDataset
13
14
  from rbms.io import load_model, save_model
14
15
  from rbms.map_model import map_model
16
+ from rbms.parser import default_args, set_args_default
17
+ from rbms.potts_bernoulli.classes import PBRBM
18
+ from rbms.potts_bernoulli.utils import ensure_zero_sum_gauge
15
19
  from rbms.utils import get_saved_updates
16
- from rbms.dataset.dataset_class import RBMDataset
17
20
 
18
21
 
19
22
  def setup_training(
20
23
  args: dict,
21
- dataset: RBMDataset,
24
+ train_dataset: RBMDataset,
25
+ test_dataset: Optional[RBMDataset] = None,
22
26
  map_model: dict[str, EBM] = map_model,
23
27
  ) -> Tuple[
24
28
  EBM,
@@ -52,14 +56,15 @@ def setup_training(
52
56
 
53
57
  # Hyperparameters
54
58
  for k, v in hyperparameters.items():
55
- if v is not None:
59
+ if args[k] is None:
56
60
  args[k] = v
57
61
 
58
- train_dataset, test_dataset = dataset.split_train_test(
59
- rng=np.random.default_rng(args["seed"]),
60
- train_size=args["train_size"],
61
- test_size=args["test_size"],
62
- )
62
+ if test_dataset is None:
63
+ train_dataset, test_dataset = train_dataset.split_train_test(
64
+ rng=np.random.default_rng(args["seed"]),
65
+ train_size=args["train_size"],
66
+ test_size=args["test_size"],
67
+ )
63
68
 
64
69
  # Open the log file if it exists
65
70
  log_filename = pathlib.Path(args["filename"]).parent / pathlib.Path(
@@ -111,6 +116,8 @@ def create_machine(
111
116
  log: bool,
112
117
  flags: List[str],
113
118
  seed: int,
119
+ L1: float,
120
+ L2: float,
114
121
  ) -> None:
115
122
  """Create a RBM and save it to a new file.
116
123
 
@@ -124,6 +131,8 @@ def create_machine(
124
131
  gibbs_steps (int): Number of Gibbs steps to perform.
125
132
  learning_rate (float): Learning rate for training.
126
133
  log (bool): Whether to enable logging.
134
+ L1 (float): Lambda parameter for L1 regularization.
135
+ L2 (float): Lambda parameter for L2 regularization.
127
136
  """
128
137
  # Permanent chains
129
138
  parallel_chains = params.init_chains(num_samples=num_chains)
@@ -139,6 +148,8 @@ def create_machine(
139
148
  hyperparameters["learning_rate"] = learning_rate
140
149
  hyperparameters["train_size"] = train_size
141
150
  hyperparameters["seed"] = seed
151
+ hyperparameters["L1"] = L1
152
+ hyperparameters["L2"] = L2
142
153
 
143
154
  save_model(
144
155
  filename=filename,
@@ -182,3 +193,44 @@ def get_checkpoints(num_updates: int, n_save: int, spacing: str = "exp") -> np.n
182
193
  )
183
194
  checkpoints = np.unique(np.append(checkpoints, num_updates))
184
195
  return checkpoints
196
+
197
+
198
+ def initialize_model_archive(
199
+ args: dict,
200
+ model_type: str,
201
+ train_dataset: RBMDataset,
202
+ test_dataset: Optional[RBMDataset],
203
+ dtype: torch.dtype,
204
+ ):
205
+ num_visibles = train_dataset.get_num_visibles()
206
+ args = set_args_default(args=args, default_args=default_args)
207
+ rng = np.random.default_rng(args["seed"])
208
+ if test_dataset is None:
209
+ train_dataset, _ = train_dataset.split_train_test(
210
+ rng, args["train_size"], args["test_size"]
211
+ )
212
+ params = map_model[model_type].init_parameters(
213
+ num_hiddens=args["num_hiddens"],
214
+ dataset=train_dataset,
215
+ device=args["device"],
216
+ dtype=dtype,
217
+ )
218
+
219
+ if isinstance(params, PBRBM):
220
+ ensure_zero_sum_gauge(params)
221
+ create_machine(
222
+ filename=args["filename"],
223
+ params=params,
224
+ num_visibles=num_visibles,
225
+ num_hiddens=args["num_hiddens"],
226
+ num_chains=args["num_chains"],
227
+ batch_size=args["batch_size"],
228
+ gibbs_steps=args["gibbs_steps"],
229
+ learning_rate=args["learning_rate"],
230
+ train_size=args["train_size"],
231
+ log=args["log"],
232
+ flags=["checkpoint"],
233
+ seed=args["seed"],
234
+ L1=args["L1"],
235
+ L2=args["L2"],
236
+ )
@@ -152,6 +152,9 @@ def sample_args(tmp_path):
152
152
  "seed": pytest.SEED,
153
153
  "train_size": 0.6,
154
154
  "test_size": None,
155
+ "no_center": False,
156
+ "L1": 0.0,
157
+ "L2": 1.0,
155
158
  }
156
159
 
157
160
 
@@ -39,6 +39,8 @@ def test_create_load_machine(tmp_path, sample_params_class_bbrbm):
39
39
  log=True,
40
40
  flags=["test"],
41
41
  seed=pytest.SEED,
42
+ L1=1.0,
43
+ L2=0.0,
42
44
  )
43
45
 
44
46
  # Check if the file was created
@@ -87,6 +89,8 @@ def test_create_load_machine_dtype(tmp_path, sample_params_class_bbrbm):
87
89
  log=True,
88
90
  flags=["test"],
89
91
  seed=pytest.SEED,
92
+ L1=0.0,
93
+ L2=1.0,
90
94
  )
91
95
 
92
96
  # Check if the file was created
@@ -35,6 +35,8 @@ def test_create_load_machine(tmp_path, sample_params_class_pbrbm):
35
35
  log=True,
36
36
  flags=["test"],
37
37
  seed=pytest.SEED,
38
+ L1=1.0,
39
+ L2=0.0,
38
40
  )
39
41
 
40
42
  # Check if the file was created
@@ -83,6 +85,8 @@ def test_create_load_machine_dtype(tmp_path, sample_params_class_pbrbm):
83
85
  log=True,
84
86
  flags=["test"],
85
87
  seed=pytest.SEED,
88
+ L1=0.0,
89
+ L2=1.0,
86
90
  )
87
91
 
88
92
  # Check if the file was created
@@ -18,6 +18,7 @@ def test_train_bbrbm(sample_dataset_bbrbm, sample_args):
18
18
  model_type = "BBRBM"
19
19
  train(
20
20
  dataset,
21
+ test_dataset,
21
22
  model_type,
22
23
  sample_args,
23
24
  torch.float32,
@@ -54,12 +55,14 @@ def test_train_bbrbm(sample_dataset_bbrbm, sample_args):
54
55
 
55
56
  def test_train_pbrbm(sample_dataset_pbrbm, sample_args):
56
57
  dataset = sample_dataset_pbrbm
58
+ test_dataset = sample_dataset_pbrbm
57
59
  checkpoints = np.arange(1, sample_args["num_updates"] + 1)
58
60
  sample_args["restore"] = False
59
61
  sample_args["batch_size"] = pytest.NUM_SAMPLES
60
62
  model_type = "PBRBM"
61
63
  train(
62
64
  dataset,
65
+ test_dataset,
63
66
  model_type,
64
67
  sample_args,
65
68
  torch.float32,
@@ -57,6 +57,9 @@ def test_use_case_train_bbrbm():
57
57
  "binarize": False,
58
58
  "overwrite": True,
59
59
  "seed": 42,
60
+ "no_center": False,
61
+ "L1": 0.0,
62
+ "L2": 1.0,
60
63
  }
61
64
  train_rbm(args)
62
65
 
@@ -57,6 +57,9 @@ def test_use_case_train_pbrbm_no_weights():
57
57
  "binarize": False,
58
58
  "overwrite": True,
59
59
  "seed": 42,
60
+ "no_center": False,
61
+ "L1": 0.0,
62
+ "L2": 1.0,
60
63
  }
61
64
  train_rbm(args)
62
65
 
@@ -130,6 +133,9 @@ def test_use_case_train_pbrbm_weights():
130
133
  "binarize": False,
131
134
  "overwrite": True,
132
135
  "seed": 42,
136
+ "no_center": False,
137
+ "L1": 1.0,
138
+ "L2": 0.0,
133
139
  }
134
140
  train_rbm(args)
135
141
 
@@ -1,100 +0,0 @@
1
- from pathlib import Path
2
- from typing import List, Optional
3
-
4
- import numpy as np
5
- import torch
6
-
7
- from rbms.dataset.dataset_class import RBMDataset
8
- from rbms.dataset.load_fasta import load_FASTA
9
- from rbms.dataset.load_h5 import load_HDF5
10
- from rbms.dataset.utils import get_subset_labels
11
-
12
-
13
- def load_dataset(
14
- dataset_name: str,
15
- subset_labels: Optional[List[int]] = None,
16
- use_weights: bool = False,
17
- train_size: float = 0.6,
18
- test_size: Optional[float] = None,
19
- binarize: bool = False,
20
- alphabet="protein",
21
- seed: int = 19023741073419046239412739401234901,
22
- device: str = "cpu",
23
- dtype: torch.dtype = torch.float32,
24
- ):
25
- rng = np.random.default_rng(seed)
26
- data = None
27
- weights = None
28
- names = None
29
- labels = None
30
- is_binary = True
31
-
32
- dataset_name = Path(dataset_name)
33
-
34
- match dataset_name.suffix:
35
- case ".h5":
36
- data, labels = load_HDF5(filename=dataset_name, binarize=binarize)
37
- case ".fasta":
38
- data, weights, names = load_FASTA(
39
- filename=dataset_name,
40
- binarize=binarize,
41
- use_weights=use_weights,
42
- alphabet=alphabet,
43
- device=device,
44
- )
45
- if not binarize:
46
- is_binary = False
47
- case _:
48
- raise ValueError(
49
- """
50
- Dataset could not be loaded as the type is not recognized.
51
- It should be either:
52
- - '.h5',
53
- - '.fasta'
54
- """
55
- )
56
-
57
- # Select subset of dataset w.r.t. labels
58
- if subset_labels is not None and labels is not None:
59
- data, labels = get_subset_labels(data, labels, subset_labels)
60
-
61
- if weights is None:
62
- weights = np.ones(data.shape[0])
63
- if names is None:
64
- names = np.arange(data.shape[0])
65
- if labels is None:
66
- labels = -np.ones(data.shape[0])
67
-
68
- # Shuffle dataset
69
- permutation_index = rng.permutation(data.shape[0])
70
-
71
- # Split train/test
72
- train_size = int(train_size * data.shape[0])
73
- if test_size is not None:
74
- test_size = int(test_size * data.shape[0])
75
- else:
76
- test_size = data.shape[0] - train_size
77
-
78
- train_dataset = RBMDataset(
79
- data=data[permutation_index[:train_size]],
80
- labels=labels[permutation_index[:train_size]],
81
- weights=weights[permutation_index[:train_size]],
82
- names=names[permutation_index[:train_size]],
83
- dataset_name=dataset_name,
84
- is_binary=is_binary,
85
- device=device,
86
- dtype=dtype,
87
- )
88
- test_dataset = None
89
- if test_size > 0:
90
- test_dataset = RBMDataset(
91
- data=data[permutation_index[train_size : train_size + test_size]],
92
- labels=labels[permutation_index[train_size : train_size + test_size]],
93
- weights=weights[permutation_index[train_size : train_size + test_size]],
94
- names=names[permutation_index[train_size : train_size + test_size]],
95
- dataset_name=dataset_name,
96
- is_binary=is_binary,
97
- device=device,
98
- dtype=dtype,
99
- )
100
- return train_dataset, test_dataset
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes