braintools 0.0.1__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,467 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ import math
4
+
5
+ import braincore as bc
6
+ import jax.numpy as jnp
7
+ import numpy as np
8
+
9
+ from ._base import Initializer
10
+
11
+ __all__ = [
12
+ 'Normal',
13
+ 'TruncatedNormal',
14
+ 'Uniform',
15
+ 'VarianceScaling',
16
+ 'KaimingUniform',
17
+ 'KaimingNormal',
18
+ 'XavierUniform',
19
+ 'XavierNormal',
20
+ 'LecunUniform',
21
+ 'LecunNormal',
22
+ 'Orthogonal',
23
+ 'DeltaOrthogonal',
24
+ ]
25
+
26
+
27
+ def calculate_gain(nonlinearity, param=None):
28
+ r"""Return the recommended gain value for the given nonlinearity function.
29
+ The values are as follows:
30
+
31
+ ================= ====================================================
32
+ nonlinearity gain
33
+ ================= ====================================================
34
+ Linear / Identity :math:`1`
35
+ Conv{1,2,3}D :math:`1`
36
+ Sigmoid :math:`1`
37
+ Tanh :math:`\frac{5}{3}`
38
+ ReLU :math:`\sqrt{2}`
39
+ Leaky Relu :math:`\sqrt{\frac{2}{1 + \text{negative\_slope}^2}}`
40
+ SELU :math:`\frac{3}{4}`
41
+ ================= ====================================================
42
+
43
+ .. warning::
44
+ In order to implement `Self-Normalizing Neural Networks`_ ,
45
+ you should use ``nonlinearity='linear'`` instead of ``nonlinearity='selu'``.
46
+ This gives the initial weights a variance of ``1 / N``,
47
+ which is necessary to induce a stable fixed point in the forward pass.
48
+ In contrast, the default gain for ``SELU`` sacrifices the normalisation
49
+ effect for more stable gradient flow in rectangular layers.
50
+
51
+ Args:
52
+ nonlinearity: the non-linear function (`nn.functional` name)
53
+ param: optional parameter for the non-linear function
54
+
55
+ .. _Self-Normalizing Neural Networks: https://papers.nips.cc/paper/2017/hash/5d44ee6f2c3f71b73125876103c8f6c4-Abstract.html
56
+ """
57
+ linear_fns = ['linear', 'conv1d', 'conv2d', 'conv3d', 'conv_transpose1d', 'conv_transpose2d', 'conv_transpose3d']
58
+ if nonlinearity in linear_fns or nonlinearity == 'sigmoid':
59
+ return 1
60
+ elif nonlinearity == 'tanh':
61
+ return 5.0 / 3
62
+ elif nonlinearity == 'relu':
63
+ return math.sqrt(2.0)
64
+ elif nonlinearity == 'leaky_relu':
65
+ if param is None:
66
+ negative_slope = 0.01
67
+ elif not isinstance(param, bool) and isinstance(param, int) or isinstance(param, float):
68
+ # True/False are instances of int, hence check above
69
+ negative_slope = param
70
+ else:
71
+ raise ValueError("negative_slope {} not a valid number".format(param))
72
+ return math.sqrt(2.0 / (1 + negative_slope ** 2))
73
+ elif nonlinearity == 'selu':
74
+ return 3.0 / 4
75
+ else:
76
+ raise ValueError("Unsupported nonlinearity {}".format(nonlinearity))
77
+
78
+
79
+ def _format_shape(shape):
80
+ if isinstance(shape, int):
81
+ return (shape,)
82
+ if len(shape) == 0:
83
+ raise ValueError('Please provide shape.')
84
+ if len(shape) == 1:
85
+ if isinstance(shape[0], (tuple, list)):
86
+ return shape[0]
87
+ else:
88
+ return shape
89
+ else:
90
+ return shape
91
+
92
+
93
+ def _compute_fans(shape, in_axis=-2, out_axis=-1):
94
+ receptive_field_size = np.prod(shape) / shape[in_axis] / shape[out_axis]
95
+ fan_in = shape[in_axis] * receptive_field_size
96
+ fan_out = shape[out_axis] * receptive_field_size
97
+ return fan_in, fan_out
98
+
99
+
100
+ class Normal(Initializer):
101
+ """Initialize weights with normal distribution.
102
+
103
+ Parameters
104
+ ----------
105
+ scale : float
106
+ The gain of the derivation of the normal distribution.
107
+
108
+ """
109
+
110
+ def __init__(self, mean=0., scale=1., dtype=None):
111
+ super(Normal, self).__init__()
112
+ self.scale = scale
113
+ self.mean = mean
114
+ self.dtype = dtype or bc.environ.dftype()
115
+
116
+ def __call__(self, *shape):
117
+ weights = bc.random.normal(size=shape, loc=self.mean, scale=self.scale, dtype=self.dtype)
118
+ return weights
119
+
120
+ def __repr__(self):
121
+ return f'{self.__class__.__name__}(scale={self.scale}, dtype={self.dtype})'
122
+
123
+
124
+ class TruncatedNormal(Initializer):
125
+ """Initialize weights with truncated normal distribution.
126
+
127
+ Parameters
128
+ ----------
129
+ loc : float, ndarray
130
+ Mean ("centre") of the distribution before truncating. Note that
131
+ the mean of the truncated distribution will not be exactly equal
132
+ to ``loc``.
133
+ scale : float
134
+ The standard deviation of the normal distribution before truncating.
135
+ lower : float, ndarray
136
+ A float or array of floats representing the lower bound for
137
+ truncation. Must be broadcast-compatible with ``upper``.
138
+ upper : float, ndarray
139
+ A float or array of floats representing the upper bound for
140
+ truncation. Must be broadcast-compatible with ``lower``.
141
+
142
+ """
143
+
144
+ def __init__(self, loc=0., scale=1., lower=None, upper=None, dtype=None):
145
+ super(TruncatedNormal, self).__init__()
146
+ assert scale > 0, '`scale` must be positive.'
147
+ self.scale = scale
148
+ self.loc = loc
149
+ self.lower = lower
150
+ self.upper = upper
151
+ self.dtype = dtype or bc.environ.dftype()
152
+
153
+ def __call__(self, *shape):
154
+ weights = bc.random.truncated_normal(
155
+ size=shape,
156
+ scale=self.scale,
157
+ lower=self.lower,
158
+ upper=self.upper,
159
+ loc=self.loc,
160
+ dtype=self.dtype
161
+ )
162
+ return weights
163
+
164
+ def __repr__(self):
165
+ return (f'{self.__class__.__name__}(loc={self.loc}, scale={self.scale}, '
166
+ f'lower={self.lower}, upper={self.upper}, dtype={self.dtype})')
167
+
168
+
169
+ class Gamma(Initializer):
170
+ """Initialize weights with Gamma distribution.
171
+
172
+ Parameters
173
+ ----------
174
+ shape: float, Array
175
+ Shape parameter.
176
+ scale: float, Array
177
+ The gain of the derivation of the Gamma distribution.
178
+
179
+ """
180
+
181
+ def __init__(self, shape, scale=None, dtype=None):
182
+ self.shape = shape
183
+ self.scale = scale
184
+ self.dtype = dtype or bc.environ.dftype()
185
+
186
+ def __call__(self, *shape):
187
+ weights = bc.random.gamma(self.shape, scale=self.scale, size=shape, dtype=self.dtype)
188
+ return weights
189
+
190
+ def __repr__(self):
191
+ return f'{self.__class__.__name__}(shape={self.shape}, scale={self.scale}, dtype={self.dtype})'
192
+
193
+
194
+ class Exponential(Initializer):
195
+ """Initialize weights with Gamma distribution.
196
+
197
+ Parameters
198
+ ----------
199
+ scale: float, Array
200
+ The gain of the derivation of the Exponential distribution.
201
+
202
+ """
203
+
204
+ def __init__(self, scale=None, dtype=None):
205
+ self.scale = scale
206
+ self.dtype = dtype or bc.environ.dftype()
207
+
208
+ def __call__(self, *shape):
209
+ weights = bc.random.exponential(scale=self.scale, size=shape, dtype=self.dtype)
210
+ return weights
211
+
212
+ def __repr__(self):
213
+ return f'{self.__class__.__name__}(scale={self.scale}, dtype={self.dtype})'
214
+
215
+
216
+ class Uniform(Initializer):
217
+ """Initialize weights with uniform distribution.
218
+
219
+ Parameters
220
+ ----------
221
+ min_val : float
222
+ The lower limit of the uniform distribution.
223
+ max_val : float
224
+ The upper limit of the uniform distribution.
225
+ """
226
+
227
+ def __init__(self, min_val: float = 0., max_val: float = 1., dtype=None):
228
+ super(Uniform, self).__init__()
229
+ self.min_val = min_val
230
+ self.max_val = max_val
231
+ self.dtype = dtype or bc.environ.dftype()
232
+
233
+ def __call__(self, *shape):
234
+ return bc.random.uniform(low=self.min_val, high=self.max_val, size=shape, dtype=self.dtype)
235
+
236
+ def __repr__(self):
237
+ return (f'{self.__class__.__name__}(min_val={self.min_val}, '
238
+ f'max_val={self.max_val}, dtype={self.dtype})')
239
+
240
+
241
+ class VarianceScaling(Initializer):
242
+ def __init__(
243
+ self,
244
+ scale: float,
245
+ mode: str,
246
+ distribution: str,
247
+ in_axis: int = -2,
248
+ out_axis: int = -1,
249
+ dtype=None
250
+ ):
251
+ assert mode in ['fan_in', 'fan_out', 'fan_avg']
252
+ assert distribution in ['truncated_normal', 'normal', 'uniform']
253
+ self.scale = scale
254
+ self.mode = mode
255
+ self.in_axis = in_axis
256
+ self.out_axis = out_axis
257
+ self.distribution = distribution
258
+ self.dtype = dtype or bc.environ.dftype()
259
+
260
+ def __call__(self, *shape):
261
+ fan_in, fan_out = _compute_fans(shape, in_axis=self.in_axis, out_axis=self.out_axis)
262
+ if self.mode == "fan_in":
263
+ denominator = fan_in
264
+ elif self.mode == "fan_out":
265
+ denominator = fan_out
266
+ elif self.mode == "fan_avg":
267
+ denominator = (fan_in + fan_out) / 2
268
+ else:
269
+ raise ValueError("invalid mode for variance scaling initializer: {}".format(self.mode))
270
+ variance = (self.scale / denominator).astype(self.dtype)
271
+ if self.distribution == "truncated_normal":
272
+ stddev = (jnp.sqrt(variance) / .87962566103423978).astype(self.dtype)
273
+ res = bc.random.truncated_normal(-2, 2, shape, dtype=self.dtype) * stddev
274
+ elif self.distribution == "normal":
275
+ res = bc.random.randn(*shape, dtype=self.dtype) * jnp.sqrt(variance).astype(self.dtype)
276
+ elif self.distribution == "uniform":
277
+ res = (bc.random.uniform(low=-1, high=1, size=shape, dtype=self.dtype) *
278
+ jnp.sqrt(3 * variance).astype(self.dtype))
279
+ else:
280
+ raise ValueError("invalid distribution for variance scaling initializer")
281
+ return res
282
+
283
+ def __repr__(self):
284
+ name = self.__class__.__name__
285
+ blank = ' ' * len(name)
286
+ return (f'{name}(scale={self.scale}, mode={self.mode}, in_axis={self.in_axis}, \n'
287
+ f'{blank}out_axis={self.out_axis}, distribution={self.distribution}, dtype={self.dtype})')
288
+
289
+
290
+ class KaimingUniform(VarianceScaling):
291
+ def __init__(
292
+ self,
293
+ scale: float = 2.0,
294
+ mode: str = "fan_in",
295
+ distribution: str = "uniform",
296
+ in_axis: int = -2,
297
+ out_axis: int = -1,
298
+ dtype=None
299
+ ):
300
+ super().__init__(scale,
301
+ mode,
302
+ distribution,
303
+ in_axis=in_axis,
304
+ out_axis=out_axis,
305
+ dtype=dtype)
306
+
307
+
308
+ class KaimingNormal(VarianceScaling):
309
+ def __init__(
310
+ self,
311
+ scale: float = 2.0,
312
+ mode: str = "fan_in",
313
+ distribution: str = "truncated_normal",
314
+ in_axis: int = -2,
315
+ out_axis: int = -1,
316
+ dtype=None
317
+ ):
318
+ super().__init__(scale,
319
+ mode,
320
+ distribution,
321
+ in_axis=in_axis,
322
+ out_axis=out_axis,
323
+ dtype=dtype)
324
+
325
+
326
+ class XavierUniform(VarianceScaling):
327
+ def __init__(
328
+ self,
329
+ scale: float = 1.0,
330
+ mode: str = "fan_avg",
331
+ distribution: str = "uniform",
332
+ in_axis: int = -2,
333
+ out_axis: int = -1,
334
+ dtype=None
335
+ ):
336
+ super().__init__(scale,
337
+ mode,
338
+ distribution,
339
+ in_axis=in_axis,
340
+ out_axis=out_axis,
341
+ dtype=dtype)
342
+
343
+
344
+ class XavierNormal(VarianceScaling):
345
+ def __init__(
346
+ self,
347
+ scale: float = 1.0,
348
+ mode: str = "fan_avg",
349
+ distribution: str = "truncated_normal",
350
+ in_axis: int = -2,
351
+ out_axis: int = -1,
352
+ dtype=None
353
+ ):
354
+ super().__init__(scale,
355
+ mode,
356
+ distribution,
357
+ in_axis=in_axis,
358
+ out_axis=out_axis,
359
+ dtype=dtype)
360
+
361
+
362
+ class LecunUniform(VarianceScaling):
363
+ def __init__(
364
+ self,
365
+ scale: float = 1.0,
366
+ mode: str = "fan_in",
367
+ distribution: str = "uniform",
368
+ in_axis: int = -2,
369
+ out_axis: int = -1,
370
+ dtype=None
371
+ ):
372
+ super().__init__(scale,
373
+ mode,
374
+ distribution,
375
+ in_axis=in_axis,
376
+ out_axis=out_axis,
377
+ dtype=dtype)
378
+
379
+
380
+ class LecunNormal(VarianceScaling):
381
+ def __init__(
382
+ self,
383
+ scale: float = 1.0,
384
+ mode: str = "fan_in",
385
+ distribution: str = "truncated_normal",
386
+ in_axis: int = -2,
387
+ out_axis: int = -1,
388
+ dtype=None
389
+ ):
390
+ super().__init__(scale,
391
+ mode,
392
+ distribution,
393
+ in_axis=in_axis,
394
+ out_axis=out_axis,
395
+ dtype=dtype)
396
+
397
+
398
+ class Orthogonal(Initializer):
399
+ """
400
+ Construct an initializer for uniformly distributed orthogonal matrices.
401
+
402
+ If the shape is not square, the matrix will have orthonormal rows or columns
403
+ depending on which side is smaller.
404
+ """
405
+
406
+ def __init__(
407
+ self,
408
+ scale: float = 1.,
409
+ axis: int = -1,
410
+ dtype=None
411
+ ):
412
+ super().__init__()
413
+ self.scale = scale
414
+ self.axis = axis
415
+ self.dtype = dtype or bc.environ.dftype()
416
+
417
+ def __call__(self, *shape):
418
+ n_rows = shape[self.axis]
419
+ n_cols = np.prod(shape) // n_rows
420
+ matrix_shape = (n_rows, n_cols) if n_rows > n_cols else (n_cols, n_rows)
421
+ norm_dst = bc.random.normal(size=matrix_shape, dtype=self.dtype)
422
+ q_mat, r_mat = jnp.linalg.qr(norm_dst)
423
+ # Enforce Q is uniformly distributed
424
+ q_mat *= jnp.sign(jnp.diag(r_mat))
425
+ if n_rows < n_cols:
426
+ q_mat = q_mat.T
427
+ q_mat = jnp.reshape(q_mat, (n_rows,) + tuple(np.delete(shape, self.axis)))
428
+ q_mat = jnp.moveaxis(q_mat, 0, self.axis)
429
+ return jnp.asarray(self.scale, dtype=self.dtype) * q_mat
430
+
431
+ def __repr__(self):
432
+ return f'{self.__class__.__name__}(scale={self.scale}, axis={self.axis}, dtype={self.dtype})'
433
+
434
+
435
+ class DeltaOrthogonal(Initializer):
436
+ """
437
+ Construct an initializer for delta orthogonal kernels; see arXiv:1806.05393.
438
+
439
+ The shape must be 3D, 4D or 5D.
440
+ """
441
+
442
+ def __init__(self, scale=1.0, axis=-1, dtype=None):
443
+ super(DeltaOrthogonal, self).__init__()
444
+ self.scale = scale
445
+ self.axis = axis
446
+ self.dtype = dtype or bc.environ.dftype()
447
+
448
+ def __call__(self, *shape):
449
+ if len(shape) not in [3, 4, 5]:
450
+ raise ValueError("Delta orthogonal initializer requires a 3D, 4D or 5D shape.")
451
+ if shape[-1] < shape[-2]:
452
+ raise ValueError("`fan_in` must be less or equal than `fan_out`. ")
453
+ ortho_matrix = Orthogonal(scale=self.scale, axis=self.axis, dtype=self.dtype)(*shape[-2:])
454
+ W = jnp.zeros(shape, dtype=self.dtype)
455
+ if len(shape) == 3:
456
+ k = shape[0]
457
+ W = W.at[(k - 1) // 2].set(ortho_matrix)
458
+ elif len(shape) == 4:
459
+ k1, k2 = shape[:2]
460
+ W = W.at[(k1 - 1) // 2, (k2 - 1) // 2].set(ortho_matrix)
461
+ else:
462
+ k1, k2, k3 = shape[:3]
463
+ W = W.at[(k1 - 1) // 2, (k2 - 1) // 2, (k3 - 1) // 2].set(ortho_matrix)
464
+ return W
465
+
466
+ def __repr__(self):
467
+ return f'{self.__class__.__name__}(scale={self.scale}, axis={self.axis}, dtype={self.dtype})'
@@ -0,0 +1,91 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ import braincore as bc
4
+ import jax.numpy as jnp
5
+
6
+ from ._base import Initializer
7
+
8
+ __all__ = [
9
+ 'ZeroInit',
10
+ 'Constant',
11
+ 'Identity',
12
+ ]
13
+
14
+
15
+ class ZeroInit(Initializer):
16
+ """Zero initializer.
17
+
18
+ Initialize the weights with zeros.
19
+ """
20
+
21
+ def __init__(self, dtype=None):
22
+ super(ZeroInit, self).__init__()
23
+ self.dtype = dtype or bc.environ.dftype()
24
+
25
+ def __call__(self, *shape):
26
+ return jnp.zeros(shape, dtype=self.dtype)
27
+
28
+ def __repr__(self):
29
+ return f"{self.__class__.__name__}(dtype={self.dtype})"
30
+
31
+
32
+ class Constant(Initializer):
33
+ """Constant initializer.
34
+
35
+ Initialize the weights with the given values.
36
+
37
+ Parameters
38
+ ----------
39
+ value : float, int, bm.ndarray
40
+ The value to specify.
41
+ """
42
+
43
+ def __init__(self, value=1., dtype=None):
44
+ super(Constant, self).__init__()
45
+ self.dtype = dtype or bc.environ.dftype()
46
+ self.value = jnp.asarray(value, dtype=self.dtype)
47
+
48
+ def __call__(self, *shape):
49
+ return jnp.full(shape, self.value, dtype=self.dtype)
50
+
51
+ def __repr__(self):
52
+ return f'{self.__class__.__name__}(value={self.value}, dtype={self.dtype})'
53
+
54
+
55
+ class Identity(Initializer):
56
+ """Returns the identity matrix.
57
+
58
+ This initializer was proposed in (Le, et al., 2015) [1]_.
59
+
60
+ Parameters
61
+ ----------
62
+ value : float
63
+ The optional scaling factor.
64
+
65
+ Returns
66
+ -------
67
+ shape: tuple of int
68
+ The weight shape/size.
69
+
70
+ References
71
+ ----------
72
+ .. [1] Le, Quoc V., Navdeep Jaitly, and Geoffrey E. Hinton. "A simple way to
73
+ initialize recurrent networks of rectified linear units." arXiv preprint
74
+ arXiv:1504.00941 (2015).
75
+ """
76
+
77
+ def __init__(self, value=1., dtype=None):
78
+ super(Identity, self).__init__()
79
+ self.dtype = dtype or bc.environ.dftype()
80
+ self.value = jnp.asarray(value, dtype=self.dtype)
81
+
82
+ def __call__(self, *shape):
83
+ if isinstance(shape, (tuple, list)):
84
+ if len(shape) > 2:
85
+ raise ValueError(f'Only support initialize 2D weights for {self.__class__.__name__}.')
86
+ r = jnp.eye(*shape, dtype=self.dtype)
87
+ r = jnp.fill_diagonal(r, self.value)
88
+ return r
89
+
90
+ def __repr__(self):
91
+ return f'{self.__class__.__name__}(value={self.value}, dtype={self.dtype})'
@@ -0,0 +1,12 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+
4
+ """
5
+ This module provides various methods to form current inputs.
6
+ You can access them through ``brainpy.inputs.XXX``.
7
+ """
8
+
9
+ from .currents import *
10
+ from .currents import __all__ as currents_all
11
+
12
+ __all__ = currents_all