torchness 2.1.1__tar.gz → 2.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. {torchness-2.1.1 → torchness-2.2.0}/PKG-INFO +1 -1
  2. {torchness-2.1.1 → torchness-2.2.0}/setup.py +1 -1
  3. torchness-2.2.0/tests/test_scaled.py +106 -0
  4. {torchness-2.1.1 → torchness-2.2.0}/torchness/motorch.py +1 -1
  5. torchness-2.2.0/torchness/scaled.py +112 -0
  6. {torchness-2.1.1 → torchness-2.2.0}/torchness.egg-info/PKG-INFO +1 -1
  7. {torchness-2.1.1 → torchness-2.2.0}/torchness.egg-info/SOURCES.txt +2 -2
  8. torchness-2.1.1/tests/test_scaled_LR.py +0 -16
  9. torchness-2.1.1/torchness/scaled_LR.py +0 -90
  10. {torchness-2.1.1 → torchness-2.2.0}/README.md +0 -0
  11. {torchness-2.1.1 → torchness-2.2.0}/setup.cfg +0 -0
  12. {torchness-2.1.1 → torchness-2.2.0}/tests/test_batcher.py +0 -0
  13. {torchness-2.1.1 → torchness-2.2.0}/tests/test_ckpt.py +0 -0
  14. {torchness-2.1.1 → torchness-2.2.0}/tests/test_devices.py +0 -0
  15. {torchness-2.1.1 → torchness-2.2.0}/tests/test_encoders.py +0 -0
  16. {torchness-2.1.1 → torchness-2.2.0}/tests/test_grad_clipping.py +0 -0
  17. {torchness-2.1.1 → torchness-2.2.0}/tests/test_initialize.py +0 -0
  18. {torchness-2.1.1 → torchness-2.2.0}/tests/test_layers.py +0 -0
  19. {torchness-2.1.1 → torchness-2.2.0}/tests/test_motorch.py +0 -0
  20. {torchness-2.1.1 → torchness-2.2.0}/tests/test_tbwr.py +0 -0
  21. {torchness-2.1.1 → torchness-2.2.0}/tests/test_tools.py +0 -0
  22. {torchness-2.1.1 → torchness-2.2.0}/tests/test_zeroes_processor.py +0 -0
  23. {torchness-2.1.1 → torchness-2.2.0}/torchness/__init__.py +0 -0
  24. {torchness-2.1.1 → torchness-2.2.0}/torchness/base.py +0 -0
  25. {torchness-2.1.1 → torchness-2.2.0}/torchness/batcher.py +0 -0
  26. {torchness-2.1.1 → torchness-2.2.0}/torchness/ckpt.py +0 -0
  27. {torchness-2.1.1 → torchness-2.2.0}/torchness/devices.py +0 -0
  28. {torchness-2.1.1 → torchness-2.2.0}/torchness/encoders.py +0 -0
  29. {torchness-2.1.1 → torchness-2.2.0}/torchness/grad_clipping.py +0 -0
  30. {torchness-2.1.1 → torchness-2.2.0}/torchness/initialize.py +0 -0
  31. {torchness-2.1.1 → torchness-2.2.0}/torchness/layers.py +0 -0
  32. {torchness-2.1.1 → torchness-2.2.0}/torchness/models/__init__.py +0 -0
  33. {torchness-2.1.1 → torchness-2.2.0}/torchness/models/simple_feats_classifier.py +0 -0
  34. {torchness-2.1.1 → torchness-2.2.0}/torchness/models/simple_text_classifier.py +0 -0
  35. {torchness-2.1.1 → torchness-2.2.0}/torchness/models/text_embbeder.py +0 -0
  36. {torchness-2.1.1 → torchness-2.2.0}/torchness/nnmodule.py +0 -0
  37. {torchness-2.1.1 → torchness-2.2.0}/torchness/onnx_export.py +0 -0
  38. {torchness-2.1.1 → torchness-2.2.0}/torchness/tbwr.py +0 -0
  39. {torchness-2.1.1 → torchness-2.2.0}/torchness/tools.py +0 -0
  40. {torchness-2.1.1 → torchness-2.2.0}/torchness/zeroes_processor.py +0 -0
  41. {torchness-2.1.1 → torchness-2.2.0}/torchness.egg-info/dependency_links.txt +0 -0
  42. {torchness-2.1.1 → torchness-2.2.0}/torchness.egg-info/requires.txt +0 -0
  43. {torchness-2.1.1 → torchness-2.2.0}/torchness.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: torchness
3
- Version: 2.1.1
3
+ Version: 2.2.0
4
4
  Summary: PyTorch tools
5
5
  Home-page: https://github.com/piteren/torchness.git
6
6
  Author: Piotr Niewinski
@@ -5,7 +5,7 @@ from pypaq.lipytools.files import get_requirements
5
5
 
6
6
  setup(
7
7
  name= 'torchness',
8
- version= 'v2.1.1',
8
+ version= 'v2.2.0',
9
9
  url= 'https://github.com/piteren/torchness.git',
10
10
  author= 'Piotr Niewinski',
11
11
  author_email= 'pioniewinski@gmail.com',
@@ -0,0 +1,106 @@
1
+ import math
2
+
3
+ import numpy as np
4
+ import pytest
5
+ import torch
6
+
7
+ from torchness.layers import LayDense
8
+ from torchness.scaled import ScaledLR, abc_profile
9
+
10
+
11
+ def test_base():
12
+ lay = LayDense(2, 2)
13
+ optimizer = torch.optim.AdamW(params=lay.parameters(), lr=1e-4)
14
+ scaler = ScaledLR(optimizer=optimizer)
15
+ lr = scaler.get_lr()[0]
16
+ print(lr)
17
+ optimizer.step()
18
+ scaler.step()
19
+ lr = scaler.get_lr()[0]
20
+ print(lr)
21
+
22
+
23
+ def test_anneal_base():
24
+ """Former scaled.py __main__ demo: exponential annealing for several anneal_base values."""
25
+ rng = 100_000
26
+ warmup_end = 1000
27
+ anneal_start = 10_000
28
+ lr0 = 1e-4
29
+ for ab in [0.999, 0.9999, 0.99999]:
30
+ lay = LayDense(2, 2)
31
+ optimizer = torch.optim.AdamW(params=lay.parameters(), lr=lr0)
32
+ scaler = ScaledLR(
33
+ optimizer=optimizer,
34
+ warmup_end=warmup_end,
35
+ anneal_start=anneal_start,
36
+ anneal_base=ab,
37
+ )
38
+ lrs = []
39
+ for _ in range(rng):
40
+ lrs.append(scaler.get_lr()[0])
41
+ scaler.step()
42
+ lrs = np.asarray(lrs)
43
+
44
+ # warm-up rising, flat until anneal_start, then decaying
45
+ assert np.all(np.diff(lrs[:warmup_end - 1]) > 0)
46
+ assert np.allclose(lrs[warmup_end:anneal_start - 1], lr0)
47
+ assert np.all(np.diff(lrs[anneal_start + 1:]) < 0)
48
+
49
+ # final value follows lr0 * ab ** (steps since anneal_start)
50
+ last_step = scaler._step - 1 # step at which lrs[-1] was computed
51
+ expected = lr0 * ab ** (last_step - anneal_start)
52
+ assert lrs[-1] == pytest.approx(expected, rel=1e-6)
53
+ print(f"anneal_base: {ab} final lr: {lrs[-1]}")
54
+
55
+
56
+ KW = dict(
57
+ a_value_start=0.0, a_value_end=1.0, b_value_end=0.8, c_value_end=0.1,
58
+ a_step_start=10, a_step_end=20, b_step_end=60, c_step_end=100,
59
+ )
60
+
61
+
62
+ def val(step, **over):
63
+ return float(abc_profile(step, **{**KW, **over}))
64
+
65
+
66
+ def test_boundaries_and_outside():
67
+ assert val(0) == pytest.approx(0.0)
68
+ assert val(10) == pytest.approx(0.0)
69
+ assert val(20) == pytest.approx(1.0)
70
+ assert val(60) == pytest.approx(0.8)
71
+ assert val(100) == pytest.approx(0.1)
72
+ assert val(1000) == pytest.approx(0.1)
73
+
74
+
75
+ def test_stage_shapes():
76
+ # A cosine: r=0.5 -> midpoint, r=0.25 -> 0.5*(1-cos(pi/4))
77
+ assert val(15) == pytest.approx(0.5)
78
+ assert val(12.5) == pytest.approx(0.5 * (1 - math.cos(math.pi / 4)))
79
+ # B linear: r=0.25 -> 1.0 - 0.25*0.2
80
+ assert val(30) == pytest.approx(0.95)
81
+ # C cosine midpoint
82
+ assert val(80) == pytest.approx(0.45)
83
+ # flags switch shapes
84
+ assert val(12.5, a_cosine=False) == pytest.approx(0.25)
85
+ assert val(30, b_cosine=True) == pytest.approx(1.0 - 0.2 * 0.5 * (1 - math.cos(math.pi / 4)))
86
+
87
+
88
+ def test_vectorised_monotone_continuous():
89
+ steps = torch.arange(0, 120, 0.5)
90
+ v = abc_profile(steps, **KW)
91
+ assert v.shape == steps.shape
92
+ d = v[1:] - v[:-1]
93
+ assert torch.all(d[steps[:-1] < 20] >= -1e-12) # A rising
94
+ assert torch.all(d[steps[:-1] >= 20] <= 1e-12) # B, C falling
95
+ # continuity at stage boundaries
96
+ eps = 1e-6
97
+ for s in (10, 20, 60, 100):
98
+ assert val(s - eps) == pytest.approx(val(s + eps), abs=1e-4)
99
+
100
+
101
+ def test_zero_length_stages():
102
+ kw = dict(a_step_start=10, a_step_end=10, b_step_end=10, c_step_end=50)
103
+ assert val(9, **kw) == pytest.approx(0.0)
104
+ assert val(10, **kw) == pytest.approx(0.8) # A and B collapsed -> starts C at b_value_end
105
+ assert val(50, **kw) == pytest.approx(0.1)
106
+
@@ -14,7 +14,7 @@ from torchness.base import TNS, DTNS, NPL
14
14
  from torchness.tools import accuracy, f1
15
15
  from torchness.devices import get_devices
16
16
  from torchness.ckpt import mrg_ckpts
17
- from torchness.scaled_LR import ScaledLR
17
+ from torchness.scaled import ScaledLR
18
18
  from torchness.grad_clipping import GradClipperMAVG
19
19
  from torchness.tbwr import TBwr
20
20
 
@@ -0,0 +1,112 @@
1
+ import logging
2
+ import numpy as np
3
+ from pypaq.lipytools.printout import nice_scin
4
+ import torch
5
+
6
+ logger = logging.getLogger(__name__)
7
+
8
+
9
+ class ScaledLR(torch.optim.lr_scheduler.LRScheduler):
10
+ """ Applies warm-up and annealing for LR of 0 group.
11
+ ScaledLR.step() should be called every step / batch update"""
12
+
13
+ def __init__(
14
+ self,
15
+ optimizer,
16
+ step: int = 0,
17
+ warmup_end: int | None = 1000,
18
+ anneal_start: int | None = 10_000,
19
+ anneal_base: float = 0.999,
20
+ anneal_mul: float = 1.0,
21
+ last_epoch = -1,
22
+ ):
23
+ self._step = step
24
+ self.w_end = warmup_end
25
+ self.a_start = anneal_start
26
+ self.a_base = anneal_base
27
+ self.a_mul = anneal_mul
28
+
29
+ super(ScaledLR, self).__init__(optimizer, last_epoch)
30
+
31
+ def update_base_lr0(self, lr: float):
32
+ """ updates LR of group 0 """
33
+ self.base_lrs[0] = lr
34
+
35
+ def get_lr(self) -> list[float]:
36
+
37
+ lrs = np.asarray(self.base_lrs)
38
+ if self.w_end and self._step < self.w_end:
39
+ w_ratio = self._step / self.w_end
40
+ lrs *= w_ratio
41
+ logger.debug(f'current warm-up ratio:{w_ratio}')
42
+
43
+ if self.a_start is not None and self.a_base != 1.0:
44
+ a_steps = max(0, self._step - self.a_start)
45
+ if a_steps > 0:
46
+ factor = self.a_base ** (a_steps * self.a_mul)
47
+ lrs *= factor
48
+ logger.debug(f'current annealing factor:{nice_scin(factor)}')
49
+
50
+ logger.debug(f'ScaledLR scheduler step:{self._step}, resulting LR:{nice_scin(lrs[0])}')
51
+ return lrs.tolist()
52
+
53
+ def step(self, epoch: int | None = None):
54
+ super(ScaledLR, self).step(epoch)
55
+ self._step += 1
56
+
57
+
58
+ def _interp(
59
+ step: torch.Tensor,
60
+ v0: float,
61
+ v1: float,
62
+ s0: float,
63
+ s1: float,
64
+ cosine: bool,
65
+ ) -> torch.Tensor:
66
+ """Interpolates from v0 (at s0) to v1 (at s1), clamped outside [s0, s1].
67
+ cosine=True uses half-cosine easing: v0 + (v1 - v0) * 0.5 * (1 - cos(pi * r)).
68
+ Zero-length range (s0 == s1) gives a jump: v0 for step < s1, v1 for step >= s1."""
69
+ length = s1 - s0
70
+ if length > 0:
71
+ r = torch.clamp((step - s0) / length, 0.0, 1.0)
72
+ else:
73
+ r = (step >= s1).to(step.dtype)
74
+ if cosine:
75
+ r = 0.5 * (1.0 - torch.cos(torch.pi * r))
76
+ return v0 + (v1 - v0) * r
77
+
78
+
79
+ def abc_profile(
80
+ step: int | float | torch.Tensor,
81
+ a_value_start: float,
82
+ a_value_end: float,
83
+ b_value_end: float,
84
+ c_value_end: float,
85
+ a_step_start: float,
86
+ a_step_end: float,
87
+ b_step_end: float,
88
+ c_step_end: float,
89
+ a_cosine: bool = True,
90
+ b_cosine: bool = False,
91
+ c_cosine: bool = True,
92
+ ) -> torch.Tensor:
93
+ """3-stage (A,B,C) value profile
94
+
95
+ step < a_step_start -> a_value_start
96
+ A: a_step_start .. a_step_end -> a_value_start .. a_value_end (cosine by default)
97
+ B: a_step_end .. b_step_end -> a_value_end .. b_value_end (linear by default)
98
+ C: b_step_end .. c_step_end -> b_value_end .. c_value_end (cosine by default)
99
+ step > c_step_end -> c_value_end
100
+
101
+ typical LR use:
102
+ A = linear warm-up (0 -> lr)
103
+ B = plateau (lr -> lr)
104
+ C = cosine decay (lr -> lr_min)
105
+ zero-length stages are allowed (value jumps at the boundary)
106
+ step may be a scalar or a tensor of steps (vectorized, shape preserved)
107
+ returns a float64 tensor"""
108
+ step = torch.as_tensor(step, dtype=torch.float64)
109
+ va = _interp(step, a_value_start, a_value_end, a_step_start, a_step_end, a_cosine)
110
+ vb = _interp(step, a_value_end, b_value_end, a_step_end, b_step_end, b_cosine)
111
+ vc = _interp(step, b_value_end, c_value_end, b_step_end, c_step_end, c_cosine)
112
+ return torch.where(step < a_step_end, va, torch.where(step < b_step_end, vb, vc))
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: torchness
3
- Version: 2.1.1
3
+ Version: 2.2.0
4
4
  Summary: PyTorch tools
5
5
  Home-page: https://github.com/piteren/torchness.git
6
6
  Author: Piotr Niewinski
@@ -8,7 +8,7 @@ tests/test_grad_clipping.py
8
8
  tests/test_initialize.py
9
9
  tests/test_layers.py
10
10
  tests/test_motorch.py
11
- tests/test_scaled_LR.py
11
+ tests/test_scaled.py
12
12
  tests/test_tbwr.py
13
13
  tests/test_tools.py
14
14
  tests/test_zeroes_processor.py
@@ -24,7 +24,7 @@ torchness/layers.py
24
24
  torchness/motorch.py
25
25
  torchness/nnmodule.py
26
26
  torchness/onnx_export.py
27
- torchness/scaled_LR.py
27
+ torchness/scaled.py
28
28
  torchness/tbwr.py
29
29
  torchness/tools.py
30
30
  torchness/zeroes_processor.py
@@ -1,16 +0,0 @@
1
- import torch
2
-
3
- from torchness.scaled_LR import ScaledLR
4
- from torchness.layers import LayDense
5
-
6
-
7
- def test_base():
8
- lay = LayDense(2, 2)
9
- optimizer = torch.optim.AdamW(params=lay.parameters(), lr=1e-4)
10
- scaler = ScaledLR(optimizer=optimizer)
11
- lr = scaler.get_lr()[0]
12
- print(lr)
13
- optimizer.step()
14
- scaler.step()
15
- lr = scaler.get_lr()[0]
16
- print(lr)
@@ -1,90 +0,0 @@
1
- import logging
2
- import numpy as np
3
- from pypaq.lipytools.printout import nice_scin
4
- import torch
5
-
6
- logger = logging.getLogger(__name__)
7
-
8
-
9
- class ScaledLR(torch.optim.lr_scheduler.LRScheduler):
10
- """ Applies warm-up and annealing for LR of 0 group.
11
- ScaledLR.step() should be called every update / batch / step """
12
-
13
- def __init__(
14
- self,
15
- optimizer,
16
- step: int = 0,
17
- warmup_end: int | None = 1000,
18
- anneal_start: int | None = 10_000,
19
- anneal_base: float = 0.999,
20
- anneal_mul: float = 1.0,
21
- last_epoch = -1,
22
- ):
23
- self._step = step
24
- self.w_end = warmup_end
25
- self.a_start = anneal_start
26
- self.a_base = anneal_base
27
- self.a_mul = anneal_mul
28
-
29
- super(ScaledLR, self).__init__(optimizer, last_epoch)
30
-
31
- def update_base_lr0(self, lr: float):
32
- """ updates LR of group 0 """
33
- self.base_lrs[0] = lr
34
-
35
- def get_lr(self) -> list[float]:
36
-
37
- lrs = np.asarray(self.base_lrs)
38
- if self.w_end and self._step < self.w_end:
39
- w_ratio = self._step / self.w_end
40
- lrs *= w_ratio
41
- logger.debug(f'current warm-up ratio:{w_ratio}')
42
-
43
- if self.a_start is not None and self.a_base != 1.0:
44
- a_steps = max(0, self._step - self.a_start)
45
- if a_steps > 0:
46
- factor = self.a_base ** (a_steps * self.a_mul)
47
- lrs *= factor
48
- logger.debug(f'current annealing factor:{nice_scin(factor)}')
49
-
50
- logger.debug(f'ScaledLR scheduler step:{self._step}, resulting LR:{nice_scin(lrs[0])}')
51
- return lrs.tolist()
52
-
53
- def step(self, epoch: int | None = None):
54
- super(ScaledLR, self).step(epoch)
55
- self._step += 1
56
-
57
-
58
- if __name__ == '__main__':
59
-
60
- from pypaq.lipytools.plots import two_dim_multi
61
- from torchness.layers import LayDense
62
-
63
- lay = LayDense(2, 2)
64
- optimizer = torch.optim.AdamW(params=lay.parameters(), lr=1e-4)
65
-
66
- rng = 1_000_000
67
- xs = list(range(rng))
68
- names = []
69
- lrs = []
70
- for ab in [
71
- 0.999,
72
- 0.9999,
73
- 0.99999,
74
- ]:
75
-
76
- scaler = ScaledLR(
77
- optimizer=optimizer,
78
- anneal_base=ab,
79
- )
80
- names.append(f'anneal_base: {ab}')
81
-
82
- _lrs = []
83
- for x in range(rng):
84
- _lrs.append(scaler.get_lr()[0])
85
- scaler.step()
86
- lrs.append(_lrs)
87
-
88
- two_dim_multi(ys=lrs, names=names, ylogscale=True)
89
- for _lrs,name in zip(lrs, names):
90
- print(f"{name} final lr: {_lrs[-1]}")
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes