kalfa 3.0.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kalfa/__init__.py +19 -0
- kalfa/api.py +284 -0
- kalfa/board/__init__.py +1 -0
- kalfa/board/answers.py +419 -0
- kalfa/board/server.py +536 -0
- kalfa/board/static/api.js +176 -0
- kalfa/board/static/app.js +122 -0
- kalfa/board/static/board.css +228 -0
- kalfa/board/static/charts.css +46 -0
- kalfa/board/static/components/badge.js +14 -0
- kalfa/board/static/components/chart.js +204 -0
- kalfa/board/static/components/diagram.js +857 -0
- kalfa/board/static/components/error.js +17 -0
- kalfa/board/static/components/loading.js +7 -0
- kalfa/board/static/components/progress.js +25 -0
- kalfa/board/static/components/settings.js +418 -0
- kalfa/board/static/components/spark.js +43 -0
- kalfa/board/static/components/table.js +156 -0
- kalfa/board/static/diagram.css +76 -0
- kalfa/board/static/format.js +138 -0
- kalfa/board/static/index.html +34 -0
- kalfa/board/static/plot.js +594 -0
- kalfa/board/static/predictions.css +20 -0
- kalfa/board/static/store.js +215 -0
- kalfa/board/static/views/compare.js +185 -0
- kalfa/board/static/views/home.js +90 -0
- kalfa/board/static/views/record/config.js +81 -0
- kalfa/board/static/views/record/curves.js +137 -0
- kalfa/board/static/views/record/data.js +378 -0
- kalfa/board/static/views/record/files.js +128 -0
- kalfa/board/static/views/record/log.js +164 -0
- kalfa/board/static/views/record/model.js +286 -0
- kalfa/board/static/views/record/overview.js +216 -0
- kalfa/board/static/views/record/plots.js +103 -0
- kalfa/board/static/views/record/predictions.js +834 -0
- kalfa/board/static/views/record.js +179 -0
- kalfa/board/static/views/runs.js +124 -0
- kalfa/board/static/views/sweep.js +282 -0
- kalfa/board/watch.py +87 -0
- kalfa/cli.py +371 -0
- kalfa/commands/__init__.py +0 -0
- kalfa/commands/common.py +8 -0
- kalfa/commands/export.py +117 -0
- kalfa/commands/generate.py +31 -0
- kalfa/commands/guide/__init__.py +101 -0
- kalfa/commands/guide/about.py +93 -0
- kalfa/commands/guide/calls.py +102 -0
- kalfa/commands/guide/sections.py +186 -0
- kalfa/commands/guide/seen.py +307 -0
- kalfa/commands/guide/show.py +106 -0
- kalfa/commands/guide/training.py +333 -0
- kalfa/commands/guide/words.py +97 -0
- kalfa/commands/legos.py +508 -0
- kalfa/commands/plots.py +27 -0
- kalfa/commands/report/__init__.py +64 -0
- kalfa/commands/report/page.py +112 -0
- kalfa/commands/report/predictions.py +216 -0
- kalfa/commands/report/sections.py +356 -0
- kalfa/commands/stop.py +60 -0
- kalfa/commands/sweep/__init__.py +202 -0
- kalfa/commands/sweep/checks.py +98 -0
- kalfa/commands/sweep/plan.py +183 -0
- kalfa/commands/sweep/points.py +123 -0
- kalfa/commands/sweep/runner.py +100 -0
- kalfa/commands/sweep/space.py +67 -0
- kalfa/commands/sweep/strategies.py +228 -0
- kalfa/core/__init__.py +0 -0
- kalfa/core/aliases.py +229 -0
- kalfa/core/build.py +727 -0
- kalfa/core/check.py +550 -0
- kalfa/core/coerce.py +36 -0
- kalfa/core/context.py +149 -0
- kalfa/core/device.py +63 -0
- kalfa/core/errors.py +45 -0
- kalfa/core/graph.py +304 -0
- kalfa/core/history.py +70 -0
- kalfa/core/loop.py +710 -0
- kalfa/core/make.py +121 -0
- kalfa/core/plan.py +156 -0
- kalfa/core/predict.py +290 -0
- kalfa/core/protocols.py +122 -0
- kalfa/core/record.py +347 -0
- kalfa/core/refs.py +88 -0
- kalfa/core/rng.py +100 -0
- kalfa/core/rules.py +513 -0
- kalfa/core/run.py +568 -0
- kalfa/core/spec.py +413 -0
- kalfa/core/state.py +100 -0
- kalfa/core/wrap.py +671 -0
- kalfa/records/__init__.py +24 -0
- kalfa/records/compare.py +86 -0
- kalfa/records/reader.py +561 -0
- kalfa/records/stats/__init__.py +36 -0
- kalfa/records/stats/choices.py +282 -0
- kalfa/records/stats/classes.py +236 -0
- kalfa/records/stats/measures.py +144 -0
- kalfa/records/stats/pairs.py +225 -0
- kalfa/records/stats/tables.py +153 -0
- kalfa/records/sweep.py +244 -0
- kalfa/std/__init__.py +17 -0
- kalfa/std/actions.py +35 -0
- kalfa/std/common.py +56 -0
- kalfa/std/data/__init__.py +1 -0
- kalfa/std/data/base.py +45 -0
- kalfa/std/data/columns.py +190 -0
- kalfa/std/data/flow.py +317 -0
- kalfa/std/data/images.py +443 -0
- kalfa/std/data/options.py +147 -0
- kalfa/std/data/pre.py +84 -0
- kalfa/std/data/splits.py +140 -0
- kalfa/std/data/stream.py +294 -0
- kalfa/std/data/table.py +642 -0
- kalfa/std/data/text.py +436 -0
- kalfa/std/data/transforms.py +327 -0
- kalfa/std/data/weights.py +113 -0
- kalfa/std/data/window.py +135 -0
- kalfa/std/figure.py +37 -0
- kalfa/std/init.py +72 -0
- kalfa/std/loggers.py +105 -0
- kalfa/std/losses.py +161 -0
- kalfa/std/metrics.py +463 -0
- kalfa/std/models.py +122 -0
- kalfa/std/notes.py +1553 -0
- kalfa/std/objectives.py +360 -0
- kalfa/std/plots/__init__.py +75 -0
- kalfa/std/plots/classification.py +207 -0
- kalfa/std/plots/common.py +44 -0
- kalfa/std/plots/curves.py +56 -0
- kalfa/std/plots/images.py +188 -0
- kalfa/std/plots/pairs.py +66 -0
- kalfa/std/plots/predictions.py +154 -0
- kalfa/std/samplers.py +261 -0
- kalfa/std/samples.py +72 -0
- kalfa/std/schedules.py +74 -0
- kalfa/std/steps.py +133 -0
- kalfa/std/triggers.py +135 -0
- kalfa/summary.py +296 -0
- kalfa/tui/__init__.py +4 -0
- kalfa/tui/app.py +107 -0
- kalfa/tui/config.py +61 -0
- kalfa/tui/guide.py +100 -0
- kalfa/tui/jobs.py +97 -0
- kalfa/tui/legos.py +62 -0
- kalfa/tui/records.py +439 -0
- kalfa-3.0.0.dev0.dist-info/METADATA +261 -0
- kalfa-3.0.0.dev0.dist-info/RECORD +148 -0
- kalfa-3.0.0.dev0.dist-info/WHEEL +4 -0
- kalfa-3.0.0.dev0.dist-info/entry_points.txt +2 -0
kalfa/core/context.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import time
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
|
|
5
|
+
from .errors import KalfaError, RunError
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def as_list(value):
|
|
9
|
+
if value is None or value is False:
|
|
10
|
+
return []
|
|
11
|
+
if isinstance(value, str):
|
|
12
|
+
return [value]
|
|
13
|
+
return list(value)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def train_of(tree):
|
|
17
|
+
settings = getattr(tree, "settings", None)
|
|
18
|
+
train = settings.get("train") if isinstance(settings, dict) else None
|
|
19
|
+
return train if isinstance(train, dict) else {}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass
|
|
23
|
+
class TrainSettings:
|
|
24
|
+
epochs: int | None = None
|
|
25
|
+
updates: dict | None = None
|
|
26
|
+
predicts: str | None = None
|
|
27
|
+
evaluate: list = field(default_factory=list)
|
|
28
|
+
checkpoint: dict | None = None
|
|
29
|
+
report: str = "last"
|
|
30
|
+
track: list = field(default_factory=list)
|
|
31
|
+
log: list = field(default_factory=list)
|
|
32
|
+
|
|
33
|
+
@classmethod
|
|
34
|
+
def from_mapping(cls, mapping):
|
|
35
|
+
mapping = dict(mapping or {})
|
|
36
|
+
epochs = mapping.get("epochs")
|
|
37
|
+
updates = mapping.get("updates")
|
|
38
|
+
checkpoint = mapping.get("checkpoint")
|
|
39
|
+
return cls(
|
|
40
|
+
epochs=None if epochs is None else int(epochs),
|
|
41
|
+
updates=None if updates is None else dict(updates),
|
|
42
|
+
predicts=mapping.get("predicts"),
|
|
43
|
+
evaluate=as_list(mapping.get("evaluate", ["valid", "test"])),
|
|
44
|
+
checkpoint=dict(checkpoint) if isinstance(checkpoint, dict) else checkpoint,
|
|
45
|
+
report=str(mapping.get("report") or "last"),
|
|
46
|
+
track=as_list(mapping.get("track")),
|
|
47
|
+
log=as_list(mapping.get("log")),
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def unit(self):
|
|
52
|
+
return "updates" if self.updates else "epochs"
|
|
53
|
+
|
|
54
|
+
def total_turns(self):
|
|
55
|
+
if self.updates:
|
|
56
|
+
total = int(self.updates["total"])
|
|
57
|
+
per_turn = int(self.updates.get("turn", total))
|
|
58
|
+
return max(1, -(-total // per_turn))
|
|
59
|
+
return int(self.epochs or 0)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class Context:
|
|
63
|
+
def __init__(self, tree, plan=None):
|
|
64
|
+
self.tree = tree
|
|
65
|
+
self.config = tree.config
|
|
66
|
+
self.plan = plan
|
|
67
|
+
self.settings = train_of(tree)
|
|
68
|
+
self.train = TrainSettings.from_mapping(self.settings)
|
|
69
|
+
self.data = None
|
|
70
|
+
self.models = {}
|
|
71
|
+
self.losses = {}
|
|
72
|
+
self.metrics = {}
|
|
73
|
+
self.optimizers = {}
|
|
74
|
+
self.step = None
|
|
75
|
+
self.rules = []
|
|
76
|
+
self.plots = {}
|
|
77
|
+
self.generate = None
|
|
78
|
+
self.loggers = []
|
|
79
|
+
self.device = None
|
|
80
|
+
self.rng = None
|
|
81
|
+
self.seed = 0
|
|
82
|
+
self.record = None
|
|
83
|
+
self.history = []
|
|
84
|
+
self.turn = 0
|
|
85
|
+
self.updates = 0
|
|
86
|
+
self.batches = 0
|
|
87
|
+
self.set_name = None
|
|
88
|
+
self.latest = {}
|
|
89
|
+
self.fired = []
|
|
90
|
+
self.stopping = None
|
|
91
|
+
self.stopped_by = None
|
|
92
|
+
self.stop_noted = False
|
|
93
|
+
self.first_calls = None
|
|
94
|
+
self.monitor = None
|
|
95
|
+
self.best = None
|
|
96
|
+
self.cursor = None
|
|
97
|
+
self.last_norms = {}
|
|
98
|
+
self.patches = []
|
|
99
|
+
self.fitted_from = None
|
|
100
|
+
self.want_sets = True
|
|
101
|
+
self.rebuild_failure = None
|
|
102
|
+
self.started = time.perf_counter()
|
|
103
|
+
self.logger = logging.getLogger("kalfa.run")
|
|
104
|
+
|
|
105
|
+
def predicts(self):
|
|
106
|
+
name = self.train.predicts if self.train is not None else (self.settings or {}).get("predicts")
|
|
107
|
+
if name is False:
|
|
108
|
+
return None
|
|
109
|
+
if name:
|
|
110
|
+
name = str(name)
|
|
111
|
+
for prefix in ("@models.", "models."):
|
|
112
|
+
if name.startswith(prefix):
|
|
113
|
+
name = name[len(prefix):]
|
|
114
|
+
entry = self.models.get(name)
|
|
115
|
+
if entry is None:
|
|
116
|
+
raise RunError(f"train.predicts names the model {name!r}; the models are {sorted(self.models)}")
|
|
117
|
+
return entry
|
|
118
|
+
if len(self.models) == 1:
|
|
119
|
+
return next(iter(self.models.values()))
|
|
120
|
+
return None
|
|
121
|
+
|
|
122
|
+
def stop(self, reason=None, by=None):
|
|
123
|
+
self.stopping = reason or "stop"
|
|
124
|
+
if by is not None:
|
|
125
|
+
self.stopped_by = by
|
|
126
|
+
|
|
127
|
+
def elapsed(self):
|
|
128
|
+
return time.perf_counter() - self.started
|
|
129
|
+
|
|
130
|
+
def loaders(self):
|
|
131
|
+
if self.data is None:
|
|
132
|
+
return {}
|
|
133
|
+
try:
|
|
134
|
+
return self.data.loaders()
|
|
135
|
+
except KalfaError:
|
|
136
|
+
if self.rebuild_failure is None:
|
|
137
|
+
raise
|
|
138
|
+
raise RunError(f"the sets of {self.fitted_from} could not be rebuilt: {self.rebuild_failure}") from None
|
|
139
|
+
|
|
140
|
+
def set_modes(self, train):
|
|
141
|
+
train = bool(train)
|
|
142
|
+
for objective in self.losses.values():
|
|
143
|
+
if hasattr(objective, "train"):
|
|
144
|
+
objective.train(train)
|
|
145
|
+
for entry in self.models.values():
|
|
146
|
+
if callable(getattr(entry, "set_mode", None)):
|
|
147
|
+
entry.set_mode(train)
|
|
148
|
+
else:
|
|
149
|
+
entry.module.train(train and entry.freeze is not True)
|
kalfa/core/device.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import contextlib
|
|
2
|
+
|
|
3
|
+
import torch
|
|
4
|
+
|
|
5
|
+
from .errors import RunError
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Device:
|
|
9
|
+
def __init__(self, spec="auto"):
|
|
10
|
+
self.spec = spec
|
|
11
|
+
self.torch = self.pick(spec)
|
|
12
|
+
|
|
13
|
+
@staticmethod
|
|
14
|
+
def pick(spec):
|
|
15
|
+
if isinstance(spec, torch.device):
|
|
16
|
+
return spec
|
|
17
|
+
if isinstance(spec, Device):
|
|
18
|
+
return spec.torch
|
|
19
|
+
if isinstance(spec, int) and not isinstance(spec, bool):
|
|
20
|
+
return torch.device("cuda", spec)
|
|
21
|
+
text = str(spec or "auto").strip().lower()
|
|
22
|
+
if text == "auto":
|
|
23
|
+
if torch.cuda.is_available():
|
|
24
|
+
return torch.device("cuda")
|
|
25
|
+
if torch.backends.mps.is_available():
|
|
26
|
+
return torch.device("mps")
|
|
27
|
+
return torch.device("cpu")
|
|
28
|
+
try:
|
|
29
|
+
return torch.device(text)
|
|
30
|
+
except RuntimeError as error:
|
|
31
|
+
raise RunError(f"device {spec!r} is not understood: {error}") from None
|
|
32
|
+
|
|
33
|
+
@property
|
|
34
|
+
def type(self):
|
|
35
|
+
return self.torch.type
|
|
36
|
+
|
|
37
|
+
def move(self, batch):
|
|
38
|
+
if isinstance(batch, torch.Tensor):
|
|
39
|
+
return batch.to(self.torch, non_blocking=self.type == "cuda")
|
|
40
|
+
if isinstance(batch, dict):
|
|
41
|
+
return {key: self.move(value) for key, value in batch.items()}
|
|
42
|
+
if isinstance(batch, list):
|
|
43
|
+
return [self.move(value) for value in batch]
|
|
44
|
+
if isinstance(batch, tuple):
|
|
45
|
+
return tuple(self.move(value) for value in batch)
|
|
46
|
+
return batch
|
|
47
|
+
|
|
48
|
+
def autocast(self, enabled):
|
|
49
|
+
if not enabled or self.type == "cpu":
|
|
50
|
+
return contextlib.nullcontext()
|
|
51
|
+
return torch.autocast(device_type=self.type)
|
|
52
|
+
|
|
53
|
+
def scaler(self, enabled):
|
|
54
|
+
if enabled and self.type == "cuda":
|
|
55
|
+
return torch.amp.GradScaler("cuda")
|
|
56
|
+
return None
|
|
57
|
+
|
|
58
|
+
def note(self):
|
|
59
|
+
name = torch.cuda.get_device_name(self.torch) if self.type == "cuda" and torch.cuda.is_available() else None
|
|
60
|
+
return {"type": self.type, "name": name, "spec": str(self.spec)}
|
|
61
|
+
|
|
62
|
+
def __str__(self):
|
|
63
|
+
return str(self.torch)
|
kalfa/core/errors.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class KalfaError(Exception):
|
|
5
|
+
pass
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ConfigError(KalfaError):
|
|
9
|
+
pass
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class BuildError(KalfaError):
|
|
13
|
+
pass
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class RunError(KalfaError):
|
|
17
|
+
pass
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class Problem:
|
|
22
|
+
code: str
|
|
23
|
+
message: str
|
|
24
|
+
path: str | None = None
|
|
25
|
+
hint: str | None = None
|
|
26
|
+
severity: str = "error"
|
|
27
|
+
|
|
28
|
+
def __str__(self):
|
|
29
|
+
where = f"{self.path}: " if self.path else ""
|
|
30
|
+
tail = f" ({self.hint})" if self.hint else ""
|
|
31
|
+
return f"{self.severity} {self.code}: {where}{self.message}{tail}"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def error(code, message, path=None, hint=None):
|
|
35
|
+
return Problem(code, message, path, hint, "error")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def warning(code, message, path=None, hint=None):
|
|
39
|
+
return Problem(code, message, path, hint, "warning")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class ProblemsError(KalfaError):
|
|
43
|
+
def __init__(self, problems):
|
|
44
|
+
self.problems = list(problems)
|
|
45
|
+
super().__init__("\n".join(str(problem) for problem in self.problems))
|
kalfa/core/graph.py
ADDED
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
import inspect
|
|
2
|
+
import weakref
|
|
3
|
+
|
|
4
|
+
import torch
|
|
5
|
+
from torch.nn.parameter import is_lazy
|
|
6
|
+
from torch.overrides import TorchFunctionMode, resolve_name
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
PASS = {"view", "view_as", "reshape", "reshape_as", "permute", "transpose", "t", "T", "mT", "H", "mH", "flatten",
|
|
10
|
+
"unflatten", "squeeze", "unsqueeze", "contiguous", "to", "float", "double", "half", "bfloat16", "int", "long",
|
|
11
|
+
"short", "bool", "byte", "char", "type", "type_as", "cpu", "cuda", "detach", "clone", "getitem", "expand",
|
|
12
|
+
"expand_as", "broadcast_to", "narrow", "select", "movedim", "moveaxis", "swapaxes", "swapdims", "split",
|
|
13
|
+
"chunk", "unbind", "tensor_split", "as_strided", "data", "requires_grad", "atleast_1d", "atleast_2d",
|
|
14
|
+
"atleast_3d", "real", "align_to", "rename", "refine_names", "pin_memory", "share_memory"}
|
|
15
|
+
FRESH = {"zeros_like", "ones_like", "empty_like", "full_like", "rand_like", "randn_like", "randint_like", "new_zeros",
|
|
16
|
+
"new_ones", "new_empty", "new_full", "new_tensor", "setitem"}
|
|
17
|
+
BINARY = {"add", "sub", "mul", "matmul", "div", "truediv", "floordiv", "mod", "pow", "and", "or", "xor", "lshift",
|
|
18
|
+
"rshift"}
|
|
19
|
+
RENAMED = {"truediv": "div", "floordiv": "floor_divide"}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def resolved(func):
|
|
23
|
+
try:
|
|
24
|
+
name = resolve_name(func)
|
|
25
|
+
except Exception:
|
|
26
|
+
name = None
|
|
27
|
+
return name or getattr(func, "__name__", None) or ""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def plain(func):
|
|
31
|
+
parts = resolved(func).split(".")
|
|
32
|
+
if parts[-1] in ("__get__", "__set__", "__delete__"):
|
|
33
|
+
if parts[-1] != "__get__" or len(parts) < 2:
|
|
34
|
+
return None
|
|
35
|
+
parts = parts[:-1]
|
|
36
|
+
word = parts[-1]
|
|
37
|
+
if len(word) > 4 and word.startswith("__") and word.endswith("__"):
|
|
38
|
+
word = word[2:-2]
|
|
39
|
+
if word[:1] in ("r", "i") and word[1:] in BINARY:
|
|
40
|
+
word = word[1:]
|
|
41
|
+
elif word.endswith("_") and not word.startswith("_"):
|
|
42
|
+
word = word.rstrip("_")
|
|
43
|
+
return RENAMED.get(word, word) or None
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def short(func):
|
|
47
|
+
name = resolved(func)
|
|
48
|
+
return name[len("torch."):] if name.startswith("torch.") else name
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def tensors_in(value):
|
|
52
|
+
found = []
|
|
53
|
+
stack = [value]
|
|
54
|
+
while stack:
|
|
55
|
+
item = stack.pop()
|
|
56
|
+
if isinstance(item, torch.Tensor):
|
|
57
|
+
found.append(item)
|
|
58
|
+
elif isinstance(item, dict):
|
|
59
|
+
stack.extend(reversed(list(item.values())))
|
|
60
|
+
elif isinstance(item, (tuple, list)):
|
|
61
|
+
stack.extend(reversed(item))
|
|
62
|
+
return found
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def qualified(module):
|
|
66
|
+
return f"{type(module).__module__}.{type(module).__qualname__}"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def dtype_text(value):
|
|
70
|
+
if isinstance(value, torch.Tensor):
|
|
71
|
+
return str(value.dtype).replace("torch.", "")
|
|
72
|
+
return type(value).__name__
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def named(result, names):
|
|
76
|
+
if isinstance(result, torch.Tensor):
|
|
77
|
+
return [(names[0] if names else "output", result)]
|
|
78
|
+
if isinstance(result, dict):
|
|
79
|
+
return [(str(key), value) for key, value in result.items()]
|
|
80
|
+
if isinstance(result, (tuple, list)):
|
|
81
|
+
return [(names[position] if position < len(names) else f"output_{position}", value)
|
|
82
|
+
for position, value in enumerate(result)]
|
|
83
|
+
return []
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class Tracer:
|
|
87
|
+
def __init__(self, module, limit):
|
|
88
|
+
self.root = module
|
|
89
|
+
self.limit = int(limit)
|
|
90
|
+
self.paths = {id(found): name for name, found in module.named_modules()}
|
|
91
|
+
self.sizes = {}
|
|
92
|
+
self.nodes = []
|
|
93
|
+
self.edges = []
|
|
94
|
+
self.links = set()
|
|
95
|
+
self.groups = []
|
|
96
|
+
self.frames = []
|
|
97
|
+
self.producer = {}
|
|
98
|
+
self.counted = 0
|
|
99
|
+
self.truncated = False
|
|
100
|
+
self.batch = None
|
|
101
|
+
self.depth = 0
|
|
102
|
+
self.busy = False
|
|
103
|
+
|
|
104
|
+
def shape(self, value):
|
|
105
|
+
if not isinstance(value, torch.Tensor):
|
|
106
|
+
return None
|
|
107
|
+
shape = [int(size) for size in value.shape]
|
|
108
|
+
if shape and self.batch is not None and shape[0] == self.batch:
|
|
109
|
+
return shape[1:]
|
|
110
|
+
return shape
|
|
111
|
+
|
|
112
|
+
def first_shape(self, value):
|
|
113
|
+
found = tensors_in(value)
|
|
114
|
+
return self.shape(found[0]) if found else None
|
|
115
|
+
|
|
116
|
+
def parameters(self, module):
|
|
117
|
+
key = id(module)
|
|
118
|
+
if key not in self.sizes:
|
|
119
|
+
self.sizes[key] = sum(parameter.numel() for parameter in module.parameters() if not is_lazy(parameter))
|
|
120
|
+
return self.sizes[key]
|
|
121
|
+
|
|
122
|
+
def mark(self, value, node):
|
|
123
|
+
for tensor in tensors_in(value):
|
|
124
|
+
self.producer[id(tensor)] = (node, weakref.ref(tensor))
|
|
125
|
+
|
|
126
|
+
def sources(self, value):
|
|
127
|
+
found = []
|
|
128
|
+
for tensor in tensors_in(value):
|
|
129
|
+
known = self.producer.get(id(tensor))
|
|
130
|
+
if known is not None and known[1]() is tensor:
|
|
131
|
+
found.append((known[0], tensor))
|
|
132
|
+
return found
|
|
133
|
+
|
|
134
|
+
def room(self):
|
|
135
|
+
if self.counted >= self.limit:
|
|
136
|
+
self.truncated = True
|
|
137
|
+
return False
|
|
138
|
+
self.counted += 1
|
|
139
|
+
return True
|
|
140
|
+
|
|
141
|
+
def add(self, kind, label, detail, path=None, cls=None, shape=None, parameters=None, group=None):
|
|
142
|
+
node = {"id": f"n{len(self.nodes)}", "kind": kind, "label": label, "detail": detail, "path": path,
|
|
143
|
+
"class": cls, "shape": shape, "parameters": parameters, "group": group}
|
|
144
|
+
self.nodes.append(node)
|
|
145
|
+
return node["id"]
|
|
146
|
+
|
|
147
|
+
def link(self, found, target):
|
|
148
|
+
for origin, tensor in found:
|
|
149
|
+
if origin == target or (origin, target) in self.links:
|
|
150
|
+
continue
|
|
151
|
+
self.links.add((origin, target))
|
|
152
|
+
self.edges.append({"from": origin, "to": target, "shape": self.shape(tensor)})
|
|
153
|
+
|
|
154
|
+
def group_at(self, position):
|
|
155
|
+
frame = self.frames[position]
|
|
156
|
+
if frame["group"] is None:
|
|
157
|
+
module = frame["module"]
|
|
158
|
+
parent = self.group_below(position)
|
|
159
|
+
frame["group"] = f"g{len(self.groups)}"
|
|
160
|
+
self.groups.append({"id": frame["group"], "path": frame["path"], "class": qualified(module),
|
|
161
|
+
"label": frame["path"], "detail": type(module).__name__,
|
|
162
|
+
"parameters": self.parameters(module), "parent": parent})
|
|
163
|
+
return frame["group"]
|
|
164
|
+
|
|
165
|
+
def group_below(self, position):
|
|
166
|
+
for back in range(position - 1, -1, -1):
|
|
167
|
+
if self.frames[back]["kind"] == "group":
|
|
168
|
+
return self.group_at(back)
|
|
169
|
+
return None
|
|
170
|
+
|
|
171
|
+
def start(self, inputs, names, keywords=None):
|
|
172
|
+
keywords = dict(keywords or {})
|
|
173
|
+
found = tensors_in([inputs, keywords])
|
|
174
|
+
if found and found[0].ndim:
|
|
175
|
+
self.batch = int(found[0].shape[0])
|
|
176
|
+
given = [(names[position] if position < len(names) else f"input_{position}", value)
|
|
177
|
+
for position, value in enumerate(inputs)]
|
|
178
|
+
for name, value in [*given, *((str(key), value) for key, value in keywords.items())]:
|
|
179
|
+
if not tensors_in(value):
|
|
180
|
+
continue
|
|
181
|
+
node = self.add("input", name, dtype_text(value), shape=self.shape(value))
|
|
182
|
+
self.mark(value, node)
|
|
183
|
+
|
|
184
|
+
def frame(self, module, args, kwargs):
|
|
185
|
+
if self.depth:
|
|
186
|
+
self.depth += 1
|
|
187
|
+
return {"module": module, "kind": "inside"}
|
|
188
|
+
path = self.paths.get(id(module))
|
|
189
|
+
if next(module.children(), None) is None:
|
|
190
|
+
self.depth += 1
|
|
191
|
+
return {"module": module, "kind": "leaf", "path": path, "sources": self.sources((args, kwargs))}
|
|
192
|
+
if module is self.root or path is None:
|
|
193
|
+
return {"module": module, "kind": "root"}
|
|
194
|
+
return {"module": module, "kind": "group", "path": path, "group": None}
|
|
195
|
+
|
|
196
|
+
def enter(self, module, args, kwargs):
|
|
197
|
+
self.busy = True
|
|
198
|
+
try:
|
|
199
|
+
self.frames.append(self.frame(module, args, kwargs))
|
|
200
|
+
finally:
|
|
201
|
+
self.busy = False
|
|
202
|
+
|
|
203
|
+
def leave(self, module, args, kwargs, output):
|
|
204
|
+
self.busy = True
|
|
205
|
+
try:
|
|
206
|
+
frame = None
|
|
207
|
+
while self.frames:
|
|
208
|
+
frame = self.frames.pop()
|
|
209
|
+
if frame["kind"] in ("inside", "leaf"):
|
|
210
|
+
self.depth -= 1
|
|
211
|
+
if frame["module"] is module:
|
|
212
|
+
break
|
|
213
|
+
if frame is not None and frame["kind"] == "leaf" and frame["module"] is module:
|
|
214
|
+
self.leaf(module, frame, output)
|
|
215
|
+
finally:
|
|
216
|
+
self.busy = False
|
|
217
|
+
|
|
218
|
+
def leaf(self, module, frame, output):
|
|
219
|
+
if not self.room():
|
|
220
|
+
return
|
|
221
|
+
path = frame["path"]
|
|
222
|
+
node = self.add("module", type(module).__name__, path or "", path=path, cls=qualified(module),
|
|
223
|
+
shape=self.first_shape(output), parameters=self.parameters(module),
|
|
224
|
+
group=self.group_below(len(self.frames)))
|
|
225
|
+
self.link(frame["sources"], node)
|
|
226
|
+
self.mark(output, node)
|
|
227
|
+
|
|
228
|
+
def op(self, func, args, kwargs, result):
|
|
229
|
+
name = plain(func)
|
|
230
|
+
if not name or name in FRESH:
|
|
231
|
+
return
|
|
232
|
+
found = self.sources((args, kwargs))
|
|
233
|
+
if not found or not tensors_in(result):
|
|
234
|
+
return
|
|
235
|
+
if name in PASS:
|
|
236
|
+
self.mark(result, found[0][0])
|
|
237
|
+
return
|
|
238
|
+
if not self.room():
|
|
239
|
+
return
|
|
240
|
+
node = self.add("op", name, short(func), shape=self.first_shape(result),
|
|
241
|
+
group=self.group_below(len(self.frames)))
|
|
242
|
+
self.link(found, node)
|
|
243
|
+
self.mark(result, node)
|
|
244
|
+
|
|
245
|
+
def finish(self, result, names):
|
|
246
|
+
for name, value in named(result, names):
|
|
247
|
+
if not tensors_in(value):
|
|
248
|
+
continue
|
|
249
|
+
node = self.add("output", name, dtype_text(value), shape=self.shape(value))
|
|
250
|
+
self.link(self.sources(value), node)
|
|
251
|
+
|
|
252
|
+
def graph(self):
|
|
253
|
+
return {"method": "eager", "truncated": self.truncated, "nodes": self.nodes, "edges": self.edges,
|
|
254
|
+
"groups": self.groups}
|
|
255
|
+
|
|
256
|
+
def release(self):
|
|
257
|
+
self.producer = {}
|
|
258
|
+
self.frames = []
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
class Watch(TorchFunctionMode):
|
|
262
|
+
def __init__(self, tracer):
|
|
263
|
+
super().__init__()
|
|
264
|
+
self.tracer = tracer
|
|
265
|
+
|
|
266
|
+
def __torch_function__(self, func, types, args=(), kwargs=None):
|
|
267
|
+
kwargs = kwargs or {}
|
|
268
|
+
result = func(*args, **kwargs)
|
|
269
|
+
tracer = self.tracer
|
|
270
|
+
if not tracer.busy and not tracer.depth:
|
|
271
|
+
tracer.op(func, args, kwargs, result)
|
|
272
|
+
return result
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def trace(module, inputs, input_names, output_names, limit=300, keywords=None):
|
|
276
|
+
tracer = Tracer(module, limit)
|
|
277
|
+
handles = []
|
|
278
|
+
try:
|
|
279
|
+
for found in module.modules():
|
|
280
|
+
handles.append(found.register_forward_pre_hook(tracer.enter, with_kwargs=True))
|
|
281
|
+
handles.append(found.register_forward_hook(tracer.leave, with_kwargs=True))
|
|
282
|
+
inputs = tuple(inputs)
|
|
283
|
+
keywords = dict(keywords or {})
|
|
284
|
+
tracer.start(inputs, list(input_names or []), keywords)
|
|
285
|
+
with torch.no_grad(), Watch(tracer):
|
|
286
|
+
result = module(*inputs, **keywords)
|
|
287
|
+
tracer.finish(result, list(output_names or []))
|
|
288
|
+
return tracer.graph()
|
|
289
|
+
except Exception:
|
|
290
|
+
return None
|
|
291
|
+
finally:
|
|
292
|
+
for handle in handles:
|
|
293
|
+
handle.remove()
|
|
294
|
+
tracer.release()
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def argument_names(module, count):
|
|
298
|
+
try:
|
|
299
|
+
signature = inspect.signature(module.forward)
|
|
300
|
+
except (TypeError, ValueError):
|
|
301
|
+
return [f"input_{position}" for position in range(count)]
|
|
302
|
+
kinds = (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)
|
|
303
|
+
names = [name for name, parameter in signature.parameters.items() if parameter.kind in kinds]
|
|
304
|
+
return [names[position] if position < len(names) else f"input_{position}" for position in range(count)]
|
kalfa/core/history.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import torch
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
PREFIX = {"train": "train", "valid": "val", "test": "test"}
|
|
5
|
+
SKIPPED = ("turn", "update", "seconds", "rules")
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def as_float(value):
|
|
9
|
+
if isinstance(value, torch.Tensor):
|
|
10
|
+
return float(value.detach())
|
|
11
|
+
return float(value)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Means:
|
|
15
|
+
def __init__(self):
|
|
16
|
+
self.sums = {}
|
|
17
|
+
self.counts = {}
|
|
18
|
+
|
|
19
|
+
def record(self, name, value, size):
|
|
20
|
+
terms = value if isinstance(value, dict) else {"loss": value}
|
|
21
|
+
for term, item in terms.items():
|
|
22
|
+
if item is None:
|
|
23
|
+
continue
|
|
24
|
+
key = name if term == "loss" else f"{name}/{term}"
|
|
25
|
+
self.sums[key] = self.sums.get(key, 0.0) + as_float(item) * size
|
|
26
|
+
self.counts[key] = self.counts.get(key, 0) + size
|
|
27
|
+
|
|
28
|
+
def result(self):
|
|
29
|
+
return {key: total / self.counts[key] for key, total in self.sums.items() if self.counts[key]}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def prefixed(results):
|
|
33
|
+
merged = {}
|
|
34
|
+
for set_name, values in results.items():
|
|
35
|
+
prefix = PREFIX.get(set_name, set_name)
|
|
36
|
+
for name, value in (values or {}).items():
|
|
37
|
+
merged[f"{prefix}/{name}"] = value
|
|
38
|
+
return merged
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def history_line(ctx, metrics, seconds):
|
|
42
|
+
line = {"turn": int(ctx.turn), "update": int(ctx.updates)}
|
|
43
|
+
line.update(metrics or {})
|
|
44
|
+
for name, entry in ctx.optimizers.items():
|
|
45
|
+
line[f"lr/{name}"] = entry.rate()
|
|
46
|
+
for label, rate in entry.rates().items():
|
|
47
|
+
line[f"lr/{name}/{label}"] = rate
|
|
48
|
+
for name, entry in ctx.optimizers.items():
|
|
49
|
+
if entry.minimizes is not None:
|
|
50
|
+
line[f"minimizes/{name}"] = entry.minimizes.name
|
|
51
|
+
line["seconds"] = round(float(seconds), 3)
|
|
52
|
+
line["rules"] = list(ctx.fired)
|
|
53
|
+
return line
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def progress_text(line, total=None, count=4):
|
|
57
|
+
turn = line.get("turn")
|
|
58
|
+
parts = [f"turn {turn}/{total}" if total else f"turn {turn}"]
|
|
59
|
+
seconds = line.get("seconds")
|
|
60
|
+
if isinstance(seconds, (int, float)):
|
|
61
|
+
parts.append(f"{float(seconds):.1f}s")
|
|
62
|
+
shown = 0
|
|
63
|
+
for key, value in line.items():
|
|
64
|
+
if key in SKIPPED or isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
65
|
+
continue
|
|
66
|
+
parts.append(f"{key} {value:.4g}")
|
|
67
|
+
shown += 1
|
|
68
|
+
if shown == count:
|
|
69
|
+
break
|
|
70
|
+
return " ".join(parts)
|