authtracker 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- authtracker/__init__.py +16 -0
- authtracker/backends/__init__.py +95 -0
- authtracker/backends/base.py +136 -0
- authtracker/backends/local.py +200 -0
- authtracker/backends/mlflow_backend.py +310 -0
- authtracker/backends/resilient.py +329 -0
- authtracker/backends/wandb_backend.py +235 -0
- authtracker/bundle.py +887 -0
- authtracker/capture.py +175 -0
- authtracker/cli.py +51 -0
- authtracker/configfile.py +84 -0
- authtracker/dataset.py +98 -0
- authtracker/errors.py +18 -0
- authtracker/registry.py +118 -0
- authtracker/run.py +346 -0
- authtracker/settings.py +171 -0
- authtracker/simple.py +241 -0
- authtracker/testing.py +82 -0
- authtracker/utils.py +321 -0
- authtracker/watch.py +59 -0
- authtracker-0.0.1.dist-info/METADATA +215 -0
- authtracker-0.0.1.dist-info/RECORD +25 -0
- authtracker-0.0.1.dist-info/WHEEL +4 -0
- authtracker-0.0.1.dist-info/entry_points.txt +2 -0
- authtracker-0.0.1.dist-info/licenses/LICENSE +202 -0
authtracker/bundle.py
ADDED
|
@@ -0,0 +1,887 @@
|
|
|
1
|
+
"""A PyTorch model as a plain folder you can read, run and reproduce.
|
|
2
|
+
|
|
3
|
+
at.save(net, flavor="bundle", init_args={"hidden": 64})
|
|
4
|
+
|
|
5
|
+
stores this folder with the run (and `at.register()` can version it):
|
|
6
|
+
|
|
7
|
+
architecture.py the file that defines your model class (and any other code you list)
|
|
8
|
+
weights.pth the weights, as a plain state_dict (loaded with weights_only=True)
|
|
9
|
+
inference.py a sample inference script. It does not need authtracker
|
|
10
|
+
requirements.txt pinned versions of torch and the packages your code imports
|
|
11
|
+
environment.txt every package installed when it was saved
|
|
12
|
+
bundle.json what to import, how to build it, file hashes, python / torch / cuda versions
|
|
13
|
+
|
|
14
|
+
`at.load(name, alias)` (or `at.load_bundle(folder)`) imports the class from the folder, builds it with
|
|
15
|
+
`init_args`, loads the weights and returns the real `torch.nn.Module`. There is no MLflow model
|
|
16
|
+
format and no pickle of your model: the weights are a state_dict, the architecture is source code.
|
|
17
|
+
|
|
18
|
+
Loading runs the architecture file, so only load bundles you trust.
|
|
19
|
+
|
|
20
|
+
Two optional extras, for when you want the exact object back (like fastai's `export.pkl`):
|
|
21
|
+
|
|
22
|
+
at.save(net, flavor="bundle", pickle=True) # the folder above, plus model.pkl
|
|
23
|
+
at.save(net, flavor="pickle") # only model.pkl, requirements and the environment
|
|
24
|
+
at.load("name", "production", pickle=True) # the exact object, attributes and all
|
|
25
|
+
|
|
26
|
+
A pickle runs code when it is loaded, and only loads on the same Python and torch versions, so it is
|
|
27
|
+
opt-in on both ends. The hashes in bundle.json are checked before anything is unpickled.
|
|
28
|
+
"""
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import ast
|
|
32
|
+
import contextlib
|
|
33
|
+
import hashlib
|
|
34
|
+
import importlib
|
|
35
|
+
import importlib.metadata
|
|
36
|
+
import inspect
|
|
37
|
+
import json
|
|
38
|
+
import linecache
|
|
39
|
+
import logging
|
|
40
|
+
import os
|
|
41
|
+
import pickle as _pickle
|
|
42
|
+
import platform
|
|
43
|
+
import re
|
|
44
|
+
import shutil
|
|
45
|
+
import subprocess
|
|
46
|
+
import symtable
|
|
47
|
+
import sys
|
|
48
|
+
import sysconfig
|
|
49
|
+
import textwrap
|
|
50
|
+
import tempfile
|
|
51
|
+
import time
|
|
52
|
+
import types
|
|
53
|
+
from pathlib import Path
|
|
54
|
+
from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Union
|
|
55
|
+
|
|
56
|
+
from .errors import ValidationError
|
|
57
|
+
from .utils import container_info, git_info, requirements_text
|
|
58
|
+
|
|
59
|
+
_log = logging.getLogger("authtracker")
|
|
60
|
+
|
|
61
|
+
FLAVOR = "bundle" # the value to pass as at.save(model, flavor=...). Rename it here.
|
|
62
|
+
PICKLE_FLAVOR = "pickle" # at.save(model, flavor="pickle"): only the pickled object
|
|
63
|
+
MODEL_TYPE = "authtracker-bundle" # recorded on the stored model so it can be recognised when loading
|
|
64
|
+
MANIFEST = "bundle.json"
|
|
65
|
+
WEIGHTS = "weights.pth"
|
|
66
|
+
INFERENCE = "inference.py"
|
|
67
|
+
REQUIREMENTS = "requirements.txt"
|
|
68
|
+
ENVIRONMENT = "environment.txt"
|
|
69
|
+
PICKLE = "model.pkl"
|
|
70
|
+
FORMAT_VERSION = 1
|
|
71
|
+
_RESERVED = {MANIFEST, WEIGHTS, INFERENCE, REQUIREMENTS, ENVIRONMENT, PICKLE}
|
|
72
|
+
TOKEN = "$BUNDLE" # in init_args: "the folder this bundle is in", filled in at load
|
|
73
|
+
ASSETS = "assets"
|
|
74
|
+
_CARRY = "_authtracker_bundle" # set on a loaded model: its code and arguments, so it can be saved again
|
|
75
|
+
_CARRY_LIMIT = 5 * 1024 * 1024 # carry the code only if it is small
|
|
76
|
+
|
|
77
|
+
PathLike = Union[str, Path]
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _torch():
|
|
81
|
+
try:
|
|
82
|
+
import torch
|
|
83
|
+
except ImportError as e: # pragma: no cover
|
|
84
|
+
raise ValidationError("A PyTorch bundle needs torch: pip install torch") from e
|
|
85
|
+
return torch
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
# ---------------------------------------------------------------------------------------- build
|
|
89
|
+
def _as_paths(code: Union[None, PathLike, Sequence[PathLike]]) -> List[Path]:
|
|
90
|
+
if code is None:
|
|
91
|
+
return []
|
|
92
|
+
if isinstance(code, (str, Path)):
|
|
93
|
+
return [Path(code)]
|
|
94
|
+
return [Path(c) for c in code]
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _is_library(source: Optional[str]) -> bool:
|
|
98
|
+
"""True for code that belongs to an installed library (torch, torchvision, timm...), not to you."""
|
|
99
|
+
if not source:
|
|
100
|
+
return False
|
|
101
|
+
stdlib = sysconfig.get_paths().get("stdlib", "\0")
|
|
102
|
+
return "site-packages" in source or "dist-packages" in source or source.startswith(stdlib)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _code_entries(cls: Any, code) -> tuple:
|
|
106
|
+
"""Which files hold the architecture, and the module name to import the class (or builder) from."""
|
|
107
|
+
entries = _as_paths(code)
|
|
108
|
+
module = cls.__module__
|
|
109
|
+
if not entries:
|
|
110
|
+
if module == "__main__":
|
|
111
|
+
raise ValidationError(
|
|
112
|
+
f"{cls.__name__} is defined in the script you are running. Put the class in its own "
|
|
113
|
+
"file (for example architecture.py), import it from there, or name the file: "
|
|
114
|
+
"code='architecture.py'") # (build() copies it out itself before it gets here)
|
|
115
|
+
try:
|
|
116
|
+
source = inspect.getsourcefile(cls)
|
|
117
|
+
except TypeError:
|
|
118
|
+
source = None
|
|
119
|
+
if _is_library(source):
|
|
120
|
+
raise ValidationError(
|
|
121
|
+
f"This model is a {module}.{cls.__name__} from a library, so there is no class of yours to "
|
|
122
|
+
"store. Give a function that builds it, in your own file or script, and pass it as "
|
|
123
|
+
"factory=:\n def build_model(**kw):\n return nn.Sequential(...) # or torchvision.models."
|
|
124
|
+
"resnet18(num_classes=2), timm.create_model(...)\n"
|
|
125
|
+
" at.save(net, flavor='bundle', factory=build_model, init_args={...})")
|
|
126
|
+
if "." in module:
|
|
127
|
+
raise ValidationError(
|
|
128
|
+
f"{cls.__name__} lives in the package module {module!r}. Pass code='{module.split('.')[0]}' "
|
|
129
|
+
"(the folder) so its structure is kept.")
|
|
130
|
+
if not source:
|
|
131
|
+
raise ValidationError(f"Cannot find the source file of {cls.__name__}; pass code='file.py'")
|
|
132
|
+
entries = [Path(source)]
|
|
133
|
+
for p in entries:
|
|
134
|
+
if not p.exists():
|
|
135
|
+
raise ValidationError(f"code path {str(p)!r} does not exist")
|
|
136
|
+
if p.name in _RESERVED:
|
|
137
|
+
raise ValidationError(f"{p.name!r} is a name the bundle uses itself; rename that file")
|
|
138
|
+
if module == "__main__": # the class was defined in a script and `code=` names the file
|
|
139
|
+
pattern = re.compile(rf"^(class|def)\s+{re.escape(cls.__name__)}\b", re.M)
|
|
140
|
+
found = [p for p in entries if p.is_file() and pattern.search(p.read_text(encoding="utf-8"))]
|
|
141
|
+
if not found:
|
|
142
|
+
raise ValidationError(f"None of the code files defines {cls.__name__}")
|
|
143
|
+
module = found[0].stem
|
|
144
|
+
return entries, module
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _globals_used(source: str) -> List[str]:
|
|
148
|
+
"""Names a piece of source reads from the module around it (imports, helpers, constants)."""
|
|
149
|
+
table = symtable.symtable(textwrap.dedent(source), "<source>", "exec")
|
|
150
|
+
used: List[str] = []
|
|
151
|
+
|
|
152
|
+
def walk(t, top: bool) -> None:
|
|
153
|
+
for sym in t.get_symbols():
|
|
154
|
+
if not sym.is_referenced() or sym.get_name() in used:
|
|
155
|
+
continue
|
|
156
|
+
if (top and not sym.is_assigned()) or (not top and sym.is_global()):
|
|
157
|
+
used.append(sym.get_name())
|
|
158
|
+
for child in t.get_children():
|
|
159
|
+
walk(child, False)
|
|
160
|
+
|
|
161
|
+
walk(table, True)
|
|
162
|
+
return used
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _source_of(obj: Any) -> str:
|
|
166
|
+
"""The source of a class or function, including ones defined in a notebook cell.
|
|
167
|
+
|
|
168
|
+
Python cannot find the source of a *class* defined in `__main__` when there is no script file
|
|
169
|
+
(a notebook): the module has no file. But the class's methods remember which cell they came from,
|
|
170
|
+
and the notebook keeps every cell's text, so the class is found through one of its methods."""
|
|
171
|
+
try:
|
|
172
|
+
return inspect.getsource(obj)
|
|
173
|
+
except (OSError, TypeError):
|
|
174
|
+
if not inspect.isclass(obj):
|
|
175
|
+
raise
|
|
176
|
+
for member in vars(obj).values():
|
|
177
|
+
code = getattr(getattr(member, "__func__", member), "__code__", None)
|
|
178
|
+
lines = linecache.getlines(code.co_filename) if code is not None else []
|
|
179
|
+
if not lines:
|
|
180
|
+
continue
|
|
181
|
+
for node in ast.walk(ast.parse("".join(lines))):
|
|
182
|
+
if isinstance(node, ast.ClassDef) and node.name == obj.__name__ \
|
|
183
|
+
and node.lineno <= code.co_firstlineno <= node.end_lineno:
|
|
184
|
+
first = min([node.lineno] + [d.lineno for d in node.decorator_list])
|
|
185
|
+
return "".join(lines[first - 1:node.end_lineno])
|
|
186
|
+
raise OSError("source code not available")
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _extract_main_class(cls: type) -> str:
|
|
190
|
+
"""Write architecture.py for a class defined in the running script or notebook.
|
|
191
|
+
|
|
192
|
+
It holds the class and what it needs from the script around it: the imports it uses, helper
|
|
193
|
+
functions and classes, and simple constants. The training code is left out.
|
|
194
|
+
"""
|
|
195
|
+
namespace = vars(sys.modules["__main__"])
|
|
196
|
+
imports: Dict[str, str] = {}
|
|
197
|
+
consts: Dict[str, str] = {}
|
|
198
|
+
defs: List[str] = []
|
|
199
|
+
seen = {cls.__name__}
|
|
200
|
+
|
|
201
|
+
def visit(source: str) -> None:
|
|
202
|
+
for name in _globals_used(source):
|
|
203
|
+
if name in seen or name not in namespace:
|
|
204
|
+
continue # already handled, or a builtin
|
|
205
|
+
value = namespace[name]
|
|
206
|
+
if isinstance(value, types.ModuleType):
|
|
207
|
+
imports[name] = f"import {value.__name__}" + (f" as {name}" if name != value.__name__ else "")
|
|
208
|
+
elif isinstance(value, (type, types.FunctionType)):
|
|
209
|
+
if value.__module__ == "__main__":
|
|
210
|
+
seen.add(name)
|
|
211
|
+
text = textwrap.dedent(_source_of(value))
|
|
212
|
+
visit(text) # what this helper needs comes first
|
|
213
|
+
defs.append(text)
|
|
214
|
+
elif "<locals>" in value.__qualname__:
|
|
215
|
+
raise ValidationError(f"{name!r} is defined inside a function, which cannot be copied")
|
|
216
|
+
else:
|
|
217
|
+
imports[name] = (f"from {value.__module__} import {value.__qualname__}"
|
|
218
|
+
+ (f" as {name}" if name != value.__qualname__ else ""))
|
|
219
|
+
else:
|
|
220
|
+
try:
|
|
221
|
+
literal = repr(value)
|
|
222
|
+
ast.literal_eval(literal)
|
|
223
|
+
except (ValueError, SyntaxError):
|
|
224
|
+
raise ValidationError(
|
|
225
|
+
f"the class uses the global {name!r} ({type(value).__name__}), which is not a plain "
|
|
226
|
+
"number, text, list or dict, so it cannot be copied") from None
|
|
227
|
+
consts[name] = f"{name} = {literal}"
|
|
228
|
+
|
|
229
|
+
source = textwrap.dedent(_source_of(cls))
|
|
230
|
+
visit(source)
|
|
231
|
+
header = '"""Copied by authtracker from the script that trained the model: the class and what it needs."""'
|
|
232
|
+
parts = [header, "\n".join(sorted(imports.values())), "\n".join(consts.values()), *defs, source]
|
|
233
|
+
return "\n\n\n".join(p.strip("\n") for p in parts if p.strip()) + "\n"
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _tokenize(value: Any, mapping: Mapping[str, str]) -> Any:
|
|
237
|
+
"""Replace paths of copied assets (in init_args) by TOKEN/assets/<name>."""
|
|
238
|
+
if isinstance(value, str):
|
|
239
|
+
absolute = os.path.abspath(value)
|
|
240
|
+
for original, token in mapping.items():
|
|
241
|
+
if absolute == original or absolute.startswith(original + os.sep):
|
|
242
|
+
return token + absolute[len(original):].replace(os.sep, "/")
|
|
243
|
+
return value
|
|
244
|
+
if isinstance(value, Mapping):
|
|
245
|
+
return {k: _tokenize(v, mapping) for k, v in value.items()}
|
|
246
|
+
if isinstance(value, (list, tuple)):
|
|
247
|
+
return [_tokenize(v, mapping) for v in value]
|
|
248
|
+
return value
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _resolve(value: Any, root: Path) -> Any:
|
|
252
|
+
"""The reverse: TOKEN/... becomes the real path inside the bundle folder."""
|
|
253
|
+
if isinstance(value, str) and value.startswith(TOKEN):
|
|
254
|
+
return str(root) + value[len(TOKEN):]
|
|
255
|
+
if isinstance(value, Mapping):
|
|
256
|
+
return {k: _resolve(v, root) for k, v in value.items()}
|
|
257
|
+
if isinstance(value, (list, tuple)):
|
|
258
|
+
return [_resolve(v, root) for v in value]
|
|
259
|
+
return value
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def _copy_assets(assets: Union[Sequence[PathLike], Mapping[PathLike, Sequence[str]]],
|
|
263
|
+
dest: Path) -> Dict[str, str]:
|
|
264
|
+
"""Copy files the constructor reads into assets/. A list copies each path whole; a dict
|
|
265
|
+
{folder: ["*.json", ...]} copies only the files that match, so a model folder with multi-GB
|
|
266
|
+
weights next to its config.json can contribute just the config."""
|
|
267
|
+
items = assets.items() if isinstance(assets, Mapping) else [(a, None) for a in assets]
|
|
268
|
+
mapping: Dict[str, str] = {}
|
|
269
|
+
for item, patterns in items:
|
|
270
|
+
source = Path(item)
|
|
271
|
+
if not source.exists():
|
|
272
|
+
raise ValidationError(f"asset {str(item)!r} does not exist")
|
|
273
|
+
target = dest / ASSETS / source.name
|
|
274
|
+
if target.exists():
|
|
275
|
+
raise ValidationError(f"two assets are called {source.name!r}; rename one")
|
|
276
|
+
target.parent.mkdir(exist_ok=True)
|
|
277
|
+
if patterns is not None:
|
|
278
|
+
if not source.is_dir():
|
|
279
|
+
raise ValidationError(f"patterns only make sense for a folder, and {str(item)!r} is a file")
|
|
280
|
+
files = sorted({f for pattern in patterns for f in source.glob(pattern) if f.is_file()})
|
|
281
|
+
if not files:
|
|
282
|
+
raise ValidationError(f"no file in {str(item)!r} matches {list(patterns)}")
|
|
283
|
+
for f in files:
|
|
284
|
+
(target / f.relative_to(source)).parent.mkdir(parents=True, exist_ok=True)
|
|
285
|
+
shutil.copy2(f, target / f.relative_to(source))
|
|
286
|
+
elif source.is_dir():
|
|
287
|
+
shutil.copytree(source, target, ignore=shutil.ignore_patterns("__pycache__", "*.pyc"))
|
|
288
|
+
else:
|
|
289
|
+
shutil.copy2(source, target)
|
|
290
|
+
mapping[os.path.abspath(source)] = f"{TOKEN}/{ASSETS}/{source.name}"
|
|
291
|
+
return mapping
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _unwrap(model: Any) -> Any:
|
|
295
|
+
"""The model inside DataParallel, DistributedDataParallel and torch.compile wrappers.
|
|
296
|
+
|
|
297
|
+
Those wrappers add a `module.` prefix to the weights and are not your architecture: what you want
|
|
298
|
+
to keep is the model inside them."""
|
|
299
|
+
torch = _torch()
|
|
300
|
+
for _ in range(5):
|
|
301
|
+
if hasattr(model, "_orig_mod"): # torch.compile
|
|
302
|
+
model = model._orig_mod
|
|
303
|
+
elif isinstance(model, (torch.nn.DataParallel, torch.nn.parallel.DistributedDataParallel)):
|
|
304
|
+
model = model.module
|
|
305
|
+
else:
|
|
306
|
+
break
|
|
307
|
+
return model
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _validate_extras(model: Any, extras: Optional[Mapping[str, Any]]) -> Dict[str, Any]:
|
|
311
|
+
"""Plain attributes to put back on the model after loading (class names, a threshold...)."""
|
|
312
|
+
extras = dict(extras or {})
|
|
313
|
+
try:
|
|
314
|
+
json.dumps(extras)
|
|
315
|
+
except TypeError as e:
|
|
316
|
+
raise ValidationError(f"extras must be JSON-friendly (numbers, text, lists, dicts): {e}") from e
|
|
317
|
+
taken = ({n for n, _ in model.named_children()} | {n for n, _ in model.named_parameters(recurse=False)}
|
|
318
|
+
| {n for n, _ in model.named_buffers(recurse=False)})
|
|
319
|
+
for name in extras:
|
|
320
|
+
if not name.isidentifier():
|
|
321
|
+
raise ValidationError(f"extras name {name!r} is not a valid attribute name")
|
|
322
|
+
if name in taken or callable(getattr(type(model), name, None)):
|
|
323
|
+
raise ValidationError(f"extras name {name!r} is already used by the model (a layer, a weight or a method)")
|
|
324
|
+
return extras
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
@contextlib.contextmanager
|
|
328
|
+
def _on_cpu(model: Any):
|
|
329
|
+
"""The model's tensors on the CPU while it is saved, and back where they were afterwards."""
|
|
330
|
+
devices = {t.device for t in list(model.parameters()) + list(model.buffers())}
|
|
331
|
+
if len(devices) != 1 or next(iter(devices)).type == "cpu":
|
|
332
|
+
yield
|
|
333
|
+
return
|
|
334
|
+
original = next(iter(devices))
|
|
335
|
+
model.to("cpu")
|
|
336
|
+
try:
|
|
337
|
+
yield
|
|
338
|
+
finally:
|
|
339
|
+
model.to(original)
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _user_modules() -> list:
|
|
343
|
+
"""Modules loaded from your own project (not from an installed library, not authtracker)."""
|
|
344
|
+
found = []
|
|
345
|
+
for name, module in list(sys.modules.items()):
|
|
346
|
+
file = getattr(module, "__file__", None)
|
|
347
|
+
if module is None or name == "__main__" or name.split(".")[0] == "authtracker":
|
|
348
|
+
continue
|
|
349
|
+
if file and file.endswith(".py") and not _is_library(file):
|
|
350
|
+
found.append(module)
|
|
351
|
+
return found
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
@contextlib.contextmanager
|
|
355
|
+
def _pickle_user_code_by_value():
|
|
356
|
+
"""Make cloudpickle store *your* classes and functions inside the pickle, not just their names,
|
|
357
|
+
so the pickle loads on a machine that does not have your files."""
|
|
358
|
+
try:
|
|
359
|
+
import cloudpickle
|
|
360
|
+
except ImportError as e: # pragma: no cover
|
|
361
|
+
raise ValidationError("Pickling needs cloudpickle: pip install cloudpickle") from e
|
|
362
|
+
registered = []
|
|
363
|
+
for module in _user_modules():
|
|
364
|
+
try:
|
|
365
|
+
cloudpickle.register_pickle_by_value(module)
|
|
366
|
+
registered.append(module)
|
|
367
|
+
except Exception:
|
|
368
|
+
continue
|
|
369
|
+
try:
|
|
370
|
+
yield cloudpickle
|
|
371
|
+
finally:
|
|
372
|
+
for module in registered:
|
|
373
|
+
try:
|
|
374
|
+
cloudpickle.unregister_pickle_by_value(module)
|
|
375
|
+
except Exception:
|
|
376
|
+
pass
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def _object_packages(model: Any) -> set:
|
|
380
|
+
"""Top-level packages of every class in the model, to pin them when only a pickle is stored."""
|
|
381
|
+
tops = set()
|
|
382
|
+
for module in model.modules():
|
|
383
|
+
for obj in (module, *vars(module).values()):
|
|
384
|
+
name = getattr(type(obj), "__module__", None)
|
|
385
|
+
if isinstance(name, str) and name:
|
|
386
|
+
tops.add(name.split(".")[0])
|
|
387
|
+
tops.discard("builtins")
|
|
388
|
+
return tops
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def _copy_code(entries: Sequence[Path], dest: Path) -> None:
|
|
392
|
+
for p in entries:
|
|
393
|
+
if p.is_dir():
|
|
394
|
+
shutil.copytree(p, dest / p.name, ignore=shutil.ignore_patterns("__pycache__", "*.pyc"))
|
|
395
|
+
else:
|
|
396
|
+
shutil.copy2(p, dest / p.name)
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def _imports(files: Iterable[Path]) -> set:
|
|
400
|
+
names = set()
|
|
401
|
+
for f in files:
|
|
402
|
+
try:
|
|
403
|
+
tree = ast.parse(f.read_text(encoding="utf-8"))
|
|
404
|
+
except (SyntaxError, UnicodeDecodeError):
|
|
405
|
+
continue
|
|
406
|
+
for node in ast.walk(tree):
|
|
407
|
+
if isinstance(node, ast.Import):
|
|
408
|
+
names.update(a.name.split(".")[0] for a in node.names)
|
|
409
|
+
elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
|
|
410
|
+
names.add(node.module.split(".")[0])
|
|
411
|
+
return names
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _namespace_packages(module: Any) -> set:
|
|
415
|
+
"""Top-level packages behind every name in a module, including names that came in through
|
|
416
|
+
`from something import *`, which reading the source cannot see."""
|
|
417
|
+
tops = set()
|
|
418
|
+
for value in vars(module).values():
|
|
419
|
+
name = value.__name__ if isinstance(value, types.ModuleType) else getattr(value, "__module__", None)
|
|
420
|
+
if isinstance(name, str) and name:
|
|
421
|
+
tops.add(name.split(".")[0])
|
|
422
|
+
return tops
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _auto_requirements(dest: Path, extra: Iterable[str] = ()) -> List[str]:
|
|
426
|
+
"""Pinned versions of torch and of every installed package the bundle's code uses."""
|
|
427
|
+
local = {p.stem for p in dest.glob("*.py")} | {p.name for p in dest.iterdir() if p.is_dir()}
|
|
428
|
+
code = [p for p in dest.rglob("*.py") if ASSETS not in p.relative_to(dest).parts[:1]]
|
|
429
|
+
owners = importlib.metadata.packages_distributions()
|
|
430
|
+
pins = set()
|
|
431
|
+
for name in _imports(code) | set(extra) | {"torch"}:
|
|
432
|
+
if name in local or name in sys.stdlib_module_names or name not in owners:
|
|
433
|
+
continue
|
|
434
|
+
dist = owners[name][0]
|
|
435
|
+
pins.add(f"{dist}=={importlib.metadata.version(dist)}")
|
|
436
|
+
return sorted(pins, key=str.lower)
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
_TEMPLATE = '''"""Sample inference for this bundle. It needs torch and requirements.txt, not authtracker.
|
|
440
|
+
|
|
441
|
+
python inference.py input.npy # a NumPy file of shape (batch, ...); prints the model output
|
|
442
|
+
|
|
443
|
+
From your own code:
|
|
444
|
+
|
|
445
|
+
from inference import load_model, predict
|
|
446
|
+
model = load_model() # the real torch.nn.Module with its weights, in eval mode
|
|
447
|
+
out = predict(x) # x: a NumPy array or tensor of float32 inputs
|
|
448
|
+
"""
|
|
449
|
+
import importlib
|
|
450
|
+
import json
|
|
451
|
+
import sys
|
|
452
|
+
from pathlib import Path
|
|
453
|
+
|
|
454
|
+
import torch
|
|
455
|
+
|
|
456
|
+
HERE = Path(__file__).resolve().parent
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def _resolve(value):
|
|
460
|
+
if isinstance(value, str) and value.startswith("$BUNDLE"):
|
|
461
|
+
return str(HERE) + value[len("$BUNDLE"):]
|
|
462
|
+
if isinstance(value, dict):
|
|
463
|
+
return {k: _resolve(v) for k, v in value.items()}
|
|
464
|
+
if isinstance(value, list):
|
|
465
|
+
return [_resolve(v) for v in value]
|
|
466
|
+
return value
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def load_model(device="cpu"):
|
|
470
|
+
manifest = json.loads((HERE / "bundle.json").read_text())
|
|
471
|
+
sys.path.insert(0, str(HERE))
|
|
472
|
+
try:
|
|
473
|
+
module = importlib.import_module(manifest["module"])
|
|
474
|
+
finally:
|
|
475
|
+
sys.path.remove(str(HERE))
|
|
476
|
+
net = getattr(module, manifest["class"])(**_resolve(manifest["init_args"]))
|
|
477
|
+
net.load_state_dict(torch.load(HERE / manifest["weights"], map_location=device, weights_only=True))
|
|
478
|
+
return net.to(device).eval()
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def predict(x, device="cpu"):
|
|
482
|
+
model = load_model(device)
|
|
483
|
+
with torch.no_grad():
|
|
484
|
+
return model(torch.as_tensor(x, dtype=torch.float32, device=device))
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
if __name__ == "__main__":
|
|
488
|
+
if len(sys.argv) != 2:
|
|
489
|
+
sys.exit("usage: python inference.py input.npy")
|
|
490
|
+
import numpy as np
|
|
491
|
+
print(predict(np.load(sys.argv[1])))
|
|
492
|
+
'''
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def _sha256(path: Path) -> str:
|
|
496
|
+
h = hashlib.sha256()
|
|
497
|
+
with open(path, "rb") as f:
|
|
498
|
+
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
|
499
|
+
h.update(chunk)
|
|
500
|
+
return h.hexdigest()
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def _hashes(root: Path) -> Dict[str, str]:
|
|
504
|
+
return {p.relative_to(root).as_posix(): _sha256(p)
|
|
505
|
+
for p in sorted(root.rglob("*"))
|
|
506
|
+
if p.is_file() and p.name != MANIFEST and "__pycache__" not in p.parts}
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def _environment(model) -> Dict[str, Any]:
|
|
510
|
+
torch = _torch()
|
|
511
|
+
devices = {t.device.type for t in list(model.parameters()) + list(model.buffers())}
|
|
512
|
+
return {
|
|
513
|
+
"python": platform.python_version(), "platform": platform.platform(),
|
|
514
|
+
"torch": torch.__version__, "cuda": torch.version.cuda,
|
|
515
|
+
"trained_on": sorted(devices)[0] if len(devices) == 1 else sorted(devices),
|
|
516
|
+
"created_utc": time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()),
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def build(model: Any, dest: PathLike, *, init_args: Optional[Mapping[str, Any]] = None,
|
|
521
|
+
code: Union[None, PathLike, Sequence[PathLike]] = None, inference: Optional[PathLike] = None,
|
|
522
|
+
requirements: Optional[Sequence[str]] = None,
|
|
523
|
+
assets: Union[None, Sequence[PathLike], Mapping[PathLike, Sequence[str]]] = None,
|
|
524
|
+
factory: Optional[Any] = None, extras: Optional[Mapping[str, Any]] = None,
|
|
525
|
+
pickle: bool = False, check: bool = True,
|
|
526
|
+
context: Optional[Mapping[str, Any]] = None) -> Dict[str, Any]:
|
|
527
|
+
"""Write the bundle for `model` into the (empty) folder `dest`. Returns the manifest.
|
|
528
|
+
|
|
529
|
+
extras: plain attributes to put back on the model after loading, for things that are not weights,
|
|
530
|
+
such as class_names=[...] or threshold=0.3 (numbers, text, lists, dicts).
|
|
531
|
+
pickle: also store model.pkl, the whole object, loadable later with pickle=True.
|
|
532
|
+
|
|
533
|
+
factory: a function `f(**init_args)` that returns a new, untrained model like `model`. Use it when
|
|
534
|
+
the model has no class of yours: nn.Sequential(...), torchvision / timm models, or a model you
|
|
535
|
+
change after creating it (adding LoRA, freezing). Without it, the model's class is shipped.
|
|
536
|
+
"""
|
|
537
|
+
torch = _torch()
|
|
538
|
+
model = _unwrap(model)
|
|
539
|
+
if not isinstance(model, torch.nn.Module):
|
|
540
|
+
raise ValidationError(f"flavor='{FLAVOR}' is for a torch.nn.Module, not {type(model).__name__}")
|
|
541
|
+
if factory is not None and not callable(factory):
|
|
542
|
+
raise ValidationError("factory must be a function that builds the model: factory=build_model")
|
|
543
|
+
dest = Path(dest)
|
|
544
|
+
dest.mkdir(parents=True, exist_ok=True)
|
|
545
|
+
carried = getattr(model, _CARRY, None) # a model that came from a bundle carries its code
|
|
546
|
+
if carried is not None and (code is not None or factory is not None or (
|
|
547
|
+
not carried.get("factory") and carried["class"] != type(model).__name__)):
|
|
548
|
+
carried = None
|
|
549
|
+
if init_args is None and carried is not None:
|
|
550
|
+
init_args = carried["init_args"]
|
|
551
|
+
init_args = dict(init_args or {})
|
|
552
|
+
kept = {k: getattr(model, k, v) for k, v in (carried or {}).get("extras", {}).items()} # as they are now
|
|
553
|
+
extras = _validate_extras(model, {**kept, **dict(extras or {})}) if (kept or extras) else {}
|
|
554
|
+
try:
|
|
555
|
+
json.dumps(init_args)
|
|
556
|
+
except TypeError as e:
|
|
557
|
+
raise ValidationError(f"init_args must be JSON-friendly (numbers, text, lists, dicts): {e}") from e
|
|
558
|
+
|
|
559
|
+
cls = factory if factory is not None else type(model) # what the loader will call
|
|
560
|
+
extracted = False # was the code copied out of a script or notebook?
|
|
561
|
+
if carried is not None:
|
|
562
|
+
module = carried["module"]
|
|
563
|
+
for name, data in carried["files"].items():
|
|
564
|
+
target = dest / name
|
|
565
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
566
|
+
target.write_bytes(data)
|
|
567
|
+
elif code is None and cls.__module__ == "__main__":
|
|
568
|
+
extracted = True
|
|
569
|
+
try:
|
|
570
|
+
text = _extract_main_class(cls)
|
|
571
|
+
except (OSError, TypeError, ValidationError) as e:
|
|
572
|
+
raise ValidationError(
|
|
573
|
+
f"{cls.__name__} is defined in the script you are running and authtracker could not copy "
|
|
574
|
+
f"it out ({e}). Put the class in its own file (for example architecture.py), import it, "
|
|
575
|
+
"or name the file: code='architecture.py'") from e
|
|
576
|
+
(dest / "architecture.py").write_text(text, encoding="utf-8")
|
|
577
|
+
module = "architecture"
|
|
578
|
+
else:
|
|
579
|
+
entries, module = _code_entries(cls, code)
|
|
580
|
+
_copy_code(entries, dest)
|
|
581
|
+
|
|
582
|
+
if assets:
|
|
583
|
+
init_args = _tokenize(init_args, _copy_assets(assets, dest)) # files the constructor needs
|
|
584
|
+
state = {k: (v.detach().cpu() if isinstance(v, torch.Tensor) else v) # the model itself stays put
|
|
585
|
+
for k, v in model.state_dict().items()}
|
|
586
|
+
torch.save(state, dest / WEIGHTS)
|
|
587
|
+
|
|
588
|
+
if inference is not None:
|
|
589
|
+
if not Path(inference).is_file():
|
|
590
|
+
raise ValidationError(f"inference script {str(inference)!r} does not exist")
|
|
591
|
+
shutil.copy2(inference, dest / INFERENCE)
|
|
592
|
+
elif not (dest / INFERENCE).exists():
|
|
593
|
+
(dest / INFERENCE).write_text(_TEMPLATE, encoding="utf-8")
|
|
594
|
+
|
|
595
|
+
(dest / ENVIRONMENT).write_text(requirements_text(), encoding="utf-8")
|
|
596
|
+
entry = carried["class"] if carried is not None else cls.__name__ # what the loader calls
|
|
597
|
+
is_factory = bool(carried["factory"]) if carried is not None else factory is not None
|
|
598
|
+
manifest = {"format": MODEL_TYPE, "version": FORMAT_VERSION, "kind": "bundle", "module": module,
|
|
599
|
+
"class": entry, "factory": is_factory, "init_args": init_args, "weights": WEIGHTS,
|
|
600
|
+
"inference": INFERENCE, "extras": extras, "pickle": PICKLE if pickle else None,
|
|
601
|
+
"environment": _environment(model), "git": git_info(), "container": _container(context)}
|
|
602
|
+
try:
|
|
603
|
+
used = _check(dest, manifest, state) if check else set()
|
|
604
|
+
except ValidationError as e:
|
|
605
|
+
if not extracted:
|
|
606
|
+
raise
|
|
607
|
+
raise ValidationError(
|
|
608
|
+
f"{cls.__name__} is defined in the script you are running, and the copy authtracker made of it "
|
|
609
|
+
f"does not work: {e}\nThe copy only gets what the class uses directly. Put the class in its own "
|
|
610
|
+
"file (for example architecture.py) and import it, or name the file: code='architecture.py'") from e
|
|
611
|
+
pins = list(requirements) if requirements is not None else _auto_requirements(dest, used)
|
|
612
|
+
(dest / REQUIREMENTS).write_text("\n".join(pins) + "\n", encoding="utf-8")
|
|
613
|
+
if pickle:
|
|
614
|
+
_write_pickle(model, dest, by_value=False) # by name: architecture.py in this folder is what it finds
|
|
615
|
+
(dest / MANIFEST).write_text(json.dumps(manifest, indent=2), encoding="utf-8")
|
|
616
|
+
if check:
|
|
617
|
+
_check_pickle(dest)
|
|
618
|
+
manifest["files"] = _hashes(dest)
|
|
619
|
+
(dest / MANIFEST).write_text(json.dumps(manifest, indent=2), encoding="utf-8")
|
|
620
|
+
return manifest
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
def _write_pickle(model: Any, dest: Path, by_value: bool) -> None:
|
|
624
|
+
"""cloudpickle the whole model, from CPU tensors. `by_value` stores your own classes inside the
|
|
625
|
+
pickle (so it loads without your files); otherwise they are stored by name."""
|
|
626
|
+
with contextlib.ExitStack() as stack:
|
|
627
|
+
if by_value:
|
|
628
|
+
cloudpickle = stack.enter_context(_pickle_user_code_by_value())
|
|
629
|
+
else:
|
|
630
|
+
import cloudpickle
|
|
631
|
+
stack.enter_context(_on_cpu(model))
|
|
632
|
+
out = stack.enter_context(open(dest / PICKLE, "wb"))
|
|
633
|
+
cloudpickle.dump(model, out, protocol=_pickle.HIGHEST_PROTOCOL)
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
def _check_pickle(root: Path) -> None:
|
|
637
|
+
"""Load the pickle in a clean process that has none of your files, the way another machine would."""
|
|
638
|
+
import authtracker
|
|
639
|
+
library = str(Path(authtracker.__file__).resolve().parent.parent)
|
|
640
|
+
code = ("import sys\nsys.path.insert(0, %r)\nfrom authtracker import bundle\n"
|
|
641
|
+
"net = bundle.load_dir(%r, pickle=True, verify=False)\nprint(type(net).__name__)") % (library, str(root))
|
|
642
|
+
with tempfile.TemporaryDirectory() as empty:
|
|
643
|
+
env = {**os.environ, "PYTHONPATH": "", "PYTHONDONTWRITEBYTECODE": "1"}
|
|
644
|
+
result = subprocess.run([sys.executable, "-c", code], cwd=empty, env=env, capture_output=True, text=True)
|
|
645
|
+
if result.returncode != 0:
|
|
646
|
+
reason = (result.stderr.strip().splitlines() or ["no error message"])[-1][:250]
|
|
647
|
+
raise ValidationError(
|
|
648
|
+
f"The pickle cannot be loaded in a clean process, so it would not load on another machine: "
|
|
649
|
+
f"{reason}. A class or function from a module that is not on the path is the usual reason. "
|
|
650
|
+
"Use the folder bundle (without pickle=True), or check=False to skip this test.")
|
|
651
|
+
|
|
652
|
+
|
|
653
|
+
def _container(context: Optional[Mapping[str, Any]]) -> Optional[Dict[str, str]]:
|
|
654
|
+
"""The container image of the run this model belongs to (passed in by the run), else what can be found."""
|
|
655
|
+
if context and "container" in context:
|
|
656
|
+
return dict(context["container"]) if context["container"] else None
|
|
657
|
+
return container_info()
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
def build_pickle(model: Any, dest: PathLike, *, requirements: Optional[Sequence[str]] = None,
|
|
661
|
+
check: bool = True, context: Optional[Mapping[str, Any]] = None,
|
|
662
|
+
**unsupported: Any) -> Dict[str, Any]:
|
|
663
|
+
"""Write only the pickled model, the requirements and the environment into the folder `dest`.
|
|
664
|
+
|
|
665
|
+
Your own classes and functions are stored inside the pickle, so it loads on a machine that does not
|
|
666
|
+
have your files. It needs the same Python and torch versions, and loading it runs code.
|
|
667
|
+
"""
|
|
668
|
+
torch = _torch()
|
|
669
|
+
if unsupported:
|
|
670
|
+
raise ValidationError(
|
|
671
|
+
f"flavor='{PICKLE_FLAVOR}' stores the whole object, so {sorted(unsupported)} are not used. "
|
|
672
|
+
f"They belong to flavor='{FLAVOR}'.")
|
|
673
|
+
model = _unwrap(model)
|
|
674
|
+
if not isinstance(model, torch.nn.Module):
|
|
675
|
+
raise ValidationError(f"flavor='{PICKLE_FLAVOR}' is for a torch.nn.Module, not {type(model).__name__}")
|
|
676
|
+
dest = Path(dest)
|
|
677
|
+
dest.mkdir(parents=True, exist_ok=True)
|
|
678
|
+
_write_pickle(model, dest, by_value=True)
|
|
679
|
+
(dest / ENVIRONMENT).write_text(requirements_text(), encoding="utf-8")
|
|
680
|
+
pins = list(requirements) if requirements is not None else _auto_requirements(dest, _object_packages(model))
|
|
681
|
+
(dest / REQUIREMENTS).write_text("\n".join(pins) + "\n", encoding="utf-8")
|
|
682
|
+
manifest = {"format": MODEL_TYPE, "version": FORMAT_VERSION, "kind": "pickle", "class": type(model).__name__,
|
|
683
|
+
"pickle": PICKLE, "environment": _environment(model), "git": git_info(),
|
|
684
|
+
"container": _container(context)}
|
|
685
|
+
(dest / MANIFEST).write_text(json.dumps(manifest, indent=2), encoding="utf-8")
|
|
686
|
+
if check:
|
|
687
|
+
_check_pickle(dest)
|
|
688
|
+
manifest["files"] = _hashes(dest)
|
|
689
|
+
(dest / MANIFEST).write_text(json.dumps(manifest, indent=2), encoding="utf-8")
|
|
690
|
+
return manifest
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
def _check(root: Path, manifest: Mapping[str, Any], state: Mapping[str, Any]) -> set:
|
|
694
|
+
"""Prove the bundle can rebuild the class, without allocating the model (a 'meta' device).
|
|
695
|
+
Returns the packages the class's module uses."""
|
|
696
|
+
torch = _torch()
|
|
697
|
+
packages: set = set()
|
|
698
|
+
cls = _import_class(root, manifest["module"], manifest["class"], packages)
|
|
699
|
+
args = _resolve(manifest["init_args"], root)
|
|
700
|
+
try:
|
|
701
|
+
with torch.device("meta"):
|
|
702
|
+
net = cls(**args)
|
|
703
|
+
except Exception:
|
|
704
|
+
try:
|
|
705
|
+
net = cls(**args)
|
|
706
|
+
except Exception as e:
|
|
707
|
+
raise ValidationError(
|
|
708
|
+
f"The bundle cannot rebuild {manifest['class']} from its files: {type(e).__name__}: {e}. "
|
|
709
|
+
"If the constructor needs arguments, pass them: init_args={...}") from e
|
|
710
|
+
shape = lambda v: tuple(v.shape) if isinstance(v, torch.Tensor) else None # noqa: E731
|
|
711
|
+
expected = {k: shape(v) for k, v in net.state_dict().items()}
|
|
712
|
+
saved = {k: shape(v) for k, v in state.items()}
|
|
713
|
+
if expected != saved:
|
|
714
|
+
missing = sorted(set(expected) - set(saved))[:3]
|
|
715
|
+
extra = sorted(set(saved) - set(expected))[:3]
|
|
716
|
+
wrong = [k for k in expected if k in saved and expected[k] != saved[k]][:3]
|
|
717
|
+
raise ValidationError(
|
|
718
|
+
f"The weights do not fit {manifest['class']} built with init_args={manifest['init_args']}: "
|
|
719
|
+
f"missing {missing}, unexpected {extra}, different shapes {wrong}. "
|
|
720
|
+
"Check init_args (the sizes your model was created with). If you change the model after "
|
|
721
|
+
"creating it (pruning, LoRA, freezing), put those steps in a function and pass it as factory=.")
|
|
722
|
+
if any(not isinstance(v, torch.Tensor) for v in state.values()):
|
|
723
|
+
try:
|
|
724
|
+
torch.load(root / WEIGHTS, map_location="cpu", weights_only=True)
|
|
725
|
+
except Exception as e:
|
|
726
|
+
raise ValidationError(
|
|
727
|
+
"Some entries in the model's state_dict are not plain tensors (for example an object "
|
|
728
|
+
"returned by get_extra_state()), and the bundle loads weights with weights_only=True so "
|
|
729
|
+
f"that loading can never run code: {type(e).__name__}: {str(e).splitlines()[0][:150]}") from e
|
|
730
|
+
return packages
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
# ---------------------------------------------------------------------------------------- load
|
|
734
|
+
def _under(root: Path, module: Any) -> bool:
|
|
735
|
+
paths = [getattr(module, "__file__", None), *getattr(module, "__path__", [])]
|
|
736
|
+
for p in paths:
|
|
737
|
+
try:
|
|
738
|
+
if p and Path(p).resolve().is_relative_to(root):
|
|
739
|
+
return True
|
|
740
|
+
except (OSError, ValueError):
|
|
741
|
+
continue
|
|
742
|
+
return False
|
|
743
|
+
|
|
744
|
+
|
|
745
|
+
@contextlib.contextmanager
|
|
746
|
+
def _bundle_modules(root: Path):
|
|
747
|
+
"""Inside this block, `import` finds the bundle's own files first, never a module of the same
|
|
748
|
+
name that is already loaded (your architecture.py in the training process, or another bundle).
|
|
749
|
+
Afterwards, the modules loaded from the bundle are forgotten and the old ones are put back."""
|
|
750
|
+
root = root.resolve()
|
|
751
|
+
tops = {p.stem for p in root.glob("*.py")} | {p.name for p in root.iterdir() if p.is_dir()}
|
|
752
|
+
shadowed = {n: sys.modules.pop(n) for n in list(sys.modules) if n.split(".")[0] in tops}
|
|
753
|
+
before = set(sys.modules)
|
|
754
|
+
sys.path.insert(0, str(root))
|
|
755
|
+
try:
|
|
756
|
+
yield
|
|
757
|
+
finally:
|
|
758
|
+
sys.path.remove(str(root))
|
|
759
|
+
for n in set(sys.modules) - before:
|
|
760
|
+
if _under(root, sys.modules[n]):
|
|
761
|
+
sys.modules.pop(n, None)
|
|
762
|
+
sys.modules.update(shadowed)
|
|
763
|
+
|
|
764
|
+
|
|
765
|
+
def _import_class(root: Path, module: str, name: str, packages: Optional[set] = None):
|
|
766
|
+
"""Import `module` from `root` only and return `name` from it."""
|
|
767
|
+
with _bundle_modules(root):
|
|
768
|
+
try:
|
|
769
|
+
mod = importlib.import_module(module)
|
|
770
|
+
if packages is not None:
|
|
771
|
+
packages.update(_namespace_packages(mod))
|
|
772
|
+
return getattr(mod, name)
|
|
773
|
+
except Exception as e:
|
|
774
|
+
raise ValidationError(
|
|
775
|
+
f"Could not import {name} from {module!r} in the bundle: {type(e).__name__}: {e}") from e
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
def read_manifest(root: PathLike) -> Dict[str, Any]:
|
|
779
|
+
path = Path(root) / MANIFEST
|
|
780
|
+
if not path.is_file():
|
|
781
|
+
raise ValidationError(f"{str(root)!r} is not an authtracker bundle (no {MANIFEST})")
|
|
782
|
+
manifest = json.loads(path.read_text(encoding="utf-8"))
|
|
783
|
+
if manifest.get("format") != MODEL_TYPE:
|
|
784
|
+
raise ValidationError(f"{str(path)!r} is not an authtracker bundle manifest")
|
|
785
|
+
if manifest.get("version", 0) > FORMAT_VERSION:
|
|
786
|
+
raise ValidationError(
|
|
787
|
+
f"This bundle uses format version {manifest['version']}; this authtracker reads up to "
|
|
788
|
+
f"{FORMAT_VERSION}. Upgrade authtracker.")
|
|
789
|
+
return manifest
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
def is_bundle(root: PathLike) -> bool:
|
|
793
|
+
try:
|
|
794
|
+
read_manifest(root)
|
|
795
|
+
return True
|
|
796
|
+
except (ValidationError, ValueError, OSError):
|
|
797
|
+
return False
|
|
798
|
+
|
|
799
|
+
|
|
800
|
+
def _warn_if_environment_differs(manifest: Mapping[str, Any]) -> None:
|
|
801
|
+
saved = manifest.get("environment", {})
|
|
802
|
+
now_python, now_torch = platform.python_version(), _torch().__version__
|
|
803
|
+
short = lambda v: ".".join(str(v).split("+")[0].split(".")[:2]) # noqa: E731
|
|
804
|
+
notes = []
|
|
805
|
+
if saved.get("python") and short(saved["python"]) != short(now_python):
|
|
806
|
+
notes.append(f"Python {saved['python']} (this is {now_python})")
|
|
807
|
+
if saved.get("torch") and short(saved["torch"]) != short(now_torch):
|
|
808
|
+
notes.append(f"torch {saved['torch']} (this is {now_torch})")
|
|
809
|
+
if notes:
|
|
810
|
+
_log.warning("model.pkl was saved with %s. A pickle may fail to load, or behave differently, "
|
|
811
|
+
"on other versions; the folder bundle (architecture.py + weights.pth) does not have "
|
|
812
|
+
"this limit.", " and ".join(notes))
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
def _load_pickle(root: Path, manifest: Mapping[str, Any], device: str, eval: bool):
|
|
816
|
+
torch = _torch()
|
|
817
|
+
name = manifest.get("pickle") or PICKLE
|
|
818
|
+
if not (root / name).is_file():
|
|
819
|
+
raise ValidationError(f"This bundle has no {name}: it was saved without pickle=True")
|
|
820
|
+
_warn_if_environment_differs(manifest)
|
|
821
|
+
with _bundle_modules(root): # a pickle that names your classes finds this bundle's copy
|
|
822
|
+
try:
|
|
823
|
+
with open(root / name, "rb") as f:
|
|
824
|
+
net = _pickle.load(f)
|
|
825
|
+
except Exception as e:
|
|
826
|
+
raise ValidationError(
|
|
827
|
+
f"Could not load {name}: {type(e).__name__}: {str(e).splitlines()[0][:200] if str(e) else ''}. "
|
|
828
|
+
"A pickle needs the same Python and torch versions it was saved with.") from e
|
|
829
|
+
if not isinstance(net, torch.nn.Module):
|
|
830
|
+
raise ValidationError(f"{name} does not hold a torch.nn.Module")
|
|
831
|
+
net.to(device)
|
|
832
|
+
if eval:
|
|
833
|
+
net.eval()
|
|
834
|
+
return net
|
|
835
|
+
|
|
836
|
+
|
|
837
|
+
def load_dir(root: PathLike, *, device: str = "cpu", eval: bool = True, verify: bool = True,
|
|
838
|
+
pickle: bool = False):
|
|
839
|
+
"""Rebuild the model from a bundle folder: the class from its source, then the weights.
|
|
840
|
+
|
|
841
|
+
pickle=True loads model.pkl instead, which gives back the exact object (attributes and all).
|
|
842
|
+
That runs code from the file, so it is opt-in; a folder saved with flavor='pickle' needs it.
|
|
843
|
+
"""
|
|
844
|
+
torch = _torch()
|
|
845
|
+
root = Path(root)
|
|
846
|
+
manifest = read_manifest(root)
|
|
847
|
+
if verify: # before anything is imported or unpickled
|
|
848
|
+
bad = [name for name, digest in manifest.get("files", {}).items()
|
|
849
|
+
if not (root / name).is_file() or _sha256(root / name) != digest]
|
|
850
|
+
if bad:
|
|
851
|
+
raise ValidationError(f"The bundle was changed or is incomplete (hash mismatch): {bad[:5]}")
|
|
852
|
+
if manifest.get("kind") == "pickle" and not pickle:
|
|
853
|
+
raise ValidationError(
|
|
854
|
+
"This model is stored as a pickle (flavor='pickle'). Loading a pickle runs code from the file, "
|
|
855
|
+
"so it is opt-in: load it with pickle=True, and only if you trust where it came from.")
|
|
856
|
+
if pickle:
|
|
857
|
+
return _load_pickle(root, manifest, device, eval)
|
|
858
|
+
cls = _import_class(root, manifest["module"], manifest["class"])
|
|
859
|
+
args = _resolve(manifest["init_args"], root.resolve())
|
|
860
|
+
try:
|
|
861
|
+
net = cls(**args)
|
|
862
|
+
except Exception as e:
|
|
863
|
+
raise ValidationError(
|
|
864
|
+
f"Could not build {manifest['class']}(**{args}): {type(e).__name__}: {e}. "
|
|
865
|
+
"If the constructor reads files (a config, a vocabulary), ship them with assets=[...]") from e
|
|
866
|
+
state = torch.load(root / manifest["weights"], map_location=device, weights_only=True)
|
|
867
|
+
net.load_state_dict(state)
|
|
868
|
+
for name, value in manifest.get("extras", {}).items(): # attributes that are not weights
|
|
869
|
+
setattr(net, name, value)
|
|
870
|
+
net.to(device)
|
|
871
|
+
if eval:
|
|
872
|
+
net.eval()
|
|
873
|
+
_carry(net, root, manifest)
|
|
874
|
+
return net
|
|
875
|
+
|
|
876
|
+
|
|
877
|
+
def _carry(net: Any, root: Path, manifest: Mapping[str, Any]) -> None:
|
|
878
|
+
"""Keep the bundle's code on the loaded model, so fine-tuning it and saving it again just works
|
|
879
|
+
(the folder it was loaded from may be gone by then)."""
|
|
880
|
+
skip = {WEIGHTS, MANIFEST, REQUIREMENTS, ENVIRONMENT}
|
|
881
|
+
files = {name: (root / name).read_bytes() for name in manifest.get("files", {})
|
|
882
|
+
if name not in skip and (root / name).is_file()}
|
|
883
|
+
if sum(len(b) for b in files.values()) <= _CARRY_LIMIT:
|
|
884
|
+
setattr(net, _CARRY, {"module": manifest["module"], "class": manifest["class"],
|
|
885
|
+
"factory": bool(manifest.get("factory")),
|
|
886
|
+
"init_args": dict(manifest["init_args"]),
|
|
887
|
+
"extras": dict(manifest.get("extras", {})), "files": files})
|