echoact 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- echoact/__init__.py +3 -0
- echoact/__main__.py +117 -0
- echoact/app.py +315 -0
- echoact/audio/__init__.py +0 -0
- echoact/audio/devices.py +192 -0
- echoact/audio/player.py +611 -0
- echoact/audio/wav.py +854 -0
- echoact/config/__init__.py +0 -0
- echoact/config/budget.py +370 -0
- echoact/config/settings.py +1244 -0
- echoact/db/__init__.py +0 -0
- echoact/db/backup.py +2429 -0
- echoact/db/migrations.py +434 -0
- echoact/db/schema.sql +214 -0
- echoact/db/store.py +2062 -0
- echoact/diagnostics.py +902 -0
- echoact/domain.py +487 -0
- echoact/engine/__init__.py +0 -0
- echoact/engine/container.py +843 -0
- echoact/engine/protocol.py +241 -0
- echoact/engine/runtime.py +324 -0
- echoact/engine/supervisor.py +961 -0
- echoact/engine/worker.py +659 -0
- echoact/errors.py +281 -0
- echoact/instance.py +172 -0
- echoact/jobs/__init__.py +0 -0
- echoact/jobs/engine.py +776 -0
- echoact/jobs/request.py +300 -0
- echoact/mcp/__init__.py +0 -0
- echoact/mcp/__main__.py +50 -0
- echoact/mcp/client.py +202 -0
- echoact/mcp/config.py +112 -0
- echoact/mcp/server.py +340 -0
- echoact/models/__init__.py +0 -0
- echoact/models/catalog.py +273 -0
- echoact/models/manifest.py +278 -0
- echoact/models/registry.py +1551 -0
- echoact/paths.py +93 -0
- echoact/policy.py +189 -0
- echoact/security/__init__.py +0 -0
- echoact/security/credentials.py +930 -0
- echoact/security/ratelimit.py +534 -0
- echoact/service/__init__.py +20 -0
- echoact/service/app.py +182 -0
- echoact/service/deps.py +563 -0
- echoact/service/errors.py +241 -0
- echoact/service/routes.py +1125 -0
- echoact/service/schemas.py +509 -0
- echoact/service/server.py +270 -0
- echoact/text/__init__.py +0 -0
- echoact/text/language.py +44 -0
- echoact/text/loader.py +577 -0
- echoact/text/normalize.py +924 -0
- echoact/text/segment.py +499 -0
- echoact/text/sniff.py +1202 -0
- echoact/ui/__init__.py +0 -0
- echoact/ui/bridge.py +50 -0
- echoact/ui/controls.py +360 -0
- echoact/ui/credential_dialog.py +131 -0
- echoact/ui/fonts.py +94 -0
- echoact/ui/i18n.py +260 -0
- echoact/ui/icons.py +440 -0
- echoact/ui/library.py +1642 -0
- echoact/ui/licence.py +162 -0
- echoact/ui/main_window.py +1202 -0
- echoact/ui/mcp_setup.py +494 -0
- echoact/ui/models_view.py +1142 -0
- echoact/ui/notifications.py +202 -0
- echoact/ui/reading.py +494 -0
- echoact/ui/settings_view.py +2258 -0
- echoact/ui/status_view.py +1193 -0
- echoact/ui/theme.py +579 -0
- echoact/util/__init__.py +0 -0
- echoact/util/ids.py +62 -0
- echoact/util/logging.py +127 -0
- echoact-0.1.0.dist-info/METADATA +162 -0
- echoact-0.1.0.dist-info/RECORD +80 -0
- echoact-0.1.0.dist-info/WHEEL +4 -0
- echoact-0.1.0.dist-info/entry_points.txt +3 -0
- echoact-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,1551 @@
|
|
|
1
|
+
"""The model lifecycle: where weights live, whether they are sound, and how
|
|
2
|
+
they get there (F-09, F-63, F-64, F-65, resolved against F-84's manifest).
|
|
3
|
+
|
|
4
|
+
The app owns its own model cache under :func:`echoact.paths.model_cache_dir`
|
|
5
|
+
rather than letting the ``supertonic`` package download into
|
|
6
|
+
``~/.cache/supertonic3``. F-73 has to size the model cache separately from
|
|
7
|
+
documents and audio, F-65 has to be able to delete it, and F-76 offers it as
|
|
8
|
+
its own deletion scope; none of that is possible for a directory a third-party
|
|
9
|
+
package owns and shares with whatever else on the machine imports it. A
|
|
10
|
+
verified copy already sitting in the package's cache is still *read* rather
|
|
11
|
+
than re-downloaded -- 385 MB is not worth a second trip to satisfy tidiness --
|
|
12
|
+
but it is never written to, and deleting the app's cache says so.
|
|
13
|
+
|
|
14
|
+
Everything that decides "sound" goes through the manifest. Nothing here
|
|
15
|
+
trusts a file because it exists, a size because a server reported it, or a
|
|
16
|
+
directory because the last run left it behind.
|
|
17
|
+
|
|
18
|
+
N-11's licence gate sits on the way to a *directory*, not only on the way to
|
|
19
|
+
a download. A model that is already on disk -- borrowed from the package
|
|
20
|
+
cache above, or prepared before a release amended the restrictions -- is
|
|
21
|
+
never downloaded again, so a gate that only guarded :meth:`ModelRegistry.download`
|
|
22
|
+
would be one the two commonest cases walk straight past.
|
|
23
|
+
|
|
24
|
+
Four requirements shape the awkward parts:
|
|
25
|
+
|
|
26
|
+
* N-21 forbids reading a whole file into memory to hash it, and a
|
|
27
|
+
verification pass over 385 MB has to be interruptible, so every read is a
|
|
28
|
+
bounded chunk with a cancellation check between chunks.
|
|
29
|
+
* F-64 wants progress, failure, cancellation, and a retry that reuses sound
|
|
30
|
+
data. Downloads therefore stream into a ``.part`` file next to the target
|
|
31
|
+
and are renamed into place only after the digest matches. A cancelled or
|
|
32
|
+
crashed download leaves a partial file that the next attempt resumes from
|
|
33
|
+
and that verification ignores. What a cancelled attempt *reports* rests on
|
|
34
|
+
digests as well -- the ones the opening pass computed and the ones the
|
|
35
|
+
transfer itself matched -- and never on a cheap size check, because a size
|
|
36
|
+
check would call a same-size tampered file sound and hand F-63's screen a
|
|
37
|
+
corrupt model described as ready.
|
|
38
|
+
* N-23 forbids a repeated request multiplying the work. Preparation of one
|
|
39
|
+
model is serialised for the whole process, so a second caller waits and
|
|
40
|
+
then finds the files already in place instead of streaming a second copy
|
|
41
|
+
through the same ``.part``.
|
|
42
|
+
* Section 5.3 forbids an external caller triggering an arbitrary large
|
|
43
|
+
download. A download asked for over REST or MCP is refused unless the
|
|
44
|
+
owner pre-authorised that model in the GUI.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
from __future__ import annotations
|
|
48
|
+
|
|
49
|
+
import errno
|
|
50
|
+
import hashlib
|
|
51
|
+
import json
|
|
52
|
+
import os
|
|
53
|
+
import shutil
|
|
54
|
+
import threading
|
|
55
|
+
from collections.abc import Callable, Iterable, Iterator
|
|
56
|
+
from contextlib import AbstractContextManager, contextmanager
|
|
57
|
+
from dataclasses import dataclass
|
|
58
|
+
from enum import StrEnum
|
|
59
|
+
from pathlib import Path
|
|
60
|
+
from typing import Protocol
|
|
61
|
+
|
|
62
|
+
from ..domain import Budget, RequestPath
|
|
63
|
+
from ..errors import Code, EchoActError
|
|
64
|
+
from ..paths import data_dir, model_cache_dir, redact
|
|
65
|
+
from ..policy import GIB, LOW_SPACE_WARNING_BYTES
|
|
66
|
+
from ..util.ids import now
|
|
67
|
+
from ..util.logging import get_logger
|
|
68
|
+
from .catalog import MANIFEST
|
|
69
|
+
from .manifest import LicenseTerms, Manifest, ModelEntry, ModelFile
|
|
70
|
+
|
|
71
|
+
log = get_logger("models.registry")
|
|
72
|
+
|
|
73
|
+
#: Read buffer for hashing and for streaming a download. Not a policy limit
|
|
74
|
+
#: -- it is the granularity at which N-21's cancellation and F-64's progress
|
|
75
|
+
#: are observable, and the size at which neither is expensive.
|
|
76
|
+
_CHUNK_BYTES = 1 << 20
|
|
77
|
+
|
|
78
|
+
#: Network timeouts for a model download. F-09 says preparation needs the
|
|
79
|
+
#: internet; nothing else in the product does, so these are not a policy
|
|
80
|
+
#: number shared with anything.
|
|
81
|
+
_CONNECT_TIMEOUT_S = 15.0
|
|
82
|
+
_READ_TIMEOUT_S = 60.0
|
|
83
|
+
|
|
84
|
+
#: How long a caller is told to wait before retrying a failed download.
|
|
85
|
+
_DOWNLOAD_RETRY_AFTER_S = 5.0
|
|
86
|
+
|
|
87
|
+
#: How long a caller queued behind another preparation of the same model
|
|
88
|
+
#: waits before looking at its cancel token again. N-22 allows five seconds
|
|
89
|
+
#: to stop; a queued attempt gives up well inside that.
|
|
90
|
+
_LOCK_POLL_S = 0.25
|
|
91
|
+
|
|
92
|
+
_PART_SUFFIX = ".part"
|
|
93
|
+
|
|
94
|
+
#: Where the ``supertonic`` package would keep the same weights. This is a
|
|
95
|
+
#: coupling to that package's layout, kept here rather than in the manifest
|
|
96
|
+
#: because it describes *another* program's cache, not the model.
|
|
97
|
+
_PACKAGE_CACHE_NAMES = {"supertonic-3": "supertonic3"}
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
# ======================================================================
|
|
101
|
+
# Cancellation
|
|
102
|
+
# ======================================================================
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class CancelToken:
|
|
106
|
+
"""A flag one thread sets and another notices between chunks.
|
|
107
|
+
|
|
108
|
+
Verification and download both run off the Qt main thread and both have
|
|
109
|
+
to stop promptly: N-22 gives five seconds to release resources, and a
|
|
110
|
+
256 MB hash does not fit in that if it can only be interrupted at file
|
|
111
|
+
boundaries.
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
__slots__ = ("_event",)
|
|
115
|
+
|
|
116
|
+
def __init__(self) -> None:
|
|
117
|
+
self._event = threading.Event()
|
|
118
|
+
|
|
119
|
+
def cancel(self) -> None:
|
|
120
|
+
self._event.set()
|
|
121
|
+
|
|
122
|
+
@property
|
|
123
|
+
def cancelled(self) -> bool:
|
|
124
|
+
return self._event.is_set()
|
|
125
|
+
|
|
126
|
+
def wait(self, timeout_s: float) -> bool:
|
|
127
|
+
return self._event.wait(timeout_s)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _cancelled(token: CancelToken | None) -> bool:
|
|
131
|
+
return token is not None and token.cancelled
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
# ======================================================================
|
|
135
|
+
# Verification results (F-65)
|
|
136
|
+
# ======================================================================
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class ModelState(StrEnum):
|
|
140
|
+
"""F-65's four answers to "are this model's files sound?".
|
|
141
|
+
|
|
142
|
+
``PARTIAL`` and ``CORRUPT`` are kept apart because they call for
|
|
143
|
+
different actions and F-64 treats them differently on retry: a partial
|
|
144
|
+
model needs the rest fetched, a corrupt one needs bad files replaced.
|
|
145
|
+
"""
|
|
146
|
+
|
|
147
|
+
NOT_PRESENT = "not_present"
|
|
148
|
+
PARTIAL = "partial"
|
|
149
|
+
CORRUPT = "corrupt"
|
|
150
|
+
READY = "ready"
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@dataclass(frozen=True, slots=True)
|
|
154
|
+
class FileStatus:
|
|
155
|
+
"""What verification learned about one manifest file."""
|
|
156
|
+
|
|
157
|
+
relative_path: str
|
|
158
|
+
expected_bytes: int
|
|
159
|
+
actual_bytes: int
|
|
160
|
+
present: bool
|
|
161
|
+
size_ok: bool
|
|
162
|
+
#: ``None`` when no digest was computed: the file was absent, the wrong
|
|
163
|
+
#: size, or the pass was a shallow one.
|
|
164
|
+
digest_ok: bool | None
|
|
165
|
+
#: ``False`` when verification stopped before reaching this file.
|
|
166
|
+
checked: bool = True
|
|
167
|
+
|
|
168
|
+
@property
|
|
169
|
+
def ok(self) -> bool:
|
|
170
|
+
return self.checked and self.present and self.size_ok and self.digest_ok is not False
|
|
171
|
+
|
|
172
|
+
@property
|
|
173
|
+
def corrupt(self) -> bool:
|
|
174
|
+
return self.present and (not self.size_ok or self.digest_ok is False)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
@dataclass(frozen=True, slots=True)
|
|
178
|
+
class VerifyReport:
|
|
179
|
+
"""The evidence behind a :class:`ModelState`.
|
|
180
|
+
|
|
181
|
+
F-63 shows disk usage and F-64 decides what to re-fetch, so the report
|
|
182
|
+
keeps per-file detail rather than collapsing to a verdict.
|
|
183
|
+
"""
|
|
184
|
+
|
|
185
|
+
model_id: str
|
|
186
|
+
root: Path
|
|
187
|
+
files: tuple[FileStatus, ...]
|
|
188
|
+
#: False for a presence-and-size pass, which cannot see a tampered file.
|
|
189
|
+
deep: bool
|
|
190
|
+
cancelled: bool = False
|
|
191
|
+
|
|
192
|
+
@property
|
|
193
|
+
def state(self) -> ModelState:
|
|
194
|
+
if any(f.corrupt for f in self.files):
|
|
195
|
+
return ModelState.CORRUPT
|
|
196
|
+
if all(f.ok for f in self.files):
|
|
197
|
+
return ModelState.READY
|
|
198
|
+
if any(f.present for f in self.files):
|
|
199
|
+
return ModelState.PARTIAL
|
|
200
|
+
return ModelState.NOT_PRESENT
|
|
201
|
+
|
|
202
|
+
@property
|
|
203
|
+
def bytes_present(self) -> int:
|
|
204
|
+
return sum(f.actual_bytes for f in self.files if f.present)
|
|
205
|
+
|
|
206
|
+
@property
|
|
207
|
+
def bytes_expected(self) -> int:
|
|
208
|
+
return sum(f.expected_bytes for f in self.files)
|
|
209
|
+
|
|
210
|
+
@property
|
|
211
|
+
def missing(self) -> tuple[str, ...]:
|
|
212
|
+
return tuple(f.relative_path for f in self.files if not f.present)
|
|
213
|
+
|
|
214
|
+
@property
|
|
215
|
+
def damaged(self) -> tuple[str, ...]:
|
|
216
|
+
return tuple(f.relative_path for f in self.files if f.corrupt)
|
|
217
|
+
|
|
218
|
+
@property
|
|
219
|
+
def unusable(self) -> tuple[str, ...]:
|
|
220
|
+
"""Everything a download would have to fetch to make this ready."""
|
|
221
|
+
return tuple(f.relative_path for f in self.files if not f.ok)
|
|
222
|
+
|
|
223
|
+
def status(self, relative_path: str) -> FileStatus | None:
|
|
224
|
+
for f in self.files:
|
|
225
|
+
if f.relative_path == relative_path:
|
|
226
|
+
return f
|
|
227
|
+
return None
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
# ======================================================================
|
|
231
|
+
# Download reporting (F-64)
|
|
232
|
+
# ======================================================================
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
class DownloadPhase(StrEnum):
|
|
236
|
+
CHECKING = "checking"
|
|
237
|
+
DOWNLOADING = "downloading"
|
|
238
|
+
COMPLETE = "complete"
|
|
239
|
+
CANCELLED = "cancelled"
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
@dataclass(frozen=True, slots=True)
|
|
243
|
+
class DownloadProgress:
|
|
244
|
+
"""One progress tick. F-64 requires progress to be visible, and F-63
|
|
245
|
+
requires the required storage to be known before the transfer starts, so
|
|
246
|
+
the totals are populated from the manifest rather than from the server."""
|
|
247
|
+
|
|
248
|
+
model_id: str
|
|
249
|
+
phase: DownloadPhase
|
|
250
|
+
relative_path: str
|
|
251
|
+
file_index: int
|
|
252
|
+
file_count: int
|
|
253
|
+
file_bytes_done: int
|
|
254
|
+
file_bytes_total: int
|
|
255
|
+
bytes_done: int
|
|
256
|
+
bytes_total: int
|
|
257
|
+
|
|
258
|
+
@property
|
|
259
|
+
def fraction(self) -> float:
|
|
260
|
+
if self.bytes_total <= 0:
|
|
261
|
+
return 1.0
|
|
262
|
+
return min(1.0, self.bytes_done / self.bytes_total)
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
ProgressCallback = Callable[[DownloadProgress], None]
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
@dataclass(frozen=True, slots=True)
|
|
269
|
+
class DownloadOutcome:
|
|
270
|
+
"""What one preparation attempt did.
|
|
271
|
+
|
|
272
|
+
Cancellation is an outcome, not an error: F-64 makes it a first-class
|
|
273
|
+
control, and raising for it would force every caller to catch an
|
|
274
|
+
exception for something the user asked for. A failure that the user did
|
|
275
|
+
not ask for still raises, so the two are never confused.
|
|
276
|
+
"""
|
|
277
|
+
|
|
278
|
+
model_id: str
|
|
279
|
+
completed: bool
|
|
280
|
+
cancelled: bool
|
|
281
|
+
bytes_downloaded: int
|
|
282
|
+
reused: tuple[str, ...]
|
|
283
|
+
fetched: tuple[str, ...]
|
|
284
|
+
report: VerifyReport
|
|
285
|
+
|
|
286
|
+
@property
|
|
287
|
+
def state(self) -> ModelState:
|
|
288
|
+
return self.report.state
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
# ======================================================================
|
|
292
|
+
# Where acceptance and authorisation are recorded (N-11, 5.3)
|
|
293
|
+
# ======================================================================
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
class ModelPreferences(Protocol):
|
|
297
|
+
"""The two owner decisions the lifecycle has to remember.
|
|
298
|
+
|
|
299
|
+
N-11 puts licence acceptance in settings, and 5.3 puts the
|
|
300
|
+
pre-authorisation of a model for external download there too. Both are
|
|
301
|
+
keyed by a fingerprint of the terms accepted, not by model id alone: if
|
|
302
|
+
a later release ships different restrictions, the old acceptance does
|
|
303
|
+
not silently cover them.
|
|
304
|
+
"""
|
|
305
|
+
|
|
306
|
+
def license_accepted(self, model_id: str, fingerprint: str) -> bool: ...
|
|
307
|
+
|
|
308
|
+
def record_license_acceptance(self, model_id: str, fingerprint: str) -> None: ...
|
|
309
|
+
|
|
310
|
+
def download_authorised(self, model_id: str) -> bool: ...
|
|
311
|
+
|
|
312
|
+
def set_download_authorised(self, model_id: str, allowed: bool) -> None: ...
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
class JsonModelPreferences:
|
|
316
|
+
"""A small JSON file beside the settings, used until
|
|
317
|
+
``echoact.config.settings`` exists to hold these two records.
|
|
318
|
+
|
|
319
|
+
It fails closed: a file that cannot be read counts as nothing accepted
|
|
320
|
+
and nothing authorised, because the failure mode of guessing the other
|
|
321
|
+
way is preparing a model whose terms the user never saw.
|
|
322
|
+
"""
|
|
323
|
+
|
|
324
|
+
def __init__(self, path: Path | None = None) -> None:
|
|
325
|
+
self._path = path
|
|
326
|
+
|
|
327
|
+
@property
|
|
328
|
+
def path(self) -> Path:
|
|
329
|
+
# Resolved per call, never at construction: ``paths.data_dir`` is
|
|
330
|
+
# cached and tests redirect it after this object exists.
|
|
331
|
+
return self._path if self._path is not None else data_dir() / "model_state.json"
|
|
332
|
+
|
|
333
|
+
def _read(self) -> dict[str, dict[str, object]]:
|
|
334
|
+
try:
|
|
335
|
+
raw = json.loads(self.path.read_text(encoding="utf-8"))
|
|
336
|
+
except (OSError, ValueError):
|
|
337
|
+
return {}
|
|
338
|
+
return raw if isinstance(raw, dict) else {}
|
|
339
|
+
|
|
340
|
+
def _write(self, data: dict[str, dict[str, object]]) -> None:
|
|
341
|
+
path = self.path
|
|
342
|
+
try:
|
|
343
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
344
|
+
tmp = path.with_suffix(".json.tmp")
|
|
345
|
+
tmp.write_text(json.dumps(data, indent=2, sort_keys=True), encoding="utf-8")
|
|
346
|
+
os.replace(tmp, path)
|
|
347
|
+
except OSError as exc:
|
|
348
|
+
code = Code.STORAGE_FULL if exc.errno == errno.ENOSPC else Code.INTERNAL
|
|
349
|
+
raise EchoActError(
|
|
350
|
+
code,
|
|
351
|
+
"The model licence decision could not be saved.",
|
|
352
|
+
detail={"path": redact(path)},
|
|
353
|
+
cause=exc,
|
|
354
|
+
) from exc
|
|
355
|
+
|
|
356
|
+
def license_accepted(self, model_id: str, fingerprint: str) -> bool:
|
|
357
|
+
record = self._read().get("licenses_accepted", {})
|
|
358
|
+
entry = record.get(model_id) if isinstance(record, dict) else None
|
|
359
|
+
return isinstance(entry, dict) and entry.get("fingerprint") == fingerprint
|
|
360
|
+
|
|
361
|
+
def record_license_acceptance(self, model_id: str, fingerprint: str) -> None:
|
|
362
|
+
data = self._read()
|
|
363
|
+
accepted = data.get("licenses_accepted")
|
|
364
|
+
if not isinstance(accepted, dict):
|
|
365
|
+
accepted = {}
|
|
366
|
+
accepted[model_id] = {"fingerprint": fingerprint, "accepted_at": now()}
|
|
367
|
+
data["licenses_accepted"] = accepted
|
|
368
|
+
self._write(data)
|
|
369
|
+
|
|
370
|
+
def download_authorised(self, model_id: str) -> bool:
|
|
371
|
+
record = self._read().get("downloads_authorised", {})
|
|
372
|
+
return isinstance(record, dict) and record.get(model_id) is True
|
|
373
|
+
|
|
374
|
+
def set_download_authorised(self, model_id: str, allowed: bool) -> None:
|
|
375
|
+
data = self._read()
|
|
376
|
+
authorised = data.get("downloads_authorised")
|
|
377
|
+
if not isinstance(authorised, dict):
|
|
378
|
+
authorised = {}
|
|
379
|
+
authorised[model_id] = bool(allowed)
|
|
380
|
+
data["downloads_authorised"] = authorised
|
|
381
|
+
self._write(data)
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
# ======================================================================
|
|
385
|
+
# Transport (F-09, F-64)
|
|
386
|
+
# ======================================================================
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
@dataclass(slots=True)
|
|
390
|
+
class RemoteBody:
|
|
391
|
+
"""A response being streamed. ``resumed`` says whether the server
|
|
392
|
+
honoured a Range request; if it did not, the caller must start over
|
|
393
|
+
rather than append the whole file to a partial one."""
|
|
394
|
+
|
|
395
|
+
chunks: Iterator[bytes]
|
|
396
|
+
resumed: bool
|
|
397
|
+
total_bytes: int | None
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
class Fetcher(Protocol):
|
|
401
|
+
def open(self, url: str, *, offset: int = 0) -> AbstractContextManager[RemoteBody]: ...
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
class HttpxFetcher:
|
|
405
|
+
"""HTTPS streaming, with the range request F-64's retry depends on.
|
|
406
|
+
|
|
407
|
+
``huggingface_hub`` is installed (a dependency of ``supertonic``) and is
|
|
408
|
+
used for the one thing it is authoritative about -- building the resolve
|
|
409
|
+
URL for a repository at a pinned revision. Its ``hf_hub_download`` is
|
|
410
|
+
not used to transfer: it cannot be cancelled part-way through a 256 MB
|
|
411
|
+
file and reports progress only through a tqdm bar, and F-64 requires
|
|
412
|
+
both. Streaming here also keeps the bytes going straight into the app's
|
|
413
|
+
own cache instead of the hub's parallel one.
|
|
414
|
+
"""
|
|
415
|
+
|
|
416
|
+
def __init__(self, *, connect_timeout_s: float = _CONNECT_TIMEOUT_S,
|
|
417
|
+
read_timeout_s: float = _READ_TIMEOUT_S) -> None:
|
|
418
|
+
self._connect_timeout_s = connect_timeout_s
|
|
419
|
+
self._read_timeout_s = read_timeout_s
|
|
420
|
+
|
|
421
|
+
@contextmanager
|
|
422
|
+
def open(self, url: str, *, offset: int = 0) -> Iterator[RemoteBody]:
|
|
423
|
+
import httpx # imported here so a GUI-only start-up never pays for it
|
|
424
|
+
|
|
425
|
+
headers = {"Range": f"bytes={offset}-"} if offset > 0 else {}
|
|
426
|
+
timeout = httpx.Timeout(
|
|
427
|
+
self._read_timeout_s, connect=self._connect_timeout_s, read=self._read_timeout_s
|
|
428
|
+
)
|
|
429
|
+
try:
|
|
430
|
+
with httpx.Client(timeout=timeout, follow_redirects=True) as client:
|
|
431
|
+
with client.stream("GET", url, headers=headers) as response:
|
|
432
|
+
if response.status_code >= 400:
|
|
433
|
+
response.read()
|
|
434
|
+
raise EchoActError(
|
|
435
|
+
Code.MODEL_DOWNLOAD_FAILED,
|
|
436
|
+
f"The model server answered {response.status_code}.",
|
|
437
|
+
detail={"status": response.status_code},
|
|
438
|
+
retry_after_s=_DOWNLOAD_RETRY_AFTER_S,
|
|
439
|
+
)
|
|
440
|
+
length = response.headers.get("content-length")
|
|
441
|
+
yield RemoteBody(
|
|
442
|
+
chunks=response.iter_bytes(_CHUNK_BYTES),
|
|
443
|
+
resumed=offset > 0 and response.status_code == 206,
|
|
444
|
+
total_bytes=int(length) if length and length.isdigit() else None,
|
|
445
|
+
)
|
|
446
|
+
except httpx.HTTPError as exc:
|
|
447
|
+
# 5.3: a network outage says why preparation failed and stays
|
|
448
|
+
# retryable; it never touches an already-prepared model.
|
|
449
|
+
raise EchoActError(
|
|
450
|
+
Code.MODEL_DOWNLOAD_FAILED,
|
|
451
|
+
"The model could not be downloaded: the connection failed.",
|
|
452
|
+
detail={"reason": type(exc).__name__},
|
|
453
|
+
retry_after_s=_DOWNLOAD_RETRY_AFTER_S,
|
|
454
|
+
cause=exc,
|
|
455
|
+
) from exc
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def resolve_url(entry: ModelEntry, file: ModelFile) -> str:
|
|
459
|
+
"""The pinned-revision URL for one manifest file."""
|
|
460
|
+
from huggingface_hub import hf_hub_url
|
|
461
|
+
|
|
462
|
+
return hf_hub_url(entry.repo_id, file.relative_path, revision=entry.revision)
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
# ======================================================================
|
|
466
|
+
# Status projection (F-53, F-63)
|
|
467
|
+
# ======================================================================
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
@dataclass(frozen=True, slots=True)
|
|
471
|
+
class ModelStatus:
|
|
472
|
+
"""One row of F-63's model management screen, and F-53's model entry.
|
|
473
|
+
|
|
474
|
+
``runnable`` and ``unavailable_reason`` travel together because F-04
|
|
475
|
+
forbids hiding a model that cannot run in the current budget and forbids
|
|
476
|
+
substituting another for it: the only correct presentation is the model,
|
|
477
|
+
shown as unavailable, with the reason.
|
|
478
|
+
"""
|
|
479
|
+
|
|
480
|
+
model_id: str
|
|
481
|
+
display_name: str
|
|
482
|
+
state: ModelState
|
|
483
|
+
sample_rate: int
|
|
484
|
+
languages: tuple[str, ...]
|
|
485
|
+
bytes_present: int
|
|
486
|
+
bytes_total: int
|
|
487
|
+
disk_bytes: int
|
|
488
|
+
runnable: bool
|
|
489
|
+
unavailable_reason: str | None
|
|
490
|
+
license_name: str
|
|
491
|
+
license_acceptance_required: bool
|
|
492
|
+
license_accepted: bool
|
|
493
|
+
download_authorised: bool
|
|
494
|
+
#: True when the only sound copy is the ``supertonic`` package's cache.
|
|
495
|
+
using_package_cache: bool
|
|
496
|
+
minimum_memory_bytes: int
|
|
497
|
+
minimum_cpu_percent: int
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
# ======================================================================
|
|
501
|
+
# The registry
|
|
502
|
+
# ======================================================================
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
class ModelRegistry:
|
|
506
|
+
"""F-63 to F-65's operations over F-84's manifest."""
|
|
507
|
+
|
|
508
|
+
def __init__(
|
|
509
|
+
self,
|
|
510
|
+
manifest: Manifest = MANIFEST,
|
|
511
|
+
*,
|
|
512
|
+
root: Path | None = None,
|
|
513
|
+
preferences: ModelPreferences | None = None,
|
|
514
|
+
fetcher: Fetcher | None = None,
|
|
515
|
+
package_cache_dirs: dict[str, Path] | None = None,
|
|
516
|
+
) -> None:
|
|
517
|
+
self.manifest = manifest
|
|
518
|
+
self._root = root
|
|
519
|
+
self.preferences: ModelPreferences = preferences or JsonModelPreferences()
|
|
520
|
+
self._fetcher = fetcher
|
|
521
|
+
self._package_cache_dirs = package_cache_dirs
|
|
522
|
+
|
|
523
|
+
# -- locations ------------------------------------------------------
|
|
524
|
+
|
|
525
|
+
@property
|
|
526
|
+
def root(self) -> Path:
|
|
527
|
+
"""Resolved per call: ``paths.model_cache_dir`` reads the
|
|
528
|
+
environment through an ``lru_cache`` that a test may have cleared."""
|
|
529
|
+
return self._root if self._root is not None else model_cache_dir()
|
|
530
|
+
|
|
531
|
+
def entry(self, model_id: str) -> ModelEntry:
|
|
532
|
+
return self.manifest.get(model_id)
|
|
533
|
+
|
|
534
|
+
def model_dir(self, model_id: str) -> Path:
|
|
535
|
+
"""The app-owned directory for a model's files."""
|
|
536
|
+
return self.root / self.entry(model_id).model_id
|
|
537
|
+
|
|
538
|
+
def package_cache_dir(self, model_id: str) -> Path | None:
|
|
539
|
+
"""Where the ``supertonic`` package keeps the same weights, if the
|
|
540
|
+
model is one it knows about.
|
|
541
|
+
|
|
542
|
+
Read-only as far as this app is concerned. F-73 reports it apart
|
|
543
|
+
from the app's own cache, and F-65's delete does not touch it: it
|
|
544
|
+
belongs to another program and may be shared with other tools.
|
|
545
|
+
"""
|
|
546
|
+
if self._package_cache_dirs is not None:
|
|
547
|
+
return self._package_cache_dirs.get(model_id)
|
|
548
|
+
name = _PACKAGE_CACHE_NAMES.get(model_id)
|
|
549
|
+
if name is None:
|
|
550
|
+
return None
|
|
551
|
+
override = os.environ.get("SUPERTONIC_CACHE_DIR")
|
|
552
|
+
if override:
|
|
553
|
+
return Path(override).expanduser()
|
|
554
|
+
return Path.home() / ".cache" / name
|
|
555
|
+
|
|
556
|
+
# -- verification (F-65) --------------------------------------------
|
|
557
|
+
|
|
558
|
+
def verify(
|
|
559
|
+
self,
|
|
560
|
+
model_id: str,
|
|
561
|
+
*,
|
|
562
|
+
deep: bool = True,
|
|
563
|
+
cancel: CancelToken | None = None,
|
|
564
|
+
progress: ProgressCallback | None = None,
|
|
565
|
+
root: Path | None = None,
|
|
566
|
+
) -> VerifyReport:
|
|
567
|
+
"""Check every manifest file's size and, when ``deep``, its digest.
|
|
568
|
+
|
|
569
|
+
Size first because a truncated download is the common case and
|
|
570
|
+
hashing it would be a second wasted; the digest is what catches the
|
|
571
|
+
tampering F-84 and A-23 care about. Reading is chunked and checks
|
|
572
|
+
the cancel token between chunks, per N-21.
|
|
573
|
+
|
|
574
|
+
A pass cut short cannot report ``READY``: a file it never reached,
|
|
575
|
+
or whose hash it abandoned part-way, counts as unchecked, and an
|
|
576
|
+
unchecked file is not sound.
|
|
577
|
+
"""
|
|
578
|
+
entry = self.entry(model_id)
|
|
579
|
+
base = root if root is not None else self.model_dir(model_id)
|
|
580
|
+
statuses: list[FileStatus] = []
|
|
581
|
+
cancelled = False
|
|
582
|
+
done = 0
|
|
583
|
+
total = entry.total_bytes
|
|
584
|
+
|
|
585
|
+
for index, file in enumerate(entry.files):
|
|
586
|
+
if _cancelled(cancel):
|
|
587
|
+
cancelled = True
|
|
588
|
+
statuses.extend(_unchecked(f) for f in entry.files[index:])
|
|
589
|
+
break
|
|
590
|
+
status, read = self._verify_file(file, base, deep=deep, cancel=cancel)
|
|
591
|
+
statuses.append(status)
|
|
592
|
+
done += read
|
|
593
|
+
if progress is not None:
|
|
594
|
+
progress(
|
|
595
|
+
DownloadProgress(
|
|
596
|
+
model_id=model_id,
|
|
597
|
+
phase=DownloadPhase.CHECKING,
|
|
598
|
+
relative_path=file.relative_path,
|
|
599
|
+
file_index=index,
|
|
600
|
+
file_count=entry.file_count,
|
|
601
|
+
file_bytes_done=status.actual_bytes,
|
|
602
|
+
file_bytes_total=file.byte_size,
|
|
603
|
+
bytes_done=done,
|
|
604
|
+
bytes_total=total,
|
|
605
|
+
)
|
|
606
|
+
)
|
|
607
|
+
if _cancelled(cancel):
|
|
608
|
+
cancelled = True
|
|
609
|
+
statuses.extend(_unchecked(f) for f in entry.files[index + 1 :])
|
|
610
|
+
break
|
|
611
|
+
|
|
612
|
+
return VerifyReport(
|
|
613
|
+
model_id=model_id,
|
|
614
|
+
root=base,
|
|
615
|
+
files=tuple(statuses),
|
|
616
|
+
deep=deep,
|
|
617
|
+
cancelled=cancelled,
|
|
618
|
+
)
|
|
619
|
+
|
|
620
|
+
def _verify_file(
|
|
621
|
+
self, file: ModelFile, base: Path, *, deep: bool, cancel: CancelToken | None
|
|
622
|
+
) -> tuple[FileStatus, int]:
|
|
623
|
+
path = file.path_under(base)
|
|
624
|
+
try:
|
|
625
|
+
size = path.stat().st_size
|
|
626
|
+
except OSError:
|
|
627
|
+
return (
|
|
628
|
+
FileStatus(
|
|
629
|
+
relative_path=file.relative_path,
|
|
630
|
+
expected_bytes=file.byte_size,
|
|
631
|
+
actual_bytes=0,
|
|
632
|
+
present=False,
|
|
633
|
+
size_ok=False,
|
|
634
|
+
digest_ok=None,
|
|
635
|
+
),
|
|
636
|
+
0,
|
|
637
|
+
)
|
|
638
|
+
size_ok = size == file.byte_size
|
|
639
|
+
digest_ok: bool | None = None
|
|
640
|
+
read = 0
|
|
641
|
+
if size_ok and deep:
|
|
642
|
+
digest, read = _sha256_file(path, cancel=cancel)
|
|
643
|
+
digest_ok = digest == file.sha256 if digest is not None else None
|
|
644
|
+
return (
|
|
645
|
+
FileStatus(
|
|
646
|
+
relative_path=file.relative_path,
|
|
647
|
+
expected_bytes=file.byte_size,
|
|
648
|
+
actual_bytes=size,
|
|
649
|
+
present=True,
|
|
650
|
+
size_ok=size_ok,
|
|
651
|
+
digest_ok=digest_ok,
|
|
652
|
+
checked=not (size_ok and deep and digest_ok is None),
|
|
653
|
+
),
|
|
654
|
+
read,
|
|
655
|
+
)
|
|
656
|
+
|
|
657
|
+
def state(self, model_id: str, *, cancel: CancelToken | None = None) -> ModelState:
|
|
658
|
+
"""F-65's question, answered against digests."""
|
|
659
|
+
return self.verify(model_id, deep=True, cancel=cancel).state
|
|
660
|
+
|
|
661
|
+
def quick_state(self, model_id: str) -> ModelState:
|
|
662
|
+
"""Presence and size only.
|
|
663
|
+
|
|
664
|
+
For a screen that refreshes: hashing 385 MB to redraw a list is
|
|
665
|
+
wasteful, and a size check catches everything except deliberate
|
|
666
|
+
tampering. Anything that is about to *load* the model calls
|
|
667
|
+
:meth:`prepared_dir`, which is deep.
|
|
668
|
+
"""
|
|
669
|
+
return self.verify(model_id, deep=False).state
|
|
670
|
+
|
|
671
|
+
def resolve_dir(
|
|
672
|
+
self, model_id: str, *, deep: bool = True, cancel: CancelToken | None = None
|
|
673
|
+
) -> tuple[Path, bool]:
|
|
674
|
+
"""The directory holding a sound copy, and whether it is the
|
|
675
|
+
package's cache rather than the app's own.
|
|
676
|
+
|
|
677
|
+
Raises rather than returning a doubtful path: F-84 says a model whose
|
|
678
|
+
files do not match the manifest is reported as corrupted, not used.
|
|
679
|
+
|
|
680
|
+
The licence is checked here and not only in :meth:`download`, because
|
|
681
|
+
this is the path a model actually reaches the engine by (N-11, F-80,
|
|
682
|
+
A-23). Neither case that matters passes through a download: the
|
|
683
|
+
package cache is read where it lies, and a release that amends
|
|
684
|
+
Attachment A leaves the files READY, so nothing would ever ask again.
|
|
685
|
+
|
|
686
|
+
It is checked *after* the files are found, not before. N-11 makes
|
|
687
|
+
acceptance a condition of preparing a model, so for a model that is
|
|
688
|
+
not on this machine the useful answer is that it is not prepared --
|
|
689
|
+
which is the flow that presents the terms. Reporting an unaccepted
|
|
690
|
+
licence for a model that also is not there would be true and
|
|
691
|
+
useless, and would send the owner to the wrong screen.
|
|
692
|
+
"""
|
|
693
|
+
own = self.verify(model_id, deep=deep, cancel=cancel)
|
|
694
|
+
if own.state is ModelState.READY:
|
|
695
|
+
self._require_license(self.entry(model_id))
|
|
696
|
+
return self.model_dir(model_id), False
|
|
697
|
+
|
|
698
|
+
fallback = self.package_cache_dir(model_id)
|
|
699
|
+
if fallback is not None and fallback.is_dir():
|
|
700
|
+
other = self.verify(model_id, deep=deep, cancel=cancel, root=fallback)
|
|
701
|
+
if other.state is ModelState.READY:
|
|
702
|
+
# The case the gate exists for: borrowed weights reach the
|
|
703
|
+
# engine without a download ever being asked for.
|
|
704
|
+
self._require_license(self.entry(model_id))
|
|
705
|
+
log.info("model %s served from the package cache %s", model_id, redact(fallback))
|
|
706
|
+
return fallback, True
|
|
707
|
+
|
|
708
|
+
if own.state is ModelState.CORRUPT:
|
|
709
|
+
raise EchoActError(
|
|
710
|
+
Code.MODEL_CORRUPT,
|
|
711
|
+
detail={"model_id": model_id, "files": list(own.damaged)},
|
|
712
|
+
)
|
|
713
|
+
raise EchoActError(
|
|
714
|
+
Code.MODEL_NOT_READY,
|
|
715
|
+
detail={"model_id": model_id, "missing": len(own.unusable)},
|
|
716
|
+
)
|
|
717
|
+
|
|
718
|
+
def prepared_dir(
|
|
719
|
+
self,
|
|
720
|
+
model_id: str,
|
|
721
|
+
*,
|
|
722
|
+
budget: Budget | None = None,
|
|
723
|
+
deep: bool = True,
|
|
724
|
+
cancel: CancelToken | None = None,
|
|
725
|
+
) -> Path:
|
|
726
|
+
"""What the engine is given for ``Load.model_dir``.
|
|
727
|
+
|
|
728
|
+
The budget check happens here as well as on the selection screen
|
|
729
|
+
because F-04's promise is that a model that cannot run is refused
|
|
730
|
+
with a reason, and the last chance to keep that promise is the
|
|
731
|
+
moment before loading.
|
|
732
|
+
"""
|
|
733
|
+
if budget is not None:
|
|
734
|
+
self.ensure_can_run(model_id, budget)
|
|
735
|
+
path, _ = self.resolve_dir(model_id, deep=deep, cancel=cancel)
|
|
736
|
+
return path
|
|
737
|
+
|
|
738
|
+
# -- budget (F-04, N-05) --------------------------------------------
|
|
739
|
+
|
|
740
|
+
def can_run(self, model_id: str, budget: Budget) -> tuple[bool, str | None]:
|
|
741
|
+
"""F-04: a model that cannot run under the current budget is shown
|
|
742
|
+
as unavailable *with the reason*, never hidden and never replaced.
|
|
743
|
+
|
|
744
|
+
Returns ``(True, None)`` or ``(False, reason)``. The reason is
|
|
745
|
+
written for a person and repeated verbatim to an API caller, so it
|
|
746
|
+
names both the requirement and the current value.
|
|
747
|
+
"""
|
|
748
|
+
entry = self.entry(model_id)
|
|
749
|
+
minimum = entry.minimum_budget
|
|
750
|
+
reasons: list[str] = []
|
|
751
|
+
if budget.memory_bytes < minimum.memory_bytes:
|
|
752
|
+
reasons.append(
|
|
753
|
+
f"{entry.display_name} is approved from a memory budget of "
|
|
754
|
+
f"{_gib(minimum.memory_bytes)}; the current budget is "
|
|
755
|
+
f"{_gib(budget.memory_bytes)}"
|
|
756
|
+
)
|
|
757
|
+
if budget.cpu_percent < minimum.cpu_percent:
|
|
758
|
+
reasons.append(
|
|
759
|
+
f"{entry.display_name} is approved from a CPU budget of "
|
|
760
|
+
f"{minimum.cpu_percent}%; the current budget is {budget.cpu_percent}%"
|
|
761
|
+
)
|
|
762
|
+
if reasons:
|
|
763
|
+
return False, "; ".join(reasons) + "."
|
|
764
|
+
return True, None
|
|
765
|
+
|
|
766
|
+
def ensure_can_run(self, model_id: str, budget: Budget) -> None:
|
|
767
|
+
runnable, reason = self.can_run(model_id, budget)
|
|
768
|
+
if not runnable:
|
|
769
|
+
raise EchoActError(
|
|
770
|
+
Code.MODEL_OVER_BUDGET,
|
|
771
|
+
reason,
|
|
772
|
+
detail={
|
|
773
|
+
"model_id": model_id,
|
|
774
|
+
"minimum_memory_bytes": self.entry(model_id).minimum_budget.memory_bytes,
|
|
775
|
+
"minimum_cpu_percent": self.entry(model_id).minimum_budget.cpu_percent,
|
|
776
|
+
},
|
|
777
|
+
)
|
|
778
|
+
|
|
779
|
+
# -- licence (N-11, F-80) -------------------------------------------
|
|
780
|
+
|
|
781
|
+
def license_terms(self, model_id: str) -> LicenseTerms:
|
|
782
|
+
return self.entry(model_id).license
|
|
783
|
+
|
|
784
|
+
def license_acceptance_required(self, model_id: str) -> bool:
|
|
785
|
+
"""True while N-11's terms still have to be shown and accepted.
|
|
786
|
+
|
|
787
|
+
The fingerprint covers the restrictions themselves, so a release
|
|
788
|
+
that ships changed terms asks again rather than inheriting consent
|
|
789
|
+
the user gave to different text.
|
|
790
|
+
"""
|
|
791
|
+
terms = self.license_terms(model_id)
|
|
792
|
+
if not terms.acceptance_required:
|
|
793
|
+
return False
|
|
794
|
+
return not self.preferences.license_accepted(model_id, _fingerprint(terms))
|
|
795
|
+
|
|
796
|
+
def accept_license(self, model_id: str) -> None:
|
|
797
|
+
"""Record that the owner accepted this model's terms (N-11)."""
|
|
798
|
+
terms = self.license_terms(model_id)
|
|
799
|
+
self.preferences.record_license_acceptance(model_id, _fingerprint(terms))
|
|
800
|
+
log.info("licence accepted for model %s (%s)", model_id, terms.name)
|
|
801
|
+
|
|
802
|
+
# -- external authorisation (5.3) -----------------------------------
|
|
803
|
+
|
|
804
|
+
def download_authorised(self, model_id: str) -> bool:
|
|
805
|
+
self.entry(model_id)
|
|
806
|
+
return self.preferences.download_authorised(model_id)
|
|
807
|
+
|
|
808
|
+
def set_download_authorised(self, model_id: str, allowed: bool) -> None:
|
|
809
|
+
"""The GUI's pre-authorisation switch from Section 5.3.
|
|
810
|
+
|
|
811
|
+
Without it, a REST or MCP request can never start a download: the
|
|
812
|
+
rule exists so that an integration cannot make the machine fetch
|
|
813
|
+
hundreds of megabytes on its own initiative.
|
|
814
|
+
"""
|
|
815
|
+
self.entry(model_id)
|
|
816
|
+
self.preferences.set_download_authorised(model_id, allowed)
|
|
817
|
+
log.info("download authorisation for %s set to %s", model_id, bool(allowed))
|
|
818
|
+
|
|
819
|
+
# -- preparation (F-09, F-64) ---------------------------------------
|
|
820
|
+
|
|
821
|
+
def download(
|
|
822
|
+
self,
|
|
823
|
+
model_id: str,
|
|
824
|
+
progress_cb: ProgressCallback | None = None,
|
|
825
|
+
cancel: CancelToken | None = None,
|
|
826
|
+
*,
|
|
827
|
+
request_path: RequestPath = RequestPath.GUI,
|
|
828
|
+
discard_partial: bool = False,
|
|
829
|
+
) -> DownloadOutcome:
|
|
830
|
+
"""Fetch whatever is missing or unsound, and nothing else (F-64).
|
|
831
|
+
|
|
832
|
+
The work is decided by a verification pass, so a retry after a
|
|
833
|
+
failure re-downloads exactly the files that are absent or corrupt and
|
|
834
|
+
reuses the rest. Within a file, a ``.part`` left by an earlier
|
|
835
|
+
attempt is resumed from with a range request; if the server ignores
|
|
836
|
+
the range, the transfer restarts rather than appending to it.
|
|
837
|
+
|
|
838
|
+
Nothing is renamed into place until its digest matches the manifest,
|
|
839
|
+
which is what makes F-64's "a cancelled download is never shown as
|
|
840
|
+
ready" true by construction rather than by a flag someone has to
|
|
841
|
+
remember to clear.
|
|
842
|
+
|
|
843
|
+
Only one attempt per model runs at a time (N-23). A second caller --
|
|
844
|
+
the GUI and a REST request, two REST requests, a repair overtaking a
|
|
845
|
+
download -- waits, and then its own verification pass finds the work
|
|
846
|
+
already done and fetches nothing.
|
|
847
|
+
"""
|
|
848
|
+
entry = self.entry(model_id)
|
|
849
|
+
self._require_license(entry)
|
|
850
|
+
self._require_authorisation(entry, request_path)
|
|
851
|
+
|
|
852
|
+
with _preparing(self.model_dir(model_id), cancel) as held:
|
|
853
|
+
if not held:
|
|
854
|
+
# Cancelled while queued behind another attempt. Nothing was
|
|
855
|
+
# examined, so nothing is claimed: the pass below runs with an
|
|
856
|
+
# already-cancelled token and reports every file unchecked.
|
|
857
|
+
return self._cancelled_outcome(
|
|
858
|
+
entry, self.verify(model_id, cancel=cancel), progress_cb, 0, (), ()
|
|
859
|
+
)
|
|
860
|
+
return self._prepare(entry, progress_cb, cancel, discard_partial=discard_partial)
|
|
861
|
+
|
|
862
|
+
def _prepare(
|
|
863
|
+
self,
|
|
864
|
+
entry: ModelEntry,
|
|
865
|
+
progress_cb: ProgressCallback | None,
|
|
866
|
+
cancel: CancelToken | None,
|
|
867
|
+
*,
|
|
868
|
+
discard_partial: bool,
|
|
869
|
+
) -> DownloadOutcome:
|
|
870
|
+
""":meth:`download`'s body, with this model's preparation lock held."""
|
|
871
|
+
model_id = entry.model_id
|
|
872
|
+
report = self.verify(model_id, cancel=cancel, progress=progress_cb)
|
|
873
|
+
if report.cancelled:
|
|
874
|
+
return self._cancelled_outcome(entry, report, progress_cb, 0, (), ())
|
|
875
|
+
|
|
876
|
+
todo = [f for f in entry.files if not _is_ok(report, f.relative_path)]
|
|
877
|
+
reused = tuple(f.relative_path for f in entry.files if _is_ok(report, f.relative_path))
|
|
878
|
+
if not todo:
|
|
879
|
+
self._emit(
|
|
880
|
+
progress_cb,
|
|
881
|
+
entry,
|
|
882
|
+
DownloadPhase.COMPLETE,
|
|
883
|
+
None,
|
|
884
|
+
0,
|
|
885
|
+
report.bytes_expected,
|
|
886
|
+
report.bytes_expected,
|
|
887
|
+
)
|
|
888
|
+
return DownloadOutcome(
|
|
889
|
+
model_id=model_id,
|
|
890
|
+
completed=True,
|
|
891
|
+
cancelled=False,
|
|
892
|
+
bytes_downloaded=0,
|
|
893
|
+
reused=reused,
|
|
894
|
+
fetched=(),
|
|
895
|
+
report=report,
|
|
896
|
+
)
|
|
897
|
+
|
|
898
|
+
base = self.model_dir(model_id)
|
|
899
|
+
remaining = sum(f.byte_size for f in todo)
|
|
900
|
+
self._require_space(base, remaining)
|
|
901
|
+
|
|
902
|
+
fetcher = self._fetcher if self._fetcher is not None else HttpxFetcher()
|
|
903
|
+
already = report.bytes_expected - remaining
|
|
904
|
+
downloaded = 0
|
|
905
|
+
fetched: list[str] = []
|
|
906
|
+
|
|
907
|
+
for index, file in enumerate(todo):
|
|
908
|
+
if _cancelled(cancel):
|
|
909
|
+
return self._cancelled_outcome(
|
|
910
|
+
entry,
|
|
911
|
+
self._interrupted_report(entry, report, fetched, None),
|
|
912
|
+
progress_cb,
|
|
913
|
+
downloaded,
|
|
914
|
+
reused,
|
|
915
|
+
tuple(fetched),
|
|
916
|
+
)
|
|
917
|
+
|
|
918
|
+
def tick(
|
|
919
|
+
file_done: int,
|
|
920
|
+
*,
|
|
921
|
+
current: ModelFile = file,
|
|
922
|
+
position: int = index,
|
|
923
|
+
prior: int = already + downloaded,
|
|
924
|
+
) -> None:
|
|
925
|
+
self._emit(
|
|
926
|
+
progress_cb,
|
|
927
|
+
entry,
|
|
928
|
+
DownloadPhase.DOWNLOADING,
|
|
929
|
+
current,
|
|
930
|
+
file_done,
|
|
931
|
+
prior + file_done,
|
|
932
|
+
report.bytes_expected,
|
|
933
|
+
file_index=position,
|
|
934
|
+
file_count=len(todo),
|
|
935
|
+
)
|
|
936
|
+
|
|
937
|
+
written, cancelled = self._download_file(
|
|
938
|
+
entry,
|
|
939
|
+
file,
|
|
940
|
+
base,
|
|
941
|
+
fetcher,
|
|
942
|
+
cancel=cancel,
|
|
943
|
+
discard_partial=discard_partial,
|
|
944
|
+
on_chunk=tick,
|
|
945
|
+
)
|
|
946
|
+
downloaded += written
|
|
947
|
+
if cancelled:
|
|
948
|
+
return self._cancelled_outcome(
|
|
949
|
+
entry,
|
|
950
|
+
self._interrupted_report(entry, report, fetched, file.relative_path),
|
|
951
|
+
progress_cb,
|
|
952
|
+
downloaded,
|
|
953
|
+
reused,
|
|
954
|
+
tuple(fetched),
|
|
955
|
+
)
|
|
956
|
+
fetched.append(file.relative_path)
|
|
957
|
+
|
|
958
|
+
final = self.verify(model_id, cancel=cancel)
|
|
959
|
+
if final.cancelled:
|
|
960
|
+
# Cancelled after the last byte landed: the files may well be
|
|
961
|
+
# sound, but this pass did not establish that, and F-64 would
|
|
962
|
+
# rather report the cancellation than a readiness it guessed.
|
|
963
|
+
return self._cancelled_outcome(
|
|
964
|
+
entry, final, progress_cb, downloaded, reused, tuple(fetched)
|
|
965
|
+
)
|
|
966
|
+
if final.state is not ModelState.READY:
|
|
967
|
+
# Everything was digest-checked on the way in, so this means the
|
|
968
|
+
# tree changed underneath us. F-84: report corrupt, do not use.
|
|
969
|
+
raise EchoActError(
|
|
970
|
+
Code.MODEL_CORRUPT,
|
|
971
|
+
detail={"model_id": model_id, "files": list(final.unusable)},
|
|
972
|
+
)
|
|
973
|
+
self._emit(
|
|
974
|
+
progress_cb,
|
|
975
|
+
entry,
|
|
976
|
+
DownloadPhase.COMPLETE,
|
|
977
|
+
None,
|
|
978
|
+
0,
|
|
979
|
+
final.bytes_expected,
|
|
980
|
+
final.bytes_expected,
|
|
981
|
+
)
|
|
982
|
+
log.info(
|
|
983
|
+
"model %s prepared: %d file(s) fetched, %d reused", model_id, len(fetched), len(reused)
|
|
984
|
+
)
|
|
985
|
+
return DownloadOutcome(
|
|
986
|
+
model_id=model_id,
|
|
987
|
+
completed=True,
|
|
988
|
+
cancelled=False,
|
|
989
|
+
bytes_downloaded=downloaded,
|
|
990
|
+
reused=reused,
|
|
991
|
+
fetched=tuple(fetched),
|
|
992
|
+
report=final,
|
|
993
|
+
)
|
|
994
|
+
|
|
995
|
+
def repair(
|
|
996
|
+
self,
|
|
997
|
+
model_id: str,
|
|
998
|
+
progress_cb: ProgressCallback | None = None,
|
|
999
|
+
cancel: CancelToken | None = None,
|
|
1000
|
+
*,
|
|
1001
|
+
request_path: RequestPath = RequestPath.GUI,
|
|
1002
|
+
) -> DownloadOutcome:
|
|
1003
|
+
"""F-65's repair: replace whatever does not match the manifest.
|
|
1004
|
+
|
|
1005
|
+
Stronger than a retry in one respect -- it discards half-downloaded
|
|
1006
|
+
parts instead of resuming them. A retry assumes the interruption was
|
|
1007
|
+
the problem; a repair is asked for because something is wrong, and a
|
|
1008
|
+
partial file is a candidate for what that is.
|
|
1009
|
+
"""
|
|
1010
|
+
return self.download(
|
|
1011
|
+
model_id,
|
|
1012
|
+
progress_cb,
|
|
1013
|
+
cancel,
|
|
1014
|
+
request_path=request_path,
|
|
1015
|
+
discard_partial=True,
|
|
1016
|
+
)
|
|
1017
|
+
|
|
1018
|
+
def _download_file(
|
|
1019
|
+
self,
|
|
1020
|
+
entry: ModelEntry,
|
|
1021
|
+
file: ModelFile,
|
|
1022
|
+
base: Path,
|
|
1023
|
+
fetcher: Fetcher,
|
|
1024
|
+
*,
|
|
1025
|
+
cancel: CancelToken | None,
|
|
1026
|
+
discard_partial: bool,
|
|
1027
|
+
on_chunk: Callable[[int], None],
|
|
1028
|
+
) -> tuple[int, bool]:
|
|
1029
|
+
"""Stream one file into place. Returns (bytes written, cancelled)."""
|
|
1030
|
+
target = file.path_under(base)
|
|
1031
|
+
part = target.with_name(target.name + _PART_SUFFIX)
|
|
1032
|
+
with _writing(target.parent):
|
|
1033
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
1034
|
+
|
|
1035
|
+
# A target that exists here failed verification, so it is not
|
|
1036
|
+
# something to keep: F-64 re-downloads corrupted data.
|
|
1037
|
+
_unlink(target)
|
|
1038
|
+
if discard_partial:
|
|
1039
|
+
_unlink(part)
|
|
1040
|
+
|
|
1041
|
+
url = resolve_url(entry, file)
|
|
1042
|
+
for attempt in (1, 2):
|
|
1043
|
+
written, cancelled, digest = self._stream_to_part(
|
|
1044
|
+
url, part, file, fetcher, cancel=cancel, on_chunk=on_chunk
|
|
1045
|
+
)
|
|
1046
|
+
if cancelled:
|
|
1047
|
+
return written, True
|
|
1048
|
+
if digest == file.sha256:
|
|
1049
|
+
_replace(part, target)
|
|
1050
|
+
return written, False
|
|
1051
|
+
# Bad bytes. The first explanation is a damaged transfer or a
|
|
1052
|
+
# stale resume, so throw the part away and fetch the whole file
|
|
1053
|
+
# once more before concluding anything about the upstream.
|
|
1054
|
+
_unlink(part)
|
|
1055
|
+
log.warning(
|
|
1056
|
+
"digest mismatch for %s (attempt %d)", redact(target), attempt
|
|
1057
|
+
)
|
|
1058
|
+
raise EchoActError(
|
|
1059
|
+
Code.MODEL_CORRUPT,
|
|
1060
|
+
"A downloaded model file does not match the manifest.",
|
|
1061
|
+
detail={"model_id": entry.model_id, "file": file.relative_path},
|
|
1062
|
+
)
|
|
1063
|
+
|
|
1064
|
+
def _stream_to_part(
|
|
1065
|
+
self,
|
|
1066
|
+
url: str,
|
|
1067
|
+
part: Path,
|
|
1068
|
+
file: ModelFile,
|
|
1069
|
+
fetcher: Fetcher,
|
|
1070
|
+
*,
|
|
1071
|
+
cancel: CancelToken | None,
|
|
1072
|
+
on_chunk: Callable[[int], None],
|
|
1073
|
+
) -> tuple[int, bool, str | None]:
|
|
1074
|
+
hasher = hashlib.sha256()
|
|
1075
|
+
offset = 0
|
|
1076
|
+
try:
|
|
1077
|
+
existing = part.stat().st_size
|
|
1078
|
+
except OSError:
|
|
1079
|
+
existing = 0
|
|
1080
|
+
if 0 < existing < file.byte_size:
|
|
1081
|
+
# Hash what is already there so the digest covers the whole file
|
|
1082
|
+
# without re-reading it after the transfer.
|
|
1083
|
+
digest_so_far, _ = _sha256_file(part, cancel=cancel, hasher=hasher)
|
|
1084
|
+
if digest_so_far is None:
|
|
1085
|
+
return 0, True, None
|
|
1086
|
+
offset = existing
|
|
1087
|
+
elif existing:
|
|
1088
|
+
_unlink(part)
|
|
1089
|
+
|
|
1090
|
+
written = 0
|
|
1091
|
+
with fetcher.open(url, offset=offset) as body:
|
|
1092
|
+
if offset and not body.resumed:
|
|
1093
|
+
# The server sent the whole file; appending would corrupt it.
|
|
1094
|
+
hasher = hashlib.sha256()
|
|
1095
|
+
offset = 0
|
|
1096
|
+
mode = "ab" if offset else "wb"
|
|
1097
|
+
# Opening, every write, and the flush at the end of the block are
|
|
1098
|
+
# all inside ``_writing``: rule 3 lets no OSError out of here, and
|
|
1099
|
+
# 5.3 wants a disk that fills mid-transfer reported as a failed
|
|
1100
|
+
# save rather than as a crash.
|
|
1101
|
+
with _writing(part), part.open(mode) as sink:
|
|
1102
|
+
for chunk in body.chunks:
|
|
1103
|
+
if _cancelled(cancel):
|
|
1104
|
+
sink.flush()
|
|
1105
|
+
return written, True, None
|
|
1106
|
+
sink.write(chunk)
|
|
1107
|
+
hasher.update(chunk)
|
|
1108
|
+
written += len(chunk)
|
|
1109
|
+
on_chunk(offset + written)
|
|
1110
|
+
return written, False, hasher.hexdigest()
|
|
1111
|
+
|
|
1112
|
+
# -- deletion (F-65, F-76) ------------------------------------------
|
|
1113
|
+
|
|
1114
|
+
def delete(self, model_id: str, *, in_use: bool = False) -> int:
|
|
1115
|
+
"""Remove the app's copy of a model. Returns the bytes freed.
|
|
1116
|
+
|
|
1117
|
+
Only the model's own directory under the model cache is touched.
|
|
1118
|
+
F-65 is explicit that deleting a model does not delete retained
|
|
1119
|
+
documents or audio results, and those live under different roots in
|
|
1120
|
+
:mod:`echoact.paths`; nothing here can reach them.
|
|
1121
|
+
|
|
1122
|
+
``in_use`` is the caller's answer to "is a job holding this model?".
|
|
1123
|
+
F-65 requires that job to be cancelled and the model released first,
|
|
1124
|
+
and the registry cannot see jobs, so it refuses and says why rather
|
|
1125
|
+
than guessing.
|
|
1126
|
+
"""
|
|
1127
|
+
entry = self.entry(model_id)
|
|
1128
|
+
if in_use:
|
|
1129
|
+
raise EchoActError(
|
|
1130
|
+
Code.DELETE_BLOCKED_IN_USE,
|
|
1131
|
+
"The model is loaded by a running job; cancel it first.",
|
|
1132
|
+
detail={"model_id": entry.model_id},
|
|
1133
|
+
)
|
|
1134
|
+
target = self.model_dir(model_id)
|
|
1135
|
+
root = self.root
|
|
1136
|
+
# Structural guard: the only thing this method may ever remove is a
|
|
1137
|
+
# named subdirectory of the model cache.
|
|
1138
|
+
if target == root or root not in target.parents:
|
|
1139
|
+
raise AssertionError(f"refusing to delete {target} outside {root}")
|
|
1140
|
+
# N-23: deleting the tree a download is streaming into would leave it
|
|
1141
|
+
# renaming files into a directory nobody expects to exist. Refused
|
|
1142
|
+
# rather than queued, because F-65's delete is a GUI action and rule 7
|
|
1143
|
+
# forbids the main thread waiting minutes for a transfer to finish.
|
|
1144
|
+
lock = _prepare_lock(target)
|
|
1145
|
+
if not lock.acquire(blocking=False):
|
|
1146
|
+
raise EchoActError(
|
|
1147
|
+
Code.DELETE_BLOCKED_IN_USE,
|
|
1148
|
+
"The model is being prepared; cancel that first.",
|
|
1149
|
+
detail={"model_id": entry.model_id},
|
|
1150
|
+
retry_after_s=_DOWNLOAD_RETRY_AFTER_S,
|
|
1151
|
+
)
|
|
1152
|
+
try:
|
|
1153
|
+
freed = _tree_bytes(target)
|
|
1154
|
+
try:
|
|
1155
|
+
shutil.rmtree(target)
|
|
1156
|
+
except FileNotFoundError:
|
|
1157
|
+
return 0
|
|
1158
|
+
except OSError as exc:
|
|
1159
|
+
raise EchoActError(
|
|
1160
|
+
Code.INTERNAL,
|
|
1161
|
+
"The model files could not be deleted.",
|
|
1162
|
+
detail={"model_id": entry.model_id, "path": redact(target)},
|
|
1163
|
+
cause=exc,
|
|
1164
|
+
) from exc
|
|
1165
|
+
finally:
|
|
1166
|
+
lock.release()
|
|
1167
|
+
log.info("deleted model %s (%d bytes)", entry.model_id, freed)
|
|
1168
|
+
return freed
|
|
1169
|
+
|
|
1170
|
+
# -- reporting (F-53, F-63, F-73) -----------------------------------
|
|
1171
|
+
|
|
1172
|
+
def disk_usage(self, model_id: str) -> int:
|
|
1173
|
+
"""Bytes the app's copy occupies, partial downloads included."""
|
|
1174
|
+
return _tree_bytes(self.model_dir(model_id))
|
|
1175
|
+
|
|
1176
|
+
def total_disk_usage(self) -> int:
|
|
1177
|
+
"""F-73's "model cache" figure: the whole app-owned cache tree."""
|
|
1178
|
+
return _tree_bytes(self.root)
|
|
1179
|
+
|
|
1180
|
+
def status(
|
|
1181
|
+
self, model_id: str, budget: Budget | None = None, *, deep: bool = False
|
|
1182
|
+
) -> ModelStatus:
|
|
1183
|
+
"""One model as F-63 and F-53 present it."""
|
|
1184
|
+
entry = self.entry(model_id)
|
|
1185
|
+
report = self.verify(model_id, deep=deep)
|
|
1186
|
+
using_package = False
|
|
1187
|
+
if report.state is not ModelState.READY:
|
|
1188
|
+
fallback = self.package_cache_dir(model_id)
|
|
1189
|
+
if fallback is not None and fallback.is_dir():
|
|
1190
|
+
other = self.verify(model_id, deep=deep, root=fallback)
|
|
1191
|
+
if other.state is ModelState.READY:
|
|
1192
|
+
report = other
|
|
1193
|
+
using_package = True
|
|
1194
|
+
runnable, reason = (True, None) if budget is None else self.can_run(model_id, budget)
|
|
1195
|
+
return ModelStatus(
|
|
1196
|
+
model_id=entry.model_id,
|
|
1197
|
+
display_name=entry.display_name,
|
|
1198
|
+
state=report.state,
|
|
1199
|
+
sample_rate=entry.sample_rate,
|
|
1200
|
+
languages=tuple(lang.value for lang in entry.languages),
|
|
1201
|
+
bytes_present=report.bytes_present,
|
|
1202
|
+
bytes_total=entry.total_bytes,
|
|
1203
|
+
disk_bytes=self.disk_usage(model_id),
|
|
1204
|
+
runnable=runnable,
|
|
1205
|
+
unavailable_reason=reason,
|
|
1206
|
+
license_name=entry.license.name,
|
|
1207
|
+
license_acceptance_required=entry.license.acceptance_required,
|
|
1208
|
+
license_accepted=not self.license_acceptance_required(model_id),
|
|
1209
|
+
download_authorised=self.preferences.download_authorised(model_id),
|
|
1210
|
+
using_package_cache=using_package,
|
|
1211
|
+
minimum_memory_bytes=entry.minimum_budget.memory_bytes,
|
|
1212
|
+
minimum_cpu_percent=entry.minimum_budget.cpu_percent,
|
|
1213
|
+
)
|
|
1214
|
+
|
|
1215
|
+
def statuses(self, budget: Budget | None = None) -> tuple[ModelStatus, ...]:
|
|
1216
|
+
return tuple(self.status(m.model_id, budget) for m in self.manifest)
|
|
1217
|
+
|
|
1218
|
+
# -- gates ----------------------------------------------------------
|
|
1219
|
+
|
|
1220
|
+
def _require_license(self, entry: ModelEntry) -> None:
|
|
1221
|
+
if self.license_acceptance_required(entry.model_id):
|
|
1222
|
+
raise EchoActError(
|
|
1223
|
+
Code.MODEL_LICENSE_NOT_ACCEPTED,
|
|
1224
|
+
detail={"model_id": entry.model_id, "license": entry.license.name},
|
|
1225
|
+
)
|
|
1226
|
+
|
|
1227
|
+
def _require_authorisation(self, entry: ModelEntry, request_path: RequestPath) -> None:
|
|
1228
|
+
"""Section 5.3: only a model the owner pre-authorised in the GUI may
|
|
1229
|
+
be downloaded on an external request."""
|
|
1230
|
+
if request_path is RequestPath.GUI:
|
|
1231
|
+
return
|
|
1232
|
+
if not self.preferences.download_authorised(entry.model_id):
|
|
1233
|
+
raise EchoActError(
|
|
1234
|
+
Code.MODEL_DOWNLOAD_FORBIDDEN,
|
|
1235
|
+
detail={"model_id": entry.model_id, "request_path": request_path.value},
|
|
1236
|
+
)
|
|
1237
|
+
|
|
1238
|
+
def _require_space(self, base: Path, needed: int) -> None:
|
|
1239
|
+
"""F-09: preparation needs storage, and running the disk to zero is
|
|
1240
|
+
worse than refusing. 4.1's low-space figure is the headroom left."""
|
|
1241
|
+
probe = base
|
|
1242
|
+
while not probe.exists() and probe.parent != probe:
|
|
1243
|
+
probe = probe.parent
|
|
1244
|
+
try:
|
|
1245
|
+
free = shutil.disk_usage(probe).free
|
|
1246
|
+
except OSError:
|
|
1247
|
+
return
|
|
1248
|
+
if free < needed + LOW_SPACE_WARNING_BYTES:
|
|
1249
|
+
raise EchoActError(
|
|
1250
|
+
Code.STORAGE_FULL,
|
|
1251
|
+
"There is not enough free disk space to prepare this model.",
|
|
1252
|
+
detail={"needed_bytes": needed, "free_bytes": free},
|
|
1253
|
+
)
|
|
1254
|
+
|
|
1255
|
+
# -- helpers --------------------------------------------------------
|
|
1256
|
+
|
|
1257
|
+
def _emit(
|
|
1258
|
+
self,
|
|
1259
|
+
cb: ProgressCallback | None,
|
|
1260
|
+
entry: ModelEntry,
|
|
1261
|
+
phase: DownloadPhase,
|
|
1262
|
+
file: ModelFile | None,
|
|
1263
|
+
file_done: int,
|
|
1264
|
+
done: int,
|
|
1265
|
+
total: int,
|
|
1266
|
+
*,
|
|
1267
|
+
file_index: int = 0,
|
|
1268
|
+
file_count: int = 0,
|
|
1269
|
+
) -> None:
|
|
1270
|
+
if cb is None:
|
|
1271
|
+
return
|
|
1272
|
+
cb(
|
|
1273
|
+
DownloadProgress(
|
|
1274
|
+
model_id=entry.model_id,
|
|
1275
|
+
phase=phase,
|
|
1276
|
+
relative_path=file.relative_path if file else "",
|
|
1277
|
+
file_index=file_index,
|
|
1278
|
+
file_count=file_count or entry.file_count,
|
|
1279
|
+
file_bytes_done=file_done,
|
|
1280
|
+
file_bytes_total=file.byte_size if file else 0,
|
|
1281
|
+
bytes_done=done,
|
|
1282
|
+
bytes_total=total,
|
|
1283
|
+
)
|
|
1284
|
+
)
|
|
1285
|
+
|
|
1286
|
+
def _interrupted_report(
|
|
1287
|
+
self,
|
|
1288
|
+
entry: ModelEntry,
|
|
1289
|
+
baseline: VerifyReport,
|
|
1290
|
+
fetched: Iterable[str],
|
|
1291
|
+
in_flight: str | None,
|
|
1292
|
+
) -> VerifyReport:
|
|
1293
|
+
"""What a cancelled attempt is entitled to say about the model.
|
|
1294
|
+
|
|
1295
|
+
F-64: a cancelled download is never shown as ready. The only
|
|
1296
|
+
evidence of soundness that exists at this point is a digest -- the
|
|
1297
|
+
one the opening deep pass computed for a file the attempt did not
|
|
1298
|
+
have to fetch, or the one the transfer matched before renaming a file
|
|
1299
|
+
into place. Verifying again here could only be a shallow pass, since
|
|
1300
|
+
N-22 allows five seconds to stop and 385 MB does not hash in that;
|
|
1301
|
+
and a shallow pass calls a same-size tampered file sound, so it would
|
|
1302
|
+
let a cancellation report READY for a model the deep pass had just
|
|
1303
|
+
called corrupt. The deep verdicts are therefore carried forward
|
|
1304
|
+
instead of being thrown away.
|
|
1305
|
+
|
|
1306
|
+
The file being written when the cancellation landed counts as
|
|
1307
|
+
unknown: its target was removed before the transfer began, and only
|
|
1308
|
+
a ``.part`` stands in its place.
|
|
1309
|
+
"""
|
|
1310
|
+
done = set(fetched)
|
|
1311
|
+
statuses: list[FileStatus] = []
|
|
1312
|
+
for file in entry.files:
|
|
1313
|
+
prior = baseline.status(file.relative_path)
|
|
1314
|
+
if file.relative_path in done:
|
|
1315
|
+
statuses.append(
|
|
1316
|
+
FileStatus(
|
|
1317
|
+
relative_path=file.relative_path,
|
|
1318
|
+
expected_bytes=file.byte_size,
|
|
1319
|
+
actual_bytes=file.byte_size,
|
|
1320
|
+
present=True,
|
|
1321
|
+
size_ok=True,
|
|
1322
|
+
digest_ok=True,
|
|
1323
|
+
)
|
|
1324
|
+
)
|
|
1325
|
+
elif prior is None or file.relative_path == in_flight:
|
|
1326
|
+
statuses.append(_unchecked(file))
|
|
1327
|
+
else:
|
|
1328
|
+
statuses.append(prior)
|
|
1329
|
+
return VerifyReport(
|
|
1330
|
+
model_id=entry.model_id,
|
|
1331
|
+
root=self.model_dir(entry.model_id),
|
|
1332
|
+
files=tuple(statuses),
|
|
1333
|
+
deep=baseline.deep,
|
|
1334
|
+
cancelled=True,
|
|
1335
|
+
)
|
|
1336
|
+
|
|
1337
|
+
def _cancelled_outcome(
|
|
1338
|
+
self,
|
|
1339
|
+
entry: ModelEntry,
|
|
1340
|
+
report: VerifyReport,
|
|
1341
|
+
cb: ProgressCallback | None,
|
|
1342
|
+
downloaded: int,
|
|
1343
|
+
reused: tuple[str, ...],
|
|
1344
|
+
fetched: tuple[str, ...],
|
|
1345
|
+
) -> DownloadOutcome:
|
|
1346
|
+
self._emit(cb, entry, DownloadPhase.CANCELLED, None, 0, report.bytes_present,
|
|
1347
|
+
entry.total_bytes)
|
|
1348
|
+
log.info("model %s preparation cancelled", entry.model_id)
|
|
1349
|
+
return DownloadOutcome(
|
|
1350
|
+
model_id=entry.model_id,
|
|
1351
|
+
completed=False,
|
|
1352
|
+
cancelled=True,
|
|
1353
|
+
bytes_downloaded=downloaded,
|
|
1354
|
+
reused=reused,
|
|
1355
|
+
fetched=fetched,
|
|
1356
|
+
report=report,
|
|
1357
|
+
)
|
|
1358
|
+
|
|
1359
|
+
|
|
1360
|
+
# ======================================================================
|
|
1361
|
+
# Module helpers
|
|
1362
|
+
# ======================================================================
|
|
1363
|
+
|
|
1364
|
+
|
|
1365
|
+
#: One preparation lock per model directory, shared by every registry in
|
|
1366
|
+
#: this process. N-23: two requests for one model must not become two
|
|
1367
|
+
#: transfers. The ``.part`` path is derived from the model id and the file
|
|
1368
|
+
#: path alone, so two attempts open the same file -- one truncating while the
|
|
1369
|
+
#: other appends -- and the digest each computes over its own stream says
|
|
1370
|
+
#: nothing about the bytes that ended up on disk. Keyed by directory rather
|
|
1371
|
+
#: than by model id, so two registries over one cache serialise and two over
|
|
1372
|
+
#: different caches do not.
|
|
1373
|
+
_PREPARE_LOCKS: dict[str, threading.RLock] = {}
|
|
1374
|
+
_PREPARE_LOCKS_GUARD = threading.Lock()
|
|
1375
|
+
|
|
1376
|
+
|
|
1377
|
+
def _prepare_lock(directory: Path) -> threading.RLock:
|
|
1378
|
+
key = os.path.normcase(os.path.abspath(directory))
|
|
1379
|
+
with _PREPARE_LOCKS_GUARD:
|
|
1380
|
+
lock = _PREPARE_LOCKS.get(key)
|
|
1381
|
+
if lock is None:
|
|
1382
|
+
lock = threading.RLock()
|
|
1383
|
+
_PREPARE_LOCKS[key] = lock
|
|
1384
|
+
return lock
|
|
1385
|
+
|
|
1386
|
+
|
|
1387
|
+
@contextmanager
|
|
1388
|
+
def _preparing(directory: Path, cancel: CancelToken | None) -> Iterator[bool]:
|
|
1389
|
+
"""Hold one model's preparation lock (N-23).
|
|
1390
|
+
|
|
1391
|
+
Yields ``True`` with the lock held, or ``False`` when the caller
|
|
1392
|
+
cancelled while waiting for the attempt in front of it -- the wait is
|
|
1393
|
+
polled rather than indefinite so that N-22's five seconds to stop hold
|
|
1394
|
+
for a queued attempt too.
|
|
1395
|
+
"""
|
|
1396
|
+
lock = _prepare_lock(directory)
|
|
1397
|
+
while not lock.acquire(timeout=_LOCK_POLL_S):
|
|
1398
|
+
if _cancelled(cancel):
|
|
1399
|
+
yield False
|
|
1400
|
+
return
|
|
1401
|
+
try:
|
|
1402
|
+
yield True
|
|
1403
|
+
finally:
|
|
1404
|
+
lock.release()
|
|
1405
|
+
|
|
1406
|
+
|
|
1407
|
+
@contextmanager
|
|
1408
|
+
def _writing(path: Path) -> Iterator[None]:
|
|
1409
|
+
"""Convert a failed write into this module's own error (rule 3, 5.3).
|
|
1410
|
+
|
|
1411
|
+
:meth:`ModelRegistry._require_space` cannot stand in for this. It asks
|
|
1412
|
+
once, before a transfer that runs for minutes: it does not see another
|
|
1413
|
+
process taking the last of the disk in the meantime, a cache directory
|
|
1414
|
+
that turns out to be read-only, or a scanner holding the ``.part`` open
|
|
1415
|
+
-- and on a volume whose free space cannot be read at all it declines to
|
|
1416
|
+
answer and lets the transfer start anyway.
|
|
1417
|
+
"""
|
|
1418
|
+
try:
|
|
1419
|
+
yield
|
|
1420
|
+
except OSError as exc:
|
|
1421
|
+
out_of_room = exc.errno in (errno.ENOSPC, errno.EDQUOT)
|
|
1422
|
+
raise EchoActError(
|
|
1423
|
+
Code.STORAGE_FULL if out_of_room else Code.MODEL_DOWNLOAD_FAILED,
|
|
1424
|
+
"There is not enough free disk space to finish preparing this model."
|
|
1425
|
+
if out_of_room
|
|
1426
|
+
else "The model file could not be written to the cache.",
|
|
1427
|
+
detail={"path": redact(path)},
|
|
1428
|
+
retry_after_s=None if out_of_room else _DOWNLOAD_RETRY_AFTER_S,
|
|
1429
|
+
cause=exc,
|
|
1430
|
+
) from exc
|
|
1431
|
+
|
|
1432
|
+
|
|
1433
|
+
def _sha256_file(
|
|
1434
|
+
path: Path, *, cancel: CancelToken | None = None, hasher: hashlib._Hash | None = None
|
|
1435
|
+
) -> tuple[str | None, int]:
|
|
1436
|
+
"""Digest a file in bounded chunks. ``None`` means cancelled.
|
|
1437
|
+
|
|
1438
|
+
N-21 forbids holding the file in memory -- ``vector_estimator.onnx`` is
|
|
1439
|
+
256 MB and the generation budget starts at 2 GiB -- and the same loop is
|
|
1440
|
+
what makes a verification pass interruptible within N-22's five seconds.
|
|
1441
|
+
"""
|
|
1442
|
+
digest = hasher if hasher is not None else hashlib.sha256()
|
|
1443
|
+
read = 0
|
|
1444
|
+
try:
|
|
1445
|
+
with path.open("rb") as handle:
|
|
1446
|
+
while True:
|
|
1447
|
+
if _cancelled(cancel):
|
|
1448
|
+
return None, read
|
|
1449
|
+
chunk = handle.read(_CHUNK_BYTES)
|
|
1450
|
+
if not chunk:
|
|
1451
|
+
break
|
|
1452
|
+
digest.update(chunk)
|
|
1453
|
+
read += len(chunk)
|
|
1454
|
+
except OSError as exc:
|
|
1455
|
+
raise EchoActError(
|
|
1456
|
+
Code.MODEL_CORRUPT,
|
|
1457
|
+
"A model file could not be read.",
|
|
1458
|
+
detail={"path": redact(path)},
|
|
1459
|
+
cause=exc,
|
|
1460
|
+
) from exc
|
|
1461
|
+
return digest.hexdigest(), read
|
|
1462
|
+
|
|
1463
|
+
|
|
1464
|
+
def _unchecked(file: ModelFile) -> FileStatus:
|
|
1465
|
+
return FileStatus(
|
|
1466
|
+
relative_path=file.relative_path,
|
|
1467
|
+
expected_bytes=file.byte_size,
|
|
1468
|
+
actual_bytes=0,
|
|
1469
|
+
present=False,
|
|
1470
|
+
size_ok=False,
|
|
1471
|
+
digest_ok=None,
|
|
1472
|
+
checked=False,
|
|
1473
|
+
)
|
|
1474
|
+
|
|
1475
|
+
|
|
1476
|
+
def _is_ok(report: VerifyReport, relative_path: str) -> bool:
|
|
1477
|
+
status = report.status(relative_path)
|
|
1478
|
+
return status is not None and status.ok
|
|
1479
|
+
|
|
1480
|
+
|
|
1481
|
+
def _tree_bytes(root: Path) -> int:
|
|
1482
|
+
total = 0
|
|
1483
|
+
if not root.exists():
|
|
1484
|
+
return 0
|
|
1485
|
+
for path in _walk(root):
|
|
1486
|
+
try:
|
|
1487
|
+
total += path.stat().st_size
|
|
1488
|
+
except OSError:
|
|
1489
|
+
continue
|
|
1490
|
+
return total
|
|
1491
|
+
|
|
1492
|
+
|
|
1493
|
+
def _walk(root: Path) -> Iterable[Path]:
|
|
1494
|
+
for dirpath, _dirnames, filenames in os.walk(root):
|
|
1495
|
+
for name in filenames:
|
|
1496
|
+
yield Path(dirpath) / name
|
|
1497
|
+
|
|
1498
|
+
|
|
1499
|
+
def _unlink(path: Path) -> None:
|
|
1500
|
+
try:
|
|
1501
|
+
path.unlink()
|
|
1502
|
+
except OSError:
|
|
1503
|
+
pass
|
|
1504
|
+
|
|
1505
|
+
|
|
1506
|
+
def _replace(source: Path, target: Path) -> None:
|
|
1507
|
+
try:
|
|
1508
|
+
os.replace(source, target)
|
|
1509
|
+
except OSError as exc:
|
|
1510
|
+
raise EchoActError(
|
|
1511
|
+
Code.MODEL_DOWNLOAD_FAILED,
|
|
1512
|
+
"A downloaded model file could not be moved into place.",
|
|
1513
|
+
detail={"path": redact(target)},
|
|
1514
|
+
retry_after_s=_DOWNLOAD_RETRY_AFTER_S,
|
|
1515
|
+
cause=exc,
|
|
1516
|
+
) from exc
|
|
1517
|
+
|
|
1518
|
+
|
|
1519
|
+
def _fingerprint(terms: LicenseTerms) -> str:
|
|
1520
|
+
"""Identify the exact terms accepted, so changed terms are re-asked."""
|
|
1521
|
+
digest = hashlib.sha256()
|
|
1522
|
+
digest.update(terms.name.encode("utf-8"))
|
|
1523
|
+
digest.update(terms.pass_through_obligation.encode("utf-8"))
|
|
1524
|
+
for restriction in terms.restrictions:
|
|
1525
|
+
digest.update(b"\x00")
|
|
1526
|
+
digest.update(restriction.encode("utf-8"))
|
|
1527
|
+
return digest.hexdigest()[:32]
|
|
1528
|
+
|
|
1529
|
+
|
|
1530
|
+
def _gib(value: int) -> str:
|
|
1531
|
+
"""4.1: memory is shown in GiB."""
|
|
1532
|
+
return f"{value / GIB:.1f} GiB"
|
|
1533
|
+
|
|
1534
|
+
|
|
1535
|
+
__all__ = [
|
|
1536
|
+
"CancelToken",
|
|
1537
|
+
"DownloadOutcome",
|
|
1538
|
+
"DownloadPhase",
|
|
1539
|
+
"DownloadProgress",
|
|
1540
|
+
"Fetcher",
|
|
1541
|
+
"FileStatus",
|
|
1542
|
+
"HttpxFetcher",
|
|
1543
|
+
"JsonModelPreferences",
|
|
1544
|
+
"ModelPreferences",
|
|
1545
|
+
"ModelRegistry",
|
|
1546
|
+
"ModelState",
|
|
1547
|
+
"ModelStatus",
|
|
1548
|
+
"RemoteBody",
|
|
1549
|
+
"VerifyReport",
|
|
1550
|
+
"resolve_url",
|
|
1551
|
+
]
|