echoact 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. echoact/__init__.py +3 -0
  2. echoact/__main__.py +117 -0
  3. echoact/app.py +315 -0
  4. echoact/audio/__init__.py +0 -0
  5. echoact/audio/devices.py +192 -0
  6. echoact/audio/player.py +611 -0
  7. echoact/audio/wav.py +854 -0
  8. echoact/config/__init__.py +0 -0
  9. echoact/config/budget.py +370 -0
  10. echoact/config/settings.py +1244 -0
  11. echoact/db/__init__.py +0 -0
  12. echoact/db/backup.py +2429 -0
  13. echoact/db/migrations.py +434 -0
  14. echoact/db/schema.sql +214 -0
  15. echoact/db/store.py +2062 -0
  16. echoact/diagnostics.py +902 -0
  17. echoact/domain.py +487 -0
  18. echoact/engine/__init__.py +0 -0
  19. echoact/engine/container.py +843 -0
  20. echoact/engine/protocol.py +241 -0
  21. echoact/engine/runtime.py +324 -0
  22. echoact/engine/supervisor.py +961 -0
  23. echoact/engine/worker.py +659 -0
  24. echoact/errors.py +281 -0
  25. echoact/instance.py +172 -0
  26. echoact/jobs/__init__.py +0 -0
  27. echoact/jobs/engine.py +776 -0
  28. echoact/jobs/request.py +300 -0
  29. echoact/mcp/__init__.py +0 -0
  30. echoact/mcp/__main__.py +50 -0
  31. echoact/mcp/client.py +202 -0
  32. echoact/mcp/config.py +112 -0
  33. echoact/mcp/server.py +340 -0
  34. echoact/models/__init__.py +0 -0
  35. echoact/models/catalog.py +273 -0
  36. echoact/models/manifest.py +278 -0
  37. echoact/models/registry.py +1551 -0
  38. echoact/paths.py +93 -0
  39. echoact/policy.py +189 -0
  40. echoact/security/__init__.py +0 -0
  41. echoact/security/credentials.py +930 -0
  42. echoact/security/ratelimit.py +534 -0
  43. echoact/service/__init__.py +20 -0
  44. echoact/service/app.py +182 -0
  45. echoact/service/deps.py +563 -0
  46. echoact/service/errors.py +241 -0
  47. echoact/service/routes.py +1125 -0
  48. echoact/service/schemas.py +509 -0
  49. echoact/service/server.py +270 -0
  50. echoact/text/__init__.py +0 -0
  51. echoact/text/language.py +44 -0
  52. echoact/text/loader.py +577 -0
  53. echoact/text/normalize.py +924 -0
  54. echoact/text/segment.py +499 -0
  55. echoact/text/sniff.py +1202 -0
  56. echoact/ui/__init__.py +0 -0
  57. echoact/ui/bridge.py +50 -0
  58. echoact/ui/controls.py +360 -0
  59. echoact/ui/credential_dialog.py +131 -0
  60. echoact/ui/fonts.py +94 -0
  61. echoact/ui/i18n.py +260 -0
  62. echoact/ui/icons.py +440 -0
  63. echoact/ui/library.py +1642 -0
  64. echoact/ui/licence.py +162 -0
  65. echoact/ui/main_window.py +1202 -0
  66. echoact/ui/mcp_setup.py +494 -0
  67. echoact/ui/models_view.py +1142 -0
  68. echoact/ui/notifications.py +202 -0
  69. echoact/ui/reading.py +494 -0
  70. echoact/ui/settings_view.py +2258 -0
  71. echoact/ui/status_view.py +1193 -0
  72. echoact/ui/theme.py +579 -0
  73. echoact/util/__init__.py +0 -0
  74. echoact/util/ids.py +62 -0
  75. echoact/util/logging.py +127 -0
  76. echoact-0.1.0.dist-info/METADATA +162 -0
  77. echoact-0.1.0.dist-info/RECORD +80 -0
  78. echoact-0.1.0.dist-info/WHEEL +4 -0
  79. echoact-0.1.0.dist-info/entry_points.txt +3 -0
  80. echoact-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,1551 @@
1
+ """The model lifecycle: where weights live, whether they are sound, and how
2
+ they get there (F-09, F-63, F-64, F-65, resolved against F-84's manifest).
3
+
4
+ The app owns its own model cache under :func:`echoact.paths.model_cache_dir`
5
+ rather than letting the ``supertonic`` package download into
6
+ ``~/.cache/supertonic3``. F-73 has to size the model cache separately from
7
+ documents and audio, F-65 has to be able to delete it, and F-76 offers it as
8
+ its own deletion scope; none of that is possible for a directory a third-party
9
+ package owns and shares with whatever else on the machine imports it. A
10
+ verified copy already sitting in the package's cache is still *read* rather
11
+ than re-downloaded -- 385 MB is not worth a second trip to satisfy tidiness --
12
+ but it is never written to, and deleting the app's cache says so.
13
+
14
+ Everything that decides "sound" goes through the manifest. Nothing here
15
+ trusts a file because it exists, a size because a server reported it, or a
16
+ directory because the last run left it behind.
17
+
18
+ N-11's licence gate sits on the way to a *directory*, not only on the way to
19
+ a download. A model that is already on disk -- borrowed from the package
20
+ cache above, or prepared before a release amended the restrictions -- is
21
+ never downloaded again, so a gate that only guarded :meth:`ModelRegistry.download`
22
+ would be one the two commonest cases walk straight past.
23
+
24
+ Four requirements shape the awkward parts:
25
+
26
+ * N-21 forbids reading a whole file into memory to hash it, and a
27
+ verification pass over 385 MB has to be interruptible, so every read is a
28
+ bounded chunk with a cancellation check between chunks.
29
+ * F-64 wants progress, failure, cancellation, and a retry that reuses sound
30
+ data. Downloads therefore stream into a ``.part`` file next to the target
31
+ and are renamed into place only after the digest matches. A cancelled or
32
+ crashed download leaves a partial file that the next attempt resumes from
33
+ and that verification ignores. What a cancelled attempt *reports* rests on
34
+ digests as well -- the ones the opening pass computed and the ones the
35
+ transfer itself matched -- and never on a cheap size check, because a size
36
+ check would call a same-size tampered file sound and hand F-63's screen a
37
+ corrupt model described as ready.
38
+ * N-23 forbids a repeated request multiplying the work. Preparation of one
39
+ model is serialised for the whole process, so a second caller waits and
40
+ then finds the files already in place instead of streaming a second copy
41
+ through the same ``.part``.
42
+ * Section 5.3 forbids an external caller triggering an arbitrary large
43
+ download. A download asked for over REST or MCP is refused unless the
44
+ owner pre-authorised that model in the GUI.
45
+ """
46
+
47
+ from __future__ import annotations
48
+
49
+ import errno
50
+ import hashlib
51
+ import json
52
+ import os
53
+ import shutil
54
+ import threading
55
+ from collections.abc import Callable, Iterable, Iterator
56
+ from contextlib import AbstractContextManager, contextmanager
57
+ from dataclasses import dataclass
58
+ from enum import StrEnum
59
+ from pathlib import Path
60
+ from typing import Protocol
61
+
62
+ from ..domain import Budget, RequestPath
63
+ from ..errors import Code, EchoActError
64
+ from ..paths import data_dir, model_cache_dir, redact
65
+ from ..policy import GIB, LOW_SPACE_WARNING_BYTES
66
+ from ..util.ids import now
67
+ from ..util.logging import get_logger
68
+ from .catalog import MANIFEST
69
+ from .manifest import LicenseTerms, Manifest, ModelEntry, ModelFile
70
+
71
+ log = get_logger("models.registry")
72
+
73
+ #: Read buffer for hashing and for streaming a download. Not a policy limit
74
+ #: -- it is the granularity at which N-21's cancellation and F-64's progress
75
+ #: are observable, and the size at which neither is expensive.
76
+ _CHUNK_BYTES = 1 << 20
77
+
78
+ #: Network timeouts for a model download. F-09 says preparation needs the
79
+ #: internet; nothing else in the product does, so these are not a policy
80
+ #: number shared with anything.
81
+ _CONNECT_TIMEOUT_S = 15.0
82
+ _READ_TIMEOUT_S = 60.0
83
+
84
+ #: How long a caller is told to wait before retrying a failed download.
85
+ _DOWNLOAD_RETRY_AFTER_S = 5.0
86
+
87
+ #: How long a caller queued behind another preparation of the same model
88
+ #: waits before looking at its cancel token again. N-22 allows five seconds
89
+ #: to stop; a queued attempt gives up well inside that.
90
+ _LOCK_POLL_S = 0.25
91
+
92
+ _PART_SUFFIX = ".part"
93
+
94
+ #: Where the ``supertonic`` package would keep the same weights. This is a
95
+ #: coupling to that package's layout, kept here rather than in the manifest
96
+ #: because it describes *another* program's cache, not the model.
97
+ _PACKAGE_CACHE_NAMES = {"supertonic-3": "supertonic3"}
98
+
99
+
100
+ # ======================================================================
101
+ # Cancellation
102
+ # ======================================================================
103
+
104
+
105
+ class CancelToken:
106
+ """A flag one thread sets and another notices between chunks.
107
+
108
+ Verification and download both run off the Qt main thread and both have
109
+ to stop promptly: N-22 gives five seconds to release resources, and a
110
+ 256 MB hash does not fit in that if it can only be interrupted at file
111
+ boundaries.
112
+ """
113
+
114
+ __slots__ = ("_event",)
115
+
116
+ def __init__(self) -> None:
117
+ self._event = threading.Event()
118
+
119
+ def cancel(self) -> None:
120
+ self._event.set()
121
+
122
+ @property
123
+ def cancelled(self) -> bool:
124
+ return self._event.is_set()
125
+
126
+ def wait(self, timeout_s: float) -> bool:
127
+ return self._event.wait(timeout_s)
128
+
129
+
130
+ def _cancelled(token: CancelToken | None) -> bool:
131
+ return token is not None and token.cancelled
132
+
133
+
134
+ # ======================================================================
135
+ # Verification results (F-65)
136
+ # ======================================================================
137
+
138
+
139
+ class ModelState(StrEnum):
140
+ """F-65's four answers to "are this model's files sound?".
141
+
142
+ ``PARTIAL`` and ``CORRUPT`` are kept apart because they call for
143
+ different actions and F-64 treats them differently on retry: a partial
144
+ model needs the rest fetched, a corrupt one needs bad files replaced.
145
+ """
146
+
147
+ NOT_PRESENT = "not_present"
148
+ PARTIAL = "partial"
149
+ CORRUPT = "corrupt"
150
+ READY = "ready"
151
+
152
+
153
+ @dataclass(frozen=True, slots=True)
154
+ class FileStatus:
155
+ """What verification learned about one manifest file."""
156
+
157
+ relative_path: str
158
+ expected_bytes: int
159
+ actual_bytes: int
160
+ present: bool
161
+ size_ok: bool
162
+ #: ``None`` when no digest was computed: the file was absent, the wrong
163
+ #: size, or the pass was a shallow one.
164
+ digest_ok: bool | None
165
+ #: ``False`` when verification stopped before reaching this file.
166
+ checked: bool = True
167
+
168
+ @property
169
+ def ok(self) -> bool:
170
+ return self.checked and self.present and self.size_ok and self.digest_ok is not False
171
+
172
+ @property
173
+ def corrupt(self) -> bool:
174
+ return self.present and (not self.size_ok or self.digest_ok is False)
175
+
176
+
177
+ @dataclass(frozen=True, slots=True)
178
+ class VerifyReport:
179
+ """The evidence behind a :class:`ModelState`.
180
+
181
+ F-63 shows disk usage and F-64 decides what to re-fetch, so the report
182
+ keeps per-file detail rather than collapsing to a verdict.
183
+ """
184
+
185
+ model_id: str
186
+ root: Path
187
+ files: tuple[FileStatus, ...]
188
+ #: False for a presence-and-size pass, which cannot see a tampered file.
189
+ deep: bool
190
+ cancelled: bool = False
191
+
192
+ @property
193
+ def state(self) -> ModelState:
194
+ if any(f.corrupt for f in self.files):
195
+ return ModelState.CORRUPT
196
+ if all(f.ok for f in self.files):
197
+ return ModelState.READY
198
+ if any(f.present for f in self.files):
199
+ return ModelState.PARTIAL
200
+ return ModelState.NOT_PRESENT
201
+
202
+ @property
203
+ def bytes_present(self) -> int:
204
+ return sum(f.actual_bytes for f in self.files if f.present)
205
+
206
+ @property
207
+ def bytes_expected(self) -> int:
208
+ return sum(f.expected_bytes for f in self.files)
209
+
210
+ @property
211
+ def missing(self) -> tuple[str, ...]:
212
+ return tuple(f.relative_path for f in self.files if not f.present)
213
+
214
+ @property
215
+ def damaged(self) -> tuple[str, ...]:
216
+ return tuple(f.relative_path for f in self.files if f.corrupt)
217
+
218
+ @property
219
+ def unusable(self) -> tuple[str, ...]:
220
+ """Everything a download would have to fetch to make this ready."""
221
+ return tuple(f.relative_path for f in self.files if not f.ok)
222
+
223
+ def status(self, relative_path: str) -> FileStatus | None:
224
+ for f in self.files:
225
+ if f.relative_path == relative_path:
226
+ return f
227
+ return None
228
+
229
+
230
+ # ======================================================================
231
+ # Download reporting (F-64)
232
+ # ======================================================================
233
+
234
+
235
+ class DownloadPhase(StrEnum):
236
+ CHECKING = "checking"
237
+ DOWNLOADING = "downloading"
238
+ COMPLETE = "complete"
239
+ CANCELLED = "cancelled"
240
+
241
+
242
+ @dataclass(frozen=True, slots=True)
243
+ class DownloadProgress:
244
+ """One progress tick. F-64 requires progress to be visible, and F-63
245
+ requires the required storage to be known before the transfer starts, so
246
+ the totals are populated from the manifest rather than from the server."""
247
+
248
+ model_id: str
249
+ phase: DownloadPhase
250
+ relative_path: str
251
+ file_index: int
252
+ file_count: int
253
+ file_bytes_done: int
254
+ file_bytes_total: int
255
+ bytes_done: int
256
+ bytes_total: int
257
+
258
+ @property
259
+ def fraction(self) -> float:
260
+ if self.bytes_total <= 0:
261
+ return 1.0
262
+ return min(1.0, self.bytes_done / self.bytes_total)
263
+
264
+
265
+ ProgressCallback = Callable[[DownloadProgress], None]
266
+
267
+
268
+ @dataclass(frozen=True, slots=True)
269
+ class DownloadOutcome:
270
+ """What one preparation attempt did.
271
+
272
+ Cancellation is an outcome, not an error: F-64 makes it a first-class
273
+ control, and raising for it would force every caller to catch an
274
+ exception for something the user asked for. A failure that the user did
275
+ not ask for still raises, so the two are never confused.
276
+ """
277
+
278
+ model_id: str
279
+ completed: bool
280
+ cancelled: bool
281
+ bytes_downloaded: int
282
+ reused: tuple[str, ...]
283
+ fetched: tuple[str, ...]
284
+ report: VerifyReport
285
+
286
+ @property
287
+ def state(self) -> ModelState:
288
+ return self.report.state
289
+
290
+
291
+ # ======================================================================
292
+ # Where acceptance and authorisation are recorded (N-11, 5.3)
293
+ # ======================================================================
294
+
295
+
296
+ class ModelPreferences(Protocol):
297
+ """The two owner decisions the lifecycle has to remember.
298
+
299
+ N-11 puts licence acceptance in settings, and 5.3 puts the
300
+ pre-authorisation of a model for external download there too. Both are
301
+ keyed by a fingerprint of the terms accepted, not by model id alone: if
302
+ a later release ships different restrictions, the old acceptance does
303
+ not silently cover them.
304
+ """
305
+
306
+ def license_accepted(self, model_id: str, fingerprint: str) -> bool: ...
307
+
308
+ def record_license_acceptance(self, model_id: str, fingerprint: str) -> None: ...
309
+
310
+ def download_authorised(self, model_id: str) -> bool: ...
311
+
312
+ def set_download_authorised(self, model_id: str, allowed: bool) -> None: ...
313
+
314
+
315
+ class JsonModelPreferences:
316
+ """A small JSON file beside the settings, used until
317
+ ``echoact.config.settings`` exists to hold these two records.
318
+
319
+ It fails closed: a file that cannot be read counts as nothing accepted
320
+ and nothing authorised, because the failure mode of guessing the other
321
+ way is preparing a model whose terms the user never saw.
322
+ """
323
+
324
+ def __init__(self, path: Path | None = None) -> None:
325
+ self._path = path
326
+
327
+ @property
328
+ def path(self) -> Path:
329
+ # Resolved per call, never at construction: ``paths.data_dir`` is
330
+ # cached and tests redirect it after this object exists.
331
+ return self._path if self._path is not None else data_dir() / "model_state.json"
332
+
333
+ def _read(self) -> dict[str, dict[str, object]]:
334
+ try:
335
+ raw = json.loads(self.path.read_text(encoding="utf-8"))
336
+ except (OSError, ValueError):
337
+ return {}
338
+ return raw if isinstance(raw, dict) else {}
339
+
340
+ def _write(self, data: dict[str, dict[str, object]]) -> None:
341
+ path = self.path
342
+ try:
343
+ path.parent.mkdir(parents=True, exist_ok=True)
344
+ tmp = path.with_suffix(".json.tmp")
345
+ tmp.write_text(json.dumps(data, indent=2, sort_keys=True), encoding="utf-8")
346
+ os.replace(tmp, path)
347
+ except OSError as exc:
348
+ code = Code.STORAGE_FULL if exc.errno == errno.ENOSPC else Code.INTERNAL
349
+ raise EchoActError(
350
+ code,
351
+ "The model licence decision could not be saved.",
352
+ detail={"path": redact(path)},
353
+ cause=exc,
354
+ ) from exc
355
+
356
+ def license_accepted(self, model_id: str, fingerprint: str) -> bool:
357
+ record = self._read().get("licenses_accepted", {})
358
+ entry = record.get(model_id) if isinstance(record, dict) else None
359
+ return isinstance(entry, dict) and entry.get("fingerprint") == fingerprint
360
+
361
+ def record_license_acceptance(self, model_id: str, fingerprint: str) -> None:
362
+ data = self._read()
363
+ accepted = data.get("licenses_accepted")
364
+ if not isinstance(accepted, dict):
365
+ accepted = {}
366
+ accepted[model_id] = {"fingerprint": fingerprint, "accepted_at": now()}
367
+ data["licenses_accepted"] = accepted
368
+ self._write(data)
369
+
370
+ def download_authorised(self, model_id: str) -> bool:
371
+ record = self._read().get("downloads_authorised", {})
372
+ return isinstance(record, dict) and record.get(model_id) is True
373
+
374
+ def set_download_authorised(self, model_id: str, allowed: bool) -> None:
375
+ data = self._read()
376
+ authorised = data.get("downloads_authorised")
377
+ if not isinstance(authorised, dict):
378
+ authorised = {}
379
+ authorised[model_id] = bool(allowed)
380
+ data["downloads_authorised"] = authorised
381
+ self._write(data)
382
+
383
+
384
+ # ======================================================================
385
+ # Transport (F-09, F-64)
386
+ # ======================================================================
387
+
388
+
389
+ @dataclass(slots=True)
390
+ class RemoteBody:
391
+ """A response being streamed. ``resumed`` says whether the server
392
+ honoured a Range request; if it did not, the caller must start over
393
+ rather than append the whole file to a partial one."""
394
+
395
+ chunks: Iterator[bytes]
396
+ resumed: bool
397
+ total_bytes: int | None
398
+
399
+
400
+ class Fetcher(Protocol):
401
+ def open(self, url: str, *, offset: int = 0) -> AbstractContextManager[RemoteBody]: ...
402
+
403
+
404
+ class HttpxFetcher:
405
+ """HTTPS streaming, with the range request F-64's retry depends on.
406
+
407
+ ``huggingface_hub`` is installed (a dependency of ``supertonic``) and is
408
+ used for the one thing it is authoritative about -- building the resolve
409
+ URL for a repository at a pinned revision. Its ``hf_hub_download`` is
410
+ not used to transfer: it cannot be cancelled part-way through a 256 MB
411
+ file and reports progress only through a tqdm bar, and F-64 requires
412
+ both. Streaming here also keeps the bytes going straight into the app's
413
+ own cache instead of the hub's parallel one.
414
+ """
415
+
416
+ def __init__(self, *, connect_timeout_s: float = _CONNECT_TIMEOUT_S,
417
+ read_timeout_s: float = _READ_TIMEOUT_S) -> None:
418
+ self._connect_timeout_s = connect_timeout_s
419
+ self._read_timeout_s = read_timeout_s
420
+
421
+ @contextmanager
422
+ def open(self, url: str, *, offset: int = 0) -> Iterator[RemoteBody]:
423
+ import httpx # imported here so a GUI-only start-up never pays for it
424
+
425
+ headers = {"Range": f"bytes={offset}-"} if offset > 0 else {}
426
+ timeout = httpx.Timeout(
427
+ self._read_timeout_s, connect=self._connect_timeout_s, read=self._read_timeout_s
428
+ )
429
+ try:
430
+ with httpx.Client(timeout=timeout, follow_redirects=True) as client:
431
+ with client.stream("GET", url, headers=headers) as response:
432
+ if response.status_code >= 400:
433
+ response.read()
434
+ raise EchoActError(
435
+ Code.MODEL_DOWNLOAD_FAILED,
436
+ f"The model server answered {response.status_code}.",
437
+ detail={"status": response.status_code},
438
+ retry_after_s=_DOWNLOAD_RETRY_AFTER_S,
439
+ )
440
+ length = response.headers.get("content-length")
441
+ yield RemoteBody(
442
+ chunks=response.iter_bytes(_CHUNK_BYTES),
443
+ resumed=offset > 0 and response.status_code == 206,
444
+ total_bytes=int(length) if length and length.isdigit() else None,
445
+ )
446
+ except httpx.HTTPError as exc:
447
+ # 5.3: a network outage says why preparation failed and stays
448
+ # retryable; it never touches an already-prepared model.
449
+ raise EchoActError(
450
+ Code.MODEL_DOWNLOAD_FAILED,
451
+ "The model could not be downloaded: the connection failed.",
452
+ detail={"reason": type(exc).__name__},
453
+ retry_after_s=_DOWNLOAD_RETRY_AFTER_S,
454
+ cause=exc,
455
+ ) from exc
456
+
457
+
458
+ def resolve_url(entry: ModelEntry, file: ModelFile) -> str:
459
+ """The pinned-revision URL for one manifest file."""
460
+ from huggingface_hub import hf_hub_url
461
+
462
+ return hf_hub_url(entry.repo_id, file.relative_path, revision=entry.revision)
463
+
464
+
465
+ # ======================================================================
466
+ # Status projection (F-53, F-63)
467
+ # ======================================================================
468
+
469
+
470
+ @dataclass(frozen=True, slots=True)
471
+ class ModelStatus:
472
+ """One row of F-63's model management screen, and F-53's model entry.
473
+
474
+ ``runnable`` and ``unavailable_reason`` travel together because F-04
475
+ forbids hiding a model that cannot run in the current budget and forbids
476
+ substituting another for it: the only correct presentation is the model,
477
+ shown as unavailable, with the reason.
478
+ """
479
+
480
+ model_id: str
481
+ display_name: str
482
+ state: ModelState
483
+ sample_rate: int
484
+ languages: tuple[str, ...]
485
+ bytes_present: int
486
+ bytes_total: int
487
+ disk_bytes: int
488
+ runnable: bool
489
+ unavailable_reason: str | None
490
+ license_name: str
491
+ license_acceptance_required: bool
492
+ license_accepted: bool
493
+ download_authorised: bool
494
+ #: True when the only sound copy is the ``supertonic`` package's cache.
495
+ using_package_cache: bool
496
+ minimum_memory_bytes: int
497
+ minimum_cpu_percent: int
498
+
499
+
500
+ # ======================================================================
501
+ # The registry
502
+ # ======================================================================
503
+
504
+
505
+ class ModelRegistry:
506
+ """F-63 to F-65's operations over F-84's manifest."""
507
+
508
+ def __init__(
509
+ self,
510
+ manifest: Manifest = MANIFEST,
511
+ *,
512
+ root: Path | None = None,
513
+ preferences: ModelPreferences | None = None,
514
+ fetcher: Fetcher | None = None,
515
+ package_cache_dirs: dict[str, Path] | None = None,
516
+ ) -> None:
517
+ self.manifest = manifest
518
+ self._root = root
519
+ self.preferences: ModelPreferences = preferences or JsonModelPreferences()
520
+ self._fetcher = fetcher
521
+ self._package_cache_dirs = package_cache_dirs
522
+
523
+ # -- locations ------------------------------------------------------
524
+
525
+ @property
526
+ def root(self) -> Path:
527
+ """Resolved per call: ``paths.model_cache_dir`` reads the
528
+ environment through an ``lru_cache`` that a test may have cleared."""
529
+ return self._root if self._root is not None else model_cache_dir()
530
+
531
+ def entry(self, model_id: str) -> ModelEntry:
532
+ return self.manifest.get(model_id)
533
+
534
+ def model_dir(self, model_id: str) -> Path:
535
+ """The app-owned directory for a model's files."""
536
+ return self.root / self.entry(model_id).model_id
537
+
538
+ def package_cache_dir(self, model_id: str) -> Path | None:
539
+ """Where the ``supertonic`` package keeps the same weights, if the
540
+ model is one it knows about.
541
+
542
+ Read-only as far as this app is concerned. F-73 reports it apart
543
+ from the app's own cache, and F-65's delete does not touch it: it
544
+ belongs to another program and may be shared with other tools.
545
+ """
546
+ if self._package_cache_dirs is not None:
547
+ return self._package_cache_dirs.get(model_id)
548
+ name = _PACKAGE_CACHE_NAMES.get(model_id)
549
+ if name is None:
550
+ return None
551
+ override = os.environ.get("SUPERTONIC_CACHE_DIR")
552
+ if override:
553
+ return Path(override).expanduser()
554
+ return Path.home() / ".cache" / name
555
+
556
+ # -- verification (F-65) --------------------------------------------
557
+
558
+ def verify(
559
+ self,
560
+ model_id: str,
561
+ *,
562
+ deep: bool = True,
563
+ cancel: CancelToken | None = None,
564
+ progress: ProgressCallback | None = None,
565
+ root: Path | None = None,
566
+ ) -> VerifyReport:
567
+ """Check every manifest file's size and, when ``deep``, its digest.
568
+
569
+ Size first because a truncated download is the common case and
570
+ hashing it would be a second wasted; the digest is what catches the
571
+ tampering F-84 and A-23 care about. Reading is chunked and checks
572
+ the cancel token between chunks, per N-21.
573
+
574
+ A pass cut short cannot report ``READY``: a file it never reached,
575
+ or whose hash it abandoned part-way, counts as unchecked, and an
576
+ unchecked file is not sound.
577
+ """
578
+ entry = self.entry(model_id)
579
+ base = root if root is not None else self.model_dir(model_id)
580
+ statuses: list[FileStatus] = []
581
+ cancelled = False
582
+ done = 0
583
+ total = entry.total_bytes
584
+
585
+ for index, file in enumerate(entry.files):
586
+ if _cancelled(cancel):
587
+ cancelled = True
588
+ statuses.extend(_unchecked(f) for f in entry.files[index:])
589
+ break
590
+ status, read = self._verify_file(file, base, deep=deep, cancel=cancel)
591
+ statuses.append(status)
592
+ done += read
593
+ if progress is not None:
594
+ progress(
595
+ DownloadProgress(
596
+ model_id=model_id,
597
+ phase=DownloadPhase.CHECKING,
598
+ relative_path=file.relative_path,
599
+ file_index=index,
600
+ file_count=entry.file_count,
601
+ file_bytes_done=status.actual_bytes,
602
+ file_bytes_total=file.byte_size,
603
+ bytes_done=done,
604
+ bytes_total=total,
605
+ )
606
+ )
607
+ if _cancelled(cancel):
608
+ cancelled = True
609
+ statuses.extend(_unchecked(f) for f in entry.files[index + 1 :])
610
+ break
611
+
612
+ return VerifyReport(
613
+ model_id=model_id,
614
+ root=base,
615
+ files=tuple(statuses),
616
+ deep=deep,
617
+ cancelled=cancelled,
618
+ )
619
+
620
+ def _verify_file(
621
+ self, file: ModelFile, base: Path, *, deep: bool, cancel: CancelToken | None
622
+ ) -> tuple[FileStatus, int]:
623
+ path = file.path_under(base)
624
+ try:
625
+ size = path.stat().st_size
626
+ except OSError:
627
+ return (
628
+ FileStatus(
629
+ relative_path=file.relative_path,
630
+ expected_bytes=file.byte_size,
631
+ actual_bytes=0,
632
+ present=False,
633
+ size_ok=False,
634
+ digest_ok=None,
635
+ ),
636
+ 0,
637
+ )
638
+ size_ok = size == file.byte_size
639
+ digest_ok: bool | None = None
640
+ read = 0
641
+ if size_ok and deep:
642
+ digest, read = _sha256_file(path, cancel=cancel)
643
+ digest_ok = digest == file.sha256 if digest is not None else None
644
+ return (
645
+ FileStatus(
646
+ relative_path=file.relative_path,
647
+ expected_bytes=file.byte_size,
648
+ actual_bytes=size,
649
+ present=True,
650
+ size_ok=size_ok,
651
+ digest_ok=digest_ok,
652
+ checked=not (size_ok and deep and digest_ok is None),
653
+ ),
654
+ read,
655
+ )
656
+
657
+ def state(self, model_id: str, *, cancel: CancelToken | None = None) -> ModelState:
658
+ """F-65's question, answered against digests."""
659
+ return self.verify(model_id, deep=True, cancel=cancel).state
660
+
661
+ def quick_state(self, model_id: str) -> ModelState:
662
+ """Presence and size only.
663
+
664
+ For a screen that refreshes: hashing 385 MB to redraw a list is
665
+ wasteful, and a size check catches everything except deliberate
666
+ tampering. Anything that is about to *load* the model calls
667
+ :meth:`prepared_dir`, which is deep.
668
+ """
669
+ return self.verify(model_id, deep=False).state
670
+
671
+ def resolve_dir(
672
+ self, model_id: str, *, deep: bool = True, cancel: CancelToken | None = None
673
+ ) -> tuple[Path, bool]:
674
+ """The directory holding a sound copy, and whether it is the
675
+ package's cache rather than the app's own.
676
+
677
+ Raises rather than returning a doubtful path: F-84 says a model whose
678
+ files do not match the manifest is reported as corrupted, not used.
679
+
680
+ The licence is checked here and not only in :meth:`download`, because
681
+ this is the path a model actually reaches the engine by (N-11, F-80,
682
+ A-23). Neither case that matters passes through a download: the
683
+ package cache is read where it lies, and a release that amends
684
+ Attachment A leaves the files READY, so nothing would ever ask again.
685
+
686
+ It is checked *after* the files are found, not before. N-11 makes
687
+ acceptance a condition of preparing a model, so for a model that is
688
+ not on this machine the useful answer is that it is not prepared --
689
+ which is the flow that presents the terms. Reporting an unaccepted
690
+ licence for a model that also is not there would be true and
691
+ useless, and would send the owner to the wrong screen.
692
+ """
693
+ own = self.verify(model_id, deep=deep, cancel=cancel)
694
+ if own.state is ModelState.READY:
695
+ self._require_license(self.entry(model_id))
696
+ return self.model_dir(model_id), False
697
+
698
+ fallback = self.package_cache_dir(model_id)
699
+ if fallback is not None and fallback.is_dir():
700
+ other = self.verify(model_id, deep=deep, cancel=cancel, root=fallback)
701
+ if other.state is ModelState.READY:
702
+ # The case the gate exists for: borrowed weights reach the
703
+ # engine without a download ever being asked for.
704
+ self._require_license(self.entry(model_id))
705
+ log.info("model %s served from the package cache %s", model_id, redact(fallback))
706
+ return fallback, True
707
+
708
+ if own.state is ModelState.CORRUPT:
709
+ raise EchoActError(
710
+ Code.MODEL_CORRUPT,
711
+ detail={"model_id": model_id, "files": list(own.damaged)},
712
+ )
713
+ raise EchoActError(
714
+ Code.MODEL_NOT_READY,
715
+ detail={"model_id": model_id, "missing": len(own.unusable)},
716
+ )
717
+
718
+ def prepared_dir(
719
+ self,
720
+ model_id: str,
721
+ *,
722
+ budget: Budget | None = None,
723
+ deep: bool = True,
724
+ cancel: CancelToken | None = None,
725
+ ) -> Path:
726
+ """What the engine is given for ``Load.model_dir``.
727
+
728
+ The budget check happens here as well as on the selection screen
729
+ because F-04's promise is that a model that cannot run is refused
730
+ with a reason, and the last chance to keep that promise is the
731
+ moment before loading.
732
+ """
733
+ if budget is not None:
734
+ self.ensure_can_run(model_id, budget)
735
+ path, _ = self.resolve_dir(model_id, deep=deep, cancel=cancel)
736
+ return path
737
+
738
+ # -- budget (F-04, N-05) --------------------------------------------
739
+
740
+ def can_run(self, model_id: str, budget: Budget) -> tuple[bool, str | None]:
741
+ """F-04: a model that cannot run under the current budget is shown
742
+ as unavailable *with the reason*, never hidden and never replaced.
743
+
744
+ Returns ``(True, None)`` or ``(False, reason)``. The reason is
745
+ written for a person and repeated verbatim to an API caller, so it
746
+ names both the requirement and the current value.
747
+ """
748
+ entry = self.entry(model_id)
749
+ minimum = entry.minimum_budget
750
+ reasons: list[str] = []
751
+ if budget.memory_bytes < minimum.memory_bytes:
752
+ reasons.append(
753
+ f"{entry.display_name} is approved from a memory budget of "
754
+ f"{_gib(minimum.memory_bytes)}; the current budget is "
755
+ f"{_gib(budget.memory_bytes)}"
756
+ )
757
+ if budget.cpu_percent < minimum.cpu_percent:
758
+ reasons.append(
759
+ f"{entry.display_name} is approved from a CPU budget of "
760
+ f"{minimum.cpu_percent}%; the current budget is {budget.cpu_percent}%"
761
+ )
762
+ if reasons:
763
+ return False, "; ".join(reasons) + "."
764
+ return True, None
765
+
766
+ def ensure_can_run(self, model_id: str, budget: Budget) -> None:
767
+ runnable, reason = self.can_run(model_id, budget)
768
+ if not runnable:
769
+ raise EchoActError(
770
+ Code.MODEL_OVER_BUDGET,
771
+ reason,
772
+ detail={
773
+ "model_id": model_id,
774
+ "minimum_memory_bytes": self.entry(model_id).minimum_budget.memory_bytes,
775
+ "minimum_cpu_percent": self.entry(model_id).minimum_budget.cpu_percent,
776
+ },
777
+ )
778
+
779
+ # -- licence (N-11, F-80) -------------------------------------------
780
+
781
+ def license_terms(self, model_id: str) -> LicenseTerms:
782
+ return self.entry(model_id).license
783
+
784
+ def license_acceptance_required(self, model_id: str) -> bool:
785
+ """True while N-11's terms still have to be shown and accepted.
786
+
787
+ The fingerprint covers the restrictions themselves, so a release
788
+ that ships changed terms asks again rather than inheriting consent
789
+ the user gave to different text.
790
+ """
791
+ terms = self.license_terms(model_id)
792
+ if not terms.acceptance_required:
793
+ return False
794
+ return not self.preferences.license_accepted(model_id, _fingerprint(terms))
795
+
796
+ def accept_license(self, model_id: str) -> None:
797
+ """Record that the owner accepted this model's terms (N-11)."""
798
+ terms = self.license_terms(model_id)
799
+ self.preferences.record_license_acceptance(model_id, _fingerprint(terms))
800
+ log.info("licence accepted for model %s (%s)", model_id, terms.name)
801
+
802
+ # -- external authorisation (5.3) -----------------------------------
803
+
804
+ def download_authorised(self, model_id: str) -> bool:
805
+ self.entry(model_id)
806
+ return self.preferences.download_authorised(model_id)
807
+
808
+ def set_download_authorised(self, model_id: str, allowed: bool) -> None:
809
+ """The GUI's pre-authorisation switch from Section 5.3.
810
+
811
+ Without it, a REST or MCP request can never start a download: the
812
+ rule exists so that an integration cannot make the machine fetch
813
+ hundreds of megabytes on its own initiative.
814
+ """
815
+ self.entry(model_id)
816
+ self.preferences.set_download_authorised(model_id, allowed)
817
+ log.info("download authorisation for %s set to %s", model_id, bool(allowed))
818
+
819
+ # -- preparation (F-09, F-64) ---------------------------------------
820
+
821
+ def download(
822
+ self,
823
+ model_id: str,
824
+ progress_cb: ProgressCallback | None = None,
825
+ cancel: CancelToken | None = None,
826
+ *,
827
+ request_path: RequestPath = RequestPath.GUI,
828
+ discard_partial: bool = False,
829
+ ) -> DownloadOutcome:
830
+ """Fetch whatever is missing or unsound, and nothing else (F-64).
831
+
832
+ The work is decided by a verification pass, so a retry after a
833
+ failure re-downloads exactly the files that are absent or corrupt and
834
+ reuses the rest. Within a file, a ``.part`` left by an earlier
835
+ attempt is resumed from with a range request; if the server ignores
836
+ the range, the transfer restarts rather than appending to it.
837
+
838
+ Nothing is renamed into place until its digest matches the manifest,
839
+ which is what makes F-64's "a cancelled download is never shown as
840
+ ready" true by construction rather than by a flag someone has to
841
+ remember to clear.
842
+
843
+ Only one attempt per model runs at a time (N-23). A second caller --
844
+ the GUI and a REST request, two REST requests, a repair overtaking a
845
+ download -- waits, and then its own verification pass finds the work
846
+ already done and fetches nothing.
847
+ """
848
+ entry = self.entry(model_id)
849
+ self._require_license(entry)
850
+ self._require_authorisation(entry, request_path)
851
+
852
+ with _preparing(self.model_dir(model_id), cancel) as held:
853
+ if not held:
854
+ # Cancelled while queued behind another attempt. Nothing was
855
+ # examined, so nothing is claimed: the pass below runs with an
856
+ # already-cancelled token and reports every file unchecked.
857
+ return self._cancelled_outcome(
858
+ entry, self.verify(model_id, cancel=cancel), progress_cb, 0, (), ()
859
+ )
860
+ return self._prepare(entry, progress_cb, cancel, discard_partial=discard_partial)
861
+
862
+ def _prepare(
863
+ self,
864
+ entry: ModelEntry,
865
+ progress_cb: ProgressCallback | None,
866
+ cancel: CancelToken | None,
867
+ *,
868
+ discard_partial: bool,
869
+ ) -> DownloadOutcome:
870
+ """:meth:`download`'s body, with this model's preparation lock held."""
871
+ model_id = entry.model_id
872
+ report = self.verify(model_id, cancel=cancel, progress=progress_cb)
873
+ if report.cancelled:
874
+ return self._cancelled_outcome(entry, report, progress_cb, 0, (), ())
875
+
876
+ todo = [f for f in entry.files if not _is_ok(report, f.relative_path)]
877
+ reused = tuple(f.relative_path for f in entry.files if _is_ok(report, f.relative_path))
878
+ if not todo:
879
+ self._emit(
880
+ progress_cb,
881
+ entry,
882
+ DownloadPhase.COMPLETE,
883
+ None,
884
+ 0,
885
+ report.bytes_expected,
886
+ report.bytes_expected,
887
+ )
888
+ return DownloadOutcome(
889
+ model_id=model_id,
890
+ completed=True,
891
+ cancelled=False,
892
+ bytes_downloaded=0,
893
+ reused=reused,
894
+ fetched=(),
895
+ report=report,
896
+ )
897
+
898
+ base = self.model_dir(model_id)
899
+ remaining = sum(f.byte_size for f in todo)
900
+ self._require_space(base, remaining)
901
+
902
+ fetcher = self._fetcher if self._fetcher is not None else HttpxFetcher()
903
+ already = report.bytes_expected - remaining
904
+ downloaded = 0
905
+ fetched: list[str] = []
906
+
907
+ for index, file in enumerate(todo):
908
+ if _cancelled(cancel):
909
+ return self._cancelled_outcome(
910
+ entry,
911
+ self._interrupted_report(entry, report, fetched, None),
912
+ progress_cb,
913
+ downloaded,
914
+ reused,
915
+ tuple(fetched),
916
+ )
917
+
918
+ def tick(
919
+ file_done: int,
920
+ *,
921
+ current: ModelFile = file,
922
+ position: int = index,
923
+ prior: int = already + downloaded,
924
+ ) -> None:
925
+ self._emit(
926
+ progress_cb,
927
+ entry,
928
+ DownloadPhase.DOWNLOADING,
929
+ current,
930
+ file_done,
931
+ prior + file_done,
932
+ report.bytes_expected,
933
+ file_index=position,
934
+ file_count=len(todo),
935
+ )
936
+
937
+ written, cancelled = self._download_file(
938
+ entry,
939
+ file,
940
+ base,
941
+ fetcher,
942
+ cancel=cancel,
943
+ discard_partial=discard_partial,
944
+ on_chunk=tick,
945
+ )
946
+ downloaded += written
947
+ if cancelled:
948
+ return self._cancelled_outcome(
949
+ entry,
950
+ self._interrupted_report(entry, report, fetched, file.relative_path),
951
+ progress_cb,
952
+ downloaded,
953
+ reused,
954
+ tuple(fetched),
955
+ )
956
+ fetched.append(file.relative_path)
957
+
958
+ final = self.verify(model_id, cancel=cancel)
959
+ if final.cancelled:
960
+ # Cancelled after the last byte landed: the files may well be
961
+ # sound, but this pass did not establish that, and F-64 would
962
+ # rather report the cancellation than a readiness it guessed.
963
+ return self._cancelled_outcome(
964
+ entry, final, progress_cb, downloaded, reused, tuple(fetched)
965
+ )
966
+ if final.state is not ModelState.READY:
967
+ # Everything was digest-checked on the way in, so this means the
968
+ # tree changed underneath us. F-84: report corrupt, do not use.
969
+ raise EchoActError(
970
+ Code.MODEL_CORRUPT,
971
+ detail={"model_id": model_id, "files": list(final.unusable)},
972
+ )
973
+ self._emit(
974
+ progress_cb,
975
+ entry,
976
+ DownloadPhase.COMPLETE,
977
+ None,
978
+ 0,
979
+ final.bytes_expected,
980
+ final.bytes_expected,
981
+ )
982
+ log.info(
983
+ "model %s prepared: %d file(s) fetched, %d reused", model_id, len(fetched), len(reused)
984
+ )
985
+ return DownloadOutcome(
986
+ model_id=model_id,
987
+ completed=True,
988
+ cancelled=False,
989
+ bytes_downloaded=downloaded,
990
+ reused=reused,
991
+ fetched=tuple(fetched),
992
+ report=final,
993
+ )
994
+
995
+ def repair(
996
+ self,
997
+ model_id: str,
998
+ progress_cb: ProgressCallback | None = None,
999
+ cancel: CancelToken | None = None,
1000
+ *,
1001
+ request_path: RequestPath = RequestPath.GUI,
1002
+ ) -> DownloadOutcome:
1003
+ """F-65's repair: replace whatever does not match the manifest.
1004
+
1005
+ Stronger than a retry in one respect -- it discards half-downloaded
1006
+ parts instead of resuming them. A retry assumes the interruption was
1007
+ the problem; a repair is asked for because something is wrong, and a
1008
+ partial file is a candidate for what that is.
1009
+ """
1010
+ return self.download(
1011
+ model_id,
1012
+ progress_cb,
1013
+ cancel,
1014
+ request_path=request_path,
1015
+ discard_partial=True,
1016
+ )
1017
+
1018
+ def _download_file(
1019
+ self,
1020
+ entry: ModelEntry,
1021
+ file: ModelFile,
1022
+ base: Path,
1023
+ fetcher: Fetcher,
1024
+ *,
1025
+ cancel: CancelToken | None,
1026
+ discard_partial: bool,
1027
+ on_chunk: Callable[[int], None],
1028
+ ) -> tuple[int, bool]:
1029
+ """Stream one file into place. Returns (bytes written, cancelled)."""
1030
+ target = file.path_under(base)
1031
+ part = target.with_name(target.name + _PART_SUFFIX)
1032
+ with _writing(target.parent):
1033
+ target.parent.mkdir(parents=True, exist_ok=True)
1034
+
1035
+ # A target that exists here failed verification, so it is not
1036
+ # something to keep: F-64 re-downloads corrupted data.
1037
+ _unlink(target)
1038
+ if discard_partial:
1039
+ _unlink(part)
1040
+
1041
+ url = resolve_url(entry, file)
1042
+ for attempt in (1, 2):
1043
+ written, cancelled, digest = self._stream_to_part(
1044
+ url, part, file, fetcher, cancel=cancel, on_chunk=on_chunk
1045
+ )
1046
+ if cancelled:
1047
+ return written, True
1048
+ if digest == file.sha256:
1049
+ _replace(part, target)
1050
+ return written, False
1051
+ # Bad bytes. The first explanation is a damaged transfer or a
1052
+ # stale resume, so throw the part away and fetch the whole file
1053
+ # once more before concluding anything about the upstream.
1054
+ _unlink(part)
1055
+ log.warning(
1056
+ "digest mismatch for %s (attempt %d)", redact(target), attempt
1057
+ )
1058
+ raise EchoActError(
1059
+ Code.MODEL_CORRUPT,
1060
+ "A downloaded model file does not match the manifest.",
1061
+ detail={"model_id": entry.model_id, "file": file.relative_path},
1062
+ )
1063
+
1064
+ def _stream_to_part(
1065
+ self,
1066
+ url: str,
1067
+ part: Path,
1068
+ file: ModelFile,
1069
+ fetcher: Fetcher,
1070
+ *,
1071
+ cancel: CancelToken | None,
1072
+ on_chunk: Callable[[int], None],
1073
+ ) -> tuple[int, bool, str | None]:
1074
+ hasher = hashlib.sha256()
1075
+ offset = 0
1076
+ try:
1077
+ existing = part.stat().st_size
1078
+ except OSError:
1079
+ existing = 0
1080
+ if 0 < existing < file.byte_size:
1081
+ # Hash what is already there so the digest covers the whole file
1082
+ # without re-reading it after the transfer.
1083
+ digest_so_far, _ = _sha256_file(part, cancel=cancel, hasher=hasher)
1084
+ if digest_so_far is None:
1085
+ return 0, True, None
1086
+ offset = existing
1087
+ elif existing:
1088
+ _unlink(part)
1089
+
1090
+ written = 0
1091
+ with fetcher.open(url, offset=offset) as body:
1092
+ if offset and not body.resumed:
1093
+ # The server sent the whole file; appending would corrupt it.
1094
+ hasher = hashlib.sha256()
1095
+ offset = 0
1096
+ mode = "ab" if offset else "wb"
1097
+ # Opening, every write, and the flush at the end of the block are
1098
+ # all inside ``_writing``: rule 3 lets no OSError out of here, and
1099
+ # 5.3 wants a disk that fills mid-transfer reported as a failed
1100
+ # save rather than as a crash.
1101
+ with _writing(part), part.open(mode) as sink:
1102
+ for chunk in body.chunks:
1103
+ if _cancelled(cancel):
1104
+ sink.flush()
1105
+ return written, True, None
1106
+ sink.write(chunk)
1107
+ hasher.update(chunk)
1108
+ written += len(chunk)
1109
+ on_chunk(offset + written)
1110
+ return written, False, hasher.hexdigest()
1111
+
1112
+ # -- deletion (F-65, F-76) ------------------------------------------
1113
+
1114
+ def delete(self, model_id: str, *, in_use: bool = False) -> int:
1115
+ """Remove the app's copy of a model. Returns the bytes freed.
1116
+
1117
+ Only the model's own directory under the model cache is touched.
1118
+ F-65 is explicit that deleting a model does not delete retained
1119
+ documents or audio results, and those live under different roots in
1120
+ :mod:`echoact.paths`; nothing here can reach them.
1121
+
1122
+ ``in_use`` is the caller's answer to "is a job holding this model?".
1123
+ F-65 requires that job to be cancelled and the model released first,
1124
+ and the registry cannot see jobs, so it refuses and says why rather
1125
+ than guessing.
1126
+ """
1127
+ entry = self.entry(model_id)
1128
+ if in_use:
1129
+ raise EchoActError(
1130
+ Code.DELETE_BLOCKED_IN_USE,
1131
+ "The model is loaded by a running job; cancel it first.",
1132
+ detail={"model_id": entry.model_id},
1133
+ )
1134
+ target = self.model_dir(model_id)
1135
+ root = self.root
1136
+ # Structural guard: the only thing this method may ever remove is a
1137
+ # named subdirectory of the model cache.
1138
+ if target == root or root not in target.parents:
1139
+ raise AssertionError(f"refusing to delete {target} outside {root}")
1140
+ # N-23: deleting the tree a download is streaming into would leave it
1141
+ # renaming files into a directory nobody expects to exist. Refused
1142
+ # rather than queued, because F-65's delete is a GUI action and rule 7
1143
+ # forbids the main thread waiting minutes for a transfer to finish.
1144
+ lock = _prepare_lock(target)
1145
+ if not lock.acquire(blocking=False):
1146
+ raise EchoActError(
1147
+ Code.DELETE_BLOCKED_IN_USE,
1148
+ "The model is being prepared; cancel that first.",
1149
+ detail={"model_id": entry.model_id},
1150
+ retry_after_s=_DOWNLOAD_RETRY_AFTER_S,
1151
+ )
1152
+ try:
1153
+ freed = _tree_bytes(target)
1154
+ try:
1155
+ shutil.rmtree(target)
1156
+ except FileNotFoundError:
1157
+ return 0
1158
+ except OSError as exc:
1159
+ raise EchoActError(
1160
+ Code.INTERNAL,
1161
+ "The model files could not be deleted.",
1162
+ detail={"model_id": entry.model_id, "path": redact(target)},
1163
+ cause=exc,
1164
+ ) from exc
1165
+ finally:
1166
+ lock.release()
1167
+ log.info("deleted model %s (%d bytes)", entry.model_id, freed)
1168
+ return freed
1169
+
1170
+ # -- reporting (F-53, F-63, F-73) -----------------------------------
1171
+
1172
+ def disk_usage(self, model_id: str) -> int:
1173
+ """Bytes the app's copy occupies, partial downloads included."""
1174
+ return _tree_bytes(self.model_dir(model_id))
1175
+
1176
+ def total_disk_usage(self) -> int:
1177
+ """F-73's "model cache" figure: the whole app-owned cache tree."""
1178
+ return _tree_bytes(self.root)
1179
+
1180
+ def status(
1181
+ self, model_id: str, budget: Budget | None = None, *, deep: bool = False
1182
+ ) -> ModelStatus:
1183
+ """One model as F-63 and F-53 present it."""
1184
+ entry = self.entry(model_id)
1185
+ report = self.verify(model_id, deep=deep)
1186
+ using_package = False
1187
+ if report.state is not ModelState.READY:
1188
+ fallback = self.package_cache_dir(model_id)
1189
+ if fallback is not None and fallback.is_dir():
1190
+ other = self.verify(model_id, deep=deep, root=fallback)
1191
+ if other.state is ModelState.READY:
1192
+ report = other
1193
+ using_package = True
1194
+ runnable, reason = (True, None) if budget is None else self.can_run(model_id, budget)
1195
+ return ModelStatus(
1196
+ model_id=entry.model_id,
1197
+ display_name=entry.display_name,
1198
+ state=report.state,
1199
+ sample_rate=entry.sample_rate,
1200
+ languages=tuple(lang.value for lang in entry.languages),
1201
+ bytes_present=report.bytes_present,
1202
+ bytes_total=entry.total_bytes,
1203
+ disk_bytes=self.disk_usage(model_id),
1204
+ runnable=runnable,
1205
+ unavailable_reason=reason,
1206
+ license_name=entry.license.name,
1207
+ license_acceptance_required=entry.license.acceptance_required,
1208
+ license_accepted=not self.license_acceptance_required(model_id),
1209
+ download_authorised=self.preferences.download_authorised(model_id),
1210
+ using_package_cache=using_package,
1211
+ minimum_memory_bytes=entry.minimum_budget.memory_bytes,
1212
+ minimum_cpu_percent=entry.minimum_budget.cpu_percent,
1213
+ )
1214
+
1215
+ def statuses(self, budget: Budget | None = None) -> tuple[ModelStatus, ...]:
1216
+ return tuple(self.status(m.model_id, budget) for m in self.manifest)
1217
+
1218
+ # -- gates ----------------------------------------------------------
1219
+
1220
+ def _require_license(self, entry: ModelEntry) -> None:
1221
+ if self.license_acceptance_required(entry.model_id):
1222
+ raise EchoActError(
1223
+ Code.MODEL_LICENSE_NOT_ACCEPTED,
1224
+ detail={"model_id": entry.model_id, "license": entry.license.name},
1225
+ )
1226
+
1227
+ def _require_authorisation(self, entry: ModelEntry, request_path: RequestPath) -> None:
1228
+ """Section 5.3: only a model the owner pre-authorised in the GUI may
1229
+ be downloaded on an external request."""
1230
+ if request_path is RequestPath.GUI:
1231
+ return
1232
+ if not self.preferences.download_authorised(entry.model_id):
1233
+ raise EchoActError(
1234
+ Code.MODEL_DOWNLOAD_FORBIDDEN,
1235
+ detail={"model_id": entry.model_id, "request_path": request_path.value},
1236
+ )
1237
+
1238
+ def _require_space(self, base: Path, needed: int) -> None:
1239
+ """F-09: preparation needs storage, and running the disk to zero is
1240
+ worse than refusing. 4.1's low-space figure is the headroom left."""
1241
+ probe = base
1242
+ while not probe.exists() and probe.parent != probe:
1243
+ probe = probe.parent
1244
+ try:
1245
+ free = shutil.disk_usage(probe).free
1246
+ except OSError:
1247
+ return
1248
+ if free < needed + LOW_SPACE_WARNING_BYTES:
1249
+ raise EchoActError(
1250
+ Code.STORAGE_FULL,
1251
+ "There is not enough free disk space to prepare this model.",
1252
+ detail={"needed_bytes": needed, "free_bytes": free},
1253
+ )
1254
+
1255
+ # -- helpers --------------------------------------------------------
1256
+
1257
+ def _emit(
1258
+ self,
1259
+ cb: ProgressCallback | None,
1260
+ entry: ModelEntry,
1261
+ phase: DownloadPhase,
1262
+ file: ModelFile | None,
1263
+ file_done: int,
1264
+ done: int,
1265
+ total: int,
1266
+ *,
1267
+ file_index: int = 0,
1268
+ file_count: int = 0,
1269
+ ) -> None:
1270
+ if cb is None:
1271
+ return
1272
+ cb(
1273
+ DownloadProgress(
1274
+ model_id=entry.model_id,
1275
+ phase=phase,
1276
+ relative_path=file.relative_path if file else "",
1277
+ file_index=file_index,
1278
+ file_count=file_count or entry.file_count,
1279
+ file_bytes_done=file_done,
1280
+ file_bytes_total=file.byte_size if file else 0,
1281
+ bytes_done=done,
1282
+ bytes_total=total,
1283
+ )
1284
+ )
1285
+
1286
+ def _interrupted_report(
1287
+ self,
1288
+ entry: ModelEntry,
1289
+ baseline: VerifyReport,
1290
+ fetched: Iterable[str],
1291
+ in_flight: str | None,
1292
+ ) -> VerifyReport:
1293
+ """What a cancelled attempt is entitled to say about the model.
1294
+
1295
+ F-64: a cancelled download is never shown as ready. The only
1296
+ evidence of soundness that exists at this point is a digest -- the
1297
+ one the opening deep pass computed for a file the attempt did not
1298
+ have to fetch, or the one the transfer matched before renaming a file
1299
+ into place. Verifying again here could only be a shallow pass, since
1300
+ N-22 allows five seconds to stop and 385 MB does not hash in that;
1301
+ and a shallow pass calls a same-size tampered file sound, so it would
1302
+ let a cancellation report READY for a model the deep pass had just
1303
+ called corrupt. The deep verdicts are therefore carried forward
1304
+ instead of being thrown away.
1305
+
1306
+ The file being written when the cancellation landed counts as
1307
+ unknown: its target was removed before the transfer began, and only
1308
+ a ``.part`` stands in its place.
1309
+ """
1310
+ done = set(fetched)
1311
+ statuses: list[FileStatus] = []
1312
+ for file in entry.files:
1313
+ prior = baseline.status(file.relative_path)
1314
+ if file.relative_path in done:
1315
+ statuses.append(
1316
+ FileStatus(
1317
+ relative_path=file.relative_path,
1318
+ expected_bytes=file.byte_size,
1319
+ actual_bytes=file.byte_size,
1320
+ present=True,
1321
+ size_ok=True,
1322
+ digest_ok=True,
1323
+ )
1324
+ )
1325
+ elif prior is None or file.relative_path == in_flight:
1326
+ statuses.append(_unchecked(file))
1327
+ else:
1328
+ statuses.append(prior)
1329
+ return VerifyReport(
1330
+ model_id=entry.model_id,
1331
+ root=self.model_dir(entry.model_id),
1332
+ files=tuple(statuses),
1333
+ deep=baseline.deep,
1334
+ cancelled=True,
1335
+ )
1336
+
1337
+ def _cancelled_outcome(
1338
+ self,
1339
+ entry: ModelEntry,
1340
+ report: VerifyReport,
1341
+ cb: ProgressCallback | None,
1342
+ downloaded: int,
1343
+ reused: tuple[str, ...],
1344
+ fetched: tuple[str, ...],
1345
+ ) -> DownloadOutcome:
1346
+ self._emit(cb, entry, DownloadPhase.CANCELLED, None, 0, report.bytes_present,
1347
+ entry.total_bytes)
1348
+ log.info("model %s preparation cancelled", entry.model_id)
1349
+ return DownloadOutcome(
1350
+ model_id=entry.model_id,
1351
+ completed=False,
1352
+ cancelled=True,
1353
+ bytes_downloaded=downloaded,
1354
+ reused=reused,
1355
+ fetched=fetched,
1356
+ report=report,
1357
+ )
1358
+
1359
+
1360
+ # ======================================================================
1361
+ # Module helpers
1362
+ # ======================================================================
1363
+
1364
+
1365
+ #: One preparation lock per model directory, shared by every registry in
1366
+ #: this process. N-23: two requests for one model must not become two
1367
+ #: transfers. The ``.part`` path is derived from the model id and the file
1368
+ #: path alone, so two attempts open the same file -- one truncating while the
1369
+ #: other appends -- and the digest each computes over its own stream says
1370
+ #: nothing about the bytes that ended up on disk. Keyed by directory rather
1371
+ #: than by model id, so two registries over one cache serialise and two over
1372
+ #: different caches do not.
1373
+ _PREPARE_LOCKS: dict[str, threading.RLock] = {}
1374
+ _PREPARE_LOCKS_GUARD = threading.Lock()
1375
+
1376
+
1377
+ def _prepare_lock(directory: Path) -> threading.RLock:
1378
+ key = os.path.normcase(os.path.abspath(directory))
1379
+ with _PREPARE_LOCKS_GUARD:
1380
+ lock = _PREPARE_LOCKS.get(key)
1381
+ if lock is None:
1382
+ lock = threading.RLock()
1383
+ _PREPARE_LOCKS[key] = lock
1384
+ return lock
1385
+
1386
+
1387
+ @contextmanager
1388
+ def _preparing(directory: Path, cancel: CancelToken | None) -> Iterator[bool]:
1389
+ """Hold one model's preparation lock (N-23).
1390
+
1391
+ Yields ``True`` with the lock held, or ``False`` when the caller
1392
+ cancelled while waiting for the attempt in front of it -- the wait is
1393
+ polled rather than indefinite so that N-22's five seconds to stop hold
1394
+ for a queued attempt too.
1395
+ """
1396
+ lock = _prepare_lock(directory)
1397
+ while not lock.acquire(timeout=_LOCK_POLL_S):
1398
+ if _cancelled(cancel):
1399
+ yield False
1400
+ return
1401
+ try:
1402
+ yield True
1403
+ finally:
1404
+ lock.release()
1405
+
1406
+
1407
+ @contextmanager
1408
+ def _writing(path: Path) -> Iterator[None]:
1409
+ """Convert a failed write into this module's own error (rule 3, 5.3).
1410
+
1411
+ :meth:`ModelRegistry._require_space` cannot stand in for this. It asks
1412
+ once, before a transfer that runs for minutes: it does not see another
1413
+ process taking the last of the disk in the meantime, a cache directory
1414
+ that turns out to be read-only, or a scanner holding the ``.part`` open
1415
+ -- and on a volume whose free space cannot be read at all it declines to
1416
+ answer and lets the transfer start anyway.
1417
+ """
1418
+ try:
1419
+ yield
1420
+ except OSError as exc:
1421
+ out_of_room = exc.errno in (errno.ENOSPC, errno.EDQUOT)
1422
+ raise EchoActError(
1423
+ Code.STORAGE_FULL if out_of_room else Code.MODEL_DOWNLOAD_FAILED,
1424
+ "There is not enough free disk space to finish preparing this model."
1425
+ if out_of_room
1426
+ else "The model file could not be written to the cache.",
1427
+ detail={"path": redact(path)},
1428
+ retry_after_s=None if out_of_room else _DOWNLOAD_RETRY_AFTER_S,
1429
+ cause=exc,
1430
+ ) from exc
1431
+
1432
+
1433
+ def _sha256_file(
1434
+ path: Path, *, cancel: CancelToken | None = None, hasher: hashlib._Hash | None = None
1435
+ ) -> tuple[str | None, int]:
1436
+ """Digest a file in bounded chunks. ``None`` means cancelled.
1437
+
1438
+ N-21 forbids holding the file in memory -- ``vector_estimator.onnx`` is
1439
+ 256 MB and the generation budget starts at 2 GiB -- and the same loop is
1440
+ what makes a verification pass interruptible within N-22's five seconds.
1441
+ """
1442
+ digest = hasher if hasher is not None else hashlib.sha256()
1443
+ read = 0
1444
+ try:
1445
+ with path.open("rb") as handle:
1446
+ while True:
1447
+ if _cancelled(cancel):
1448
+ return None, read
1449
+ chunk = handle.read(_CHUNK_BYTES)
1450
+ if not chunk:
1451
+ break
1452
+ digest.update(chunk)
1453
+ read += len(chunk)
1454
+ except OSError as exc:
1455
+ raise EchoActError(
1456
+ Code.MODEL_CORRUPT,
1457
+ "A model file could not be read.",
1458
+ detail={"path": redact(path)},
1459
+ cause=exc,
1460
+ ) from exc
1461
+ return digest.hexdigest(), read
1462
+
1463
+
1464
+ def _unchecked(file: ModelFile) -> FileStatus:
1465
+ return FileStatus(
1466
+ relative_path=file.relative_path,
1467
+ expected_bytes=file.byte_size,
1468
+ actual_bytes=0,
1469
+ present=False,
1470
+ size_ok=False,
1471
+ digest_ok=None,
1472
+ checked=False,
1473
+ )
1474
+
1475
+
1476
+ def _is_ok(report: VerifyReport, relative_path: str) -> bool:
1477
+ status = report.status(relative_path)
1478
+ return status is not None and status.ok
1479
+
1480
+
1481
+ def _tree_bytes(root: Path) -> int:
1482
+ total = 0
1483
+ if not root.exists():
1484
+ return 0
1485
+ for path in _walk(root):
1486
+ try:
1487
+ total += path.stat().st_size
1488
+ except OSError:
1489
+ continue
1490
+ return total
1491
+
1492
+
1493
+ def _walk(root: Path) -> Iterable[Path]:
1494
+ for dirpath, _dirnames, filenames in os.walk(root):
1495
+ for name in filenames:
1496
+ yield Path(dirpath) / name
1497
+
1498
+
1499
+ def _unlink(path: Path) -> None:
1500
+ try:
1501
+ path.unlink()
1502
+ except OSError:
1503
+ pass
1504
+
1505
+
1506
+ def _replace(source: Path, target: Path) -> None:
1507
+ try:
1508
+ os.replace(source, target)
1509
+ except OSError as exc:
1510
+ raise EchoActError(
1511
+ Code.MODEL_DOWNLOAD_FAILED,
1512
+ "A downloaded model file could not be moved into place.",
1513
+ detail={"path": redact(target)},
1514
+ retry_after_s=_DOWNLOAD_RETRY_AFTER_S,
1515
+ cause=exc,
1516
+ ) from exc
1517
+
1518
+
1519
+ def _fingerprint(terms: LicenseTerms) -> str:
1520
+ """Identify the exact terms accepted, so changed terms are re-asked."""
1521
+ digest = hashlib.sha256()
1522
+ digest.update(terms.name.encode("utf-8"))
1523
+ digest.update(terms.pass_through_obligation.encode("utf-8"))
1524
+ for restriction in terms.restrictions:
1525
+ digest.update(b"\x00")
1526
+ digest.update(restriction.encode("utf-8"))
1527
+ return digest.hexdigest()[:32]
1528
+
1529
+
1530
+ def _gib(value: int) -> str:
1531
+ """4.1: memory is shown in GiB."""
1532
+ return f"{value / GIB:.1f} GiB"
1533
+
1534
+
1535
+ __all__ = [
1536
+ "CancelToken",
1537
+ "DownloadOutcome",
1538
+ "DownloadPhase",
1539
+ "DownloadProgress",
1540
+ "Fetcher",
1541
+ "FileStatus",
1542
+ "HttpxFetcher",
1543
+ "JsonModelPreferences",
1544
+ "ModelPreferences",
1545
+ "ModelRegistry",
1546
+ "ModelState",
1547
+ "ModelStatus",
1548
+ "RemoteBody",
1549
+ "VerifyReport",
1550
+ "resolve_url",
1551
+ ]