dinkster-api 0.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (238) hide show
  1. dinkster_api/__init__.py +9 -0
  2. dinkster_api/py.typed +0 -0
  3. dinkster_api/v1.py +935 -0
  4. dinkster_api-0.0.2.dist-info/METADATA +16 -0
  5. dinkster_api-0.0.2.dist-info/RECORD +238 -0
  6. dinkster_api-0.0.2.dist-info/WHEEL +4 -0
  7. dinkster_api-0.0.2.dist-info/entry_points.txt +2 -0
  8. dinkster_assets/__init__.py +337 -0
  9. dinkster_assets/acquire.py +226 -0
  10. dinkster_assets/acquisition_plan.py +145 -0
  11. dinkster_assets/audio.py +119 -0
  12. dinkster_assets/catalog.py +156 -0
  13. dinkster_assets/component_manifest.py +110 -0
  14. dinkster_assets/declared.py +218 -0
  15. dinkster_assets/fetch.py +495 -0
  16. dinkster_assets/guess.py +136 -0
  17. dinkster_assets/identity.py +72 -0
  18. dinkster_assets/integrity.py +310 -0
  19. dinkster_assets/json_metadata.py +41 -0
  20. dinkster_assets/kind.py +52 -0
  21. dinkster_assets/latent_format.py +346 -0
  22. dinkster_assets/library.py +816 -0
  23. dinkster_assets/media.py +1385 -0
  24. dinkster_assets/migrations/env.py +38 -0
  25. dinkster_assets/migrations/script.py.mako +28 -0
  26. dinkster_assets/migrations/versions/0001_asset_records_create_asset_record_catalog.py +109 -0
  27. dinkster_assets/migrations/versions/0002_asset_observations.py +36 -0
  28. dinkster_assets/model.py +242 -0
  29. dinkster_assets/mounts.py +1278 -0
  30. dinkster_assets/need.py +168 -0
  31. dinkster_assets/pack_assets.py +217 -0
  32. dinkster_assets/probe.py +118 -0
  33. dinkster_assets/provenance.py +254 -0
  34. dinkster_assets/public_acquisition.py +424 -0
  35. dinkster_assets/py.typed +0 -0
  36. dinkster_assets/records.py +899 -0
  37. dinkster_assets/resolution.py +778 -0
  38. dinkster_assets/resolver_index.py +364 -0
  39. dinkster_assets/resolver_subscription.py +1149 -0
  40. dinkster_assets/save_target.py +139 -0
  41. dinkster_assets/schema.py +73 -0
  42. dinkster_assets/selections.py +256 -0
  43. dinkster_assets/transport.py +243 -0
  44. dinkster_assets/value.py +258 -0
  45. dinkster_assets/vault.py +276 -0
  46. dinkster_assets/writer.py +765 -0
  47. dinkster_caches/__init__.py +19 -0
  48. dinkster_caches/cas.py +310 -0
  49. dinkster_caches/disk.py +330 -0
  50. dinkster_caches/layered.py +150 -0
  51. dinkster_caches/memory.py +208 -0
  52. dinkster_caches/py.typed +0 -0
  53. dinkster_caches/wire.py +346 -0
  54. dinkster_image_document/__init__.py +39 -0
  55. dinkster_image_document/blend.py +264 -0
  56. dinkster_image_document/compat.py +211 -0
  57. dinkster_image_document/document.py +374 -0
  58. dinkster_image_document/format.py +569 -0
  59. dinkster_image_document/interchange.py +572 -0
  60. dinkster_image_document/py.typed +1 -0
  61. dinkster_image_document/render.py +455 -0
  62. dinkster_inference_wire/__init__.py +141 -0
  63. dinkster_inference_wire/camera_wire.py +127 -0
  64. dinkster_inference_wire/conditioning.py +752 -0
  65. dinkster_inference_wire/conditioning_wire.py +1143 -0
  66. dinkster_inference_wire/devices.py +285 -0
  67. dinkster_inference_wire/generation.py +446 -0
  68. dinkster_inference_wire/hook_values.py +168 -0
  69. dinkster_inference_wire/ic_lora_parameters.py +69 -0
  70. dinkster_inference_wire/latents.py +244 -0
  71. dinkster_inference_wire/media_wire.py +169 -0
  72. dinkster_inference_wire/openai_generation.py +1035 -0
  73. dinkster_inference_wire/pose_keypoint.py +83 -0
  74. dinkster_inference_wire/process.py +33 -0
  75. dinkster_inference_wire/py.typed +0 -0
  76. dinkster_inference_wire/registry.py +116 -0
  77. dinkster_inference_wire/sam3_wire.py +68 -0
  78. dinkster_inference_wire/sampling_wire.py +279 -0
  79. dinkster_inference_wire/sources.py +399 -0
  80. dinkster_inference_wire/spaces.py +344 -0
  81. dinkster_inference_wire/sparse.py +372 -0
  82. dinkster_inference_wire/splat_wire.py +23 -0
  83. dinkster_inference_wire/types.py +256 -0
  84. dinkster_inference_wire/vision_wire.py +53 -0
  85. dinkster_inference_wire/voxel_wire.py +72 -0
  86. dinkster_inference_wire/wan_wire.py +93 -0
  87. dinkster_inference_wire/weights.py +219 -0
  88. dinkster_memory/__init__.py +92 -0
  89. dinkster_memory/accelerator.py +96 -0
  90. dinkster_memory/budgets.py +96 -0
  91. dinkster_memory/details.py +91 -0
  92. dinkster_memory/governor.py +554 -0
  93. dinkster_memory/leases.py +176 -0
  94. dinkster_memory/py.typed +0 -0
  95. dinkster_memory/release.py +131 -0
  96. dinkster_memory/reported.py +101 -0
  97. dinkster_memory/reservations.py +101 -0
  98. dinkster_memory/resident.py +600 -0
  99. dinkster_memory/system.py +614 -0
  100. dinkster_memory/tenants.py +67 -0
  101. dinkster_protocol/__init__.py +1008 -0
  102. dinkster_protocol/attention.py +652 -0
  103. dinkster_protocol/extension_contribution_kinds.json +21 -0
  104. dinkster_protocol/extension_contribution_kinds.py +43 -0
  105. dinkster_protocol/extensions.py +742 -0
  106. dinkster_protocol/frontend_modules.py +145 -0
  107. dinkster_protocol/multigpu.py +107 -0
  108. dinkster_protocol/pack_surfaces.py +489 -0
  109. dinkster_protocol/payload.py +63 -0
  110. dinkster_protocol/preview.py +86 -0
  111. dinkster_protocol/py.typed +0 -0
  112. dinkster_protocol/result_algebra.py +380 -0
  113. dinkster_protocol/result_document.py +244 -0
  114. dinkster_protocol/run_control.py +96 -0
  115. dinkster_protocol/training.py +399 -0
  116. dinkster_protocol/workgroup.py +1014 -0
  117. dinkster_registry/__init__.py +135 -0
  118. dinkster_registry/admission.py +372 -0
  119. dinkster_registry/artifact.py +128 -0
  120. dinkster_registry/composition.py +152 -0
  121. dinkster_registry/grants.py +206 -0
  122. dinkster_registry/install.py +1017 -0
  123. dinkster_registry/model.py +414 -0
  124. dinkster_registry/principals.py +494 -0
  125. dinkster_registry/py.typed +0 -0
  126. dinkster_schema/__init__.py +347 -0
  127. dinkster_schema/capture.py +240 -0
  128. dinkster_schema/coerce.py +186 -0
  129. dinkster_schema/comfy_alias.py +496 -0
  130. dinkster_schema/comfy_group.py +768 -0
  131. dinkster_schema/elaborate.py +676 -0
  132. dinkster_schema/execution.py +24 -0
  133. dinkster_schema/expansion.py +123 -0
  134. dinkster_schema/log.py +206 -0
  135. dinkster_schema/media.py +77 -0
  136. dinkster_schema/model.py +2920 -0
  137. dinkster_schema/names.py +25 -0
  138. dinkster_schema/node.py +137 -0
  139. dinkster_schema/output_descriptors.py +63 -0
  140. dinkster_schema/py.typed +0 -0
  141. dinkster_schema/replace.py +1853 -0
  142. dinkster_schema/reporting.py +441 -0
  143. dinkster_schema/solve.py +178 -0
  144. dinkster_schema/wire.py +1876 -0
  145. dinkster_values/__init__.py +625 -0
  146. dinkster_values/_media_array.py +7 -0
  147. dinkster_values/absent.py +77 -0
  148. dinkster_values/asset_folders.py +64 -0
  149. dinkster_values/assets.py +106 -0
  150. dinkster_values/audio_codec.py +594 -0
  151. dinkster_values/audio_lazy.py +871 -0
  152. dinkster_values/builtin_types.py +181 -0
  153. dinkster_values/curve.py +260 -0
  154. dinkster_values/detection_codec.py +286 -0
  155. dinkster_values/image_codec.py +691 -0
  156. dinkster_values/image_components.py +219 -0
  157. dinkster_values/latent_codec.py +514 -0
  158. dinkster_values/limits.py +52 -0
  159. dinkster_values/lists.py +216 -0
  160. dinkster_values/media_containers.py +70 -0
  161. dinkster_values/model.py +417 -0
  162. dinkster_values/model3d_codec.py +167 -0
  163. dinkster_values/names.py +168 -0
  164. dinkster_values/pins.py +77 -0
  165. dinkster_values/py.typed +0 -0
  166. dinkster_values/registry.py +1105 -0
  167. dinkster_values/resident.py +210 -0
  168. dinkster_values/resources.py +210 -0
  169. dinkster_values/splat_codec.py +477 -0
  170. dinkster_values/storage.py +133 -0
  171. dinkster_values/streams.py +241 -0
  172. dinkster_values/svg_codec.py +120 -0
  173. dinkster_values/timeline_video.py +68 -0
  174. dinkster_values/tuples.py +61 -0
  175. dinkster_values/video_codec.py +897 -0
  176. dinkster_values/video_document.py +315 -0
  177. dinkster_values/video_edits.py +253 -0
  178. dinkster_values/video_probe.py +221 -0
  179. dinkster_video/__init__.py +58 -0
  180. dinkster_video/document.py +352 -0
  181. dinkster_video/formats.py +722 -0
  182. dinkster_video/image_math.py +175 -0
  183. dinkster_video/preview/dinkster-pack.toml +36 -0
  184. dinkster_video/preview/src/dinkster_video_preview/__init__.py +84 -0
  185. dinkster_video/preview/src/dinkster_video_preview/preview.js +45 -0
  186. dinkster_video/py.typed +1 -0
  187. dinkster_video/runtime.py +1561 -0
  188. dinkster_video/timeline.py +576 -0
  189. dinkster_video/timeline_runtime.py +305 -0
  190. dinkster_video_preview/__init__.py +84 -0
  191. dinkster_video_preview/preview.js +45 -0
  192. dinkster_workers/__init__.py +207 -0
  193. dinkster_workers/_doctor_probe.py +359 -0
  194. dinkster_workers/accelerator.py +156 -0
  195. dinkster_workers/aimdo_bootstrap.py +81 -0
  196. dinkster_workers/backend_env.py +1485 -0
  197. dinkster_workers/blobs.py +402 -0
  198. dinkster_workers/boundary.py +1918 -0
  199. dinkster_workers/catalog.py +182 -0
  200. dinkster_workers/devices.py +164 -0
  201. dinkster_workers/diagnostics.py +64 -0
  202. dinkster_workers/dispatch.py +298 -0
  203. dinkster_workers/doctor.py +1382 -0
  204. dinkster_workers/egress.py +344 -0
  205. dinkster_workers/error_hints.py +175 -0
  206. dinkster_workers/execution.py +160 -0
  207. dinkster_workers/governed.py +123 -0
  208. dinkster_workers/headroom.py +156 -0
  209. dinkster_workers/host.py +3116 -0
  210. dinkster_workers/in_process.py +1325 -0
  211. dinkster_workers/interpreter.py +84 -0
  212. dinkster_workers/isolated.py +862 -0
  213. dinkster_workers/launch.py +72 -0
  214. dinkster_workers/manifest.py +3332 -0
  215. dinkster_workers/media.py +196 -0
  216. dinkster_workers/pack_environment.py +473 -0
  217. dinkster_workers/pack_lock.py +359 -0
  218. dinkster_workers/placement.py +215 -0
  219. dinkster_workers/preview_artifacts.py +233 -0
  220. dinkster_workers/produced_assets.py +218 -0
  221. dinkster_workers/provision.py +597 -0
  222. dinkster_workers/py.typed +0 -0
  223. dinkster_workers/relay.py +762 -0
  224. dinkster_workers/remote.py +921 -0
  225. dinkster_workers/resume.py +490 -0
  226. dinkster_workers/routing.py +134 -0
  227. dinkster_workers/run_control.py +275 -0
  228. dinkster_workers/sandbox.py +798 -0
  229. dinkster_workers/saved_artifacts.py +275 -0
  230. dinkster_workers/service.py +593 -0
  231. dinkster_workers/session.py +2605 -0
  232. dinkster_workers/single_job_multi_gpu.py +7 -0
  233. dinkster_workers/staging.py +230 -0
  234. dinkster_workers/transport.py +161 -0
  235. dinkster_workers/workgroup.py +729 -0
  236. dinkster_workers/workgroup_compose.py +135 -0
  237. dinkster_workers/workgroup_config.py +196 -0
  238. dinkster_workers/workgroup_session.py +340 -0
@@ -0,0 +1,72 @@
1
+ """Asset identity: a canonical content digest (DESIGN 3.12).
2
+
3
+ Identity is a property of bytes - it survives renames, moves, duplicate
4
+ copies, and machine hops. The canonical form is ``blake3:<64 hex>``,
5
+ deliberately the same form ComfyUI's asset database uses, so digests
6
+ interoperate with hashes the existing ecosystem already computed.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import re
12
+ from pathlib import Path
13
+ from typing import TYPE_CHECKING
14
+
15
+ from dinkster_values import MEBIBYTE
16
+
17
+ if TYPE_CHECKING:
18
+ from blake3 import blake3
19
+
20
+ DIGEST_PREFIX = "blake3:"
21
+ _DIGEST_RE = re.compile(r"^blake3:[0-9a-f]{64}$")
22
+
23
+ CHUNK_SIZE = 8 * MEBIBYTE
24
+ """Hash files in 8 MiB chunks (matches ComfyUI's scanner) - bounded memory
25
+ regardless of file size."""
26
+
27
+
28
+ class AssetError(Exception):
29
+ """An asset could not be identified, cataloged, or materialized."""
30
+
31
+
32
+ def is_digest(text: str) -> bool:
33
+ return _DIGEST_RE.match(text) is not None
34
+
35
+
36
+ def require_digest(text: str) -> str:
37
+ if not is_digest(text):
38
+ raise AssetError(f"not a canonical asset digest (expected 'blake3:<64 hex>'): {text!r}")
39
+ return text
40
+
41
+
42
+ def _blake3() -> type[blake3]:
43
+ """Lazy: hashing happens where Dinkster proper runs (blake3 installed);
44
+ workers that only *resolve* digests - compat children riding a foreign
45
+ interpreter via PYTHONPATH - never need it."""
46
+ try:
47
+ from blake3 import blake3 as hasher
48
+ except ImportError: # pragma: no cover - environment-specific
49
+ raise AssetError(
50
+ "computing asset digests requires the 'blake3' package, which "
51
+ "is not installed in this interpreter; compute digests where "
52
+ "Dinkster is installed (workers resolve pre-indexed digests only)"
53
+ ) from None
54
+ return hasher
55
+
56
+
57
+ def new_hasher() -> blake3:
58
+ """A fresh incremental hasher for streaming digest computation (vault
59
+ ingest, resumable scans). Same lazy-import posture as digest_file."""
60
+ return _blake3()()
61
+
62
+
63
+ def digest_bytes(data: bytes) -> str:
64
+ return DIGEST_PREFIX + _blake3()(data).hexdigest()
65
+
66
+
67
+ def digest_file(path: Path) -> str:
68
+ hasher = _blake3()()
69
+ with path.open("rb") as handle:
70
+ while chunk := handle.read(CHUNK_SIZE):
71
+ hasher.update(chunk)
72
+ return DIGEST_PREFIX + hasher.hexdigest()
@@ -0,0 +1,310 @@
1
+ """Descriptor-bound verification for content-addressed asset reads.
2
+
3
+ Ingest records bind a digest to stable descriptor metadata. Loaders can trust
4
+ that record without rereading the payload when the opened descriptor still
5
+ matches it. Missing records retain full descriptor-bound hashing, and stale
6
+ records fail closed.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import os
12
+ import threading
13
+ from collections import OrderedDict
14
+ from collections.abc import Mapping
15
+ from dataclasses import dataclass
16
+ from pathlib import Path
17
+ from typing import BinaryIO, Literal, cast
18
+
19
+ from .identity import CHUNK_SIZE, DIGEST_PREFIX, AssetError, new_hasher
20
+
21
+ IntegrityFailure = Literal[
22
+ "digest_mismatch",
23
+ "changed_during_verification",
24
+ "path_rebound",
25
+ "stale_ingest_record",
26
+ ]
27
+
28
+
29
+ class AssetIntegrityError(AssetError):
30
+ """The bytes at a resolved path do not (or can no longer be proven to)
31
+ match the digest that named them. Carries the expected digest, the
32
+ actual digest when a hash completed (None when the file changed under
33
+ verification or the path stopped naming the verified inode), and the
34
+ offending path - for logs; public surfaces should not echo the path."""
35
+
36
+ def __init__(
37
+ self,
38
+ expected_digest: str,
39
+ actual_digest: str | None,
40
+ path: Path,
41
+ reason: IntegrityFailure,
42
+ ) -> None:
43
+ self.expected_digest = expected_digest
44
+ self.actual_digest = actual_digest
45
+ self.path = path
46
+ self.reason = reason
47
+ detail = {
48
+ "digest_mismatch": f"content hashes to {actual_digest}",
49
+ "changed_during_verification": "the file changed while being hashed",
50
+ "path_rebound": "the path stopped naming the verified file",
51
+ "stale_ingest_record": "the file metadata differs from its ingest record",
52
+ }[reason]
53
+ super().__init__(
54
+ f"asset integrity failure ({reason}): {path} does not hold the "
55
+ f"bytes named by {expected_digest} - {detail}"
56
+ )
57
+
58
+
59
+ _Fingerprint = tuple[int, int, int, int, int]
60
+
61
+
62
+ @dataclass(frozen=True)
63
+ class AssetVerificationRecord:
64
+ """Descriptor metadata captured by a content-verified ingest."""
65
+
66
+ digest: str
67
+ scheme: str
68
+ device: int
69
+ inode: int
70
+ size: int
71
+ mtime_ns: int
72
+ ctime_ns: int | None
73
+
74
+ def matches(self, stat: os.stat_result) -> bool:
75
+ if self.scheme != _stat_scheme():
76
+ return False
77
+ actual = (stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns)
78
+ expected = (self.device, self.inode, self.size, self.mtime_ns)
79
+ return actual == expected and (self.ctime_ns is None or stat.st_ctime_ns == self.ctime_ns)
80
+
81
+ def to_json(self) -> dict[str, object]:
82
+ return {
83
+ "digest": self.digest,
84
+ "scheme": self.scheme,
85
+ "device": self.device,
86
+ "inode": self.inode,
87
+ "size": self.size,
88
+ "mtimeNs": self.mtime_ns,
89
+ **({"ctimeNs": self.ctime_ns} if self.ctime_ns is not None else {}),
90
+ }
91
+
92
+ @classmethod
93
+ def from_json(cls, digest: str, value: object) -> AssetVerificationRecord | None:
94
+ if not isinstance(value, Mapping):
95
+ return None
96
+ row = cast("Mapping[str, object]", value)
97
+ recorded_digest = row.get("digest")
98
+ scheme = row.get("scheme")
99
+ device = row.get("device")
100
+ inode = row.get("inode")
101
+ size = row.get("size")
102
+ mtime_ns = row.get("mtimeNs")
103
+ ctime_ns = row.get("ctimeNs")
104
+ if (
105
+ recorded_digest != digest
106
+ or scheme not in ("posix-v1", "windows-v1")
107
+ or not all(isinstance(field, int) for field in (device, inode, size, mtime_ns))
108
+ or (ctime_ns is not None and not isinstance(ctime_ns, int))
109
+ ):
110
+ return None
111
+ if scheme == "posix-v1" and not isinstance(ctime_ns, int):
112
+ return None
113
+ return cls(
114
+ digest,
115
+ cast("str", scheme),
116
+ cast("int", device),
117
+ cast("int", inode),
118
+ cast("int", size),
119
+ cast("int", mtime_ns),
120
+ ctime_ns,
121
+ )
122
+
123
+
124
+ def _stat_scheme() -> str | None:
125
+ if os.name == "posix":
126
+ return "posix-v1"
127
+ if os.name == "nt":
128
+ return "windows-v1"
129
+ return None
130
+
131
+
132
+ def verification_record(digest: str, stat: os.stat_result) -> AssetVerificationRecord | None:
133
+ """Create a load-time proof only where device/inode identity is usable."""
134
+ scheme = _stat_scheme()
135
+ if scheme is None or stat.st_dev == 0 or stat.st_ino == 0:
136
+ return None
137
+ return AssetVerificationRecord(
138
+ digest,
139
+ scheme,
140
+ stat.st_dev,
141
+ stat.st_ino,
142
+ stat.st_size,
143
+ stat.st_mtime_ns,
144
+ stat.st_ctime_ns if scheme == "posix-v1" else None,
145
+ )
146
+
147
+
148
+ def verification_record_for_publication(
149
+ digest: str,
150
+ verified: os.stat_result,
151
+ published: os.stat_result,
152
+ ) -> AssetVerificationRecord | None:
153
+ """Bind verified temporary bytes to their atomically published name."""
154
+ verified_identity = (verified.st_dev, verified.st_ino)
155
+ published_identity = (published.st_dev, published.st_ino)
156
+ identities_stable = all(verified_identity) and all(published_identity)
157
+ if identities_stable and verified_identity != published_identity:
158
+ raise AssetError("asset path was rebound while it was being published")
159
+ if (verified.st_size, verified.st_mtime_ns) != (published.st_size, published.st_mtime_ns):
160
+ raise AssetError("asset changed while it was being published")
161
+ if not identities_stable:
162
+ return None
163
+ return verification_record(digest, published)
164
+
165
+
166
+ _CACHE_LIMIT = 4096
167
+ """Verified fingerprints retained per process. Bounds memory and shrinks
168
+ the window in which a recycled inode number could meet a matching stale
169
+ entry; real libraries hold thousands of assets, not millions."""
170
+
171
+ _TRUST_FINGERPRINTS = os.name == "posix"
172
+ """Whether a matching fingerprint may skip the rehash. Requires st_ctime_ns
173
+ to be a change stamp (POSIX); on Windows it is creation time, which an
174
+ in-place rewrite with a restored mtime would not move."""
175
+
176
+ _verified: OrderedDict[_Fingerprint, str] = OrderedDict()
177
+ _verified_lock = threading.Lock()
178
+
179
+
180
+ def _reset_after_fork() -> None:
181
+ # A forked child inherits the dict but not the right to trust it being
182
+ # in sync with whatever the parent does next - and possibly a lock a
183
+ # now-nonexistent parent thread held. Fresh lock, empty cache.
184
+ global _verified_lock
185
+ _verified_lock = threading.Lock()
186
+ _verified.clear()
187
+
188
+
189
+ register_at_fork = getattr(os, "register_at_fork", None)
190
+ if register_at_fork is not None: # pragma: no branch - POSIX
191
+ register_at_fork(after_in_child=_reset_after_fork)
192
+
193
+
194
+ def clear_verified_cache() -> None:
195
+ """Forget every recorded verification (tests; long-lived hosts that
196
+ want to force a re-hash)."""
197
+ with _verified_lock:
198
+ _verified.clear()
199
+
200
+
201
+ def _fingerprint(stat: os.stat_result) -> _Fingerprint:
202
+ return (stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns)
203
+
204
+
205
+ def _path_binding_fingerprint(fingerprint: _Fingerprint) -> tuple[int, ...]:
206
+ # Windows creation time can differ between stat() and fstat() for the same file.
207
+ return fingerprint[:4] if os.name == "nt" else fingerprint
208
+
209
+
210
+ def _hash_handle(handle: BinaryIO) -> str:
211
+ """Hash the handle's content from position 0. A seam on purpose:
212
+ integrity tests monkeypatch this to count hashes and to interleave
213
+ replacements mid-verification."""
214
+ handle.seek(0)
215
+ hasher = new_hasher()
216
+ while chunk := handle.read(CHUNK_SIZE):
217
+ hasher.update(chunk)
218
+ return DIGEST_PREFIX + hasher.hexdigest()
219
+
220
+
221
+ def digest_file_with_record(
222
+ path: Path,
223
+ ) -> tuple[str, AssetVerificationRecord | None]:
224
+ """Hash one ingest candidate and bind its digest to the same descriptor."""
225
+ with path.open("rb") as handle:
226
+ before = _fingerprint(os.fstat(handle.fileno()))
227
+ digest = _hash_handle(handle)
228
+ after_stat = os.fstat(handle.fileno())
229
+ if _fingerprint(after_stat) != before:
230
+ raise AssetError(f"asset changed while being ingested: {path}")
231
+ return digest, verification_record(digest, after_stat)
232
+
233
+
234
+ def open_verified(
235
+ path: Path,
236
+ expected_digest: str,
237
+ record: AssetVerificationRecord | None = None,
238
+ ) -> BinaryIO:
239
+ """Open ``path`` and return a handle positioned at 0 whose bytes are
240
+ proven to hash to ``expected_digest``.
241
+
242
+ The proof and the returned handle share one file descriptor, so a
243
+ rename/replace after (or during) verification cannot swap the bytes:
244
+ the descriptor keeps the verified inode. Raises AssetIntegrityError
245
+ on any mismatch; raises AssetError when hashing is unavailable in
246
+ this interpreter (missing blake3) - never a silent downgrade."""
247
+ handle = path.open("rb")
248
+ try:
249
+ opened_stat = os.fstat(handle.fileno())
250
+ before = _fingerprint(opened_stat)
251
+ if record is not None:
252
+ if record.digest != expected_digest or not record.matches(opened_stat):
253
+ raise AssetIntegrityError(expected_digest, None, path, "stale_ingest_record")
254
+ return handle
255
+ actual: str | None = None
256
+ if _TRUST_FINGERPRINTS:
257
+ with _verified_lock:
258
+ actual = _verified.get(before)
259
+ if actual is not None:
260
+ _verified.move_to_end(before)
261
+ if actual is None:
262
+ actual = _hash_handle(handle)
263
+ after = _fingerprint(os.fstat(handle.fileno()))
264
+ if after != before:
265
+ raise AssetIntegrityError(
266
+ expected_digest, None, path, "changed_during_verification"
267
+ )
268
+ if _TRUST_FINGERPRINTS:
269
+ # Cache what the content IS (its proven hash), not what the
270
+ # caller hoped for: a later open with the honest digest may
271
+ # hit; one with a different digest still compares and fails.
272
+ with _verified_lock:
273
+ _verified[after] = actual
274
+ _verified.move_to_end(after)
275
+ while len(_verified) > _CACHE_LIMIT:
276
+ _verified.popitem(last=False)
277
+ if actual != expected_digest:
278
+ raise AssetIntegrityError(expected_digest, actual, path, "digest_mismatch")
279
+ handle.seek(0)
280
+ return handle
281
+ except BaseException:
282
+ handle.close()
283
+ raise
284
+
285
+
286
+ def verified_local_path(
287
+ path: Path,
288
+ expected_digest: str,
289
+ record: AssetVerificationRecord | None = None,
290
+ ) -> Path:
291
+ """Verify ``path``'s content against ``expected_digest`` and return the
292
+ path, for consumers that need a real filesystem name (checkpoint
293
+ loaders). Weaker than :func:`open_verified` by nature: after the final
294
+ check the name can be rebound, so the guarantee is "the path named the
295
+ verified bytes at return time", with the window shrunk from unbounded
296
+ to the instant of return. Prefer open_verified where a handle works."""
297
+ verified = (
298
+ open_verified(path, expected_digest)
299
+ if record is None
300
+ else open_verified(path, expected_digest, record)
301
+ )
302
+ with verified as handle:
303
+ fd_print = _fingerprint(os.fstat(handle.fileno()))
304
+ try:
305
+ path_print = _fingerprint(os.stat(path))
306
+ except OSError:
307
+ raise AssetIntegrityError(expected_digest, None, path, "path_rebound") from None
308
+ if _path_binding_fingerprint(path_print) != _path_binding_fingerprint(fd_print):
309
+ raise AssetIntegrityError(expected_digest, None, path, "path_rebound")
310
+ return path
@@ -0,0 +1,41 @@
1
+ """Immutable validated JSON metadata values."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from collections.abc import Mapping, Sequence
7
+ from types import MappingProxyType
8
+ from typing import cast
9
+
10
+ from .identity import AssetError
11
+
12
+
13
+ def freeze_json(value: object, path: str = "metadata") -> object:
14
+ if value is None or isinstance(value, (str, bool, int)):
15
+ return value
16
+ if isinstance(value, float):
17
+ if not math.isfinite(value):
18
+ raise AssetError(f"{path} must contain finite JSON numbers")
19
+ return value
20
+ if isinstance(value, Mapping):
21
+ frozen: dict[str, object] = {}
22
+ for key, item in cast("Mapping[object, object]", value).items():
23
+ if not isinstance(key, str):
24
+ raise AssetError(f"{path} keys must be strings")
25
+ frozen[key] = freeze_json(item, f"{path}.{key}")
26
+ return MappingProxyType(dict(sorted(frozen.items())))
27
+ if isinstance(value, (list, tuple)):
28
+ sequence = cast("Sequence[object]", value)
29
+ return tuple(freeze_json(item, f"{path}[]") for item in sequence)
30
+ raise AssetError(f"{path} must contain only JSON values")
31
+
32
+
33
+ def thaw_json(value: object) -> object:
34
+ if isinstance(value, Mapping):
35
+ return {
36
+ str(key): thaw_json(item)
37
+ for key, item in cast("Mapping[object, object]", value).items()
38
+ }
39
+ if isinstance(value, tuple):
40
+ return [thaw_json(item) for item in cast("tuple[object, ...]", value)]
41
+ return value
@@ -0,0 +1,52 @@
1
+ """Asset kinds: an open, namespaced vocabulary (DESIGN 3.12 + roadmap
2
+ "templates/asset distribution").
3
+
4
+ A kind says what an asset *is for* - the axis pickers filter on and
5
+ substitution UX groups by - without repeating ComfyUI's folder taxonomy
6
+ (where "kind" was literally which directory a file sat in, so a new model
7
+ family meant patching folder_paths). Kinds are plain namespaced strings,
8
+ ``namespace/name`` with optional deeper segments: ``media/image``,
9
+ ``model/lora``, ``model/diffusion``. The vocabulary is OPEN - packs and
10
+ templates may mint kinds freely - while the grammar is closed, so kinds
11
+ sort, compare, and prefix-filter mechanically forever.
12
+
13
+ Kind is presentation/index metadata, never execution identity: it joins
14
+ widgets, needs, and catalogs, and it never joins schema signatures or
15
+ cache keys.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import re
21
+
22
+ from .identity import AssetError
23
+
24
+ _KIND_RE = re.compile(r"^[a-z0-9][a-z0-9-]*(/[a-z0-9][a-z0-9-]*)+$")
25
+
26
+ # Conventional core kinds. A convention, not an enum: validation accepts
27
+ # any grammatical kind, and unknown kinds are first-class everywhere.
28
+ KIND_MEDIA_IMAGE = "media/image"
29
+ KIND_MEDIA_VIDEO = "media/video"
30
+ KIND_MEDIA_AUDIO = "media/audio"
31
+ KIND_MEDIA_MODEL3D = "media/model3d"
32
+ KIND_MODEL_CHECKPOINT = "model/checkpoint"
33
+ KIND_MODEL_DIFFUSION = "model/diffusion"
34
+ KIND_MODEL_LORA = "model/lora"
35
+ KIND_MODEL_VAE = "model/vae"
36
+ KIND_MODEL_TEXT_ENCODER = "model/text-encoder"
37
+ KIND_MODEL_EMBEDDING = "model/embedding"
38
+
39
+
40
+ def is_asset_kind(text: str) -> bool:
41
+ return _KIND_RE.match(text) is not None
42
+
43
+
44
+ def require_asset_kind(text: str) -> str:
45
+ """Validate a kind string: lowercase alphanumeric/hyphen segments,
46
+ '/'-separated, at least two segments (a bare word is not namespaced)."""
47
+ if not is_asset_kind(text):
48
+ raise AssetError(
49
+ "not a valid asset kind (expected 'namespace/name' with "
50
+ f"lowercase [a-z0-9-] segments): {text!r}"
51
+ )
52
+ return text