dinkster-api 0.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (238) hide show
  1. dinkster_api/__init__.py +9 -0
  2. dinkster_api/py.typed +0 -0
  3. dinkster_api/v1.py +935 -0
  4. dinkster_api-0.0.2.dist-info/METADATA +16 -0
  5. dinkster_api-0.0.2.dist-info/RECORD +238 -0
  6. dinkster_api-0.0.2.dist-info/WHEEL +4 -0
  7. dinkster_api-0.0.2.dist-info/entry_points.txt +2 -0
  8. dinkster_assets/__init__.py +337 -0
  9. dinkster_assets/acquire.py +226 -0
  10. dinkster_assets/acquisition_plan.py +145 -0
  11. dinkster_assets/audio.py +119 -0
  12. dinkster_assets/catalog.py +156 -0
  13. dinkster_assets/component_manifest.py +110 -0
  14. dinkster_assets/declared.py +218 -0
  15. dinkster_assets/fetch.py +495 -0
  16. dinkster_assets/guess.py +136 -0
  17. dinkster_assets/identity.py +72 -0
  18. dinkster_assets/integrity.py +310 -0
  19. dinkster_assets/json_metadata.py +41 -0
  20. dinkster_assets/kind.py +52 -0
  21. dinkster_assets/latent_format.py +346 -0
  22. dinkster_assets/library.py +816 -0
  23. dinkster_assets/media.py +1385 -0
  24. dinkster_assets/migrations/env.py +38 -0
  25. dinkster_assets/migrations/script.py.mako +28 -0
  26. dinkster_assets/migrations/versions/0001_asset_records_create_asset_record_catalog.py +109 -0
  27. dinkster_assets/migrations/versions/0002_asset_observations.py +36 -0
  28. dinkster_assets/model.py +242 -0
  29. dinkster_assets/mounts.py +1278 -0
  30. dinkster_assets/need.py +168 -0
  31. dinkster_assets/pack_assets.py +217 -0
  32. dinkster_assets/probe.py +118 -0
  33. dinkster_assets/provenance.py +254 -0
  34. dinkster_assets/public_acquisition.py +424 -0
  35. dinkster_assets/py.typed +0 -0
  36. dinkster_assets/records.py +899 -0
  37. dinkster_assets/resolution.py +778 -0
  38. dinkster_assets/resolver_index.py +364 -0
  39. dinkster_assets/resolver_subscription.py +1149 -0
  40. dinkster_assets/save_target.py +139 -0
  41. dinkster_assets/schema.py +73 -0
  42. dinkster_assets/selections.py +256 -0
  43. dinkster_assets/transport.py +243 -0
  44. dinkster_assets/value.py +258 -0
  45. dinkster_assets/vault.py +276 -0
  46. dinkster_assets/writer.py +765 -0
  47. dinkster_caches/__init__.py +19 -0
  48. dinkster_caches/cas.py +310 -0
  49. dinkster_caches/disk.py +330 -0
  50. dinkster_caches/layered.py +150 -0
  51. dinkster_caches/memory.py +208 -0
  52. dinkster_caches/py.typed +0 -0
  53. dinkster_caches/wire.py +346 -0
  54. dinkster_image_document/__init__.py +39 -0
  55. dinkster_image_document/blend.py +264 -0
  56. dinkster_image_document/compat.py +211 -0
  57. dinkster_image_document/document.py +374 -0
  58. dinkster_image_document/format.py +569 -0
  59. dinkster_image_document/interchange.py +572 -0
  60. dinkster_image_document/py.typed +1 -0
  61. dinkster_image_document/render.py +455 -0
  62. dinkster_inference_wire/__init__.py +141 -0
  63. dinkster_inference_wire/camera_wire.py +127 -0
  64. dinkster_inference_wire/conditioning.py +752 -0
  65. dinkster_inference_wire/conditioning_wire.py +1143 -0
  66. dinkster_inference_wire/devices.py +285 -0
  67. dinkster_inference_wire/generation.py +446 -0
  68. dinkster_inference_wire/hook_values.py +168 -0
  69. dinkster_inference_wire/ic_lora_parameters.py +69 -0
  70. dinkster_inference_wire/latents.py +244 -0
  71. dinkster_inference_wire/media_wire.py +169 -0
  72. dinkster_inference_wire/openai_generation.py +1035 -0
  73. dinkster_inference_wire/pose_keypoint.py +83 -0
  74. dinkster_inference_wire/process.py +33 -0
  75. dinkster_inference_wire/py.typed +0 -0
  76. dinkster_inference_wire/registry.py +116 -0
  77. dinkster_inference_wire/sam3_wire.py +68 -0
  78. dinkster_inference_wire/sampling_wire.py +279 -0
  79. dinkster_inference_wire/sources.py +399 -0
  80. dinkster_inference_wire/spaces.py +344 -0
  81. dinkster_inference_wire/sparse.py +372 -0
  82. dinkster_inference_wire/splat_wire.py +23 -0
  83. dinkster_inference_wire/types.py +256 -0
  84. dinkster_inference_wire/vision_wire.py +53 -0
  85. dinkster_inference_wire/voxel_wire.py +72 -0
  86. dinkster_inference_wire/wan_wire.py +93 -0
  87. dinkster_inference_wire/weights.py +219 -0
  88. dinkster_memory/__init__.py +92 -0
  89. dinkster_memory/accelerator.py +96 -0
  90. dinkster_memory/budgets.py +96 -0
  91. dinkster_memory/details.py +91 -0
  92. dinkster_memory/governor.py +554 -0
  93. dinkster_memory/leases.py +176 -0
  94. dinkster_memory/py.typed +0 -0
  95. dinkster_memory/release.py +131 -0
  96. dinkster_memory/reported.py +101 -0
  97. dinkster_memory/reservations.py +101 -0
  98. dinkster_memory/resident.py +600 -0
  99. dinkster_memory/system.py +614 -0
  100. dinkster_memory/tenants.py +67 -0
  101. dinkster_protocol/__init__.py +1008 -0
  102. dinkster_protocol/attention.py +652 -0
  103. dinkster_protocol/extension_contribution_kinds.json +21 -0
  104. dinkster_protocol/extension_contribution_kinds.py +43 -0
  105. dinkster_protocol/extensions.py +742 -0
  106. dinkster_protocol/frontend_modules.py +145 -0
  107. dinkster_protocol/multigpu.py +107 -0
  108. dinkster_protocol/pack_surfaces.py +489 -0
  109. dinkster_protocol/payload.py +63 -0
  110. dinkster_protocol/preview.py +86 -0
  111. dinkster_protocol/py.typed +0 -0
  112. dinkster_protocol/result_algebra.py +380 -0
  113. dinkster_protocol/result_document.py +244 -0
  114. dinkster_protocol/run_control.py +96 -0
  115. dinkster_protocol/training.py +399 -0
  116. dinkster_protocol/workgroup.py +1014 -0
  117. dinkster_registry/__init__.py +135 -0
  118. dinkster_registry/admission.py +372 -0
  119. dinkster_registry/artifact.py +128 -0
  120. dinkster_registry/composition.py +152 -0
  121. dinkster_registry/grants.py +206 -0
  122. dinkster_registry/install.py +1017 -0
  123. dinkster_registry/model.py +414 -0
  124. dinkster_registry/principals.py +494 -0
  125. dinkster_registry/py.typed +0 -0
  126. dinkster_schema/__init__.py +347 -0
  127. dinkster_schema/capture.py +240 -0
  128. dinkster_schema/coerce.py +186 -0
  129. dinkster_schema/comfy_alias.py +496 -0
  130. dinkster_schema/comfy_group.py +768 -0
  131. dinkster_schema/elaborate.py +676 -0
  132. dinkster_schema/execution.py +24 -0
  133. dinkster_schema/expansion.py +123 -0
  134. dinkster_schema/log.py +206 -0
  135. dinkster_schema/media.py +77 -0
  136. dinkster_schema/model.py +2920 -0
  137. dinkster_schema/names.py +25 -0
  138. dinkster_schema/node.py +137 -0
  139. dinkster_schema/output_descriptors.py +63 -0
  140. dinkster_schema/py.typed +0 -0
  141. dinkster_schema/replace.py +1853 -0
  142. dinkster_schema/reporting.py +441 -0
  143. dinkster_schema/solve.py +178 -0
  144. dinkster_schema/wire.py +1876 -0
  145. dinkster_values/__init__.py +625 -0
  146. dinkster_values/_media_array.py +7 -0
  147. dinkster_values/absent.py +77 -0
  148. dinkster_values/asset_folders.py +64 -0
  149. dinkster_values/assets.py +106 -0
  150. dinkster_values/audio_codec.py +594 -0
  151. dinkster_values/audio_lazy.py +871 -0
  152. dinkster_values/builtin_types.py +181 -0
  153. dinkster_values/curve.py +260 -0
  154. dinkster_values/detection_codec.py +286 -0
  155. dinkster_values/image_codec.py +691 -0
  156. dinkster_values/image_components.py +219 -0
  157. dinkster_values/latent_codec.py +514 -0
  158. dinkster_values/limits.py +52 -0
  159. dinkster_values/lists.py +216 -0
  160. dinkster_values/media_containers.py +70 -0
  161. dinkster_values/model.py +417 -0
  162. dinkster_values/model3d_codec.py +167 -0
  163. dinkster_values/names.py +168 -0
  164. dinkster_values/pins.py +77 -0
  165. dinkster_values/py.typed +0 -0
  166. dinkster_values/registry.py +1105 -0
  167. dinkster_values/resident.py +210 -0
  168. dinkster_values/resources.py +210 -0
  169. dinkster_values/splat_codec.py +477 -0
  170. dinkster_values/storage.py +133 -0
  171. dinkster_values/streams.py +241 -0
  172. dinkster_values/svg_codec.py +120 -0
  173. dinkster_values/timeline_video.py +68 -0
  174. dinkster_values/tuples.py +61 -0
  175. dinkster_values/video_codec.py +897 -0
  176. dinkster_values/video_document.py +315 -0
  177. dinkster_values/video_edits.py +253 -0
  178. dinkster_values/video_probe.py +221 -0
  179. dinkster_video/__init__.py +58 -0
  180. dinkster_video/document.py +352 -0
  181. dinkster_video/formats.py +722 -0
  182. dinkster_video/image_math.py +175 -0
  183. dinkster_video/preview/dinkster-pack.toml +36 -0
  184. dinkster_video/preview/src/dinkster_video_preview/__init__.py +84 -0
  185. dinkster_video/preview/src/dinkster_video_preview/preview.js +45 -0
  186. dinkster_video/py.typed +1 -0
  187. dinkster_video/runtime.py +1561 -0
  188. dinkster_video/timeline.py +576 -0
  189. dinkster_video/timeline_runtime.py +305 -0
  190. dinkster_video_preview/__init__.py +84 -0
  191. dinkster_video_preview/preview.js +45 -0
  192. dinkster_workers/__init__.py +207 -0
  193. dinkster_workers/_doctor_probe.py +359 -0
  194. dinkster_workers/accelerator.py +156 -0
  195. dinkster_workers/aimdo_bootstrap.py +81 -0
  196. dinkster_workers/backend_env.py +1485 -0
  197. dinkster_workers/blobs.py +402 -0
  198. dinkster_workers/boundary.py +1918 -0
  199. dinkster_workers/catalog.py +182 -0
  200. dinkster_workers/devices.py +164 -0
  201. dinkster_workers/diagnostics.py +64 -0
  202. dinkster_workers/dispatch.py +298 -0
  203. dinkster_workers/doctor.py +1382 -0
  204. dinkster_workers/egress.py +344 -0
  205. dinkster_workers/error_hints.py +175 -0
  206. dinkster_workers/execution.py +160 -0
  207. dinkster_workers/governed.py +123 -0
  208. dinkster_workers/headroom.py +156 -0
  209. dinkster_workers/host.py +3116 -0
  210. dinkster_workers/in_process.py +1325 -0
  211. dinkster_workers/interpreter.py +84 -0
  212. dinkster_workers/isolated.py +862 -0
  213. dinkster_workers/launch.py +72 -0
  214. dinkster_workers/manifest.py +3332 -0
  215. dinkster_workers/media.py +196 -0
  216. dinkster_workers/pack_environment.py +473 -0
  217. dinkster_workers/pack_lock.py +359 -0
  218. dinkster_workers/placement.py +215 -0
  219. dinkster_workers/preview_artifacts.py +233 -0
  220. dinkster_workers/produced_assets.py +218 -0
  221. dinkster_workers/provision.py +597 -0
  222. dinkster_workers/py.typed +0 -0
  223. dinkster_workers/relay.py +762 -0
  224. dinkster_workers/remote.py +921 -0
  225. dinkster_workers/resume.py +490 -0
  226. dinkster_workers/routing.py +134 -0
  227. dinkster_workers/run_control.py +275 -0
  228. dinkster_workers/sandbox.py +798 -0
  229. dinkster_workers/saved_artifacts.py +275 -0
  230. dinkster_workers/service.py +593 -0
  231. dinkster_workers/session.py +2605 -0
  232. dinkster_workers/single_job_multi_gpu.py +7 -0
  233. dinkster_workers/staging.py +230 -0
  234. dinkster_workers/transport.py +161 -0
  235. dinkster_workers/workgroup.py +729 -0
  236. dinkster_workers/workgroup_compose.py +135 -0
  237. dinkster_workers/workgroup_config.py +196 -0
  238. dinkster_workers/workgroup_session.py +340 -0
@@ -0,0 +1,226 @@
1
+ """Acquisition: turning a declared need into verified local bytes
2
+ (DESIGN 3.12 + roadmap "templates/asset distribution").
3
+
4
+ One function, one contract: ``acquire_need`` resolves a need in strict
5
+ cost order - already-local, packaged, remote - and every source must
6
+ verify before vault publication, so a wrong file in a pack, a lying mirror,
7
+ or a truncated download lands NOTHING. The digest the need declared is the
8
+ only authority.
9
+
10
+ Consent lives ABOVE this function on purpose. Acquisition mutates disk
11
+ and possibly touches the network, so callers (job preflight, template
12
+ install, CLI) decide per digest whether to call it at all; the function
13
+ itself never asks. A digestless need returns "unverifiable" without
14
+ touching any source - the reserved seat for a future name-guess tier
15
+ with its own explicit acceptance, never a silent same-name fallback.
16
+
17
+ Provenance accretes on success: a remote URL that actually produced
18
+ verifying bytes is worth remembering, so it merges into the store for
19
+ the next machine that consults it.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ from collections.abc import Callable, Mapping, Sequence
25
+ from dataclasses import dataclass
26
+ from functools import partial
27
+ from pathlib import Path
28
+ from typing import Literal
29
+
30
+ from dinkster_values import MEBIBYTE
31
+
32
+ from .fetch import (
33
+ DEFAULT_FETCH_TIMEOUT,
34
+ FetchAborted,
35
+ fetch_asset,
36
+ fetch_public_asset,
37
+ )
38
+ from .identity import AssetError
39
+ from .model import AssetRef, AssetResolver
40
+ from .need import AssetNeed, AssetSource, PackagedSource, RemoteSource
41
+ from .provenance import ProvenanceRecord, ProvenanceStore
42
+ from .public_acquisition import PublicAcquisitionReceiptStore, PublicAcquisitionSourceV1
43
+ from .vault import AssetVault
44
+
45
+ _COPY_CHUNK_SIZE = 8 * MEBIBYTE
46
+
47
+ AcquisitionStatus = Literal["held", "acquired", "unverifiable", "failed"]
48
+
49
+
50
+ @dataclass(frozen=True)
51
+ class AcquisitionResult:
52
+ """What happened for one need. ``path`` is set for held/acquired;
53
+ ``detail`` is a human-facing explanation for the other outcomes."""
54
+
55
+ need: AssetNeed
56
+ status: AcquisitionStatus
57
+ path: Path | None = None
58
+ detail: str = ""
59
+
60
+ @property
61
+ def ok(self) -> bool:
62
+ return self.status in ("held", "acquired")
63
+
64
+
65
+ def _packaged_file(source: PackagedSource, pack_roots: Mapping[str, Path]) -> Path | None:
66
+ """Resolve a packaged source to a real file, confined to its pack's
67
+ root. The path grammar already bans '..'; the containment check makes
68
+ escape structurally impossible even through symlinks."""
69
+ root = pack_roots.get(source.pack)
70
+ if root is None:
71
+ return None
72
+ candidate = (root / source.path).resolve()
73
+ try:
74
+ candidate.relative_to(root.resolve())
75
+ except ValueError:
76
+ return None
77
+ return candidate if candidate.is_file() else None
78
+
79
+
80
+ def acquire_need(
81
+ need: AssetNeed,
82
+ vault: AssetVault,
83
+ *,
84
+ resolver: AssetResolver | None = None,
85
+ pack_roots: Mapping[str, Path] | None = None,
86
+ provenance: ProvenanceStore | None = None,
87
+ public_sources: Sequence[PublicAcquisitionSourceV1] = (),
88
+ receipts: PublicAcquisitionReceiptStore | None = None,
89
+ timeout: float = DEFAULT_FETCH_TIMEOUT,
90
+ before_publish: Callable[[AssetSource, int], None] | None = None,
91
+ ) -> AcquisitionResult:
92
+ """Materialize one need, verified, in cost order.
93
+
94
+ 1. Local: the vault, then ``resolver`` (mounts, libraries) - free.
95
+ 2. Packaged: copy out of an installed pack under ``pack_roots``,
96
+ digest-verified through the vault writer.
97
+ 3. Remote: the need's URLs plus any ``provenance`` leads for the
98
+ digest, via ``fetch_asset`` (verified the same way).
99
+
100
+ Success from a non-local source records the winning provenance.
101
+ ``before_publish`` observes only a source whose bytes verified; a
102
+ metadata failure aborts publication rather than trying another source."""
103
+ if not need.digest:
104
+ return AcquisitionResult(
105
+ need,
106
+ "unverifiable",
107
+ detail=(
108
+ "need declares no digest; acquisition requires content "
109
+ "identity (name-based matching is a separate, explicitly "
110
+ "accepted tier)"
111
+ ),
112
+ )
113
+
114
+ for local in (vault, resolver):
115
+ if local is None:
116
+ continue
117
+ try:
118
+ held = AssetRef(need.digest, need.name, 0, resolver=local).local_path()
119
+ except (AssetError, OSError):
120
+ continue
121
+ return AcquisitionResult(need, "held", path=held)
122
+
123
+ failures: list[str] = []
124
+ for source in need.sources:
125
+ if not isinstance(source, PackagedSource):
126
+ continue
127
+ file = _packaged_file(source, pack_roots or {})
128
+ if file is None:
129
+ failures.append(f"packaged {source.pack}:{source.path}: not present")
130
+ continue
131
+ try:
132
+ with file.open("rb") as handle, vault.writer(need.digest) as writer:
133
+ while chunk := handle.read(_COPY_CHUNK_SIZE):
134
+ writer.write(chunk)
135
+ path = writer.commit(
136
+ before_publish=(
137
+ partial(before_publish, source) if before_publish is not None else None
138
+ )
139
+ )
140
+ except (OSError, AssetError) as exc:
141
+ failures.append(f"packaged {source.pack}:{source.path}: {exc}")
142
+ continue
143
+ _record_provenance(need, provenance, note=f"packaged by {source.pack}")
144
+ return AcquisitionResult(need, "acquired", path=path)
145
+
146
+ urls = [source.url for source in need.sources if isinstance(source, RemoteSource)]
147
+ if provenance is not None:
148
+ urls.extend(url for url in provenance.sources(need.digest) if url not in urls)
149
+ if urls:
150
+ if receipts is not None:
151
+ eligible = tuple(
152
+ source
153
+ for source in public_sources
154
+ if source.digest == need.digest and any(url in source.listed_urls for url in urls)
155
+ )
156
+ attempted: set[tuple[str, int]] = set()
157
+ for url in urls:
158
+ for source in eligible:
159
+ if url not in source.listed_urls or (url, source.size_bytes) in attempted:
160
+ continue
161
+ attempted.add((url, source.size_bytes))
162
+ try:
163
+ fetched = fetch_public_asset(
164
+ need.digest,
165
+ url,
166
+ source.size_bytes,
167
+ vault,
168
+ timeout=timeout,
169
+ before_publish=(
170
+ partial(before_publish, RemoteSource(url))
171
+ if before_publish is not None
172
+ else None
173
+ ),
174
+ )
175
+ except FetchAborted:
176
+ raise
177
+ except (OSError, AssetError, TimeoutError):
178
+ continue
179
+ for matching in eligible:
180
+ if (
181
+ matching.size_bytes == fetched.size_bytes
182
+ and fetched.listed_url in matching.listed_urls
183
+ ):
184
+ try:
185
+ receipts.record(matching, fetched)
186
+ except (OSError, AssetError):
187
+ pass
188
+ _record_provenance(need, provenance)
189
+ return AcquisitionResult(need, "acquired", path=fetched.path)
190
+ path = fetch_asset(
191
+ need.digest,
192
+ urls,
193
+ vault,
194
+ timeout=timeout,
195
+ before_publish=(
196
+ (lambda url, size: before_publish(RemoteSource(url), size))
197
+ if before_publish is not None
198
+ else None
199
+ ),
200
+ )
201
+ if path is not None:
202
+ _record_provenance(need, provenance)
203
+ return AcquisitionResult(need, "acquired", path=path)
204
+ failures.append(f"remote: all {len(urls)} candidate URL(s) failed")
205
+
206
+ if not failures:
207
+ failures.append("no usable sources declared or known")
208
+ return AcquisitionResult(need, "failed", detail="; ".join(failures))
209
+
210
+
211
+ def _record_provenance(need: AssetNeed, provenance: ProvenanceStore | None, note: str = "") -> None:
212
+ if provenance is None:
213
+ return
214
+ try:
215
+ provenance.add(
216
+ ProvenanceRecord(
217
+ digest=need.digest,
218
+ sources=tuple(
219
+ source.url for source in need.sources if isinstance(source, RemoteSource)
220
+ ),
221
+ note=note,
222
+ )
223
+ )
224
+ except OSError:
225
+ # Verified bytes remain valid when their advisory provenance cannot persist.
226
+ return
@@ -0,0 +1,145 @@
1
+ """Trusted acquisition source facts."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from typing import Literal, TypeVar
7
+ from urllib.parse import urlsplit
8
+
9
+ from .identity import AssetError, require_digest
10
+ from .kind import require_asset_kind
11
+ from .resolution import SourceIdentity
12
+
13
+ _REASON_LIMIT = 256
14
+ _OPAQUE_ID_LIMIT = 256
15
+ _T = TypeVar("_T")
16
+
17
+ SourceState = Literal["available", "unavailable"]
18
+
19
+
20
+ def _require_type(value: object, expected: type[_T], message: str) -> _T:
21
+ if not isinstance(value, expected):
22
+ raise AssetError(message)
23
+ return value
24
+
25
+
26
+ def _digest(value: object, field_name: str = "digest") -> str:
27
+ if not isinstance(value, str):
28
+ raise AssetError(f"{field_name} must be a canonical asset digest string")
29
+ return require_digest(value)
30
+
31
+
32
+ def _asset_kind(value: object) -> str:
33
+ if not isinstance(value, str):
34
+ raise AssetError("asset kind must be a string")
35
+ return require_asset_kind(value)
36
+
37
+
38
+ def _opaque_id(value: object, field_name: str) -> str:
39
+ if not isinstance(value, str) or not value or value != value.strip():
40
+ raise AssetError(f"{field_name} must be a non-empty opaque id")
41
+ if len(value) > _OPAQUE_ID_LIMIT or any(ord(char) < 32 for char in value):
42
+ raise AssetError(f"{field_name} must be a bounded printable opaque id")
43
+ return value
44
+
45
+
46
+ def _reason(value: object, field_name: str) -> str:
47
+ if not isinstance(value, str):
48
+ raise AssetError(f"{field_name} must be a string")
49
+ if len(value) > _REASON_LIMIT:
50
+ raise AssetError(f"{field_name} exceeds {_REASON_LIMIT} characters")
51
+ return value
52
+
53
+
54
+ def _integer(value: object, field_name: str, *, minimum: int | None = None) -> int:
55
+ if not isinstance(value, int) or isinstance(value, bool):
56
+ raise AssetError(f"{field_name} must be an integer")
57
+ if minimum is not None and value < minimum:
58
+ raise AssetError(f"{field_name} must be an integer >= {minimum}")
59
+ return value
60
+
61
+
62
+ def _http_locator(value: object) -> str:
63
+ if not isinstance(value, str) or not value or value != value.strip():
64
+ raise AssetError("authorized locator must be a non-empty http(s) URL")
65
+ if len(value) > 2048 or any(char.isspace() or ord(char) < 32 for char in value):
66
+ raise AssetError("authorized locator must be a bounded URL without raw whitespace")
67
+ try:
68
+ parsed = urlsplit(value)
69
+ hostname = parsed.hostname
70
+ username = parsed.username
71
+ password = parsed.password
72
+ port = parsed.port
73
+ except ValueError as exc:
74
+ raise AssetError("authorized locator is malformed") from exc
75
+ if parsed.scheme not in ("http", "https") or not parsed.netloc or hostname is None:
76
+ raise AssetError("authorized locator must be an absolute http(s) URL")
77
+ if username is not None or password is not None:
78
+ raise AssetError("authorized locator must not contain userinfo credentials")
79
+ if parsed.query or parsed.fragment:
80
+ raise AssetError("authorized locator must not contain a query or fragment")
81
+ if parsed.netloc.endswith(":") or (port is not None and not 1 <= port <= 65535):
82
+ raise AssetError("authorized locator contains an invalid port")
83
+ return value
84
+
85
+
86
+ @dataclass(frozen=True)
87
+ class AcquisitionSource:
88
+ """Trusted source authority, separate from advisory provider metadata.
89
+
90
+ The locator is an absolute HTTP(S) URL with a hostname, optional valid
91
+ port, and optional path. Raw whitespace, userinfo, query, and fragment are
92
+ forbidden so credentials and bearer-style signed URL values cannot enter
93
+ the durable source fact.
94
+ """
95
+
96
+ source: SourceIdentity
97
+ digest: str
98
+ asset_kind: str
99
+ state: SourceState
100
+ authorized_locator: str
101
+ expected_size: int
102
+ media_type: str
103
+ provider_priority: int = 0
104
+ unavailable_reason: str = ""
105
+ credential_requirement_id: str | None = None
106
+ license_requirement_id: str | None = None
107
+ cost_requirement_id: str | None = None
108
+ policy_override_requirement_id: str | None = None
109
+
110
+ def __post_init__(self) -> None:
111
+ _require_type(self.source, SourceIdentity, "source must be a SourceIdentity")
112
+ object.__setattr__(self, "digest", _digest(self.digest))
113
+ object.__setattr__(self, "asset_kind", _asset_kind(self.asset_kind))
114
+ if self.state not in ("available", "unavailable"):
115
+ raise AssetError(f"unknown acquisition source state: {self.state!r}")
116
+ object.__setattr__(self, "authorized_locator", _http_locator(self.authorized_locator))
117
+ object.__setattr__(
118
+ self,
119
+ "expected_size",
120
+ _integer(self.expected_size, "expected size", minimum=0),
121
+ )
122
+ media_type = _require_type(self.media_type, str, "media type must be a non-empty string")
123
+ if not media_type.strip():
124
+ raise AssetError("media type must be a non-empty string")
125
+ object.__setattr__(self, "media_type", media_type.strip())
126
+ object.__setattr__(
127
+ self,
128
+ "provider_priority",
129
+ _integer(self.provider_priority, "provider priority"),
130
+ )
131
+ reason = _reason(self.unavailable_reason, "unavailable reason")
132
+ if self.state == "available" and reason:
133
+ raise AssetError("available acquisition source cannot carry an unavailable reason")
134
+ if self.state == "unavailable" and not reason:
135
+ raise AssetError("unavailable acquisition source requires a reason")
136
+ object.__setattr__(self, "unavailable_reason", reason)
137
+ for field_name in (
138
+ "credential_requirement_id",
139
+ "license_requirement_id",
140
+ "cost_requirement_id",
141
+ "policy_override_requirement_id",
142
+ ):
143
+ value: object = getattr(self, field_name)
144
+ if value is not None:
145
+ object.__setattr__(self, field_name, _opaque_id(value, field_name))
@@ -0,0 +1,119 @@
1
+ """Bind portable AUDIO sources and publish large compatibility PCM without copying it."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import struct
7
+ import tempfile
8
+ from collections.abc import Mapping
9
+ from typing import Any, cast
10
+
11
+ from dinkster_values import MEBIBYTE, TypeRegistry
12
+ from dinkster_values.audio_codec import (
13
+ AUDIO_INLINE_LIMIT,
14
+ AUDIO_WAVEFORM_LIMITS,
15
+ AUDIO_WAVEFORM_VERSION,
16
+ AUDIO_WINDOW_LIMITS,
17
+ AUDIO_WINDOW_VERSION,
18
+ audio_fingerprint,
19
+ audio_from_source,
20
+ audio_meta,
21
+ bind_audio_sources,
22
+ coerce_audio,
23
+ decode_audio,
24
+ encode_audio,
25
+ normalize_audio_waveform_request,
26
+ normalize_audio_window_request,
27
+ render_audio_wav,
28
+ render_audio_waveform,
29
+ render_audio_window,
30
+ validate_audio_encoded,
31
+ )
32
+ from dinkster_values.audio_lazy import LazyAudio
33
+ from dinkster_values.storage import audio_input
34
+
35
+ from .identity import AssetError, new_hasher
36
+ from .model import AssetRef, AssetResolver
37
+ from .vault import AssetVault
38
+
39
+
40
+ def bind_audio_value(obj: object, resolver: AssetResolver | None = None) -> LazyAudio:
41
+ value = coerce_audio(obj)
42
+ source = value["source"]
43
+ if isinstance(source, Mapping) and "pcm" in source:
44
+ pcm = cast(Any, source["pcm"])
45
+ if pcm.nbytes > AUDIO_INLINE_LIMIT:
46
+ import numpy as np
47
+
48
+ root = os.environ.get("DINKSTER_ASSET_VAULT")
49
+ if not root:
50
+ raise AssetError("large AUDIO PCM requires a configured DINKSTER_ASSET_VAULT")
51
+ vault = AssetVault(root)
52
+ with tempfile.TemporaryFile() as file:
53
+ file.write(struct.pack("<Q", cast(int, source["sample_rate"])))
54
+ # numpy streams contiguous arrays directly to disk, rather than a BytesIO copy.
55
+ np.save(file, np.ascontiguousarray(pcm), allow_pickle=False)
56
+ size = file.tell()
57
+ file.seek(0)
58
+ hasher = new_hasher()
59
+ while chunk := file.read(MEBIBYTE):
60
+ hasher.update(chunk)
61
+ digest = "blake3:" + hasher.hexdigest()
62
+ file.seek(0)
63
+ with vault.writer(digest) as writer:
64
+ while chunk := file.read(MEBIBYTE):
65
+ writer.write(chunk)
66
+ writer.commit()
67
+ published = audio_from_source(AssetRef(digest, "audio.pcm", size, resolver=vault))
68
+ cast(dict[str, object], published["probe"])["layout"] = cast(
69
+ Mapping[str, object], value["probe"]
70
+ )["layout"]
71
+ published["edits"] = value["edits"]
72
+ value = published
73
+ elif isinstance(source, AssetRef) and source.resolver is None and resolver is not None:
74
+ value["source"] = AssetRef.from_wire(source.to_wire(), resolver)
75
+ value["edits"] = [
76
+ {"concat": [bind_audio_value(child, resolver) for child in edit["concat"]]}
77
+ if "concat" in edit
78
+ else edit
79
+ for edit in cast("list[dict[str, Any]]", value["edits"])
80
+ ]
81
+ return bind_audio_sources(value, lambda wire: AssetRef.from_wire(wire, resolver))
82
+
83
+
84
+ def register_audio_value_type(
85
+ registry: TypeRegistry, type_id: str, resolver: AssetResolver | None = None
86
+ ) -> None:
87
+ registry.register(
88
+ type_id,
89
+ encode=encode_audio,
90
+ decode=lambda data: bind_audio_value(decode_audio(data), resolver),
91
+ coerce=lambda obj: bind_audio_value(obj, resolver),
92
+ fingerprint=audio_fingerprint(type_id),
93
+ meta=audio_meta,
94
+ input_convert=audio_input,
95
+ validate_encoded_buffer=validate_audio_encoded,
96
+ )
97
+ registry.register_rendition(type_id, "wav", mime="audio/wav", render=render_audio_wav)
98
+ registry.register_rendition(
99
+ type_id,
100
+ "waveform",
101
+ mime="image/png",
102
+ render=render_audio_waveform,
103
+ version=AUDIO_WAVEFORM_VERSION,
104
+ parameters=("batch", "waveform"),
105
+ defaults={"batch": "0"},
106
+ limits=AUDIO_WAVEFORM_LIMITS,
107
+ normalize=normalize_audio_waveform_request,
108
+ )
109
+ registry.register_rendition(
110
+ type_id,
111
+ "window",
112
+ mime="audio/wav",
113
+ render=render_audio_window,
114
+ version=AUDIO_WINDOW_VERSION,
115
+ parameters=("batch", "window"),
116
+ defaults={"batch": "0"},
117
+ limits=AUDIO_WINDOW_LIMITS,
118
+ normalize=normalize_audio_window_request,
119
+ )
@@ -0,0 +1,156 @@
1
+ """The asset catalog: file-like semantics without filesystem access.
2
+
3
+ The catalog separates what legacy Comfy conflates (DESIGN 3.12): *content*
4
+ (one digest per unique blob, however many names it has) and *references*
5
+ (virtual paths in a namespace pointing at content). Virtual folders are
6
+ namespace prefixes: listing, globbing, and prefix queries behave like a
7
+ filesystem - the part ComfyUI's asset work is missing - while nodes and
8
+ clients never touch real paths.
9
+
10
+ Virtual paths are ``/``-separated, relative, and validated: no empty
11
+ segments, no ``.``/``..``, no backslashes. A catalog is a queryable index;
12
+ where bytes live is a store concern (see AssetResolver).
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import re
18
+ from collections.abc import Iterable, Mapping
19
+ from dataclasses import dataclass, field
20
+
21
+ from .identity import AssetError, require_digest
22
+ from .model import AssetRef, AssetResolver
23
+
24
+
25
+ def glob_regex(pattern: str) -> re.Pattern[str]:
26
+ parts: list[str] = []
27
+ index = 0
28
+ while index < len(pattern):
29
+ char = pattern[index]
30
+ if char == "*":
31
+ if pattern[index : index + 2] == "**":
32
+ parts.append(".*")
33
+ index += 2
34
+ else:
35
+ parts.append("[^/]*")
36
+ index += 1
37
+ elif char == "?":
38
+ parts.append("[^/]")
39
+ index += 1
40
+ else:
41
+ parts.append(re.escape(char))
42
+ index += 1
43
+ return re.compile("".join(parts) + r"\Z")
44
+
45
+
46
+ def validate_virtual_path(path: str) -> str:
47
+ if "\\" in path:
48
+ raise AssetError(f"virtual paths use '/' separators only: {path!r}")
49
+ segments = path.split("/")
50
+ if any(segment in ("", ".", "..") for segment in segments):
51
+ raise AssetError(f"invalid virtual path (empty/./.. segment): {path!r}")
52
+ return path
53
+
54
+
55
+ @dataclass(frozen=True)
56
+ class AssetEntry:
57
+ """One reference: a virtual path bound to a content identity. Many
58
+ entries may share one digest (duplicate files, aliases)."""
59
+
60
+ virtual_path: str
61
+ digest: str
62
+ size: int
63
+ media_type: str = "application/octet-stream"
64
+ tags: tuple[str, ...] = ()
65
+ metadata: Mapping[str, object] = field(default_factory=dict[str, object])
66
+
67
+ def __post_init__(self) -> None:
68
+ validate_virtual_path(self.virtual_path)
69
+ require_digest(self.digest)
70
+
71
+ @property
72
+ def name(self) -> str:
73
+ return self.virtual_path.rsplit("/", 1)[-1]
74
+
75
+ def ref(self, resolver: AssetResolver | None = None) -> AssetRef:
76
+ return AssetRef(
77
+ digest=self.digest,
78
+ name=self.name,
79
+ size=self.size,
80
+ media_type=self.media_type,
81
+ virtual_path=self.virtual_path,
82
+ resolver=resolver,
83
+ )
84
+
85
+
86
+ @dataclass(frozen=True)
87
+ class FolderListing:
88
+ """One level of a virtual folder: child folder names and the entries
89
+ directly inside - the shape a file picker renders."""
90
+
91
+ folders: tuple[str, ...]
92
+ entries: tuple[AssetEntry, ...]
93
+
94
+
95
+ class AssetCatalog:
96
+ """In-memory reference index. Entries are keyed by virtual path (unique);
97
+ digests are secondary keys (one-to-many)."""
98
+
99
+ def __init__(self, entries: Iterable[AssetEntry] = ()) -> None:
100
+ self._by_path: dict[str, AssetEntry] = {}
101
+ self._by_digest: dict[str, list[AssetEntry]] = {}
102
+ for entry in entries:
103
+ self.add(entry)
104
+
105
+ def add(self, entry: AssetEntry) -> None:
106
+ existing = self._by_path.get(entry.virtual_path)
107
+ if existing is not None:
108
+ self._by_digest[existing.digest].remove(existing)
109
+ self._by_path[entry.virtual_path] = entry
110
+ self._by_digest.setdefault(entry.digest, []).append(entry)
111
+
112
+ def remove(self, virtual_path: str) -> None:
113
+ entry = self._by_path.pop(virtual_path, None)
114
+ if entry is not None:
115
+ self._by_digest[entry.digest].remove(entry)
116
+
117
+ def get(self, virtual_path: str) -> AssetEntry | None:
118
+ return self._by_path.get(virtual_path)
119
+
120
+ def by_digest(self, digest: str) -> tuple[AssetEntry, ...]:
121
+ """All virtual paths carrying this content - renames and duplicate
122
+ copies are the same asset."""
123
+ return tuple(self._by_digest.get(require_digest(digest), ()))
124
+
125
+ def entries(self) -> tuple[AssetEntry, ...]:
126
+ return tuple(self._by_path[path] for path in sorted(self._by_path))
127
+
128
+ def list_folder(self, prefix: str = "") -> FolderListing:
129
+ """Immediate children of a virtual folder, files and subfolders,
130
+ like a directory listing."""
131
+ if prefix:
132
+ validate_virtual_path(prefix)
133
+ prefix += "/"
134
+ folders: set[str] = set()
135
+ entries: list[AssetEntry] = []
136
+ for path in sorted(self._by_path):
137
+ if not path.startswith(prefix):
138
+ continue
139
+ remainder = path[len(prefix) :]
140
+ head, separator, _ = remainder.partition("/")
141
+ if separator:
142
+ folders.add(head)
143
+ else:
144
+ entries.append(self._by_path[path])
145
+ return FolderListing(folders=tuple(sorted(folders)), entries=tuple(entries))
146
+
147
+ def glob(self, pattern: str) -> tuple[AssetEntry, ...]:
148
+ """Match virtual paths against a glob pattern with filesystem
149
+ semantics: ``*`` and ``?`` stay within one folder segment,
150
+ ``**`` spans folders (``models/loras/*.safetensors``,
151
+ ``models/**`` for a whole subtree)."""
152
+ regex = glob_regex(pattern)
153
+ return tuple(self._by_path[path] for path in sorted(self._by_path) if regex.match(path))
154
+
155
+ def __len__(self) -> int:
156
+ return len(self._by_path)