dinkster-api 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dinkster_api/__init__.py +9 -0
- dinkster_api/py.typed +0 -0
- dinkster_api/v1.py +935 -0
- dinkster_api-0.0.2.dist-info/METADATA +16 -0
- dinkster_api-0.0.2.dist-info/RECORD +238 -0
- dinkster_api-0.0.2.dist-info/WHEEL +4 -0
- dinkster_api-0.0.2.dist-info/entry_points.txt +2 -0
- dinkster_assets/__init__.py +337 -0
- dinkster_assets/acquire.py +226 -0
- dinkster_assets/acquisition_plan.py +145 -0
- dinkster_assets/audio.py +119 -0
- dinkster_assets/catalog.py +156 -0
- dinkster_assets/component_manifest.py +110 -0
- dinkster_assets/declared.py +218 -0
- dinkster_assets/fetch.py +495 -0
- dinkster_assets/guess.py +136 -0
- dinkster_assets/identity.py +72 -0
- dinkster_assets/integrity.py +310 -0
- dinkster_assets/json_metadata.py +41 -0
- dinkster_assets/kind.py +52 -0
- dinkster_assets/latent_format.py +346 -0
- dinkster_assets/library.py +816 -0
- dinkster_assets/media.py +1385 -0
- dinkster_assets/migrations/env.py +38 -0
- dinkster_assets/migrations/script.py.mako +28 -0
- dinkster_assets/migrations/versions/0001_asset_records_create_asset_record_catalog.py +109 -0
- dinkster_assets/migrations/versions/0002_asset_observations.py +36 -0
- dinkster_assets/model.py +242 -0
- dinkster_assets/mounts.py +1278 -0
- dinkster_assets/need.py +168 -0
- dinkster_assets/pack_assets.py +217 -0
- dinkster_assets/probe.py +118 -0
- dinkster_assets/provenance.py +254 -0
- dinkster_assets/public_acquisition.py +424 -0
- dinkster_assets/py.typed +0 -0
- dinkster_assets/records.py +899 -0
- dinkster_assets/resolution.py +778 -0
- dinkster_assets/resolver_index.py +364 -0
- dinkster_assets/resolver_subscription.py +1149 -0
- dinkster_assets/save_target.py +139 -0
- dinkster_assets/schema.py +73 -0
- dinkster_assets/selections.py +256 -0
- dinkster_assets/transport.py +243 -0
- dinkster_assets/value.py +258 -0
- dinkster_assets/vault.py +276 -0
- dinkster_assets/writer.py +765 -0
- dinkster_caches/__init__.py +19 -0
- dinkster_caches/cas.py +310 -0
- dinkster_caches/disk.py +330 -0
- dinkster_caches/layered.py +150 -0
- dinkster_caches/memory.py +208 -0
- dinkster_caches/py.typed +0 -0
- dinkster_caches/wire.py +346 -0
- dinkster_image_document/__init__.py +39 -0
- dinkster_image_document/blend.py +264 -0
- dinkster_image_document/compat.py +211 -0
- dinkster_image_document/document.py +374 -0
- dinkster_image_document/format.py +569 -0
- dinkster_image_document/interchange.py +572 -0
- dinkster_image_document/py.typed +1 -0
- dinkster_image_document/render.py +455 -0
- dinkster_inference_wire/__init__.py +141 -0
- dinkster_inference_wire/camera_wire.py +127 -0
- dinkster_inference_wire/conditioning.py +752 -0
- dinkster_inference_wire/conditioning_wire.py +1143 -0
- dinkster_inference_wire/devices.py +285 -0
- dinkster_inference_wire/generation.py +446 -0
- dinkster_inference_wire/hook_values.py +168 -0
- dinkster_inference_wire/ic_lora_parameters.py +69 -0
- dinkster_inference_wire/latents.py +244 -0
- dinkster_inference_wire/media_wire.py +169 -0
- dinkster_inference_wire/openai_generation.py +1035 -0
- dinkster_inference_wire/pose_keypoint.py +83 -0
- dinkster_inference_wire/process.py +33 -0
- dinkster_inference_wire/py.typed +0 -0
- dinkster_inference_wire/registry.py +116 -0
- dinkster_inference_wire/sam3_wire.py +68 -0
- dinkster_inference_wire/sampling_wire.py +279 -0
- dinkster_inference_wire/sources.py +399 -0
- dinkster_inference_wire/spaces.py +344 -0
- dinkster_inference_wire/sparse.py +372 -0
- dinkster_inference_wire/splat_wire.py +23 -0
- dinkster_inference_wire/types.py +256 -0
- dinkster_inference_wire/vision_wire.py +53 -0
- dinkster_inference_wire/voxel_wire.py +72 -0
- dinkster_inference_wire/wan_wire.py +93 -0
- dinkster_inference_wire/weights.py +219 -0
- dinkster_memory/__init__.py +92 -0
- dinkster_memory/accelerator.py +96 -0
- dinkster_memory/budgets.py +96 -0
- dinkster_memory/details.py +91 -0
- dinkster_memory/governor.py +554 -0
- dinkster_memory/leases.py +176 -0
- dinkster_memory/py.typed +0 -0
- dinkster_memory/release.py +131 -0
- dinkster_memory/reported.py +101 -0
- dinkster_memory/reservations.py +101 -0
- dinkster_memory/resident.py +600 -0
- dinkster_memory/system.py +614 -0
- dinkster_memory/tenants.py +67 -0
- dinkster_protocol/__init__.py +1008 -0
- dinkster_protocol/attention.py +652 -0
- dinkster_protocol/extension_contribution_kinds.json +21 -0
- dinkster_protocol/extension_contribution_kinds.py +43 -0
- dinkster_protocol/extensions.py +742 -0
- dinkster_protocol/frontend_modules.py +145 -0
- dinkster_protocol/multigpu.py +107 -0
- dinkster_protocol/pack_surfaces.py +489 -0
- dinkster_protocol/payload.py +63 -0
- dinkster_protocol/preview.py +86 -0
- dinkster_protocol/py.typed +0 -0
- dinkster_protocol/result_algebra.py +380 -0
- dinkster_protocol/result_document.py +244 -0
- dinkster_protocol/run_control.py +96 -0
- dinkster_protocol/training.py +399 -0
- dinkster_protocol/workgroup.py +1014 -0
- dinkster_registry/__init__.py +135 -0
- dinkster_registry/admission.py +372 -0
- dinkster_registry/artifact.py +128 -0
- dinkster_registry/composition.py +152 -0
- dinkster_registry/grants.py +206 -0
- dinkster_registry/install.py +1017 -0
- dinkster_registry/model.py +414 -0
- dinkster_registry/principals.py +494 -0
- dinkster_registry/py.typed +0 -0
- dinkster_schema/__init__.py +347 -0
- dinkster_schema/capture.py +240 -0
- dinkster_schema/coerce.py +186 -0
- dinkster_schema/comfy_alias.py +496 -0
- dinkster_schema/comfy_group.py +768 -0
- dinkster_schema/elaborate.py +676 -0
- dinkster_schema/execution.py +24 -0
- dinkster_schema/expansion.py +123 -0
- dinkster_schema/log.py +206 -0
- dinkster_schema/media.py +77 -0
- dinkster_schema/model.py +2920 -0
- dinkster_schema/names.py +25 -0
- dinkster_schema/node.py +137 -0
- dinkster_schema/output_descriptors.py +63 -0
- dinkster_schema/py.typed +0 -0
- dinkster_schema/replace.py +1853 -0
- dinkster_schema/reporting.py +441 -0
- dinkster_schema/solve.py +178 -0
- dinkster_schema/wire.py +1876 -0
- dinkster_values/__init__.py +625 -0
- dinkster_values/_media_array.py +7 -0
- dinkster_values/absent.py +77 -0
- dinkster_values/asset_folders.py +64 -0
- dinkster_values/assets.py +106 -0
- dinkster_values/audio_codec.py +594 -0
- dinkster_values/audio_lazy.py +871 -0
- dinkster_values/builtin_types.py +181 -0
- dinkster_values/curve.py +260 -0
- dinkster_values/detection_codec.py +286 -0
- dinkster_values/image_codec.py +691 -0
- dinkster_values/image_components.py +219 -0
- dinkster_values/latent_codec.py +514 -0
- dinkster_values/limits.py +52 -0
- dinkster_values/lists.py +216 -0
- dinkster_values/media_containers.py +70 -0
- dinkster_values/model.py +417 -0
- dinkster_values/model3d_codec.py +167 -0
- dinkster_values/names.py +168 -0
- dinkster_values/pins.py +77 -0
- dinkster_values/py.typed +0 -0
- dinkster_values/registry.py +1105 -0
- dinkster_values/resident.py +210 -0
- dinkster_values/resources.py +210 -0
- dinkster_values/splat_codec.py +477 -0
- dinkster_values/storage.py +133 -0
- dinkster_values/streams.py +241 -0
- dinkster_values/svg_codec.py +120 -0
- dinkster_values/timeline_video.py +68 -0
- dinkster_values/tuples.py +61 -0
- dinkster_values/video_codec.py +897 -0
- dinkster_values/video_document.py +315 -0
- dinkster_values/video_edits.py +253 -0
- dinkster_values/video_probe.py +221 -0
- dinkster_video/__init__.py +58 -0
- dinkster_video/document.py +352 -0
- dinkster_video/formats.py +722 -0
- dinkster_video/image_math.py +175 -0
- dinkster_video/preview/dinkster-pack.toml +36 -0
- dinkster_video/preview/src/dinkster_video_preview/__init__.py +84 -0
- dinkster_video/preview/src/dinkster_video_preview/preview.js +45 -0
- dinkster_video/py.typed +1 -0
- dinkster_video/runtime.py +1561 -0
- dinkster_video/timeline.py +576 -0
- dinkster_video/timeline_runtime.py +305 -0
- dinkster_video_preview/__init__.py +84 -0
- dinkster_video_preview/preview.js +45 -0
- dinkster_workers/__init__.py +207 -0
- dinkster_workers/_doctor_probe.py +359 -0
- dinkster_workers/accelerator.py +156 -0
- dinkster_workers/aimdo_bootstrap.py +81 -0
- dinkster_workers/backend_env.py +1485 -0
- dinkster_workers/blobs.py +402 -0
- dinkster_workers/boundary.py +1918 -0
- dinkster_workers/catalog.py +182 -0
- dinkster_workers/devices.py +164 -0
- dinkster_workers/diagnostics.py +64 -0
- dinkster_workers/dispatch.py +298 -0
- dinkster_workers/doctor.py +1382 -0
- dinkster_workers/egress.py +344 -0
- dinkster_workers/error_hints.py +175 -0
- dinkster_workers/execution.py +160 -0
- dinkster_workers/governed.py +123 -0
- dinkster_workers/headroom.py +156 -0
- dinkster_workers/host.py +3116 -0
- dinkster_workers/in_process.py +1325 -0
- dinkster_workers/interpreter.py +84 -0
- dinkster_workers/isolated.py +862 -0
- dinkster_workers/launch.py +72 -0
- dinkster_workers/manifest.py +3332 -0
- dinkster_workers/media.py +196 -0
- dinkster_workers/pack_environment.py +473 -0
- dinkster_workers/pack_lock.py +359 -0
- dinkster_workers/placement.py +215 -0
- dinkster_workers/preview_artifacts.py +233 -0
- dinkster_workers/produced_assets.py +218 -0
- dinkster_workers/provision.py +597 -0
- dinkster_workers/py.typed +0 -0
- dinkster_workers/relay.py +762 -0
- dinkster_workers/remote.py +921 -0
- dinkster_workers/resume.py +490 -0
- dinkster_workers/routing.py +134 -0
- dinkster_workers/run_control.py +275 -0
- dinkster_workers/sandbox.py +798 -0
- dinkster_workers/saved_artifacts.py +275 -0
- dinkster_workers/service.py +593 -0
- dinkster_workers/session.py +2605 -0
- dinkster_workers/single_job_multi_gpu.py +7 -0
- dinkster_workers/staging.py +230 -0
- dinkster_workers/transport.py +161 -0
- dinkster_workers/workgroup.py +729 -0
- dinkster_workers/workgroup_compose.py +135 -0
- dinkster_workers/workgroup_config.py +196 -0
- dinkster_workers/workgroup_session.py +340 -0
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""Acquisition: turning a declared need into verified local bytes
|
|
2
|
+
(DESIGN 3.12 + roadmap "templates/asset distribution").
|
|
3
|
+
|
|
4
|
+
One function, one contract: ``acquire_need`` resolves a need in strict
|
|
5
|
+
cost order - already-local, packaged, remote - and every source must
|
|
6
|
+
verify before vault publication, so a wrong file in a pack, a lying mirror,
|
|
7
|
+
or a truncated download lands NOTHING. The digest the need declared is the
|
|
8
|
+
only authority.
|
|
9
|
+
|
|
10
|
+
Consent lives ABOVE this function on purpose. Acquisition mutates disk
|
|
11
|
+
and possibly touches the network, so callers (job preflight, template
|
|
12
|
+
install, CLI) decide per digest whether to call it at all; the function
|
|
13
|
+
itself never asks. A digestless need returns "unverifiable" without
|
|
14
|
+
touching any source - the reserved seat for a future name-guess tier
|
|
15
|
+
with its own explicit acceptance, never a silent same-name fallback.
|
|
16
|
+
|
|
17
|
+
Provenance accretes on success: a remote URL that actually produced
|
|
18
|
+
verifying bytes is worth remembering, so it merges into the store for
|
|
19
|
+
the next machine that consults it.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
25
|
+
from dataclasses import dataclass
|
|
26
|
+
from functools import partial
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
from typing import Literal
|
|
29
|
+
|
|
30
|
+
from dinkster_values import MEBIBYTE
|
|
31
|
+
|
|
32
|
+
from .fetch import (
|
|
33
|
+
DEFAULT_FETCH_TIMEOUT,
|
|
34
|
+
FetchAborted,
|
|
35
|
+
fetch_asset,
|
|
36
|
+
fetch_public_asset,
|
|
37
|
+
)
|
|
38
|
+
from .identity import AssetError
|
|
39
|
+
from .model import AssetRef, AssetResolver
|
|
40
|
+
from .need import AssetNeed, AssetSource, PackagedSource, RemoteSource
|
|
41
|
+
from .provenance import ProvenanceRecord, ProvenanceStore
|
|
42
|
+
from .public_acquisition import PublicAcquisitionReceiptStore, PublicAcquisitionSourceV1
|
|
43
|
+
from .vault import AssetVault
|
|
44
|
+
|
|
45
|
+
_COPY_CHUNK_SIZE = 8 * MEBIBYTE
|
|
46
|
+
|
|
47
|
+
AcquisitionStatus = Literal["held", "acquired", "unverifiable", "failed"]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(frozen=True)
|
|
51
|
+
class AcquisitionResult:
|
|
52
|
+
"""What happened for one need. ``path`` is set for held/acquired;
|
|
53
|
+
``detail`` is a human-facing explanation for the other outcomes."""
|
|
54
|
+
|
|
55
|
+
need: AssetNeed
|
|
56
|
+
status: AcquisitionStatus
|
|
57
|
+
path: Path | None = None
|
|
58
|
+
detail: str = ""
|
|
59
|
+
|
|
60
|
+
@property
|
|
61
|
+
def ok(self) -> bool:
|
|
62
|
+
return self.status in ("held", "acquired")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _packaged_file(source: PackagedSource, pack_roots: Mapping[str, Path]) -> Path | None:
|
|
66
|
+
"""Resolve a packaged source to a real file, confined to its pack's
|
|
67
|
+
root. The path grammar already bans '..'; the containment check makes
|
|
68
|
+
escape structurally impossible even through symlinks."""
|
|
69
|
+
root = pack_roots.get(source.pack)
|
|
70
|
+
if root is None:
|
|
71
|
+
return None
|
|
72
|
+
candidate = (root / source.path).resolve()
|
|
73
|
+
try:
|
|
74
|
+
candidate.relative_to(root.resolve())
|
|
75
|
+
except ValueError:
|
|
76
|
+
return None
|
|
77
|
+
return candidate if candidate.is_file() else None
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def acquire_need(
|
|
81
|
+
need: AssetNeed,
|
|
82
|
+
vault: AssetVault,
|
|
83
|
+
*,
|
|
84
|
+
resolver: AssetResolver | None = None,
|
|
85
|
+
pack_roots: Mapping[str, Path] | None = None,
|
|
86
|
+
provenance: ProvenanceStore | None = None,
|
|
87
|
+
public_sources: Sequence[PublicAcquisitionSourceV1] = (),
|
|
88
|
+
receipts: PublicAcquisitionReceiptStore | None = None,
|
|
89
|
+
timeout: float = DEFAULT_FETCH_TIMEOUT,
|
|
90
|
+
before_publish: Callable[[AssetSource, int], None] | None = None,
|
|
91
|
+
) -> AcquisitionResult:
|
|
92
|
+
"""Materialize one need, verified, in cost order.
|
|
93
|
+
|
|
94
|
+
1. Local: the vault, then ``resolver`` (mounts, libraries) - free.
|
|
95
|
+
2. Packaged: copy out of an installed pack under ``pack_roots``,
|
|
96
|
+
digest-verified through the vault writer.
|
|
97
|
+
3. Remote: the need's URLs plus any ``provenance`` leads for the
|
|
98
|
+
digest, via ``fetch_asset`` (verified the same way).
|
|
99
|
+
|
|
100
|
+
Success from a non-local source records the winning provenance.
|
|
101
|
+
``before_publish`` observes only a source whose bytes verified; a
|
|
102
|
+
metadata failure aborts publication rather than trying another source."""
|
|
103
|
+
if not need.digest:
|
|
104
|
+
return AcquisitionResult(
|
|
105
|
+
need,
|
|
106
|
+
"unverifiable",
|
|
107
|
+
detail=(
|
|
108
|
+
"need declares no digest; acquisition requires content "
|
|
109
|
+
"identity (name-based matching is a separate, explicitly "
|
|
110
|
+
"accepted tier)"
|
|
111
|
+
),
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
for local in (vault, resolver):
|
|
115
|
+
if local is None:
|
|
116
|
+
continue
|
|
117
|
+
try:
|
|
118
|
+
held = AssetRef(need.digest, need.name, 0, resolver=local).local_path()
|
|
119
|
+
except (AssetError, OSError):
|
|
120
|
+
continue
|
|
121
|
+
return AcquisitionResult(need, "held", path=held)
|
|
122
|
+
|
|
123
|
+
failures: list[str] = []
|
|
124
|
+
for source in need.sources:
|
|
125
|
+
if not isinstance(source, PackagedSource):
|
|
126
|
+
continue
|
|
127
|
+
file = _packaged_file(source, pack_roots or {})
|
|
128
|
+
if file is None:
|
|
129
|
+
failures.append(f"packaged {source.pack}:{source.path}: not present")
|
|
130
|
+
continue
|
|
131
|
+
try:
|
|
132
|
+
with file.open("rb") as handle, vault.writer(need.digest) as writer:
|
|
133
|
+
while chunk := handle.read(_COPY_CHUNK_SIZE):
|
|
134
|
+
writer.write(chunk)
|
|
135
|
+
path = writer.commit(
|
|
136
|
+
before_publish=(
|
|
137
|
+
partial(before_publish, source) if before_publish is not None else None
|
|
138
|
+
)
|
|
139
|
+
)
|
|
140
|
+
except (OSError, AssetError) as exc:
|
|
141
|
+
failures.append(f"packaged {source.pack}:{source.path}: {exc}")
|
|
142
|
+
continue
|
|
143
|
+
_record_provenance(need, provenance, note=f"packaged by {source.pack}")
|
|
144
|
+
return AcquisitionResult(need, "acquired", path=path)
|
|
145
|
+
|
|
146
|
+
urls = [source.url for source in need.sources if isinstance(source, RemoteSource)]
|
|
147
|
+
if provenance is not None:
|
|
148
|
+
urls.extend(url for url in provenance.sources(need.digest) if url not in urls)
|
|
149
|
+
if urls:
|
|
150
|
+
if receipts is not None:
|
|
151
|
+
eligible = tuple(
|
|
152
|
+
source
|
|
153
|
+
for source in public_sources
|
|
154
|
+
if source.digest == need.digest and any(url in source.listed_urls for url in urls)
|
|
155
|
+
)
|
|
156
|
+
attempted: set[tuple[str, int]] = set()
|
|
157
|
+
for url in urls:
|
|
158
|
+
for source in eligible:
|
|
159
|
+
if url not in source.listed_urls or (url, source.size_bytes) in attempted:
|
|
160
|
+
continue
|
|
161
|
+
attempted.add((url, source.size_bytes))
|
|
162
|
+
try:
|
|
163
|
+
fetched = fetch_public_asset(
|
|
164
|
+
need.digest,
|
|
165
|
+
url,
|
|
166
|
+
source.size_bytes,
|
|
167
|
+
vault,
|
|
168
|
+
timeout=timeout,
|
|
169
|
+
before_publish=(
|
|
170
|
+
partial(before_publish, RemoteSource(url))
|
|
171
|
+
if before_publish is not None
|
|
172
|
+
else None
|
|
173
|
+
),
|
|
174
|
+
)
|
|
175
|
+
except FetchAborted:
|
|
176
|
+
raise
|
|
177
|
+
except (OSError, AssetError, TimeoutError):
|
|
178
|
+
continue
|
|
179
|
+
for matching in eligible:
|
|
180
|
+
if (
|
|
181
|
+
matching.size_bytes == fetched.size_bytes
|
|
182
|
+
and fetched.listed_url in matching.listed_urls
|
|
183
|
+
):
|
|
184
|
+
try:
|
|
185
|
+
receipts.record(matching, fetched)
|
|
186
|
+
except (OSError, AssetError):
|
|
187
|
+
pass
|
|
188
|
+
_record_provenance(need, provenance)
|
|
189
|
+
return AcquisitionResult(need, "acquired", path=fetched.path)
|
|
190
|
+
path = fetch_asset(
|
|
191
|
+
need.digest,
|
|
192
|
+
urls,
|
|
193
|
+
vault,
|
|
194
|
+
timeout=timeout,
|
|
195
|
+
before_publish=(
|
|
196
|
+
(lambda url, size: before_publish(RemoteSource(url), size))
|
|
197
|
+
if before_publish is not None
|
|
198
|
+
else None
|
|
199
|
+
),
|
|
200
|
+
)
|
|
201
|
+
if path is not None:
|
|
202
|
+
_record_provenance(need, provenance)
|
|
203
|
+
return AcquisitionResult(need, "acquired", path=path)
|
|
204
|
+
failures.append(f"remote: all {len(urls)} candidate URL(s) failed")
|
|
205
|
+
|
|
206
|
+
if not failures:
|
|
207
|
+
failures.append("no usable sources declared or known")
|
|
208
|
+
return AcquisitionResult(need, "failed", detail="; ".join(failures))
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _record_provenance(need: AssetNeed, provenance: ProvenanceStore | None, note: str = "") -> None:
|
|
212
|
+
if provenance is None:
|
|
213
|
+
return
|
|
214
|
+
try:
|
|
215
|
+
provenance.add(
|
|
216
|
+
ProvenanceRecord(
|
|
217
|
+
digest=need.digest,
|
|
218
|
+
sources=tuple(
|
|
219
|
+
source.url for source in need.sources if isinstance(source, RemoteSource)
|
|
220
|
+
),
|
|
221
|
+
note=note,
|
|
222
|
+
)
|
|
223
|
+
)
|
|
224
|
+
except OSError:
|
|
225
|
+
# Verified bytes remain valid when their advisory provenance cannot persist.
|
|
226
|
+
return
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""Trusted acquisition source facts."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Literal, TypeVar
|
|
7
|
+
from urllib.parse import urlsplit
|
|
8
|
+
|
|
9
|
+
from .identity import AssetError, require_digest
|
|
10
|
+
from .kind import require_asset_kind
|
|
11
|
+
from .resolution import SourceIdentity
|
|
12
|
+
|
|
13
|
+
_REASON_LIMIT = 256
|
|
14
|
+
_OPAQUE_ID_LIMIT = 256
|
|
15
|
+
_T = TypeVar("_T")
|
|
16
|
+
|
|
17
|
+
SourceState = Literal["available", "unavailable"]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _require_type(value: object, expected: type[_T], message: str) -> _T:
|
|
21
|
+
if not isinstance(value, expected):
|
|
22
|
+
raise AssetError(message)
|
|
23
|
+
return value
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _digest(value: object, field_name: str = "digest") -> str:
|
|
27
|
+
if not isinstance(value, str):
|
|
28
|
+
raise AssetError(f"{field_name} must be a canonical asset digest string")
|
|
29
|
+
return require_digest(value)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _asset_kind(value: object) -> str:
|
|
33
|
+
if not isinstance(value, str):
|
|
34
|
+
raise AssetError("asset kind must be a string")
|
|
35
|
+
return require_asset_kind(value)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _opaque_id(value: object, field_name: str) -> str:
|
|
39
|
+
if not isinstance(value, str) or not value or value != value.strip():
|
|
40
|
+
raise AssetError(f"{field_name} must be a non-empty opaque id")
|
|
41
|
+
if len(value) > _OPAQUE_ID_LIMIT or any(ord(char) < 32 for char in value):
|
|
42
|
+
raise AssetError(f"{field_name} must be a bounded printable opaque id")
|
|
43
|
+
return value
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _reason(value: object, field_name: str) -> str:
|
|
47
|
+
if not isinstance(value, str):
|
|
48
|
+
raise AssetError(f"{field_name} must be a string")
|
|
49
|
+
if len(value) > _REASON_LIMIT:
|
|
50
|
+
raise AssetError(f"{field_name} exceeds {_REASON_LIMIT} characters")
|
|
51
|
+
return value
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _integer(value: object, field_name: str, *, minimum: int | None = None) -> int:
|
|
55
|
+
if not isinstance(value, int) or isinstance(value, bool):
|
|
56
|
+
raise AssetError(f"{field_name} must be an integer")
|
|
57
|
+
if minimum is not None and value < minimum:
|
|
58
|
+
raise AssetError(f"{field_name} must be an integer >= {minimum}")
|
|
59
|
+
return value
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _http_locator(value: object) -> str:
|
|
63
|
+
if not isinstance(value, str) or not value or value != value.strip():
|
|
64
|
+
raise AssetError("authorized locator must be a non-empty http(s) URL")
|
|
65
|
+
if len(value) > 2048 or any(char.isspace() or ord(char) < 32 for char in value):
|
|
66
|
+
raise AssetError("authorized locator must be a bounded URL without raw whitespace")
|
|
67
|
+
try:
|
|
68
|
+
parsed = urlsplit(value)
|
|
69
|
+
hostname = parsed.hostname
|
|
70
|
+
username = parsed.username
|
|
71
|
+
password = parsed.password
|
|
72
|
+
port = parsed.port
|
|
73
|
+
except ValueError as exc:
|
|
74
|
+
raise AssetError("authorized locator is malformed") from exc
|
|
75
|
+
if parsed.scheme not in ("http", "https") or not parsed.netloc or hostname is None:
|
|
76
|
+
raise AssetError("authorized locator must be an absolute http(s) URL")
|
|
77
|
+
if username is not None or password is not None:
|
|
78
|
+
raise AssetError("authorized locator must not contain userinfo credentials")
|
|
79
|
+
if parsed.query or parsed.fragment:
|
|
80
|
+
raise AssetError("authorized locator must not contain a query or fragment")
|
|
81
|
+
if parsed.netloc.endswith(":") or (port is not None and not 1 <= port <= 65535):
|
|
82
|
+
raise AssetError("authorized locator contains an invalid port")
|
|
83
|
+
return value
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass(frozen=True)
|
|
87
|
+
class AcquisitionSource:
|
|
88
|
+
"""Trusted source authority, separate from advisory provider metadata.
|
|
89
|
+
|
|
90
|
+
The locator is an absolute HTTP(S) URL with a hostname, optional valid
|
|
91
|
+
port, and optional path. Raw whitespace, userinfo, query, and fragment are
|
|
92
|
+
forbidden so credentials and bearer-style signed URL values cannot enter
|
|
93
|
+
the durable source fact.
|
|
94
|
+
"""
|
|
95
|
+
|
|
96
|
+
source: SourceIdentity
|
|
97
|
+
digest: str
|
|
98
|
+
asset_kind: str
|
|
99
|
+
state: SourceState
|
|
100
|
+
authorized_locator: str
|
|
101
|
+
expected_size: int
|
|
102
|
+
media_type: str
|
|
103
|
+
provider_priority: int = 0
|
|
104
|
+
unavailable_reason: str = ""
|
|
105
|
+
credential_requirement_id: str | None = None
|
|
106
|
+
license_requirement_id: str | None = None
|
|
107
|
+
cost_requirement_id: str | None = None
|
|
108
|
+
policy_override_requirement_id: str | None = None
|
|
109
|
+
|
|
110
|
+
def __post_init__(self) -> None:
|
|
111
|
+
_require_type(self.source, SourceIdentity, "source must be a SourceIdentity")
|
|
112
|
+
object.__setattr__(self, "digest", _digest(self.digest))
|
|
113
|
+
object.__setattr__(self, "asset_kind", _asset_kind(self.asset_kind))
|
|
114
|
+
if self.state not in ("available", "unavailable"):
|
|
115
|
+
raise AssetError(f"unknown acquisition source state: {self.state!r}")
|
|
116
|
+
object.__setattr__(self, "authorized_locator", _http_locator(self.authorized_locator))
|
|
117
|
+
object.__setattr__(
|
|
118
|
+
self,
|
|
119
|
+
"expected_size",
|
|
120
|
+
_integer(self.expected_size, "expected size", minimum=0),
|
|
121
|
+
)
|
|
122
|
+
media_type = _require_type(self.media_type, str, "media type must be a non-empty string")
|
|
123
|
+
if not media_type.strip():
|
|
124
|
+
raise AssetError("media type must be a non-empty string")
|
|
125
|
+
object.__setattr__(self, "media_type", media_type.strip())
|
|
126
|
+
object.__setattr__(
|
|
127
|
+
self,
|
|
128
|
+
"provider_priority",
|
|
129
|
+
_integer(self.provider_priority, "provider priority"),
|
|
130
|
+
)
|
|
131
|
+
reason = _reason(self.unavailable_reason, "unavailable reason")
|
|
132
|
+
if self.state == "available" and reason:
|
|
133
|
+
raise AssetError("available acquisition source cannot carry an unavailable reason")
|
|
134
|
+
if self.state == "unavailable" and not reason:
|
|
135
|
+
raise AssetError("unavailable acquisition source requires a reason")
|
|
136
|
+
object.__setattr__(self, "unavailable_reason", reason)
|
|
137
|
+
for field_name in (
|
|
138
|
+
"credential_requirement_id",
|
|
139
|
+
"license_requirement_id",
|
|
140
|
+
"cost_requirement_id",
|
|
141
|
+
"policy_override_requirement_id",
|
|
142
|
+
):
|
|
143
|
+
value: object = getattr(self, field_name)
|
|
144
|
+
if value is not None:
|
|
145
|
+
object.__setattr__(self, field_name, _opaque_id(value, field_name))
|
dinkster_assets/audio.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Bind portable AUDIO sources and publish large compatibility PCM without copying it."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import struct
|
|
7
|
+
import tempfile
|
|
8
|
+
from collections.abc import Mapping
|
|
9
|
+
from typing import Any, cast
|
|
10
|
+
|
|
11
|
+
from dinkster_values import MEBIBYTE, TypeRegistry
|
|
12
|
+
from dinkster_values.audio_codec import (
|
|
13
|
+
AUDIO_INLINE_LIMIT,
|
|
14
|
+
AUDIO_WAVEFORM_LIMITS,
|
|
15
|
+
AUDIO_WAVEFORM_VERSION,
|
|
16
|
+
AUDIO_WINDOW_LIMITS,
|
|
17
|
+
AUDIO_WINDOW_VERSION,
|
|
18
|
+
audio_fingerprint,
|
|
19
|
+
audio_from_source,
|
|
20
|
+
audio_meta,
|
|
21
|
+
bind_audio_sources,
|
|
22
|
+
coerce_audio,
|
|
23
|
+
decode_audio,
|
|
24
|
+
encode_audio,
|
|
25
|
+
normalize_audio_waveform_request,
|
|
26
|
+
normalize_audio_window_request,
|
|
27
|
+
render_audio_wav,
|
|
28
|
+
render_audio_waveform,
|
|
29
|
+
render_audio_window,
|
|
30
|
+
validate_audio_encoded,
|
|
31
|
+
)
|
|
32
|
+
from dinkster_values.audio_lazy import LazyAudio
|
|
33
|
+
from dinkster_values.storage import audio_input
|
|
34
|
+
|
|
35
|
+
from .identity import AssetError, new_hasher
|
|
36
|
+
from .model import AssetRef, AssetResolver
|
|
37
|
+
from .vault import AssetVault
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def bind_audio_value(obj: object, resolver: AssetResolver | None = None) -> LazyAudio:
|
|
41
|
+
value = coerce_audio(obj)
|
|
42
|
+
source = value["source"]
|
|
43
|
+
if isinstance(source, Mapping) and "pcm" in source:
|
|
44
|
+
pcm = cast(Any, source["pcm"])
|
|
45
|
+
if pcm.nbytes > AUDIO_INLINE_LIMIT:
|
|
46
|
+
import numpy as np
|
|
47
|
+
|
|
48
|
+
root = os.environ.get("DINKSTER_ASSET_VAULT")
|
|
49
|
+
if not root:
|
|
50
|
+
raise AssetError("large AUDIO PCM requires a configured DINKSTER_ASSET_VAULT")
|
|
51
|
+
vault = AssetVault(root)
|
|
52
|
+
with tempfile.TemporaryFile() as file:
|
|
53
|
+
file.write(struct.pack("<Q", cast(int, source["sample_rate"])))
|
|
54
|
+
# numpy streams contiguous arrays directly to disk, rather than a BytesIO copy.
|
|
55
|
+
np.save(file, np.ascontiguousarray(pcm), allow_pickle=False)
|
|
56
|
+
size = file.tell()
|
|
57
|
+
file.seek(0)
|
|
58
|
+
hasher = new_hasher()
|
|
59
|
+
while chunk := file.read(MEBIBYTE):
|
|
60
|
+
hasher.update(chunk)
|
|
61
|
+
digest = "blake3:" + hasher.hexdigest()
|
|
62
|
+
file.seek(0)
|
|
63
|
+
with vault.writer(digest) as writer:
|
|
64
|
+
while chunk := file.read(MEBIBYTE):
|
|
65
|
+
writer.write(chunk)
|
|
66
|
+
writer.commit()
|
|
67
|
+
published = audio_from_source(AssetRef(digest, "audio.pcm", size, resolver=vault))
|
|
68
|
+
cast(dict[str, object], published["probe"])["layout"] = cast(
|
|
69
|
+
Mapping[str, object], value["probe"]
|
|
70
|
+
)["layout"]
|
|
71
|
+
published["edits"] = value["edits"]
|
|
72
|
+
value = published
|
|
73
|
+
elif isinstance(source, AssetRef) and source.resolver is None and resolver is not None:
|
|
74
|
+
value["source"] = AssetRef.from_wire(source.to_wire(), resolver)
|
|
75
|
+
value["edits"] = [
|
|
76
|
+
{"concat": [bind_audio_value(child, resolver) for child in edit["concat"]]}
|
|
77
|
+
if "concat" in edit
|
|
78
|
+
else edit
|
|
79
|
+
for edit in cast("list[dict[str, Any]]", value["edits"])
|
|
80
|
+
]
|
|
81
|
+
return bind_audio_sources(value, lambda wire: AssetRef.from_wire(wire, resolver))
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def register_audio_value_type(
|
|
85
|
+
registry: TypeRegistry, type_id: str, resolver: AssetResolver | None = None
|
|
86
|
+
) -> None:
|
|
87
|
+
registry.register(
|
|
88
|
+
type_id,
|
|
89
|
+
encode=encode_audio,
|
|
90
|
+
decode=lambda data: bind_audio_value(decode_audio(data), resolver),
|
|
91
|
+
coerce=lambda obj: bind_audio_value(obj, resolver),
|
|
92
|
+
fingerprint=audio_fingerprint(type_id),
|
|
93
|
+
meta=audio_meta,
|
|
94
|
+
input_convert=audio_input,
|
|
95
|
+
validate_encoded_buffer=validate_audio_encoded,
|
|
96
|
+
)
|
|
97
|
+
registry.register_rendition(type_id, "wav", mime="audio/wav", render=render_audio_wav)
|
|
98
|
+
registry.register_rendition(
|
|
99
|
+
type_id,
|
|
100
|
+
"waveform",
|
|
101
|
+
mime="image/png",
|
|
102
|
+
render=render_audio_waveform,
|
|
103
|
+
version=AUDIO_WAVEFORM_VERSION,
|
|
104
|
+
parameters=("batch", "waveform"),
|
|
105
|
+
defaults={"batch": "0"},
|
|
106
|
+
limits=AUDIO_WAVEFORM_LIMITS,
|
|
107
|
+
normalize=normalize_audio_waveform_request,
|
|
108
|
+
)
|
|
109
|
+
registry.register_rendition(
|
|
110
|
+
type_id,
|
|
111
|
+
"window",
|
|
112
|
+
mime="audio/wav",
|
|
113
|
+
render=render_audio_window,
|
|
114
|
+
version=AUDIO_WINDOW_VERSION,
|
|
115
|
+
parameters=("batch", "window"),
|
|
116
|
+
defaults={"batch": "0"},
|
|
117
|
+
limits=AUDIO_WINDOW_LIMITS,
|
|
118
|
+
normalize=normalize_audio_window_request,
|
|
119
|
+
)
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"""The asset catalog: file-like semantics without filesystem access.
|
|
2
|
+
|
|
3
|
+
The catalog separates what legacy Comfy conflates (DESIGN 3.12): *content*
|
|
4
|
+
(one digest per unique blob, however many names it has) and *references*
|
|
5
|
+
(virtual paths in a namespace pointing at content). Virtual folders are
|
|
6
|
+
namespace prefixes: listing, globbing, and prefix queries behave like a
|
|
7
|
+
filesystem - the part ComfyUI's asset work is missing - while nodes and
|
|
8
|
+
clients never touch real paths.
|
|
9
|
+
|
|
10
|
+
Virtual paths are ``/``-separated, relative, and validated: no empty
|
|
11
|
+
segments, no ``.``/``..``, no backslashes. A catalog is a queryable index;
|
|
12
|
+
where bytes live is a store concern (see AssetResolver).
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import re
|
|
18
|
+
from collections.abc import Iterable, Mapping
|
|
19
|
+
from dataclasses import dataclass, field
|
|
20
|
+
|
|
21
|
+
from .identity import AssetError, require_digest
|
|
22
|
+
from .model import AssetRef, AssetResolver
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def glob_regex(pattern: str) -> re.Pattern[str]:
|
|
26
|
+
parts: list[str] = []
|
|
27
|
+
index = 0
|
|
28
|
+
while index < len(pattern):
|
|
29
|
+
char = pattern[index]
|
|
30
|
+
if char == "*":
|
|
31
|
+
if pattern[index : index + 2] == "**":
|
|
32
|
+
parts.append(".*")
|
|
33
|
+
index += 2
|
|
34
|
+
else:
|
|
35
|
+
parts.append("[^/]*")
|
|
36
|
+
index += 1
|
|
37
|
+
elif char == "?":
|
|
38
|
+
parts.append("[^/]")
|
|
39
|
+
index += 1
|
|
40
|
+
else:
|
|
41
|
+
parts.append(re.escape(char))
|
|
42
|
+
index += 1
|
|
43
|
+
return re.compile("".join(parts) + r"\Z")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def validate_virtual_path(path: str) -> str:
|
|
47
|
+
if "\\" in path:
|
|
48
|
+
raise AssetError(f"virtual paths use '/' separators only: {path!r}")
|
|
49
|
+
segments = path.split("/")
|
|
50
|
+
if any(segment in ("", ".", "..") for segment in segments):
|
|
51
|
+
raise AssetError(f"invalid virtual path (empty/./.. segment): {path!r}")
|
|
52
|
+
return path
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass(frozen=True)
|
|
56
|
+
class AssetEntry:
|
|
57
|
+
"""One reference: a virtual path bound to a content identity. Many
|
|
58
|
+
entries may share one digest (duplicate files, aliases)."""
|
|
59
|
+
|
|
60
|
+
virtual_path: str
|
|
61
|
+
digest: str
|
|
62
|
+
size: int
|
|
63
|
+
media_type: str = "application/octet-stream"
|
|
64
|
+
tags: tuple[str, ...] = ()
|
|
65
|
+
metadata: Mapping[str, object] = field(default_factory=dict[str, object])
|
|
66
|
+
|
|
67
|
+
def __post_init__(self) -> None:
|
|
68
|
+
validate_virtual_path(self.virtual_path)
|
|
69
|
+
require_digest(self.digest)
|
|
70
|
+
|
|
71
|
+
@property
|
|
72
|
+
def name(self) -> str:
|
|
73
|
+
return self.virtual_path.rsplit("/", 1)[-1]
|
|
74
|
+
|
|
75
|
+
def ref(self, resolver: AssetResolver | None = None) -> AssetRef:
|
|
76
|
+
return AssetRef(
|
|
77
|
+
digest=self.digest,
|
|
78
|
+
name=self.name,
|
|
79
|
+
size=self.size,
|
|
80
|
+
media_type=self.media_type,
|
|
81
|
+
virtual_path=self.virtual_path,
|
|
82
|
+
resolver=resolver,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass(frozen=True)
|
|
87
|
+
class FolderListing:
|
|
88
|
+
"""One level of a virtual folder: child folder names and the entries
|
|
89
|
+
directly inside - the shape a file picker renders."""
|
|
90
|
+
|
|
91
|
+
folders: tuple[str, ...]
|
|
92
|
+
entries: tuple[AssetEntry, ...]
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class AssetCatalog:
|
|
96
|
+
"""In-memory reference index. Entries are keyed by virtual path (unique);
|
|
97
|
+
digests are secondary keys (one-to-many)."""
|
|
98
|
+
|
|
99
|
+
def __init__(self, entries: Iterable[AssetEntry] = ()) -> None:
|
|
100
|
+
self._by_path: dict[str, AssetEntry] = {}
|
|
101
|
+
self._by_digest: dict[str, list[AssetEntry]] = {}
|
|
102
|
+
for entry in entries:
|
|
103
|
+
self.add(entry)
|
|
104
|
+
|
|
105
|
+
def add(self, entry: AssetEntry) -> None:
|
|
106
|
+
existing = self._by_path.get(entry.virtual_path)
|
|
107
|
+
if existing is not None:
|
|
108
|
+
self._by_digest[existing.digest].remove(existing)
|
|
109
|
+
self._by_path[entry.virtual_path] = entry
|
|
110
|
+
self._by_digest.setdefault(entry.digest, []).append(entry)
|
|
111
|
+
|
|
112
|
+
def remove(self, virtual_path: str) -> None:
|
|
113
|
+
entry = self._by_path.pop(virtual_path, None)
|
|
114
|
+
if entry is not None:
|
|
115
|
+
self._by_digest[entry.digest].remove(entry)
|
|
116
|
+
|
|
117
|
+
def get(self, virtual_path: str) -> AssetEntry | None:
|
|
118
|
+
return self._by_path.get(virtual_path)
|
|
119
|
+
|
|
120
|
+
def by_digest(self, digest: str) -> tuple[AssetEntry, ...]:
|
|
121
|
+
"""All virtual paths carrying this content - renames and duplicate
|
|
122
|
+
copies are the same asset."""
|
|
123
|
+
return tuple(self._by_digest.get(require_digest(digest), ()))
|
|
124
|
+
|
|
125
|
+
def entries(self) -> tuple[AssetEntry, ...]:
|
|
126
|
+
return tuple(self._by_path[path] for path in sorted(self._by_path))
|
|
127
|
+
|
|
128
|
+
def list_folder(self, prefix: str = "") -> FolderListing:
|
|
129
|
+
"""Immediate children of a virtual folder, files and subfolders,
|
|
130
|
+
like a directory listing."""
|
|
131
|
+
if prefix:
|
|
132
|
+
validate_virtual_path(prefix)
|
|
133
|
+
prefix += "/"
|
|
134
|
+
folders: set[str] = set()
|
|
135
|
+
entries: list[AssetEntry] = []
|
|
136
|
+
for path in sorted(self._by_path):
|
|
137
|
+
if not path.startswith(prefix):
|
|
138
|
+
continue
|
|
139
|
+
remainder = path[len(prefix) :]
|
|
140
|
+
head, separator, _ = remainder.partition("/")
|
|
141
|
+
if separator:
|
|
142
|
+
folders.add(head)
|
|
143
|
+
else:
|
|
144
|
+
entries.append(self._by_path[path])
|
|
145
|
+
return FolderListing(folders=tuple(sorted(folders)), entries=tuple(entries))
|
|
146
|
+
|
|
147
|
+
def glob(self, pattern: str) -> tuple[AssetEntry, ...]:
|
|
148
|
+
"""Match virtual paths against a glob pattern with filesystem
|
|
149
|
+
semantics: ``*`` and ``?`` stay within one folder segment,
|
|
150
|
+
``**`` spans folders (``models/loras/*.safetensors``,
|
|
151
|
+
``models/**`` for a whole subtree)."""
|
|
152
|
+
regex = glob_regex(pattern)
|
|
153
|
+
return tuple(self._by_path[path] for path in sorted(self._by_path) if regex.match(path))
|
|
154
|
+
|
|
155
|
+
def __len__(self) -> int:
|
|
156
|
+
return len(self._by_path)
|