dinkster-api 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dinkster_api/__init__.py +9 -0
- dinkster_api/py.typed +0 -0
- dinkster_api/v1.py +935 -0
- dinkster_api-0.0.2.dist-info/METADATA +16 -0
- dinkster_api-0.0.2.dist-info/RECORD +238 -0
- dinkster_api-0.0.2.dist-info/WHEEL +4 -0
- dinkster_api-0.0.2.dist-info/entry_points.txt +2 -0
- dinkster_assets/__init__.py +337 -0
- dinkster_assets/acquire.py +226 -0
- dinkster_assets/acquisition_plan.py +145 -0
- dinkster_assets/audio.py +119 -0
- dinkster_assets/catalog.py +156 -0
- dinkster_assets/component_manifest.py +110 -0
- dinkster_assets/declared.py +218 -0
- dinkster_assets/fetch.py +495 -0
- dinkster_assets/guess.py +136 -0
- dinkster_assets/identity.py +72 -0
- dinkster_assets/integrity.py +310 -0
- dinkster_assets/json_metadata.py +41 -0
- dinkster_assets/kind.py +52 -0
- dinkster_assets/latent_format.py +346 -0
- dinkster_assets/library.py +816 -0
- dinkster_assets/media.py +1385 -0
- dinkster_assets/migrations/env.py +38 -0
- dinkster_assets/migrations/script.py.mako +28 -0
- dinkster_assets/migrations/versions/0001_asset_records_create_asset_record_catalog.py +109 -0
- dinkster_assets/migrations/versions/0002_asset_observations.py +36 -0
- dinkster_assets/model.py +242 -0
- dinkster_assets/mounts.py +1278 -0
- dinkster_assets/need.py +168 -0
- dinkster_assets/pack_assets.py +217 -0
- dinkster_assets/probe.py +118 -0
- dinkster_assets/provenance.py +254 -0
- dinkster_assets/public_acquisition.py +424 -0
- dinkster_assets/py.typed +0 -0
- dinkster_assets/records.py +899 -0
- dinkster_assets/resolution.py +778 -0
- dinkster_assets/resolver_index.py +364 -0
- dinkster_assets/resolver_subscription.py +1149 -0
- dinkster_assets/save_target.py +139 -0
- dinkster_assets/schema.py +73 -0
- dinkster_assets/selections.py +256 -0
- dinkster_assets/transport.py +243 -0
- dinkster_assets/value.py +258 -0
- dinkster_assets/vault.py +276 -0
- dinkster_assets/writer.py +765 -0
- dinkster_caches/__init__.py +19 -0
- dinkster_caches/cas.py +310 -0
- dinkster_caches/disk.py +330 -0
- dinkster_caches/layered.py +150 -0
- dinkster_caches/memory.py +208 -0
- dinkster_caches/py.typed +0 -0
- dinkster_caches/wire.py +346 -0
- dinkster_image_document/__init__.py +39 -0
- dinkster_image_document/blend.py +264 -0
- dinkster_image_document/compat.py +211 -0
- dinkster_image_document/document.py +374 -0
- dinkster_image_document/format.py +569 -0
- dinkster_image_document/interchange.py +572 -0
- dinkster_image_document/py.typed +1 -0
- dinkster_image_document/render.py +455 -0
- dinkster_inference_wire/__init__.py +141 -0
- dinkster_inference_wire/camera_wire.py +127 -0
- dinkster_inference_wire/conditioning.py +752 -0
- dinkster_inference_wire/conditioning_wire.py +1143 -0
- dinkster_inference_wire/devices.py +285 -0
- dinkster_inference_wire/generation.py +446 -0
- dinkster_inference_wire/hook_values.py +168 -0
- dinkster_inference_wire/ic_lora_parameters.py +69 -0
- dinkster_inference_wire/latents.py +244 -0
- dinkster_inference_wire/media_wire.py +169 -0
- dinkster_inference_wire/openai_generation.py +1035 -0
- dinkster_inference_wire/pose_keypoint.py +83 -0
- dinkster_inference_wire/process.py +33 -0
- dinkster_inference_wire/py.typed +0 -0
- dinkster_inference_wire/registry.py +116 -0
- dinkster_inference_wire/sam3_wire.py +68 -0
- dinkster_inference_wire/sampling_wire.py +279 -0
- dinkster_inference_wire/sources.py +399 -0
- dinkster_inference_wire/spaces.py +344 -0
- dinkster_inference_wire/sparse.py +372 -0
- dinkster_inference_wire/splat_wire.py +23 -0
- dinkster_inference_wire/types.py +256 -0
- dinkster_inference_wire/vision_wire.py +53 -0
- dinkster_inference_wire/voxel_wire.py +72 -0
- dinkster_inference_wire/wan_wire.py +93 -0
- dinkster_inference_wire/weights.py +219 -0
- dinkster_memory/__init__.py +92 -0
- dinkster_memory/accelerator.py +96 -0
- dinkster_memory/budgets.py +96 -0
- dinkster_memory/details.py +91 -0
- dinkster_memory/governor.py +554 -0
- dinkster_memory/leases.py +176 -0
- dinkster_memory/py.typed +0 -0
- dinkster_memory/release.py +131 -0
- dinkster_memory/reported.py +101 -0
- dinkster_memory/reservations.py +101 -0
- dinkster_memory/resident.py +600 -0
- dinkster_memory/system.py +614 -0
- dinkster_memory/tenants.py +67 -0
- dinkster_protocol/__init__.py +1008 -0
- dinkster_protocol/attention.py +652 -0
- dinkster_protocol/extension_contribution_kinds.json +21 -0
- dinkster_protocol/extension_contribution_kinds.py +43 -0
- dinkster_protocol/extensions.py +742 -0
- dinkster_protocol/frontend_modules.py +145 -0
- dinkster_protocol/multigpu.py +107 -0
- dinkster_protocol/pack_surfaces.py +489 -0
- dinkster_protocol/payload.py +63 -0
- dinkster_protocol/preview.py +86 -0
- dinkster_protocol/py.typed +0 -0
- dinkster_protocol/result_algebra.py +380 -0
- dinkster_protocol/result_document.py +244 -0
- dinkster_protocol/run_control.py +96 -0
- dinkster_protocol/training.py +399 -0
- dinkster_protocol/workgroup.py +1014 -0
- dinkster_registry/__init__.py +135 -0
- dinkster_registry/admission.py +372 -0
- dinkster_registry/artifact.py +128 -0
- dinkster_registry/composition.py +152 -0
- dinkster_registry/grants.py +206 -0
- dinkster_registry/install.py +1017 -0
- dinkster_registry/model.py +414 -0
- dinkster_registry/principals.py +494 -0
- dinkster_registry/py.typed +0 -0
- dinkster_schema/__init__.py +347 -0
- dinkster_schema/capture.py +240 -0
- dinkster_schema/coerce.py +186 -0
- dinkster_schema/comfy_alias.py +496 -0
- dinkster_schema/comfy_group.py +768 -0
- dinkster_schema/elaborate.py +676 -0
- dinkster_schema/execution.py +24 -0
- dinkster_schema/expansion.py +123 -0
- dinkster_schema/log.py +206 -0
- dinkster_schema/media.py +77 -0
- dinkster_schema/model.py +2920 -0
- dinkster_schema/names.py +25 -0
- dinkster_schema/node.py +137 -0
- dinkster_schema/output_descriptors.py +63 -0
- dinkster_schema/py.typed +0 -0
- dinkster_schema/replace.py +1853 -0
- dinkster_schema/reporting.py +441 -0
- dinkster_schema/solve.py +178 -0
- dinkster_schema/wire.py +1876 -0
- dinkster_values/__init__.py +625 -0
- dinkster_values/_media_array.py +7 -0
- dinkster_values/absent.py +77 -0
- dinkster_values/asset_folders.py +64 -0
- dinkster_values/assets.py +106 -0
- dinkster_values/audio_codec.py +594 -0
- dinkster_values/audio_lazy.py +871 -0
- dinkster_values/builtin_types.py +181 -0
- dinkster_values/curve.py +260 -0
- dinkster_values/detection_codec.py +286 -0
- dinkster_values/image_codec.py +691 -0
- dinkster_values/image_components.py +219 -0
- dinkster_values/latent_codec.py +514 -0
- dinkster_values/limits.py +52 -0
- dinkster_values/lists.py +216 -0
- dinkster_values/media_containers.py +70 -0
- dinkster_values/model.py +417 -0
- dinkster_values/model3d_codec.py +167 -0
- dinkster_values/names.py +168 -0
- dinkster_values/pins.py +77 -0
- dinkster_values/py.typed +0 -0
- dinkster_values/registry.py +1105 -0
- dinkster_values/resident.py +210 -0
- dinkster_values/resources.py +210 -0
- dinkster_values/splat_codec.py +477 -0
- dinkster_values/storage.py +133 -0
- dinkster_values/streams.py +241 -0
- dinkster_values/svg_codec.py +120 -0
- dinkster_values/timeline_video.py +68 -0
- dinkster_values/tuples.py +61 -0
- dinkster_values/video_codec.py +897 -0
- dinkster_values/video_document.py +315 -0
- dinkster_values/video_edits.py +253 -0
- dinkster_values/video_probe.py +221 -0
- dinkster_video/__init__.py +58 -0
- dinkster_video/document.py +352 -0
- dinkster_video/formats.py +722 -0
- dinkster_video/image_math.py +175 -0
- dinkster_video/preview/dinkster-pack.toml +36 -0
- dinkster_video/preview/src/dinkster_video_preview/__init__.py +84 -0
- dinkster_video/preview/src/dinkster_video_preview/preview.js +45 -0
- dinkster_video/py.typed +1 -0
- dinkster_video/runtime.py +1561 -0
- dinkster_video/timeline.py +576 -0
- dinkster_video/timeline_runtime.py +305 -0
- dinkster_video_preview/__init__.py +84 -0
- dinkster_video_preview/preview.js +45 -0
- dinkster_workers/__init__.py +207 -0
- dinkster_workers/_doctor_probe.py +359 -0
- dinkster_workers/accelerator.py +156 -0
- dinkster_workers/aimdo_bootstrap.py +81 -0
- dinkster_workers/backend_env.py +1485 -0
- dinkster_workers/blobs.py +402 -0
- dinkster_workers/boundary.py +1918 -0
- dinkster_workers/catalog.py +182 -0
- dinkster_workers/devices.py +164 -0
- dinkster_workers/diagnostics.py +64 -0
- dinkster_workers/dispatch.py +298 -0
- dinkster_workers/doctor.py +1382 -0
- dinkster_workers/egress.py +344 -0
- dinkster_workers/error_hints.py +175 -0
- dinkster_workers/execution.py +160 -0
- dinkster_workers/governed.py +123 -0
- dinkster_workers/headroom.py +156 -0
- dinkster_workers/host.py +3116 -0
- dinkster_workers/in_process.py +1325 -0
- dinkster_workers/interpreter.py +84 -0
- dinkster_workers/isolated.py +862 -0
- dinkster_workers/launch.py +72 -0
- dinkster_workers/manifest.py +3332 -0
- dinkster_workers/media.py +196 -0
- dinkster_workers/pack_environment.py +473 -0
- dinkster_workers/pack_lock.py +359 -0
- dinkster_workers/placement.py +215 -0
- dinkster_workers/preview_artifacts.py +233 -0
- dinkster_workers/produced_assets.py +218 -0
- dinkster_workers/provision.py +597 -0
- dinkster_workers/py.typed +0 -0
- dinkster_workers/relay.py +762 -0
- dinkster_workers/remote.py +921 -0
- dinkster_workers/resume.py +490 -0
- dinkster_workers/routing.py +134 -0
- dinkster_workers/run_control.py +275 -0
- dinkster_workers/sandbox.py +798 -0
- dinkster_workers/saved_artifacts.py +275 -0
- dinkster_workers/service.py +593 -0
- dinkster_workers/session.py +2605 -0
- dinkster_workers/single_job_multi_gpu.py +7 -0
- dinkster_workers/staging.py +230 -0
- dinkster_workers/transport.py +161 -0
- dinkster_workers/workgroup.py +729 -0
- dinkster_workers/workgroup_compose.py +135 -0
- dinkster_workers/workgroup_config.py +196 -0
- dinkster_workers/workgroup_session.py +340 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Asset identity: a canonical content digest (DESIGN 3.12).
|
|
2
|
+
|
|
3
|
+
Identity is a property of bytes - it survives renames, moves, duplicate
|
|
4
|
+
copies, and machine hops. The canonical form is ``blake3:<64 hex>``,
|
|
5
|
+
deliberately the same form ComfyUI's asset database uses, so digests
|
|
6
|
+
interoperate with hashes the existing ecosystem already computed.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import TYPE_CHECKING
|
|
14
|
+
|
|
15
|
+
from dinkster_values import MEBIBYTE
|
|
16
|
+
|
|
17
|
+
if TYPE_CHECKING:
|
|
18
|
+
from blake3 import blake3
|
|
19
|
+
|
|
20
|
+
DIGEST_PREFIX = "blake3:"
|
|
21
|
+
_DIGEST_RE = re.compile(r"^blake3:[0-9a-f]{64}$")
|
|
22
|
+
|
|
23
|
+
CHUNK_SIZE = 8 * MEBIBYTE
|
|
24
|
+
"""Hash files in 8 MiB chunks (matches ComfyUI's scanner) - bounded memory
|
|
25
|
+
regardless of file size."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class AssetError(Exception):
|
|
29
|
+
"""An asset could not be identified, cataloged, or materialized."""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def is_digest(text: str) -> bool:
|
|
33
|
+
return _DIGEST_RE.match(text) is not None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def require_digest(text: str) -> str:
|
|
37
|
+
if not is_digest(text):
|
|
38
|
+
raise AssetError(f"not a canonical asset digest (expected 'blake3:<64 hex>'): {text!r}")
|
|
39
|
+
return text
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _blake3() -> type[blake3]:
|
|
43
|
+
"""Lazy: hashing happens where Dinkster proper runs (blake3 installed);
|
|
44
|
+
workers that only *resolve* digests - compat children riding a foreign
|
|
45
|
+
interpreter via PYTHONPATH - never need it."""
|
|
46
|
+
try:
|
|
47
|
+
from blake3 import blake3 as hasher
|
|
48
|
+
except ImportError: # pragma: no cover - environment-specific
|
|
49
|
+
raise AssetError(
|
|
50
|
+
"computing asset digests requires the 'blake3' package, which "
|
|
51
|
+
"is not installed in this interpreter; compute digests where "
|
|
52
|
+
"Dinkster is installed (workers resolve pre-indexed digests only)"
|
|
53
|
+
) from None
|
|
54
|
+
return hasher
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def new_hasher() -> blake3:
|
|
58
|
+
"""A fresh incremental hasher for streaming digest computation (vault
|
|
59
|
+
ingest, resumable scans). Same lazy-import posture as digest_file."""
|
|
60
|
+
return _blake3()()
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def digest_bytes(data: bytes) -> str:
|
|
64
|
+
return DIGEST_PREFIX + _blake3()(data).hexdigest()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def digest_file(path: Path) -> str:
|
|
68
|
+
hasher = _blake3()()
|
|
69
|
+
with path.open("rb") as handle:
|
|
70
|
+
while chunk := handle.read(CHUNK_SIZE):
|
|
71
|
+
hasher.update(chunk)
|
|
72
|
+
return DIGEST_PREFIX + hasher.hexdigest()
|
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
"""Descriptor-bound verification for content-addressed asset reads.
|
|
2
|
+
|
|
3
|
+
Ingest records bind a digest to stable descriptor metadata. Loaders can trust
|
|
4
|
+
that record without rereading the payload when the opened descriptor still
|
|
5
|
+
matches it. Missing records retain full descriptor-bound hashing, and stale
|
|
6
|
+
records fail closed.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import os
|
|
12
|
+
import threading
|
|
13
|
+
from collections import OrderedDict
|
|
14
|
+
from collections.abc import Mapping
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import BinaryIO, Literal, cast
|
|
18
|
+
|
|
19
|
+
from .identity import CHUNK_SIZE, DIGEST_PREFIX, AssetError, new_hasher
|
|
20
|
+
|
|
21
|
+
IntegrityFailure = Literal[
|
|
22
|
+
"digest_mismatch",
|
|
23
|
+
"changed_during_verification",
|
|
24
|
+
"path_rebound",
|
|
25
|
+
"stale_ingest_record",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class AssetIntegrityError(AssetError):
|
|
30
|
+
"""The bytes at a resolved path do not (or can no longer be proven to)
|
|
31
|
+
match the digest that named them. Carries the expected digest, the
|
|
32
|
+
actual digest when a hash completed (None when the file changed under
|
|
33
|
+
verification or the path stopped naming the verified inode), and the
|
|
34
|
+
offending path - for logs; public surfaces should not echo the path."""
|
|
35
|
+
|
|
36
|
+
def __init__(
|
|
37
|
+
self,
|
|
38
|
+
expected_digest: str,
|
|
39
|
+
actual_digest: str | None,
|
|
40
|
+
path: Path,
|
|
41
|
+
reason: IntegrityFailure,
|
|
42
|
+
) -> None:
|
|
43
|
+
self.expected_digest = expected_digest
|
|
44
|
+
self.actual_digest = actual_digest
|
|
45
|
+
self.path = path
|
|
46
|
+
self.reason = reason
|
|
47
|
+
detail = {
|
|
48
|
+
"digest_mismatch": f"content hashes to {actual_digest}",
|
|
49
|
+
"changed_during_verification": "the file changed while being hashed",
|
|
50
|
+
"path_rebound": "the path stopped naming the verified file",
|
|
51
|
+
"stale_ingest_record": "the file metadata differs from its ingest record",
|
|
52
|
+
}[reason]
|
|
53
|
+
super().__init__(
|
|
54
|
+
f"asset integrity failure ({reason}): {path} does not hold the "
|
|
55
|
+
f"bytes named by {expected_digest} - {detail}"
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
_Fingerprint = tuple[int, int, int, int, int]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True)
|
|
63
|
+
class AssetVerificationRecord:
|
|
64
|
+
"""Descriptor metadata captured by a content-verified ingest."""
|
|
65
|
+
|
|
66
|
+
digest: str
|
|
67
|
+
scheme: str
|
|
68
|
+
device: int
|
|
69
|
+
inode: int
|
|
70
|
+
size: int
|
|
71
|
+
mtime_ns: int
|
|
72
|
+
ctime_ns: int | None
|
|
73
|
+
|
|
74
|
+
def matches(self, stat: os.stat_result) -> bool:
|
|
75
|
+
if self.scheme != _stat_scheme():
|
|
76
|
+
return False
|
|
77
|
+
actual = (stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns)
|
|
78
|
+
expected = (self.device, self.inode, self.size, self.mtime_ns)
|
|
79
|
+
return actual == expected and (self.ctime_ns is None or stat.st_ctime_ns == self.ctime_ns)
|
|
80
|
+
|
|
81
|
+
def to_json(self) -> dict[str, object]:
|
|
82
|
+
return {
|
|
83
|
+
"digest": self.digest,
|
|
84
|
+
"scheme": self.scheme,
|
|
85
|
+
"device": self.device,
|
|
86
|
+
"inode": self.inode,
|
|
87
|
+
"size": self.size,
|
|
88
|
+
"mtimeNs": self.mtime_ns,
|
|
89
|
+
**({"ctimeNs": self.ctime_ns} if self.ctime_ns is not None else {}),
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
@classmethod
|
|
93
|
+
def from_json(cls, digest: str, value: object) -> AssetVerificationRecord | None:
|
|
94
|
+
if not isinstance(value, Mapping):
|
|
95
|
+
return None
|
|
96
|
+
row = cast("Mapping[str, object]", value)
|
|
97
|
+
recorded_digest = row.get("digest")
|
|
98
|
+
scheme = row.get("scheme")
|
|
99
|
+
device = row.get("device")
|
|
100
|
+
inode = row.get("inode")
|
|
101
|
+
size = row.get("size")
|
|
102
|
+
mtime_ns = row.get("mtimeNs")
|
|
103
|
+
ctime_ns = row.get("ctimeNs")
|
|
104
|
+
if (
|
|
105
|
+
recorded_digest != digest
|
|
106
|
+
or scheme not in ("posix-v1", "windows-v1")
|
|
107
|
+
or not all(isinstance(field, int) for field in (device, inode, size, mtime_ns))
|
|
108
|
+
or (ctime_ns is not None and not isinstance(ctime_ns, int))
|
|
109
|
+
):
|
|
110
|
+
return None
|
|
111
|
+
if scheme == "posix-v1" and not isinstance(ctime_ns, int):
|
|
112
|
+
return None
|
|
113
|
+
return cls(
|
|
114
|
+
digest,
|
|
115
|
+
cast("str", scheme),
|
|
116
|
+
cast("int", device),
|
|
117
|
+
cast("int", inode),
|
|
118
|
+
cast("int", size),
|
|
119
|
+
cast("int", mtime_ns),
|
|
120
|
+
ctime_ns,
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _stat_scheme() -> str | None:
|
|
125
|
+
if os.name == "posix":
|
|
126
|
+
return "posix-v1"
|
|
127
|
+
if os.name == "nt":
|
|
128
|
+
return "windows-v1"
|
|
129
|
+
return None
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def verification_record(digest: str, stat: os.stat_result) -> AssetVerificationRecord | None:
|
|
133
|
+
"""Create a load-time proof only where device/inode identity is usable."""
|
|
134
|
+
scheme = _stat_scheme()
|
|
135
|
+
if scheme is None or stat.st_dev == 0 or stat.st_ino == 0:
|
|
136
|
+
return None
|
|
137
|
+
return AssetVerificationRecord(
|
|
138
|
+
digest,
|
|
139
|
+
scheme,
|
|
140
|
+
stat.st_dev,
|
|
141
|
+
stat.st_ino,
|
|
142
|
+
stat.st_size,
|
|
143
|
+
stat.st_mtime_ns,
|
|
144
|
+
stat.st_ctime_ns if scheme == "posix-v1" else None,
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def verification_record_for_publication(
|
|
149
|
+
digest: str,
|
|
150
|
+
verified: os.stat_result,
|
|
151
|
+
published: os.stat_result,
|
|
152
|
+
) -> AssetVerificationRecord | None:
|
|
153
|
+
"""Bind verified temporary bytes to their atomically published name."""
|
|
154
|
+
verified_identity = (verified.st_dev, verified.st_ino)
|
|
155
|
+
published_identity = (published.st_dev, published.st_ino)
|
|
156
|
+
identities_stable = all(verified_identity) and all(published_identity)
|
|
157
|
+
if identities_stable and verified_identity != published_identity:
|
|
158
|
+
raise AssetError("asset path was rebound while it was being published")
|
|
159
|
+
if (verified.st_size, verified.st_mtime_ns) != (published.st_size, published.st_mtime_ns):
|
|
160
|
+
raise AssetError("asset changed while it was being published")
|
|
161
|
+
if not identities_stable:
|
|
162
|
+
return None
|
|
163
|
+
return verification_record(digest, published)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
_CACHE_LIMIT = 4096
|
|
167
|
+
"""Verified fingerprints retained per process. Bounds memory and shrinks
|
|
168
|
+
the window in which a recycled inode number could meet a matching stale
|
|
169
|
+
entry; real libraries hold thousands of assets, not millions."""
|
|
170
|
+
|
|
171
|
+
_TRUST_FINGERPRINTS = os.name == "posix"
|
|
172
|
+
"""Whether a matching fingerprint may skip the rehash. Requires st_ctime_ns
|
|
173
|
+
to be a change stamp (POSIX); on Windows it is creation time, which an
|
|
174
|
+
in-place rewrite with a restored mtime would not move."""
|
|
175
|
+
|
|
176
|
+
_verified: OrderedDict[_Fingerprint, str] = OrderedDict()
|
|
177
|
+
_verified_lock = threading.Lock()
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _reset_after_fork() -> None:
|
|
181
|
+
# A forked child inherits the dict but not the right to trust it being
|
|
182
|
+
# in sync with whatever the parent does next - and possibly a lock a
|
|
183
|
+
# now-nonexistent parent thread held. Fresh lock, empty cache.
|
|
184
|
+
global _verified_lock
|
|
185
|
+
_verified_lock = threading.Lock()
|
|
186
|
+
_verified.clear()
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
register_at_fork = getattr(os, "register_at_fork", None)
|
|
190
|
+
if register_at_fork is not None: # pragma: no branch - POSIX
|
|
191
|
+
register_at_fork(after_in_child=_reset_after_fork)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def clear_verified_cache() -> None:
|
|
195
|
+
"""Forget every recorded verification (tests; long-lived hosts that
|
|
196
|
+
want to force a re-hash)."""
|
|
197
|
+
with _verified_lock:
|
|
198
|
+
_verified.clear()
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _fingerprint(stat: os.stat_result) -> _Fingerprint:
|
|
202
|
+
return (stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _path_binding_fingerprint(fingerprint: _Fingerprint) -> tuple[int, ...]:
|
|
206
|
+
# Windows creation time can differ between stat() and fstat() for the same file.
|
|
207
|
+
return fingerprint[:4] if os.name == "nt" else fingerprint
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _hash_handle(handle: BinaryIO) -> str:
|
|
211
|
+
"""Hash the handle's content from position 0. A seam on purpose:
|
|
212
|
+
integrity tests monkeypatch this to count hashes and to interleave
|
|
213
|
+
replacements mid-verification."""
|
|
214
|
+
handle.seek(0)
|
|
215
|
+
hasher = new_hasher()
|
|
216
|
+
while chunk := handle.read(CHUNK_SIZE):
|
|
217
|
+
hasher.update(chunk)
|
|
218
|
+
return DIGEST_PREFIX + hasher.hexdigest()
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def digest_file_with_record(
|
|
222
|
+
path: Path,
|
|
223
|
+
) -> tuple[str, AssetVerificationRecord | None]:
|
|
224
|
+
"""Hash one ingest candidate and bind its digest to the same descriptor."""
|
|
225
|
+
with path.open("rb") as handle:
|
|
226
|
+
before = _fingerprint(os.fstat(handle.fileno()))
|
|
227
|
+
digest = _hash_handle(handle)
|
|
228
|
+
after_stat = os.fstat(handle.fileno())
|
|
229
|
+
if _fingerprint(after_stat) != before:
|
|
230
|
+
raise AssetError(f"asset changed while being ingested: {path}")
|
|
231
|
+
return digest, verification_record(digest, after_stat)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def open_verified(
|
|
235
|
+
path: Path,
|
|
236
|
+
expected_digest: str,
|
|
237
|
+
record: AssetVerificationRecord | None = None,
|
|
238
|
+
) -> BinaryIO:
|
|
239
|
+
"""Open ``path`` and return a handle positioned at 0 whose bytes are
|
|
240
|
+
proven to hash to ``expected_digest``.
|
|
241
|
+
|
|
242
|
+
The proof and the returned handle share one file descriptor, so a
|
|
243
|
+
rename/replace after (or during) verification cannot swap the bytes:
|
|
244
|
+
the descriptor keeps the verified inode. Raises AssetIntegrityError
|
|
245
|
+
on any mismatch; raises AssetError when hashing is unavailable in
|
|
246
|
+
this interpreter (missing blake3) - never a silent downgrade."""
|
|
247
|
+
handle = path.open("rb")
|
|
248
|
+
try:
|
|
249
|
+
opened_stat = os.fstat(handle.fileno())
|
|
250
|
+
before = _fingerprint(opened_stat)
|
|
251
|
+
if record is not None:
|
|
252
|
+
if record.digest != expected_digest or not record.matches(opened_stat):
|
|
253
|
+
raise AssetIntegrityError(expected_digest, None, path, "stale_ingest_record")
|
|
254
|
+
return handle
|
|
255
|
+
actual: str | None = None
|
|
256
|
+
if _TRUST_FINGERPRINTS:
|
|
257
|
+
with _verified_lock:
|
|
258
|
+
actual = _verified.get(before)
|
|
259
|
+
if actual is not None:
|
|
260
|
+
_verified.move_to_end(before)
|
|
261
|
+
if actual is None:
|
|
262
|
+
actual = _hash_handle(handle)
|
|
263
|
+
after = _fingerprint(os.fstat(handle.fileno()))
|
|
264
|
+
if after != before:
|
|
265
|
+
raise AssetIntegrityError(
|
|
266
|
+
expected_digest, None, path, "changed_during_verification"
|
|
267
|
+
)
|
|
268
|
+
if _TRUST_FINGERPRINTS:
|
|
269
|
+
# Cache what the content IS (its proven hash), not what the
|
|
270
|
+
# caller hoped for: a later open with the honest digest may
|
|
271
|
+
# hit; one with a different digest still compares and fails.
|
|
272
|
+
with _verified_lock:
|
|
273
|
+
_verified[after] = actual
|
|
274
|
+
_verified.move_to_end(after)
|
|
275
|
+
while len(_verified) > _CACHE_LIMIT:
|
|
276
|
+
_verified.popitem(last=False)
|
|
277
|
+
if actual != expected_digest:
|
|
278
|
+
raise AssetIntegrityError(expected_digest, actual, path, "digest_mismatch")
|
|
279
|
+
handle.seek(0)
|
|
280
|
+
return handle
|
|
281
|
+
except BaseException:
|
|
282
|
+
handle.close()
|
|
283
|
+
raise
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def verified_local_path(
|
|
287
|
+
path: Path,
|
|
288
|
+
expected_digest: str,
|
|
289
|
+
record: AssetVerificationRecord | None = None,
|
|
290
|
+
) -> Path:
|
|
291
|
+
"""Verify ``path``'s content against ``expected_digest`` and return the
|
|
292
|
+
path, for consumers that need a real filesystem name (checkpoint
|
|
293
|
+
loaders). Weaker than :func:`open_verified` by nature: after the final
|
|
294
|
+
check the name can be rebound, so the guarantee is "the path named the
|
|
295
|
+
verified bytes at return time", with the window shrunk from unbounded
|
|
296
|
+
to the instant of return. Prefer open_verified where a handle works."""
|
|
297
|
+
verified = (
|
|
298
|
+
open_verified(path, expected_digest)
|
|
299
|
+
if record is None
|
|
300
|
+
else open_verified(path, expected_digest, record)
|
|
301
|
+
)
|
|
302
|
+
with verified as handle:
|
|
303
|
+
fd_print = _fingerprint(os.fstat(handle.fileno()))
|
|
304
|
+
try:
|
|
305
|
+
path_print = _fingerprint(os.stat(path))
|
|
306
|
+
except OSError:
|
|
307
|
+
raise AssetIntegrityError(expected_digest, None, path, "path_rebound") from None
|
|
308
|
+
if _path_binding_fingerprint(path_print) != _path_binding_fingerprint(fd_print):
|
|
309
|
+
raise AssetIntegrityError(expected_digest, None, path, "path_rebound")
|
|
310
|
+
return path
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Immutable validated JSON metadata values."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from collections.abc import Mapping, Sequence
|
|
7
|
+
from types import MappingProxyType
|
|
8
|
+
from typing import cast
|
|
9
|
+
|
|
10
|
+
from .identity import AssetError
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def freeze_json(value: object, path: str = "metadata") -> object:
|
|
14
|
+
if value is None or isinstance(value, (str, bool, int)):
|
|
15
|
+
return value
|
|
16
|
+
if isinstance(value, float):
|
|
17
|
+
if not math.isfinite(value):
|
|
18
|
+
raise AssetError(f"{path} must contain finite JSON numbers")
|
|
19
|
+
return value
|
|
20
|
+
if isinstance(value, Mapping):
|
|
21
|
+
frozen: dict[str, object] = {}
|
|
22
|
+
for key, item in cast("Mapping[object, object]", value).items():
|
|
23
|
+
if not isinstance(key, str):
|
|
24
|
+
raise AssetError(f"{path} keys must be strings")
|
|
25
|
+
frozen[key] = freeze_json(item, f"{path}.{key}")
|
|
26
|
+
return MappingProxyType(dict(sorted(frozen.items())))
|
|
27
|
+
if isinstance(value, (list, tuple)):
|
|
28
|
+
sequence = cast("Sequence[object]", value)
|
|
29
|
+
return tuple(freeze_json(item, f"{path}[]") for item in sequence)
|
|
30
|
+
raise AssetError(f"{path} must contain only JSON values")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def thaw_json(value: object) -> object:
|
|
34
|
+
if isinstance(value, Mapping):
|
|
35
|
+
return {
|
|
36
|
+
str(key): thaw_json(item)
|
|
37
|
+
for key, item in cast("Mapping[object, object]", value).items()
|
|
38
|
+
}
|
|
39
|
+
if isinstance(value, tuple):
|
|
40
|
+
return [thaw_json(item) for item in cast("tuple[object, ...]", value)]
|
|
41
|
+
return value
|
dinkster_assets/kind.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Asset kinds: an open, namespaced vocabulary (DESIGN 3.12 + roadmap
|
|
2
|
+
"templates/asset distribution").
|
|
3
|
+
|
|
4
|
+
A kind says what an asset *is for* - the axis pickers filter on and
|
|
5
|
+
substitution UX groups by - without repeating ComfyUI's folder taxonomy
|
|
6
|
+
(where "kind" was literally which directory a file sat in, so a new model
|
|
7
|
+
family meant patching folder_paths). Kinds are plain namespaced strings,
|
|
8
|
+
``namespace/name`` with optional deeper segments: ``media/image``,
|
|
9
|
+
``model/lora``, ``model/diffusion``. The vocabulary is OPEN - packs and
|
|
10
|
+
templates may mint kinds freely - while the grammar is closed, so kinds
|
|
11
|
+
sort, compare, and prefix-filter mechanically forever.
|
|
12
|
+
|
|
13
|
+
Kind is presentation/index metadata, never execution identity: it joins
|
|
14
|
+
widgets, needs, and catalogs, and it never joins schema signatures or
|
|
15
|
+
cache keys.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import re
|
|
21
|
+
|
|
22
|
+
from .identity import AssetError
|
|
23
|
+
|
|
24
|
+
_KIND_RE = re.compile(r"^[a-z0-9][a-z0-9-]*(/[a-z0-9][a-z0-9-]*)+$")
|
|
25
|
+
|
|
26
|
+
# Conventional core kinds. A convention, not an enum: validation accepts
|
|
27
|
+
# any grammatical kind, and unknown kinds are first-class everywhere.
|
|
28
|
+
KIND_MEDIA_IMAGE = "media/image"
|
|
29
|
+
KIND_MEDIA_VIDEO = "media/video"
|
|
30
|
+
KIND_MEDIA_AUDIO = "media/audio"
|
|
31
|
+
KIND_MEDIA_MODEL3D = "media/model3d"
|
|
32
|
+
KIND_MODEL_CHECKPOINT = "model/checkpoint"
|
|
33
|
+
KIND_MODEL_DIFFUSION = "model/diffusion"
|
|
34
|
+
KIND_MODEL_LORA = "model/lora"
|
|
35
|
+
KIND_MODEL_VAE = "model/vae"
|
|
36
|
+
KIND_MODEL_TEXT_ENCODER = "model/text-encoder"
|
|
37
|
+
KIND_MODEL_EMBEDDING = "model/embedding"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def is_asset_kind(text: str) -> bool:
|
|
41
|
+
return _KIND_RE.match(text) is not None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def require_asset_kind(text: str) -> str:
|
|
45
|
+
"""Validate a kind string: lowercase alphanumeric/hyphen segments,
|
|
46
|
+
'/'-separated, at least two segments (a bare word is not namespaced)."""
|
|
47
|
+
if not is_asset_kind(text):
|
|
48
|
+
raise AssetError(
|
|
49
|
+
"not a valid asset kind (expected 'namespace/name' with "
|
|
50
|
+
f"lowercase [a-z0-9-] segments): {text!r}"
|
|
51
|
+
)
|
|
52
|
+
return text
|