dinkster-api 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dinkster_api/__init__.py +9 -0
- dinkster_api/py.typed +0 -0
- dinkster_api/v1.py +935 -0
- dinkster_api-0.0.2.dist-info/METADATA +16 -0
- dinkster_api-0.0.2.dist-info/RECORD +238 -0
- dinkster_api-0.0.2.dist-info/WHEEL +4 -0
- dinkster_api-0.0.2.dist-info/entry_points.txt +2 -0
- dinkster_assets/__init__.py +337 -0
- dinkster_assets/acquire.py +226 -0
- dinkster_assets/acquisition_plan.py +145 -0
- dinkster_assets/audio.py +119 -0
- dinkster_assets/catalog.py +156 -0
- dinkster_assets/component_manifest.py +110 -0
- dinkster_assets/declared.py +218 -0
- dinkster_assets/fetch.py +495 -0
- dinkster_assets/guess.py +136 -0
- dinkster_assets/identity.py +72 -0
- dinkster_assets/integrity.py +310 -0
- dinkster_assets/json_metadata.py +41 -0
- dinkster_assets/kind.py +52 -0
- dinkster_assets/latent_format.py +346 -0
- dinkster_assets/library.py +816 -0
- dinkster_assets/media.py +1385 -0
- dinkster_assets/migrations/env.py +38 -0
- dinkster_assets/migrations/script.py.mako +28 -0
- dinkster_assets/migrations/versions/0001_asset_records_create_asset_record_catalog.py +109 -0
- dinkster_assets/migrations/versions/0002_asset_observations.py +36 -0
- dinkster_assets/model.py +242 -0
- dinkster_assets/mounts.py +1278 -0
- dinkster_assets/need.py +168 -0
- dinkster_assets/pack_assets.py +217 -0
- dinkster_assets/probe.py +118 -0
- dinkster_assets/provenance.py +254 -0
- dinkster_assets/public_acquisition.py +424 -0
- dinkster_assets/py.typed +0 -0
- dinkster_assets/records.py +899 -0
- dinkster_assets/resolution.py +778 -0
- dinkster_assets/resolver_index.py +364 -0
- dinkster_assets/resolver_subscription.py +1149 -0
- dinkster_assets/save_target.py +139 -0
- dinkster_assets/schema.py +73 -0
- dinkster_assets/selections.py +256 -0
- dinkster_assets/transport.py +243 -0
- dinkster_assets/value.py +258 -0
- dinkster_assets/vault.py +276 -0
- dinkster_assets/writer.py +765 -0
- dinkster_caches/__init__.py +19 -0
- dinkster_caches/cas.py +310 -0
- dinkster_caches/disk.py +330 -0
- dinkster_caches/layered.py +150 -0
- dinkster_caches/memory.py +208 -0
- dinkster_caches/py.typed +0 -0
- dinkster_caches/wire.py +346 -0
- dinkster_image_document/__init__.py +39 -0
- dinkster_image_document/blend.py +264 -0
- dinkster_image_document/compat.py +211 -0
- dinkster_image_document/document.py +374 -0
- dinkster_image_document/format.py +569 -0
- dinkster_image_document/interchange.py +572 -0
- dinkster_image_document/py.typed +1 -0
- dinkster_image_document/render.py +455 -0
- dinkster_inference_wire/__init__.py +141 -0
- dinkster_inference_wire/camera_wire.py +127 -0
- dinkster_inference_wire/conditioning.py +752 -0
- dinkster_inference_wire/conditioning_wire.py +1143 -0
- dinkster_inference_wire/devices.py +285 -0
- dinkster_inference_wire/generation.py +446 -0
- dinkster_inference_wire/hook_values.py +168 -0
- dinkster_inference_wire/ic_lora_parameters.py +69 -0
- dinkster_inference_wire/latents.py +244 -0
- dinkster_inference_wire/media_wire.py +169 -0
- dinkster_inference_wire/openai_generation.py +1035 -0
- dinkster_inference_wire/pose_keypoint.py +83 -0
- dinkster_inference_wire/process.py +33 -0
- dinkster_inference_wire/py.typed +0 -0
- dinkster_inference_wire/registry.py +116 -0
- dinkster_inference_wire/sam3_wire.py +68 -0
- dinkster_inference_wire/sampling_wire.py +279 -0
- dinkster_inference_wire/sources.py +399 -0
- dinkster_inference_wire/spaces.py +344 -0
- dinkster_inference_wire/sparse.py +372 -0
- dinkster_inference_wire/splat_wire.py +23 -0
- dinkster_inference_wire/types.py +256 -0
- dinkster_inference_wire/vision_wire.py +53 -0
- dinkster_inference_wire/voxel_wire.py +72 -0
- dinkster_inference_wire/wan_wire.py +93 -0
- dinkster_inference_wire/weights.py +219 -0
- dinkster_memory/__init__.py +92 -0
- dinkster_memory/accelerator.py +96 -0
- dinkster_memory/budgets.py +96 -0
- dinkster_memory/details.py +91 -0
- dinkster_memory/governor.py +554 -0
- dinkster_memory/leases.py +176 -0
- dinkster_memory/py.typed +0 -0
- dinkster_memory/release.py +131 -0
- dinkster_memory/reported.py +101 -0
- dinkster_memory/reservations.py +101 -0
- dinkster_memory/resident.py +600 -0
- dinkster_memory/system.py +614 -0
- dinkster_memory/tenants.py +67 -0
- dinkster_protocol/__init__.py +1008 -0
- dinkster_protocol/attention.py +652 -0
- dinkster_protocol/extension_contribution_kinds.json +21 -0
- dinkster_protocol/extension_contribution_kinds.py +43 -0
- dinkster_protocol/extensions.py +742 -0
- dinkster_protocol/frontend_modules.py +145 -0
- dinkster_protocol/multigpu.py +107 -0
- dinkster_protocol/pack_surfaces.py +489 -0
- dinkster_protocol/payload.py +63 -0
- dinkster_protocol/preview.py +86 -0
- dinkster_protocol/py.typed +0 -0
- dinkster_protocol/result_algebra.py +380 -0
- dinkster_protocol/result_document.py +244 -0
- dinkster_protocol/run_control.py +96 -0
- dinkster_protocol/training.py +399 -0
- dinkster_protocol/workgroup.py +1014 -0
- dinkster_registry/__init__.py +135 -0
- dinkster_registry/admission.py +372 -0
- dinkster_registry/artifact.py +128 -0
- dinkster_registry/composition.py +152 -0
- dinkster_registry/grants.py +206 -0
- dinkster_registry/install.py +1017 -0
- dinkster_registry/model.py +414 -0
- dinkster_registry/principals.py +494 -0
- dinkster_registry/py.typed +0 -0
- dinkster_schema/__init__.py +347 -0
- dinkster_schema/capture.py +240 -0
- dinkster_schema/coerce.py +186 -0
- dinkster_schema/comfy_alias.py +496 -0
- dinkster_schema/comfy_group.py +768 -0
- dinkster_schema/elaborate.py +676 -0
- dinkster_schema/execution.py +24 -0
- dinkster_schema/expansion.py +123 -0
- dinkster_schema/log.py +206 -0
- dinkster_schema/media.py +77 -0
- dinkster_schema/model.py +2920 -0
- dinkster_schema/names.py +25 -0
- dinkster_schema/node.py +137 -0
- dinkster_schema/output_descriptors.py +63 -0
- dinkster_schema/py.typed +0 -0
- dinkster_schema/replace.py +1853 -0
- dinkster_schema/reporting.py +441 -0
- dinkster_schema/solve.py +178 -0
- dinkster_schema/wire.py +1876 -0
- dinkster_values/__init__.py +625 -0
- dinkster_values/_media_array.py +7 -0
- dinkster_values/absent.py +77 -0
- dinkster_values/asset_folders.py +64 -0
- dinkster_values/assets.py +106 -0
- dinkster_values/audio_codec.py +594 -0
- dinkster_values/audio_lazy.py +871 -0
- dinkster_values/builtin_types.py +181 -0
- dinkster_values/curve.py +260 -0
- dinkster_values/detection_codec.py +286 -0
- dinkster_values/image_codec.py +691 -0
- dinkster_values/image_components.py +219 -0
- dinkster_values/latent_codec.py +514 -0
- dinkster_values/limits.py +52 -0
- dinkster_values/lists.py +216 -0
- dinkster_values/media_containers.py +70 -0
- dinkster_values/model.py +417 -0
- dinkster_values/model3d_codec.py +167 -0
- dinkster_values/names.py +168 -0
- dinkster_values/pins.py +77 -0
- dinkster_values/py.typed +0 -0
- dinkster_values/registry.py +1105 -0
- dinkster_values/resident.py +210 -0
- dinkster_values/resources.py +210 -0
- dinkster_values/splat_codec.py +477 -0
- dinkster_values/storage.py +133 -0
- dinkster_values/streams.py +241 -0
- dinkster_values/svg_codec.py +120 -0
- dinkster_values/timeline_video.py +68 -0
- dinkster_values/tuples.py +61 -0
- dinkster_values/video_codec.py +897 -0
- dinkster_values/video_document.py +315 -0
- dinkster_values/video_edits.py +253 -0
- dinkster_values/video_probe.py +221 -0
- dinkster_video/__init__.py +58 -0
- dinkster_video/document.py +352 -0
- dinkster_video/formats.py +722 -0
- dinkster_video/image_math.py +175 -0
- dinkster_video/preview/dinkster-pack.toml +36 -0
- dinkster_video/preview/src/dinkster_video_preview/__init__.py +84 -0
- dinkster_video/preview/src/dinkster_video_preview/preview.js +45 -0
- dinkster_video/py.typed +1 -0
- dinkster_video/runtime.py +1561 -0
- dinkster_video/timeline.py +576 -0
- dinkster_video/timeline_runtime.py +305 -0
- dinkster_video_preview/__init__.py +84 -0
- dinkster_video_preview/preview.js +45 -0
- dinkster_workers/__init__.py +207 -0
- dinkster_workers/_doctor_probe.py +359 -0
- dinkster_workers/accelerator.py +156 -0
- dinkster_workers/aimdo_bootstrap.py +81 -0
- dinkster_workers/backend_env.py +1485 -0
- dinkster_workers/blobs.py +402 -0
- dinkster_workers/boundary.py +1918 -0
- dinkster_workers/catalog.py +182 -0
- dinkster_workers/devices.py +164 -0
- dinkster_workers/diagnostics.py +64 -0
- dinkster_workers/dispatch.py +298 -0
- dinkster_workers/doctor.py +1382 -0
- dinkster_workers/egress.py +344 -0
- dinkster_workers/error_hints.py +175 -0
- dinkster_workers/execution.py +160 -0
- dinkster_workers/governed.py +123 -0
- dinkster_workers/headroom.py +156 -0
- dinkster_workers/host.py +3116 -0
- dinkster_workers/in_process.py +1325 -0
- dinkster_workers/interpreter.py +84 -0
- dinkster_workers/isolated.py +862 -0
- dinkster_workers/launch.py +72 -0
- dinkster_workers/manifest.py +3332 -0
- dinkster_workers/media.py +196 -0
- dinkster_workers/pack_environment.py +473 -0
- dinkster_workers/pack_lock.py +359 -0
- dinkster_workers/placement.py +215 -0
- dinkster_workers/preview_artifacts.py +233 -0
- dinkster_workers/produced_assets.py +218 -0
- dinkster_workers/provision.py +597 -0
- dinkster_workers/py.typed +0 -0
- dinkster_workers/relay.py +762 -0
- dinkster_workers/remote.py +921 -0
- dinkster_workers/resume.py +490 -0
- dinkster_workers/routing.py +134 -0
- dinkster_workers/run_control.py +275 -0
- dinkster_workers/sandbox.py +798 -0
- dinkster_workers/saved_artifacts.py +275 -0
- dinkster_workers/service.py +593 -0
- dinkster_workers/session.py +2605 -0
- dinkster_workers/single_job_multi_gpu.py +7 -0
- dinkster_workers/staging.py +230 -0
- dinkster_workers/transport.py +161 -0
- dinkster_workers/workgroup.py +729 -0
- dinkster_workers/workgroup_compose.py +135 -0
- dinkster_workers/workgroup_config.py +196 -0
- dinkster_workers/workgroup_session.py +340 -0
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Declarative component metadata for combined model assets."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping, Sequence
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from typing import cast
|
|
8
|
+
|
|
9
|
+
from .identity import AssetError
|
|
10
|
+
from .json_metadata import freeze_json, thaw_json
|
|
11
|
+
from .kind import KIND_MODEL_CHECKPOINT, require_asset_kind
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class AssetComponent:
|
|
16
|
+
"""One declared component contained in a combined model asset."""
|
|
17
|
+
|
|
18
|
+
kind: str
|
|
19
|
+
architecture: str = ""
|
|
20
|
+
dtype: str = ""
|
|
21
|
+
metadata: Mapping[str, object] = field(default_factory=dict[str, object])
|
|
22
|
+
|
|
23
|
+
def __post_init__(self) -> None:
|
|
24
|
+
kind = cast("object", self.kind)
|
|
25
|
+
if not isinstance(kind, str):
|
|
26
|
+
raise AssetError("component kind must be a string")
|
|
27
|
+
object.__setattr__(self, "kind", require_asset_kind(kind))
|
|
28
|
+
for name in ("architecture", "dtype"):
|
|
29
|
+
if not isinstance(getattr(self, name), str):
|
|
30
|
+
raise AssetError(f"component {name} must be a string")
|
|
31
|
+
metadata = cast("object", self.metadata)
|
|
32
|
+
if not isinstance(metadata, Mapping):
|
|
33
|
+
raise AssetError("component metadata must be an object")
|
|
34
|
+
object.__setattr__(
|
|
35
|
+
self,
|
|
36
|
+
"metadata",
|
|
37
|
+
cast(
|
|
38
|
+
"Mapping[str, object]",
|
|
39
|
+
freeze_json(cast("Mapping[object, object]", metadata), "component metadata"),
|
|
40
|
+
),
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
def to_wire(self) -> dict[str, object]:
|
|
44
|
+
wire: dict[str, object] = {"kind": self.kind}
|
|
45
|
+
if self.architecture:
|
|
46
|
+
wire["architecture"] = self.architecture
|
|
47
|
+
if self.dtype:
|
|
48
|
+
wire["dtype"] = self.dtype
|
|
49
|
+
if self.metadata:
|
|
50
|
+
wire["metadata"] = thaw_json(self.metadata)
|
|
51
|
+
return wire
|
|
52
|
+
|
|
53
|
+
@classmethod
|
|
54
|
+
def from_wire(cls, wire: Mapping[str, object]) -> AssetComponent:
|
|
55
|
+
metadata = wire.get("metadata", {})
|
|
56
|
+
if not isinstance(metadata, Mapping):
|
|
57
|
+
raise AssetError("component metadata must be an object")
|
|
58
|
+
return cls(
|
|
59
|
+
kind=wire.get("kind", ""), # type: ignore[arg-type]
|
|
60
|
+
architecture=wire.get("architecture", ""), # type: ignore[arg-type]
|
|
61
|
+
dtype=wire.get("dtype", ""), # type: ignore[arg-type]
|
|
62
|
+
metadata=cast("Mapping[str, object]", metadata),
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True)
|
|
67
|
+
class AssetComponentManifest:
|
|
68
|
+
"""An ordered declaration of components contained in one asset."""
|
|
69
|
+
|
|
70
|
+
components: tuple[AssetComponent, ...]
|
|
71
|
+
|
|
72
|
+
def __post_init__(self) -> None:
|
|
73
|
+
raw_components = cast("object", self.components)
|
|
74
|
+
if not isinstance(raw_components, (list, tuple)):
|
|
75
|
+
raise AssetError("component manifest must be a list")
|
|
76
|
+
components = tuple(cast("list[object] | tuple[object, ...]", raw_components))
|
|
77
|
+
if any(not isinstance(component, AssetComponent) for component in components):
|
|
78
|
+
raise AssetError("component manifest must contain AssetComponent values")
|
|
79
|
+
object.__setattr__(self, "components", cast("tuple[AssetComponent, ...]", components))
|
|
80
|
+
|
|
81
|
+
def to_wire(self) -> list[dict[str, object]]:
|
|
82
|
+
return [component.to_wire() for component in self.components]
|
|
83
|
+
|
|
84
|
+
@classmethod
|
|
85
|
+
def from_wire(cls, wire: object) -> AssetComponentManifest:
|
|
86
|
+
if not isinstance(wire, Sequence) or isinstance(wire, (str, bytes)):
|
|
87
|
+
raise AssetError("component manifest must be a list")
|
|
88
|
+
components: list[AssetComponent] = []
|
|
89
|
+
for entry in cast("Sequence[object]", wire):
|
|
90
|
+
if not isinstance(entry, Mapping):
|
|
91
|
+
raise AssetError("component manifest entries must be objects")
|
|
92
|
+
components.append(AssetComponent.from_wire(cast("Mapping[str, object]", entry)))
|
|
93
|
+
return cls(tuple(components))
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def asset_kind_matches(
|
|
97
|
+
asset_kind: str,
|
|
98
|
+
accepted_kind: str,
|
|
99
|
+
component_manifest: AssetComponentManifest | None = None,
|
|
100
|
+
) -> bool:
|
|
101
|
+
"""Match a direct kind or a checkpoint's declared component kind."""
|
|
102
|
+
require_asset_kind(asset_kind)
|
|
103
|
+
require_asset_kind(accepted_kind)
|
|
104
|
+
if asset_kind == accepted_kind:
|
|
105
|
+
return True
|
|
106
|
+
return (
|
|
107
|
+
asset_kind == KIND_MODEL_CHECKPOINT
|
|
108
|
+
and component_manifest is not None
|
|
109
|
+
and any(component.kind == accepted_kind for component in component_manifest.components)
|
|
110
|
+
)
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
"""Worker-side declared-asset resolution: pack code reaching its own
|
|
2
|
+
``[[pack.assets]]`` declarations by pack-local id (DESIGN 3.12; the
|
|
3
|
+
controlnet-aux story's last leg).
|
|
4
|
+
|
|
5
|
+
Declaration, preflight, consent, and verified acquisition all live
|
|
6
|
+
host-side (pack_assets.py, acquire.py). What was missing is the read
|
|
7
|
+
end: node code that wants the bytes its manifest declared had to know
|
|
8
|
+
the digest and walk the raw resolver chain itself. :func:`declared_asset`
|
|
9
|
+
closes that: pack code names the declaration's stable pack-local id and
|
|
10
|
+
gets back a resolver-bound :class:`AssetRef` - or a loud AssetError that
|
|
11
|
+
says exactly which contract was not met. It NEVER downloads: execution
|
|
12
|
+
reads what consented acquisition already landed, nothing else.
|
|
13
|
+
|
|
14
|
+
Attribution is explicit and context-local. A host may install several
|
|
15
|
+
pack tables, but pack load and invocation run under exactly one active
|
|
16
|
+
pack context. Tables are never merged, so another pack's local ids are
|
|
17
|
+
not visible, and reads outside a known active context refuse loudly.
|
|
18
|
+
|
|
19
|
+
:func:`resolver_from_env` is the one place the worker-side store chain
|
|
20
|
+
is assembled from the environment contract (``DINKSTER_ASSET_VAULT``,
|
|
21
|
+
``DINKSTER_MOUNTS_SNAPSHOT``, ``DINKSTER_ASSET_ROOT``) - shared by the compat
|
|
22
|
+
pack's type registration and the worker host, so the chain's composition
|
|
23
|
+
can never drift between the two.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import logging
|
|
29
|
+
import os
|
|
30
|
+
from collections.abc import Generator, Iterable, Mapping
|
|
31
|
+
from contextlib import contextmanager
|
|
32
|
+
from contextvars import ContextVar
|
|
33
|
+
from dataclasses import dataclass
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
|
|
36
|
+
from .fetch import ChainResolver
|
|
37
|
+
from .identity import AssetError
|
|
38
|
+
from .kind import require_asset_kind
|
|
39
|
+
from .library import IndexedAssetResolver
|
|
40
|
+
from .model import AssetRef, AssetResolver
|
|
41
|
+
from .mounts import MountSnapshotResolver
|
|
42
|
+
from .pack_assets import DeclaredAsset
|
|
43
|
+
from .vault import AssetVault
|
|
44
|
+
|
|
45
|
+
__all__ = [
|
|
46
|
+
"clear_declared_assets",
|
|
47
|
+
"declared_asset",
|
|
48
|
+
"install_declared_assets",
|
|
49
|
+
"mount_roots_for_kind",
|
|
50
|
+
"resolver_from_env",
|
|
51
|
+
"use_declared_asset_pack",
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
VAULT_ENV = "DINKSTER_ASSET_VAULT"
|
|
55
|
+
MOUNTS_ENV = "DINKSTER_MOUNTS_SNAPSHOT"
|
|
56
|
+
ROOT_ENV = "DINKSTER_ASSET_ROOT"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def mount_roots_for_kind(kind: str, *, env: Mapping[str, str] | None = None) -> tuple[Path, ...]:
|
|
60
|
+
"""Read exact-kind directory grants from the worker's live mounts snapshot.
|
|
61
|
+
|
|
62
|
+
Ordered by ascending priority then mount id. Unconfigured or missing
|
|
63
|
+
authority returns no roots, with no vault, library or filesystem fallback.
|
|
64
|
+
"""
|
|
65
|
+
require_asset_kind(kind)
|
|
66
|
+
environment = os.environ if env is None else env
|
|
67
|
+
snapshot = environment.get(MOUNTS_ENV)
|
|
68
|
+
return MountSnapshotResolver(snapshot).roots_for_kind(kind) if snapshot else ()
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def produced_output_scratch(env: Mapping[str, str] | None = None) -> AssetVault | None:
|
|
72
|
+
"""Worker publications use writable scratch, not the sandbox's read-only input vault."""
|
|
73
|
+
environment = os.environ if env is None else env
|
|
74
|
+
scratch = environment.get("DINKSTER_PACK_SCRATCH")
|
|
75
|
+
if scratch:
|
|
76
|
+
return AssetVault(Path(scratch) / "produced-assets")
|
|
77
|
+
root = environment.get(VAULT_ENV)
|
|
78
|
+
return AssetVault(root) if root else None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def resolver_from_env(env: Mapping[str, str] | None = None) -> AssetResolver | None:
|
|
82
|
+
"""The process's asset store chain from its environment, cheapest
|
|
83
|
+
first: verified vault CAS, then the live mounts snapshot, then the
|
|
84
|
+
indexed library root. None when no store is configured (refs still
|
|
85
|
+
flow; only reads need a store). Pass ``env`` explicitly in tests;
|
|
86
|
+
production callers read the real environment."""
|
|
87
|
+
if env is None:
|
|
88
|
+
env = os.environ
|
|
89
|
+
resolvers: list[AssetResolver] = []
|
|
90
|
+
if env.get("DINKSTER_PACK_SCRATCH"):
|
|
91
|
+
produced = produced_output_scratch(env)
|
|
92
|
+
assert produced is not None
|
|
93
|
+
resolvers.append(produced)
|
|
94
|
+
vault_root = env.get(VAULT_ENV, "")
|
|
95
|
+
if vault_root:
|
|
96
|
+
resolvers.append(AssetVault(vault_root))
|
|
97
|
+
snapshot = env.get(MOUNTS_ENV, "")
|
|
98
|
+
if snapshot:
|
|
99
|
+
# Re-reads the engine-published snapshot per resolve, so folders
|
|
100
|
+
# granted after this process started materialize without restart.
|
|
101
|
+
resolvers.append(MountSnapshotResolver(snapshot))
|
|
102
|
+
root = env.get(ROOT_ENV, "")
|
|
103
|
+
if root:
|
|
104
|
+
try:
|
|
105
|
+
resolvers.append(IndexedAssetResolver(root))
|
|
106
|
+
except AssetError as error:
|
|
107
|
+
# The indexed root is one advisory store among several, and
|
|
108
|
+
# EVERY worker assembles this chain at startup now - a stale
|
|
109
|
+
# or unscanned library must not stop a pack that never reads
|
|
110
|
+
# assets from loading. Reads that needed it still fail loudly
|
|
111
|
+
# ("not acquired"/"not materializable") at the read site.
|
|
112
|
+
logging.getLogger("dinkster.assets").warning(
|
|
113
|
+
"ignoring unreadable asset root %s: %s", root, error
|
|
114
|
+
)
|
|
115
|
+
if not resolvers:
|
|
116
|
+
return None
|
|
117
|
+
return resolvers[0] if len(resolvers) == 1 else ChainResolver(*resolvers)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@dataclass(frozen=True)
|
|
121
|
+
class _DeclaredTable:
|
|
122
|
+
pack: str
|
|
123
|
+
assets: Mapping[str, DeclaredAsset]
|
|
124
|
+
resolver: AssetResolver | None
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
_installed: dict[str, _DeclaredTable] = {}
|
|
128
|
+
_active_pack: ContextVar[str | None] = ContextVar("dinkster_declared_asset_pack", default=None)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def install_declared_assets(
|
|
132
|
+
pack: str,
|
|
133
|
+
assets: Iterable[DeclaredAsset],
|
|
134
|
+
resolver: AssetResolver | None,
|
|
135
|
+
) -> None:
|
|
136
|
+
"""Install one hosted pack's table before its entry code runs."""
|
|
137
|
+
_installed[pack] = _DeclaredTable(
|
|
138
|
+
pack=pack,
|
|
139
|
+
assets={asset.id: asset for asset in assets},
|
|
140
|
+
resolver=resolver,
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@contextmanager
|
|
145
|
+
def use_declared_asset_pack(pack: str) -> Generator[None]:
|
|
146
|
+
"""Attribute declared-asset reads to exactly one hosted pack."""
|
|
147
|
+
if pack not in _installed:
|
|
148
|
+
raise AssetError(f"unknown declared-asset pack context {pack!r}")
|
|
149
|
+
token = _active_pack.set(pack)
|
|
150
|
+
try:
|
|
151
|
+
yield
|
|
152
|
+
finally:
|
|
153
|
+
_active_pack.reset(token)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def clear_declared_assets() -> None:
|
|
157
|
+
"""Remove the installed table (test hygiene; production workers never
|
|
158
|
+
clear - the table lives as long as the process)."""
|
|
159
|
+
_installed.clear()
|
|
160
|
+
_active_pack.set(None)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def declared_asset(asset_id: str) -> AssetRef:
|
|
164
|
+
"""Resolve one of the hosting pack's ``[[pack.assets]]`` declarations
|
|
165
|
+
to a resolver-bound :class:`AssetRef` whose bytes are readable NOW.
|
|
166
|
+
|
|
167
|
+
Fails loudly - never downloads, never guesses:
|
|
168
|
+
|
|
169
|
+
- no table installed: this code is not running inside a Dinkster worker
|
|
170
|
+
(or the host predates the hook);
|
|
171
|
+
- unknown id: the manifest does not declare it (declared ids are
|
|
172
|
+
listed in the error - the id is the contract, typos surface here);
|
|
173
|
+
- no store configured: the worker has nowhere to read bytes from;
|
|
174
|
+
- not acquired: the declaration exists but consented acquisition has
|
|
175
|
+
not landed the bytes on this machine - the error names the digest
|
|
176
|
+
so it correlates with the preflight plan that offers it.
|
|
177
|
+
"""
|
|
178
|
+
pack = _active_pack.get()
|
|
179
|
+
if pack is None:
|
|
180
|
+
raise AssetError(
|
|
181
|
+
f"declared asset {asset_id!r} requested, but no declaration "
|
|
182
|
+
"table is installed in this process - declared_asset() only "
|
|
183
|
+
"works inside a Dinkster worker hosting the declaring pack"
|
|
184
|
+
)
|
|
185
|
+
table = _installed.get(pack)
|
|
186
|
+
if table is None:
|
|
187
|
+
raise AssetError(f"unknown declared-asset pack context {pack!r}")
|
|
188
|
+
declared = table.assets.get(asset_id)
|
|
189
|
+
if declared is None:
|
|
190
|
+
known = ", ".join(sorted(table.assets)) or "none"
|
|
191
|
+
raise AssetError(
|
|
192
|
+
f"pack {table.pack!r} declares no asset {asset_id!r} in its "
|
|
193
|
+
f"[[pack.assets]] (declared ids: {known})"
|
|
194
|
+
)
|
|
195
|
+
need = declared.need
|
|
196
|
+
if table.resolver is None:
|
|
197
|
+
raise AssetError(
|
|
198
|
+
f"declared asset {asset_id!r} ({need.digest}) cannot be read: "
|
|
199
|
+
"this worker has no asset store configured (no "
|
|
200
|
+
f"{VAULT_ENV}/{MOUNTS_ENV}/{ROOT_ENV})"
|
|
201
|
+
)
|
|
202
|
+
path = table.resolver.resolve(need.digest)
|
|
203
|
+
if path is None:
|
|
204
|
+
raise AssetError(
|
|
205
|
+
f"declared asset {asset_id!r} ({need.digest}) is not acquired "
|
|
206
|
+
f"on this machine; pack {table.pack!r} declares where it can "
|
|
207
|
+
"come from, but assets never download during execution - "
|
|
208
|
+
"acquire it through job preflight consent (acquireAssets) or "
|
|
209
|
+
"the assets API, then rerun"
|
|
210
|
+
)
|
|
211
|
+
size = need.size if need.size >= 0 else path.stat().st_size
|
|
212
|
+
return AssetRef(
|
|
213
|
+
digest=need.digest,
|
|
214
|
+
name=need.name or asset_id,
|
|
215
|
+
size=size,
|
|
216
|
+
media_type=need.media_type or "application/octet-stream",
|
|
217
|
+
resolver=table.resolver,
|
|
218
|
+
)
|