k-dash 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- k_dash/__init__.py +5 -0
- k_dash/api.py +77 -0
- k_dash/artifact.py +104 -0
- k_dash/builder.py +217 -0
- k_dash/cache.py +126 -0
- k_dash/canonical.py +124 -0
- k_dash/cli.py +150 -0
- k_dash/config.py +93 -0
- k_dash/errors.py +48 -0
- k_dash/model.py +70 -0
- k_dash/oci.py +288 -0
- k_dash/project.py +183 -0
- k_dash/publish.py +259 -0
- k_dash/runtime.py +146 -0
- k_dash/target.py +78 -0
- k_dash/template_data/cpp-build.nix +98 -0
- k_dash/template_data/cutedsl-build.nix +72 -0
- k_dash/template_data/cutedsl-flake.lock +52 -0
- k_dash/template_data/cutedsl-flake.nix +16 -0
- k_dash/template_data/flake.lock +38 -0
- k_dash/templates.py +123 -0
- k_dash/validation.py +319 -0
- k_dash-0.1.0.dist-info/METADATA +80 -0
- k_dash-0.1.0.dist-info/RECORD +26 -0
- k_dash-0.1.0.dist-info/WHEEL +4 -0
- k_dash-0.1.0.dist-info/entry_points.txt +2 -0
k_dash/__init__.py
ADDED
k_dash/api.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""Synchronous public Python load API."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import threading
|
|
6
|
+
import json
|
|
7
|
+
from collections.abc import Mapping
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from .cache import Cache
|
|
12
|
+
from .canonical import normalize_args
|
|
13
|
+
from .config import load_registry_set
|
|
14
|
+
from .errors import ContractError, OfflineCacheMiss
|
|
15
|
+
from .project import validate_kernel_name
|
|
16
|
+
from .runtime import materialize, pull_release, resolve_release
|
|
17
|
+
from .target import detect_target
|
|
18
|
+
from .validation import validate_host_cxx_runtime, validate_host_dependencies, validate_tvm_ffi_runtime
|
|
19
|
+
|
|
20
|
+
_modules: dict[str, Any] = {}
|
|
21
|
+
_module_locks: dict[str, threading.Lock] = {}
|
|
22
|
+
_guard = threading.Lock()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def get(kernel: str, *, jit_args: Mapping[str, Any] | None = None, version: str) -> Path:
|
|
26
|
+
"""Resolve, materialize and return the absolute path to ``kernel.so``.
|
|
27
|
+
|
|
28
|
+
This is the download-path solver: registry / offline cache lookup, Binary
|
|
29
|
+
Miss local JIT when needed, then a stable cache path callers can load.
|
|
30
|
+
"""
|
|
31
|
+
validate_kernel_name(kernel)
|
|
32
|
+
if not isinstance(version, str) or not version:
|
|
33
|
+
raise ContractError("version must be a non-empty string", stage="load")
|
|
34
|
+
if jit_args is not None and not isinstance(jit_args, Mapping):
|
|
35
|
+
raise ContractError("jit_args must be a mapping", stage="load")
|
|
36
|
+
registries = load_registry_set()
|
|
37
|
+
cache = Cache()
|
|
38
|
+
release_digest, authority = resolve_release(registries, kernel, version, cache)
|
|
39
|
+
cached_release = cache.get_release(release_digest)
|
|
40
|
+
if cached_release is None:
|
|
41
|
+
import os
|
|
42
|
+
|
|
43
|
+
if os.environ.get("K_DASH_OFFLINE") == "1":
|
|
44
|
+
raise OfflineCacheMiss(
|
|
45
|
+
"release is absent from the offline cache",
|
|
46
|
+
stage="offline",
|
|
47
|
+
context={"release_digest": release_digest},
|
|
48
|
+
)
|
|
49
|
+
release = pull_release(authority, kernel, release_digest)
|
|
50
|
+
cache.commit_release(release_digest, release.config, release.layers[0])
|
|
51
|
+
release_config = release.config
|
|
52
|
+
else:
|
|
53
|
+
release_config = cached_release[0]
|
|
54
|
+
args = normalize_args(dict(jit_args or {}), release_config["args_schema"])
|
|
55
|
+
target = detect_target()
|
|
56
|
+
_, module_path = materialize(
|
|
57
|
+
registries, authority, kernel, release_digest, args, target, cache
|
|
58
|
+
)
|
|
59
|
+
return Path(module_path).resolve()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def load(kernel: str, *, jit_args: Mapping[str, Any] | None = None, version: str):
|
|
63
|
+
"""Resolve, materialize, validate and load one TVM-FFI module."""
|
|
64
|
+
module_path = get(kernel, jit_args=jit_args, version=version)
|
|
65
|
+
key = str(module_path)
|
|
66
|
+
with _guard:
|
|
67
|
+
lock = _module_locks.setdefault(key, threading.Lock())
|
|
68
|
+
with lock:
|
|
69
|
+
if key not in _modules:
|
|
70
|
+
import tvm_ffi
|
|
71
|
+
|
|
72
|
+
build_config = json.loads(module_path.with_name("config.json").read_text())
|
|
73
|
+
validate_tvm_ffi_runtime(build_config, tvm_ffi.__version__)
|
|
74
|
+
validate_host_dependencies(build_config)
|
|
75
|
+
validate_host_cxx_runtime(build_config)
|
|
76
|
+
_modules[key] = tvm_ffi.load_module(str(module_path))
|
|
77
|
+
return _modules[key]
|
k_dash/artifact.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""OCI release/build artifact construction and verification."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import io
|
|
6
|
+
import tarfile
|
|
7
|
+
from pathlib import PurePosixPath
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
import zstandard
|
|
11
|
+
|
|
12
|
+
from .canonical import build_key, canonical_json, digest_bytes, loads_no_duplicates
|
|
13
|
+
from .errors import ArtifactIntegrityError
|
|
14
|
+
from .model import BuildSpec
|
|
15
|
+
|
|
16
|
+
OCI_MANIFEST = "application/vnd.oci.image.manifest.v1+json"
|
|
17
|
+
RELEASE_TYPE = "application/vnd.k-dash.release.v1"
|
|
18
|
+
BUILD_TYPE = "application/vnd.k-dash.build.v1"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def binary_archive(kernel_so: bytes) -> bytes:
|
|
22
|
+
stream = io.BytesIO()
|
|
23
|
+
with tarfile.open(fileobj=stream, mode="w", format=tarfile.USTAR_FORMAT) as archive:
|
|
24
|
+
info = tarfile.TarInfo("kernel.so")
|
|
25
|
+
info.size = len(kernel_so)
|
|
26
|
+
info.mode = 0o755
|
|
27
|
+
info.mtime = 0
|
|
28
|
+
info.uid = info.gid = 0
|
|
29
|
+
archive.addfile(info, io.BytesIO(kernel_so))
|
|
30
|
+
return zstandard.ZstdCompressor(level=19, write_checksum=True).compress(stream.getvalue())
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def extract_binary_archive(payload: bytes) -> bytes:
|
|
34
|
+
try:
|
|
35
|
+
raw = zstandard.ZstdDecompressor().decompress(payload)
|
|
36
|
+
except zstandard.ZstdError as error:
|
|
37
|
+
raise ArtifactIntegrityError("invalid binary compression", stage="build-artifact") from error
|
|
38
|
+
with tarfile.open(fileobj=io.BytesIO(raw), mode="r:") as archive:
|
|
39
|
+
members = archive.getmembers()
|
|
40
|
+
if len(members) != 1 or members[0].name != "kernel.so" or not members[0].isfile():
|
|
41
|
+
raise ArtifactIntegrityError("binary layer must contain only kernel.so", stage="build-artifact")
|
|
42
|
+
if PurePosixPath(members[0].name).is_absolute():
|
|
43
|
+
raise ArtifactIntegrityError("absolute archive path", stage="build-artifact")
|
|
44
|
+
source = archive.extractfile(members[0])
|
|
45
|
+
assert source is not None
|
|
46
|
+
return source.read()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def build_objects(
|
|
50
|
+
spec: BuildSpec,
|
|
51
|
+
kernel_so: bytes,
|
|
52
|
+
*,
|
|
53
|
+
provenance: dict[str, Any],
|
|
54
|
+
host_dependencies: list[dict[str, str]],
|
|
55
|
+
cxx_runtime: dict[str, Any] | None = None,
|
|
56
|
+
) -> tuple[dict[str, Any], bytes, bytes, bytes, str]:
|
|
57
|
+
key = build_key(spec)
|
|
58
|
+
layer = binary_archive(kernel_so)
|
|
59
|
+
config = {
|
|
60
|
+
"protocol_version": 1,
|
|
61
|
+
"release_digest": spec.release_digest,
|
|
62
|
+
"buildspec": spec.as_dict(),
|
|
63
|
+
"build_key": key,
|
|
64
|
+
"provenance": provenance,
|
|
65
|
+
"host_dependencies": host_dependencies,
|
|
66
|
+
"cxx_runtime": cxx_runtime,
|
|
67
|
+
"kernel_so_digest": digest_bytes(kernel_so),
|
|
68
|
+
"binary": {"digest": digest_bytes(layer), "size": len(layer)},
|
|
69
|
+
}
|
|
70
|
+
config_bytes = canonical_json(config)
|
|
71
|
+
manifest = {
|
|
72
|
+
"schemaVersion": 2,
|
|
73
|
+
"mediaType": OCI_MANIFEST,
|
|
74
|
+
"artifactType": BUILD_TYPE,
|
|
75
|
+
"config": {
|
|
76
|
+
"mediaType": "application/vnd.k-dash.build.config.v1+json",
|
|
77
|
+
"digest": digest_bytes(config_bytes),
|
|
78
|
+
"size": len(config_bytes),
|
|
79
|
+
},
|
|
80
|
+
"layers": [{
|
|
81
|
+
"mediaType": "application/vnd.k-dash.binary.v1.tar+zstd",
|
|
82
|
+
"digest": digest_bytes(layer),
|
|
83
|
+
"size": len(layer),
|
|
84
|
+
}],
|
|
85
|
+
}
|
|
86
|
+
manifest_bytes = canonical_json(manifest)
|
|
87
|
+
return config, config_bytes, layer, manifest_bytes, digest_bytes(manifest_bytes)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def parse_json_blob(data: bytes, *, stage: str) -> dict[str, Any]:
|
|
91
|
+
try:
|
|
92
|
+
value = loads_no_duplicates(data, stage=stage)
|
|
93
|
+
except (ContractError, UnicodeDecodeError) as error:
|
|
94
|
+
raise ArtifactIntegrityError("artifact JSON is invalid", stage=stage) from error
|
|
95
|
+
if not isinstance(value, dict):
|
|
96
|
+
raise ArtifactIntegrityError("artifact JSON must be an object", stage=stage)
|
|
97
|
+
return value
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def verify_descriptor(descriptor: dict[str, Any], payload: bytes, *, stage: str) -> None:
|
|
101
|
+
expected_digest = descriptor.get("digest")
|
|
102
|
+
expected_size = descriptor.get("size")
|
|
103
|
+
if digest_bytes(payload) != expected_digest or len(payload) != expected_size:
|
|
104
|
+
raise ArtifactIntegrityError("descriptor digest or size mismatch", stage=stage)
|
k_dash/builder.py
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
"""Nix Local JIT and archive-stream Docker AOT execution."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import shutil
|
|
8
|
+
import subprocess
|
|
9
|
+
import tempfile
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from .canonical import canonical_json
|
|
14
|
+
from .errors import ContractError, KDashError
|
|
15
|
+
from .model import BuildSpec
|
|
16
|
+
from .project import materialize_source_tree
|
|
17
|
+
|
|
18
|
+
DEFAULT_BUILDER_IMAGE = "nixos/nix:2.24.9"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _driver_expression(source: Path, spec: BuildSpec, system: str) -> str:
|
|
22
|
+
# Kernel flakes own and lock nixpkgs. build.nix owns its build helpers, so
|
|
23
|
+
# the Release remains self-contained and has no separate kdlib dependency.
|
|
24
|
+
return f'''let
|
|
25
|
+
source = builtins.path {{ path = {json.dumps(str(source))}; name = "kernel-source"; }};
|
|
26
|
+
flake = builtins.getFlake (
|
|
27
|
+
builtins.unsafeDiscardStringContext ("path:" + toString source)
|
|
28
|
+
);
|
|
29
|
+
system = {json.dumps(system)};
|
|
30
|
+
hasCuteDsl = builtins.hasAttr "cutedsl-toolchain" flake.inputs;
|
|
31
|
+
hasCompatNixpkgs = builtins.hasAttr "compat-nixpkgs" flake.inputs;
|
|
32
|
+
cuteDslOverlays =
|
|
33
|
+
if hasCuteDsl then [
|
|
34
|
+
(import ((builtins.getAttr "cutedsl-toolchain" flake.inputs) + "/nix-builder/overlay.nix") {{
|
|
35
|
+
builderProvenance = null;
|
|
36
|
+
}})
|
|
37
|
+
] else [ ];
|
|
38
|
+
basePkgs = import flake.inputs.nixpkgs {{
|
|
39
|
+
inherit system;
|
|
40
|
+
config.allowUnfree = true;
|
|
41
|
+
}};
|
|
42
|
+
cudaPkgs = import flake.inputs.cuda-nixpkgs {{
|
|
43
|
+
inherit system;
|
|
44
|
+
config.allowUnfree = true;
|
|
45
|
+
overlays = cuteDslOverlays;
|
|
46
|
+
}};
|
|
47
|
+
compatPkgs = if hasCompatNixpkgs then
|
|
48
|
+
import flake.inputs.compat-nixpkgs {{
|
|
49
|
+
inherit system;
|
|
50
|
+
config.allowUnfree = true;
|
|
51
|
+
}}
|
|
52
|
+
else basePkgs;
|
|
53
|
+
manylinuxHostCc = if hasCompatNixpkgs then
|
|
54
|
+
basePkgs.wrapCCWith {{
|
|
55
|
+
cc = compatPkgs.gcc11.cc;
|
|
56
|
+
bintools = basePkgs.stdenv.cc.bintools;
|
|
57
|
+
libc = basePkgs.glibc;
|
|
58
|
+
}}
|
|
59
|
+
else basePkgs.stdenv.cc;
|
|
60
|
+
pkgs = basePkgs // {{
|
|
61
|
+
cudaPackages_12_8 = cudaPkgs.cudaPackages_12_8;
|
|
62
|
+
# GCC 11 supplies the compiler and C++ headers. The wrapper supplies the
|
|
63
|
+
# base pin's glibc 2.27 headers, CRT and linker paths; kernel build.nix can
|
|
64
|
+
# then select a compatible dynamic libstdc++/libgcc at final link time.
|
|
65
|
+
manylinuxHostStdenv = basePkgs.overrideCC basePkgs.stdenv manylinuxHostCc;
|
|
66
|
+
}} // (if builtins.hasAttr "cudaPackages_13" cudaPkgs then {{
|
|
67
|
+
cudaPackages_13 = cudaPkgs.cudaPackages_13;
|
|
68
|
+
}} else {{ }}) // (if builtins.hasAttr "cudaPackages_13_0" cudaPkgs then {{
|
|
69
|
+
cudaPackages_13_0 = cudaPkgs.cudaPackages_13_0;
|
|
70
|
+
}} else {{ }}) // (if hasCuteDsl then {{
|
|
71
|
+
cutePythonEnv = cudaPkgs.python313.withPackages (ps: [ ps.nvidia-cutlass-dsl ]);
|
|
72
|
+
}} else {{ }});
|
|
73
|
+
buildSpec = builtins.fromJSON {json.dumps(canonical_json(spec.as_dict()).decode())};
|
|
74
|
+
in import (source + "/build.nix") {{ inherit pkgs buildSpec; }}
|
|
75
|
+
'''
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _nix_system(spec: BuildSpec) -> str:
|
|
79
|
+
return {"x86_64": "x86_64-linux", "aarch64": "aarch64-linux"}[spec.target["arch"]]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def local_jit(source: Path, spec: BuildSpec) -> tuple[bytes, dict[str, Any]]:
|
|
83
|
+
if shutil.which("nix") is None:
|
|
84
|
+
raise KDashError("Nix is unavailable after Binary Miss", stage="local-jit")
|
|
85
|
+
with tempfile.TemporaryDirectory(prefix="k-dash-jit-") as temporary:
|
|
86
|
+
temporary_path = Path(temporary)
|
|
87
|
+
expression = _driver_expression(source.resolve(), spec, _nix_system(spec))
|
|
88
|
+
expression_path = temporary_path / "driver.nix"
|
|
89
|
+
expression_path.write_text(expression)
|
|
90
|
+
environment = {
|
|
91
|
+
"PATH": os.environ.get("PATH", ""),
|
|
92
|
+
"HOME": os.environ.get("HOME", ""),
|
|
93
|
+
"NIX_CONFIG": "experimental-features = nix-command flakes\nsandbox = true\npure-eval = true",
|
|
94
|
+
}
|
|
95
|
+
timeout = int(os.environ.get("K_DASH_JIT_TIMEOUT", "1800"))
|
|
96
|
+
jobs = os.environ.get("K_DASH_NIX_MAX_JOBS", "auto")
|
|
97
|
+
try:
|
|
98
|
+
command = [
|
|
99
|
+
"nix", "build", "--no-link", "--print-out-paths", "--max-jobs", jobs,
|
|
100
|
+
"--file", str(expression_path),
|
|
101
|
+
]
|
|
102
|
+
if os.environ.get("K_DASH_OFFLINE") == "1":
|
|
103
|
+
command.insert(2, "--offline")
|
|
104
|
+
result = subprocess.run(
|
|
105
|
+
command,
|
|
106
|
+
check=True,
|
|
107
|
+
capture_output=True,
|
|
108
|
+
text=True,
|
|
109
|
+
env=environment,
|
|
110
|
+
timeout=timeout,
|
|
111
|
+
)
|
|
112
|
+
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error:
|
|
113
|
+
stderr = getattr(error, "stderr", None)
|
|
114
|
+
context = {"stderr": stderr[-4000:]} if stderr else None
|
|
115
|
+
raise KDashError(
|
|
116
|
+
"Nix Local JIT failed", stage="local-jit", context=context
|
|
117
|
+
) from error
|
|
118
|
+
output = Path(result.stdout.strip().splitlines()[-1])
|
|
119
|
+
module = output / "kernel.so"
|
|
120
|
+
if not module.is_file() or any(path.name != "kernel.so" for path in output.iterdir()):
|
|
121
|
+
raise ContractError("build output must contain only kernel.so", stage="local-jit")
|
|
122
|
+
return module.read_bytes(), {"mode": "local-jit", "nix_output": output.name}
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def nix_aot(source: Path, spec: BuildSpec) -> tuple[bytes, dict[str, Any]]:
|
|
126
|
+
"""Build inside an already isolated CI container without nested namespaces."""
|
|
127
|
+
if shutil.which("nix") is None:
|
|
128
|
+
raise KDashError("Nix is unavailable", stage="nix-aot")
|
|
129
|
+
with tempfile.TemporaryDirectory(prefix="k-dash-nix-aot-") as temporary:
|
|
130
|
+
driver = Path(temporary) / "driver.nix"
|
|
131
|
+
driver.write_text(_driver_expression(source.resolve(), spec, _nix_system(spec)))
|
|
132
|
+
environment = {
|
|
133
|
+
"PATH": os.environ.get("PATH", ""),
|
|
134
|
+
"HOME": temporary,
|
|
135
|
+
"NIX_CONFIG": "experimental-features = nix-command flakes\nsandbox = false\npure-eval = true\nbuild-users-group =\nfilter-syscalls = false",
|
|
136
|
+
}
|
|
137
|
+
try:
|
|
138
|
+
result = subprocess.run(
|
|
139
|
+
["nix", "build", "--no-link", "--print-out-paths", "--max-jobs",
|
|
140
|
+
os.environ.get("K_DASH_NIX_MAX_JOBS", "auto"), "--file", str(driver)],
|
|
141
|
+
check=True, stdout=subprocess.PIPE, text=True, env=environment,
|
|
142
|
+
timeout=int(os.environ.get("K_DASH_AOT_TIMEOUT", "1800")),
|
|
143
|
+
)
|
|
144
|
+
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error:
|
|
145
|
+
raise KDashError("Nix AOT failed", stage="nix-aot") from error
|
|
146
|
+
output = Path(result.stdout.strip().splitlines()[-1])
|
|
147
|
+
module = output / "kernel.so"
|
|
148
|
+
if not module.is_file() or any(path.name != "kernel.so" for path in output.iterdir()):
|
|
149
|
+
raise ContractError("build output must contain only kernel.so", stage="nix-aot")
|
|
150
|
+
return module.read_bytes(), {"mode": "nix-aot", "nix_output": output.name}
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def docker_aot(
|
|
154
|
+
source: Path,
|
|
155
|
+
spec: BuildSpec,
|
|
156
|
+
*,
|
|
157
|
+
builder_image: str = DEFAULT_BUILDER_IMAGE,
|
|
158
|
+
) -> tuple[bytes, dict[str, Any]]:
|
|
159
|
+
try:
|
|
160
|
+
import docker
|
|
161
|
+
except ImportError as error:
|
|
162
|
+
raise KDashError("install k-dash[build] for Docker AOT", stage="docker-aot") from error
|
|
163
|
+
client = docker.from_env()
|
|
164
|
+
try:
|
|
165
|
+
client.volumes.get("k-dash-nix-store")
|
|
166
|
+
except docker.errors.NotFound:
|
|
167
|
+
client.volumes.create(name="k-dash-nix-store", labels={"org.k-dash.cache": "nix-store"})
|
|
168
|
+
with tempfile.TemporaryDirectory(prefix="k-dash-aot-") as temporary:
|
|
169
|
+
temporary_path = Path(temporary)
|
|
170
|
+
stage = temporary_path / "work"
|
|
171
|
+
materialize_source_tree(source, stage / "source")
|
|
172
|
+
expression = _driver_expression(Path("/tmp/source"), spec, _nix_system(spec))
|
|
173
|
+
(stage / "driver.nix").write_text(expression)
|
|
174
|
+
archive = shutil.make_archive(str(temporary_path / "context"), "tar", root_dir=stage, base_dir=".")
|
|
175
|
+
container = client.containers.create(
|
|
176
|
+
builder_image,
|
|
177
|
+
command=[
|
|
178
|
+
"sh", "-lc",
|
|
179
|
+
"nix --extra-experimental-features 'nix-command flakes' build "
|
|
180
|
+
"--option sandbox true --option filter-syscalls false "
|
|
181
|
+
"--no-link --print-out-paths --file /tmp/driver.nix > /tmp/output && "
|
|
182
|
+
"cp $(tail -n1 /tmp/output)/kernel.so /tmp/kernel.so",
|
|
183
|
+
],
|
|
184
|
+
platform="linux/amd64" if spec.target["arch"] == "x86_64" else "linux/arm64",
|
|
185
|
+
network_disabled=False,
|
|
186
|
+
user="0:0",
|
|
187
|
+
security_opt=["seccomp=unconfined"],
|
|
188
|
+
# Very large daemon defaults make Nix 2.24 close billions of FDs
|
|
189
|
+
# before exec. Bound this per-container startup work.
|
|
190
|
+
ulimits=[docker.types.Ulimit(name="nofile", soft=65536, hard=65536)],
|
|
191
|
+
volumes={"k-dash-nix-store": {"bind": "/nix", "mode": "rw"}},
|
|
192
|
+
)
|
|
193
|
+
try:
|
|
194
|
+
with open(archive, "rb") as stream:
|
|
195
|
+
if not container.put_archive("/tmp", stream):
|
|
196
|
+
raise KDashError("failed to stream source into builder", stage="docker-aot")
|
|
197
|
+
container.start()
|
|
198
|
+
# A cold /nix volume has to fetch and unpack the whole CUDA
|
|
199
|
+
# toolchain before it ever reaches nvcc, which outlasts any timeout
|
|
200
|
+
# that is reasonable once the store is warm.
|
|
201
|
+
status = container.wait(timeout=int(os.environ.get("K_DASH_AOT_TIMEOUT", "1800")))
|
|
202
|
+
if status.get("StatusCode") != 0:
|
|
203
|
+
log = container.logs(stdout=True, stderr=True, tail=80).decode(errors="replace")
|
|
204
|
+
raise KDashError("Docker AOT failed", stage="docker-aot", context={"log": log})
|
|
205
|
+
bits, _ = container.get_archive("/tmp/kernel.so")
|
|
206
|
+
import io
|
|
207
|
+
import tarfile
|
|
208
|
+
|
|
209
|
+
with tarfile.open(fileobj=io.BytesIO(b"".join(bits)), mode="r:") as tar:
|
|
210
|
+
extracted = tar.extractfile("kernel.so")
|
|
211
|
+
if extracted is None:
|
|
212
|
+
raise KDashError("builder returned no kernel.so", stage="docker-aot")
|
|
213
|
+
payload = extracted.read()
|
|
214
|
+
image = client.images.get(builder_image)
|
|
215
|
+
return payload, {"mode": "docker-aot", "builder_image": builder_image, "builder_digest": image.id}
|
|
216
|
+
finally:
|
|
217
|
+
container.remove(force=True)
|
k_dash/cache.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""Content-addressed cache with per-BuildKey advisory locking."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import contextlib
|
|
6
|
+
import fcntl
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import shutil
|
|
10
|
+
import tempfile
|
|
11
|
+
from collections.abc import Iterator
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from .canonical import canonical_json
|
|
16
|
+
from .errors import ArtifactIntegrityError
|
|
17
|
+
from .project import extract_source_archive
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Cache:
|
|
21
|
+
def __init__(self, root: Path | None = None):
|
|
22
|
+
self.root = root or Path(os.environ.get("K_DASH_CACHE_DIR", "~/.cache/k-dash")).expanduser()
|
|
23
|
+
for child in ("releases", "builds", "resolutions", "locks", "tmp"):
|
|
24
|
+
(self.root / child).mkdir(parents=True, exist_ok=True)
|
|
25
|
+
|
|
26
|
+
def build_dir(self, build_key: str) -> Path:
|
|
27
|
+
return self.root / "builds" / build_key.removeprefix("sha256:")
|
|
28
|
+
|
|
29
|
+
def release_dir(self, digest: str) -> Path:
|
|
30
|
+
return self.root / "releases" / digest.removeprefix("sha256:")
|
|
31
|
+
|
|
32
|
+
def get_release(self, digest: str) -> tuple[dict[str, Any], bytes, Path] | None:
|
|
33
|
+
path = self.release_dir(digest)
|
|
34
|
+
config_path = path / "config.json"
|
|
35
|
+
archive_path = path / "source.tar.zst"
|
|
36
|
+
source_path = path / "source"
|
|
37
|
+
if not config_path.is_file() or not archive_path.is_file() or not source_path.is_dir():
|
|
38
|
+
return None
|
|
39
|
+
config = json.loads(config_path.read_text())
|
|
40
|
+
if config.get("source", {}).get("digest") is None:
|
|
41
|
+
raise ArtifactIntegrityError("cached Release config is incomplete", stage="cache")
|
|
42
|
+
return config, archive_path.read_bytes(), source_path
|
|
43
|
+
|
|
44
|
+
def commit_release(self, digest: str, config: dict[str, Any], source_archive: bytes) -> Path:
|
|
45
|
+
destination = self.release_dir(digest)
|
|
46
|
+
if destination.exists():
|
|
47
|
+
return destination
|
|
48
|
+
temporary = Path(tempfile.mkdtemp(prefix="release-", dir=self.root / "tmp"))
|
|
49
|
+
try:
|
|
50
|
+
(temporary / "config.json").write_bytes(canonical_json(config))
|
|
51
|
+
(temporary / "source.tar.zst").write_bytes(source_archive)
|
|
52
|
+
extract_source_archive(source_archive, temporary / "source")
|
|
53
|
+
try:
|
|
54
|
+
temporary.replace(destination)
|
|
55
|
+
except FileExistsError:
|
|
56
|
+
pass
|
|
57
|
+
finally:
|
|
58
|
+
if temporary.exists():
|
|
59
|
+
shutil.rmtree(temporary)
|
|
60
|
+
return destination
|
|
61
|
+
|
|
62
|
+
def get_build(self, build_key: str) -> tuple[dict[str, Any], Path] | None:
|
|
63
|
+
path = self.build_dir(build_key)
|
|
64
|
+
config_path = path / "config.json"
|
|
65
|
+
module_path = path / "kernel.so"
|
|
66
|
+
if not config_path.is_file() or not module_path.is_file():
|
|
67
|
+
return None
|
|
68
|
+
config = json.loads(config_path.read_text())
|
|
69
|
+
if config.get("build_key") != build_key:
|
|
70
|
+
raise ArtifactIntegrityError("cached BuildKey mismatch", stage="cache", context={"path": str(path)})
|
|
71
|
+
return config, module_path
|
|
72
|
+
|
|
73
|
+
def commit_build(self, build_key: str, config: dict[str, Any], kernel_so: bytes) -> Path:
|
|
74
|
+
destination = self.build_dir(build_key)
|
|
75
|
+
if destination.exists():
|
|
76
|
+
return destination
|
|
77
|
+
temporary = Path(tempfile.mkdtemp(prefix="build-", dir=self.root / "tmp"))
|
|
78
|
+
try:
|
|
79
|
+
(temporary / "config.json").write_bytes(canonical_json(config))
|
|
80
|
+
(temporary / "kernel.so").write_bytes(kernel_so)
|
|
81
|
+
(temporary / "kernel.so").chmod(0o755)
|
|
82
|
+
try:
|
|
83
|
+
temporary.replace(destination)
|
|
84
|
+
except FileExistsError:
|
|
85
|
+
pass
|
|
86
|
+
finally:
|
|
87
|
+
if temporary.exists():
|
|
88
|
+
shutil.rmtree(temporary)
|
|
89
|
+
return destination
|
|
90
|
+
|
|
91
|
+
@contextlib.contextmanager
|
|
92
|
+
def build_lock(self, build_key: str) -> Iterator[None]:
|
|
93
|
+
path = self.root / "locks" / (build_key.removeprefix("sha256:") + ".lock")
|
|
94
|
+
with path.open("a+b") as handle:
|
|
95
|
+
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
|
|
96
|
+
try:
|
|
97
|
+
yield
|
|
98
|
+
finally:
|
|
99
|
+
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
|
100
|
+
|
|
101
|
+
def resolution_path(self, registry: str, kernel: str, version: str) -> Path:
|
|
102
|
+
safe_version = version.replace("/", "%2F")
|
|
103
|
+
return self.root / "resolutions" / registry / kernel / f"{safe_version}.json"
|
|
104
|
+
|
|
105
|
+
def put_resolution(self, registry: str, kernel: str, version: str, digest: str) -> None:
|
|
106
|
+
path = self.resolution_path(registry, kernel, version)
|
|
107
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
108
|
+
temporary = path.with_suffix(".tmp")
|
|
109
|
+
temporary.write_bytes(canonical_json({"release_digest": digest}))
|
|
110
|
+
temporary.replace(path)
|
|
111
|
+
|
|
112
|
+
def get_resolution(self, registry: str, kernel: str, version: str) -> str | None:
|
|
113
|
+
path = self.resolution_path(registry, kernel, version)
|
|
114
|
+
if not path.is_file():
|
|
115
|
+
return None
|
|
116
|
+
return json.loads(path.read_text()).get("release_digest")
|
|
117
|
+
|
|
118
|
+
def size(self) -> int:
|
|
119
|
+
return sum(path.stat().st_size for path in self.root.rglob("*") if path.is_file())
|
|
120
|
+
|
|
121
|
+
def clean(self) -> None:
|
|
122
|
+
for child in ("releases", "builds", "resolutions", "tmp"):
|
|
123
|
+
path = self.root / child
|
|
124
|
+
if path.exists():
|
|
125
|
+
shutil.rmtree(path)
|
|
126
|
+
path.mkdir(parents=True, exist_ok=True)
|
k_dash/canonical.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Args normalization and RFC 8785 BuildKey calculation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import copy
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import math
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
import jsonschema
|
|
12
|
+
import rfc8785
|
|
13
|
+
|
|
14
|
+
from .errors import ContractError
|
|
15
|
+
from .model import BuildSpec
|
|
16
|
+
|
|
17
|
+
MAX_SAFE_INTEGER = (1 << 53) - 1
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def loads_no_duplicates(payload: str | bytes, *, stage: str) -> Any:
|
|
21
|
+
def pairs(pairs_value: list[tuple[str, Any]]) -> dict[str, Any]:
|
|
22
|
+
output: dict[str, Any] = {}
|
|
23
|
+
for key, value in pairs_value:
|
|
24
|
+
if key in output:
|
|
25
|
+
raise ContractError(
|
|
26
|
+
"duplicate JSON object key",
|
|
27
|
+
stage=stage,
|
|
28
|
+
context={"key": key},
|
|
29
|
+
)
|
|
30
|
+
output[key] = value
|
|
31
|
+
return output
|
|
32
|
+
|
|
33
|
+
try:
|
|
34
|
+
return json.loads(payload, object_pairs_hook=pairs)
|
|
35
|
+
except json.JSONDecodeError as error:
|
|
36
|
+
raise ContractError("invalid JSON", stage=stage) from error
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _apply_defaults(instance: Any, schema: dict[str, Any]) -> Any:
|
|
40
|
+
if isinstance(instance, dict):
|
|
41
|
+
properties = schema.get("properties", {})
|
|
42
|
+
for key, subschema in properties.items():
|
|
43
|
+
if key not in instance and "default" in subschema:
|
|
44
|
+
instance[key] = copy.deepcopy(subschema["default"])
|
|
45
|
+
if key in instance:
|
|
46
|
+
instance[key] = _apply_defaults(instance[key], subschema)
|
|
47
|
+
elif isinstance(instance, list):
|
|
48
|
+
item_schema = schema.get("items", {})
|
|
49
|
+
for index, value in enumerate(instance):
|
|
50
|
+
instance[index] = _apply_defaults(value, item_schema)
|
|
51
|
+
return instance
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _validate_i_json(value: Any, path: str = "$") -> None:
|
|
55
|
+
if isinstance(value, bool) or value is None or isinstance(value, str):
|
|
56
|
+
return
|
|
57
|
+
if isinstance(value, int):
|
|
58
|
+
if not -MAX_SAFE_INTEGER <= value <= MAX_SAFE_INTEGER:
|
|
59
|
+
raise ContractError(
|
|
60
|
+
"integer is outside the I-JSON exact range",
|
|
61
|
+
stage="args",
|
|
62
|
+
context={"path": path, "value": value},
|
|
63
|
+
)
|
|
64
|
+
return
|
|
65
|
+
if isinstance(value, float):
|
|
66
|
+
if not math.isfinite(value) or (value == 0.0 and math.copysign(1.0, value) < 0):
|
|
67
|
+
raise ContractError(
|
|
68
|
+
"non-finite and negative-zero numbers are forbidden",
|
|
69
|
+
stage="args",
|
|
70
|
+
context={"path": path},
|
|
71
|
+
)
|
|
72
|
+
return
|
|
73
|
+
if isinstance(value, list):
|
|
74
|
+
for index, item in enumerate(value):
|
|
75
|
+
_validate_i_json(item, f"{path}[{index}]")
|
|
76
|
+
return
|
|
77
|
+
if isinstance(value, dict):
|
|
78
|
+
for key, item in value.items():
|
|
79
|
+
if not isinstance(key, str):
|
|
80
|
+
raise ContractError("object keys must be strings", stage="args", context={"path": path})
|
|
81
|
+
_validate_i_json(item, f"{path}.{key}")
|
|
82
|
+
return
|
|
83
|
+
raise ContractError("value is not JSON-compatible", stage="args", context={"path": path})
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def normalize_args(raw: dict[str, Any] | None, schema: dict[str, Any]) -> dict[str, Any]:
|
|
87
|
+
if raw is None:
|
|
88
|
+
raw = {}
|
|
89
|
+
if not isinstance(raw, dict):
|
|
90
|
+
raise ContractError("kernel args must be a JSON object", stage="args")
|
|
91
|
+
normalized = _apply_defaults(copy.deepcopy(raw), schema)
|
|
92
|
+
try:
|
|
93
|
+
jsonschema.Draft202012Validator(schema).validate(normalized)
|
|
94
|
+
except jsonschema.ValidationError as error:
|
|
95
|
+
raise ContractError(
|
|
96
|
+
error.message,
|
|
97
|
+
stage="args",
|
|
98
|
+
context={"path": ".".join(str(part) for part in error.absolute_path) or "$"},
|
|
99
|
+
) from error
|
|
100
|
+
_validate_i_json(normalized)
|
|
101
|
+
return normalized
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def canonical_json(value: Any) -> bytes:
|
|
105
|
+
_validate_i_json(value)
|
|
106
|
+
try:
|
|
107
|
+
return rfc8785.dumps(value)
|
|
108
|
+
except (TypeError, ValueError) as error:
|
|
109
|
+
raise ContractError(str(error), stage="canonical-json") from error
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def digest_bytes(data: bytes) -> str:
|
|
113
|
+
return "sha256:" + hashlib.sha256(data).hexdigest()
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def build_key(spec: BuildSpec | dict[str, Any]) -> str:
|
|
117
|
+
value = spec.as_dict() if isinstance(spec, BuildSpec) else spec
|
|
118
|
+
return digest_bytes(canonical_json(value))
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def build_tag(key: str) -> str:
|
|
122
|
+
if not key.startswith("sha256:") or len(key) != 71:
|
|
123
|
+
raise ContractError("invalid BuildKey", stage="build-key", context={"build_key": key})
|
|
124
|
+
return "build-sha256-" + key.removeprefix("sha256:")
|