xqo-toolchains 3.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xqo_toolchains/__init__.py +32 -0
- xqo_toolchains/chains.py +717 -0
- xqo_toolchains-3.1.0.dist-info/METADATA +49 -0
- xqo_toolchains-3.1.0.dist-info/RECORD +8 -0
- xqo_toolchains-3.1.0.dist-info/WHEEL +5 -0
- xqo_toolchains-3.1.0.dist-info/entry_points.txt +2 -0
- xqo_toolchains-3.1.0.dist-info/licenses/LICENSE +21 -0
- xqo_toolchains-3.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""xqo-toolchains: Node (incl. Nuxt/Next), Go, Rust and .NET for run_code, next to Python's uv venv (core).
|
|
2
|
+
|
|
3
|
+
It installs itself into the sandbox (`xqo.api.use_toolchains`): component detection, versions through mise,
|
|
4
|
+
dependency setup with digest stamps and shared download caches, for the local backend and (with xqo-docker)
|
|
5
|
+
toolchain images. Without it (or with `"plugins": {"disable": ["toolchains"]}`) runs still work: Python projects get
|
|
6
|
+
their venv, other projects run with whatever is on the host's PATH and no setup step. Per HQ it can also be switched
|
|
7
|
+
off with `"sandbox": {"toolchains": false}`. The details are in chains.py."""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import shutil
|
|
11
|
+
|
|
12
|
+
from xqo.api import Plugin, use_toolchains
|
|
13
|
+
|
|
14
|
+
__version__ = "3.1.0"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ToolchainsPlugin(Plugin):
|
|
18
|
+
name = "toolchains"
|
|
19
|
+
description = "Node / Go / Rust / .NET setup for run_code"
|
|
20
|
+
|
|
21
|
+
@classmethod
|
|
22
|
+
def register(cls) -> None:
|
|
23
|
+
from . import chains
|
|
24
|
+
use_toolchains(chains)
|
|
25
|
+
|
|
26
|
+
def configure(self, ctx, cfg):
|
|
27
|
+
from . import chains
|
|
28
|
+
on = (cfg.get("sandbox") or {}).get("toolchains", True)
|
|
29
|
+
return {"kinds": [k for k in chains.KINDS if k != "python"], "enabled": bool(on)}
|
|
30
|
+
|
|
31
|
+
def status(self) -> dict:
|
|
32
|
+
return {**self.obj, "mise": bool(shutil.which("mise"))}
|
xqo_toolchains/chains.py
ADDED
|
@@ -0,0 +1,717 @@
|
|
|
1
|
+
"""Language toolchains for the local sandbox backend: Node (incl. Nuxt/Next), Go, Rust and .NET, next to Python's uv venv.
|
|
2
|
+
|
|
3
|
+
A workspace is split into *components*: folders with a marker file (package.json, go.mod, Cargo.toml, *.sln/*.csproj),
|
|
4
|
+
so a Go service with a Nuxt app in web/ is two components. Each run, before the command:
|
|
5
|
+
|
|
6
|
+
* versions: wanted versions come from .nvmrc/.node-version/package.json engines, go.mod (`toolchain`/`go`),
|
|
7
|
+
global.json and mise's own files (.tool-versions, mise.toml). With `mise` on PATH they are installed once (shared,
|
|
8
|
+
in mise's data dir) and put first on PATH; without it the host's toolchains are used and a mismatch is reported.
|
|
9
|
+
Rust follows rust-toolchain(.toml) through rustup by itself.
|
|
10
|
+
* setup: dependencies are installed when a component's manifest or lockfile changed (digest stamp), else reused:
|
|
11
|
+
npm ci / pnpm / yarn / bun into ``artifacts/<task>/.deps/node@<path>/`` (linked into the run copy as node_modules,
|
|
12
|
+
never copied), ``go mod download``, ``cargo fetch``, ``dotnet restore``.
|
|
13
|
+
* caches: download caches (npm, pnpm store, yarn, Go modules, cargo registry, NuGet) are shared by all tasks under
|
|
14
|
+
`cache_dir`; build outputs (node_modules, GOCACHE, cargo target, dotnet CLI home) belong to the task.
|
|
15
|
+
|
|
16
|
+
The local backend is NOT a security boundary: commands run as xqo's user, with network."""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import hashlib
|
|
20
|
+
import json
|
|
21
|
+
import os
|
|
22
|
+
import re
|
|
23
|
+
import shutil
|
|
24
|
+
from dataclasses import dataclass, field
|
|
25
|
+
from pathlib import Path, PurePosixPath
|
|
26
|
+
|
|
27
|
+
KINDS = ("node", "go", "rust", "dotnet", "python")
|
|
28
|
+
# never searched for components, never linked (dependency, build and VCS folders)
|
|
29
|
+
SKIP_DIRS = {".git", "node_modules", "vendor", "target", "bin", "obj", ".venv", "venv", "dist", "build", ".next",
|
|
30
|
+
".nuxt", ".output", "__pycache__", ".skills", ".pytest_cache", ".mypy_cache", ".ruff_cache", ".yarn",
|
|
31
|
+
".svelte-kit", ".turbo", ".cache", "coverage"}
|
|
32
|
+
MAX_DEPTH = 3
|
|
33
|
+
PY_MARKERS = ("pyproject.toml", "setup.py", "setup.cfg", "requirements.txt", "Pipfile")
|
|
34
|
+
NODE_LOCKS = ("package-lock.json", "npm-shrinkwrap.json", "pnpm-lock.yaml", "yarn.lock", "bun.lockb", "bun.lock")
|
|
35
|
+
NODE_EXTRA = (".npmrc", ".yarnrc", ".yarnrc.yml", ".pnpmfile.cjs", "pnpm-workspace.yaml")
|
|
36
|
+
DOTNET_PROJ = (".sln", ".slnx", ".csproj", ".fsproj", ".vbproj")
|
|
37
|
+
DOTNET_FILES = {"global.json", "nuget.config", "directory.build.props", "directory.build.targets",
|
|
38
|
+
"directory.packages.props", "packages.lock.json"}
|
|
39
|
+
# build outputs a run makes again: never copied into a run (only next to their project's marker file)
|
|
40
|
+
BUILD_DIRS = {"node": (".next", ".nuxt", ".output", ".svelte-kit", ".turbo"), "rust": ("target",),
|
|
41
|
+
"dotnet": ("bin", "obj")}
|
|
42
|
+
# what toolchains need from our environment: proxies, certificates, registries, rustup/mise homes
|
|
43
|
+
PASS = {"HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", "SSL_CERT_FILE", "SSL_CERT_DIR", "REQUESTS_CA_BUNDLE",
|
|
44
|
+
"NODE_EXTRA_CA_CERTS", "GOPROXY", "GOPRIVATE", "GONOPROXY", "GONOSUMDB", "GOSUMDB", "GOINSECURE",
|
|
45
|
+
"NPM_CONFIG_REGISTRY", "CARGO_HTTP_CAINFO", "CARGO_NET_GIT_FETCH_WITH_CLI", "RUSTUP_HOME", "RUSTUP_TOOLCHAIN",
|
|
46
|
+
"MISE_DATA_DIR", "MISE_CONFIG_DIR", "MISE_CACHE_DIR", "LOCALAPPDATA", "APPDATA", "USERPROFILE", "TEMP", "TMP",
|
|
47
|
+
"PROGRAMFILES", "PROGRAMDATA", "DOTNET_ROOT"}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass
|
|
51
|
+
class Component:
|
|
52
|
+
path: str # posix path relative to the workspace, "." for the root
|
|
53
|
+
kind: str # node | go | rust | dotnet | python
|
|
54
|
+
mode: str = "" # node: link | clone | inplace
|
|
55
|
+
|
|
56
|
+
@property
|
|
57
|
+
def slug(self) -> str:
|
|
58
|
+
return f"{self.kind}@{'root' if self.path == '.' else re.sub(r'[^A-Za-z0-9._-]', '_', self.path)}"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass
|
|
62
|
+
class Prepared:
|
|
63
|
+
env: dict = field(default_factory=dict) # variables for the run (and setup)
|
|
64
|
+
path: list = field(default_factory=list) # dirs to put first on PATH
|
|
65
|
+
info: list = field(default_factory=list) # one dict per component (and one for versions)
|
|
66
|
+
managed: list = field(default_factory=list) # components whose deps the sandbox installs
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
# ---- detection -------------------------------------------------------------------------------------------------------
|
|
70
|
+
def _read_json(p: Path) -> dict:
|
|
71
|
+
try:
|
|
72
|
+
d = json.loads(p.read_text(encoding="utf-8-sig"))
|
|
73
|
+
return d if isinstance(d, dict) else {}
|
|
74
|
+
except (OSError, ValueError):
|
|
75
|
+
return {}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _kinds_at(d: Path, names: set[str]) -> list[str]:
|
|
79
|
+
kinds = []
|
|
80
|
+
if "package.json" in names:
|
|
81
|
+
kinds.append("node")
|
|
82
|
+
if "go.mod" in names:
|
|
83
|
+
kinds.append("go")
|
|
84
|
+
if "Cargo.toml" in names:
|
|
85
|
+
kinds.append("rust")
|
|
86
|
+
if any(n.lower().endswith(DOTNET_PROJ) for n in names):
|
|
87
|
+
kinds.append("dotnet")
|
|
88
|
+
if any(n in names for n in PY_MARKERS) or any(n.startswith("requirements") and n.endswith(".txt") for n in names):
|
|
89
|
+
kinds.append("python")
|
|
90
|
+
return kinds
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _workspace_root(d: Path, kind: str) -> bool:
|
|
94
|
+
"""A root that owns the components of the same kind below it (npm/pnpm/yarn workspaces, go.work, cargo
|
|
95
|
+
[workspace], a .sln)."""
|
|
96
|
+
if kind == "node":
|
|
97
|
+
return bool(_read_json(d / "package.json").get("workspaces")) or (d / "pnpm-workspace.yaml").is_file()
|
|
98
|
+
if kind == "go":
|
|
99
|
+
return (d / "go.work").is_file()
|
|
100
|
+
if kind == "rust":
|
|
101
|
+
try:
|
|
102
|
+
return re.search(r"(?m)^\s*\[workspace\]", (d / "Cargo.toml").read_text(encoding="utf-8")) is not None
|
|
103
|
+
except OSError:
|
|
104
|
+
return False
|
|
105
|
+
if kind == "dotnet":
|
|
106
|
+
return any(p.suffix.lower() in (".sln", ".slnx") for p in d.iterdir())
|
|
107
|
+
return False
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def detect(work: Path, overrides: list | None = None, node_modules: str = "auto") -> list[Component]:
|
|
111
|
+
"""Components of the workspace, breadth first, at most MAX_DEPTH folders deep. `overrides` ([{path, toolchain}])
|
|
112
|
+
replaces detection."""
|
|
113
|
+
if overrides:
|
|
114
|
+
comps = [Component(str(o.get("path") or ".").strip("/") or ".", str(o.get("toolchain") or o.get("kind")))
|
|
115
|
+
for o in overrides if isinstance(o, dict) and (o.get("toolchain") or o.get("kind")) in KINDS]
|
|
116
|
+
else:
|
|
117
|
+
comps, owners = [], [] # owners: (path, kind) of workspace roots
|
|
118
|
+
queue = [(work, 0)]
|
|
119
|
+
while queue:
|
|
120
|
+
d, depth = queue.pop(0)
|
|
121
|
+
try:
|
|
122
|
+
entries = sorted(d.iterdir(), key=lambda p: p.name)
|
|
123
|
+
except OSError:
|
|
124
|
+
continue
|
|
125
|
+
names = {p.name for p in entries if p.is_file()}
|
|
126
|
+
rel = "." if d == work else d.relative_to(work).as_posix()
|
|
127
|
+
for kind in _kinds_at(d, names):
|
|
128
|
+
if any(o[1] == kind and _under(rel, o[0]) for o in owners):
|
|
129
|
+
continue # part of a workspace root found higher up
|
|
130
|
+
if kind == "python" and rel != ".":
|
|
131
|
+
continue # the Python venv is per workspace (uv), from the root's files
|
|
132
|
+
comps.append(Component(rel, kind))
|
|
133
|
+
if _workspace_root(d, kind):
|
|
134
|
+
owners.append((rel, kind))
|
|
135
|
+
if depth < MAX_DEPTH:
|
|
136
|
+
queue += [(p, depth + 1) for p in entries if p.is_dir() and p.name not in SKIP_DIRS
|
|
137
|
+
and not p.is_symlink() and not p.name.startswith(".")]
|
|
138
|
+
for c in comps:
|
|
139
|
+
if c.kind == "node":
|
|
140
|
+
c.mode = _node_mode(work / c.path, node_modules)
|
|
141
|
+
return comps
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _under(rel: str, root: str) -> bool:
|
|
145
|
+
return root == "." or rel == root or rel.startswith(root + "/")
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _node_mode(d: Path, pref: str) -> str:
|
|
149
|
+
pkg = _read_json(d / "package.json")
|
|
150
|
+
deps = {**(pkg.get("dependencies") or {}), **(pkg.get("devDependencies") or {}),
|
|
151
|
+
**(pkg.get("optionalDependencies") or {})}
|
|
152
|
+
local = any(isinstance(v, str) and v.startswith(("file:", "link:", "workspace:", "portal:")) for v in deps.values())
|
|
153
|
+
if pkg.get("workspaces") or (d / "pnpm-workspace.yaml").is_file() or local:
|
|
154
|
+
return "inplace" # the install needs the sources too: done in the run copy
|
|
155
|
+
if pref in ("link", "clone"):
|
|
156
|
+
return pref
|
|
157
|
+
# Turbopack (Next) refuses a node_modules that links outside the project: give Next a real folder
|
|
158
|
+
return "clone" if "next" in deps else "link"
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def is_python_project(work: Path) -> bool:
|
|
162
|
+
"""Python markers at the root, or .py files in a workspace with no other toolchain's marker."""
|
|
163
|
+
try:
|
|
164
|
+
names = {p.name for p in work.iterdir() if p.is_file()}
|
|
165
|
+
except OSError:
|
|
166
|
+
return False
|
|
167
|
+
kinds = _kinds_at(work, names)
|
|
168
|
+
if "python" in kinds:
|
|
169
|
+
return True
|
|
170
|
+
return not kinds and any(walk(work, "*.py"))
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def walk(work: Path, pattern: str):
|
|
174
|
+
"""Files matching `pattern` below `work`, never inside SKIP_DIRS (node_modules ships .py files: node-gyp)."""
|
|
175
|
+
for root, dirs, files in os.walk(work):
|
|
176
|
+
dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
|
|
177
|
+
for f in files:
|
|
178
|
+
if Path(f).match(pattern):
|
|
179
|
+
yield Path(root) / f
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def copy_ignore(src: Path, comps: list[Component], base: tuple):
|
|
183
|
+
"""shutil.copytree ignore: `base` names everywhere, plus build outputs next to their marker file and the
|
|
184
|
+
node_modules of components whose deps the sandbox links in."""
|
|
185
|
+
managed = {os.path.normcase(str((src / c.path).resolve())) for c in comps if c.kind == "node"
|
|
186
|
+
and c.mode in ("link", "clone")}
|
|
187
|
+
kinds_with_builds = {c.kind for c in comps}
|
|
188
|
+
|
|
189
|
+
def ignore(d: str, names: list[str]) -> set[str]:
|
|
190
|
+
out = {n for n in names if n in base}
|
|
191
|
+
here = set(names)
|
|
192
|
+
if "package.json" in here and "node" in kinds_with_builds:
|
|
193
|
+
out |= here & set(BUILD_DIRS["node"])
|
|
194
|
+
if "node_modules" in here and os.path.normcase(str(Path(d).resolve())) in managed:
|
|
195
|
+
out.add("node_modules")
|
|
196
|
+
if "Cargo.toml" in here and "rust" in kinds_with_builds:
|
|
197
|
+
out |= here & set(BUILD_DIRS["rust"])
|
|
198
|
+
if "dotnet" in kinds_with_builds and any(n.lower().endswith(DOTNET_PROJ) for n in here):
|
|
199
|
+
out |= here & set(BUILD_DIRS["dotnet"])
|
|
200
|
+
return out
|
|
201
|
+
return ignore
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
# ---- versions --------------------------------------------------------------------------------------------------------
|
|
205
|
+
_PLAIN = re.compile(r"^\s*[v=]?(\d+(?:\.\d+){0,2})(?:\.x)?\s*$")
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def wanted_versions(work: Path, comps: list[Component]) -> dict[str, str]:
|
|
209
|
+
"""{tool: version} the project asks for (mise tool names: node, go, dotnet)."""
|
|
210
|
+
want: dict[str, str] = {}
|
|
211
|
+
for c in comps:
|
|
212
|
+
d = work / c.path
|
|
213
|
+
v = ""
|
|
214
|
+
if c.kind == "node":
|
|
215
|
+
for f in (".nvmrc", ".node-version"):
|
|
216
|
+
if (d / f).is_file() or (work / f).is_file():
|
|
217
|
+
raw = ((d / f) if (d / f).is_file() else (work / f)).read_text(encoding="utf-8").strip()
|
|
218
|
+
v = "lts" if raw.lower().startswith("lts") else raw.lstrip("v")
|
|
219
|
+
break
|
|
220
|
+
else:
|
|
221
|
+
eng = (_read_json(d / "package.json").get("engines") or {}).get("node", "")
|
|
222
|
+
m = _PLAIN.match(eng) if isinstance(eng, str) else None
|
|
223
|
+
v = m.group(1) if m else ""
|
|
224
|
+
elif c.kind == "go":
|
|
225
|
+
try:
|
|
226
|
+
text = (d / "go.mod").read_text(encoding="utf-8")
|
|
227
|
+
except OSError:
|
|
228
|
+
text = ""
|
|
229
|
+
m = re.search(r"(?m)^toolchain\s+go(\S+)", text) or re.search(r"(?m)^go\s+(\d+\.\d+(?:\.\d+)?)", text)
|
|
230
|
+
v = m.group(1) if m else ""
|
|
231
|
+
elif c.kind == "dotnet":
|
|
232
|
+
g = d / "global.json" if (d / "global.json").is_file() else work / "global.json"
|
|
233
|
+
v = str((_read_json(g).get("sdk") or {}).get("version") or "")
|
|
234
|
+
if v and c.kind in ("node", "go", "dotnet"):
|
|
235
|
+
want.setdefault(c.kind, v)
|
|
236
|
+
return want
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def mise_files(work: Path) -> list[Path]:
|
|
240
|
+
return [work / f for f in (".tool-versions", "mise.toml", ".mise.toml") if (work / f).is_file()]
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def host_version(tool: str, env: dict) -> str:
|
|
244
|
+
exe = shutil.which({"node": "node", "go": "go", "dotnet": "dotnet"}[tool], path=env.get("PATH"))
|
|
245
|
+
if not exe:
|
|
246
|
+
return ""
|
|
247
|
+
import subprocess
|
|
248
|
+
args = {"node": ["--version"], "go": ["version"], "dotnet": ["--version"]}[tool]
|
|
249
|
+
try:
|
|
250
|
+
out = subprocess.run([exe, *args], capture_output=True, text=True, timeout=20, env=env).stdout
|
|
251
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
252
|
+
return ""
|
|
253
|
+
m = re.search(r"(\d+\.\d+(?:\.\d+)?)", out)
|
|
254
|
+
return m.group(1) if m else ""
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def version_matches(want: str, have: str, tool: str) -> bool:
|
|
258
|
+
if not want or want == "lts":
|
|
259
|
+
return True
|
|
260
|
+
w, h = want.split("."), have.split(".")
|
|
261
|
+
n = 1 if tool == "node" else min(len(w), 2 if tool == "go" else 3) # node: major; go: minor; dotnet: as given
|
|
262
|
+
return w[:n] == h[:n]
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
# ---- environment -----------------------------------------------------------------------------------------------------
|
|
266
|
+
def cache_root(cfg_dir: str) -> Path:
|
|
267
|
+
d = os.getenv("XQO_CACHE_DIR") or cfg_dir or str(Path.home() / ".cache" / "xqo" / "toolchains")
|
|
268
|
+
return Path(d).expanduser().resolve()
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def base_env(cache, deps, comps: list[Component], host: bool = True) -> dict:
|
|
272
|
+
"""Variables that point every tool's caches at the shared cache dir or the task's deps dir (as the tools see
|
|
273
|
+
them: host paths, or /cache and /deps in a container). `host`: also pass our proxy/certificate/rustup settings."""
|
|
274
|
+
kinds = {c.kind for c in comps}
|
|
275
|
+
env = {k: v for k, v in os.environ.items() if k.upper() in PASS} if host else {}
|
|
276
|
+
if host and "RUSTUP_HOME" not in env and (Path.home() / ".rustup").is_dir():
|
|
277
|
+
env["RUSTUP_HOME"] = str(Path.home() / ".rustup") # HOME is the run copy: rustup must still find toolchains
|
|
278
|
+
if "node" in kinds:
|
|
279
|
+
env.update({"npm_config_cache": str(cache / "npm"), "npm_config_store_dir": str(cache / "pnpm-store"),
|
|
280
|
+
"npm_config_update_notifier": "false", "npm_config_fund": "false", "npm_config_audit": "false",
|
|
281
|
+
"YARN_CACHE_FOLDER": str(cache / "yarn"), "YARN_ENABLE_GLOBAL_CACHE": "false",
|
|
282
|
+
"YARN_NODE_LINKER": "node-modules", "YARN_ENABLE_TELEMETRY": "0",
|
|
283
|
+
"COREPACK_ENABLE_DOWNLOAD_PROMPT": "0", "COREPACK_HOME": str(cache / "corepack"),
|
|
284
|
+
"BUN_INSTALL_CACHE_DIR": str(cache / "bun"),
|
|
285
|
+
"NEXT_TELEMETRY_DISABLED": "1", "NUXT_TELEMETRY_DISABLED": "1"})
|
|
286
|
+
if "go" in kinds:
|
|
287
|
+
env.update({"GOMODCACHE": str(cache / "gomod"), "GOCACHE": str(deps / "go" / "build"),
|
|
288
|
+
"GOPATH": str(deps / "go" / "path"), "GOFLAGS": "-modcacherw", "GOTELEMETRY": "off"})
|
|
289
|
+
if "rust" in kinds:
|
|
290
|
+
env.update({"CARGO_HOME": str(cache / "cargo"), "CARGO_TARGET_DIR": str(deps / "rust-target")})
|
|
291
|
+
if "dotnet" in kinds:
|
|
292
|
+
env.update({"NUGET_PACKAGES": str(cache / "nuget"), "NUGET_HTTP_CACHE_PATH": str(cache / "nuget-http"),
|
|
293
|
+
"DOTNET_CLI_HOME": str(deps / "dotnet"), "DOTNET_CLI_TELEMETRY_OPTOUT": "1", "DOTNET_NOLOGO": "1",
|
|
294
|
+
"DOTNET_SKIP_FIRST_TIME_EXPERIENCE": "1", "DOTNET_GENERATE_ASPNET_CERTIFICATE": "false",
|
|
295
|
+
# no build servers left running: they hold the run's output pipes open until the timeout
|
|
296
|
+
"MSBUILDDISABLENODEREUSE": "1", "DOTNET_CLI_USE_MSBUILD_SERVER": "0", "UseSharedCompilation": "false"})
|
|
297
|
+
return env
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def digest(*parts) -> str:
|
|
301
|
+
return hashlib.sha256(json.dumps(parts, sort_keys=True, default=str).encode()).hexdigest()
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def manifest_files(d: Path, kind: str) -> list[Path]:
|
|
305
|
+
"""Files whose content decides a component's dependencies."""
|
|
306
|
+
if kind == "node":
|
|
307
|
+
return [d / n for n in ("package.json", *NODE_LOCKS, *NODE_EXTRA) if (d / n).is_file()]
|
|
308
|
+
if kind == "go":
|
|
309
|
+
return [d / n for n in ("go.mod", "go.sum", "go.work", "go.work.sum") if (d / n).is_file()]
|
|
310
|
+
if kind == "rust":
|
|
311
|
+
return sorted(p for p in walk(d, "Cargo.toml")) + [d / n for n in ("Cargo.lock", "rust-toolchain",
|
|
312
|
+
"rust-toolchain.toml") if (d / n).is_file()]
|
|
313
|
+
if kind == "dotnet":
|
|
314
|
+
return sorted(p for p in walk(d, "*") if p.suffix.lower() in DOTNET_PROJ or p.name.lower() in DOTNET_FILES)
|
|
315
|
+
return []
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def read_all(files: list[Path]) -> list:
|
|
319
|
+
out = []
|
|
320
|
+
for f in files:
|
|
321
|
+
try:
|
|
322
|
+
out.append((f.name, hashlib.sha256(f.read_bytes()).hexdigest()))
|
|
323
|
+
except OSError:
|
|
324
|
+
pass
|
|
325
|
+
return out
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def node_pm(d: Path) -> str:
|
|
329
|
+
pm = str(_read_json(d / "package.json").get("packageManager") or "").split("@")[0]
|
|
330
|
+
if pm in ("npm", "pnpm", "yarn", "bun"):
|
|
331
|
+
return pm
|
|
332
|
+
for lock, name in (("pnpm-lock.yaml", "pnpm"), ("yarn.lock", "yarn"), ("bun.lockb", "bun"), ("bun.lock", "bun")):
|
|
333
|
+
if (d / lock).is_file():
|
|
334
|
+
return name
|
|
335
|
+
return "npm"
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def node_install_argv(d: Path, pm: str, scripts: bool) -> list[str]:
|
|
339
|
+
if pm == "npm":
|
|
340
|
+
locked = (d / "package-lock.json").is_file() or (d / "npm-shrinkwrap.json").is_file()
|
|
341
|
+
argv = ["npm", "ci" if locked else "install", "--no-audit", "--no-fund"]
|
|
342
|
+
elif pm == "pnpm":
|
|
343
|
+
argv = ["pnpm", "install"] + (["--frozen-lockfile"] if (d / "pnpm-lock.yaml").is_file() else [])
|
|
344
|
+
elif pm == "yarn":
|
|
345
|
+
argv = ["yarn", "install"]
|
|
346
|
+
else:
|
|
347
|
+
argv = ["bun", "install"]
|
|
348
|
+
if not scripts and pm != "yarn": # yarn berry has no --ignore-scripts
|
|
349
|
+
argv.append("--ignore-scripts")
|
|
350
|
+
return argv
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def resolve_exe(argv: list[str], env: dict) -> list[str] | None:
|
|
354
|
+
"""argv with its program resolved on the run's PATH (Windows: npm.cmd); pnpm/yarn through corepack if needed."""
|
|
355
|
+
exe = shutil.which(argv[0], path=env.get("PATH"))
|
|
356
|
+
if exe:
|
|
357
|
+
return [exe, *argv[1:]]
|
|
358
|
+
if argv[0] in ("pnpm", "yarn") and (cp := shutil.which("corepack", path=env.get("PATH"))):
|
|
359
|
+
return [cp, *argv]
|
|
360
|
+
return None
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def dotnet_target(d: Path) -> list[str]:
|
|
364
|
+
"""`dotnet restore` needs one project or solution when a folder has several."""
|
|
365
|
+
files = sorted(p.name for p in d.iterdir() if p.is_file() and p.suffix.lower() in DOTNET_PROJ)
|
|
366
|
+
sln = [f for f in files if f.lower().endswith((".sln", ".slnx"))]
|
|
367
|
+
return [sln[0]] if sln else files[:1]
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def link_dir(target: Path, link: Path) -> None:
|
|
371
|
+
"""A directory symlink (a junction on Windows, which needs no privilege)."""
|
|
372
|
+
if os.name == "nt":
|
|
373
|
+
try:
|
|
374
|
+
import _winapi
|
|
375
|
+
_winapi.CreateJunction(str(target), str(link))
|
|
376
|
+
return
|
|
377
|
+
except (ImportError, OSError):
|
|
378
|
+
pass
|
|
379
|
+
os.symlink(target, link, target_is_directory=True)
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def clone_tree(src: Path, dst: Path) -> None:
|
|
383
|
+
"""Hard-link a tree (fast, no extra space), copying where links fail (another drive). Symlinks stay links."""
|
|
384
|
+
can_link = [True]
|
|
385
|
+
|
|
386
|
+
def cp(s, d):
|
|
387
|
+
if can_link[0]:
|
|
388
|
+
try:
|
|
389
|
+
return os.link(s, d)
|
|
390
|
+
except OSError:
|
|
391
|
+
can_link[0] = False # another drive / filesystem: copy the rest without trying again
|
|
392
|
+
shutil.copy2(s, d)
|
|
393
|
+
shutil.copytree(src, dst, symlinks=True, copy_function=cp)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
_INSTALL = re.compile(r"(?:^|[;&|]\s*)(?:(?:npm|pnpm|bun)\s+(?:install|i|add)\s+(?!-)\S|yarn\s+add\s|go\s+get\s|"
|
|
397
|
+
r"cargo\s+add\s|dotnet\s+add\s.*\bpackage\s)")
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def install_note(command: str) -> str:
|
|
401
|
+
if _INSTALL.search(command):
|
|
402
|
+
return ("this run works on a copy that is thrown away: packages added here are not in the workspace's "
|
|
403
|
+
"package.json / go.mod / Cargo.toml / project file. Add them to that file with your file tools; the "
|
|
404
|
+
"sandbox installs them before the next run.")
|
|
405
|
+
return ""
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def describe(info: dict) -> str:
|
|
409
|
+
if info.get("kind") == "image":
|
|
410
|
+
what = ("built" if info.get("built") else "") + (f"; ERROR: {info['error']}" if info.get("error") else "")
|
|
411
|
+
tools = ", ".join(f"{k} {v}" for k, v in info.get("tools", {}).items()) if info.get("built") or info.get(
|
|
412
|
+
"reused") or info.get("error") else ""
|
|
413
|
+
return (f"[image {info['image']}" + (f" ({tools})" if tools else "") + (f": {what}" if what else "") + "]"
|
|
414
|
+
+ (f"\n{info.get('log', '')[-3000:]}" if info.get("error") else ""))
|
|
415
|
+
if info.get("kind") == "versions":
|
|
416
|
+
parts = [f"{t} {v}" for t, v in info.get("tools", {}).items()]
|
|
417
|
+
head = f"[versions via {info['manager']}: {', '.join(parts) or 'from mise files'}"
|
|
418
|
+
return head + (f"; ERROR: {info['error']}]" if info.get("error") else "]") + "".join(
|
|
419
|
+
f"\n[warning: {w}]" for w in info.get("warnings", []))
|
|
420
|
+
where = f"{info['kind']} {info['path']}"
|
|
421
|
+
if info.get("error"):
|
|
422
|
+
return f"[{where}: setup ERROR: {info['error']}]\n{info.get('log', '')[-3000:]}"
|
|
423
|
+
what = info.get("setup") or ("reused" if info.get("reused") else "ready")
|
|
424
|
+
return f"[{where}: {what}]" + "".join(f"\n[warning: {w}]" for w in info.get("warnings", []))
|
|
425
|
+
|
|
426
|
+
# ---- runners ---------------------------------------------------------------------------------------------------------
|
|
427
|
+
class HostRunner:
|
|
428
|
+
"""Setup commands on this machine (local backend): host paths, host toolchains, mise for versions."""
|
|
429
|
+
versions = True # wanted versions come from mise / the host (a container gets them from its image)
|
|
430
|
+
host_env = True
|
|
431
|
+
|
|
432
|
+
def __init__(self, sh, cfg):
|
|
433
|
+
self._sh = sh
|
|
434
|
+
self.cache = cache_root(cfg.cache_dir)
|
|
435
|
+
|
|
436
|
+
def deps_view(self, deps: Path):
|
|
437
|
+
return deps
|
|
438
|
+
|
|
439
|
+
def exe(self, argv: list[str], env: dict) -> list[str] | None:
|
|
440
|
+
return resolve_exe(argv, env)
|
|
441
|
+
|
|
442
|
+
async def sh(self, argv, cwd: Path, env: dict, timeout: float):
|
|
443
|
+
return await self._sh(argv, cwd, env, timeout)
|
|
444
|
+
|
|
445
|
+
def link(self, target: Path, link: Path) -> None:
|
|
446
|
+
link_dir(target, link)
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
# ---- setup -----------------------------------------------------------------------------------------------------------
|
|
450
|
+
_locks: dict = {}
|
|
451
|
+
_host_versions: dict = {}
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _lock(key: str):
|
|
455
|
+
import asyncio
|
|
456
|
+
return _locks.setdefault(key, asyncio.Lock())
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def _stamp_ok(sdir: Path, key: str) -> bool:
|
|
460
|
+
try:
|
|
461
|
+
return json.loads((sdir / "stamp.json").read_text(encoding="utf-8"))["key"] == key
|
|
462
|
+
except (OSError, ValueError, KeyError, TypeError):
|
|
463
|
+
return False
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def _stamp(sdir: Path, key: str, **extra) -> None:
|
|
467
|
+
sdir.mkdir(parents=True, exist_ok=True)
|
|
468
|
+
(sdir / "stamp.json").write_text(json.dumps({"key": key, **extra}), encoding="utf-8")
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def _json_in(text: str) -> dict:
|
|
472
|
+
"""The JSON object in a command's output (stderr warnings are mixed in)."""
|
|
473
|
+
a, b = text.find("{"), text.rfind("}")
|
|
474
|
+
try:
|
|
475
|
+
d = json.loads(text[a:b + 1]) if a >= 0 and b > a else {}
|
|
476
|
+
return d if isinstance(d, dict) else {}
|
|
477
|
+
except ValueError:
|
|
478
|
+
return {}
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
async def prepare(runner, work: Path, comps: list[Component], deps: Path, cfg, run_env: dict) -> Prepared:
|
|
482
|
+
"""Versions, caches and dependencies for every non-Python component of the run copy `work`.
|
|
483
|
+
`runner`: HostRunner, or the Docker backend's runner (setup containers); `deps`: the task's deps dir on the
|
|
484
|
+
host; `run_env`: the run's environment so far."""
|
|
485
|
+
out = Prepared()
|
|
486
|
+
tcomps = [c for c in comps if c.kind != "python"]
|
|
487
|
+
if not tcomps and not (runner.versions and mise_files(work)):
|
|
488
|
+
return out
|
|
489
|
+
deps.mkdir(parents=True, exist_ok=True)
|
|
490
|
+
runner.cache.mkdir(parents=True, exist_ok=True)
|
|
491
|
+
cache_view = runner.cache if runner.host_env else PurePosixPath("/cache")
|
|
492
|
+
env = {**run_env, **base_env(cache_view, runner.deps_view(deps), tcomps, host=runner.host_env)}
|
|
493
|
+
vinfo = await _versions(runner.sh, work, wanted_versions(work, tcomps), deps, cfg, env) if runner.versions else {}
|
|
494
|
+
if vinfo:
|
|
495
|
+
out.info.append(vinfo)
|
|
496
|
+
if vinfo.get("path"):
|
|
497
|
+
env["PATH"] = os.pathsep.join([*vinfo["path"], env.get("PATH", "")])
|
|
498
|
+
env.update(vinfo.get("env") or {})
|
|
499
|
+
if vinfo.get("manager") == "mise" and "go" in (vinfo.get("tools") or {}):
|
|
500
|
+
env["GOTOOLCHAIN"] = "local" # mise installed the wanted Go: don't let go fetch another one
|
|
501
|
+
for c in tcomps:
|
|
502
|
+
out.info.append(await _setup(runner, work, c, deps, cfg, env, vinfo))
|
|
503
|
+
out.managed.append(c)
|
|
504
|
+
out.env = env
|
|
505
|
+
return out
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
async def _versions(sh, work: Path, want: dict, deps: Path, cfg, env: dict) -> dict:
|
|
509
|
+
files = mise_files(work)
|
|
510
|
+
if not want and not files:
|
|
511
|
+
return {}
|
|
512
|
+
mode = (cfg.versions or "auto").lower()
|
|
513
|
+
mise = shutil.which("mise") if mode in ("auto", "mise") else None
|
|
514
|
+
info = {"kind": "versions", "manager": "mise" if mise else "host", "tools": dict(want)}
|
|
515
|
+
if mise:
|
|
516
|
+
specs = [f"{t}@{v}" for t, v in want.items()]
|
|
517
|
+
key = digest(specs, read_all(files), mise)
|
|
518
|
+
sdir = deps / "versions"
|
|
519
|
+
async with _lock(str(sdir)):
|
|
520
|
+
if _stamp_ok(sdir, key):
|
|
521
|
+
st = json.loads((sdir / "stamp.json").read_text(encoding="utf-8"))
|
|
522
|
+
return {**info, "reused": True, "path": st.get("path", []), "env": st.get("env", {})}
|
|
523
|
+
# mise runs with our own environment (real HOME, its data dir); the run copy's mise files are trusted
|
|
524
|
+
menv = {**os.environ, "MISE_TRUSTED_CONFIG_PATHS": str(work), "MISE_YES": "1"}
|
|
525
|
+
logs = []
|
|
526
|
+
for argv in ([mise, "install"] if files else None, [mise, "install", *specs] if specs else None):
|
|
527
|
+
if argv:
|
|
528
|
+
code, o = await sh(argv, work, menv, cfg.setup_timeout)
|
|
529
|
+
logs.append(f"$ {' '.join(argv)}\n{o}")
|
|
530
|
+
if code != 0:
|
|
531
|
+
return {**info, "error": f"mise install failed (exit {code})", "log": "\n".join(logs)}
|
|
532
|
+
code, o = await sh([mise, "env", "--json", *specs], work, menv, 120)
|
|
533
|
+
data = _json_in(o) if code == 0 else {}
|
|
534
|
+
if not data:
|
|
535
|
+
return {**info, "error": f"mise env failed (exit {code})", "log": "\n".join(logs + [o])}
|
|
536
|
+
have = set(env.get("PATH", "").split(os.pathsep)) | set(os.environ.get("PATH", "").split(os.pathsep))
|
|
537
|
+
path = [p for p in str(data.get("PATH", "")).split(os.pathsep) if p and p not in have]
|
|
538
|
+
extra = {k: v for k, v in data.items() if k != "PATH" and isinstance(v, str) and not k.startswith("MISE_")}
|
|
539
|
+
_stamp(sdir, key, path=path, env=extra)
|
|
540
|
+
return {**info, "installed": True, "path": path, "env": extra, "log": "\n".join(logs)}
|
|
541
|
+
warnings = []
|
|
542
|
+
if mode == "mise":
|
|
543
|
+
warnings.append("mise is not installed: using the host's toolchains")
|
|
544
|
+
if files and not mise:
|
|
545
|
+
warnings.append(f"{', '.join(f.name for f in files)} found but mise is not installed: versions not applied")
|
|
546
|
+
for tool, v in want.items():
|
|
547
|
+
if tool == "go":
|
|
548
|
+
continue # go fetches the toolchain go.mod asks for by itself (GOTOOLCHAIN=auto, Go 1.21+)
|
|
549
|
+
k = (tool, env.get("PATH"))
|
|
550
|
+
if k not in _host_versions:
|
|
551
|
+
_host_versions[k] = host_version(tool, env)
|
|
552
|
+
have = _host_versions[k]
|
|
553
|
+
if not have:
|
|
554
|
+
warnings.append(f"the project wants {tool} {v}, but {tool} is not installed")
|
|
555
|
+
elif not version_matches(v, have, tool):
|
|
556
|
+
warnings.append(f"the project wants {tool} {v}, the host has {have} (install mise to get {v})")
|
|
557
|
+
return {**info, "warnings": warnings} if warnings else {} # host toolchains that fit: nothing to say
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
async def _setup(runner, work: Path, c: Component, deps: Path, cfg, env: dict, vinfo: dict) -> dict:
|
|
561
|
+
d = work / c.path
|
|
562
|
+
sdir = deps / c.slug
|
|
563
|
+
info = {"kind": c.kind, "path": c.path}
|
|
564
|
+
key = digest(c.kind, c.mode, read_all(manifest_files(d, c.kind)), vinfo.get("tools"), vinfo.get("path"),
|
|
565
|
+
cfg.install_scripts, getattr(runner, "image", "")) # another image: binaries built again
|
|
566
|
+
if c.kind == "node":
|
|
567
|
+
return await _setup_node(runner, d, c, sdir, key, cfg, env, info)
|
|
568
|
+
if c.kind == "go":
|
|
569
|
+
try:
|
|
570
|
+
needs = re.search(r"(?m)^require\b", (d / "go.mod").read_text(encoding="utf-8")) is not None
|
|
571
|
+
except OSError:
|
|
572
|
+
needs = False
|
|
573
|
+
argv = ["go", "mod", "download"] if needs else None
|
|
574
|
+
elif c.kind == "rust":
|
|
575
|
+
argv = ["cargo", "fetch"]
|
|
576
|
+
else:
|
|
577
|
+
argv = ["dotnet", "restore", *dotnet_target(d)]
|
|
578
|
+
async with _lock(str(sdir)):
|
|
579
|
+
if _stamp_ok(sdir, key):
|
|
580
|
+
return {**info, "reused": True}
|
|
581
|
+
if argv is None:
|
|
582
|
+
_stamp(sdir, key)
|
|
583
|
+
return {**info, "setup": "no dependencies"}
|
|
584
|
+
exe = runner.exe(argv, env)
|
|
585
|
+
if exe is None:
|
|
586
|
+
return {**info, "error": f"{argv[0]} is not installed (or not on PATH)"}
|
|
587
|
+
code, o = await runner.sh(exe, d, env, cfg.setup_timeout)
|
|
588
|
+
if code != 0:
|
|
589
|
+
return {**info, "error": f"{' '.join(argv)} failed (exit {code})", "log": o}
|
|
590
|
+
_stamp(sdir, key)
|
|
591
|
+
return {**info, "setup": f"dependencies fetched with {' '.join(argv)}", "log": o}
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
async def _setup_node(runner, d: Path, c: Component, sdir: Path, key: str, cfg, env: dict, info: dict) -> dict:
|
|
595
|
+
pm = node_pm(d)
|
|
596
|
+
argv = node_install_argv(d, pm, cfg.install_scripts)
|
|
597
|
+
info["manager"] = pm
|
|
598
|
+
exe = runner.exe(argv, env)
|
|
599
|
+
if c.mode == "inplace": # workspaces / local file deps: the install needs the sources, so it happens here
|
|
600
|
+
if (d / "node_modules").is_dir():
|
|
601
|
+
return {**info, "setup": "node_modules from the workspace"}
|
|
602
|
+
if exe is None:
|
|
603
|
+
return {**info, "error": f"{pm} is not installed (or not on PATH)"}
|
|
604
|
+
code, o = await runner.sh(exe, d, env, cfg.setup_timeout)
|
|
605
|
+
if code != 0:
|
|
606
|
+
return {**info, "error": f"{' '.join(argv)} failed (exit {code})", "log": o}
|
|
607
|
+
return {**info, "setup": f"installed in this run's copy with {' '.join(argv)} (workspaces or local deps: "
|
|
608
|
+
"not kept between runs)", "log": o}
|
|
609
|
+
pkg = sdir / "pkg"
|
|
610
|
+
async with _lock(str(sdir)):
|
|
611
|
+
if _stamp_ok(sdir, key) and (pkg / "node_modules").is_dir():
|
|
612
|
+
info["reused"] = True
|
|
613
|
+
else:
|
|
614
|
+
if exe is None:
|
|
615
|
+
return {**info, "error": f"{pm} is not installed (or not on PATH)"}
|
|
616
|
+
pkg.mkdir(parents=True, exist_ok=True)
|
|
617
|
+
for old in pkg.iterdir(): # only the manifests of now, so a deleted lockfile is gone here too
|
|
618
|
+
if old.name != "node_modules":
|
|
619
|
+
shutil.rmtree(old, ignore_errors=True) if old.is_dir() and not old.is_symlink() else old.unlink()
|
|
620
|
+
for f in manifest_files(d, "node"):
|
|
621
|
+
shutil.copy2(f, pkg / f.name)
|
|
622
|
+
for extra in ("patches", ".yarn"): # patch-package / pnpm patches, yarn releases and plugins
|
|
623
|
+
if (d / extra).is_dir():
|
|
624
|
+
shutil.copytree(d / extra, pkg / extra, ignore=shutil.ignore_patterns("cache", "install-state.gz"))
|
|
625
|
+
code, o = await runner.sh(exe, pkg, env, cfg.setup_timeout)
|
|
626
|
+
if code != 0:
|
|
627
|
+
(sdir / "stamp.json").unlink(missing_ok=True)
|
|
628
|
+
return {**info, "error": f"{' '.join(argv)} failed (exit {code})", "log": o}
|
|
629
|
+
(pkg / "node_modules").mkdir(exist_ok=True) # no dependencies: still something to link
|
|
630
|
+
_stamp(sdir, key)
|
|
631
|
+
info.update(setup=f"installed with {' '.join(argv)}", log=o)
|
|
632
|
+
nm = d / "node_modules"
|
|
633
|
+
if nm.is_symlink() or nm.is_file():
|
|
634
|
+
nm.unlink()
|
|
635
|
+
elif nm.is_dir():
|
|
636
|
+
shutil.rmtree(nm, ignore_errors=True)
|
|
637
|
+
(runner.link if c.mode == "link" else clone_tree)(pkg / "node_modules", nm)
|
|
638
|
+
info["node_modules"] = "linked" if c.mode == "link" else "cloned"
|
|
639
|
+
return info
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
# ---- docker images ---------------------------------------------------------------------------------------------------
|
|
643
|
+
UV_IMAGE = "astral/uv:0.9.30" # uv for the venv inside a composite image
|
|
644
|
+
COMPOSITE_BASE = "buildpack-deps:bookworm" # compilers, make, python3, git: what native packages need
|
|
645
|
+
# toolchain dirs on PATH in every image we run (missing ones are harmless)
|
|
646
|
+
DOCKER_TOOL_PATH = "/usr/local/go/bin:/usr/local/cargo/bin:/usr/share/dotnet"
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
def rust_channel(work: Path, comps: list[Component]) -> str:
|
|
650
|
+
for c in comps:
|
|
651
|
+
if c.kind != "rust":
|
|
652
|
+
continue
|
|
653
|
+
d = work / c.path
|
|
654
|
+
for f in (d / "rust-toolchain.toml", d / "rust-toolchain", work / "rust-toolchain.toml"):
|
|
655
|
+
if f.is_file():
|
|
656
|
+
text = f.read_text(encoding="utf-8")
|
|
657
|
+
m = re.search(r'channel\s*=\s*"([^"]+)"', text) or re.match(r"\s*(\S+)", text)
|
|
658
|
+
return m.group(1) if m else ""
|
|
659
|
+
return ""
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
def default_image(kind: str, version: str) -> str:
|
|
663
|
+
"""The official image for a toolchain at a version (bookworm variants: they have compilers for native deps)."""
|
|
664
|
+
if kind == "node":
|
|
665
|
+
return f"node:{version or '22'}-bookworm"
|
|
666
|
+
if kind == "go":
|
|
667
|
+
return f"golang:{version}-bookworm" if version else "golang:bookworm"
|
|
668
|
+
if kind == "rust":
|
|
669
|
+
v = version if re.match(r"^\d+(\.\d+){0,2}$", version or "") else "1" # stable/nightly: the latest 1.x
|
|
670
|
+
return f"rust:{v}-bookworm"
|
|
671
|
+
if kind == "dotnet":
|
|
672
|
+
m = re.match(r"^(\d+\.\d+)", version or "")
|
|
673
|
+
return f"mcr.microsoft.com/dotnet/sdk:{m.group(1) if m else '8.0'}"
|
|
674
|
+
raise ValueError(kind)
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def composite_dockerfile(images: dict[str, str], python: bool, base: str) -> str:
|
|
678
|
+
"""One image with every toolchain the project uses, copied out of their official images."""
|
|
679
|
+
lines = [f"FROM {base or COMPOSITE_BASE}"]
|
|
680
|
+
if "node" in images:
|
|
681
|
+
n = images["node"]
|
|
682
|
+
lines += [f"COPY --from={n} /usr/local/bin/node /usr/local/bin/node",
|
|
683
|
+
f"COPY --from={n} /usr/local/lib/node_modules /usr/local/lib/node_modules",
|
|
684
|
+
"RUN ln -sf ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm"
|
|
685
|
+
" && ln -sf ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx"
|
|
686
|
+
" && if [ -e /usr/local/lib/node_modules/corepack ]; then"
|
|
687
|
+
" ln -sf ../lib/node_modules/corepack/dist/corepack.js /usr/local/bin/corepack; fi"]
|
|
688
|
+
if "go" in images:
|
|
689
|
+
lines += [f"COPY --from={images['go']} /usr/local/go /usr/local/go"]
|
|
690
|
+
if "rust" in images:
|
|
691
|
+
lines += [f"COPY --from={images['rust']} /usr/local/rustup /usr/local/rustup",
|
|
692
|
+
f"COPY --from={images['rust']} /usr/local/cargo /usr/local/cargo",
|
|
693
|
+
"ENV RUSTUP_HOME=/usr/local/rustup"]
|
|
694
|
+
if "dotnet" in images:
|
|
695
|
+
lines += [f"COPY --from={images['dotnet']} /usr/share/dotnet /usr/share/dotnet",
|
|
696
|
+
"RUN ln -sf /usr/share/dotnet/dotnet /usr/local/bin/dotnet",
|
|
697
|
+
"ENV DOTNET_ROOT=/usr/share/dotnet"]
|
|
698
|
+
if python:
|
|
699
|
+
lines += [f"COPY --from={UV_IMAGE} /uv /uvx /usr/local/bin/"]
|
|
700
|
+
lines += [f"ENV PATH=/usr/local/sbin:/usr/local/bin:{DOCKER_TOOL_PATH}:/usr/sbin:/usr/bin:/sbin:/bin"]
|
|
701
|
+
return "\n".join(lines) + "\n"
|
|
702
|
+
|
|
703
|
+
|
|
704
|
+
def docker_plan(work: Path, comps: list[Component], cfg) -> dict:
|
|
705
|
+
"""Which image runs this workspace: {"image", "dockerfile" (None: pull as is), "tools": {kind: image}}.
|
|
706
|
+
One toolchain: its official image. Several (or Python plus one): a composite image built once, tagged by its
|
|
707
|
+
Dockerfile. Images can be overridden per toolchain with sandbox.images."""
|
|
708
|
+
kinds = sorted({c.kind for c in comps})
|
|
709
|
+
want = wanted_versions(work, comps)
|
|
710
|
+
want["rust"] = rust_channel(work, comps)
|
|
711
|
+
overrides = getattr(cfg, "images", None) or {}
|
|
712
|
+
tools = {k: overrides.get(k) or default_image(k, want.get(k, "")) for k in kinds if k != "python"}
|
|
713
|
+
if len(tools) == 1 and "python" not in kinds:
|
|
714
|
+
return {"image": next(iter(tools.values())), "dockerfile": None, "tools": tools}
|
|
715
|
+
df = composite_dockerfile(tools, "python" in kinds, getattr(cfg, "composite_base", "") or COMPOSITE_BASE)
|
|
716
|
+
return {"image": f"orch-toolchains:{hashlib.sha256(df.encode()).hexdigest()[:12]}", "dockerfile": df,
|
|
717
|
+
"tools": {**tools, **({"python": "uv"} if "python" in kinds else {})}}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: xqo-toolchains
|
|
3
|
+
Version: 3.1.0
|
|
4
|
+
Summary: Node, Go, Rust and .NET setup for xqo run_code
|
|
5
|
+
Author: dudikkocak
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/dudikkocak/xqo
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Dist: xqo<4,>=3.0
|
|
12
|
+
Provides-Extra: dev
|
|
13
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
14
|
+
Requires-Dist: pytest-timeout>=2.3; extra == "dev"
|
|
15
|
+
Dynamic: license-file
|
|
16
|
+
|
|
17
|
+
# xqo-toolchains
|
|
18
|
+
|
|
19
|
+
Node (including Nuxt and Next), Go, Rust and .NET for [xqo](https://github.com/dudikkocak/xqo)'s `run_code`, next to
|
|
20
|
+
the Python venv the base package already makes.
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
pip install xqo-toolchains
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Before each run it:
|
|
27
|
+
|
|
28
|
+
- **splits the workspace into components**: folders with `package.json`, `go.mod`, `Cargo.toml` or `*.sln`/`*.csproj`,
|
|
29
|
+
so a Go service with a Nuxt app in `web/` is two components;
|
|
30
|
+
- **picks versions** from `.nvmrc`, `.node-version`, `package.json` engines, `go.mod`, `global.json`,
|
|
31
|
+
`rust-toolchain.toml`, `.tool-versions` and `mise.toml`. With `mise` on PATH they are installed once and shared;
|
|
32
|
+
without it the host's toolchains are used and a mismatch is reported;
|
|
33
|
+
- **installs dependencies** only when a manifest or lockfile changed: `npm ci` / `pnpm` / `yarn` / `bun`,
|
|
34
|
+
`go mod download`, `cargo fetch`, `dotnet restore`. Download caches are shared by all tasks, build outputs belong
|
|
35
|
+
to the task.
|
|
36
|
+
|
|
37
|
+
With [xqo-docker](https://pypi.org/project/xqo-docker/) installed, runs use each project's official image at the
|
|
38
|
+
version it asks for (`node:<v>-bookworm`, `golang:<v>-bookworm`, `rust:<channel>-bookworm`,
|
|
39
|
+
`mcr.microsoft.com/dotnet/sdk:<v>`), and a mixed project gets a composite image built once.
|
|
40
|
+
|
|
41
|
+
| `sandbox` key | Default | |
|
|
42
|
+
|---|---|---|
|
|
43
|
+
| `toolchains` | `true` | `false` turns the setup off for this HQ |
|
|
44
|
+
| `versions` | `auto` | `auto` (mise if installed, else the host's), `mise` or `host` |
|
|
45
|
+
| `cache_dir` | `~/.cache/xqo/toolchains` | shared download caches (or `XQO_CACHE_DIR`) |
|
|
46
|
+
| `node_modules` | `auto` | `auto` (link; a real copy for Next), `link` or `clone` |
|
|
47
|
+
| `images` | | per-toolchain image overrides for the docker backend, e.g. `{"node": "node:20-bookworm"}` |
|
|
48
|
+
|
|
49
|
+
The local backend is not a security boundary: commands run as xqo's user, with network. Use xqo-docker for isolation.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
xqo_toolchains/__init__.py,sha256=tOQ8IFc2gstoXNyo_8TaY1qNNLuod2bsxH_QRUq4Ptc,1298
|
|
2
|
+
xqo_toolchains/chains.py,sha256=cq7uRUhUWNRffUcLEN7juP43SVL0zj332mvnTS6AdDE,35969
|
|
3
|
+
xqo_toolchains-3.1.0.dist-info/licenses/LICENSE,sha256=PHtf4TI2P6oulwhOKBF8SgeOkDLp9YnLdq46mw6506s,1067
|
|
4
|
+
xqo_toolchains-3.1.0.dist-info/METADATA,sha256=0_CrOUcUnhV1U0jYGKSEGA61Ovq6JtSkxTeCd0rs3eM,2315
|
|
5
|
+
xqo_toolchains-3.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
6
|
+
xqo_toolchains-3.1.0.dist-info/entry_points.txt,sha256=exFguh22m_TpJsL4C6yyb1y8X_SAYXxkFdWYY4CQexM,59
|
|
7
|
+
xqo_toolchains-3.1.0.dist-info/top_level.txt,sha256=BWSafzmq3SUKP2tkTpa-Ly0AeB7jPbO9T_iEN0mR2R8,15
|
|
8
|
+
xqo_toolchains-3.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 dudikkocak
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
xqo_toolchains
|