spawnfile 0.1.19 → 0.1.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/compiler/containerArtifactsPlans.js +1 -0
- package/dist/compiler/containerArtifactsTypes.d.ts +8 -0
- package/dist/compiler/containerDaimonGrokWorkerProvisioning.js +21 -1
- package/dist/compiler/containerDaimonGrokWorkerRender.d.ts +9 -0
- package/dist/compiler/containerDaimonGrokWorkerRender.js +7 -2
- package/dist/compiler/training/broker/declaration.d.ts +7 -0
- package/dist/compiler/training/broker/declaration.js +13 -1
- package/dist/compiler/training/broker/entrypoint.js +7 -10
- package/dist/compiler/training/broker/index.d.ts +2 -1
- package/dist/compiler/training/broker/index.js +2 -1
- package/dist/compiler/training/broker/projection.d.ts +9 -0
- package/dist/compiler/training/broker/projection.js +20 -1
- package/dist/compiler/training/broker/recycleClient.d.ts +33 -0
- package/dist/compiler/training/broker/recycleClient.js +112 -0
- package/dist/compiler/training/broker/registration.d.ts +3 -0
- package/dist/compiler/training/broker/registration.js +3 -0
- package/dist/compiler/training/broker/rootScript.d.ts +32 -0
- package/dist/compiler/training/broker/rootScript.js +33 -0
- package/dist/compiler/training/broker/runtime.js +4 -3
- package/dist/compiler/training/index.d.ts +3 -0
- package/dist/compiler/training/index.js +3 -0
- package/dist/compiler/training/preparation/assets/package-lock.json +2 -2
- package/dist/compiler/training/preparation/broker.d.ts +10 -1
- package/dist/compiler/training/preparation/broker.js +25 -3
- package/dist/compiler/training/preparation/prepare.js +5 -4
- package/dist/compiler/training/subjectDeclaration.d.ts +25 -0
- package/dist/compiler/training/subjectDeclaration.js +52 -0
- package/dist/compiler/training/subjectLease.d.ts +114 -0
- package/dist/compiler/training/subjectLease.js +175 -0
- package/dist/runtime/daimon/adapter.d.ts +7 -9
- package/dist/runtime/daimon/adapter.js +14 -19
- package/dist/runtime/daimon/config.js +13 -1
- package/dist/runtime/daimon/grokSkillsSupport.d.ts +20 -0
- package/dist/runtime/daimon/grokSkillsSupport.js +61 -0
- package/dist/runtime/daimon/grokWorkerConfigBytes.d.ts +126 -0
- package/dist/runtime/daimon/grokWorkerConfigBytes.js +157 -0
- package/dist/runtime/daimon/grokWorkerSkills.d.ts +61 -0
- package/dist/runtime/daimon/grokWorkerSkills.js +107 -0
- package/dist/runtime/types.d.ts +8 -0
- package/package.json +1 -1
- package/runtimes.yaml +5 -5
|
@@ -103,6 +103,7 @@ export const createRuntimeTargetPlans = async (plan, compiledNodes, worldBinding
|
|
|
103
103
|
...(target.engineByNodeId ? { engineByNodeId: target.engineByNodeId } : {}),
|
|
104
104
|
...(target.grokModelByNodeId ? { grokModelByNodeId: target.grokModelByNodeId } : {}),
|
|
105
105
|
...(target.grokTurnLimitsByNodeId ? { grokTurnLimitsByNodeId: target.grokTurnLimitsByNodeId } : {}),
|
|
106
|
+
...(target.grokSkillsByNodeId ? { grokSkillsByNodeId: target.grokSkillsByNodeId } : {}),
|
|
106
107
|
envFiles: resolveTargetEnvFiles(instancePaths.configPath, target),
|
|
107
108
|
packages: resolveTargetPackages(target, targetInputs),
|
|
108
109
|
id: target.id,
|
|
@@ -30,6 +30,14 @@ export interface RuntimeTargetPlan {
|
|
|
30
30
|
maxTokens?: number;
|
|
31
31
|
timeoutMs?: number;
|
|
32
32
|
}>;
|
|
33
|
+
/** Passthrough of `ContainerTarget.grokSkillsByNodeId` for the Daimon broker worker render. */
|
|
34
|
+
grokSkillsByNodeId?: Record<string, {
|
|
35
|
+
name: string;
|
|
36
|
+
files: {
|
|
37
|
+
path: string;
|
|
38
|
+
content: string;
|
|
39
|
+
}[];
|
|
40
|
+
}[]>;
|
|
33
41
|
packages?: ResolvedPackage[];
|
|
34
42
|
envFiles: Array<{
|
|
35
43
|
envName: string;
|
|
@@ -24,8 +24,27 @@ const programRegistration = (entry) => ({
|
|
|
24
24
|
spillDirectory: entry.spillDirectory,
|
|
25
25
|
slot: entry.slot,
|
|
26
26
|
uid: entry.uid,
|
|
27
|
-
workspace: entry.workspace
|
|
27
|
+
workspace: entry.workspace,
|
|
28
|
+
// By digest only: the tree itself is embedded once per program (`grokWorkerSkillTrees`), never per slot.
|
|
29
|
+
...(entry.skills ? { skillsSha256: entry.skills.sha256 } : {})
|
|
28
30
|
});
|
|
31
|
+
/**
|
|
32
|
+
* `$GROK_HOME/skills`, exactly Daimon's `renderGrokWorkerSkillsTree` entries
|
|
33
|
+
* (`../runtime/daimon/grokWorkerSkills.ts`): rendered only when some worker
|
|
34
|
+
* declares skills, so a skill-less program is byte-identical to before. It runs
|
|
35
|
+
* while `$GROK_HOME` is still `root:root 0700` (pass 1), first removing any
|
|
36
|
+
* previous tree (a restart, or a worker-planted entry) after reclaiming each
|
|
37
|
+
* directory, because root holds neither `CAP_FOWNER` nor `CAP_DAC_OVERRIDE`.
|
|
38
|
+
* Files are created `wx` (single link) and moded `root:root 0444`; directories
|
|
39
|
+
* are created `0700` and only moded `root:root 0555` once every child exists.
|
|
40
|
+
* No other user skill root (`GROK_WORKER_FOREIGN_SKILL_ROOTS`) is created.
|
|
41
|
+
*/
|
|
42
|
+
const renderDaimonGrokWorkerSkillsProvisioning = (registrations) => registrations.some((entry) => entry.skills) ? [
|
|
43
|
+
`const grokWorkerSkillTrees = ${JSON.stringify(Object.fromEntries(registrations.flatMap((entry) => entry.skills ? [[entry.skills.sha256, entry.skills.tree]] : [])))};`,
|
|
44
|
+
"const removeSkillsTree = (target) => { let info; try { info = fs.lstatSync(target); } catch (error) { if (error.code === 'ENOENT') return; throw error; } if (info.isDirectory() && !info.isSymbolicLink()) { fs.chownSync(target, 0, 0); fs.chmodSync(target, 0o700); for (const name of fs.readdirSync(target)) removeSkillsTree(`${target}/${name}`); fs.rmdirSync(target); } else fs.unlinkSync(target); };",
|
|
45
|
+
"const provisionSkills = (entry) => { removeSkillsTree(`${entry.grokHome}/skills`); const tree = grokWorkerSkillTrees[entry.skillsSha256]; if (!Array.isArray(tree) || tree.length === 0) throw new Error(`Grok worker skills ${entry.skillsSha256} are not embedded`); const directories = []; for (const item of tree) { const target = `${entry.grokHome}/${item.path}`; if ((item.path !== 'skills' && !item.path.startsWith('skills/')) || item.path.split('/').some((part) => part === '' || part === '.' || part === '..') || item.uid !== 0 || item.gid !== 0) throw new Error(`unsafe Grok worker skill entry: ${item.path}`); if (item.kind === 'directory') { fs.mkdirSync(target, { mode: 0o700 }); fs.chownSync(target, 0, 0); fs.chmodSync(target, 0o700); directories.push([target, item.mode]); continue; } fs.writeFileSync(target, item.content, { mode: 0o600, flag: 'wx' }); const info = fs.lstatSync(target); if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1) throw new Error(`unsafe Grok worker skill file: ${target}`); fs.chownSync(target, 0, 0); fs.chmodSync(target, item.mode); } for (const [target, mode] of directories.reverse()) { fs.chownSync(target, 0, 0); fs.chmodSync(target, mode); } };",
|
|
46
|
+
"for (const entry of grokWorkers) if (entry.skillsSha256) provisionSkills(entry);"
|
|
47
|
+
] : [];
|
|
29
48
|
/**
|
|
30
49
|
* Root provisioning for every brokered Grok worker, as lines of the broker's
|
|
31
50
|
* node provisioning program (which already defines `crypto`, `fs`, and
|
|
@@ -79,6 +98,7 @@ optionalDenyPaths = [...DAIMON_GROK_OPTIONAL_DENY_PATHS, ...DAIMON_GROK_DENIED_S
|
|
|
79
98
|
`for (const entry of grokWorkers) { traversable(entry.workspace); secureWorkspace(entry.workspace, entry.uid); assertCanonical(entry.workspace, 'workspace'); ${scope === "container" ? "ensureDirectory(entry.home, 0, 0, 0o700);" : ""} ensureDirectory(entry.grokHome, 0, 0, 0o700); ensureDirectory(\`\${entry.grokHome}/sessions\`, 0, 0, 0o700); for (const target of [entry.home, entry.grokHome]) assertCanonical(target, 'home'); }`,
|
|
80
99
|
"for (const denied of optionalDenyPaths) { try { fs.lstatSync(denied); } catch (error) { if (error.code !== 'ENOENT') throw error; fs.mkdirSync(denied, { mode: 0o700 }); fs.chownSync(denied, 0, 0); fs.chmodSync(denied, 0o700); } }",
|
|
81
100
|
"for (const entry of grokWorkers) for (const denied of entry.denyPaths) { try { assertCanonical(denied, 'deny path'); } catch (error) { if (error.code === 'ENOENT' && entry.deferredDenyPaths.includes(denied)) continue; if (error.code === 'ENOENT') throw new Error(`Grok worker deny path is missing: ${denied}`); throw error; } }",
|
|
101
|
+
...renderDaimonGrokWorkerSkillsProvisioning(registrations),
|
|
82
102
|
// Pass 2: exact root-owned read-only files, the events file, then the final sticky modes.
|
|
83
103
|
`for (const entry of grokWorkers) { ensureExactFile(\`\${entry.grokHome}/config.toml\`, entry.config, 0o444); ensureExactFile(entry.profilePath, entry.profile, 0o444); for (const name of ${JSON.stringify(DAIMON_GROK_WORKER_READ_ONLY_FILES.filter((name) => name !== "config.toml" && name !== "sandbox.toml"))}) ensureExactFile(\`\${entry.grokHome}/\${name}\`, '', 0o444); ensureEventsFile(entry.eventsPath, entry.uid); ensureDirectory(\`\${entry.grokHome}/sessions\`, 0, entry.uid, 0o1771); ensureDirectory(entry.grokHome, 0, entry.uid, 0o1771); ${scope === "container" ? `ensureDirectory(entry.privateTmp, entry.uid, entry.uid, 0o700); ensureDirectory(entry.home, entry.uid, ${DAIMON_BROKER_UID}, 0o710);` : ""} for (const target of [entry.profilePath, entry.eventsPath, \`\${entry.grokHome}/config.toml\`]) assertCanonical(target, 'home file'); }`,
|
|
84
104
|
// Worker-private temp: the launcher compiles TMPDIR=<home>/tmp, the only temp the worker may write. It is
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type DaimonGrokBrokerModel, type DaimonGrokBrokerReasoningEffort } from "../runtime/daimon/contractManifest.js";
|
|
2
|
+
import { type DaimonGrokWorkerSkills } from "../runtime/daimon/grokWorkerSkills.js";
|
|
2
3
|
import type { RuntimeTargetPlan } from "./containerArtifactsTypes.js";
|
|
3
4
|
export declare const DAIMON_WORKER_ROOT = "/var/lib/daimon-workers";
|
|
4
5
|
export declare const DAIMON_GROK_WORKER_READ_ONLY_FILES: readonly ["config.toml", "managed_config.toml", "requirements.toml", "sandbox.toml", "trusted_folders.toml"];
|
|
@@ -77,6 +78,12 @@ export interface DaimonGrokRegistration {
|
|
|
77
78
|
runtimeHome: string;
|
|
78
79
|
/** Persistent mounts inside this agent's runtime home; Spawnfile keeps each `0700` under the traversable home. */
|
|
79
80
|
runtimeHomeMounts: string[];
|
|
81
|
+
/**
|
|
82
|
+
* The agent's declared skills as Daimon renders them into `$GROK_HOME/skills`,
|
|
83
|
+
* with their digest; absent when it declares none, which keeps the
|
|
84
|
+
* registration, `service.json` and provisioning byte-identical.
|
|
85
|
+
*/
|
|
86
|
+
skills?: DaimonGrokWorkerSkills;
|
|
80
87
|
/** `<runtimeHome>/tool-output`: setgid spill directory in the worker's group. */
|
|
81
88
|
spillDirectory: string;
|
|
82
89
|
slot: number;
|
|
@@ -149,6 +156,7 @@ export interface DaimonGrokServiceConfigOptions {
|
|
|
149
156
|
/** `service.json` v2, exactly the shape Daimon's strict `parseEngineBrokerServiceConfig` accepts. */
|
|
150
157
|
export declare const renderDaimonGrokServiceConfig: (registrations: readonly DaimonGrokRegistration[], options?: DaimonGrokServiceConfigOptions) => {
|
|
151
158
|
slotPolicies?: {
|
|
159
|
+
skillsSha256?: string | undefined;
|
|
152
160
|
slot: number;
|
|
153
161
|
usageLedgerPath: string;
|
|
154
162
|
limits: {
|
|
@@ -166,6 +174,7 @@ export declare const renderDaimonGrokServiceConfig: (registrations: readonly Dai
|
|
|
166
174
|
};
|
|
167
175
|
}[] | undefined;
|
|
168
176
|
registrations: {
|
|
177
|
+
skillsSha256?: string | undefined;
|
|
169
178
|
usageLedgerPath?: string | undefined;
|
|
170
179
|
limits?: {
|
|
171
180
|
maxRequests: 32;
|
|
@@ -4,6 +4,7 @@ import { resolveDaimonGrokRegistrationLimits } from "../runtime/daimon/grokTurnL
|
|
|
4
4
|
import { DAIMON_ORGANIZATION_TARGET_ID, DAIMON_RUNTIME_ACCEPTANCE_STORE_DIRECTORY, DAIMON_RUNTIME_HOMES_DIRECTORY, DAIMON_WAKE_FUSE_DIRECTORY } from "../runtime/daimon/config.js";
|
|
5
5
|
import { DAIMON_AGY_SUBSCRIPTION_REALM, DAIMON_GROK_BROKER_MODELS, DAIMON_GROK_BROKER_REASONING_EFFORTS, DAIMON_GROK_ENGINE_BROKER, DAIMON_GROK_SUBSCRIPTION_REALM, DAIMON_GROK_TURN_USAGE_LEDGER, DAIMON_RUNTIME_HOME_ROOT } from "../runtime/daimon/contractManifest.js";
|
|
6
6
|
import { DAIMON_GROK_WORKER_HOME_DIRECTORY, daimonGrokWorkerSandboxProfileSha256, renderDaimonGrokWorkerSandboxProfile, resolveDaimonGrokWorkerConfig } from "../runtime/daimon/grokWorkerContract.js";
|
|
7
|
+
import { resolveDaimonGrokWorkerSkills } from "../runtime/daimon/grokWorkerSkills.js";
|
|
7
8
|
import { DAIMON_FIRST_WORKER_UID } from "../runtime/daimon/runtimeIdentity.js";
|
|
8
9
|
export const DAIMON_WORKER_ROOT = "/var/lib/daimon-workers";
|
|
9
10
|
export const DAIMON_GROK_WORKER_READ_ONLY_FILES = DAIMON_GROK_ENGINE_BROKER.worker.home.readOnlyFiles.names;
|
|
@@ -205,7 +206,9 @@ export const resolveDaimonGrokRegistrations = (plans) => {
|
|
|
205
206
|
if (path.posix.dirname(path.posix.dirname(path.posix.join(grokHome, "sandbox.toml"))) !== home)
|
|
206
207
|
fail(`Grok worker ${agentId} profile is not under its registered home`);
|
|
207
208
|
const backings = new Set(plans.flatMap((candidate) => candidate.resources ?? []).map((resource) => resource.backingPath));
|
|
209
|
+
const skills = resolveDaimonGrokWorkerSkills(plan.grokSkillsByNodeId?.[agentId]);
|
|
208
210
|
return {
|
|
211
|
+
...(skills ? { skills } : {}),
|
|
209
212
|
deferredDenyPaths: denyPaths.filter((entry) => backings.has(entry)),
|
|
210
213
|
limits,
|
|
211
214
|
agentId,
|
|
@@ -271,11 +274,13 @@ export const renderDaimonGrokServiceConfig = (registrations, options = {}) => ({
|
|
|
271
274
|
profileSha256: entry.profileSha256,
|
|
272
275
|
...(options.version === "v3" ? {} : { usageLedgerPath: entry.usageLedgerPath,
|
|
273
276
|
limits: { ...(entry.limits ?? DAIMON_GROK_ENGINE_BROKER.turnLimits.v1Defaults) },
|
|
274
|
-
model: { id: entry.model, reasoningEffort: entry.reasoningEffort }
|
|
277
|
+
model: { id: entry.model, reasoningEffort: entry.reasoningEffort },
|
|
278
|
+
...(entry.skills ? { skillsSha256: entry.skills.sha256 } : {}) })
|
|
275
279
|
})),
|
|
276
280
|
...(options.version === "v3" ? { slotPolicies: registrations.map(entry => ({
|
|
277
281
|
slot: entry.slot, usageLedgerPath: entry.usageLedgerPath,
|
|
278
282
|
limits: { ...(entry.limits ?? DAIMON_GROK_ENGINE_BROKER.turnLimits.v1Defaults) },
|
|
279
|
-
model: { id: entry.model, reasoningEffort: entry.reasoningEffort }
|
|
283
|
+
model: { id: entry.model, reasoningEffort: entry.reasoningEffort },
|
|
284
|
+
...(entry.skills ? { skillsSha256: entry.skills.sha256 } : {})
|
|
280
285
|
})) } : {})
|
|
281
286
|
});
|
|
@@ -42,6 +42,13 @@ export declare const trainingBrokerDeclarationSchema: z.ZodObject<{
|
|
|
42
42
|
refuse: "refuse";
|
|
43
43
|
"profile-only": "profile-only";
|
|
44
44
|
}>>;
|
|
45
|
+
skills: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
46
|
+
name: z.ZodString;
|
|
47
|
+
files: z.ZodArray<z.ZodObject<{
|
|
48
|
+
path: z.ZodString;
|
|
49
|
+
content: z.ZodString;
|
|
50
|
+
}, z.core.$strict>>;
|
|
51
|
+
}, z.core.$strict>>>;
|
|
45
52
|
}, z.core.$strict>;
|
|
46
53
|
export type TrainingBrokerDeclaration = z.infer<typeof trainingBrokerDeclarationSchema>;
|
|
47
54
|
export declare const parseTrainingBrokerDeclaration: (value: unknown) => TrainingBrokerDeclaration;
|
|
@@ -52,6 +52,18 @@ export const trainingBrokerDeclarationSchema = z.strictObject({
|
|
|
52
52
|
* datasets are held by DAC on their ancestor under both values, and a slot
|
|
53
53
|
* that cannot prove that denial is refused whatever this says.
|
|
54
54
|
*/
|
|
55
|
-
unenforcedBindPolicy: z.enum(["refuse", "profile-only"]).default("refuse")
|
|
55
|
+
unenforcedBindPolicy: z.enum(["refuse", "profile-only"]).default("refuse"),
|
|
56
|
+
/**
|
|
57
|
+
* The subject's declared skills in Daimon's engine-neutral form (the
|
|
58
|
+
* compiled `.agents/skills` tree), omitted when it declares none. Root
|
|
59
|
+
* provisioning renders them into every slot's `$GROK_HOME/skills`, and the
|
|
60
|
+
* entrypoint, supervisor and lease all resolve the projection from this one
|
|
61
|
+
* read-only input, so they bind the same `skillsSha256`. Daimon's own parser
|
|
62
|
+
* (mirrored in `runtime/daimon/grokWorkerSkills.ts`) enforces the bounds.
|
|
63
|
+
*/
|
|
64
|
+
skills: z.array(z.strictObject({
|
|
65
|
+
name: z.string().min(1).max(128),
|
|
66
|
+
files: z.array(z.strictObject({ path: z.string().min(1).max(1_024), content: z.string() })).min(1).max(64)
|
|
67
|
+
})).min(1).max(64).optional()
|
|
56
68
|
}).strict();
|
|
57
69
|
export const parseTrainingBrokerDeclaration = (value) => trainingBrokerDeclarationSchema.parse(value);
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
2
|
import { randomBytes } from "node:crypto";
|
|
3
3
|
import { lstat, readFile } from "node:fs/promises";
|
|
4
4
|
import os from "node:os";
|
|
5
|
-
import { promisify } from "node:util";
|
|
6
5
|
import { renderTrainingBrokerProvisioning } from "./provisioning.js";
|
|
6
|
+
import { runTrainingRootScript } from "./rootScript.js";
|
|
7
7
|
import { stageTrainingEvaluatorSecret } from "./evaluatorSecrets.js";
|
|
8
8
|
import { startBrokerProcesses, stopBrokerProcesses } from "./processes.js";
|
|
9
9
|
import { resetTrainingBrokerSlotAsOrganization } from "./reset.js";
|
|
@@ -11,8 +11,7 @@ import path from "node:path";
|
|
|
11
11
|
import { SpawnfileError } from "../../../shared/index.js";
|
|
12
12
|
import { DAIMON_GROK_ENGINE_BROKER } from "../../../runtime/daimon/contractManifest.js";
|
|
13
13
|
import { parseTrainingBrokerDeclaration } from "./declaration.js";
|
|
14
|
-
import { loadDaimonProjectionModule,
|
|
15
|
-
import { resolveTrainingGrokRegistration } from "./registration.js";
|
|
14
|
+
import { loadDaimonProjectionModule, resolveTrainingSlotProjection } from "./projection.js";
|
|
16
15
|
import { createTrainingSlotRuntime } from "./runtime.js";
|
|
17
16
|
import { createTrainingSlotSupervisor, serveTrainingSlotSupervisor, startTrainingSlot } from "./supervisor.js";
|
|
18
17
|
import { trainingSlotDescriptors, assertTrainingSlotsDocument, TRAINING_SLOTS_FILE, DAIMON_ORGANIZATION_UID, TRAINING_BOOTSTRAP_MOUNT, TRAINING_BROKER_DECLARATION_FILE, TRAINING_GRANT_HOME_ROOT, TRAINING_SUPERVISOR_SOCKET } from "./paths.js";
|
|
@@ -65,15 +64,13 @@ export const runTrainingBrokerEntrypoint = async (options) => {
|
|
|
65
64
|
const descriptors = trainingSlotDescriptors(declaration.slots);
|
|
66
65
|
const document = JSON.parse(await readFile(TRAINING_SLOTS_FILE, "utf8"));
|
|
67
66
|
assertTrainingSlotsDocument(document, declaration.slots);
|
|
68
|
-
const registrations = descriptors.map(p => resolveTrainingGrokRegistration({ ...declaration, slot: p.slot }));
|
|
69
67
|
const daimon = await (options.loadDaimon ?? loadDaimonProjectionModule)();
|
|
70
68
|
const reset = resetTrainingBrokerSlotAsOrganization;
|
|
71
|
-
const
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
}));
|
|
69
|
+
const slots = await Promise.all(descriptors.map(p => resolveTrainingSlotProjection(declaration, p.slot, daimon)));
|
|
70
|
+
const registrations = slots.map(slot => slot.registration);
|
|
71
|
+
const runtimes = slots.map(({ registration, projectionSha256 }) => createTrainingSlotRuntime({ declaration, registration, projectionSha256, reset }));
|
|
75
72
|
await stageTrainingEvaluatorSecret();
|
|
76
|
-
await
|
|
73
|
+
await runTrainingRootScript(renderTrainingBrokerProvisioning(registrations, declaration.maxRunRequests).join("\n"), { maxBuffer: 8 * 1024 * 1024 });
|
|
77
74
|
const children = await startBrokerProcesses({ log });
|
|
78
75
|
let server;
|
|
79
76
|
try {
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
export { parseTrainingBrokerDeclaration, trainingBrokerDeclarationSchema, type TrainingBrokerDeclaration } from "./declaration.js";
|
|
2
2
|
export { assertNotDesktopGrokAuth, desktopGrokAuthPaths, runTrainingBrokerEntrypoint, trainingChildArgv, TRAINING_GROK_BROKER_CONTROL_SOCKET_ENV, TRAINING_GROK_GRANT_HOME_ROOT_ENV } from "./entrypoint.js";
|
|
3
3
|
export { brokerProcessPlan, startBrokerProcesses, stopBrokerProcesses } from "./processes.js";
|
|
4
|
-
export { loadDaimonProjectionModule, resolveTrainingGrokProjection, trainingOrganizationRuntimeConfig } from "./projection.js";
|
|
4
|
+
export { loadDaimonProjectionModule, resolveTrainingGrokProjection, resolveTrainingSlotProjection, trainingOrganizationRuntimeConfig } from "./projection.js";
|
|
5
|
+
export { recycleTrainingSlot, TRAINING_RECYCLE_DEFAULT_TIMEOUT_MS, TRAINING_RECYCLE_MAX_REPLY_BYTES, type RecycleTrainingSlotOptions, type TrainingSlotRecycleReceipt } from "./recycleClient.js";
|
|
5
6
|
export { renderTrainingBrokerProvisioning, renderTrainingIdentities, renderTrainingSlotProvisioning, trainingSlotDirectories } from "./provisioning.js";
|
|
6
7
|
export { buildTrainingSlotReceipt, readGrokExecutableSha256, resolveBackingFilesystem, resolveTrainingCanaries } from "./receipt.js";
|
|
7
8
|
export { resolveTrainingGrokDenyPaths, resolveTrainingGrokRegistration } from "./registration.js";
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
export { parseTrainingBrokerDeclaration, trainingBrokerDeclarationSchema } from "./declaration.js";
|
|
2
2
|
export { assertNotDesktopGrokAuth, desktopGrokAuthPaths, runTrainingBrokerEntrypoint, trainingChildArgv, TRAINING_GROK_BROKER_CONTROL_SOCKET_ENV, TRAINING_GROK_GRANT_HOME_ROOT_ENV } from "./entrypoint.js";
|
|
3
3
|
export { brokerProcessPlan, startBrokerProcesses, stopBrokerProcesses } from "./processes.js";
|
|
4
|
-
export { loadDaimonProjectionModule, resolveTrainingGrokProjection, trainingOrganizationRuntimeConfig } from "./projection.js";
|
|
4
|
+
export { loadDaimonProjectionModule, resolveTrainingGrokProjection, resolveTrainingSlotProjection, trainingOrganizationRuntimeConfig } from "./projection.js";
|
|
5
|
+
export { recycleTrainingSlot, TRAINING_RECYCLE_DEFAULT_TIMEOUT_MS, TRAINING_RECYCLE_MAX_REPLY_BYTES } from "./recycleClient.js";
|
|
5
6
|
export { renderTrainingBrokerProvisioning, renderTrainingIdentities, renderTrainingSlotProvisioning, trainingSlotDirectories } from "./provisioning.js";
|
|
6
7
|
export { buildTrainingSlotReceipt, readGrokExecutableSha256, resolveBackingFilesystem, resolveTrainingCanaries } from "./receipt.js";
|
|
7
8
|
export { resolveTrainingGrokDenyPaths, resolveTrainingGrokRegistration } from "./registration.js";
|
|
@@ -50,3 +50,12 @@ export interface TrainingGrokProjection {
|
|
|
50
50
|
* list drifted from the bytes provisioning wrote.
|
|
51
51
|
*/
|
|
52
52
|
export declare const resolveTrainingGrokProjection: (declaration: TrainingBrokerDeclaration, registration: DaimonGrokRegistration, daimon: DaimonProjectionModule) => Promise<TrainingGrokProjection>;
|
|
53
|
+
/**
|
|
54
|
+
* One slot's registration and projection, exactly as the root entrypoint binds
|
|
55
|
+
* them into that slot's preflight receipt. The entrypoint and the public lease
|
|
56
|
+
* both call this, so the digest a lease hands Paideia cannot drift from the
|
|
57
|
+
* one the supervisor certified.
|
|
58
|
+
*/
|
|
59
|
+
export declare const resolveTrainingSlotProjection: (declaration: TrainingBrokerDeclaration, slot: number, daimon: DaimonProjectionModule) => Promise<TrainingGrokProjection & {
|
|
60
|
+
registration: DaimonGrokRegistration;
|
|
61
|
+
}>;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { SpawnfileError } from "../../../shared/index.js";
|
|
2
2
|
import { DAIMON_GROK_ENGINE_BROKER } from "../../../runtime/daimon/contractManifest.js";
|
|
3
3
|
import { trainingSlotPaths } from "./paths.js";
|
|
4
|
+
import { resolveTrainingGrokRegistration } from "./registration.js";
|
|
4
5
|
/** Indirect specifier: Daimon is an in-image runtime peer of the training container, never a Spawnfile source dependency. */
|
|
5
6
|
const loadRuntime = async () => {
|
|
6
7
|
const packageName = "@noopolis/daimon/runtime";
|
|
@@ -68,8 +69,16 @@ export const resolveTrainingGrokProjection = async (declaration, registration, d
|
|
|
68
69
|
acceptanceStorePath: trainingSlotPaths(registration.slot).state,
|
|
69
70
|
denyPaths: registration.denyPaths,
|
|
70
71
|
seccompProfileSha256: declaration.seccompProfileSha256,
|
|
71
|
-
profileSha256: registration.profileSha256
|
|
72
|
+
profileSha256: registration.profileSha256,
|
|
73
|
+
// The declaration's skills, exactly as root provisioning renders them; omitted for none so a
|
|
74
|
+
// skill-less projection stays byte-identical to one computed before skills existed.
|
|
75
|
+
...(declaration.skills?.length ? { skills: declaration.skills } : {})
|
|
72
76
|
});
|
|
77
|
+
// A Daimon older than 0.2.2 ignores `skills`, and a drifted mirror would provision a tree the broker
|
|
78
|
+
// refuses every turn: either way the projection must carry exactly the provisioned skills digest.
|
|
79
|
+
if (projection.skillsSha256 !== registration.skills?.sha256) {
|
|
80
|
+
throw new SpawnfileError("runtime_error", "Daimon's Grok broker projection skills digest differs from the provisioned skills; the training image needs @noopolis/daimon >= 0.2.2");
|
|
81
|
+
}
|
|
73
82
|
const clock = DAIMON_GROK_ENGINE_BROKER.worker.taskClock;
|
|
74
83
|
const expectedClock = { ...clock, directoryPath: clock.slotDirectoryPattern.replace("{slot}", String(registration.slot)),
|
|
75
84
|
timestampFilePath: clock.slotFilePattern.replace("{slot}", String(registration.slot)) };
|
|
@@ -86,3 +95,13 @@ export const resolveTrainingGrokProjection = async (declaration, registration, d
|
|
|
86
95
|
}
|
|
87
96
|
return { projection, projectionSha256: daimon.grokBrokerProjectionSha256(projection) };
|
|
88
97
|
};
|
|
98
|
+
/**
|
|
99
|
+
* One slot's registration and projection, exactly as the root entrypoint binds
|
|
100
|
+
* them into that slot's preflight receipt. The entrypoint and the public lease
|
|
101
|
+
* both call this, so the digest a lease hands Paideia cannot drift from the
|
|
102
|
+
* one the supervisor certified.
|
|
103
|
+
*/
|
|
104
|
+
export const resolveTrainingSlotProjection = async (declaration, slot, daimon) => {
|
|
105
|
+
const registration = resolveTrainingGrokRegistration({ ...declaration, slot });
|
|
106
|
+
return { registration, ...await resolveTrainingGrokProjection(declaration, registration, daimon) };
|
|
107
|
+
};
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { type Socket } from "node:net";
|
|
2
|
+
/** A reply is one short JSON line; anything longer is not this supervisor. */
|
|
3
|
+
export declare const TRAINING_RECYCLE_MAX_REPLY_BYTES = 65536;
|
|
4
|
+
/** Above the supervisor's 120 s drain deadline plus wipe, provision and canaries. */
|
|
5
|
+
export declare const TRAINING_RECYCLE_DEFAULT_TIMEOUT_MS = 180000;
|
|
6
|
+
export interface RecycleTrainingSlotOptions {
|
|
7
|
+
slot: number;
|
|
8
|
+
/** This trial's 32-byte hex nonce; the published receipt must echo it. */
|
|
9
|
+
nonce: string;
|
|
10
|
+
signal?: AbortSignal;
|
|
11
|
+
/** Opens the supervisor connection; defaults to the container's fixed supervisor socket. */
|
|
12
|
+
connect?: () => Socket;
|
|
13
|
+
timeoutMs?: number;
|
|
14
|
+
}
|
|
15
|
+
export interface TrainingSlotRecycleReceipt {
|
|
16
|
+
slot: number;
|
|
17
|
+
generation: number;
|
|
18
|
+
nonce: string;
|
|
19
|
+
/** The receipt path the supervisor says it published; callers compare it to the slot descriptor. */
|
|
20
|
+
receipt: string;
|
|
21
|
+
durationMs: number;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* One `recycle` over the root slot supervisor socket (`spawnfile.training-slot-supervisor.v2`).
|
|
25
|
+
*
|
|
26
|
+
* The verb takes a slot and a nonce, never a path or a command: the container
|
|
27
|
+
* replays its own audited provisioning and publishes the slot preflight receipt
|
|
28
|
+
* echoing this nonce. The answer must bind this request exactly (protocol,
|
|
29
|
+
* verb, slot, nonce, a generation of at least 1) or it is refused; a supervisor
|
|
30
|
+
* refusal surfaces with its own reason. The reply is bounded, and the call is
|
|
31
|
+
* bounded by `timeoutMs` and `signal`.
|
|
32
|
+
*/
|
|
33
|
+
export declare const recycleTrainingSlot: (options: RecycleTrainingSlotOptions) => Promise<TrainingSlotRecycleReceipt>;
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import { createConnection } from "node:net";
|
|
2
|
+
import { SpawnfileError } from "../../../shared/index.js";
|
|
3
|
+
import { TRAINING_MAX_SLOTS, TRAINING_SUPERVISOR_SOCKET } from "./paths.js";
|
|
4
|
+
import { TRAINING_SUPERVISOR_PROTOCOL } from "./supervisor.js";
|
|
5
|
+
/** A reply is one short JSON line; anything longer is not this supervisor. */
|
|
6
|
+
export const TRAINING_RECYCLE_MAX_REPLY_BYTES = 65_536;
|
|
7
|
+
/** Above the supervisor's 120 s drain deadline plus wipe, provision and canaries. */
|
|
8
|
+
export const TRAINING_RECYCLE_DEFAULT_TIMEOUT_MS = 180_000;
|
|
9
|
+
const NONCE = /^[a-f0-9]{64}$/u;
|
|
10
|
+
const REPLY_KEYS = ["durationMs", "generation", "nonce", "ok", "receipt", "slot", "v", "verb"];
|
|
11
|
+
const refuse = (message) => new SpawnfileError("runtime_error", message);
|
|
12
|
+
/** The supervisor's refusal text is diagnostic only; keep it one bounded printable line. */
|
|
13
|
+
const refusalText = (value) => typeof value === "string" ? value.replace(/[^\x20-\x7e]/gu, "?").slice(0, 512) || "unknown" : "unknown";
|
|
14
|
+
const bindReply = (answer, slot, nonce) => {
|
|
15
|
+
if (answer.ok !== true)
|
|
16
|
+
throw refuse(`Training slot ${slot} recycle refused: ${refusalText(answer.error)}`);
|
|
17
|
+
if (Object.keys(answer).sort().join(",") !== REPLY_KEYS.join(",") || answer.verb !== "recycle") {
|
|
18
|
+
throw refuse("Training slot recycle answer is not a recycle receipt");
|
|
19
|
+
}
|
|
20
|
+
if (answer.slot !== slot)
|
|
21
|
+
throw refuse(`Training slot recycle answer names slot ${String(answer.slot)}, not the leased slot ${slot}`);
|
|
22
|
+
if (answer.nonce !== nonce)
|
|
23
|
+
throw refuse("Training slot recycle answer does not echo this trial's nonce");
|
|
24
|
+
if (!Number.isSafeInteger(answer.generation) || answer.generation < 1) {
|
|
25
|
+
throw refuse("Training slot recycle answer carries no valid generation");
|
|
26
|
+
}
|
|
27
|
+
if (typeof answer.receipt !== "string" || answer.receipt.length === 0)
|
|
28
|
+
throw refuse("Training slot recycle answer names no receipt");
|
|
29
|
+
if (typeof answer.durationMs !== "number" || !Number.isFinite(answer.durationMs) || answer.durationMs < 0) {
|
|
30
|
+
throw refuse("Training slot recycle answer carries no valid duration");
|
|
31
|
+
}
|
|
32
|
+
return { slot, generation: answer.generation, nonce, receipt: answer.receipt, durationMs: answer.durationMs };
|
|
33
|
+
};
|
|
34
|
+
/**
|
|
35
|
+
* One `recycle` over the root slot supervisor socket (`spawnfile.training-slot-supervisor.v2`).
|
|
36
|
+
*
|
|
37
|
+
* The verb takes a slot and a nonce, never a path or a command: the container
|
|
38
|
+
* replays its own audited provisioning and publishes the slot preflight receipt
|
|
39
|
+
* echoing this nonce. The answer must bind this request exactly (protocol,
|
|
40
|
+
* verb, slot, nonce, a generation of at least 1) or it is refused; a supervisor
|
|
41
|
+
* refusal surfaces with its own reason. The reply is bounded, and the call is
|
|
42
|
+
* bounded by `timeoutMs` and `signal`.
|
|
43
|
+
*/
|
|
44
|
+
export const recycleTrainingSlot = (options) => {
|
|
45
|
+
const { slot, nonce } = options;
|
|
46
|
+
if (!Number.isInteger(slot) || slot < 0 || slot >= TRAINING_MAX_SLOTS) {
|
|
47
|
+
return Promise.reject(new SpawnfileError("validation_error", `Training slot recycle requires a slot index from 0 to ${TRAINING_MAX_SLOTS - 1}`));
|
|
48
|
+
}
|
|
49
|
+
if (!NONCE.test(nonce))
|
|
50
|
+
return Promise.reject(new SpawnfileError("validation_error", "Training slot recycle requires a 32-byte hex nonce"));
|
|
51
|
+
try {
|
|
52
|
+
options.signal?.throwIfAborted();
|
|
53
|
+
}
|
|
54
|
+
catch (error) {
|
|
55
|
+
return Promise.reject(error);
|
|
56
|
+
}
|
|
57
|
+
return new Promise((resolve, reject) => {
|
|
58
|
+
const socket = (options.connect ?? (() => createConnection(TRAINING_SUPERVISOR_SOCKET)))();
|
|
59
|
+
let buffer = Buffer.alloc(0), settled = false;
|
|
60
|
+
const onAbort = () => finish(options.signal?.reason instanceof Error ? options.signal.reason : refuse("Training slot recycle was aborted"));
|
|
61
|
+
const timer = setTimeout(() => finish(refuse(`Training slot ${slot} recycle timed out before the supervisor answered`)), options.timeoutMs ?? TRAINING_RECYCLE_DEFAULT_TIMEOUT_MS);
|
|
62
|
+
function finish(error, value) {
|
|
63
|
+
if (settled)
|
|
64
|
+
return;
|
|
65
|
+
settled = true;
|
|
66
|
+
clearTimeout(timer);
|
|
67
|
+
options.signal?.removeEventListener("abort", onAbort);
|
|
68
|
+
socket.destroy();
|
|
69
|
+
if (error === undefined)
|
|
70
|
+
resolve(value);
|
|
71
|
+
else
|
|
72
|
+
reject(error);
|
|
73
|
+
}
|
|
74
|
+
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
75
|
+
socket.on("error", (error) => finish(refuse(`Training slot recycle could not reach the root supervisor: ${error.message}`)));
|
|
76
|
+
socket.on("close", () => finish(refuse("Training slot recycle connection closed before an answer")));
|
|
77
|
+
socket.on("connect", () => socket.write(`${JSON.stringify({ v: TRAINING_SUPERVISOR_PROTOCOL, verb: "recycle", slot, nonce })}\n`));
|
|
78
|
+
socket.on("data", (chunk) => {
|
|
79
|
+
buffer = Buffer.concat([buffer, chunk]);
|
|
80
|
+
const end = buffer.indexOf(0x0a);
|
|
81
|
+
if (end < 0 ? buffer.length > TRAINING_RECYCLE_MAX_REPLY_BYTES : end > TRAINING_RECYCLE_MAX_REPLY_BYTES) {
|
|
82
|
+
finish(refuse("Training slot recycle answer exceeded its bound"));
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
if (end < 0)
|
|
86
|
+
return;
|
|
87
|
+
let answer;
|
|
88
|
+
try {
|
|
89
|
+
answer = JSON.parse(buffer.subarray(0, end).toString("utf8"));
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
finish(refuse("Training slot recycle answer is not JSON"));
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
if (!answer || typeof answer !== "object" || Array.isArray(answer)) {
|
|
96
|
+
finish(refuse("Training slot recycle answer is not an object"));
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
const record = answer;
|
|
100
|
+
if (record.v !== TRAINING_SUPERVISOR_PROTOCOL) {
|
|
101
|
+
finish(refuse("Training slot recycle answer names another protocol"));
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
try {
|
|
105
|
+
finish(undefined, bindReply(record, slot, nonce));
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
finish(error);
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
});
|
|
112
|
+
};
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type DaimonGrokBrokerModel, type DaimonGrokBrokerReasoningEffort } from "../../../runtime/daimon/contractManifest.js";
|
|
2
|
+
import { type DaimonAgentSkill } from "../../../runtime/daimon/grokWorkerSkills.js";
|
|
2
3
|
import { type DaimonGrokRegistration } from "../../containerDaimonGrokWorkerRender.js";
|
|
3
4
|
/**
|
|
4
5
|
* The one training slot's deny list: Daimon's own protected set for a
|
|
@@ -32,6 +33,8 @@ export interface TrainingGrokSlotInput {
|
|
|
32
33
|
maxTokens: number;
|
|
33
34
|
timeoutMs: number;
|
|
34
35
|
};
|
|
36
|
+
/** The declaration's skills; rendered into the slot's `$GROK_HOME/skills` and bound as `skillsSha256`. */
|
|
37
|
+
skills?: readonly DaimonAgentSkill[];
|
|
35
38
|
}
|
|
36
39
|
/**
|
|
37
40
|
* The single brokered Grok registration a training container runs, at the
|
|
@@ -2,6 +2,7 @@ import path from "node:path";
|
|
|
2
2
|
import { SpawnfileError } from "../../../shared/index.js";
|
|
3
3
|
import { DAIMON_GROK_BROKER_MODELS, DAIMON_GROK_BROKER_REASONING_EFFORTS, DAIMON_GROK_ENGINE_BROKER } from "../../../runtime/daimon/contractManifest.js";
|
|
4
4
|
import { DAIMON_GROK_WORKER_HOME_DIRECTORY, daimonGrokWorkerSandboxProfileSha256, renderDaimonGrokWorkerSandboxProfile, resolveDaimonGrokClockedWorkerConfig } from "../../../runtime/daimon/grokWorkerContract.js";
|
|
5
|
+
import { resolveDaimonGrokWorkerSkills } from "../../../runtime/daimon/grokWorkerSkills.js";
|
|
5
6
|
import { DAIMON_GROK_BASE_PROFILE_GRANTS, assertCanonicalRegisteredPath } from "../../containerDaimonGrokWorkerRender.js";
|
|
6
7
|
import { trainingSlotPaths, trainingEvaluatorPaths, TRAINING_BOOTSTRAP_MOUNT, TRAINING_REALM_MOUNT, TRAINING_DEFERRED_DENY_PATHS } from "./paths.js";
|
|
7
8
|
const fail = (message) => {
|
|
@@ -79,7 +80,9 @@ export const resolveTrainingGrokRegistration = (input) => {
|
|
|
79
80
|
["workspace", p.workspace], ["runtime home", p.runtimeHome],
|
|
80
81
|
["usage ledger", p.usageLedger]])
|
|
81
82
|
assertCanonicalRegisteredPath(label, value);
|
|
83
|
+
const skills = resolveDaimonGrokWorkerSkills(input.skills);
|
|
82
84
|
return {
|
|
85
|
+
...(skills ? { skills } : {}),
|
|
83
86
|
agentId: input.agentId,
|
|
84
87
|
config: config.bytes,
|
|
85
88
|
configVariant: "clocked",
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Linux `MAX_ARG_STRLEN`: the most bytes one `execve` argument may carry. A
|
|
3
|
+
* generated root program passed as `bash -c <script>` dies with `E2BIG` past
|
|
4
|
+
* it — and a slot program embeds its declared skills, so the size grows with
|
|
5
|
+
* the declaration. macOS has no per-argument limit, which is why this is
|
|
6
|
+
* asserted in tests rather than observed.
|
|
7
|
+
*/
|
|
8
|
+
export declare const LINUX_MAX_ARG_STRLEN = 131072;
|
|
9
|
+
export type TrainingRootScriptRunner = (file: string, args: readonly string[], options: {
|
|
10
|
+
maxBuffer: number;
|
|
11
|
+
}) => Promise<{
|
|
12
|
+
stdout: string;
|
|
13
|
+
stderr: string;
|
|
14
|
+
}>;
|
|
15
|
+
export interface TrainingRootScriptOptions {
|
|
16
|
+
shell?: string;
|
|
17
|
+
maxBuffer?: number;
|
|
18
|
+
run?: TrainingRootScriptRunner;
|
|
19
|
+
tmpRoot?: string;
|
|
20
|
+
}
|
|
21
|
+
/** The argv a generated script runs under: bounded and independent of the script's size. */
|
|
22
|
+
export declare const trainingRootScriptArgv: (scriptFile: string) => string[];
|
|
23
|
+
/**
|
|
24
|
+
* Runs a generated root shell program from a fresh `0700` directory holding a
|
|
25
|
+
* `0600` script file, never from argv and never from stdin (a child that reads
|
|
26
|
+
* stdin would consume the rest of the program). The directory is removed
|
|
27
|
+
* afterwards, whatever the outcome.
|
|
28
|
+
*/
|
|
29
|
+
export declare const runTrainingRootScript: (script: string, options?: TrainingRootScriptOptions) => Promise<{
|
|
30
|
+
stdout: string;
|
|
31
|
+
stderr: string;
|
|
32
|
+
}>;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
3
|
+
import os from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { promisify } from "node:util";
|
|
6
|
+
/**
|
|
7
|
+
* Linux `MAX_ARG_STRLEN`: the most bytes one `execve` argument may carry. A
|
|
8
|
+
* generated root program passed as `bash -c <script>` dies with `E2BIG` past
|
|
9
|
+
* it — and a slot program embeds its declared skills, so the size grows with
|
|
10
|
+
* the declaration. macOS has no per-argument limit, which is why this is
|
|
11
|
+
* asserted in tests rather than observed.
|
|
12
|
+
*/
|
|
13
|
+
export const LINUX_MAX_ARG_STRLEN = 131_072;
|
|
14
|
+
const execFileAsync = promisify(execFile);
|
|
15
|
+
/** The argv a generated script runs under: bounded and independent of the script's size. */
|
|
16
|
+
export const trainingRootScriptArgv = (scriptFile) => ["--noprofile", "--norc", "-eu", scriptFile];
|
|
17
|
+
/**
|
|
18
|
+
* Runs a generated root shell program from a fresh `0700` directory holding a
|
|
19
|
+
* `0600` script file, never from argv and never from stdin (a child that reads
|
|
20
|
+
* stdin would consume the rest of the program). The directory is removed
|
|
21
|
+
* afterwards, whatever the outcome.
|
|
22
|
+
*/
|
|
23
|
+
export const runTrainingRootScript = async (script, options = {}) => {
|
|
24
|
+
const directory = await mkdtemp(path.join(options.tmpRoot ?? os.tmpdir(), "spawnfile-root-script-"));
|
|
25
|
+
try {
|
|
26
|
+
const file = path.join(directory, "script.sh");
|
|
27
|
+
await writeFile(file, script, { mode: 0o600, flag: "wx" });
|
|
28
|
+
return await (options.run ?? execFileAsync)(options.shell ?? "/bin/bash", trainingRootScriptArgv(file), { maxBuffer: options.maxBuffer ?? 1024 * 1024 });
|
|
29
|
+
}
|
|
30
|
+
finally {
|
|
31
|
+
await rm(directory, { recursive: true, force: true });
|
|
32
|
+
}
|
|
33
|
+
};
|
|
@@ -9,6 +9,7 @@ import { MOUNT_AWARE_CLEAR_HELPER } from "../../containerDaimonBrokerRender.js";
|
|
|
9
9
|
import { probeTrainingSiblingClock } from "./clockProbe.js";
|
|
10
10
|
import { renderTrainingTaskClockAssertions } from "./taskClock.js";
|
|
11
11
|
import { renderTrainingSlotProvisioning } from "./provisioning.js";
|
|
12
|
+
import { runTrainingRootScript } from "./rootScript.js";
|
|
12
13
|
import { buildTrainingSlotReceipt, readGrokExecutableSha256, resolveTrainingCanaries } from "./receipt.js";
|
|
13
14
|
import { DAIMON_ORGANIZATION_UID, TRAINING_HOST_BIND_DENY_PATHS, TRAINING_SEALED_DENY_PATHS, trainingSlotPaths, trainingWipeTargets, trainingClearTargets, TRAINING_SUPERVISOR_LOG } from "./paths.js";
|
|
14
15
|
const run = promisify(execFile);
|
|
@@ -38,13 +39,13 @@ export const createTrainingSlotRuntime = (options) => {
|
|
|
38
39
|
// Root holds `CAP_CHOWN` and `CAP_DAC_READ_SEARCH` but not `CAP_DAC_OVERRIDE`, so it cannot delete
|
|
39
40
|
// inside a directory it handed to the worker or the broker. Reclaim each tree first, then remove it.
|
|
40
41
|
const script = renderTrainingSlotWipe(p.slot).join("\n");
|
|
41
|
-
await
|
|
42
|
+
await runTrainingRootScript(script, { shell, maxBuffer: 1024 * 1024 }).catch((error) => {
|
|
42
43
|
const detail = error;
|
|
43
44
|
throw new SpawnfileError("runtime_error", `Training slot wipe failed: ${(detail.stderr ?? "").slice(-1024).trim() || "no diagnostic"}`);
|
|
44
45
|
});
|
|
45
46
|
},
|
|
46
47
|
provision: async () => {
|
|
47
|
-
const result = await
|
|
48
|
+
const result = await runTrainingRootScript(script, { shell, maxBuffer: 8 * 1024 * 1024 }).catch((error) => {
|
|
48
49
|
const detail = error;
|
|
49
50
|
throw new SpawnfileError("runtime_error", `Training slot provisioning failed: ${(detail.stderr ?? detail.stdout ?? "").slice(-2048).trim()}`);
|
|
50
51
|
});
|
|
@@ -52,7 +53,7 @@ export const createTrainingSlotRuntime = (options) => {
|
|
|
52
53
|
log(`provisioning: ${result.stdout.trim().slice(-2048)}`);
|
|
53
54
|
},
|
|
54
55
|
canaries: async () => {
|
|
55
|
-
await
|
|
56
|
+
await runTrainingRootScript(renderTrainingTaskClockAssertions(options.registration).join("\n"), { shell });
|
|
56
57
|
return resolveTrainingCanaries({
|
|
57
58
|
denyPaths: options.registration.denyPaths, probe, log, hostBindPaths: TRAINING_HOST_BIND_DENY_PATHS,
|
|
58
59
|
sealedPaths: TRAINING_SEALED_DENY_PATHS,
|
|
@@ -10,3 +10,6 @@ export * from "./execution.js";
|
|
|
10
10
|
export * from "./moments.js";
|
|
11
11
|
export * from "./compiledAgent.js";
|
|
12
12
|
export { TRAINING_FAKETIME_LIBRARY_PATH } from "./container/contract.js";
|
|
13
|
+
export * from "./subjectLease.js";
|
|
14
|
+
export * from "./subjectDeclaration.js";
|
|
15
|
+
export { recycleTrainingSlot, type RecycleTrainingSlotOptions, type TrainingSlotRecycleReceipt } from "./broker/recycleClient.js";
|
|
@@ -8,3 +8,6 @@ export * from "./execution.js";
|
|
|
8
8
|
export * from "./moments.js";
|
|
9
9
|
export * from "./compiledAgent.js";
|
|
10
10
|
export { TRAINING_FAKETIME_LIBRARY_PATH } from "./container/contract.js";
|
|
11
|
+
export * from "./subjectLease.js";
|
|
12
|
+
export * from "./subjectDeclaration.js";
|
|
13
|
+
export { recycleTrainingSlot } from "./broker/recycleClient.js";
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "spawnfile",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.20",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "spawnfile",
|
|
9
|
-
"version": "0.1.
|
|
9
|
+
"version": "0.1.20",
|
|
10
10
|
"license": "MIT",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"@noopolis/stele": "0.0.2",
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { type DaimonAgentSkill } from "../../../runtime/daimon/grokWorkerSkills.js";
|
|
2
|
+
import type { TrainingContext } from "../contract.js";
|
|
1
3
|
import type { TrainingPreparationConfig } from "./contract.js";
|
|
2
4
|
export interface PreparedTrainingBroker {
|
|
3
5
|
declarationPath: string;
|
|
@@ -18,4 +20,11 @@ export interface PreparedTrainingBroker {
|
|
|
18
20
|
* `docker create` — a dry run, a failed build — must still refuse the
|
|
19
21
|
* developer's desktop Grok login rather than report it later.
|
|
20
22
|
*/
|
|
21
|
-
|
|
23
|
+
/**
|
|
24
|
+
* The subject's declared skills, read from the context's pinned sources and
|
|
25
|
+
* refused if any byte changed since capture: exactly the `SKILL.md` files the
|
|
26
|
+
* compiler emits into its `.agents/skills`, in Daimon's engine-neutral form.
|
|
27
|
+
* `undefined` for none keeps a skill-less declaration byte-identical.
|
|
28
|
+
*/
|
|
29
|
+
export declare const trainingBrokerSkills: (context: Pick<TrainingContext, "skills">) => Promise<DaimonAgentSkill[] | undefined>;
|
|
30
|
+
export declare const prepareTrainingBroker: (broker: NonNullable<TrainingPreparationConfig["broker"]>, root: string, staging: string, write: boolean, slots?: number, skills?: readonly DaimonAgentSkill[]) => Promise<PreparedTrainingBroker>;
|