software-defence-factory 0.3.5 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -19,10 +19,12 @@ Choose the part you need:
19
19
 
20
20
  | Outcome | Command / guide |
21
21
  | --- | --- |
22
+ | Set up an operator, worker and application | [Setup plan and acceptance checklist](docs/setup.md) |
22
23
  | Use the method with your existing agent | `software-defence-factory kit --output ./factory-kit` — exports a new staging directory |
23
24
  | Try the runtime without inference | `software-defence-factory demo` — Docker required; synthetic sample only |
24
25
  | Connect an existing repository | [Runtime quickstart](docs/quickstart.md) |
25
26
  | Understand installation and updates | [npm and npx](docs/npm.md) |
27
+ | Restore a dashboard after boot or reconnect remotely | [Services and SSH tunnels](docs/services.md) |
26
28
  | Review the evidence and limits | [Qualification](docs/proof.md) |
27
29
 
28
30
  The runtime supplies policy and six focused skills to its isolated jobs. `init` configures a private installation; it does not modify the application or start work. Model access and the application's real check command must be configured before using it for delivery.
@@ -68,6 +70,7 @@ npm run check
68
70
  ```
69
71
 
70
72
  CI builds the dashboard and checks Node 22/24. A version increase merged to `main` is published to npm through the configured release workflow. Installed CLIs can update on invocation when all installations are stopped. See [release and update behavior](docs/npm.md).
73
+ Managed Linux services can also opt into daily updates that reserve idle controllers, preserve stopped projects and restore the prior release if startup fails. See [service operation](docs/services.md).
71
74
 
72
75
  This is a test release. Synthetic qualification demonstrates control flow and isolation, not model quality, application correctness or production readiness. Follow [AGENTS.md](AGENTS.md) for contributions and [SECURITY.md](SECURITY.md) for the trust boundaries.
73
76
 
@@ -8,6 +8,7 @@ import { ROOT, PINS, DEFAULT_STATE, configAt, save, json, run, stream, digest, a
8
8
  import { admitIncident } from '../factory/incident.mjs';
9
9
  import { DEFAULT_DEMO_STATE } from '../factory/paths.mjs';
10
10
  import { bootstrap, registerInstallation, VERSION } from '../factory/updates.mjs';
11
+ import { hasService, manageService, serviceDefinition, withServiceOperation, isManagedLaunch } from '../factory/services.mjs';
11
12
 
12
13
  try {
13
14
  const handled = await bootstrap(process.argv.slice(2));
@@ -94,7 +95,8 @@ async function stop() {
94
95
  const {pid}=json(lock);
95
96
  if(alive(pid)) {
96
97
  const identity=run('ps',['-p',String(pid),'-o','command=']);
97
- if(!identity.includes(join(ROOT,'factory/supervisor.mjs'))||!identity.includes(state))throw new Error('PID identity changed; refusing to signal an unrelated process');
98
+ const controllerProcess=identity.includes(join(ROOT,'factory/supervisor.mjs'))||identity.includes(join(ROOT,'bin/software-defence-factory.mjs')+' serve');
99
+ if(!controllerProcess||!identity.includes(state))throw new Error('PID identity changed; refusing to signal an unrelated process');
98
100
  process.kill(pid,'SIGTERM');
99
101
  for(let i=0;i<40&&existsSync(lock);i++)await sleep(250);
100
102
  if(existsSync(lock))throw new Error('Stop unconfirmed; inspect supervisor and do not start replacement workers');
@@ -134,18 +136,27 @@ async function jobAction(action) {
134
136
 
135
137
  try {
136
138
  if(command==='init') { if(!flags.repo)throw new Error('init requires --repo /path/to/existing/git/repo');init(flags.repo,flags.agent,flags.check,flags.port); }
137
- else if(command==='install')await install();
138
- else if(command==='up')await up();
139
- else if(command==='stop')await stop();
139
+ else if(command==='install')await withServiceOperation('install',install);
140
+ else if(command==='up') { if(hasService(state))await manageService('controller','start',state);else await withServiceOperation('up',up); }
141
+ else if(command==='stop') { if(hasService(state))await manageService('controller','stop',state);else await withServiceOperation('stop',stop); }
140
142
  else if(command==='serve') {
143
+ const managed=isManagedLaunch(state);
144
+ if(hasService(state)&&!managed)throw new Error('This installation is managed; use service start instead of foreground serve');
145
+ const launch=async()=>{
141
146
  if(process.getuid()===0)throw new Error('Use a dedicated unprivileged operator account');
142
147
  const lock=join(state,'supervisor.json');if(existsSync(lock)){if(alive(json(lock).pid))throw new Error('Supervisor already running');rmSync(lock);}
143
- registerInstallation(state);await portFree(configAt(state).port);await stream(process.execPath,[join(ROOT,'factory/supervisor.mjs'),state]);
148
+ if(!existsSync(join(state,'engine.json')))throw new Error('Run install first');
149
+ run('docker',['image','inspect',configAt(state).image]);
150
+ registerInstallation(state);await portFree(configAt(state).port);
151
+ const { supervise } = await import('../factory/supervisor.mjs');await supervise(state);
152
+ };
153
+ if(managed)await launch();else await withServiceOperation('serve',launch);
144
154
  }
145
155
  else if(command==='service') {
146
- configAt(state);const q=s=>JSON.stringify(s.replaceAll('%','%%'));
147
- console.log(`[Unit]\nDescription=Software & Defence Factory\nAfter=network-online.target\n\n[Service]\nType=simple\nWorkingDirectory=${q(ROOT)}\nExecStart=${q(process.execPath)} ${q(join(ROOT,'bin/software-defence-factory.mjs'))} serve --state ${q(state)}\nRestart=on-failure\nRestartSec=10\nTimeoutStopSec=30\nKillMode=control-group\nUMask=0077\n\n[Install]\nWantedBy=default.target`);
156
+ if(!positional.length || positional[0]==='print')console.log(serviceDefinition(state));
157
+ else await manageService('controller',positional[0],state,flags);
148
158
  }
159
+ else if(command==='tunnel')await manageService('tunnel',positional[0],state,flags);
149
160
  else if(command==='status') { const snapshot=await api(state,'/api/v1/status');delete snapshot.csrf_token;console.log(JSON.stringify(snapshot,null,2)); }
150
161
  else if(command==='doctor') {
151
162
  const config=configAt(state);console.log(JSON.stringify({node:process.version,docker:run('docker',['info','--format','{{.ServerVersion}}']),engineInstalled:existsSync(join(state,'engine.json')),repo:config.repo,agent:config.agent,checksConfigured:!!config.check?.trim(),inference:'Not called or verified',dashboard:`http://127.0.0.1:${config.port}`},null,2));
@@ -175,7 +186,7 @@ try {
175
186
  run('git',['-C',repo,'-c','user.name=Factory demo','-c','user.email=demo@localhost','commit','-m','Synthetic fixture']);
176
187
  init(repo,'mock',"test \"$(cat value.txt)\" = fixed",Number(flags.port || 7332));
177
188
  } else if(configAt(state).agent!=='mock')throw new Error('Demo requires a mock configuration');
178
- await install();await up();console.log(JSON.stringify(await submit('software','Synthetic installation qualification: fix value.txt. No inference is used.')));
189
+ await withServiceOperation('demo startup',async()=>{await install();await up();});console.log(JSON.stringify(await submit('software','Synthetic installation qualification: fix value.txt. No inference is used.')));
179
190
  console.log('Review the synthetic change in the dashboard and approve its handoff.');
180
191
  } else if(['version','--version','-v'].includes(command))console.log(VERSION);
181
192
  else if(command==='qualify') {
@@ -183,7 +194,7 @@ try {
183
194
  } else if(command==='kit') {
184
195
  if(!flags.output)throw new Error('kit requires --output NEW_DIRECTORY');
185
196
  await stream(process.execPath,[join(ROOT,'scripts/export-kit.mjs'),resolve(flags.output)]);
186
- } else if(command==='help')console.log(`Software & Defence Factory ${VERSION} (test release)
197
+ } else if(['help','--help','-h'].includes(command))console.log(`Software & Defence Factory ${VERSION} (test release)
187
198
 
188
199
  kit --output NEW_DIRECTORY Export the portable method without a runtime
189
200
  demo Install and run a synthetic sample (no model key)
@@ -191,8 +202,16 @@ try {
191
202
  init --repo PATH --agent codex|pi|custom --check "npm ci && npm test"
192
203
  install Build the isolated job image; the controller ships with the CLI
193
204
  doctor | up | status | stop Inspect / operate your private installation
194
- serve Foreground supervisor for a VPS service
195
- service Print a systemd user-service definition
205
+ serve Foreground supervisor
206
+ service [print] Print a systemd user-service definition
207
+ service install|start|stop|restart Manage a Linux user service (--state PATH)
208
+ install accepts --group EXISTING_GROUP
209
+ service status|logs|uninstall Diagnose / remove service, preserve private state
210
+ service update Update idle managed controllers, restore on failure
211
+ service updates --auto on|off|status Daily idle updates through a systemd timer
212
+ service resume Release a reconciled maintenance reservation
213
+ tunnel install|start|stop|status|logs|uninstall --host SSH_ALIAS --port PORT
214
+ Persistent loopback SSH tunnel (macOS/Linux)
196
215
  run --file task.md | --issue URL Submit one software vertical slice
197
216
  incident --file incident.json Submit a private, read-only incident draft
198
217
  approve JOB_ID | cancel JOB_ID Review gate / stop this attempt
@@ -205,6 +224,7 @@ Runtime commands accept --state PATH. Default: ${DEFAULT_STATE}
205
224
  Demo default: ${DEFAULT_DEMO_STATE}
206
225
  The npm CLI keeps state outside the package; updates wait for stopped installations.
207
226
  Dashboard binds only to loopback; use SSH for remote access.
208
- See docs/quickstart.md for inference, hosting, evidence and recovery.`);
227
+ Setup plan: ${join(ROOT, 'docs/setup.md')}
228
+ See docs/quickstart.md for task execution, evidence and recovery.`);
209
229
  else throw new Error(`Unknown command: ${command}`);
210
230
  } catch(error) {console.error(`Factory: ${error.message}`);process.exitCode=1;}
package/docs/npm.md CHANGED
@@ -42,7 +42,12 @@ An npm-installed CLI checks npm's `latest` stable release at most once a day
42
42
  when invoked. It downloads and activates a newer release when all registered
43
43
  installations are stopped and no executor requires reconciliation. It does not
44
44
  run a background updater or interrupt a job. Routine `status`, `stop`, `cancel`,
45
- `serve` and version commands do not initiate automatic downloads.
45
+ `serve`, service/tunnel management, help and version commands do not initiate automatic downloads.
46
+
47
+ For an always-running Linux installation, opt into the separate managed daily
48
+ timer with `service updates --auto on`. It reserves idle controllers, updates
49
+ and restores their prior running set without interrupting a job. See
50
+ [services](services.md) for installation, maintenance recovery and limitations.
46
51
 
47
52
  ```sh
48
53
  software-defence-factory update --check
@@ -1,5 +1,7 @@
1
1
  # Runtime quickstart
2
2
 
3
+ For a new worker or remote operator, start with the [setup plan](setup.md).
4
+
3
5
  Install Node 22.13+, Git and Docker Engine/Desktop. Use an unprivileged account with Docker access. Install `software-defence-factory` through npm, or invoke the same package with npx. No factory source checkout is required.
4
6
 
5
7
  ## Qualify a synthetic installation
@@ -55,7 +57,7 @@ ssh -N -L 127.0.0.1:7331:127.0.0.1:7331 your-host
55
57
 
56
58
  Open http://127.0.0.1:7331 on the client. Use the same local/remote port because the HTTP service validates its Host header. The SSH connection must remain open. Access also works across different networks when your configured private network connects the hosts.
57
59
 
58
- Use `status`, `cancel JOB_ID`, `retry JOB_ID` and `stop`, always with the selected `--state`. `service` prints a systemd user-service definition; review and install it through the host's normal service management. It is not enabled by printing it. See [recovery](recovery.md).
60
+ Use `status`, `cancel JOB_ID`, `retry JOB_ID` and `stop`, always with the selected `--state`. `service install --state PATH` installs and enables the supported Linux user service; `service status`, `service logs` and `service uninstall` operate it. Bare `service` only prints a definition. For managed startup, persistent SSH tunnels and daily idle updates, follow [services](services.md). See [recovery](recovery.md) for interrupted attempts.
59
61
 
60
62
  ## Native application builds
61
63
 
@@ -0,0 +1,23 @@
1
+ # Recovery and retained evidence
2
+
3
+ Use `status --state PATH` and the private supervisor.log to identify the active installation. Stop it before replacing its package or container image.
4
+
5
+ - A normal `stop` signals the controller, waits for the executor process group, removes its labelled containers and retains the database and artifacts.
6
+ - An unconfirmed running attempt becomes `interrupted` on controller restart. It is never silently considered successful.
7
+ - `retry JOB_ID` reconciles the previous process group and containers. A live or unknown writer blocks retry. For a new build/defence attempt, the prior checkout is retained as previous-checkout-*.
8
+ - A failed verification can retry the same unchanged candidate after the check environment is repaired. A changed candidate needs a fresh verification/review sequence.
9
+ - A requested revision retains previous evidence and starts a new build from the source repository with the accumulated feedback. It does not reuse earlier approval.
10
+ - Removing a stopped task from the dashboard hides its queue record. Private artifacts and its deleted_at record remain on disk; this is not secure erasure.
11
+
12
+ Do not remove active.json merely to unblock a job. Establish that its PID, process group and labelled containers are stopped. PID reuse or missing process identity requires operator investigation. Preserve logs and work before cleanup.
13
+
14
+ For backup, stop the installation and copy the complete private state directory, including SQLite files, factory.json and credentials, to an authorized private destination. Restore only while stopped. Update the repository path if it moved, verify ownership/permissions and the pinned image, then inspect state before any retry. Keep previous backups; no automatic destructive schema migration is provided.
15
+
16
+ Earlier experimental engines use a different journal. Start a new state directory for the native 0.3 runtime; preserve old journals separately. There is no automatic import of their jobs or approval state.
17
+
18
+ Verification cleanup makes owned scratch directories traversable before removing
19
+ them and never follows their symlinks. It runs only after container stop is
20
+ confirmed. If a filesystem error still prevents cleanup, the attempt fails and
21
+ retains the original check exit and private log path alongside the cleanup error.
22
+ Inspect that retained attempt before manual removal; never substitute the source
23
+ candidate path for the scratch path.
@@ -0,0 +1,156 @@
1
+ # Persistent controllers and remote dashboards
2
+
3
+ For the complete host-to-application sequence and reboot checklist, start with
4
+ [the setup plan](setup.md). Install the npm CLI first. A Factory controller is a Linux systemd **user**
5
+ service; SSH tunnels support Linux systemd and macOS launchd. Neither needs a
6
+ root controller. macOS can still run a local controller with `up`.
7
+
8
+ ## Linux controller
9
+
10
+ Configure and install a runtime using the [quickstart](quickstart.md), then:
11
+
12
+ ```sh
13
+ software-defence-factory stop --state /absolute/private/state
14
+ software-defence-factory service install --state /absolute/private/state
15
+ software-defence-factory service status --state /absolute/private/state
16
+ ```
17
+
18
+ Installation enables and starts exactly this controller. It does not submit a
19
+ new task. Existing queued jobs will execute when any controller starts; inspect
20
+ the queue before adopting an installation. Interrupted attempts still require
21
+ the normal explicit reconciliation/retry. To keep a product paused, leave its
22
+ controller stopped and do not install a service for it.
23
+
24
+ The current user must have Docker access; Docker itself must be enabled at boot.
25
+ The CLI records the current PATH, Node executable and private state/data homes.
26
+ The launcher and installed runtime are retained outside the npm/npx cache.
27
+ Removing a Node installation referenced by the service still breaks startup:
28
+ reinstall the service using the intended Node executable after stopping it.
29
+ No credentials are copied into the unit or job containers.
30
+ Each service uses an immutable launcher for its installation version. Installing
31
+ another controller or enabling a timer cannot rewrite an existing launcher's
32
+ base. An explicit managed update controls subsequent release selection in
33
+ `service-release.json`. The separate ordinary CLI `updates.json` preference
34
+ cannot move an existing service to another release, even while all services
35
+ are stopped.
36
+
37
+ If Docker group membership was added after the user service manager started,
38
+ an SSH shell may have access while user services still get permission denied.
39
+ Refresh the login session at a suitable time, or install with the explicit
40
+ `service install --group docker --state PATH` option. This uses `sg` or a
41
+ command-capable util-linux `newgrp` to apply a group the user **already belongs
42
+ to**. It grants no new membership, needs no sudo and keeps the controller's user
43
+ identity. The group launcher is checked for support before installation.
44
+
45
+ For boot **without logging in**, an administrator must enable user lingering:
46
+
47
+ ```sh
48
+ sudo loginctl enable-linger USERNAME
49
+ ```
50
+
51
+ `service status` reports lingering, boot versus login startup, service enablement
52
+ and actual HTTP health separately. The CLI never silently grants sudo access or
53
+ changes system-wide Docker, firewall or login configuration. Systemd retries
54
+ startup every 15 seconds if Docker or the configured image is unavailable.
55
+
56
+ ```sh
57
+ software-defence-factory service stop --state /absolute/private/state
58
+ software-defence-factory service start --state /absolute/private/state
59
+ software-defence-factory service restart --state /absolute/private/state
60
+ software-defence-factory service logs --state /absolute/private/state
61
+ software-defence-factory service uninstall --state /absolute/private/state
62
+ ```
63
+
64
+ `up` and `stop` detect a managed installation and use its service manager, so a
65
+ stop does not immediately respawn a second controller. An explicit stop/restart
66
+ may interrupt a job; use it deliberately. `stop` leaves boot enablement in place.
67
+ `uninstall` disables and removes only the registered service definition. It
68
+ preserves the database, credentials, artifacts, retained runtimes and history.
69
+ Repeated installation/removal is safe. Edited/unregistered service definitions
70
+ are refused rather than overwritten. Bare `service` (or `service print`) retains
71
+ the original print-only systemd-unit interface.
72
+
73
+ ## Persistent SSH tunnel on the client
74
+
75
+ First establish key authentication and a verified host key using normal SSH.
76
+ The remote dashboard must bind to loopback and the machines must be reachable
77
+ (for example through an existing private VPN). Use a configured SSH alias:
78
+
79
+ ```sh
80
+ software-defence-factory tunnel install --host worker --port 7345
81
+ software-defence-factory tunnel status --host worker --port 7345
82
+ software-defence-factory tunnel logs --host worker --port 7345
83
+ software-defence-factory tunnel stop --host worker --port 7345
84
+ software-defence-factory tunnel start --host worker --port 7345
85
+ software-defence-factory tunnel uninstall --host worker --port 7345
86
+ ```
87
+
88
+ Visit `http://127.0.0.1:7345`. The local and remote ports are intentionally equal
89
+ to preserve the dashboard's Host/Origin protection. The tunnel uses only
90
+ loopback addresses, strict host-key checking and noninteractive authentication.
91
+ It exits on forwarding failure, detects dead connections and restarts through
92
+ the OS service manager. It never disables SSH checks or requests a password.
93
+ Stop an existing manual tunnel before installation; occupied ports are refused.
94
+
95
+ On macOS this is a LaunchAgent: it starts at **login**, not before FileVault is
96
+ unlocked. On Linux, lingering determines boot versus login startup. A sleeping
97
+ or offline peer is still unavailable; the tunnel reconnects when connectivity
98
+ returns. Tailscale/SSH/system power configuration remains machine infrastructure,
99
+ not a Factory network dependency. No root LaunchDaemon is installed.
100
+
101
+ ## Updating an always-running controller
102
+
103
+ ```sh
104
+ software-defence-factory service update
105
+ software-defence-factory service updates --auto on
106
+ software-defence-factory service updates --auto status
107
+ software-defence-factory service updates --auto off
108
+ ```
109
+
110
+ `service update` manages all registered controllers for the current user. It
111
+ refuses unmanaged live controllers or unresolved executor fences. Every running
112
+ controller must atomically accept an idle maintenance reservation: queued work,
113
+ an active phase or a concurrent action defers the update. A reservation blocks
114
+ new tasks and actions and persists across restarts until explicitly released.
115
+ Downloads are immutable. Only the previously running managed controllers are
116
+ stopped and restarted; intentionally stopped projects stay stopped. HTTP health
117
+ must recover. A failed release startup restores the prior selection and starts
118
+ the old services before releasing maintenance. Images and application code are
119
+ never rebuilt or silently changed by this operation.
120
+ Lifecycle changes and updates share an exclusive operation lock. Concurrent
121
+ start/stop/install/uninstall commands fail clearly rather than undoing an update
122
+ or an operator's stop. A managed installation refuses ordinary foreground
123
+ `serve`; operate it through `service start`. Stop also checks for a remaining
124
+ controller outside the service before reporting success. Restart health must
125
+ report the expected runtime version, including during rollback.
126
+
127
+ The opt-in Linux timer checks daily, including a catch-up after downtime, with
128
+ up to one hour of random delay. A busy/unavailable installation causes a failed
129
+ attempt and remains unchanged; the next scheduled invocation retries. Inspect
130
+ `journalctl --user -u software-defence-factory-update.service`. This timer is
131
+ separate from `update --auto on|off`, which controls ordinary CLI invocations.
132
+ Remove the timer with `service updates --auto off` when retiring this setup.
133
+
134
+ ## Recovery and proof limits
135
+
136
+ If a service operation is killed or the machine loses power, inspect
137
+ `STATE_HOME/software-defence-factory/service-update.lock`, its PID and the
138
+ service logs. The lock records the owning PID and action. Do not remove a lock belonging to a live or unknown process.
139
+ After confirming the updater stopped, remove that stale lock, inspect the
140
+ selected release and start/verify each prior service. Then release each remaining
141
+ reservation using `service resume --state PATH`. There is no automatic timeout
142
+ that might reopen a queue while an updater is still working.
143
+
144
+ If a generated service was edited outside the CLI, preserve the edit and
145
+ reconcile its recorded definition before using lifecycle commands. A service
146
+ health failure leaves installation and logs in place for diagnosis. Keep the
147
+ prior release, private state and pinned image; see [recovery](recovery.md).
148
+
149
+ Service enablement and process-restart tests are not proof of a full machine
150
+ reboot, pre-login disk availability or connectivity across two physical networks.
151
+ Record those observations separately. A healthy dashboard proves neither model
152
+ quality nor that a product agent should start.
153
+
154
+ References: [systemd service semantics](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html),
155
+ [loginctl lingering](https://www.freedesktop.org/software/systemd/man/latest/loginctl.html),
156
+ [Apple launchd definitions](https://github.com/apple-oss-distributions/launchd/blob/main/man/launchd.plist.5).
package/docs/setup.md ADDED
@@ -0,0 +1,235 @@
1
+ # Setup plan: operator, worker and application
2
+
3
+ Use this plan for a new installation or when moving an existing Factory to a
4
+ worker machine. Complete the applicable checkpoints in order and record the
5
+ result in a **private** copy of the checklist below. The plan applies to any
6
+ operator/worker names and any suitable private network; it requires no personal
7
+ context system, particular VPN provider or Factory source checkout.
8
+
9
+ The operator owns access and infrastructure choices. Factory maintainers own
10
+ this plan and the linked command guides and update them with behavior changes.
11
+ Paths beginning `/private/state` or `/absolute/path` below are placeholders;
12
+ replace them with user-owned absolute directories outside the application and
13
+ installed package.
14
+
15
+ ## 1. Choose the scope and hosts
16
+
17
+ | Choice | Record before installation |
18
+ | --- | --- |
19
+ | Method only or runtime | Method export needs no Docker, background service or model |
20
+ | Operator/client | Local machine, user and how the dashboard will be opened |
21
+ | Worker | Linux/systemd for managed controllers; macOS can use manual `up` |
22
+ | Applications | Canonical repository, branch, preserved WIP and responsible owner |
23
+ | Runtime | Stable Node executable (22.13+), Git, Docker, CPU/RAM/disk budget |
24
+ | State | Private state path and unused loopback port for each installation |
25
+ | Inference | Provider/model, credential reference, job image and network path |
26
+ | Automation | Which controllers may start, update policy and who may submit work |
27
+ | Recovery | Private backup location, retained runtime/image and recovery contact |
28
+
29
+ One installation owns one repository and port. A synthetic qualification
30
+ installation can provide a running dashboard while product controllers remain
31
+ stopped. There is no multi-project controller or second scheduler hidden in
32
+ this plan. Keep unrelated personal agent configuration on its existing host.
33
+
34
+ For method-only adoption, run `software-defence-factory kit --output NEW_DIRECTORY`
35
+ and follow [the adoption guide](../kit/README.md). The remaining host/runtime
36
+ steps apply only when using Factory's optional controller.
37
+
38
+ ## 2. Establish host access and boot prerequisites
39
+
40
+ Use an unprivileged operator account. Check `node --version`, `git --version`
41
+ and `docker info` from the actual login/SSH session. Install dependencies using
42
+ the host's supported method. For a Linux worker, Docker, SSH and any private
43
+ network client must start at boot; inspect their actual unit names instead of
44
+ assuming every distribution uses the same service names.
45
+
46
+ For remote access:
47
+
48
+ 1. Establish a reachable private address, including across networks if needed.
49
+ 2. Enroll the worker's verified SSH host key and configure key authentication.
50
+ 3. Create an SSH alias such as `factory-worker`; verify
51
+ `ssh -o BatchMode=yes factory-worker 'id -un'` from the operator machine.
52
+ 4. Verify firewall/interface scope. Keep the dashboard on loopback; do not
53
+ expose it publicly or disable host-key checking to make the tunnel work.
54
+
55
+ Administrator access is a separate host choice. Installing/running Factory user
56
+ services does **not** require root or unrestricted passwordless sudo. If the
57
+ owner deliberately chooses passwordless administration, inspect `sudo -n -l`
58
+ for the intended `NOPASSWD` policy and repeat the permitted command after a
59
+ fresh reboot. A successful `sudo -n` command can merely reuse cached
60
+ credentials. A missing rule must be installed by an authenticated administrator;
61
+ record that remaining step without blocking unprivileged Factory operation.
62
+
63
+ Record disk unlock, login and power behavior explicitly. Linux user lingering
64
+ can start services without desktop login **after the OS and home are available**;
65
+ it cannot unlock an encrypted disk. A sleeping laptop is not an available
66
+ worker. Choose AC/battery/sleep behavior with the owner; do not disable disk
67
+ security or change power policy as a side effect of Factory setup.
68
+
69
+ Checkpoint: noninteractive SSH (when needed), Docker and the chosen Node binary
70
+ work, and required host dependencies have a documented startup/recovery owner.
71
+
72
+ ## 3. Install the package and qualify the runtime
73
+
74
+ ```sh
75
+ npm install --global software-defence-factory
76
+ software-defence-factory --version
77
+ software-defence-factory help
78
+ ```
79
+
80
+ Use a user-writable npm prefix or the existing Node manager; do not turn the
81
+ controller into a root process to work around installation permissions. For
82
+ example, Linux users choosing `--prefix "$HOME/.local"` must also make
83
+ `$HOME/.local/bin` available in their login PATH. `npx` uses the same package;
84
+ managed services retain a runtime outside its cache. Keep the service's recorded
85
+ Node executable available. See [installation and updates](npm.md).
86
+
87
+ Use a **separate synthetic state and unused port** to exercise the runtime:
88
+
89
+ ```sh
90
+ software-defence-factory demo --state /private/state/runtime-proof --port 7345
91
+ # Inspect and finish the sample handoff before qualification.
92
+ software-defence-factory qualify --state /private/state/runtime-proof
93
+ ```
94
+
95
+ These commands deliberately create synthetic jobs, including failed/interrupted
96
+ attempts. Never use application state for `demo` or `qualify`. They make no model
97
+ calls and prove neither model quality nor application readiness. Preserve the
98
+ qualification record; stop this controller unless it is the selected dashboard.
99
+
100
+ ## 4. Prepare each application and inference profile
101
+
102
+ Preserve existing commits, branches, uncommitted work and licenses before moving
103
+ anything. Verify the canonical GitHub origin and main branch's tracking target;
104
+ a fork may still track its upstream product. Move application sources into the
105
+ chosen workspace, not into the npm package or private runtime state.
106
+
107
+ Before admitting development work, establish:
108
+
109
+ - Reproducible toolchain/dependency pins and an actual build/check command.
110
+ - A compatible job image for native libraries, browser/mobile tools or custom
111
+ model adapters; the standard image does not supply every application stack.
112
+ - Meaningful checks in GitHub CI on the selected revision, plus the intended
113
+ review/branch policy. Record plan/access limits if enforcement is unavailable.
114
+ - Applicable application instructions, design/scope, resource limits, secret
115
+ references and a clear delivery destination. Keep unfinished WIP separate
116
+ until intentionally integrated; jobs clone committed source only.
117
+
118
+ Configure a new installation using [the quickstart](quickstart.md):
119
+
120
+ ```sh
121
+ software-defence-factory init --repo /absolute/path/to/app --agent pi --check "npm ci && npm test" --state /private/state/my-app --port 7331
122
+ software-defence-factory install --state /private/state/my-app
123
+ software-defence-factory doctor --state /private/state/my-app
124
+ ```
125
+
126
+ Replace the agent/check/paths with the accepted application profile. `install`
127
+ builds the standard image: do not use it to overwrite an existing custom image
128
+ selection. Follow the application image's own build/pinning procedure instead.
129
+ `init` does not edit the app, install personal skills or submit a task. Runtime
130
+ jobs receive the bundled method and six skills automatically.
131
+
132
+ For a local model, verify the existing model service, intended model name and
133
+ its startup. Test the model API from a disposable container using the **selected
134
+ job image and network**. Host `127.0.0.1` inside a container is not the host's
135
+ loopback. Any host bridge/proxy and narrow firewall rule are explicit machine
136
+ infrastructure; they need their own startup and reboot checks. A bridge address may appear
137
+ after the user service manager starts; configure retry/readiness and verify
138
+ recovery rather than assuming startup order from enablement alone. Avoid duplicate
139
+ model servers, public listeners, Docker socket mounts or whole account folders.
140
+ Cloud inference likewise needs a real provider/model connectivity check without
141
+ printing credentials. A model-list/health response is connectivity evidence;
142
+ qualifying model output requires a separately accepted bounded task.
143
+
144
+ Checkpoint: record the exact source revision, image ID, check command, resource
145
+ limits, inference connectivity and CI result. Keep product controllers stopped
146
+ until their tasks are explicitly ready to run.
147
+
148
+ ## 5. Enable only the intended background services
149
+
150
+ A newly initialized state has no jobs. Before adopting older state, establish
151
+ that its queue may resume; starting **any** controller executes queued work.
152
+ If queue ownership/state is uncertain, reconcile it before enabling autostart.
153
+
154
+ On the Linux worker, select the state that should stay available:
155
+
156
+ ```sh
157
+ software-defence-factory stop --state /private/state/runtime-proof
158
+ software-defence-factory service install --state /private/state/runtime-proof
159
+ software-defence-factory service status --state /private/state/runtime-proof
160
+ software-defence-factory service updates --auto on
161
+ software-defence-factory service updates --auto status
162
+ ```
163
+
164
+ Follow [services](services.md) for user lingering, existing Docker group
165
+ membership, logs, stop/start, uninstall and maintenance recovery. A user manager
166
+ started before Docker group membership changed may need `--group docker`; this
167
+ applies an existing group and grants no new membership. Enable lingering only
168
+ for the intended account through the host's administrator.
169
+
170
+ On the operator machine, stop any old manual tunnel, then:
171
+
172
+ ```sh
173
+ software-defence-factory tunnel install --host factory-worker --port 7345
174
+ software-defence-factory tunnel status --host factory-worker --port 7345
175
+ ```
176
+
177
+ Use the actual selected dashboard port on both sides. macOS tunnels start at
178
+ user login; Linux uses its user service manager. Open `http://127.0.0.1:7345`.
179
+ This is the worker's loopback dashboard forwarded through SSH.
180
+
181
+ Managed daily updates reserve idle controllers and preserve the prior running
182
+ set; they defer when busy and attempt rollback if the new runtime is unhealthy.
183
+ A deliberate `service stop` leaves boot enablement in place: use `uninstall`
184
+ when a controller must also remain disabled across reboots. Preserve private
185
+ state and previous releases. The timer does not rebuild application images,
186
+ start product tasks, change a model or publish application changes.
187
+
188
+ ## 6. Prove reboot recovery and hand over
189
+
190
+ Choose a reboot window with disk unlock/recovery available. Save a private
191
+ baseline of the boot ID (`cat /proc/sys/kernel/random/boot_id`), runtime version,
192
+ job IDs/states/attempts, selected enabled services and paused installations.
193
+ After the owner reboots/unlocks the worker, check **before manually starting
194
+ anything**:
195
+
196
+ - SSH returns and a changed boot ID confirms a new boot.
197
+ - Docker, private network and model/proxy services are healthy, with no relevant
198
+ failed units or repeated restart loop. Inspect their current-boot logs.
199
+ - `service status --state PATH` reports enabled/running/healthy, the intended
200
+ runtime version and no unexpected maintenance reservation.
201
+ - The update timer is enabled and has a next execution time.
202
+ - The operator's tunnel reconnects and the dashboard responds at the same URL.
203
+ - Job history is unchanged and intentionally disabled product controllers have
204
+ no listeners or executor processes. Probe model connectivity from the job
205
+ container again without starting a product agent.
206
+ - The intended administrator policy still works, if that capability was chosen.
207
+
208
+ Record actual observations and failures separately. An observed boot after
209
+ manual unlock/login does not prove an unattended cold boot; a worker reboot
210
+ does not prove operator-machine login startup. Process crash recovery and tests
211
+ across two physical networks are also distinct checks. Use [recovery](recovery.md)
212
+ and [service recovery](services.md#recovery-and-proof-limits) for failures; do not
213
+ start a second controller or clear unknown process locks to make status green.
214
+
215
+ Copy this private completion record into the installation's handoff:
216
+
217
+ | Checkpoint | Result | Evidence / remaining action |
218
+ | --- | --- | --- |
219
+ | Host roles, owner, source and private state selected | Pending | |
220
+ | SSH, Docker, Node and host startup prerequisites | Pending | |
221
+ | Synthetic runtime qualification | Pending | |
222
+ | Application image, real checks/CI and preserved WIP | Pending | |
223
+ | Inference path and chosen model | Pending | |
224
+ | Controller/tunnel/timer installation and recovery | Pending | |
225
+ | Worker reboot after required unlock/login | Pending | |
226
+ | Operator login recovery / separate networks | Pending | Record separately |
227
+ | History and intentionally stopped products preserved | Pending | |
228
+ | Backups, logs, stop/update/rollback owner and guide | Pending | |
229
+ | First bounded application task | Not started | Separate task authority and proof |
230
+
231
+ Use Pass, Fail or Not applicable with a reason; never infer success from an
232
+ installed file. Keep host identities, credentials, raw logs and customer details
233
+ out of public issues and package contents. A ready worker is only the foundation:
234
+ follow [the method](../kit/README.md#first-real-task) for the first explicitly
235
+ accepted application task and revision-bound checks/review/handoff.
package/factory/queue.mjs CHANGED
@@ -1,6 +1,7 @@
1
1
  import { DatabaseSync } from 'node:sqlite';
2
2
  import { randomBytes } from 'node:crypto';
3
3
  import { join } from 'node:path';
4
+ import { existsSync, writeFileSync, rmSync } from 'node:fs';
4
5
 
5
6
  const workflows = { software: ['build', 'verify', 'review', 'handoff'], defence: ['defence'] };
6
7
  const id = prefix => prefix + '_' + randomBytes(12).toString('hex');
@@ -14,7 +15,8 @@ export class JobQueue {
14
15
  this.db = new DatabaseSync(join(state, 'jobs.sqlite'));
15
16
  this.db.exec('PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000; CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, data TEXT NOT NULL);');
16
17
  this.execute = execute; this.stop = stop; this.reconcile = reconcile;
17
- this.active = null; this.closing = false; this.pumping = false; this.actions = new Set();
18
+ this.maintenanceFile = join(state, 'maintenance.json');
19
+ this.active = null; this.closing = false; this.pumping = false; this.actions = new Set(); this.maintenance = existsSync(this.maintenanceFile);
18
20
  for (const job of this.all()) if (['running', 'cancelling'].includes(job.state)) {
19
21
  job.state = 'interrupted';
20
22
  Object.assign(job.runs.at(-1), { state: 'interrupted', completed_at: now(), error: 'Controller stopped before completion was confirmed.' });
@@ -32,6 +34,7 @@ export class JobQueue {
32
34
  save(job) { job.updated_at = now(); this.db.prepare('INSERT INTO jobs VALUES (?,?) ON CONFLICT(id) DO UPDATE SET data=excluded.data').run(job.id, JSON.stringify(job)); return job; }
33
35
  submit(input) {
34
36
  if (this.closing) throw new QueueError('Controller is stopping');
37
+ if (this.maintenance) throw new QueueError('Controller is reserved for maintenance');
35
38
  if (input?.source_url && (typeof input.source_url !== 'string' || !/^https?:\/\/[^\s]+$/.test(input.source_url) || input.source_url.length > 2048)) throw new QueueError('Expected an HTTP(S) source link', 400);
36
39
  if (input?.model && (typeof input.model !== 'string' || !/^[\w.:/+-]{1,128}$/.test(input.model))) throw new QueueError('Invalid model identifier', 400);
37
40
  if (input?.source_url && !input.spec?.trim()) input = { ...input, spec: `Investigate the linked requirements within this repository's scope: ${input.source_url}` };
@@ -42,10 +45,19 @@ export class JobQueue {
42
45
  state: 'queued', created_at: now(), runs: [] };
43
46
  this.save(job); this.schedule(); return { id: job.id };
44
47
  }
48
+ setMaintenance(enabled) {
49
+ if (typeof enabled !== 'boolean') throw new QueueError('Expected enabled: true or false', 400);
50
+ if (enabled && (this.closing || this.active || this.actions.size || this.all().some(job => ['queued', 'running', 'cancelling'].includes(job.state)))) throw new QueueError('Controller is busy; update deferred');
51
+ if (enabled) writeFileSync(this.maintenanceFile, JSON.stringify({ startedAt: now() }), { mode: 0o600 });
52
+ else rmSync(this.maintenanceFile, { force: true });
53
+ this.maintenance = enabled;
54
+ if (!enabled) this.schedule();
55
+ return { maintenance: enabled };
56
+ }
45
57
  schedule() { if (!this.closing && !this.pumping) { this.pumping = true; queueMicrotask(() => this.pump()); } }
46
58
  async pump() {
47
59
  try {
48
- while (!this.closing) {
60
+ while (!this.closing && !this.maintenance) {
49
61
  let job = this.all().find(item => item.state === 'queued'); if (!job) break;
50
62
  const step = job.workflow.current_step, phase = job.workflow.steps[step];
51
63
  let attempt = job.runs.at(-1);
@@ -77,7 +89,7 @@ export class JobQueue {
77
89
  } finally { this.pumping = false; }
78
90
  }
79
91
  async exclusive(jobId, perform) {
80
- if (this.closing || this.actions.has(jobId)) throw new QueueError('Job is already changing; reload before acting');
92
+ if (this.closing || this.maintenance || this.actions.has(jobId)) throw new QueueError('Job is already changing or controller is reserved for maintenance; reload before acting');
81
93
  this.actions.add(jobId);
82
94
  try { return await perform(); } finally { this.actions.delete(jobId); }
83
95
  }
@@ -6,6 +6,7 @@ import { hostname } from 'node:os';
6
6
  import { JobQueue, QueueError } from './queue.mjs';
7
7
  import { executors } from './processes.mjs';
8
8
  import { configAt, ROOT } from './lib.mjs';
9
+ import { VERSION } from './updates.mjs';
9
10
 
10
11
  function equal(a, b) { return typeof a === 'string' && Buffer.byteLength(a) === Buffer.byteLength(b) && timingSafeEqual(Buffer.from(a), Buffer.from(b)); }
11
12
  async function body(request) {
@@ -42,7 +43,7 @@ export function createController(state, adapter = executors(state)) {
42
43
  outcome: attempt.outcome || (attempt.state === 'succeeded' ? 'complete' : undefined),
43
44
  executor: ['verify','handoff'].includes(attempt.command) ? 'deterministic' : config.agent,
44
45
  worker_name: hostname(), model: job.model || config.model })) }));
45
- return send(200, { version: 1, workflows: ['software', 'defence'], commands: [], triggers: [], jobs, csrf_token: csrf,
46
+ return send(200, { version: 1, runtime_version: VERSION, maintenance: queue.maintenance, workflows: ['software', 'defence'], commands: [], triggers: [], jobs, csrf_token: csrf,
46
47
  workers: [{ name: hostname(), instance_id: 'local-executor', repositories: ['app'], connected: !queue.closing, last_seen_at: new Date().toISOString() }],
47
48
  repositories: ['app'], repo: config.repo, agent: config.agent });
48
49
  }
@@ -73,6 +74,10 @@ export function createController(state, adapter = executors(state)) {
73
74
  if (!authenticated) throw new QueueError('Session required', 403);
74
75
  if (!(request.headers['content-type'] || '').startsWith('application/json')) throw new QueueError('Use application/json', 415);
75
76
  const input = await body(request);
77
+ if (url.pathname === '/api/v1/maintenance') {
78
+ if (!equal(request.headers.authorization, `Bearer ${token}`)) throw new QueueError('Operator token required for maintenance', 403);
79
+ return send(200, queue.setMaintenance(input.enabled));
80
+ }
76
81
  if (url.pathname === '/api/v1/jobs') {
77
82
  if (input.model && input.model !== config.model && !['codex','pi'].includes(config.agent)) throw new QueueError('Model overrides require a codex or pi executor', 400);
78
83
  return send(201, queue.submit(input));
@@ -0,0 +1,40 @@
1
+ import { createHash } from 'node:crypto';
2
+ import { join } from 'node:path';
3
+
4
+ export function serviceId(kind, identity) {
5
+ return `software-defence-factory-${kind}-${createHash('sha256').update(identity).digest('hex').slice(0, 16)}`;
6
+ }
7
+ function text(value) {
8
+ if (typeof value !== 'string' || /[\x00-\x1f\x7f]/.test(value)) throw new Error('Service values cannot contain control characters');
9
+ return value;
10
+ }
11
+ const unit = value => JSON.stringify(text(value).replaceAll('%', '%%'));
12
+ const argument = value => unit(value.replaceAll('$', () => '$$'));
13
+ const xml = value => text(value).replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;').replaceAll('"', '&quot;');
14
+
15
+ export function systemdUnit({ description, argv, directory, environment = {}, oneshot = false }) {
16
+ return `[Unit]\nDescription=${text(description)}\nStartLimitIntervalSec=0\n\n[Service]\nType=${oneshot ? 'oneshot' : 'exec'}\nWorkingDirectory=${text(directory).replaceAll('%', '%%')}\nExecStart=${argv.map(argument).join(' ')}\n${Object.entries(environment).map(([key, value]) => {
17
+ if (!/^[A-Z_][A-Z0-9_]*$/.test(key)) throw new Error('Invalid environment name');
18
+ return `Environment=${unit(`${key}=${value}`)}\n`;
19
+ }).join('')}${oneshot ? 'TimeoutStartSec=15min\n' : 'Restart=always\nRestartSec=15\n'}TimeoutStopSec=60\nKillMode=control-group\nUMask=0077\n\n[Install]\nWantedBy=default.target\n`;
20
+ }
21
+ export function launchAgent({ id, argv, directory, log, environment = {} }) {
22
+ return `<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n<plist version="1.0"><dict>\n<key>Label</key><string>${xml(id)}</string>\n<key>ProgramArguments</key><array>${argv.map(value => `<string>${xml(value)}</string>`).join('')}</array>\n<key>WorkingDirectory</key><string>${xml(directory)}</string>\n<key>EnvironmentVariables</key><dict>${Object.entries(environment).map(([key, value]) => `<key>${xml(key)}</key><string>${xml(value)}</string>`).join('')}</dict>\n<key>RunAtLoad</key><true/><key>KeepAlive</key><true/>\n<key>ThrottleInterval</key><integer>15</integer>\n<key>ExitTimeOut</key><integer>60</integer>\n<key>Umask</key><integer>63</integer>\n<key>StandardOutPath</key><string>${xml(log)}</string>\n<key>StandardErrorPath</key><string>${xml(log)}</string>\n</dict></plist>\n`;
23
+ }
24
+ export function tunnelArguments(host, port) {
25
+ if (!/^[a-zA-Z0-9][a-zA-Z0-9_.@-]*$/.test(host || '')) throw new Error('Use an SSH hostname or configured alias without spaces or options');
26
+ if (!Number.isInteger(port) || port < 1024 || port > 65535) throw new Error('Tunnel port must be 1024–65535');
27
+ return ['/usr/bin/ssh', '-N', '-T', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=yes', '-o', 'ExitOnForwardFailure=yes',
28
+ '-o', 'ConnectTimeout=10', '-o', 'ServerAliveInterval=15', '-o', 'ServerAliveCountMax=3', '-L', `127.0.0.1:${port}:127.0.0.1:${port}`, host];
29
+ }
30
+ export function groupArguments(argv, group, executable) {
31
+ if (!/^[a-z_][a-z0-9_-]*[$]?$/.test(group || '')) throw new Error('Expected an existing Unix group name');
32
+ if (!['/usr/bin/sg', '/usr/bin/newgrp'].includes(executable)) throw new Error('Unsupported group launcher');
33
+ const quote = value => `'${text(value).replaceAll("'", "'\\''")}'`;
34
+ return [executable, group, '-c', 'exec ' + argv.map(quote).join(' ')];
35
+ }
36
+
37
+ // Immutable launcher: ordinary CLI preferences never select a service release.
38
+ export function runtimeLauncher({ runtime, stateHome, dataHome }) {
39
+ return `import { readFileSync, existsSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { pathToFileURL } from 'node:url';\nlet root = ${JSON.stringify(runtime)};\nconst settings = ${JSON.stringify(join(stateHome, 'service-release.json'))};\nif (existsSync(settings)) {\n const version = JSON.parse(readFileSync(settings, 'utf8')).version;\n if (/^\\d+\\.\\d+\\.\\d+$/.test(version || '')) {\n const candidate = join(${JSON.stringify(dataHome)}, 'releases', version, 'node_modules/software-defence-factory');\n if (existsSync(join(candidate, 'package.json'))) {\n const pkg = JSON.parse(readFileSync(join(candidate, 'package.json'), 'utf8'));\n const current = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8')).version;\n const a = version.split('.').map(Number), b = current.split('.').map(Number);\n const index = a.findIndex((value, i) => value !== b[i]);\n if (pkg.name === 'software-defence-factory' && pkg.version === version && index >= 0 && a[index] > b[index]) root = candidate;\n }\n }\n}\nprocess.env.SDF_AUTO_UPDATE = '0';\nprocess.env.SDF_BOOTSTRAPPED = '1';\nawait import(pathToFileURL(join(root, 'bin/software-defence-factory.mjs')).href);\n`;
40
+ }
@@ -0,0 +1,298 @@
1
+ import { existsSync, mkdirSync, readFileSync, writeFileSync, rmSync, cpSync, readdirSync, renameSync } from 'node:fs';
2
+ import { join, dirname } from 'node:path';
3
+ import { homedir, userInfo } from 'node:os';
4
+ import { spawnSync } from 'node:child_process';
5
+ import { createServer } from 'node:net';
6
+ import { ROOT, STATE_HOME, DATA_HOME, SOURCE_CHECKOUT } from './paths.mjs';
7
+ import { configAt, json, save, run, api, sleep, digest } from './lib.mjs';
8
+ import { VERSION, newer, latestVersion, installRelease, busyInstallations } from './updates.mjs';
9
+ import { serviceId, systemdUnit, launchAgent, tunnelArguments, groupArguments, runtimeLauncher } from './service-files.mjs';
10
+
11
+ const records = join(STATE_HOME, 'services');
12
+ const configHome = process.env.XDG_CONFIG_HOME || join(homedir(), '.config');
13
+ const userUnits = join(configHome, 'systemd/user');
14
+ const updater = 'software-defence-factory-update';
15
+ const recordPath = id => join(records, `${id}.json`);
16
+ const systemctl = (...args) => run('systemctl', ['--user', ...args]);
17
+ const uid = () => process.getuid();
18
+ function userOnly() { if (uid() === 0) throw new Error('Install and operate user services without sudo; never run the controller as root'); }
19
+ function alive(pid) { try { process.kill(pid, 0); return true; } catch (error) { if (error.code === 'ESRCH') return false; throw error; } }
20
+ export const controllerService = state => recordPath(serviceId('controller', state));
21
+ export function hasService(state) { return existsSync(controllerService(state)); }
22
+ export async function withServiceOperation(action, perform) {
23
+ const lock = join(STATE_HOME, 'service-update.lock');
24
+ mkdirSync(STATE_HOME, { recursive: true, mode: 0o700 });
25
+ try { writeFileSync(lock, JSON.stringify({ pid: process.pid, action }), { flag: 'wx', mode: 0o600 }); }
26
+ catch (error) { if (error.code === 'EEXIST') throw new Error(`Another service operation may own ${lock}; reconcile its PID before removing the lock`); throw error; }
27
+ try { return await perform(); } finally { rmSync(lock); }
28
+ }
29
+ export function isManagedLaunch(state) {
30
+ return hasService(state) && process.env.SDF_MANAGED_SERVICE === serviceId('controller', state);
31
+ }
32
+ function owned(record) {
33
+ if (!existsSync(record.file) || digest(readFileSync(record.file, 'utf8')) !== record.definitionHash) throw new Error(`Service definition changed outside the CLI; reconcile ${record.file} before changing it`);
34
+ }
35
+ function loaded(record) {
36
+ if (record.platform === 'linux') return systemctl('show', record.unit, '--property=LoadState', '--value') !== 'not-found';
37
+ const result = spawnSync('/bin/launchctl', ['print', `gui/${uid()}/${record.id}`], { encoding: 'utf8' });
38
+ if (result.error) throw result.error;
39
+ if (result.status === 0) return true;
40
+ if (/Could not find service|Could not find specified service/.test(result.stderr)) return false;
41
+ throw new Error(result.stderr || 'Cannot inspect launchd service');
42
+ }
43
+ function active(record) {
44
+ if (record.platform === 'linux') return ['active', 'activating', 'reloading'].includes(systemctl('show', record.unit, '--property=ActiveState', '--value'));
45
+ return loaded(record);
46
+ }
47
+ function start(record) {
48
+ owned(record);
49
+ if (record.platform === 'linux') systemctl('start', record.unit);
50
+ else if (!loaded(record)) run('/bin/launchctl', ['bootstrap', `gui/${uid()}`, record.file]);
51
+ }
52
+ function stop(record) {
53
+ owned(record);
54
+ if (record.platform === 'linux') { if (active(record)) systemctl('stop', record.unit); }
55
+ else if (loaded(record)) run('/bin/launchctl', ['bootout', `gui/${uid()}/${record.id}`]);
56
+ if (record.kind === 'controller') {
57
+ const lock = join(record.state, 'supervisor.json');
58
+ if (existsSync(lock) && alive(json(lock).pid)) throw new Error('A controller remains alive outside the stopped service; reconcile its supervisor PID before proceeding');
59
+ }
60
+ }
61
+ export async function waitHealthy(record, expectedVersion) {
62
+ const deadline = Date.now() + 30000;
63
+ while (Date.now() < deadline) {
64
+ try {
65
+ if (record.kind === 'controller') {
66
+ const status = await api(record.state, '/api/v1/status');
67
+ if (status.workers?.some(worker => worker.connected) && (!expectedVersion || status.runtime_version === expectedVersion)) return status;
68
+ } else {
69
+ const response = await fetch(`http://127.0.0.1:${record.port}/api/v1/status`, { signal: AbortSignal.timeout(1000) });
70
+ if (response.ok && (await response.json()).workers) return;
71
+ }
72
+ } catch { /* A service may be waiting for Docker or the SSH peer. */ }
73
+ await sleep(500);
74
+ }
75
+ throw new Error(`Service is installed but dashboard is not ready; inspect ${record.file} and service logs. It will keep retrying.`);
76
+ }
77
+ async function portFree(port) {
78
+ await new Promise((resolve, reject) => {
79
+ const server = createServer(); server.once('error', reject);
80
+ server.listen(port, '127.0.0.1', () => server.close(resolve));
81
+ });
82
+ }
83
+ // Retain only installed package content. A source tree or npx cache is never
84
+ // the service's enduring runtime. Old snapshots remain available for recovery.
85
+ function retainRuntime() {
86
+ if (SOURCE_CHECKOUT) throw new Error('Install the npm artifact before installing a service; source worktrees are not service runtimes');
87
+ const destination = join(DATA_HOME, 'services/runtimes', VERSION);
88
+ if (!existsSync(destination)) {
89
+ mkdirSync(dirname(destination), { recursive: true, mode: 0o700 });
90
+ const staging = `${destination}.${process.pid}.tmp`;
91
+ try {
92
+ mkdirSync(staging, { mode: 0o700 });
93
+ for (const name of ['package.json', 'bin', 'factory', 'kit', '.agents', 'scripts']) if (existsSync(join(ROOT, name))) cpSync(join(ROOT, name), join(staging, name), { recursive: true });
94
+ renameSync(staging, destination);
95
+ } finally { rmSync(staging, { recursive: true, force: true }); }
96
+ }
97
+ if (json(join(destination, 'package.json')).version !== VERSION) throw new Error('Retained service runtime identity mismatch');
98
+ return destination;
99
+ }
100
+ function launcher() {
101
+ const runtime = retainRuntime(), file = join(DATA_HOME, 'services', `launch-${VERSION}-${digest(STATE_HOME).slice(0, 16)}.mjs`);
102
+ const content = runtimeLauncher({ runtime, stateHome: STATE_HOME, dataHome: DATA_HOME });
103
+ if (existsSync(file)) {
104
+ if (readFileSync(file, 'utf8') !== content) throw new Error('Retained launcher content differs; preserve and reconcile it before installation');
105
+ } else writeFileSync(file, content, { flag: 'wx', mode: 0o600 });
106
+ return file;
107
+ }
108
+ function environment() {
109
+ return { PATH: process.env.PATH || '/usr/local/bin:/usr/bin:/bin', HOME: homedir(), XDG_STATE_HOME: dirname(STATE_HOME), XDG_DATA_HOME: dirname(DATA_HOME), XDG_CONFIG_HOME: configHome, SDF_AUTO_UPDATE: '0' };
110
+ }
111
+ export function serviceDefinition(state) {
112
+ configAt(state);
113
+ return systemdUnit({ description: 'Software & Defence Factory', argv: [process.execPath, join(ROOT, 'bin/software-defence-factory.mjs'), 'serve', '--state', state], directory: homedir(), environment: environment() });
114
+ }
115
+ async function install(kind, state, flags) {
116
+ userOnly();
117
+ if (kind === 'controller' && process.platform !== 'linux') throw new Error('Controller services currently require Linux/systemd; use up on macOS. Tunnel services support macOS and Linux.');
118
+ if (!['linux', 'darwin'].includes(process.platform)) throw new Error('Services require Linux/systemd or macOS/launchd');
119
+ const port = kind === 'controller' ? configAt(state).port : Number(flags.port);
120
+ const id = kind === 'controller' ? serviceId(kind, state) : serviceId(kind, `${flags.host}:${port}`);
121
+ if (existsSync(recordPath(id))) {
122
+ const previous = json(recordPath(id)); owned(previous);
123
+ if (kind === 'controller' && previous.state !== state) throw new Error('Service identity mismatch');
124
+ if (previous.port !== port || (flags.group && previous.group !== flags.group)) throw new Error('Service configuration changed; uninstall and reinstall the service while preserving its private state');
125
+ if (previous.platform === 'linux') systemctl('enable', previous.unit);
126
+ start(previous); await waitHealthy(previous);
127
+ console.log(JSON.stringify(await serviceStatus(previous), null, 2)); return;
128
+ }
129
+ if (kind === 'controller') {
130
+ if (!existsSync(join(state, 'engine.json'))) throw new Error('Run install for this state before installing its service');
131
+ if (existsSync(join(state, 'supervisor.json')) && alive(json(join(state, 'supervisor.json')).pid)) throw new Error('Stop the manually started controller before adopting it as a service');
132
+ run('docker', ['image', 'inspect', configAt(state).image]);
133
+ }
134
+ let argv = kind === 'controller' ? [process.execPath, launcher(), 'serve', '--state', state] : tunnelArguments(flags.host, port);
135
+ if (flags.group) {
136
+ if (kind !== 'controller') throw new Error('--group applies only to Linux controller services');
137
+ if (!run('id', ['-nG', userInfo().username]).split(/\s+/).includes(flags.group)) throw new Error('The current user must already belong to the requested group');
138
+ const executable = existsSync('/usr/bin/sg') ? '/usr/bin/sg' : '/usr/bin/newgrp';
139
+ if (executable.endsWith('/newgrp') && !run(executable, ['--help']).includes('--command')) throw new Error('This newgrp does not support commands; use a system with sg or refresh the login session');
140
+ argv = groupArguments(argv, flags.group, executable);
141
+ }
142
+ await portFree(port);
143
+ const record = { id, kind, platform: process.platform, version: VERSION, port, state: kind === 'controller' ? state : undefined, host: flags.host, group: flags.group, argv, environment: environment() };
144
+ if (kind === 'controller') record.environment.SDF_MANAGED_SERVICE = id;
145
+ record.unit = `${id}.service`;
146
+ record.file = process.platform === 'linux' ? join(userUnits, record.unit) : join(homedir(), 'Library/LaunchAgents', `${id}.plist`);
147
+ record.log = join(records, `${id}.log`);
148
+ const definition = process.platform === 'linux' ? systemdUnit({ description: `Software & Defence Factory ${kind}`, argv, directory: homedir(), environment: record.environment })
149
+ : launchAgent({ id, argv, directory: homedir(), log: record.log, environment: record.environment });
150
+ if (existsSync(record.file)) throw new Error(`Refusing to overwrite an unregistered service: ${record.file}`);
151
+ mkdirSync(dirname(record.file), { recursive: true, mode: 0o700 }); mkdirSync(records, { recursive: true, mode: 0o700 });
152
+ writeFileSync(record.file, definition, { mode: 0o600 }); record.definitionHash = digest(definition); save(recordPath(id), record);
153
+ if (process.platform === 'linux') { run('systemd-analyze', ['--user', 'verify', record.file]); systemctl('daemon-reload'); systemctl('enable', record.unit); }
154
+ start(record);
155
+ await waitHealthy(record);
156
+ console.log(JSON.stringify(await serviceStatus(record), null, 2));
157
+ }
158
+ async function serviceStatus(record) {
159
+ owned(record);
160
+ const result = { id: record.id, kind: record.kind, definition: record.file, dashboard: `http://127.0.0.1:${record.port}`, running: active(record) };
161
+ if (record.platform === 'linux') {
162
+ result.enabled = systemctl('show', record.unit, '--property=UnitFileState', '--value') === 'enabled';
163
+ const linger = run('loginctl', ['show-user', userInfo().username, '-p', 'Linger', '--value']);
164
+ result.linger = linger === 'yes'; result.startsAt = result.linger ? 'boot' : 'login';
165
+ if (!result.linger) result.next = `For boot without login, an administrator can run: loginctl enable-linger ${userInfo().username}`;
166
+ result.logs = `journalctl --user -u ${record.unit} -n 100 --no-pager`;
167
+ } else { result.startsAt = 'login'; result.logs = record.log; }
168
+ try {
169
+ const response = await fetch(`http://127.0.0.1:${record.port}/api/v1/status`, { signal: AbortSignal.timeout(1500) });
170
+ const snapshot = await response.json(); result.healthy = response.ok && !!snapshot.workers;
171
+ result.runtimeVersion = snapshot.runtime_version; result.maintenance = snapshot.maintenance;
172
+ }
173
+ catch { result.healthy = false; }
174
+ return result;
175
+ }
176
+ export async function manageService(kind, action, state, flags = {}) {
177
+ userOnly();
178
+ if (kind === 'tunnel') tunnelArguments(flags.host, Number(flags.port));
179
+ if (kind === 'controller' && action === 'update') return updateServices();
180
+ const readonly = ['status', 'logs'].includes(action) || (action === 'updates' && flags.auto === 'status');
181
+ return readonly ? performService(kind, action, state, flags) : withServiceOperation(`${kind} ${action}`, () => performService(kind, action, state, flags));
182
+ }
183
+ async function performService(kind, action, state, flags) {
184
+ if (kind === 'controller' && action === 'updates') return automaticUpdates(flags.auto);
185
+ if (kind === 'controller' && action === 'resume') {
186
+ console.log(JSON.stringify(await api(state, '/api/v1/maintenance', { enabled: false }))); return;
187
+ }
188
+ if (action === 'install') return install(kind, state, flags);
189
+ const id = kind === 'controller' ? serviceId(kind, state) : serviceId(kind, `${flags.host}:${Number(flags.port)}`);
190
+ if (!['start', 'stop', 'restart', 'status', 'uninstall', 'logs'].includes(action)) throw new Error('Use service/tunnel install|start|stop|restart|status|logs|uninstall; service update; service updates --auto on|off|status');
191
+ if (!existsSync(recordPath(id))) {
192
+ if (action === 'status') { console.log(JSON.stringify({ installed: false, id })); return; }
193
+ if (action === 'uninstall') { console.log('Service is already uninstalled; private state was preserved.'); return; }
194
+ throw new Error('Service is not installed');
195
+ }
196
+ const record = json(recordPath(id)); owned(record);
197
+ if (action === 'status') { console.log(JSON.stringify(await serviceStatus(record), null, 2)); return; }
198
+ if (action === 'logs') {
199
+ console.log(record.platform === 'linux' ? run('journalctl', ['--user', '-u', record.unit, '-n', '100', '--no-pager']) : (existsSync(record.log) ? readFileSync(record.log, 'utf8').split('\n').slice(-100).join('\n') : 'No service log yet.')); return;
200
+ }
201
+ if (['stop', 'restart', 'uninstall'].includes(action)) stop(record);
202
+ if (['start', 'restart'].includes(action)) { start(record); await waitHealthy(record); }
203
+ if (action === 'uninstall') {
204
+ if (record.platform === 'linux') systemctl('disable', record.unit);
205
+ rmSync(record.file); rmSync(recordPath(id));
206
+ if (record.platform === 'linux') systemctl('daemon-reload');
207
+ }
208
+ console.log(`${action}: ${id}. Private state and retained runtimes were preserved.`);
209
+ }
210
+
211
+ function controllers() {
212
+ if (!existsSync(records)) return [];
213
+ return readdirSync(records).filter(name => name.endsWith('.json')).map(name => json(join(records, name))).filter(record => record.kind === 'controller');
214
+ }
215
+ export async function updateServices({ latest = latestVersion, download = installRelease } = {}) {
216
+ userOnly();
217
+ return withServiceOperation('update', () => performUpdate(latest, download));
218
+ }
219
+ async function performUpdate(latest, download) {
220
+ const held = [], stopped = [];
221
+ const preferencesPath = join(STATE_HOME, 'updates.json'), selectionPath = join(STATE_HOME, 'service-release.json');
222
+ const previousPreferences = existsSync(preferencesPath) ? json(preferencesPath) : { enabled: true, lastCheckedAt: 0 };
223
+ const previous = existsSync(selectionPath) ? json(selectionPath) : {};
224
+ let activated = false, releaseMaintenance = true;
225
+ const priorVersions = new Map();
226
+ try {
227
+ const managed = controllers();
228
+ for (const record of managed) owned(record);
229
+ const live = managed.filter(active);
230
+ const installed = managed.map(record => record.version || VERSION).reduce((minimum, version) => newer(minimum, version) ? version : minimum, VERSION);
231
+ const current = previous.version || installed;
232
+ const version = [await latest(), VERSION, current].reduce((selected, candidate) => newer(candidate, selected) ? candidate : selected);
233
+ const runningVersions = await Promise.all(live.map(async record => {
234
+ const value = (await api(record.state, '/api/v1/status')).runtime_version;
235
+ priorVersions.set(record.id, value); return value;
236
+ }));
237
+ if (runningVersions.some(value => !/^\d+\.\d+\.\d+$/.test(value || ''))) throw new Error('Cannot verify the runtime version of a managed controller');
238
+ if (!newer(version, current) && runningVersions.every(value => !newer(version, value))) { console.log(`software-defence-factory ${current} is up to date.`); return; }
239
+ const unowned = busyInstallations().filter(state => !live.some(record => record.state === state));
240
+ if (unowned.length) throw new Error('Unmanaged controllers or executor fences block the update: ' + unowned.join(', '));
241
+ // Reserve idle controllers before downloads. All mutating API actions are
242
+ // refused until restart or explicit release, eliminating submit/stop races.
243
+ for (const record of live) { await api(record.state, '/api/v1/maintenance', { enabled: true }); held.push(record); }
244
+ download(version);
245
+ for (const record of live) { stop(record); stopped.push(record); }
246
+ if (busyInstallations().length) throw new Error('An installation or executor still blocks activation');
247
+ // Ordinary CLI downloads must never change what an existing service starts.
248
+ // Only this reserved update transaction may advance the managed selection.
249
+ activated = true;
250
+ save(selectionPath, { version });
251
+ save(preferencesPath, { ...previousPreferences, version, lastCheckedAt: Date.now() });
252
+ for (const record of stopped) { start(record); await waitHealthy(record, version); }
253
+ console.log(`Updated to ${version}; restarted ${stopped.length} previously active services. Jobs and history preserved.`);
254
+ } catch (error) {
255
+ releaseMaintenance = false;
256
+ try {
257
+ if (activated) {
258
+ for (const record of stopped) stop(record);
259
+ save(selectionPath, previous);
260
+ save(preferencesPath, previousPreferences);
261
+ }
262
+ for (const record of stopped) { start(record); await waitHealthy(record, priorVersions.get(record.id)); }
263
+ releaseMaintenance = true;
264
+ } catch (recoveryError) {
265
+ throw new Error(`Update failed: ${error.message}. Recovery unconfirmed: ${recoveryError.message}. Maintenance reservations were retained.`);
266
+ }
267
+ throw error;
268
+ } finally {
269
+ const failures = [];
270
+ if (releaseMaintenance) for (const record of held) try { await api(record.state, '/api/v1/maintenance', { enabled: false }); } catch (error) { failures.push(`${record.id}: ${error.message}`); }
271
+ if (failures.length) throw new Error('Maintenance release unconfirmed: ' + failures.join('; '));
272
+ }
273
+ }
274
+ function automaticUpdates(mode) {
275
+ if (process.platform !== 'linux') throw new Error('Scheduled service updates currently require Linux/systemd');
276
+ if (!['on', 'off', 'status'].includes(mode)) throw new Error('Use service updates --auto on|off|status');
277
+ const timer = join(userUnits, `${updater}.timer`), service = join(userUnits, `${updater}.service`), record = join(records, 'updates.json');
278
+ if (mode === 'status') { console.log(JSON.stringify({ installed: existsSync(record), ...(existsSync(record) ? { timer: systemctl('show', `${updater}.timer`, '-p', 'ActiveState', '-p', 'NextElapseUSecRealtime') } : {}) })); return; }
279
+ if (mode === 'off') {
280
+ if (existsSync(record)) {
281
+ if (systemctl('show', `${updater}.service`, '-p', 'ActiveState', '--value') === 'activating') throw new Error('An update is running; wait for it to finish before disabling the timer');
282
+ const data = json(record);
283
+ for (const [file, hash] of [[timer, data.timerHash], [service, data.serviceHash]]) if (!existsSync(file) || digest(readFileSync(file, 'utf8')) !== hash) throw new Error('Update service was edited; reconcile it before removal');
284
+ systemctl('disable', '--now', `${updater}.timer`);
285
+ systemctl('stop', `${updater}.service`);
286
+ rmSync(timer); rmSync(service); rmSync(record); systemctl('daemon-reload');
287
+ }
288
+ console.log('Scheduled updates disabled.'); return;
289
+ }
290
+ if (existsSync(record)) { console.log('Scheduled updates are already installed.'); return; }
291
+ if (existsSync(timer) || existsSync(service)) throw new Error('Refusing to overwrite an unregistered update unit');
292
+ const serviceText = systemdUnit({ description: 'Software & Defence Factory idle update', argv: [process.execPath, launcher(), 'service', 'update'], directory: homedir(), environment: environment(), oneshot: true });
293
+ const timerText = `[Unit]\nDescription=Check Factory updates daily when idle\n\n[Timer]\nOnCalendar=daily\nRandomizedDelaySec=1h\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n`;
294
+ mkdirSync(userUnits, { recursive: true, mode: 0o700 });
295
+ writeFileSync(service, serviceText, { mode: 0o600 }); writeFileSync(timer, timerText, { mode: 0o600 });
296
+ save(record, { serviceHash: digest(serviceText), timerHash: digest(timerText) });
297
+ systemctl('daemon-reload'); systemctl('enable', '--now', `${updater}.timer`); console.log('Daily idle service updates enabled. Busy/unknown installations defer the update.');
298
+ }
@@ -1,8 +1,10 @@
1
1
  import { writeFileSync, rmSync } from 'node:fs';
2
2
  import { join } from 'node:path';
3
+ import { pathToFileURL } from 'node:url';
3
4
  import { configAt, stopContainers } from './lib.mjs';
4
5
  import { createController } from './server.mjs';
5
- const state = process.argv[2], config = configAt(state), lock = join(state, 'supervisor.json');
6
+ export async function supervise(state) {
7
+ const config = configAt(state), lock = join(state, 'supervisor.json');
6
8
  writeFileSync(lock, JSON.stringify({ pid: process.pid }), { flag: 'wx', mode: 0o600 });
7
9
  let controller, stopping = false;
8
10
  async function shutdown(code = 0) {
@@ -18,3 +20,5 @@ try {
18
20
  controller.server.on('error', error => { console.error(error.message); shutdown(1); });
19
21
  controller.server.listen(config.port, '127.0.0.1', () => console.log(`Factory controller ready on 127.0.0.1:${config.port}`));
20
22
  } catch (error) { console.error(error.message); await shutdown(1); }
23
+ }
24
+ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) await supervise(process.argv[2]);
@@ -104,7 +104,7 @@ export async function bootstrap(args) {
104
104
  const busy = busyInstallations();
105
105
  const cachedVersion = installedEntry(DATA_HOME, preferences.version) ? preferences.version : VERSION;
106
106
  let selectedVersion = newer(cachedVersion, VERSION) ? cachedVersion : VERSION;
107
- const automatic = !['stop', 'cancel', 'status', 'serve', 'version', '--version', '-v'].includes(command)
107
+ const automatic = !['stop', 'cancel', 'status', 'serve', 'service', 'tunnel', 'help', '--help', '-h', 'version', '--version', '-v'].includes(command)
108
108
  && preferences.enabled && process.env.SDF_AUTO_UPDATE !== '0'
109
109
  && Date.now() - preferences.lastCheckedAt >= DAY && busy.length === 0;
110
110
  if (explicit || automatic) {
@@ -10,6 +10,7 @@
10
10
  | Control surface | Existing issue/check/PR workflow unless otherwise selected |
11
11
  | Harness, version, model/provider and billing | Unselected |
12
12
  | Execution environment and necessary tools | Files, Git, shell and tests; browser for UI; other tools as needed |
13
+ | Worker boot and operator login | Disk unlock, SSH/network/Docker/model startup, selected services, timer and observed reboot evidence; not applicable for method-only use |
13
14
  | Start, stop, routing owner and recovery | Manual until automation is qualified; one owner per job |
14
15
  | Authority and allowed network/credential references | Identify permissions without key values |
15
16
  | Time, concurrency, attempts and actual provider stop limits | Unknown until exercised |
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "software-defence-factory",
3
- "version": "0.3.5",
3
+ "version": "0.4.1",
4
4
  "description": "Scoped software delivery and defence investigations with isolated jobs, evidence and review",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -29,6 +29,9 @@
29
29
  "scripts/export-kit.mjs",
30
30
  "scripts/probe-platform.mjs",
31
31
  "docs/npm.md",
32
+ "docs/setup.md",
33
+ "docs/services.md",
34
+ "docs/recovery.md",
32
35
  "docs/quickstart.md",
33
36
  "docs/defence-integration.md",
34
37
  "docs/ownership.md",
@@ -68,7 +68,7 @@ try {
68
68
  assert(!existsSync(join(state,'jobs',changed.id,'accepted.json')));record('changed candidate cannot inherit earlier approval');
69
69
 
70
70
  const policy=await work('Synthetic policy change guard');await waitState(policy.id,'awaiting_approval');
71
- setConfig({check:'true'});await cli('approve',policy.id);await waitState(policy.id,'failed');
71
+ setConfig({check:original.check==='true'?':':'true'});await cli('approve',policy.id);await waitState(policy.id,'failed');
72
72
  assert(!existsSync(join(state,'jobs',policy.id,'accepted.json')));setConfig({});record('changed check policy cannot inherit earlier approval');
73
73
 
74
74
  setConfig({check:'exit 17'});