software-defence-factory 0.3.5 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/bin/software-defence-factory.mjs +32 -12
- package/docs/npm.md +6 -1
- package/docs/quickstart.md +3 -1
- package/docs/recovery.md +23 -0
- package/docs/services.md +156 -0
- package/docs/setup.md +235 -0
- package/factory/queue.mjs +15 -3
- package/factory/server.mjs +6 -1
- package/factory/service-files.mjs +40 -0
- package/factory/services.mjs +298 -0
- package/factory/supervisor.mjs +5 -1
- package/factory/updates.mjs +1 -1
- package/kit/installation.md +1 -0
- package/package.json +4 -1
- package/scripts/probe-platform.mjs +1 -1
package/README.md
CHANGED
|
@@ -19,10 +19,12 @@ Choose the part you need:
|
|
|
19
19
|
|
|
20
20
|
| Outcome | Command / guide |
|
|
21
21
|
| --- | --- |
|
|
22
|
+
| Set up an operator, worker and application | [Setup plan and acceptance checklist](docs/setup.md) |
|
|
22
23
|
| Use the method with your existing agent | `software-defence-factory kit --output ./factory-kit` — exports a new staging directory |
|
|
23
24
|
| Try the runtime without inference | `software-defence-factory demo` — Docker required; synthetic sample only |
|
|
24
25
|
| Connect an existing repository | [Runtime quickstart](docs/quickstart.md) |
|
|
25
26
|
| Understand installation and updates | [npm and npx](docs/npm.md) |
|
|
27
|
+
| Restore a dashboard after boot or reconnect remotely | [Services and SSH tunnels](docs/services.md) |
|
|
26
28
|
| Review the evidence and limits | [Qualification](docs/proof.md) |
|
|
27
29
|
|
|
28
30
|
The runtime supplies policy and six focused skills to its isolated jobs. `init` configures a private installation; it does not modify the application or start work. Model access and the application's real check command must be configured before using it for delivery.
|
|
@@ -68,6 +70,7 @@ npm run check
|
|
|
68
70
|
```
|
|
69
71
|
|
|
70
72
|
CI builds the dashboard and checks Node 22/24. A version increase merged to `main` is published to npm through the configured release workflow. Installed CLIs can update on invocation when all installations are stopped. See [release and update behavior](docs/npm.md).
|
|
73
|
+
Managed Linux services can also opt into daily updates that reserve idle controllers, preserve stopped projects and restore the prior release if startup fails. See [service operation](docs/services.md).
|
|
71
74
|
|
|
72
75
|
This is a test release. Synthetic qualification demonstrates control flow and isolation, not model quality, application correctness or production readiness. Follow [AGENTS.md](AGENTS.md) for contributions and [SECURITY.md](SECURITY.md) for the trust boundaries.
|
|
73
76
|
|
|
@@ -8,6 +8,7 @@ import { ROOT, PINS, DEFAULT_STATE, configAt, save, json, run, stream, digest, a
|
|
|
8
8
|
import { admitIncident } from '../factory/incident.mjs';
|
|
9
9
|
import { DEFAULT_DEMO_STATE } from '../factory/paths.mjs';
|
|
10
10
|
import { bootstrap, registerInstallation, VERSION } from '../factory/updates.mjs';
|
|
11
|
+
import { hasService, manageService, serviceDefinition, withServiceOperation, isManagedLaunch } from '../factory/services.mjs';
|
|
11
12
|
|
|
12
13
|
try {
|
|
13
14
|
const handled = await bootstrap(process.argv.slice(2));
|
|
@@ -94,7 +95,8 @@ async function stop() {
|
|
|
94
95
|
const {pid}=json(lock);
|
|
95
96
|
if(alive(pid)) {
|
|
96
97
|
const identity=run('ps',['-p',String(pid),'-o','command=']);
|
|
97
|
-
|
|
98
|
+
const controllerProcess=identity.includes(join(ROOT,'factory/supervisor.mjs'))||identity.includes(join(ROOT,'bin/software-defence-factory.mjs')+' serve');
|
|
99
|
+
if(!controllerProcess||!identity.includes(state))throw new Error('PID identity changed; refusing to signal an unrelated process');
|
|
98
100
|
process.kill(pid,'SIGTERM');
|
|
99
101
|
for(let i=0;i<40&&existsSync(lock);i++)await sleep(250);
|
|
100
102
|
if(existsSync(lock))throw new Error('Stop unconfirmed; inspect supervisor and do not start replacement workers');
|
|
@@ -134,18 +136,27 @@ async function jobAction(action) {
|
|
|
134
136
|
|
|
135
137
|
try {
|
|
136
138
|
if(command==='init') { if(!flags.repo)throw new Error('init requires --repo /path/to/existing/git/repo');init(flags.repo,flags.agent,flags.check,flags.port); }
|
|
137
|
-
else if(command==='install')await install
|
|
138
|
-
else if(command==='up')await
|
|
139
|
-
else if(command==='stop')await stop();
|
|
139
|
+
else if(command==='install')await withServiceOperation('install',install);
|
|
140
|
+
else if(command==='up') { if(hasService(state))await manageService('controller','start',state);else await withServiceOperation('up',up); }
|
|
141
|
+
else if(command==='stop') { if(hasService(state))await manageService('controller','stop',state);else await withServiceOperation('stop',stop); }
|
|
140
142
|
else if(command==='serve') {
|
|
143
|
+
const managed=isManagedLaunch(state);
|
|
144
|
+
if(hasService(state)&&!managed)throw new Error('This installation is managed; use service start instead of foreground serve');
|
|
145
|
+
const launch=async()=>{
|
|
141
146
|
if(process.getuid()===0)throw new Error('Use a dedicated unprivileged operator account');
|
|
142
147
|
const lock=join(state,'supervisor.json');if(existsSync(lock)){if(alive(json(lock).pid))throw new Error('Supervisor already running');rmSync(lock);}
|
|
143
|
-
|
|
148
|
+
if(!existsSync(join(state,'engine.json')))throw new Error('Run install first');
|
|
149
|
+
run('docker',['image','inspect',configAt(state).image]);
|
|
150
|
+
registerInstallation(state);await portFree(configAt(state).port);
|
|
151
|
+
const { supervise } = await import('../factory/supervisor.mjs');await supervise(state);
|
|
152
|
+
};
|
|
153
|
+
if(managed)await launch();else await withServiceOperation('serve',launch);
|
|
144
154
|
}
|
|
145
155
|
else if(command==='service') {
|
|
146
|
-
|
|
147
|
-
|
|
156
|
+
if(!positional.length || positional[0]==='print')console.log(serviceDefinition(state));
|
|
157
|
+
else await manageService('controller',positional[0],state,flags);
|
|
148
158
|
}
|
|
159
|
+
else if(command==='tunnel')await manageService('tunnel',positional[0],state,flags);
|
|
149
160
|
else if(command==='status') { const snapshot=await api(state,'/api/v1/status');delete snapshot.csrf_token;console.log(JSON.stringify(snapshot,null,2)); }
|
|
150
161
|
else if(command==='doctor') {
|
|
151
162
|
const config=configAt(state);console.log(JSON.stringify({node:process.version,docker:run('docker',['info','--format','{{.ServerVersion}}']),engineInstalled:existsSync(join(state,'engine.json')),repo:config.repo,agent:config.agent,checksConfigured:!!config.check?.trim(),inference:'Not called or verified',dashboard:`http://127.0.0.1:${config.port}`},null,2));
|
|
@@ -175,7 +186,7 @@ try {
|
|
|
175
186
|
run('git',['-C',repo,'-c','user.name=Factory demo','-c','user.email=demo@localhost','commit','-m','Synthetic fixture']);
|
|
176
187
|
init(repo,'mock',"test \"$(cat value.txt)\" = fixed",Number(flags.port || 7332));
|
|
177
188
|
} else if(configAt(state).agent!=='mock')throw new Error('Demo requires a mock configuration');
|
|
178
|
-
await install();await up();console.log(JSON.stringify(await submit('software','Synthetic installation qualification: fix value.txt. No inference is used.')));
|
|
189
|
+
await withServiceOperation('demo startup',async()=>{await install();await up();});console.log(JSON.stringify(await submit('software','Synthetic installation qualification: fix value.txt. No inference is used.')));
|
|
179
190
|
console.log('Review the synthetic change in the dashboard and approve its handoff.');
|
|
180
191
|
} else if(['version','--version','-v'].includes(command))console.log(VERSION);
|
|
181
192
|
else if(command==='qualify') {
|
|
@@ -183,7 +194,7 @@ try {
|
|
|
183
194
|
} else if(command==='kit') {
|
|
184
195
|
if(!flags.output)throw new Error('kit requires --output NEW_DIRECTORY');
|
|
185
196
|
await stream(process.execPath,[join(ROOT,'scripts/export-kit.mjs'),resolve(flags.output)]);
|
|
186
|
-
} else if(
|
|
197
|
+
} else if(['help','--help','-h'].includes(command))console.log(`Software & Defence Factory ${VERSION} (test release)
|
|
187
198
|
|
|
188
199
|
kit --output NEW_DIRECTORY Export the portable method without a runtime
|
|
189
200
|
demo Install and run a synthetic sample (no model key)
|
|
@@ -191,8 +202,16 @@ try {
|
|
|
191
202
|
init --repo PATH --agent codex|pi|custom --check "npm ci && npm test"
|
|
192
203
|
install Build the isolated job image; the controller ships with the CLI
|
|
193
204
|
doctor | up | status | stop Inspect / operate your private installation
|
|
194
|
-
serve Foreground supervisor
|
|
195
|
-
service
|
|
205
|
+
serve Foreground supervisor
|
|
206
|
+
service [print] Print a systemd user-service definition
|
|
207
|
+
service install|start|stop|restart Manage a Linux user service (--state PATH)
|
|
208
|
+
install accepts --group EXISTING_GROUP
|
|
209
|
+
service status|logs|uninstall Diagnose / remove service, preserve private state
|
|
210
|
+
service update Update idle managed controllers, restore on failure
|
|
211
|
+
service updates --auto on|off|status Daily idle updates through a systemd timer
|
|
212
|
+
service resume Release a reconciled maintenance reservation
|
|
213
|
+
tunnel install|start|stop|status|logs|uninstall --host SSH_ALIAS --port PORT
|
|
214
|
+
Persistent loopback SSH tunnel (macOS/Linux)
|
|
196
215
|
run --file task.md | --issue URL Submit one software vertical slice
|
|
197
216
|
incident --file incident.json Submit a private, read-only incident draft
|
|
198
217
|
approve JOB_ID | cancel JOB_ID Review gate / stop this attempt
|
|
@@ -205,6 +224,7 @@ Runtime commands accept --state PATH. Default: ${DEFAULT_STATE}
|
|
|
205
224
|
Demo default: ${DEFAULT_DEMO_STATE}
|
|
206
225
|
The npm CLI keeps state outside the package; updates wait for stopped installations.
|
|
207
226
|
Dashboard binds only to loopback; use SSH for remote access.
|
|
208
|
-
|
|
227
|
+
Setup plan: ${join(ROOT, 'docs/setup.md')}
|
|
228
|
+
See docs/quickstart.md for task execution, evidence and recovery.`);
|
|
209
229
|
else throw new Error(`Unknown command: ${command}`);
|
|
210
230
|
} catch(error) {console.error(`Factory: ${error.message}`);process.exitCode=1;}
|
package/docs/npm.md
CHANGED
|
@@ -42,7 +42,12 @@ An npm-installed CLI checks npm's `latest` stable release at most once a day
|
|
|
42
42
|
when invoked. It downloads and activates a newer release when all registered
|
|
43
43
|
installations are stopped and no executor requires reconciliation. It does not
|
|
44
44
|
run a background updater or interrupt a job. Routine `status`, `stop`, `cancel`,
|
|
45
|
-
`serve
|
|
45
|
+
`serve`, service/tunnel management, help and version commands do not initiate automatic downloads.
|
|
46
|
+
|
|
47
|
+
For an always-running Linux installation, opt into the separate managed daily
|
|
48
|
+
timer with `service updates --auto on`. It reserves idle controllers, updates
|
|
49
|
+
and restores their prior running set without interrupting a job. See
|
|
50
|
+
[services](services.md) for installation, maintenance recovery and limitations.
|
|
46
51
|
|
|
47
52
|
```sh
|
|
48
53
|
software-defence-factory update --check
|
package/docs/quickstart.md
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# Runtime quickstart
|
|
2
2
|
|
|
3
|
+
For a new worker or remote operator, start with the [setup plan](setup.md).
|
|
4
|
+
|
|
3
5
|
Install Node 22.13+, Git and Docker Engine/Desktop. Use an unprivileged account with Docker access. Install `software-defence-factory` through npm, or invoke the same package with npx. No factory source checkout is required.
|
|
4
6
|
|
|
5
7
|
## Qualify a synthetic installation
|
|
@@ -55,7 +57,7 @@ ssh -N -L 127.0.0.1:7331:127.0.0.1:7331 your-host
|
|
|
55
57
|
|
|
56
58
|
Open http://127.0.0.1:7331 on the client. Use the same local/remote port because the HTTP service validates its Host header. The SSH connection must remain open. Access also works across different networks when your configured private network connects the hosts.
|
|
57
59
|
|
|
58
|
-
Use `status`, `cancel JOB_ID`, `retry JOB_ID` and `stop`, always with the selected `--state`. `service`
|
|
60
|
+
Use `status`, `cancel JOB_ID`, `retry JOB_ID` and `stop`, always with the selected `--state`. `service install --state PATH` installs and enables the supported Linux user service; `service status`, `service logs` and `service uninstall` operate it. Bare `service` only prints a definition. For managed startup, persistent SSH tunnels and daily idle updates, follow [services](services.md). See [recovery](recovery.md) for interrupted attempts.
|
|
59
61
|
|
|
60
62
|
## Native application builds
|
|
61
63
|
|
package/docs/recovery.md
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Recovery and retained evidence
|
|
2
|
+
|
|
3
|
+
Use `status --state PATH` and the private supervisor.log to identify the active installation. Stop it before replacing its package or container image.
|
|
4
|
+
|
|
5
|
+
- A normal `stop` signals the controller, waits for the executor process group, removes its labelled containers and retains the database and artifacts.
|
|
6
|
+
- An unconfirmed running attempt becomes `interrupted` on controller restart. It is never silently considered successful.
|
|
7
|
+
- `retry JOB_ID` reconciles the previous process group and containers. A live or unknown writer blocks retry. For a new build/defence attempt, the prior checkout is retained as previous-checkout-*.
|
|
8
|
+
- A failed verification can retry the same unchanged candidate after the check environment is repaired. A changed candidate needs a fresh verification/review sequence.
|
|
9
|
+
- A requested revision retains previous evidence and starts a new build from the source repository with the accumulated feedback. It does not reuse earlier approval.
|
|
10
|
+
- Removing a stopped task from the dashboard hides its queue record. Private artifacts and its deleted_at record remain on disk; this is not secure erasure.
|
|
11
|
+
|
|
12
|
+
Do not remove active.json merely to unblock a job. Establish that its PID, process group and labelled containers are stopped. PID reuse or missing process identity requires operator investigation. Preserve logs and work before cleanup.
|
|
13
|
+
|
|
14
|
+
For backup, stop the installation and copy the complete private state directory, including SQLite files, factory.json and credentials, to an authorized private destination. Restore only while stopped. Update the repository path if it moved, verify ownership/permissions and the pinned image, then inspect state before any retry. Keep previous backups; no automatic destructive schema migration is provided.
|
|
15
|
+
|
|
16
|
+
Earlier experimental engines use a different journal. Start a new state directory for the native 0.3 runtime; preserve old journals separately. There is no automatic import of their jobs or approval state.
|
|
17
|
+
|
|
18
|
+
Verification cleanup makes owned scratch directories traversable before removing
|
|
19
|
+
them and never follows their symlinks. It runs only after container stop is
|
|
20
|
+
confirmed. If a filesystem error still prevents cleanup, the attempt fails and
|
|
21
|
+
retains the original check exit and private log path alongside the cleanup error.
|
|
22
|
+
Inspect that retained attempt before manual removal; never substitute the source
|
|
23
|
+
candidate path for the scratch path.
|
package/docs/services.md
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# Persistent controllers and remote dashboards
|
|
2
|
+
|
|
3
|
+
For the complete host-to-application sequence and reboot checklist, start with
|
|
4
|
+
[the setup plan](setup.md). Install the npm CLI first. A Factory controller is a Linux systemd **user**
|
|
5
|
+
service; SSH tunnels support Linux systemd and macOS launchd. Neither needs a
|
|
6
|
+
root controller. macOS can still run a local controller with `up`.
|
|
7
|
+
|
|
8
|
+
## Linux controller
|
|
9
|
+
|
|
10
|
+
Configure and install a runtime using the [quickstart](quickstart.md), then:
|
|
11
|
+
|
|
12
|
+
```sh
|
|
13
|
+
software-defence-factory stop --state /absolute/private/state
|
|
14
|
+
software-defence-factory service install --state /absolute/private/state
|
|
15
|
+
software-defence-factory service status --state /absolute/private/state
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Installation enables and starts exactly this controller. It does not submit a
|
|
19
|
+
new task. Existing queued jobs will execute when any controller starts; inspect
|
|
20
|
+
the queue before adopting an installation. Interrupted attempts still require
|
|
21
|
+
the normal explicit reconciliation/retry. To keep a product paused, leave its
|
|
22
|
+
controller stopped and do not install a service for it.
|
|
23
|
+
|
|
24
|
+
The current user must have Docker access; Docker itself must be enabled at boot.
|
|
25
|
+
The CLI records the current PATH, Node executable and private state/data homes.
|
|
26
|
+
The launcher and installed runtime are retained outside the npm/npx cache.
|
|
27
|
+
Removing a Node installation referenced by the service still breaks startup:
|
|
28
|
+
reinstall the service using the intended Node executable after stopping it.
|
|
29
|
+
No credentials are copied into the unit or job containers.
|
|
30
|
+
Each service uses an immutable launcher for its installation version. Installing
|
|
31
|
+
another controller or enabling a timer cannot rewrite an existing launcher's
|
|
32
|
+
base. An explicit managed update controls subsequent release selection in
|
|
33
|
+
`service-release.json`. The separate ordinary CLI `updates.json` preference
|
|
34
|
+
cannot move an existing service to another release, even while all services
|
|
35
|
+
are stopped.
|
|
36
|
+
|
|
37
|
+
If Docker group membership was added after the user service manager started,
|
|
38
|
+
an SSH shell may have access while user services still get permission denied.
|
|
39
|
+
Refresh the login session at a suitable time, or install with the explicit
|
|
40
|
+
`service install --group docker --state PATH` option. This uses `sg` or a
|
|
41
|
+
command-capable util-linux `newgrp` to apply a group the user **already belongs
|
|
42
|
+
to**. It grants no new membership, needs no sudo and keeps the controller's user
|
|
43
|
+
identity. The group launcher is checked for support before installation.
|
|
44
|
+
|
|
45
|
+
For boot **without logging in**, an administrator must enable user lingering:
|
|
46
|
+
|
|
47
|
+
```sh
|
|
48
|
+
sudo loginctl enable-linger USERNAME
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
`service status` reports lingering, boot versus login startup, service enablement
|
|
52
|
+
and actual HTTP health separately. The CLI never silently grants sudo access or
|
|
53
|
+
changes system-wide Docker, firewall or login configuration. Systemd retries
|
|
54
|
+
startup every 15 seconds if Docker or the configured image is unavailable.
|
|
55
|
+
|
|
56
|
+
```sh
|
|
57
|
+
software-defence-factory service stop --state /absolute/private/state
|
|
58
|
+
software-defence-factory service start --state /absolute/private/state
|
|
59
|
+
software-defence-factory service restart --state /absolute/private/state
|
|
60
|
+
software-defence-factory service logs --state /absolute/private/state
|
|
61
|
+
software-defence-factory service uninstall --state /absolute/private/state
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
`up` and `stop` detect a managed installation and use its service manager, so a
|
|
65
|
+
stop does not immediately respawn a second controller. An explicit stop/restart
|
|
66
|
+
may interrupt a job; use it deliberately. `stop` leaves boot enablement in place.
|
|
67
|
+
`uninstall` disables and removes only the registered service definition. It
|
|
68
|
+
preserves the database, credentials, artifacts, retained runtimes and history.
|
|
69
|
+
Repeated installation/removal is safe. Edited/unregistered service definitions
|
|
70
|
+
are refused rather than overwritten. Bare `service` (or `service print`) retains
|
|
71
|
+
the original print-only systemd-unit interface.
|
|
72
|
+
|
|
73
|
+
## Persistent SSH tunnel on the client
|
|
74
|
+
|
|
75
|
+
First establish key authentication and a verified host key using normal SSH.
|
|
76
|
+
The remote dashboard must bind to loopback and the machines must be reachable
|
|
77
|
+
(for example through an existing private VPN). Use a configured SSH alias:
|
|
78
|
+
|
|
79
|
+
```sh
|
|
80
|
+
software-defence-factory tunnel install --host worker --port 7345
|
|
81
|
+
software-defence-factory tunnel status --host worker --port 7345
|
|
82
|
+
software-defence-factory tunnel logs --host worker --port 7345
|
|
83
|
+
software-defence-factory tunnel stop --host worker --port 7345
|
|
84
|
+
software-defence-factory tunnel start --host worker --port 7345
|
|
85
|
+
software-defence-factory tunnel uninstall --host worker --port 7345
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Visit `http://127.0.0.1:7345`. The local and remote ports are intentionally equal
|
|
89
|
+
to preserve the dashboard's Host/Origin protection. The tunnel uses only
|
|
90
|
+
loopback addresses, strict host-key checking and noninteractive authentication.
|
|
91
|
+
It exits on forwarding failure, detects dead connections and restarts through
|
|
92
|
+
the OS service manager. It never disables SSH checks or requests a password.
|
|
93
|
+
Stop an existing manual tunnel before installation; occupied ports are refused.
|
|
94
|
+
|
|
95
|
+
On macOS this is a LaunchAgent: it starts at **login**, not before FileVault is
|
|
96
|
+
unlocked. On Linux, lingering determines boot versus login startup. A sleeping
|
|
97
|
+
or offline peer is still unavailable; the tunnel reconnects when connectivity
|
|
98
|
+
returns. Tailscale/SSH/system power configuration remains machine infrastructure,
|
|
99
|
+
not a Factory network dependency. No root LaunchDaemon is installed.
|
|
100
|
+
|
|
101
|
+
## Updating an always-running controller
|
|
102
|
+
|
|
103
|
+
```sh
|
|
104
|
+
software-defence-factory service update
|
|
105
|
+
software-defence-factory service updates --auto on
|
|
106
|
+
software-defence-factory service updates --auto status
|
|
107
|
+
software-defence-factory service updates --auto off
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
`service update` manages all registered controllers for the current user. It
|
|
111
|
+
refuses unmanaged live controllers or unresolved executor fences. Every running
|
|
112
|
+
controller must atomically accept an idle maintenance reservation: queued work,
|
|
113
|
+
an active phase or a concurrent action defers the update. A reservation blocks
|
|
114
|
+
new tasks and actions and persists across restarts until explicitly released.
|
|
115
|
+
Downloads are immutable. Only the previously running managed controllers are
|
|
116
|
+
stopped and restarted; intentionally stopped projects stay stopped. HTTP health
|
|
117
|
+
must recover. A failed release startup restores the prior selection and starts
|
|
118
|
+
the old services before releasing maintenance. Images and application code are
|
|
119
|
+
never rebuilt or silently changed by this operation.
|
|
120
|
+
Lifecycle changes and updates share an exclusive operation lock. Concurrent
|
|
121
|
+
start/stop/install/uninstall commands fail clearly rather than undoing an update
|
|
122
|
+
or an operator's stop. A managed installation refuses ordinary foreground
|
|
123
|
+
`serve`; operate it through `service start`. Stop also checks for a remaining
|
|
124
|
+
controller outside the service before reporting success. Restart health must
|
|
125
|
+
report the expected runtime version, including during rollback.
|
|
126
|
+
|
|
127
|
+
The opt-in Linux timer checks daily, including a catch-up after downtime, with
|
|
128
|
+
up to one hour of random delay. A busy/unavailable installation causes a failed
|
|
129
|
+
attempt and remains unchanged; the next scheduled invocation retries. Inspect
|
|
130
|
+
`journalctl --user -u software-defence-factory-update.service`. This timer is
|
|
131
|
+
separate from `update --auto on|off`, which controls ordinary CLI invocations.
|
|
132
|
+
Remove the timer with `service updates --auto off` when retiring this setup.
|
|
133
|
+
|
|
134
|
+
## Recovery and proof limits
|
|
135
|
+
|
|
136
|
+
If a service operation is killed or the machine loses power, inspect
|
|
137
|
+
`STATE_HOME/software-defence-factory/service-update.lock`, its PID and the
|
|
138
|
+
service logs. The lock records the owning PID and action. Do not remove a lock belonging to a live or unknown process.
|
|
139
|
+
After confirming the updater stopped, remove that stale lock, inspect the
|
|
140
|
+
selected release and start/verify each prior service. Then release each remaining
|
|
141
|
+
reservation using `service resume --state PATH`. There is no automatic timeout
|
|
142
|
+
that might reopen a queue while an updater is still working.
|
|
143
|
+
|
|
144
|
+
If a generated service was edited outside the CLI, preserve the edit and
|
|
145
|
+
reconcile its recorded definition before using lifecycle commands. A service
|
|
146
|
+
health failure leaves installation and logs in place for diagnosis. Keep the
|
|
147
|
+
prior release, private state and pinned image; see [recovery](recovery.md).
|
|
148
|
+
|
|
149
|
+
Service enablement and process-restart tests are not proof of a full machine
|
|
150
|
+
reboot, pre-login disk availability or connectivity across two physical networks.
|
|
151
|
+
Record those observations separately. A healthy dashboard proves neither model
|
|
152
|
+
quality nor that a product agent should start.
|
|
153
|
+
|
|
154
|
+
References: [systemd service semantics](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html),
|
|
155
|
+
[loginctl lingering](https://www.freedesktop.org/software/systemd/man/latest/loginctl.html),
|
|
156
|
+
[Apple launchd definitions](https://github.com/apple-oss-distributions/launchd/blob/main/man/launchd.plist.5).
|
package/docs/setup.md
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
# Setup plan: operator, worker and application
|
|
2
|
+
|
|
3
|
+
Use this plan for a new installation or when moving an existing Factory to a
|
|
4
|
+
worker machine. Complete the applicable checkpoints in order and record the
|
|
5
|
+
result in a **private** copy of the checklist below. The plan applies to any
|
|
6
|
+
operator/worker names and any suitable private network; it requires no personal
|
|
7
|
+
context system, particular VPN provider or Factory source checkout.
|
|
8
|
+
|
|
9
|
+
The operator owns access and infrastructure choices. Factory maintainers own
|
|
10
|
+
this plan and the linked command guides and update them with behavior changes.
|
|
11
|
+
Paths beginning `/private/state` or `/absolute/path` below are placeholders;
|
|
12
|
+
replace them with user-owned absolute directories outside the application and
|
|
13
|
+
installed package.
|
|
14
|
+
|
|
15
|
+
## 1. Choose the scope and hosts
|
|
16
|
+
|
|
17
|
+
| Choice | Record before installation |
|
|
18
|
+
| --- | --- |
|
|
19
|
+
| Method only or runtime | Method export needs no Docker, background service or model |
|
|
20
|
+
| Operator/client | Local machine, user and how the dashboard will be opened |
|
|
21
|
+
| Worker | Linux/systemd for managed controllers; macOS can use manual `up` |
|
|
22
|
+
| Applications | Canonical repository, branch, preserved WIP and responsible owner |
|
|
23
|
+
| Runtime | Stable Node executable (22.13+), Git, Docker, CPU/RAM/disk budget |
|
|
24
|
+
| State | Private state path and unused loopback port for each installation |
|
|
25
|
+
| Inference | Provider/model, credential reference, job image and network path |
|
|
26
|
+
| Automation | Which controllers may start, update policy and who may submit work |
|
|
27
|
+
| Recovery | Private backup location, retained runtime/image and recovery contact |
|
|
28
|
+
|
|
29
|
+
One installation owns one repository and port. A synthetic qualification
|
|
30
|
+
installation can provide a running dashboard while product controllers remain
|
|
31
|
+
stopped. There is no multi-project controller or second scheduler hidden in
|
|
32
|
+
this plan. Keep unrelated personal agent configuration on its existing host.
|
|
33
|
+
|
|
34
|
+
For method-only adoption, run `software-defence-factory kit --output NEW_DIRECTORY`
|
|
35
|
+
and follow [the adoption guide](../kit/README.md). The remaining host/runtime
|
|
36
|
+
steps apply only when using Factory's optional controller.
|
|
37
|
+
|
|
38
|
+
## 2. Establish host access and boot prerequisites
|
|
39
|
+
|
|
40
|
+
Use an unprivileged operator account. Check `node --version`, `git --version`
|
|
41
|
+
and `docker info` from the actual login/SSH session. Install dependencies using
|
|
42
|
+
the host's supported method. For a Linux worker, Docker, SSH and any private
|
|
43
|
+
network client must start at boot; inspect their actual unit names instead of
|
|
44
|
+
assuming every distribution uses the same service names.
|
|
45
|
+
|
|
46
|
+
For remote access:
|
|
47
|
+
|
|
48
|
+
1. Establish a reachable private address, including across networks if needed.
|
|
49
|
+
2. Enroll the worker's verified SSH host key and configure key authentication.
|
|
50
|
+
3. Create an SSH alias such as `factory-worker`; verify
|
|
51
|
+
`ssh -o BatchMode=yes factory-worker 'id -un'` from the operator machine.
|
|
52
|
+
4. Verify firewall/interface scope. Keep the dashboard on loopback; do not
|
|
53
|
+
expose it publicly or disable host-key checking to make the tunnel work.
|
|
54
|
+
|
|
55
|
+
Administrator access is a separate host choice. Installing/running Factory user
|
|
56
|
+
services does **not** require root or unrestricted passwordless sudo. If the
|
|
57
|
+
owner deliberately chooses passwordless administration, inspect `sudo -n -l`
|
|
58
|
+
for the intended `NOPASSWD` policy and repeat the permitted command after a
|
|
59
|
+
fresh reboot. A successful `sudo -n` command can merely reuse cached
|
|
60
|
+
credentials. A missing rule must be installed by an authenticated administrator;
|
|
61
|
+
record that remaining step without blocking unprivileged Factory operation.
|
|
62
|
+
|
|
63
|
+
Record disk unlock, login and power behavior explicitly. Linux user lingering
|
|
64
|
+
can start services without desktop login **after the OS and home are available**;
|
|
65
|
+
it cannot unlock an encrypted disk. A sleeping laptop is not an available
|
|
66
|
+
worker. Choose AC/battery/sleep behavior with the owner; do not disable disk
|
|
67
|
+
security or change power policy as a side effect of Factory setup.
|
|
68
|
+
|
|
69
|
+
Checkpoint: noninteractive SSH (when needed), Docker and the chosen Node binary
|
|
70
|
+
work, and required host dependencies have a documented startup/recovery owner.
|
|
71
|
+
|
|
72
|
+
## 3. Install the package and qualify the runtime
|
|
73
|
+
|
|
74
|
+
```sh
|
|
75
|
+
npm install --global software-defence-factory
|
|
76
|
+
software-defence-factory --version
|
|
77
|
+
software-defence-factory help
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Use a user-writable npm prefix or the existing Node manager; do not turn the
|
|
81
|
+
controller into a root process to work around installation permissions. For
|
|
82
|
+
example, Linux users choosing `--prefix "$HOME/.local"` must also make
|
|
83
|
+
`$HOME/.local/bin` available in their login PATH. `npx` uses the same package;
|
|
84
|
+
managed services retain a runtime outside its cache. Keep the service's recorded
|
|
85
|
+
Node executable available. See [installation and updates](npm.md).
|
|
86
|
+
|
|
87
|
+
Use a **separate synthetic state and unused port** to exercise the runtime:
|
|
88
|
+
|
|
89
|
+
```sh
|
|
90
|
+
software-defence-factory demo --state /private/state/runtime-proof --port 7345
|
|
91
|
+
# Inspect and finish the sample handoff before qualification.
|
|
92
|
+
software-defence-factory qualify --state /private/state/runtime-proof
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
These commands deliberately create synthetic jobs, including failed/interrupted
|
|
96
|
+
attempts. Never use application state for `demo` or `qualify`. They make no model
|
|
97
|
+
calls and prove neither model quality nor application readiness. Preserve the
|
|
98
|
+
qualification record; stop this controller unless it is the selected dashboard.
|
|
99
|
+
|
|
100
|
+
## 4. Prepare each application and inference profile
|
|
101
|
+
|
|
102
|
+
Preserve existing commits, branches, uncommitted work and licenses before moving
|
|
103
|
+
anything. Verify the canonical GitHub origin and main branch's tracking target;
|
|
104
|
+
a fork may still track its upstream product. Move application sources into the
|
|
105
|
+
chosen workspace, not into the npm package or private runtime state.
|
|
106
|
+
|
|
107
|
+
Before admitting development work, establish:
|
|
108
|
+
|
|
109
|
+
- Reproducible toolchain/dependency pins and an actual build/check command.
|
|
110
|
+
- A compatible job image for native libraries, browser/mobile tools or custom
|
|
111
|
+
model adapters; the standard image does not supply every application stack.
|
|
112
|
+
- Meaningful checks in GitHub CI on the selected revision, plus the intended
|
|
113
|
+
review/branch policy. Record plan/access limits if enforcement is unavailable.
|
|
114
|
+
- Applicable application instructions, design/scope, resource limits, secret
|
|
115
|
+
references and a clear delivery destination. Keep unfinished WIP separate
|
|
116
|
+
until intentionally integrated; jobs clone committed source only.
|
|
117
|
+
|
|
118
|
+
Configure a new installation using [the quickstart](quickstart.md):
|
|
119
|
+
|
|
120
|
+
```sh
|
|
121
|
+
software-defence-factory init --repo /absolute/path/to/app --agent pi --check "npm ci && npm test" --state /private/state/my-app --port 7331
|
|
122
|
+
software-defence-factory install --state /private/state/my-app
|
|
123
|
+
software-defence-factory doctor --state /private/state/my-app
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
Replace the agent/check/paths with the accepted application profile. `install`
|
|
127
|
+
builds the standard image: do not use it to overwrite an existing custom image
|
|
128
|
+
selection. Follow the application image's own build/pinning procedure instead.
|
|
129
|
+
`init` does not edit the app, install personal skills or submit a task. Runtime
|
|
130
|
+
jobs receive the bundled method and six skills automatically.
|
|
131
|
+
|
|
132
|
+
For a local model, verify the existing model service, intended model name and
|
|
133
|
+
its startup. Test the model API from a disposable container using the **selected
|
|
134
|
+
job image and network**. Host `127.0.0.1` inside a container is not the host's
|
|
135
|
+
loopback. Any host bridge/proxy and narrow firewall rule are explicit machine
|
|
136
|
+
infrastructure; they need their own startup and reboot checks. A bridge address may appear
|
|
137
|
+
after the user service manager starts; configure retry/readiness and verify
|
|
138
|
+
recovery rather than assuming startup order from enablement alone. Avoid duplicate
|
|
139
|
+
model servers, public listeners, Docker socket mounts or whole account folders.
|
|
140
|
+
Cloud inference likewise needs a real provider/model connectivity check without
|
|
141
|
+
printing credentials. A model-list/health response is connectivity evidence;
|
|
142
|
+
qualifying model output requires a separately accepted bounded task.
|
|
143
|
+
|
|
144
|
+
Checkpoint: record the exact source revision, image ID, check command, resource
|
|
145
|
+
limits, inference connectivity and CI result. Keep product controllers stopped
|
|
146
|
+
until their tasks are explicitly ready to run.
|
|
147
|
+
|
|
148
|
+
## 5. Enable only the intended background services
|
|
149
|
+
|
|
150
|
+
A newly initialized state has no jobs. Before adopting older state, establish
|
|
151
|
+
that its queue may resume; starting **any** controller executes queued work.
|
|
152
|
+
If queue ownership/state is uncertain, reconcile it before enabling autostart.
|
|
153
|
+
|
|
154
|
+
On the Linux worker, select the state that should stay available:
|
|
155
|
+
|
|
156
|
+
```sh
|
|
157
|
+
software-defence-factory stop --state /private/state/runtime-proof
|
|
158
|
+
software-defence-factory service install --state /private/state/runtime-proof
|
|
159
|
+
software-defence-factory service status --state /private/state/runtime-proof
|
|
160
|
+
software-defence-factory service updates --auto on
|
|
161
|
+
software-defence-factory service updates --auto status
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
Follow [services](services.md) for user lingering, existing Docker group
|
|
165
|
+
membership, logs, stop/start, uninstall and maintenance recovery. A user manager
|
|
166
|
+
started before Docker group membership changed may need `--group docker`; this
|
|
167
|
+
applies an existing group and grants no new membership. Enable lingering only
|
|
168
|
+
for the intended account through the host's administrator.
|
|
169
|
+
|
|
170
|
+
On the operator machine, stop any old manual tunnel, then:
|
|
171
|
+
|
|
172
|
+
```sh
|
|
173
|
+
software-defence-factory tunnel install --host factory-worker --port 7345
|
|
174
|
+
software-defence-factory tunnel status --host factory-worker --port 7345
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
Use the actual selected dashboard port on both sides. macOS tunnels start at
|
|
178
|
+
user login; Linux uses its user service manager. Open `http://127.0.0.1:7345`.
|
|
179
|
+
This is the worker's loopback dashboard forwarded through SSH.
|
|
180
|
+
|
|
181
|
+
Managed daily updates reserve idle controllers and preserve the prior running
|
|
182
|
+
set; they defer when busy and attempt rollback if the new runtime is unhealthy.
|
|
183
|
+
A deliberate `service stop` leaves boot enablement in place: use `uninstall`
|
|
184
|
+
when a controller must also remain disabled across reboots. Preserve private
|
|
185
|
+
state and previous releases. The timer does not rebuild application images,
|
|
186
|
+
start product tasks, change a model or publish application changes.
|
|
187
|
+
|
|
188
|
+
## 6. Prove reboot recovery and hand over
|
|
189
|
+
|
|
190
|
+
Choose a reboot window with disk unlock/recovery available. Save a private
|
|
191
|
+
baseline of the boot ID (`cat /proc/sys/kernel/random/boot_id`), runtime version,
|
|
192
|
+
job IDs/states/attempts, selected enabled services and paused installations.
|
|
193
|
+
After the owner reboots/unlocks the worker, check **before manually starting
|
|
194
|
+
anything**:
|
|
195
|
+
|
|
196
|
+
- SSH returns and a changed boot ID confirms a new boot.
|
|
197
|
+
- Docker, private network and model/proxy services are healthy, with no relevant
|
|
198
|
+
failed units or repeated restart loop. Inspect their current-boot logs.
|
|
199
|
+
- `service status --state PATH` reports enabled/running/healthy, the intended
|
|
200
|
+
runtime version and no unexpected maintenance reservation.
|
|
201
|
+
- The update timer is enabled and has a next execution time.
|
|
202
|
+
- The operator's tunnel reconnects and the dashboard responds at the same URL.
|
|
203
|
+
- Job history is unchanged and intentionally disabled product controllers have
|
|
204
|
+
no listeners or executor processes. Probe model connectivity from the job
|
|
205
|
+
container again without starting a product agent.
|
|
206
|
+
- The intended administrator policy still works, if that capability was chosen.
|
|
207
|
+
|
|
208
|
+
Record actual observations and failures separately. An observed boot after
|
|
209
|
+
manual unlock/login does not prove an unattended cold boot; a worker reboot
|
|
210
|
+
does not prove operator-machine login startup. Process crash recovery and tests
|
|
211
|
+
across two physical networks are also distinct checks. Use [recovery](recovery.md)
|
|
212
|
+
and [service recovery](services.md#recovery-and-proof-limits) for failures; do not
|
|
213
|
+
start a second controller or clear unknown process locks to make status green.
|
|
214
|
+
|
|
215
|
+
Copy this private completion record into the installation's handoff:
|
|
216
|
+
|
|
217
|
+
| Checkpoint | Result | Evidence / remaining action |
|
|
218
|
+
| --- | --- | --- |
|
|
219
|
+
| Host roles, owner, source and private state selected | Pending | |
|
|
220
|
+
| SSH, Docker, Node and host startup prerequisites | Pending | |
|
|
221
|
+
| Synthetic runtime qualification | Pending | |
|
|
222
|
+
| Application image, real checks/CI and preserved WIP | Pending | |
|
|
223
|
+
| Inference path and chosen model | Pending | |
|
|
224
|
+
| Controller/tunnel/timer installation and recovery | Pending | |
|
|
225
|
+
| Worker reboot after required unlock/login | Pending | |
|
|
226
|
+
| Operator login recovery / separate networks | Pending | Record separately |
|
|
227
|
+
| History and intentionally stopped products preserved | Pending | |
|
|
228
|
+
| Backups, logs, stop/update/rollback owner and guide | Pending | |
|
|
229
|
+
| First bounded application task | Not started | Separate task authority and proof |
|
|
230
|
+
|
|
231
|
+
Use Pass, Fail or Not applicable with a reason; never infer success from an
|
|
232
|
+
installed file. Keep host identities, credentials, raw logs and customer details
|
|
233
|
+
out of public issues and package contents. A ready worker is only the foundation:
|
|
234
|
+
follow [the method](../kit/README.md#first-real-task) for the first explicitly
|
|
235
|
+
accepted application task and revision-bound checks/review/handoff.
|
package/factory/queue.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { DatabaseSync } from 'node:sqlite';
|
|
2
2
|
import { randomBytes } from 'node:crypto';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
|
+
import { existsSync, writeFileSync, rmSync } from 'node:fs';
|
|
4
5
|
|
|
5
6
|
const workflows = { software: ['build', 'verify', 'review', 'handoff'], defence: ['defence'] };
|
|
6
7
|
const id = prefix => prefix + '_' + randomBytes(12).toString('hex');
|
|
@@ -14,7 +15,8 @@ export class JobQueue {
|
|
|
14
15
|
this.db = new DatabaseSync(join(state, 'jobs.sqlite'));
|
|
15
16
|
this.db.exec('PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000; CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, data TEXT NOT NULL);');
|
|
16
17
|
this.execute = execute; this.stop = stop; this.reconcile = reconcile;
|
|
17
|
-
this.
|
|
18
|
+
this.maintenanceFile = join(state, 'maintenance.json');
|
|
19
|
+
this.active = null; this.closing = false; this.pumping = false; this.actions = new Set(); this.maintenance = existsSync(this.maintenanceFile);
|
|
18
20
|
for (const job of this.all()) if (['running', 'cancelling'].includes(job.state)) {
|
|
19
21
|
job.state = 'interrupted';
|
|
20
22
|
Object.assign(job.runs.at(-1), { state: 'interrupted', completed_at: now(), error: 'Controller stopped before completion was confirmed.' });
|
|
@@ -32,6 +34,7 @@ export class JobQueue {
|
|
|
32
34
|
save(job) { job.updated_at = now(); this.db.prepare('INSERT INTO jobs VALUES (?,?) ON CONFLICT(id) DO UPDATE SET data=excluded.data').run(job.id, JSON.stringify(job)); return job; }
|
|
33
35
|
submit(input) {
|
|
34
36
|
if (this.closing) throw new QueueError('Controller is stopping');
|
|
37
|
+
if (this.maintenance) throw new QueueError('Controller is reserved for maintenance');
|
|
35
38
|
if (input?.source_url && (typeof input.source_url !== 'string' || !/^https?:\/\/[^\s]+$/.test(input.source_url) || input.source_url.length > 2048)) throw new QueueError('Expected an HTTP(S) source link', 400);
|
|
36
39
|
if (input?.model && (typeof input.model !== 'string' || !/^[\w.:/+-]{1,128}$/.test(input.model))) throw new QueueError('Invalid model identifier', 400);
|
|
37
40
|
if (input?.source_url && !input.spec?.trim()) input = { ...input, spec: `Investigate the linked requirements within this repository's scope: ${input.source_url}` };
|
|
@@ -42,10 +45,19 @@ export class JobQueue {
|
|
|
42
45
|
state: 'queued', created_at: now(), runs: [] };
|
|
43
46
|
this.save(job); this.schedule(); return { id: job.id };
|
|
44
47
|
}
|
|
48
|
+
setMaintenance(enabled) {
|
|
49
|
+
if (typeof enabled !== 'boolean') throw new QueueError('Expected enabled: true or false', 400);
|
|
50
|
+
if (enabled && (this.closing || this.active || this.actions.size || this.all().some(job => ['queued', 'running', 'cancelling'].includes(job.state)))) throw new QueueError('Controller is busy; update deferred');
|
|
51
|
+
if (enabled) writeFileSync(this.maintenanceFile, JSON.stringify({ startedAt: now() }), { mode: 0o600 });
|
|
52
|
+
else rmSync(this.maintenanceFile, { force: true });
|
|
53
|
+
this.maintenance = enabled;
|
|
54
|
+
if (!enabled) this.schedule();
|
|
55
|
+
return { maintenance: enabled };
|
|
56
|
+
}
|
|
45
57
|
schedule() { if (!this.closing && !this.pumping) { this.pumping = true; queueMicrotask(() => this.pump()); } }
|
|
46
58
|
async pump() {
|
|
47
59
|
try {
|
|
48
|
-
while (!this.closing) {
|
|
60
|
+
while (!this.closing && !this.maintenance) {
|
|
49
61
|
let job = this.all().find(item => item.state === 'queued'); if (!job) break;
|
|
50
62
|
const step = job.workflow.current_step, phase = job.workflow.steps[step];
|
|
51
63
|
let attempt = job.runs.at(-1);
|
|
@@ -77,7 +89,7 @@ export class JobQueue {
|
|
|
77
89
|
} finally { this.pumping = false; }
|
|
78
90
|
}
|
|
79
91
|
async exclusive(jobId, perform) {
|
|
80
|
-
if (this.closing || this.actions.has(jobId)) throw new QueueError('Job is already changing; reload before acting');
|
|
92
|
+
if (this.closing || this.maintenance || this.actions.has(jobId)) throw new QueueError('Job is already changing or controller is reserved for maintenance; reload before acting');
|
|
81
93
|
this.actions.add(jobId);
|
|
82
94
|
try { return await perform(); } finally { this.actions.delete(jobId); }
|
|
83
95
|
}
|
package/factory/server.mjs
CHANGED
|
@@ -6,6 +6,7 @@ import { hostname } from 'node:os';
|
|
|
6
6
|
import { JobQueue, QueueError } from './queue.mjs';
|
|
7
7
|
import { executors } from './processes.mjs';
|
|
8
8
|
import { configAt, ROOT } from './lib.mjs';
|
|
9
|
+
import { VERSION } from './updates.mjs';
|
|
9
10
|
|
|
10
11
|
function equal(a, b) { return typeof a === 'string' && Buffer.byteLength(a) === Buffer.byteLength(b) && timingSafeEqual(Buffer.from(a), Buffer.from(b)); }
|
|
11
12
|
async function body(request) {
|
|
@@ -42,7 +43,7 @@ export function createController(state, adapter = executors(state)) {
|
|
|
42
43
|
outcome: attempt.outcome || (attempt.state === 'succeeded' ? 'complete' : undefined),
|
|
43
44
|
executor: ['verify','handoff'].includes(attempt.command) ? 'deterministic' : config.agent,
|
|
44
45
|
worker_name: hostname(), model: job.model || config.model })) }));
|
|
45
|
-
return send(200, { version: 1, workflows: ['software', 'defence'], commands: [], triggers: [], jobs, csrf_token: csrf,
|
|
46
|
+
return send(200, { version: 1, runtime_version: VERSION, maintenance: queue.maintenance, workflows: ['software', 'defence'], commands: [], triggers: [], jobs, csrf_token: csrf,
|
|
46
47
|
workers: [{ name: hostname(), instance_id: 'local-executor', repositories: ['app'], connected: !queue.closing, last_seen_at: new Date().toISOString() }],
|
|
47
48
|
repositories: ['app'], repo: config.repo, agent: config.agent });
|
|
48
49
|
}
|
|
@@ -73,6 +74,10 @@ export function createController(state, adapter = executors(state)) {
|
|
|
73
74
|
if (!authenticated) throw new QueueError('Session required', 403);
|
|
74
75
|
if (!(request.headers['content-type'] || '').startsWith('application/json')) throw new QueueError('Use application/json', 415);
|
|
75
76
|
const input = await body(request);
|
|
77
|
+
if (url.pathname === '/api/v1/maintenance') {
|
|
78
|
+
if (!equal(request.headers.authorization, `Bearer ${token}`)) throw new QueueError('Operator token required for maintenance', 403);
|
|
79
|
+
return send(200, queue.setMaintenance(input.enabled));
|
|
80
|
+
}
|
|
76
81
|
if (url.pathname === '/api/v1/jobs') {
|
|
77
82
|
if (input.model && input.model !== config.model && !['codex','pi'].includes(config.agent)) throw new QueueError('Model overrides require a codex or pi executor', 400);
|
|
78
83
|
return send(201, queue.submit(input));
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
|
|
4
|
+
export function serviceId(kind, identity) {
|
|
5
|
+
return `software-defence-factory-${kind}-${createHash('sha256').update(identity).digest('hex').slice(0, 16)}`;
|
|
6
|
+
}
|
|
7
|
+
function text(value) {
|
|
8
|
+
if (typeof value !== 'string' || /[\x00-\x1f\x7f]/.test(value)) throw new Error('Service values cannot contain control characters');
|
|
9
|
+
return value;
|
|
10
|
+
}
|
|
11
|
+
const unit = value => JSON.stringify(text(value).replaceAll('%', '%%'));
|
|
12
|
+
const argument = value => unit(value.replaceAll('$', () => '$$'));
|
|
13
|
+
const xml = value => text(value).replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>').replaceAll('"', '"');
|
|
14
|
+
|
|
15
|
+
export function systemdUnit({ description, argv, directory, environment = {}, oneshot = false }) {
|
|
16
|
+
return `[Unit]\nDescription=${text(description)}\nStartLimitIntervalSec=0\n\n[Service]\nType=${oneshot ? 'oneshot' : 'exec'}\nWorkingDirectory=${text(directory).replaceAll('%', '%%')}\nExecStart=${argv.map(argument).join(' ')}\n${Object.entries(environment).map(([key, value]) => {
|
|
17
|
+
if (!/^[A-Z_][A-Z0-9_]*$/.test(key)) throw new Error('Invalid environment name');
|
|
18
|
+
return `Environment=${unit(`${key}=${value}`)}\n`;
|
|
19
|
+
}).join('')}${oneshot ? 'TimeoutStartSec=15min\n' : 'Restart=always\nRestartSec=15\n'}TimeoutStopSec=60\nKillMode=control-group\nUMask=0077\n\n[Install]\nWantedBy=default.target\n`;
|
|
20
|
+
}
|
|
21
|
+
export function launchAgent({ id, argv, directory, log, environment = {} }) {
|
|
22
|
+
return `<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n<plist version="1.0"><dict>\n<key>Label</key><string>${xml(id)}</string>\n<key>ProgramArguments</key><array>${argv.map(value => `<string>${xml(value)}</string>`).join('')}</array>\n<key>WorkingDirectory</key><string>${xml(directory)}</string>\n<key>EnvironmentVariables</key><dict>${Object.entries(environment).map(([key, value]) => `<key>${xml(key)}</key><string>${xml(value)}</string>`).join('')}</dict>\n<key>RunAtLoad</key><true/><key>KeepAlive</key><true/>\n<key>ThrottleInterval</key><integer>15</integer>\n<key>ExitTimeOut</key><integer>60</integer>\n<key>Umask</key><integer>63</integer>\n<key>StandardOutPath</key><string>${xml(log)}</string>\n<key>StandardErrorPath</key><string>${xml(log)}</string>\n</dict></plist>\n`;
|
|
23
|
+
}
|
|
24
|
+
export function tunnelArguments(host, port) {
|
|
25
|
+
if (!/^[a-zA-Z0-9][a-zA-Z0-9_.@-]*$/.test(host || '')) throw new Error('Use an SSH hostname or configured alias without spaces or options');
|
|
26
|
+
if (!Number.isInteger(port) || port < 1024 || port > 65535) throw new Error('Tunnel port must be 1024–65535');
|
|
27
|
+
return ['/usr/bin/ssh', '-N', '-T', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=yes', '-o', 'ExitOnForwardFailure=yes',
|
|
28
|
+
'-o', 'ConnectTimeout=10', '-o', 'ServerAliveInterval=15', '-o', 'ServerAliveCountMax=3', '-L', `127.0.0.1:${port}:127.0.0.1:${port}`, host];
|
|
29
|
+
}
|
|
30
|
+
export function groupArguments(argv, group, executable) {
|
|
31
|
+
if (!/^[a-z_][a-z0-9_-]*[$]?$/.test(group || '')) throw new Error('Expected an existing Unix group name');
|
|
32
|
+
if (!['/usr/bin/sg', '/usr/bin/newgrp'].includes(executable)) throw new Error('Unsupported group launcher');
|
|
33
|
+
const quote = value => `'${text(value).replaceAll("'", "'\\''")}'`;
|
|
34
|
+
return [executable, group, '-c', 'exec ' + argv.map(quote).join(' ')];
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// Immutable launcher: ordinary CLI preferences never select a service release.
|
|
38
|
+
export function runtimeLauncher({ runtime, stateHome, dataHome }) {
|
|
39
|
+
return `import { readFileSync, existsSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { pathToFileURL } from 'node:url';\nlet root = ${JSON.stringify(runtime)};\nconst settings = ${JSON.stringify(join(stateHome, 'service-release.json'))};\nif (existsSync(settings)) {\n const version = JSON.parse(readFileSync(settings, 'utf8')).version;\n if (/^\\d+\\.\\d+\\.\\d+$/.test(version || '')) {\n const candidate = join(${JSON.stringify(dataHome)}, 'releases', version, 'node_modules/software-defence-factory');\n if (existsSync(join(candidate, 'package.json'))) {\n const pkg = JSON.parse(readFileSync(join(candidate, 'package.json'), 'utf8'));\n const current = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8')).version;\n const a = version.split('.').map(Number), b = current.split('.').map(Number);\n const index = a.findIndex((value, i) => value !== b[i]);\n if (pkg.name === 'software-defence-factory' && pkg.version === version && index >= 0 && a[index] > b[index]) root = candidate;\n }\n }\n}\nprocess.env.SDF_AUTO_UPDATE = '0';\nprocess.env.SDF_BOOTSTRAPPED = '1';\nawait import(pathToFileURL(join(root, 'bin/software-defence-factory.mjs')).href);\n`;
|
|
40
|
+
}
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync, rmSync, cpSync, readdirSync, renameSync } from 'node:fs';
|
|
2
|
+
import { join, dirname } from 'node:path';
|
|
3
|
+
import { homedir, userInfo } from 'node:os';
|
|
4
|
+
import { spawnSync } from 'node:child_process';
|
|
5
|
+
import { createServer } from 'node:net';
|
|
6
|
+
import { ROOT, STATE_HOME, DATA_HOME, SOURCE_CHECKOUT } from './paths.mjs';
|
|
7
|
+
import { configAt, json, save, run, api, sleep, digest } from './lib.mjs';
|
|
8
|
+
import { VERSION, newer, latestVersion, installRelease, busyInstallations } from './updates.mjs';
|
|
9
|
+
import { serviceId, systemdUnit, launchAgent, tunnelArguments, groupArguments, runtimeLauncher } from './service-files.mjs';
|
|
10
|
+
|
|
11
|
+
const records = join(STATE_HOME, 'services');
|
|
12
|
+
const configHome = process.env.XDG_CONFIG_HOME || join(homedir(), '.config');
|
|
13
|
+
const userUnits = join(configHome, 'systemd/user');
|
|
14
|
+
const updater = 'software-defence-factory-update';
|
|
15
|
+
const recordPath = id => join(records, `${id}.json`);
|
|
16
|
+
const systemctl = (...args) => run('systemctl', ['--user', ...args]);
|
|
17
|
+
const uid = () => process.getuid();
|
|
18
|
+
function userOnly() { if (uid() === 0) throw new Error('Install and operate user services without sudo; never run the controller as root'); }
|
|
19
|
+
function alive(pid) { try { process.kill(pid, 0); return true; } catch (error) { if (error.code === 'ESRCH') return false; throw error; } }
|
|
20
|
+
export const controllerService = state => recordPath(serviceId('controller', state));
|
|
21
|
+
export function hasService(state) { return existsSync(controllerService(state)); }
|
|
22
|
+
export async function withServiceOperation(action, perform) {
|
|
23
|
+
const lock = join(STATE_HOME, 'service-update.lock');
|
|
24
|
+
mkdirSync(STATE_HOME, { recursive: true, mode: 0o700 });
|
|
25
|
+
try { writeFileSync(lock, JSON.stringify({ pid: process.pid, action }), { flag: 'wx', mode: 0o600 }); }
|
|
26
|
+
catch (error) { if (error.code === 'EEXIST') throw new Error(`Another service operation may own ${lock}; reconcile its PID before removing the lock`); throw error; }
|
|
27
|
+
try { return await perform(); } finally { rmSync(lock); }
|
|
28
|
+
}
|
|
29
|
+
export function isManagedLaunch(state) {
|
|
30
|
+
return hasService(state) && process.env.SDF_MANAGED_SERVICE === serviceId('controller', state);
|
|
31
|
+
}
|
|
32
|
+
function owned(record) {
|
|
33
|
+
if (!existsSync(record.file) || digest(readFileSync(record.file, 'utf8')) !== record.definitionHash) throw new Error(`Service definition changed outside the CLI; reconcile ${record.file} before changing it`);
|
|
34
|
+
}
|
|
35
|
+
function loaded(record) {
|
|
36
|
+
if (record.platform === 'linux') return systemctl('show', record.unit, '--property=LoadState', '--value') !== 'not-found';
|
|
37
|
+
const result = spawnSync('/bin/launchctl', ['print', `gui/${uid()}/${record.id}`], { encoding: 'utf8' });
|
|
38
|
+
if (result.error) throw result.error;
|
|
39
|
+
if (result.status === 0) return true;
|
|
40
|
+
if (/Could not find service|Could not find specified service/.test(result.stderr)) return false;
|
|
41
|
+
throw new Error(result.stderr || 'Cannot inspect launchd service');
|
|
42
|
+
}
|
|
43
|
+
function active(record) {
|
|
44
|
+
if (record.platform === 'linux') return ['active', 'activating', 'reloading'].includes(systemctl('show', record.unit, '--property=ActiveState', '--value'));
|
|
45
|
+
return loaded(record);
|
|
46
|
+
}
|
|
47
|
+
function start(record) {
|
|
48
|
+
owned(record);
|
|
49
|
+
if (record.platform === 'linux') systemctl('start', record.unit);
|
|
50
|
+
else if (!loaded(record)) run('/bin/launchctl', ['bootstrap', `gui/${uid()}`, record.file]);
|
|
51
|
+
}
|
|
52
|
+
function stop(record) {
|
|
53
|
+
owned(record);
|
|
54
|
+
if (record.platform === 'linux') { if (active(record)) systemctl('stop', record.unit); }
|
|
55
|
+
else if (loaded(record)) run('/bin/launchctl', ['bootout', `gui/${uid()}/${record.id}`]);
|
|
56
|
+
if (record.kind === 'controller') {
|
|
57
|
+
const lock = join(record.state, 'supervisor.json');
|
|
58
|
+
if (existsSync(lock) && alive(json(lock).pid)) throw new Error('A controller remains alive outside the stopped service; reconcile its supervisor PID before proceeding');
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
export async function waitHealthy(record, expectedVersion) {
|
|
62
|
+
const deadline = Date.now() + 30000;
|
|
63
|
+
while (Date.now() < deadline) {
|
|
64
|
+
try {
|
|
65
|
+
if (record.kind === 'controller') {
|
|
66
|
+
const status = await api(record.state, '/api/v1/status');
|
|
67
|
+
if (status.workers?.some(worker => worker.connected) && (!expectedVersion || status.runtime_version === expectedVersion)) return status;
|
|
68
|
+
} else {
|
|
69
|
+
const response = await fetch(`http://127.0.0.1:${record.port}/api/v1/status`, { signal: AbortSignal.timeout(1000) });
|
|
70
|
+
if (response.ok && (await response.json()).workers) return;
|
|
71
|
+
}
|
|
72
|
+
} catch { /* A service may be waiting for Docker or the SSH peer. */ }
|
|
73
|
+
await sleep(500);
|
|
74
|
+
}
|
|
75
|
+
throw new Error(`Service is installed but dashboard is not ready; inspect ${record.file} and service logs. It will keep retrying.`);
|
|
76
|
+
}
|
|
77
|
+
async function portFree(port) {
|
|
78
|
+
await new Promise((resolve, reject) => {
|
|
79
|
+
const server = createServer(); server.once('error', reject);
|
|
80
|
+
server.listen(port, '127.0.0.1', () => server.close(resolve));
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
// Retain only installed package content. A source tree or npx cache is never
|
|
84
|
+
// the service's enduring runtime. Old snapshots remain available for recovery.
|
|
85
|
+
function retainRuntime() {
|
|
86
|
+
if (SOURCE_CHECKOUT) throw new Error('Install the npm artifact before installing a service; source worktrees are not service runtimes');
|
|
87
|
+
const destination = join(DATA_HOME, 'services/runtimes', VERSION);
|
|
88
|
+
if (!existsSync(destination)) {
|
|
89
|
+
mkdirSync(dirname(destination), { recursive: true, mode: 0o700 });
|
|
90
|
+
const staging = `${destination}.${process.pid}.tmp`;
|
|
91
|
+
try {
|
|
92
|
+
mkdirSync(staging, { mode: 0o700 });
|
|
93
|
+
for (const name of ['package.json', 'bin', 'factory', 'kit', '.agents', 'scripts']) if (existsSync(join(ROOT, name))) cpSync(join(ROOT, name), join(staging, name), { recursive: true });
|
|
94
|
+
renameSync(staging, destination);
|
|
95
|
+
} finally { rmSync(staging, { recursive: true, force: true }); }
|
|
96
|
+
}
|
|
97
|
+
if (json(join(destination, 'package.json')).version !== VERSION) throw new Error('Retained service runtime identity mismatch');
|
|
98
|
+
return destination;
|
|
99
|
+
}
|
|
100
|
+
function launcher() {
|
|
101
|
+
const runtime = retainRuntime(), file = join(DATA_HOME, 'services', `launch-${VERSION}-${digest(STATE_HOME).slice(0, 16)}.mjs`);
|
|
102
|
+
const content = runtimeLauncher({ runtime, stateHome: STATE_HOME, dataHome: DATA_HOME });
|
|
103
|
+
if (existsSync(file)) {
|
|
104
|
+
if (readFileSync(file, 'utf8') !== content) throw new Error('Retained launcher content differs; preserve and reconcile it before installation');
|
|
105
|
+
} else writeFileSync(file, content, { flag: 'wx', mode: 0o600 });
|
|
106
|
+
return file;
|
|
107
|
+
}
|
|
108
|
+
function environment() {
|
|
109
|
+
return { PATH: process.env.PATH || '/usr/local/bin:/usr/bin:/bin', HOME: homedir(), XDG_STATE_HOME: dirname(STATE_HOME), XDG_DATA_HOME: dirname(DATA_HOME), XDG_CONFIG_HOME: configHome, SDF_AUTO_UPDATE: '0' };
|
|
110
|
+
}
|
|
111
|
+
export function serviceDefinition(state) {
|
|
112
|
+
configAt(state);
|
|
113
|
+
return systemdUnit({ description: 'Software & Defence Factory', argv: [process.execPath, join(ROOT, 'bin/software-defence-factory.mjs'), 'serve', '--state', state], directory: homedir(), environment: environment() });
|
|
114
|
+
}
|
|
115
|
+
async function install(kind, state, flags) {
|
|
116
|
+
userOnly();
|
|
117
|
+
if (kind === 'controller' && process.platform !== 'linux') throw new Error('Controller services currently require Linux/systemd; use up on macOS. Tunnel services support macOS and Linux.');
|
|
118
|
+
if (!['linux', 'darwin'].includes(process.platform)) throw new Error('Services require Linux/systemd or macOS/launchd');
|
|
119
|
+
const port = kind === 'controller' ? configAt(state).port : Number(flags.port);
|
|
120
|
+
const id = kind === 'controller' ? serviceId(kind, state) : serviceId(kind, `${flags.host}:${port}`);
|
|
121
|
+
if (existsSync(recordPath(id))) {
|
|
122
|
+
const previous = json(recordPath(id)); owned(previous);
|
|
123
|
+
if (kind === 'controller' && previous.state !== state) throw new Error('Service identity mismatch');
|
|
124
|
+
if (previous.port !== port || (flags.group && previous.group !== flags.group)) throw new Error('Service configuration changed; uninstall and reinstall the service while preserving its private state');
|
|
125
|
+
if (previous.platform === 'linux') systemctl('enable', previous.unit);
|
|
126
|
+
start(previous); await waitHealthy(previous);
|
|
127
|
+
console.log(JSON.stringify(await serviceStatus(previous), null, 2)); return;
|
|
128
|
+
}
|
|
129
|
+
if (kind === 'controller') {
|
|
130
|
+
if (!existsSync(join(state, 'engine.json'))) throw new Error('Run install for this state before installing its service');
|
|
131
|
+
if (existsSync(join(state, 'supervisor.json')) && alive(json(join(state, 'supervisor.json')).pid)) throw new Error('Stop the manually started controller before adopting it as a service');
|
|
132
|
+
run('docker', ['image', 'inspect', configAt(state).image]);
|
|
133
|
+
}
|
|
134
|
+
let argv = kind === 'controller' ? [process.execPath, launcher(), 'serve', '--state', state] : tunnelArguments(flags.host, port);
|
|
135
|
+
if (flags.group) {
|
|
136
|
+
if (kind !== 'controller') throw new Error('--group applies only to Linux controller services');
|
|
137
|
+
if (!run('id', ['-nG', userInfo().username]).split(/\s+/).includes(flags.group)) throw new Error('The current user must already belong to the requested group');
|
|
138
|
+
const executable = existsSync('/usr/bin/sg') ? '/usr/bin/sg' : '/usr/bin/newgrp';
|
|
139
|
+
if (executable.endsWith('/newgrp') && !run(executable, ['--help']).includes('--command')) throw new Error('This newgrp does not support commands; use a system with sg or refresh the login session');
|
|
140
|
+
argv = groupArguments(argv, flags.group, executable);
|
|
141
|
+
}
|
|
142
|
+
await portFree(port);
|
|
143
|
+
const record = { id, kind, platform: process.platform, version: VERSION, port, state: kind === 'controller' ? state : undefined, host: flags.host, group: flags.group, argv, environment: environment() };
|
|
144
|
+
if (kind === 'controller') record.environment.SDF_MANAGED_SERVICE = id;
|
|
145
|
+
record.unit = `${id}.service`;
|
|
146
|
+
record.file = process.platform === 'linux' ? join(userUnits, record.unit) : join(homedir(), 'Library/LaunchAgents', `${id}.plist`);
|
|
147
|
+
record.log = join(records, `${id}.log`);
|
|
148
|
+
const definition = process.platform === 'linux' ? systemdUnit({ description: `Software & Defence Factory ${kind}`, argv, directory: homedir(), environment: record.environment })
|
|
149
|
+
: launchAgent({ id, argv, directory: homedir(), log: record.log, environment: record.environment });
|
|
150
|
+
if (existsSync(record.file)) throw new Error(`Refusing to overwrite an unregistered service: ${record.file}`);
|
|
151
|
+
mkdirSync(dirname(record.file), { recursive: true, mode: 0o700 }); mkdirSync(records, { recursive: true, mode: 0o700 });
|
|
152
|
+
writeFileSync(record.file, definition, { mode: 0o600 }); record.definitionHash = digest(definition); save(recordPath(id), record);
|
|
153
|
+
if (process.platform === 'linux') { run('systemd-analyze', ['--user', 'verify', record.file]); systemctl('daemon-reload'); systemctl('enable', record.unit); }
|
|
154
|
+
start(record);
|
|
155
|
+
await waitHealthy(record);
|
|
156
|
+
console.log(JSON.stringify(await serviceStatus(record), null, 2));
|
|
157
|
+
}
|
|
158
|
+
async function serviceStatus(record) {
|
|
159
|
+
owned(record);
|
|
160
|
+
const result = { id: record.id, kind: record.kind, definition: record.file, dashboard: `http://127.0.0.1:${record.port}`, running: active(record) };
|
|
161
|
+
if (record.platform === 'linux') {
|
|
162
|
+
result.enabled = systemctl('show', record.unit, '--property=UnitFileState', '--value') === 'enabled';
|
|
163
|
+
const linger = run('loginctl', ['show-user', userInfo().username, '-p', 'Linger', '--value']);
|
|
164
|
+
result.linger = linger === 'yes'; result.startsAt = result.linger ? 'boot' : 'login';
|
|
165
|
+
if (!result.linger) result.next = `For boot without login, an administrator can run: loginctl enable-linger ${userInfo().username}`;
|
|
166
|
+
result.logs = `journalctl --user -u ${record.unit} -n 100 --no-pager`;
|
|
167
|
+
} else { result.startsAt = 'login'; result.logs = record.log; }
|
|
168
|
+
try {
|
|
169
|
+
const response = await fetch(`http://127.0.0.1:${record.port}/api/v1/status`, { signal: AbortSignal.timeout(1500) });
|
|
170
|
+
const snapshot = await response.json(); result.healthy = response.ok && !!snapshot.workers;
|
|
171
|
+
result.runtimeVersion = snapshot.runtime_version; result.maintenance = snapshot.maintenance;
|
|
172
|
+
}
|
|
173
|
+
catch { result.healthy = false; }
|
|
174
|
+
return result;
|
|
175
|
+
}
|
|
176
|
+
export async function manageService(kind, action, state, flags = {}) {
|
|
177
|
+
userOnly();
|
|
178
|
+
if (kind === 'tunnel') tunnelArguments(flags.host, Number(flags.port));
|
|
179
|
+
if (kind === 'controller' && action === 'update') return updateServices();
|
|
180
|
+
const readonly = ['status', 'logs'].includes(action) || (action === 'updates' && flags.auto === 'status');
|
|
181
|
+
return readonly ? performService(kind, action, state, flags) : withServiceOperation(`${kind} ${action}`, () => performService(kind, action, state, flags));
|
|
182
|
+
}
|
|
183
|
+
async function performService(kind, action, state, flags) {
|
|
184
|
+
if (kind === 'controller' && action === 'updates') return automaticUpdates(flags.auto);
|
|
185
|
+
if (kind === 'controller' && action === 'resume') {
|
|
186
|
+
console.log(JSON.stringify(await api(state, '/api/v1/maintenance', { enabled: false }))); return;
|
|
187
|
+
}
|
|
188
|
+
if (action === 'install') return install(kind, state, flags);
|
|
189
|
+
const id = kind === 'controller' ? serviceId(kind, state) : serviceId(kind, `${flags.host}:${Number(flags.port)}`);
|
|
190
|
+
if (!['start', 'stop', 'restart', 'status', 'uninstall', 'logs'].includes(action)) throw new Error('Use service/tunnel install|start|stop|restart|status|logs|uninstall; service update; service updates --auto on|off|status');
|
|
191
|
+
if (!existsSync(recordPath(id))) {
|
|
192
|
+
if (action === 'status') { console.log(JSON.stringify({ installed: false, id })); return; }
|
|
193
|
+
if (action === 'uninstall') { console.log('Service is already uninstalled; private state was preserved.'); return; }
|
|
194
|
+
throw new Error('Service is not installed');
|
|
195
|
+
}
|
|
196
|
+
const record = json(recordPath(id)); owned(record);
|
|
197
|
+
if (action === 'status') { console.log(JSON.stringify(await serviceStatus(record), null, 2)); return; }
|
|
198
|
+
if (action === 'logs') {
|
|
199
|
+
console.log(record.platform === 'linux' ? run('journalctl', ['--user', '-u', record.unit, '-n', '100', '--no-pager']) : (existsSync(record.log) ? readFileSync(record.log, 'utf8').split('\n').slice(-100).join('\n') : 'No service log yet.')); return;
|
|
200
|
+
}
|
|
201
|
+
if (['stop', 'restart', 'uninstall'].includes(action)) stop(record);
|
|
202
|
+
if (['start', 'restart'].includes(action)) { start(record); await waitHealthy(record); }
|
|
203
|
+
if (action === 'uninstall') {
|
|
204
|
+
if (record.platform === 'linux') systemctl('disable', record.unit);
|
|
205
|
+
rmSync(record.file); rmSync(recordPath(id));
|
|
206
|
+
if (record.platform === 'linux') systemctl('daemon-reload');
|
|
207
|
+
}
|
|
208
|
+
console.log(`${action}: ${id}. Private state and retained runtimes were preserved.`);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function controllers() {
|
|
212
|
+
if (!existsSync(records)) return [];
|
|
213
|
+
return readdirSync(records).filter(name => name.endsWith('.json')).map(name => json(join(records, name))).filter(record => record.kind === 'controller');
|
|
214
|
+
}
|
|
215
|
+
export async function updateServices({ latest = latestVersion, download = installRelease } = {}) {
|
|
216
|
+
userOnly();
|
|
217
|
+
return withServiceOperation('update', () => performUpdate(latest, download));
|
|
218
|
+
}
|
|
219
|
+
async function performUpdate(latest, download) {
|
|
220
|
+
const held = [], stopped = [];
|
|
221
|
+
const preferencesPath = join(STATE_HOME, 'updates.json'), selectionPath = join(STATE_HOME, 'service-release.json');
|
|
222
|
+
const previousPreferences = existsSync(preferencesPath) ? json(preferencesPath) : { enabled: true, lastCheckedAt: 0 };
|
|
223
|
+
const previous = existsSync(selectionPath) ? json(selectionPath) : {};
|
|
224
|
+
let activated = false, releaseMaintenance = true;
|
|
225
|
+
const priorVersions = new Map();
|
|
226
|
+
try {
|
|
227
|
+
const managed = controllers();
|
|
228
|
+
for (const record of managed) owned(record);
|
|
229
|
+
const live = managed.filter(active);
|
|
230
|
+
const installed = managed.map(record => record.version || VERSION).reduce((minimum, version) => newer(minimum, version) ? version : minimum, VERSION);
|
|
231
|
+
const current = previous.version || installed;
|
|
232
|
+
const version = [await latest(), VERSION, current].reduce((selected, candidate) => newer(candidate, selected) ? candidate : selected);
|
|
233
|
+
const runningVersions = await Promise.all(live.map(async record => {
|
|
234
|
+
const value = (await api(record.state, '/api/v1/status')).runtime_version;
|
|
235
|
+
priorVersions.set(record.id, value); return value;
|
|
236
|
+
}));
|
|
237
|
+
if (runningVersions.some(value => !/^\d+\.\d+\.\d+$/.test(value || ''))) throw new Error('Cannot verify the runtime version of a managed controller');
|
|
238
|
+
if (!newer(version, current) && runningVersions.every(value => !newer(version, value))) { console.log(`software-defence-factory ${current} is up to date.`); return; }
|
|
239
|
+
const unowned = busyInstallations().filter(state => !live.some(record => record.state === state));
|
|
240
|
+
if (unowned.length) throw new Error('Unmanaged controllers or executor fences block the update: ' + unowned.join(', '));
|
|
241
|
+
// Reserve idle controllers before downloads. All mutating API actions are
|
|
242
|
+
// refused until restart or explicit release, eliminating submit/stop races.
|
|
243
|
+
for (const record of live) { await api(record.state, '/api/v1/maintenance', { enabled: true }); held.push(record); }
|
|
244
|
+
download(version);
|
|
245
|
+
for (const record of live) { stop(record); stopped.push(record); }
|
|
246
|
+
if (busyInstallations().length) throw new Error('An installation or executor still blocks activation');
|
|
247
|
+
// Ordinary CLI downloads must never change what an existing service starts.
|
|
248
|
+
// Only this reserved update transaction may advance the managed selection.
|
|
249
|
+
activated = true;
|
|
250
|
+
save(selectionPath, { version });
|
|
251
|
+
save(preferencesPath, { ...previousPreferences, version, lastCheckedAt: Date.now() });
|
|
252
|
+
for (const record of stopped) { start(record); await waitHealthy(record, version); }
|
|
253
|
+
console.log(`Updated to ${version}; restarted ${stopped.length} previously active services. Jobs and history preserved.`);
|
|
254
|
+
} catch (error) {
|
|
255
|
+
releaseMaintenance = false;
|
|
256
|
+
try {
|
|
257
|
+
if (activated) {
|
|
258
|
+
for (const record of stopped) stop(record);
|
|
259
|
+
save(selectionPath, previous);
|
|
260
|
+
save(preferencesPath, previousPreferences);
|
|
261
|
+
}
|
|
262
|
+
for (const record of stopped) { start(record); await waitHealthy(record, priorVersions.get(record.id)); }
|
|
263
|
+
releaseMaintenance = true;
|
|
264
|
+
} catch (recoveryError) {
|
|
265
|
+
throw new Error(`Update failed: ${error.message}. Recovery unconfirmed: ${recoveryError.message}. Maintenance reservations were retained.`);
|
|
266
|
+
}
|
|
267
|
+
throw error;
|
|
268
|
+
} finally {
|
|
269
|
+
const failures = [];
|
|
270
|
+
if (releaseMaintenance) for (const record of held) try { await api(record.state, '/api/v1/maintenance', { enabled: false }); } catch (error) { failures.push(`${record.id}: ${error.message}`); }
|
|
271
|
+
if (failures.length) throw new Error('Maintenance release unconfirmed: ' + failures.join('; '));
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
function automaticUpdates(mode) {
|
|
275
|
+
if (process.platform !== 'linux') throw new Error('Scheduled service updates currently require Linux/systemd');
|
|
276
|
+
if (!['on', 'off', 'status'].includes(mode)) throw new Error('Use service updates --auto on|off|status');
|
|
277
|
+
const timer = join(userUnits, `${updater}.timer`), service = join(userUnits, `${updater}.service`), record = join(records, 'updates.json');
|
|
278
|
+
if (mode === 'status') { console.log(JSON.stringify({ installed: existsSync(record), ...(existsSync(record) ? { timer: systemctl('show', `${updater}.timer`, '-p', 'ActiveState', '-p', 'NextElapseUSecRealtime') } : {}) })); return; }
|
|
279
|
+
if (mode === 'off') {
|
|
280
|
+
if (existsSync(record)) {
|
|
281
|
+
if (systemctl('show', `${updater}.service`, '-p', 'ActiveState', '--value') === 'activating') throw new Error('An update is running; wait for it to finish before disabling the timer');
|
|
282
|
+
const data = json(record);
|
|
283
|
+
for (const [file, hash] of [[timer, data.timerHash], [service, data.serviceHash]]) if (!existsSync(file) || digest(readFileSync(file, 'utf8')) !== hash) throw new Error('Update service was edited; reconcile it before removal');
|
|
284
|
+
systemctl('disable', '--now', `${updater}.timer`);
|
|
285
|
+
systemctl('stop', `${updater}.service`);
|
|
286
|
+
rmSync(timer); rmSync(service); rmSync(record); systemctl('daemon-reload');
|
|
287
|
+
}
|
|
288
|
+
console.log('Scheduled updates disabled.'); return;
|
|
289
|
+
}
|
|
290
|
+
if (existsSync(record)) { console.log('Scheduled updates are already installed.'); return; }
|
|
291
|
+
if (existsSync(timer) || existsSync(service)) throw new Error('Refusing to overwrite an unregistered update unit');
|
|
292
|
+
const serviceText = systemdUnit({ description: 'Software & Defence Factory idle update', argv: [process.execPath, launcher(), 'service', 'update'], directory: homedir(), environment: environment(), oneshot: true });
|
|
293
|
+
const timerText = `[Unit]\nDescription=Check Factory updates daily when idle\n\n[Timer]\nOnCalendar=daily\nRandomizedDelaySec=1h\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n`;
|
|
294
|
+
mkdirSync(userUnits, { recursive: true, mode: 0o700 });
|
|
295
|
+
writeFileSync(service, serviceText, { mode: 0o600 }); writeFileSync(timer, timerText, { mode: 0o600 });
|
|
296
|
+
save(record, { serviceHash: digest(serviceText), timerHash: digest(timerText) });
|
|
297
|
+
systemctl('daemon-reload'); systemctl('enable', '--now', `${updater}.timer`); console.log('Daily idle service updates enabled. Busy/unknown installations defer the update.');
|
|
298
|
+
}
|
package/factory/supervisor.mjs
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { writeFileSync, rmSync } from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
+
import { pathToFileURL } from 'node:url';
|
|
3
4
|
import { configAt, stopContainers } from './lib.mjs';
|
|
4
5
|
import { createController } from './server.mjs';
|
|
5
|
-
|
|
6
|
+
export async function supervise(state) {
|
|
7
|
+
const config = configAt(state), lock = join(state, 'supervisor.json');
|
|
6
8
|
writeFileSync(lock, JSON.stringify({ pid: process.pid }), { flag: 'wx', mode: 0o600 });
|
|
7
9
|
let controller, stopping = false;
|
|
8
10
|
async function shutdown(code = 0) {
|
|
@@ -18,3 +20,5 @@ try {
|
|
|
18
20
|
controller.server.on('error', error => { console.error(error.message); shutdown(1); });
|
|
19
21
|
controller.server.listen(config.port, '127.0.0.1', () => console.log(`Factory controller ready on 127.0.0.1:${config.port}`));
|
|
20
22
|
} catch (error) { console.error(error.message); await shutdown(1); }
|
|
23
|
+
}
|
|
24
|
+
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) await supervise(process.argv[2]);
|
package/factory/updates.mjs
CHANGED
|
@@ -104,7 +104,7 @@ export async function bootstrap(args) {
|
|
|
104
104
|
const busy = busyInstallations();
|
|
105
105
|
const cachedVersion = installedEntry(DATA_HOME, preferences.version) ? preferences.version : VERSION;
|
|
106
106
|
let selectedVersion = newer(cachedVersion, VERSION) ? cachedVersion : VERSION;
|
|
107
|
-
const automatic = !['stop', 'cancel', 'status', 'serve', 'version', '--version', '-v'].includes(command)
|
|
107
|
+
const automatic = !['stop', 'cancel', 'status', 'serve', 'service', 'tunnel', 'help', '--help', '-h', 'version', '--version', '-v'].includes(command)
|
|
108
108
|
&& preferences.enabled && process.env.SDF_AUTO_UPDATE !== '0'
|
|
109
109
|
&& Date.now() - preferences.lastCheckedAt >= DAY && busy.length === 0;
|
|
110
110
|
if (explicit || automatic) {
|
package/kit/installation.md
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
| Control surface | Existing issue/check/PR workflow unless otherwise selected |
|
|
11
11
|
| Harness, version, model/provider and billing | Unselected |
|
|
12
12
|
| Execution environment and necessary tools | Files, Git, shell and tests; browser for UI; other tools as needed |
|
|
13
|
+
| Worker boot and operator login | Disk unlock, SSH/network/Docker/model startup, selected services, timer and observed reboot evidence; not applicable for method-only use |
|
|
13
14
|
| Start, stop, routing owner and recovery | Manual until automation is qualified; one owner per job |
|
|
14
15
|
| Authority and allowed network/credential references | Identify permissions without key values |
|
|
15
16
|
| Time, concurrency, attempts and actual provider stop limits | Unknown until exercised |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "software-defence-factory",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.1",
|
|
4
4
|
"description": "Scoped software delivery and defence investigations with isolated jobs, evidence and review",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -29,6 +29,9 @@
|
|
|
29
29
|
"scripts/export-kit.mjs",
|
|
30
30
|
"scripts/probe-platform.mjs",
|
|
31
31
|
"docs/npm.md",
|
|
32
|
+
"docs/setup.md",
|
|
33
|
+
"docs/services.md",
|
|
34
|
+
"docs/recovery.md",
|
|
32
35
|
"docs/quickstart.md",
|
|
33
36
|
"docs/defence-integration.md",
|
|
34
37
|
"docs/ownership.md",
|
|
@@ -68,7 +68,7 @@ try {
|
|
|
68
68
|
assert(!existsSync(join(state,'jobs',changed.id,'accepted.json')));record('changed candidate cannot inherit earlier approval');
|
|
69
69
|
|
|
70
70
|
const policy=await work('Synthetic policy change guard');await waitState(policy.id,'awaiting_approval');
|
|
71
|
-
setConfig({check:'true'});await cli('approve',policy.id);await waitState(policy.id,'failed');
|
|
71
|
+
setConfig({check:original.check==='true'?':':'true'});await cli('approve',policy.id);await waitState(policy.id,'failed');
|
|
72
72
|
assert(!existsSync(join(state,'jobs',policy.id,'accepted.json')));setConfig({});record('changed check policy cannot inherit earlier approval');
|
|
73
73
|
|
|
74
74
|
setConfig({check:'exit 17'});
|