software-defence-factory 0.3.5 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/bin/software-defence-factory.mjs +30 -11
- package/docs/npm.md +6 -1
- package/docs/recovery.md +23 -0
- package/docs/services.md +155 -0
- package/factory/queue.mjs +15 -3
- package/factory/server.mjs +6 -1
- package/factory/service-files.mjs +40 -0
- package/factory/services.mjs +298 -0
- package/factory/supervisor.mjs +5 -1
- package/factory/updates.mjs +1 -1
- package/package.json +3 -1
- package/scripts/probe-platform.mjs +1 -1
package/README.md
CHANGED
|
@@ -23,6 +23,7 @@ Choose the part you need:
|
|
|
23
23
|
| Try the runtime without inference | `software-defence-factory demo` — Docker required; synthetic sample only |
|
|
24
24
|
| Connect an existing repository | [Runtime quickstart](docs/quickstart.md) |
|
|
25
25
|
| Understand installation and updates | [npm and npx](docs/npm.md) |
|
|
26
|
+
| Restore a dashboard after boot or reconnect remotely | [Services and SSH tunnels](docs/services.md) |
|
|
26
27
|
| Review the evidence and limits | [Qualification](docs/proof.md) |
|
|
27
28
|
|
|
28
29
|
The runtime supplies policy and six focused skills to its isolated jobs. `init` configures a private installation; it does not modify the application or start work. Model access and the application's real check command must be configured before using it for delivery.
|
|
@@ -68,6 +69,7 @@ npm run check
|
|
|
68
69
|
```
|
|
69
70
|
|
|
70
71
|
CI builds the dashboard and checks Node 22/24. A version increase merged to `main` is published to npm through the configured release workflow. Installed CLIs can update on invocation when all installations are stopped. See [release and update behavior](docs/npm.md).
|
|
72
|
+
Managed Linux services can also opt into daily updates that reserve idle controllers, preserve stopped projects and restore the prior release if startup fails. See [service operation](docs/services.md).
|
|
71
73
|
|
|
72
74
|
This is a test release. Synthetic qualification demonstrates control flow and isolation, not model quality, application correctness or production readiness. Follow [AGENTS.md](AGENTS.md) for contributions and [SECURITY.md](SECURITY.md) for the trust boundaries.
|
|
73
75
|
|
|
@@ -8,6 +8,7 @@ import { ROOT, PINS, DEFAULT_STATE, configAt, save, json, run, stream, digest, a
|
|
|
8
8
|
import { admitIncident } from '../factory/incident.mjs';
|
|
9
9
|
import { DEFAULT_DEMO_STATE } from '../factory/paths.mjs';
|
|
10
10
|
import { bootstrap, registerInstallation, VERSION } from '../factory/updates.mjs';
|
|
11
|
+
import { hasService, manageService, serviceDefinition, withServiceOperation, isManagedLaunch } from '../factory/services.mjs';
|
|
11
12
|
|
|
12
13
|
try {
|
|
13
14
|
const handled = await bootstrap(process.argv.slice(2));
|
|
@@ -94,7 +95,8 @@ async function stop() {
|
|
|
94
95
|
const {pid}=json(lock);
|
|
95
96
|
if(alive(pid)) {
|
|
96
97
|
const identity=run('ps',['-p',String(pid),'-o','command=']);
|
|
97
|
-
|
|
98
|
+
const controllerProcess=identity.includes(join(ROOT,'factory/supervisor.mjs'))||identity.includes(join(ROOT,'bin/software-defence-factory.mjs')+' serve');
|
|
99
|
+
if(!controllerProcess||!identity.includes(state))throw new Error('PID identity changed; refusing to signal an unrelated process');
|
|
98
100
|
process.kill(pid,'SIGTERM');
|
|
99
101
|
for(let i=0;i<40&&existsSync(lock);i++)await sleep(250);
|
|
100
102
|
if(existsSync(lock))throw new Error('Stop unconfirmed; inspect supervisor and do not start replacement workers');
|
|
@@ -134,18 +136,27 @@ async function jobAction(action) {
|
|
|
134
136
|
|
|
135
137
|
try {
|
|
136
138
|
if(command==='init') { if(!flags.repo)throw new Error('init requires --repo /path/to/existing/git/repo');init(flags.repo,flags.agent,flags.check,flags.port); }
|
|
137
|
-
else if(command==='install')await install
|
|
138
|
-
else if(command==='up')await
|
|
139
|
-
else if(command==='stop')await stop();
|
|
139
|
+
else if(command==='install')await withServiceOperation('install',install);
|
|
140
|
+
else if(command==='up') { if(hasService(state))await manageService('controller','start',state);else await withServiceOperation('up',up); }
|
|
141
|
+
else if(command==='stop') { if(hasService(state))await manageService('controller','stop',state);else await withServiceOperation('stop',stop); }
|
|
140
142
|
else if(command==='serve') {
|
|
143
|
+
const managed=isManagedLaunch(state);
|
|
144
|
+
if(hasService(state)&&!managed)throw new Error('This installation is managed; use service start instead of foreground serve');
|
|
145
|
+
const launch=async()=>{
|
|
141
146
|
if(process.getuid()===0)throw new Error('Use a dedicated unprivileged operator account');
|
|
142
147
|
const lock=join(state,'supervisor.json');if(existsSync(lock)){if(alive(json(lock).pid))throw new Error('Supervisor already running');rmSync(lock);}
|
|
143
|
-
|
|
148
|
+
if(!existsSync(join(state,'engine.json')))throw new Error('Run install first');
|
|
149
|
+
run('docker',['image','inspect',configAt(state).image]);
|
|
150
|
+
registerInstallation(state);await portFree(configAt(state).port);
|
|
151
|
+
const { supervise } = await import('../factory/supervisor.mjs');await supervise(state);
|
|
152
|
+
};
|
|
153
|
+
if(managed)await launch();else await withServiceOperation('serve',launch);
|
|
144
154
|
}
|
|
145
155
|
else if(command==='service') {
|
|
146
|
-
|
|
147
|
-
|
|
156
|
+
if(!positional.length || positional[0]==='print')console.log(serviceDefinition(state));
|
|
157
|
+
else await manageService('controller',positional[0],state,flags);
|
|
148
158
|
}
|
|
159
|
+
else if(command==='tunnel')await manageService('tunnel',positional[0],state,flags);
|
|
149
160
|
else if(command==='status') { const snapshot=await api(state,'/api/v1/status');delete snapshot.csrf_token;console.log(JSON.stringify(snapshot,null,2)); }
|
|
150
161
|
else if(command==='doctor') {
|
|
151
162
|
const config=configAt(state);console.log(JSON.stringify({node:process.version,docker:run('docker',['info','--format','{{.ServerVersion}}']),engineInstalled:existsSync(join(state,'engine.json')),repo:config.repo,agent:config.agent,checksConfigured:!!config.check?.trim(),inference:'Not called or verified',dashboard:`http://127.0.0.1:${config.port}`},null,2));
|
|
@@ -175,7 +186,7 @@ try {
|
|
|
175
186
|
run('git',['-C',repo,'-c','user.name=Factory demo','-c','user.email=demo@localhost','commit','-m','Synthetic fixture']);
|
|
176
187
|
init(repo,'mock',"test \"$(cat value.txt)\" = fixed",Number(flags.port || 7332));
|
|
177
188
|
} else if(configAt(state).agent!=='mock')throw new Error('Demo requires a mock configuration');
|
|
178
|
-
await install();await up();console.log(JSON.stringify(await submit('software','Synthetic installation qualification: fix value.txt. No inference is used.')));
|
|
189
|
+
await withServiceOperation('demo startup',async()=>{await install();await up();});console.log(JSON.stringify(await submit('software','Synthetic installation qualification: fix value.txt. No inference is used.')));
|
|
179
190
|
console.log('Review the synthetic change in the dashboard and approve its handoff.');
|
|
180
191
|
} else if(['version','--version','-v'].includes(command))console.log(VERSION);
|
|
181
192
|
else if(command==='qualify') {
|
|
@@ -183,7 +194,7 @@ try {
|
|
|
183
194
|
} else if(command==='kit') {
|
|
184
195
|
if(!flags.output)throw new Error('kit requires --output NEW_DIRECTORY');
|
|
185
196
|
await stream(process.execPath,[join(ROOT,'scripts/export-kit.mjs'),resolve(flags.output)]);
|
|
186
|
-
} else if(
|
|
197
|
+
} else if(['help','--help','-h'].includes(command))console.log(`Software & Defence Factory ${VERSION} (test release)
|
|
187
198
|
|
|
188
199
|
kit --output NEW_DIRECTORY Export the portable method without a runtime
|
|
189
200
|
demo Install and run a synthetic sample (no model key)
|
|
@@ -191,8 +202,16 @@ try {
|
|
|
191
202
|
init --repo PATH --agent codex|pi|custom --check "npm ci && npm test"
|
|
192
203
|
install Build the isolated job image; the controller ships with the CLI
|
|
193
204
|
doctor | up | status | stop Inspect / operate your private installation
|
|
194
|
-
serve Foreground supervisor
|
|
195
|
-
service
|
|
205
|
+
serve Foreground supervisor
|
|
206
|
+
service [print] Print a systemd user-service definition
|
|
207
|
+
service install|start|stop|restart Manage a Linux user service (--state PATH)
|
|
208
|
+
install accepts --group EXISTING_GROUP
|
|
209
|
+
service status|logs|uninstall Diagnose / remove service, preserve private state
|
|
210
|
+
service update Update idle managed controllers, restore on failure
|
|
211
|
+
service updates --auto on|off|status Daily idle updates through a systemd timer
|
|
212
|
+
service resume Release a reconciled maintenance reservation
|
|
213
|
+
tunnel install|start|stop|status|logs|uninstall --host SSH_ALIAS --port PORT
|
|
214
|
+
Persistent loopback SSH tunnel (macOS/Linux)
|
|
196
215
|
run --file task.md | --issue URL Submit one software vertical slice
|
|
197
216
|
incident --file incident.json Submit a private, read-only incident draft
|
|
198
217
|
approve JOB_ID | cancel JOB_ID Review gate / stop this attempt
|
package/docs/npm.md
CHANGED
|
@@ -42,7 +42,12 @@ An npm-installed CLI checks npm's `latest` stable release at most once a day
|
|
|
42
42
|
when invoked. It downloads and activates a newer release when all registered
|
|
43
43
|
installations are stopped and no executor requires reconciliation. It does not
|
|
44
44
|
run a background updater or interrupt a job. Routine `status`, `stop`, `cancel`,
|
|
45
|
-
`serve
|
|
45
|
+
`serve`, service/tunnel management, help and version commands do not initiate automatic downloads.
|
|
46
|
+
|
|
47
|
+
For an always-running Linux installation, opt into the separate managed daily
|
|
48
|
+
timer with `service updates --auto on`. It reserves idle controllers, updates
|
|
49
|
+
and restores their prior running set without interrupting a job. See
|
|
50
|
+
[services](services.md) for installation, maintenance recovery and limitations.
|
|
46
51
|
|
|
47
52
|
```sh
|
|
48
53
|
software-defence-factory update --check
|
package/docs/recovery.md
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Recovery and retained evidence
|
|
2
|
+
|
|
3
|
+
Use `status --state PATH` and the private supervisor.log to identify the active installation. Stop it before replacing its package or container image.
|
|
4
|
+
|
|
5
|
+
- A normal `stop` signals the controller, waits for the executor process group, removes its labelled containers and retains the database and artifacts.
|
|
6
|
+
- An unconfirmed running attempt becomes `interrupted` on controller restart. It is never silently considered successful.
|
|
7
|
+
- `retry JOB_ID` reconciles the previous process group and containers. A live or unknown writer blocks retry. For a new build/defence attempt, the prior checkout is retained as previous-checkout-*.
|
|
8
|
+
- A failed verification can retry the same unchanged candidate after the check environment is repaired. A changed candidate needs a fresh verification/review sequence.
|
|
9
|
+
- A requested revision retains previous evidence and starts a new build from the source repository with the accumulated feedback. It does not reuse earlier approval.
|
|
10
|
+
- Removing a stopped task from the dashboard hides its queue record. Private artifacts and its deleted_at record remain on disk; this is not secure erasure.
|
|
11
|
+
|
|
12
|
+
Do not remove active.json merely to unblock a job. Establish that its PID, process group and labelled containers are stopped. PID reuse or missing process identity requires operator investigation. Preserve logs and work before cleanup.
|
|
13
|
+
|
|
14
|
+
For backup, stop the installation and copy the complete private state directory, including SQLite files, factory.json and credentials, to an authorized private destination. Restore only while stopped. Update the repository path if it moved, verify ownership/permissions and the pinned image, then inspect state before any retry. Keep previous backups; no automatic destructive schema migration is provided.
|
|
15
|
+
|
|
16
|
+
Earlier experimental engines use a different journal. Start a new state directory for the native 0.3 runtime; preserve old journals separately. There is no automatic import of their jobs or approval state.
|
|
17
|
+
|
|
18
|
+
Verification cleanup makes owned scratch directories traversable before removing
|
|
19
|
+
them and never follows their symlinks. It runs only after container stop is
|
|
20
|
+
confirmed. If a filesystem error still prevents cleanup, the attempt fails and
|
|
21
|
+
retains the original check exit and private log path alongside the cleanup error.
|
|
22
|
+
Inspect that retained attempt before manual removal; never substitute the source
|
|
23
|
+
candidate path for the scratch path.
|
package/docs/services.md
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
# Persistent controllers and remote dashboards
|
|
2
|
+
|
|
3
|
+
Install the npm CLI first. A Factory controller is a Linux systemd **user**
|
|
4
|
+
service; SSH tunnels support Linux systemd and macOS launchd. Neither needs a
|
|
5
|
+
root controller. macOS can still run a local controller with `up`.
|
|
6
|
+
|
|
7
|
+
## Linux controller
|
|
8
|
+
|
|
9
|
+
Configure and install a runtime using the [quickstart](quickstart.md), then:
|
|
10
|
+
|
|
11
|
+
```sh
|
|
12
|
+
software-defence-factory stop --state /absolute/private/state
|
|
13
|
+
software-defence-factory service install --state /absolute/private/state
|
|
14
|
+
software-defence-factory service status --state /absolute/private/state
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Installation enables and starts exactly this controller. It does not submit a
|
|
18
|
+
new task. Existing queued jobs will execute when any controller starts; inspect
|
|
19
|
+
the queue before adopting an installation. Interrupted attempts still require
|
|
20
|
+
the normal explicit reconciliation/retry. To keep a product paused, leave its
|
|
21
|
+
controller stopped and do not install a service for it.
|
|
22
|
+
|
|
23
|
+
The current user must have Docker access; Docker itself must be enabled at boot.
|
|
24
|
+
The CLI records the current PATH, Node executable and private state/data homes.
|
|
25
|
+
The launcher and installed runtime are retained outside the npm/npx cache.
|
|
26
|
+
Removing a Node installation referenced by the service still breaks startup:
|
|
27
|
+
reinstall the service using the intended Node executable after stopping it.
|
|
28
|
+
No credentials are copied into the unit or job containers.
|
|
29
|
+
Each service uses an immutable launcher for its installation version. Installing
|
|
30
|
+
another controller or enabling a timer cannot rewrite an existing launcher's
|
|
31
|
+
base. An explicit managed update controls subsequent release selection in
|
|
32
|
+
`service-release.json`. The separate ordinary CLI `updates.json` preference
|
|
33
|
+
cannot move an existing service to another release, even while all services
|
|
34
|
+
are stopped.
|
|
35
|
+
|
|
36
|
+
If Docker group membership was added after the user service manager started,
|
|
37
|
+
an SSH shell may have access while user services still get permission denied.
|
|
38
|
+
Refresh the login session at a suitable time, or install with the explicit
|
|
39
|
+
`service install --group docker --state PATH` option. This uses `sg` or a
|
|
40
|
+
command-capable util-linux `newgrp` to apply a group the user **already belongs
|
|
41
|
+
to**. It grants no new membership, needs no sudo and keeps the controller's user
|
|
42
|
+
identity. The group launcher is checked for support before installation.
|
|
43
|
+
|
|
44
|
+
For boot **without logging in**, an administrator must enable user lingering:
|
|
45
|
+
|
|
46
|
+
```sh
|
|
47
|
+
sudo loginctl enable-linger USERNAME
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
`service status` reports lingering, boot versus login startup, service enablement
|
|
51
|
+
and actual HTTP health separately. The CLI never silently grants sudo access or
|
|
52
|
+
changes system-wide Docker, firewall or login configuration. Systemd retries
|
|
53
|
+
startup every 15 seconds if Docker or the configured image is unavailable.
|
|
54
|
+
|
|
55
|
+
```sh
|
|
56
|
+
software-defence-factory service stop --state /absolute/private/state
|
|
57
|
+
software-defence-factory service start --state /absolute/private/state
|
|
58
|
+
software-defence-factory service restart --state /absolute/private/state
|
|
59
|
+
software-defence-factory service logs --state /absolute/private/state
|
|
60
|
+
software-defence-factory service uninstall --state /absolute/private/state
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
`up` and `stop` detect a managed installation and use its service manager, so a
|
|
64
|
+
stop does not immediately respawn a second controller. An explicit stop/restart
|
|
65
|
+
may interrupt a job; use it deliberately. `stop` leaves boot enablement in place.
|
|
66
|
+
`uninstall` disables and removes only the registered service definition. It
|
|
67
|
+
preserves the database, credentials, artifacts, retained runtimes and history.
|
|
68
|
+
Repeated installation/removal is safe. Edited/unregistered service definitions
|
|
69
|
+
are refused rather than overwritten. Bare `service` (or `service print`) retains
|
|
70
|
+
the original print-only systemd-unit interface.
|
|
71
|
+
|
|
72
|
+
## Persistent SSH tunnel on the client
|
|
73
|
+
|
|
74
|
+
First establish key authentication and a verified host key using normal SSH.
|
|
75
|
+
The remote dashboard must bind to loopback and the machines must be reachable
|
|
76
|
+
(for example through an existing private VPN). Use a configured SSH alias:
|
|
77
|
+
|
|
78
|
+
```sh
|
|
79
|
+
software-defence-factory tunnel install --host worker --port 7345
|
|
80
|
+
software-defence-factory tunnel status --host worker --port 7345
|
|
81
|
+
software-defence-factory tunnel logs --host worker --port 7345
|
|
82
|
+
software-defence-factory tunnel stop --host worker --port 7345
|
|
83
|
+
software-defence-factory tunnel start --host worker --port 7345
|
|
84
|
+
software-defence-factory tunnel uninstall --host worker --port 7345
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Visit `http://127.0.0.1:7345`. The local and remote ports are intentionally equal
|
|
88
|
+
to preserve the dashboard's Host/Origin protection. The tunnel uses only
|
|
89
|
+
loopback addresses, strict host-key checking and noninteractive authentication.
|
|
90
|
+
It exits on forwarding failure, detects dead connections and restarts through
|
|
91
|
+
the OS service manager. It never disables SSH checks or requests a password.
|
|
92
|
+
Stop an existing manual tunnel before installation; occupied ports are refused.
|
|
93
|
+
|
|
94
|
+
On macOS this is a LaunchAgent: it starts at **login**, not before FileVault is
|
|
95
|
+
unlocked. On Linux, lingering determines boot versus login startup. A sleeping
|
|
96
|
+
or offline peer is still unavailable; the tunnel reconnects when connectivity
|
|
97
|
+
returns. Tailscale/SSH/system power configuration remains machine infrastructure,
|
|
98
|
+
not a Factory network dependency. No root LaunchDaemon is installed.
|
|
99
|
+
|
|
100
|
+
## Updating an always-running controller
|
|
101
|
+
|
|
102
|
+
```sh
|
|
103
|
+
software-defence-factory service update
|
|
104
|
+
software-defence-factory service updates --auto on
|
|
105
|
+
software-defence-factory service updates --auto status
|
|
106
|
+
software-defence-factory service updates --auto off
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
`service update` manages all registered controllers for the current user. It
|
|
110
|
+
refuses unmanaged live controllers or unresolved executor fences. Every running
|
|
111
|
+
controller must atomically accept an idle maintenance reservation: queued work,
|
|
112
|
+
an active phase or a concurrent action defers the update. A reservation blocks
|
|
113
|
+
new tasks and actions and persists across restarts until explicitly released.
|
|
114
|
+
Downloads are immutable. Only the previously running managed controllers are
|
|
115
|
+
stopped and restarted; intentionally stopped projects stay stopped. HTTP health
|
|
116
|
+
must recover. A failed release startup restores the prior selection and starts
|
|
117
|
+
the old services before releasing maintenance. Images and application code are
|
|
118
|
+
never rebuilt or silently changed by this operation.
|
|
119
|
+
Lifecycle changes and updates share an exclusive operation lock. Concurrent
|
|
120
|
+
start/stop/install/uninstall commands fail clearly rather than undoing an update
|
|
121
|
+
or an operator's stop. A managed installation refuses ordinary foreground
|
|
122
|
+
`serve`; operate it through `service start`. Stop also checks for a remaining
|
|
123
|
+
controller outside the service before reporting success. Restart health must
|
|
124
|
+
report the expected runtime version, including during rollback.
|
|
125
|
+
|
|
126
|
+
The opt-in Linux timer checks daily, including a catch-up after downtime, with
|
|
127
|
+
up to one hour of random delay. A busy/unavailable installation causes a failed
|
|
128
|
+
attempt and remains unchanged; the next scheduled invocation retries. Inspect
|
|
129
|
+
`journalctl --user -u software-defence-factory-update.service`. This timer is
|
|
130
|
+
separate from `update --auto on|off`, which controls ordinary CLI invocations.
|
|
131
|
+
Remove the timer with `service updates --auto off` when retiring this setup.
|
|
132
|
+
|
|
133
|
+
## Recovery and proof limits
|
|
134
|
+
|
|
135
|
+
If a service operation is killed or the machine loses power, inspect
|
|
136
|
+
`STATE_HOME/software-defence-factory/service-update.lock`, its PID and the
|
|
137
|
+
service logs. The lock records the owning PID and action. Do not remove a lock belonging to a live or unknown process.
|
|
138
|
+
After confirming the updater stopped, remove that stale lock, inspect the
|
|
139
|
+
selected release and start/verify each prior service. Then release each remaining
|
|
140
|
+
reservation using `service resume --state PATH`. There is no automatic timeout
|
|
141
|
+
that might reopen a queue while an updater is still working.
|
|
142
|
+
|
|
143
|
+
If a generated service was edited outside the CLI, preserve the edit and
|
|
144
|
+
reconcile its recorded definition before using lifecycle commands. A service
|
|
145
|
+
health failure leaves installation and logs in place for diagnosis. Keep the
|
|
146
|
+
prior release, private state and pinned image; see [recovery](recovery.md).
|
|
147
|
+
|
|
148
|
+
Service enablement and process-restart tests are not proof of a full machine
|
|
149
|
+
reboot, pre-login disk availability or connectivity across two physical networks.
|
|
150
|
+
Record those observations separately. A healthy dashboard proves neither model
|
|
151
|
+
quality nor that a product agent should start.
|
|
152
|
+
|
|
153
|
+
References: [systemd service semantics](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html),
|
|
154
|
+
[loginctl lingering](https://www.freedesktop.org/software/systemd/man/latest/loginctl.html),
|
|
155
|
+
[Apple launchd definitions](https://github.com/apple-oss-distributions/launchd/blob/main/man/launchd.plist.5).
|
package/factory/queue.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { DatabaseSync } from 'node:sqlite';
|
|
2
2
|
import { randomBytes } from 'node:crypto';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
|
+
import { existsSync, writeFileSync, rmSync } from 'node:fs';
|
|
4
5
|
|
|
5
6
|
const workflows = { software: ['build', 'verify', 'review', 'handoff'], defence: ['defence'] };
|
|
6
7
|
const id = prefix => prefix + '_' + randomBytes(12).toString('hex');
|
|
@@ -14,7 +15,8 @@ export class JobQueue {
|
|
|
14
15
|
this.db = new DatabaseSync(join(state, 'jobs.sqlite'));
|
|
15
16
|
this.db.exec('PRAGMA journal_mode=WAL; PRAGMA busy_timeout=5000; CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, data TEXT NOT NULL);');
|
|
16
17
|
this.execute = execute; this.stop = stop; this.reconcile = reconcile;
|
|
17
|
-
this.
|
|
18
|
+
this.maintenanceFile = join(state, 'maintenance.json');
|
|
19
|
+
this.active = null; this.closing = false; this.pumping = false; this.actions = new Set(); this.maintenance = existsSync(this.maintenanceFile);
|
|
18
20
|
for (const job of this.all()) if (['running', 'cancelling'].includes(job.state)) {
|
|
19
21
|
job.state = 'interrupted';
|
|
20
22
|
Object.assign(job.runs.at(-1), { state: 'interrupted', completed_at: now(), error: 'Controller stopped before completion was confirmed.' });
|
|
@@ -32,6 +34,7 @@ export class JobQueue {
|
|
|
32
34
|
save(job) { job.updated_at = now(); this.db.prepare('INSERT INTO jobs VALUES (?,?) ON CONFLICT(id) DO UPDATE SET data=excluded.data').run(job.id, JSON.stringify(job)); return job; }
|
|
33
35
|
submit(input) {
|
|
34
36
|
if (this.closing) throw new QueueError('Controller is stopping');
|
|
37
|
+
if (this.maintenance) throw new QueueError('Controller is reserved for maintenance');
|
|
35
38
|
if (input?.source_url && (typeof input.source_url !== 'string' || !/^https?:\/\/[^\s]+$/.test(input.source_url) || input.source_url.length > 2048)) throw new QueueError('Expected an HTTP(S) source link', 400);
|
|
36
39
|
if (input?.model && (typeof input.model !== 'string' || !/^[\w.:/+-]{1,128}$/.test(input.model))) throw new QueueError('Invalid model identifier', 400);
|
|
37
40
|
if (input?.source_url && !input.spec?.trim()) input = { ...input, spec: `Investigate the linked requirements within this repository's scope: ${input.source_url}` };
|
|
@@ -42,10 +45,19 @@ export class JobQueue {
|
|
|
42
45
|
state: 'queued', created_at: now(), runs: [] };
|
|
43
46
|
this.save(job); this.schedule(); return { id: job.id };
|
|
44
47
|
}
|
|
48
|
+
setMaintenance(enabled) {
|
|
49
|
+
if (typeof enabled !== 'boolean') throw new QueueError('Expected enabled: true or false', 400);
|
|
50
|
+
if (enabled && (this.closing || this.active || this.actions.size || this.all().some(job => ['queued', 'running', 'cancelling'].includes(job.state)))) throw new QueueError('Controller is busy; update deferred');
|
|
51
|
+
if (enabled) writeFileSync(this.maintenanceFile, JSON.stringify({ startedAt: now() }), { mode: 0o600 });
|
|
52
|
+
else rmSync(this.maintenanceFile, { force: true });
|
|
53
|
+
this.maintenance = enabled;
|
|
54
|
+
if (!enabled) this.schedule();
|
|
55
|
+
return { maintenance: enabled };
|
|
56
|
+
}
|
|
45
57
|
schedule() { if (!this.closing && !this.pumping) { this.pumping = true; queueMicrotask(() => this.pump()); } }
|
|
46
58
|
async pump() {
|
|
47
59
|
try {
|
|
48
|
-
while (!this.closing) {
|
|
60
|
+
while (!this.closing && !this.maintenance) {
|
|
49
61
|
let job = this.all().find(item => item.state === 'queued'); if (!job) break;
|
|
50
62
|
const step = job.workflow.current_step, phase = job.workflow.steps[step];
|
|
51
63
|
let attempt = job.runs.at(-1);
|
|
@@ -77,7 +89,7 @@ export class JobQueue {
|
|
|
77
89
|
} finally { this.pumping = false; }
|
|
78
90
|
}
|
|
79
91
|
async exclusive(jobId, perform) {
|
|
80
|
-
if (this.closing || this.actions.has(jobId)) throw new QueueError('Job is already changing; reload before acting');
|
|
92
|
+
if (this.closing || this.maintenance || this.actions.has(jobId)) throw new QueueError('Job is already changing or controller is reserved for maintenance; reload before acting');
|
|
81
93
|
this.actions.add(jobId);
|
|
82
94
|
try { return await perform(); } finally { this.actions.delete(jobId); }
|
|
83
95
|
}
|
package/factory/server.mjs
CHANGED
|
@@ -6,6 +6,7 @@ import { hostname } from 'node:os';
|
|
|
6
6
|
import { JobQueue, QueueError } from './queue.mjs';
|
|
7
7
|
import { executors } from './processes.mjs';
|
|
8
8
|
import { configAt, ROOT } from './lib.mjs';
|
|
9
|
+
import { VERSION } from './updates.mjs';
|
|
9
10
|
|
|
10
11
|
function equal(a, b) { return typeof a === 'string' && Buffer.byteLength(a) === Buffer.byteLength(b) && timingSafeEqual(Buffer.from(a), Buffer.from(b)); }
|
|
11
12
|
async function body(request) {
|
|
@@ -42,7 +43,7 @@ export function createController(state, adapter = executors(state)) {
|
|
|
42
43
|
outcome: attempt.outcome || (attempt.state === 'succeeded' ? 'complete' : undefined),
|
|
43
44
|
executor: ['verify','handoff'].includes(attempt.command) ? 'deterministic' : config.agent,
|
|
44
45
|
worker_name: hostname(), model: job.model || config.model })) }));
|
|
45
|
-
return send(200, { version: 1, workflows: ['software', 'defence'], commands: [], triggers: [], jobs, csrf_token: csrf,
|
|
46
|
+
return send(200, { version: 1, runtime_version: VERSION, maintenance: queue.maintenance, workflows: ['software', 'defence'], commands: [], triggers: [], jobs, csrf_token: csrf,
|
|
46
47
|
workers: [{ name: hostname(), instance_id: 'local-executor', repositories: ['app'], connected: !queue.closing, last_seen_at: new Date().toISOString() }],
|
|
47
48
|
repositories: ['app'], repo: config.repo, agent: config.agent });
|
|
48
49
|
}
|
|
@@ -73,6 +74,10 @@ export function createController(state, adapter = executors(state)) {
|
|
|
73
74
|
if (!authenticated) throw new QueueError('Session required', 403);
|
|
74
75
|
if (!(request.headers['content-type'] || '').startsWith('application/json')) throw new QueueError('Use application/json', 415);
|
|
75
76
|
const input = await body(request);
|
|
77
|
+
if (url.pathname === '/api/v1/maintenance') {
|
|
78
|
+
if (!equal(request.headers.authorization, `Bearer ${token}`)) throw new QueueError('Operator token required for maintenance', 403);
|
|
79
|
+
return send(200, queue.setMaintenance(input.enabled));
|
|
80
|
+
}
|
|
76
81
|
if (url.pathname === '/api/v1/jobs') {
|
|
77
82
|
if (input.model && input.model !== config.model && !['codex','pi'].includes(config.agent)) throw new QueueError('Model overrides require a codex or pi executor', 400);
|
|
78
83
|
return send(201, queue.submit(input));
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
|
|
4
|
+
export function serviceId(kind, identity) {
|
|
5
|
+
return `software-defence-factory-${kind}-${createHash('sha256').update(identity).digest('hex').slice(0, 16)}`;
|
|
6
|
+
}
|
|
7
|
+
function text(value) {
|
|
8
|
+
if (typeof value !== 'string' || /[\x00-\x1f\x7f]/.test(value)) throw new Error('Service values cannot contain control characters');
|
|
9
|
+
return value;
|
|
10
|
+
}
|
|
11
|
+
const unit = value => JSON.stringify(text(value).replaceAll('%', '%%'));
|
|
12
|
+
const argument = value => unit(value.replaceAll('$', () => '$$'));
|
|
13
|
+
const xml = value => text(value).replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>').replaceAll('"', '"');
|
|
14
|
+
|
|
15
|
+
export function systemdUnit({ description, argv, directory, environment = {}, oneshot = false }) {
|
|
16
|
+
return `[Unit]\nDescription=${text(description)}\nStartLimitIntervalSec=0\n\n[Service]\nType=${oneshot ? 'oneshot' : 'exec'}\nWorkingDirectory=${text(directory).replaceAll('%', '%%')}\nExecStart=${argv.map(argument).join(' ')}\n${Object.entries(environment).map(([key, value]) => {
|
|
17
|
+
if (!/^[A-Z_][A-Z0-9_]*$/.test(key)) throw new Error('Invalid environment name');
|
|
18
|
+
return `Environment=${unit(`${key}=${value}`)}\n`;
|
|
19
|
+
}).join('')}${oneshot ? 'TimeoutStartSec=15min\n' : 'Restart=always\nRestartSec=15\n'}TimeoutStopSec=60\nKillMode=control-group\nUMask=0077\n\n[Install]\nWantedBy=default.target\n`;
|
|
20
|
+
}
|
|
21
|
+
export function launchAgent({ id, argv, directory, log, environment = {} }) {
|
|
22
|
+
return `<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n<plist version="1.0"><dict>\n<key>Label</key><string>${xml(id)}</string>\n<key>ProgramArguments</key><array>${argv.map(value => `<string>${xml(value)}</string>`).join('')}</array>\n<key>WorkingDirectory</key><string>${xml(directory)}</string>\n<key>EnvironmentVariables</key><dict>${Object.entries(environment).map(([key, value]) => `<key>${xml(key)}</key><string>${xml(value)}</string>`).join('')}</dict>\n<key>RunAtLoad</key><true/><key>KeepAlive</key><true/>\n<key>ThrottleInterval</key><integer>15</integer>\n<key>ExitTimeOut</key><integer>60</integer>\n<key>Umask</key><integer>63</integer>\n<key>StandardOutPath</key><string>${xml(log)}</string>\n<key>StandardErrorPath</key><string>${xml(log)}</string>\n</dict></plist>\n`;
|
|
23
|
+
}
|
|
24
|
+
export function tunnelArguments(host, port) {
|
|
25
|
+
if (!/^[a-zA-Z0-9][a-zA-Z0-9_.@-]*$/.test(host || '')) throw new Error('Use an SSH hostname or configured alias without spaces or options');
|
|
26
|
+
if (!Number.isInteger(port) || port < 1024 || port > 65535) throw new Error('Tunnel port must be 1024–65535');
|
|
27
|
+
return ['/usr/bin/ssh', '-N', '-T', '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=yes', '-o', 'ExitOnForwardFailure=yes',
|
|
28
|
+
'-o', 'ConnectTimeout=10', '-o', 'ServerAliveInterval=15', '-o', 'ServerAliveCountMax=3', '-L', `127.0.0.1:${port}:127.0.0.1:${port}`, host];
|
|
29
|
+
}
|
|
30
|
+
export function groupArguments(argv, group, executable) {
|
|
31
|
+
if (!/^[a-z_][a-z0-9_-]*[$]?$/.test(group || '')) throw new Error('Expected an existing Unix group name');
|
|
32
|
+
if (!['/usr/bin/sg', '/usr/bin/newgrp'].includes(executable)) throw new Error('Unsupported group launcher');
|
|
33
|
+
const quote = value => `'${text(value).replaceAll("'", "'\\''")}'`;
|
|
34
|
+
return [executable, group, '-c', 'exec ' + argv.map(quote).join(' ')];
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// Immutable launcher: ordinary CLI preferences never select a service release.
|
|
38
|
+
export function runtimeLauncher({ runtime, stateHome, dataHome }) {
|
|
39
|
+
return `import { readFileSync, existsSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { pathToFileURL } from 'node:url';\nlet root = ${JSON.stringify(runtime)};\nconst settings = ${JSON.stringify(join(stateHome, 'service-release.json'))};\nif (existsSync(settings)) {\n const version = JSON.parse(readFileSync(settings, 'utf8')).version;\n if (/^\\d+\\.\\d+\\.\\d+$/.test(version || '')) {\n const candidate = join(${JSON.stringify(dataHome)}, 'releases', version, 'node_modules/software-defence-factory');\n if (existsSync(join(candidate, 'package.json'))) {\n const pkg = JSON.parse(readFileSync(join(candidate, 'package.json'), 'utf8'));\n const current = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8')).version;\n const a = version.split('.').map(Number), b = current.split('.').map(Number);\n const index = a.findIndex((value, i) => value !== b[i]);\n if (pkg.name === 'software-defence-factory' && pkg.version === version && index >= 0 && a[index] > b[index]) root = candidate;\n }\n }\n}\nprocess.env.SDF_AUTO_UPDATE = '0';\nprocess.env.SDF_BOOTSTRAPPED = '1';\nawait import(pathToFileURL(join(root, 'bin/software-defence-factory.mjs')).href);\n`;
|
|
40
|
+
}
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync, rmSync, cpSync, readdirSync, renameSync } from 'node:fs';
|
|
2
|
+
import { join, dirname } from 'node:path';
|
|
3
|
+
import { homedir, userInfo } from 'node:os';
|
|
4
|
+
import { spawnSync } from 'node:child_process';
|
|
5
|
+
import { createServer } from 'node:net';
|
|
6
|
+
import { ROOT, STATE_HOME, DATA_HOME, SOURCE_CHECKOUT } from './paths.mjs';
|
|
7
|
+
import { configAt, json, save, run, api, sleep, digest } from './lib.mjs';
|
|
8
|
+
import { VERSION, newer, latestVersion, installRelease, busyInstallations } from './updates.mjs';
|
|
9
|
+
import { serviceId, systemdUnit, launchAgent, tunnelArguments, groupArguments, runtimeLauncher } from './service-files.mjs';
|
|
10
|
+
|
|
11
|
+
const records = join(STATE_HOME, 'services');
|
|
12
|
+
const configHome = process.env.XDG_CONFIG_HOME || join(homedir(), '.config');
|
|
13
|
+
const userUnits = join(configHome, 'systemd/user');
|
|
14
|
+
const updater = 'software-defence-factory-update';
|
|
15
|
+
const recordPath = id => join(records, `${id}.json`);
|
|
16
|
+
const systemctl = (...args) => run('systemctl', ['--user', ...args]);
|
|
17
|
+
const uid = () => process.getuid();
|
|
18
|
+
function userOnly() { if (uid() === 0) throw new Error('Install and operate user services without sudo; never run the controller as root'); }
|
|
19
|
+
function alive(pid) { try { process.kill(pid, 0); return true; } catch (error) { if (error.code === 'ESRCH') return false; throw error; } }
|
|
20
|
+
export const controllerService = state => recordPath(serviceId('controller', state));
|
|
21
|
+
export function hasService(state) { return existsSync(controllerService(state)); }
|
|
22
|
+
export async function withServiceOperation(action, perform) {
|
|
23
|
+
const lock = join(STATE_HOME, 'service-update.lock');
|
|
24
|
+
mkdirSync(STATE_HOME, { recursive: true, mode: 0o700 });
|
|
25
|
+
try { writeFileSync(lock, JSON.stringify({ pid: process.pid, action }), { flag: 'wx', mode: 0o600 }); }
|
|
26
|
+
catch (error) { if (error.code === 'EEXIST') throw new Error(`Another service operation may own ${lock}; reconcile its PID before removing the lock`); throw error; }
|
|
27
|
+
try { return await perform(); } finally { rmSync(lock); }
|
|
28
|
+
}
|
|
29
|
+
export function isManagedLaunch(state) {
|
|
30
|
+
return hasService(state) && process.env.SDF_MANAGED_SERVICE === serviceId('controller', state);
|
|
31
|
+
}
|
|
32
|
+
function owned(record) {
|
|
33
|
+
if (!existsSync(record.file) || digest(readFileSync(record.file, 'utf8')) !== record.definitionHash) throw new Error(`Service definition changed outside the CLI; reconcile ${record.file} before changing it`);
|
|
34
|
+
}
|
|
35
|
+
function loaded(record) {
|
|
36
|
+
if (record.platform === 'linux') return systemctl('show', record.unit, '--property=LoadState', '--value') !== 'not-found';
|
|
37
|
+
const result = spawnSync('/bin/launchctl', ['print', `gui/${uid()}/${record.id}`], { encoding: 'utf8' });
|
|
38
|
+
if (result.error) throw result.error;
|
|
39
|
+
if (result.status === 0) return true;
|
|
40
|
+
if (/Could not find service|Could not find specified service/.test(result.stderr)) return false;
|
|
41
|
+
throw new Error(result.stderr || 'Cannot inspect launchd service');
|
|
42
|
+
}
|
|
43
|
+
function active(record) {
|
|
44
|
+
if (record.platform === 'linux') return ['active', 'activating', 'reloading'].includes(systemctl('show', record.unit, '--property=ActiveState', '--value'));
|
|
45
|
+
return loaded(record);
|
|
46
|
+
}
|
|
47
|
+
function start(record) {
|
|
48
|
+
owned(record);
|
|
49
|
+
if (record.platform === 'linux') systemctl('start', record.unit);
|
|
50
|
+
else if (!loaded(record)) run('/bin/launchctl', ['bootstrap', `gui/${uid()}`, record.file]);
|
|
51
|
+
}
|
|
52
|
+
function stop(record) {
|
|
53
|
+
owned(record);
|
|
54
|
+
if (record.platform === 'linux') { if (active(record)) systemctl('stop', record.unit); }
|
|
55
|
+
else if (loaded(record)) run('/bin/launchctl', ['bootout', `gui/${uid()}/${record.id}`]);
|
|
56
|
+
if (record.kind === 'controller') {
|
|
57
|
+
const lock = join(record.state, 'supervisor.json');
|
|
58
|
+
if (existsSync(lock) && alive(json(lock).pid)) throw new Error('A controller remains alive outside the stopped service; reconcile its supervisor PID before proceeding');
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
export async function waitHealthy(record, expectedVersion) {
|
|
62
|
+
const deadline = Date.now() + 30000;
|
|
63
|
+
while (Date.now() < deadline) {
|
|
64
|
+
try {
|
|
65
|
+
if (record.kind === 'controller') {
|
|
66
|
+
const status = await api(record.state, '/api/v1/status');
|
|
67
|
+
if (status.workers?.some(worker => worker.connected) && (!expectedVersion || status.runtime_version === expectedVersion)) return status;
|
|
68
|
+
} else {
|
|
69
|
+
const response = await fetch(`http://127.0.0.1:${record.port}/api/v1/status`, { signal: AbortSignal.timeout(1000) });
|
|
70
|
+
if (response.ok && (await response.json()).workers) return;
|
|
71
|
+
}
|
|
72
|
+
} catch { /* A service may be waiting for Docker or the SSH peer. */ }
|
|
73
|
+
await sleep(500);
|
|
74
|
+
}
|
|
75
|
+
throw new Error(`Service is installed but dashboard is not ready; inspect ${record.file} and service logs. It will keep retrying.`);
|
|
76
|
+
}
|
|
77
|
+
async function portFree(port) {
|
|
78
|
+
await new Promise((resolve, reject) => {
|
|
79
|
+
const server = createServer(); server.once('error', reject);
|
|
80
|
+
server.listen(port, '127.0.0.1', () => server.close(resolve));
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
// Retain only installed package content. A source tree or npx cache is never
|
|
84
|
+
// the service's enduring runtime. Old snapshots remain available for recovery.
|
|
85
|
+
function retainRuntime() {
|
|
86
|
+
if (SOURCE_CHECKOUT) throw new Error('Install the npm artifact before installing a service; source worktrees are not service runtimes');
|
|
87
|
+
const destination = join(DATA_HOME, 'services/runtimes', VERSION);
|
|
88
|
+
if (!existsSync(destination)) {
|
|
89
|
+
mkdirSync(dirname(destination), { recursive: true, mode: 0o700 });
|
|
90
|
+
const staging = `${destination}.${process.pid}.tmp`;
|
|
91
|
+
try {
|
|
92
|
+
mkdirSync(staging, { mode: 0o700 });
|
|
93
|
+
for (const name of ['package.json', 'bin', 'factory', 'kit', '.agents', 'scripts']) if (existsSync(join(ROOT, name))) cpSync(join(ROOT, name), join(staging, name), { recursive: true });
|
|
94
|
+
renameSync(staging, destination);
|
|
95
|
+
} finally { rmSync(staging, { recursive: true, force: true }); }
|
|
96
|
+
}
|
|
97
|
+
if (json(join(destination, 'package.json')).version !== VERSION) throw new Error('Retained service runtime identity mismatch');
|
|
98
|
+
return destination;
|
|
99
|
+
}
|
|
100
|
+
function launcher() {
|
|
101
|
+
const runtime = retainRuntime(), file = join(DATA_HOME, 'services', `launch-${VERSION}-${digest(STATE_HOME).slice(0, 16)}.mjs`);
|
|
102
|
+
const content = runtimeLauncher({ runtime, stateHome: STATE_HOME, dataHome: DATA_HOME });
|
|
103
|
+
if (existsSync(file)) {
|
|
104
|
+
if (readFileSync(file, 'utf8') !== content) throw new Error('Retained launcher content differs; preserve and reconcile it before installation');
|
|
105
|
+
} else writeFileSync(file, content, { flag: 'wx', mode: 0o600 });
|
|
106
|
+
return file;
|
|
107
|
+
}
|
|
108
|
+
function environment() {
|
|
109
|
+
return { PATH: process.env.PATH || '/usr/local/bin:/usr/bin:/bin', HOME: homedir(), XDG_STATE_HOME: dirname(STATE_HOME), XDG_DATA_HOME: dirname(DATA_HOME), XDG_CONFIG_HOME: configHome, SDF_AUTO_UPDATE: '0' };
|
|
110
|
+
}
|
|
111
|
+
export function serviceDefinition(state) {
|
|
112
|
+
configAt(state);
|
|
113
|
+
return systemdUnit({ description: 'Software & Defence Factory', argv: [process.execPath, join(ROOT, 'bin/software-defence-factory.mjs'), 'serve', '--state', state], directory: homedir(), environment: environment() });
|
|
114
|
+
}
|
|
115
|
+
async function install(kind, state, flags) {
|
|
116
|
+
userOnly();
|
|
117
|
+
if (kind === 'controller' && process.platform !== 'linux') throw new Error('Controller services currently require Linux/systemd; use up on macOS. Tunnel services support macOS and Linux.');
|
|
118
|
+
if (!['linux', 'darwin'].includes(process.platform)) throw new Error('Services require Linux/systemd or macOS/launchd');
|
|
119
|
+
const port = kind === 'controller' ? configAt(state).port : Number(flags.port);
|
|
120
|
+
const id = kind === 'controller' ? serviceId(kind, state) : serviceId(kind, `${flags.host}:${port}`);
|
|
121
|
+
if (existsSync(recordPath(id))) {
|
|
122
|
+
const previous = json(recordPath(id)); owned(previous);
|
|
123
|
+
if (kind === 'controller' && previous.state !== state) throw new Error('Service identity mismatch');
|
|
124
|
+
if (previous.port !== port || (flags.group && previous.group !== flags.group)) throw new Error('Service configuration changed; uninstall and reinstall the service while preserving its private state');
|
|
125
|
+
if (previous.platform === 'linux') systemctl('enable', previous.unit);
|
|
126
|
+
start(previous); await waitHealthy(previous);
|
|
127
|
+
console.log(JSON.stringify(await serviceStatus(previous), null, 2)); return;
|
|
128
|
+
}
|
|
129
|
+
if (kind === 'controller') {
|
|
130
|
+
if (!existsSync(join(state, 'engine.json'))) throw new Error('Run install for this state before installing its service');
|
|
131
|
+
if (existsSync(join(state, 'supervisor.json')) && alive(json(join(state, 'supervisor.json')).pid)) throw new Error('Stop the manually started controller before adopting it as a service');
|
|
132
|
+
run('docker', ['image', 'inspect', configAt(state).image]);
|
|
133
|
+
}
|
|
134
|
+
let argv = kind === 'controller' ? [process.execPath, launcher(), 'serve', '--state', state] : tunnelArguments(flags.host, port);
|
|
135
|
+
if (flags.group) {
|
|
136
|
+
if (kind !== 'controller') throw new Error('--group applies only to Linux controller services');
|
|
137
|
+
if (!run('id', ['-nG', userInfo().username]).split(/\s+/).includes(flags.group)) throw new Error('The current user must already belong to the requested group');
|
|
138
|
+
const executable = existsSync('/usr/bin/sg') ? '/usr/bin/sg' : '/usr/bin/newgrp';
|
|
139
|
+
if (executable.endsWith('/newgrp') && !run(executable, ['--help']).includes('--command')) throw new Error('This newgrp does not support commands; use a system with sg or refresh the login session');
|
|
140
|
+
argv = groupArguments(argv, flags.group, executable);
|
|
141
|
+
}
|
|
142
|
+
await portFree(port);
|
|
143
|
+
const record = { id, kind, platform: process.platform, version: VERSION, port, state: kind === 'controller' ? state : undefined, host: flags.host, group: flags.group, argv, environment: environment() };
|
|
144
|
+
if (kind === 'controller') record.environment.SDF_MANAGED_SERVICE = id;
|
|
145
|
+
record.unit = `${id}.service`;
|
|
146
|
+
record.file = process.platform === 'linux' ? join(userUnits, record.unit) : join(homedir(), 'Library/LaunchAgents', `${id}.plist`);
|
|
147
|
+
record.log = join(records, `${id}.log`);
|
|
148
|
+
const definition = process.platform === 'linux' ? systemdUnit({ description: `Software & Defence Factory ${kind}`, argv, directory: homedir(), environment: record.environment })
|
|
149
|
+
: launchAgent({ id, argv, directory: homedir(), log: record.log, environment: record.environment });
|
|
150
|
+
if (existsSync(record.file)) throw new Error(`Refusing to overwrite an unregistered service: ${record.file}`);
|
|
151
|
+
mkdirSync(dirname(record.file), { recursive: true, mode: 0o700 }); mkdirSync(records, { recursive: true, mode: 0o700 });
|
|
152
|
+
writeFileSync(record.file, definition, { mode: 0o600 }); record.definitionHash = digest(definition); save(recordPath(id), record);
|
|
153
|
+
if (process.platform === 'linux') { run('systemd-analyze', ['--user', 'verify', record.file]); systemctl('daemon-reload'); systemctl('enable', record.unit); }
|
|
154
|
+
start(record);
|
|
155
|
+
await waitHealthy(record);
|
|
156
|
+
console.log(JSON.stringify(await serviceStatus(record), null, 2));
|
|
157
|
+
}
|
|
158
|
+
async function serviceStatus(record) {
|
|
159
|
+
owned(record);
|
|
160
|
+
const result = { id: record.id, kind: record.kind, definition: record.file, dashboard: `http://127.0.0.1:${record.port}`, running: active(record) };
|
|
161
|
+
if (record.platform === 'linux') {
|
|
162
|
+
result.enabled = systemctl('show', record.unit, '--property=UnitFileState', '--value') === 'enabled';
|
|
163
|
+
const linger = run('loginctl', ['show-user', userInfo().username, '-p', 'Linger', '--value']);
|
|
164
|
+
result.linger = linger === 'yes'; result.startsAt = result.linger ? 'boot' : 'login';
|
|
165
|
+
if (!result.linger) result.next = `For boot without login, an administrator can run: loginctl enable-linger ${userInfo().username}`;
|
|
166
|
+
result.logs = `journalctl --user -u ${record.unit} -n 100 --no-pager`;
|
|
167
|
+
} else { result.startsAt = 'login'; result.logs = record.log; }
|
|
168
|
+
try {
|
|
169
|
+
const response = await fetch(`http://127.0.0.1:${record.port}/api/v1/status`, { signal: AbortSignal.timeout(1500) });
|
|
170
|
+
const snapshot = await response.json(); result.healthy = response.ok && !!snapshot.workers;
|
|
171
|
+
result.runtimeVersion = snapshot.runtime_version; result.maintenance = snapshot.maintenance;
|
|
172
|
+
}
|
|
173
|
+
catch { result.healthy = false; }
|
|
174
|
+
return result;
|
|
175
|
+
}
|
|
176
|
+
export async function manageService(kind, action, state, flags = {}) {
|
|
177
|
+
userOnly();
|
|
178
|
+
if (kind === 'tunnel') tunnelArguments(flags.host, Number(flags.port));
|
|
179
|
+
if (kind === 'controller' && action === 'update') return updateServices();
|
|
180
|
+
const readonly = ['status', 'logs'].includes(action) || (action === 'updates' && flags.auto === 'status');
|
|
181
|
+
return readonly ? performService(kind, action, state, flags) : withServiceOperation(`${kind} ${action}`, () => performService(kind, action, state, flags));
|
|
182
|
+
}
|
|
183
|
+
async function performService(kind, action, state, flags) {
|
|
184
|
+
if (kind === 'controller' && action === 'updates') return automaticUpdates(flags.auto);
|
|
185
|
+
if (kind === 'controller' && action === 'resume') {
|
|
186
|
+
console.log(JSON.stringify(await api(state, '/api/v1/maintenance', { enabled: false }))); return;
|
|
187
|
+
}
|
|
188
|
+
if (action === 'install') return install(kind, state, flags);
|
|
189
|
+
const id = kind === 'controller' ? serviceId(kind, state) : serviceId(kind, `${flags.host}:${Number(flags.port)}`);
|
|
190
|
+
if (!['start', 'stop', 'restart', 'status', 'uninstall', 'logs'].includes(action)) throw new Error('Use service/tunnel install|start|stop|restart|status|logs|uninstall; service update; service updates --auto on|off|status');
|
|
191
|
+
if (!existsSync(recordPath(id))) {
|
|
192
|
+
if (action === 'status') { console.log(JSON.stringify({ installed: false, id })); return; }
|
|
193
|
+
if (action === 'uninstall') { console.log('Service is already uninstalled; private state was preserved.'); return; }
|
|
194
|
+
throw new Error('Service is not installed');
|
|
195
|
+
}
|
|
196
|
+
const record = json(recordPath(id)); owned(record);
|
|
197
|
+
if (action === 'status') { console.log(JSON.stringify(await serviceStatus(record), null, 2)); return; }
|
|
198
|
+
if (action === 'logs') {
|
|
199
|
+
console.log(record.platform === 'linux' ? run('journalctl', ['--user', '-u', record.unit, '-n', '100', '--no-pager']) : (existsSync(record.log) ? readFileSync(record.log, 'utf8').split('\n').slice(-100).join('\n') : 'No service log yet.')); return;
|
|
200
|
+
}
|
|
201
|
+
if (['stop', 'restart', 'uninstall'].includes(action)) stop(record);
|
|
202
|
+
if (['start', 'restart'].includes(action)) { start(record); await waitHealthy(record); }
|
|
203
|
+
if (action === 'uninstall') {
|
|
204
|
+
if (record.platform === 'linux') systemctl('disable', record.unit);
|
|
205
|
+
rmSync(record.file); rmSync(recordPath(id));
|
|
206
|
+
if (record.platform === 'linux') systemctl('daemon-reload');
|
|
207
|
+
}
|
|
208
|
+
console.log(`${action}: ${id}. Private state and retained runtimes were preserved.`);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function controllers() {
|
|
212
|
+
if (!existsSync(records)) return [];
|
|
213
|
+
return readdirSync(records).filter(name => name.endsWith('.json')).map(name => json(join(records, name))).filter(record => record.kind === 'controller');
|
|
214
|
+
}
|
|
215
|
+
export async function updateServices({ latest = latestVersion, download = installRelease } = {}) {
|
|
216
|
+
userOnly();
|
|
217
|
+
return withServiceOperation('update', () => performUpdate(latest, download));
|
|
218
|
+
}
|
|
219
|
+
async function performUpdate(latest, download) {
|
|
220
|
+
const held = [], stopped = [];
|
|
221
|
+
const preferencesPath = join(STATE_HOME, 'updates.json'), selectionPath = join(STATE_HOME, 'service-release.json');
|
|
222
|
+
const previousPreferences = existsSync(preferencesPath) ? json(preferencesPath) : { enabled: true, lastCheckedAt: 0 };
|
|
223
|
+
const previous = existsSync(selectionPath) ? json(selectionPath) : {};
|
|
224
|
+
let activated = false, releaseMaintenance = true;
|
|
225
|
+
const priorVersions = new Map();
|
|
226
|
+
try {
|
|
227
|
+
const managed = controllers();
|
|
228
|
+
for (const record of managed) owned(record);
|
|
229
|
+
const live = managed.filter(active);
|
|
230
|
+
const installed = managed.map(record => record.version || VERSION).reduce((minimum, version) => newer(minimum, version) ? version : minimum, VERSION);
|
|
231
|
+
const current = previous.version || installed;
|
|
232
|
+
const version = [await latest(), VERSION, current].reduce((selected, candidate) => newer(candidate, selected) ? candidate : selected);
|
|
233
|
+
const runningVersions = await Promise.all(live.map(async record => {
|
|
234
|
+
const value = (await api(record.state, '/api/v1/status')).runtime_version;
|
|
235
|
+
priorVersions.set(record.id, value); return value;
|
|
236
|
+
}));
|
|
237
|
+
if (runningVersions.some(value => !/^\d+\.\d+\.\d+$/.test(value || ''))) throw new Error('Cannot verify the runtime version of a managed controller');
|
|
238
|
+
if (!newer(version, current) && runningVersions.every(value => !newer(version, value))) { console.log(`software-defence-factory ${current} is up to date.`); return; }
|
|
239
|
+
const unowned = busyInstallations().filter(state => !live.some(record => record.state === state));
|
|
240
|
+
if (unowned.length) throw new Error('Unmanaged controllers or executor fences block the update: ' + unowned.join(', '));
|
|
241
|
+
// Reserve idle controllers before downloads. All mutating API actions are
|
|
242
|
+
// refused until restart or explicit release, eliminating submit/stop races.
|
|
243
|
+
for (const record of live) { await api(record.state, '/api/v1/maintenance', { enabled: true }); held.push(record); }
|
|
244
|
+
download(version);
|
|
245
|
+
for (const record of live) { stop(record); stopped.push(record); }
|
|
246
|
+
if (busyInstallations().length) throw new Error('An installation or executor still blocks activation');
|
|
247
|
+
// Ordinary CLI downloads must never change what an existing service starts.
|
|
248
|
+
// Only this reserved update transaction may advance the managed selection.
|
|
249
|
+
activated = true;
|
|
250
|
+
save(selectionPath, { version });
|
|
251
|
+
save(preferencesPath, { ...previousPreferences, version, lastCheckedAt: Date.now() });
|
|
252
|
+
for (const record of stopped) { start(record); await waitHealthy(record, version); }
|
|
253
|
+
console.log(`Updated to ${version}; restarted ${stopped.length} previously active services. Jobs and history preserved.`);
|
|
254
|
+
} catch (error) {
|
|
255
|
+
releaseMaintenance = false;
|
|
256
|
+
try {
|
|
257
|
+
if (activated) {
|
|
258
|
+
for (const record of stopped) stop(record);
|
|
259
|
+
save(selectionPath, previous);
|
|
260
|
+
save(preferencesPath, previousPreferences);
|
|
261
|
+
}
|
|
262
|
+
for (const record of stopped) { start(record); await waitHealthy(record, priorVersions.get(record.id)); }
|
|
263
|
+
releaseMaintenance = true;
|
|
264
|
+
} catch (recoveryError) {
|
|
265
|
+
throw new Error(`Update failed: ${error.message}. Recovery unconfirmed: ${recoveryError.message}. Maintenance reservations were retained.`);
|
|
266
|
+
}
|
|
267
|
+
throw error;
|
|
268
|
+
} finally {
|
|
269
|
+
const failures = [];
|
|
270
|
+
if (releaseMaintenance) for (const record of held) try { await api(record.state, '/api/v1/maintenance', { enabled: false }); } catch (error) { failures.push(`${record.id}: ${error.message}`); }
|
|
271
|
+
if (failures.length) throw new Error('Maintenance release unconfirmed: ' + failures.join('; '));
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
function automaticUpdates(mode) {
|
|
275
|
+
if (process.platform !== 'linux') throw new Error('Scheduled service updates currently require Linux/systemd');
|
|
276
|
+
if (!['on', 'off', 'status'].includes(mode)) throw new Error('Use service updates --auto on|off|status');
|
|
277
|
+
const timer = join(userUnits, `${updater}.timer`), service = join(userUnits, `${updater}.service`), record = join(records, 'updates.json');
|
|
278
|
+
if (mode === 'status') { console.log(JSON.stringify({ installed: existsSync(record), ...(existsSync(record) ? { timer: systemctl('show', `${updater}.timer`, '-p', 'ActiveState', '-p', 'NextElapseUSecRealtime') } : {}) })); return; }
|
|
279
|
+
if (mode === 'off') {
|
|
280
|
+
if (existsSync(record)) {
|
|
281
|
+
if (systemctl('show', `${updater}.service`, '-p', 'ActiveState', '--value') === 'activating') throw new Error('An update is running; wait for it to finish before disabling the timer');
|
|
282
|
+
const data = json(record);
|
|
283
|
+
for (const [file, hash] of [[timer, data.timerHash], [service, data.serviceHash]]) if (!existsSync(file) || digest(readFileSync(file, 'utf8')) !== hash) throw new Error('Update service was edited; reconcile it before removal');
|
|
284
|
+
systemctl('disable', '--now', `${updater}.timer`);
|
|
285
|
+
systemctl('stop', `${updater}.service`);
|
|
286
|
+
rmSync(timer); rmSync(service); rmSync(record); systemctl('daemon-reload');
|
|
287
|
+
}
|
|
288
|
+
console.log('Scheduled updates disabled.'); return;
|
|
289
|
+
}
|
|
290
|
+
if (existsSync(record)) { console.log('Scheduled updates are already installed.'); return; }
|
|
291
|
+
if (existsSync(timer) || existsSync(service)) throw new Error('Refusing to overwrite an unregistered update unit');
|
|
292
|
+
const serviceText = systemdUnit({ description: 'Software & Defence Factory idle update', argv: [process.execPath, launcher(), 'service', 'update'], directory: homedir(), environment: environment(), oneshot: true });
|
|
293
|
+
const timerText = `[Unit]\nDescription=Check Factory updates daily when idle\n\n[Timer]\nOnCalendar=daily\nRandomizedDelaySec=1h\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n`;
|
|
294
|
+
mkdirSync(userUnits, { recursive: true, mode: 0o700 });
|
|
295
|
+
writeFileSync(service, serviceText, { mode: 0o600 }); writeFileSync(timer, timerText, { mode: 0o600 });
|
|
296
|
+
save(record, { serviceHash: digest(serviceText), timerHash: digest(timerText) });
|
|
297
|
+
systemctl('daemon-reload'); systemctl('enable', '--now', `${updater}.timer`); console.log('Daily idle service updates enabled. Busy/unknown installations defer the update.');
|
|
298
|
+
}
|
package/factory/supervisor.mjs
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { writeFileSync, rmSync } from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
+
import { pathToFileURL } from 'node:url';
|
|
3
4
|
import { configAt, stopContainers } from './lib.mjs';
|
|
4
5
|
import { createController } from './server.mjs';
|
|
5
|
-
|
|
6
|
+
export async function supervise(state) {
|
|
7
|
+
const config = configAt(state), lock = join(state, 'supervisor.json');
|
|
6
8
|
writeFileSync(lock, JSON.stringify({ pid: process.pid }), { flag: 'wx', mode: 0o600 });
|
|
7
9
|
let controller, stopping = false;
|
|
8
10
|
async function shutdown(code = 0) {
|
|
@@ -18,3 +20,5 @@ try {
|
|
|
18
20
|
controller.server.on('error', error => { console.error(error.message); shutdown(1); });
|
|
19
21
|
controller.server.listen(config.port, '127.0.0.1', () => console.log(`Factory controller ready on 127.0.0.1:${config.port}`));
|
|
20
22
|
} catch (error) { console.error(error.message); await shutdown(1); }
|
|
23
|
+
}
|
|
24
|
+
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) await supervise(process.argv[2]);
|
package/factory/updates.mjs
CHANGED
|
@@ -104,7 +104,7 @@ export async function bootstrap(args) {
|
|
|
104
104
|
const busy = busyInstallations();
|
|
105
105
|
const cachedVersion = installedEntry(DATA_HOME, preferences.version) ? preferences.version : VERSION;
|
|
106
106
|
let selectedVersion = newer(cachedVersion, VERSION) ? cachedVersion : VERSION;
|
|
107
|
-
const automatic = !['stop', 'cancel', 'status', 'serve', 'version', '--version', '-v'].includes(command)
|
|
107
|
+
const automatic = !['stop', 'cancel', 'status', 'serve', 'service', 'tunnel', 'help', '--help', '-h', 'version', '--version', '-v'].includes(command)
|
|
108
108
|
&& preferences.enabled && process.env.SDF_AUTO_UPDATE !== '0'
|
|
109
109
|
&& Date.now() - preferences.lastCheckedAt >= DAY && busy.length === 0;
|
|
110
110
|
if (explicit || automatic) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "software-defence-factory",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "Scoped software delivery and defence investigations with isolated jobs, evidence and review",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -29,6 +29,8 @@
|
|
|
29
29
|
"scripts/export-kit.mjs",
|
|
30
30
|
"scripts/probe-platform.mjs",
|
|
31
31
|
"docs/npm.md",
|
|
32
|
+
"docs/services.md",
|
|
33
|
+
"docs/recovery.md",
|
|
32
34
|
"docs/quickstart.md",
|
|
33
35
|
"docs/defence-integration.md",
|
|
34
36
|
"docs/ownership.md",
|
|
@@ -68,7 +68,7 @@ try {
|
|
|
68
68
|
assert(!existsSync(join(state,'jobs',changed.id,'accepted.json')));record('changed candidate cannot inherit earlier approval');
|
|
69
69
|
|
|
70
70
|
const policy=await work('Synthetic policy change guard');await waitState(policy.id,'awaiting_approval');
|
|
71
|
-
setConfig({check:'true'});await cli('approve',policy.id);await waitState(policy.id,'failed');
|
|
71
|
+
setConfig({check:original.check==='true'?':':'true'});await cli('approve',policy.id);await waitState(policy.id,'failed');
|
|
72
72
|
assert(!existsSync(join(state,'jobs',policy.id,'accepted.json')));setConfig({});record('changed check policy cannot inherit earlier approval');
|
|
73
73
|
|
|
74
74
|
setConfig({check:'exit 17'});
|