velocious 1.0.725 → 1.0.727
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/build/background-jobs/adapter.js +16 -0
- package/build/background-jobs/job-record.js +3 -1
- package/build/background-jobs/main.js +62 -2
- package/build/background-jobs/owner-liveness.js +27 -5
- package/build/background-jobs/store.js +74 -0
- package/build/background-jobs/types.js +3 -0
- package/build/configuration-types.js +1 -1
- package/build/configuration.js +53 -3
- package/build/database/advisory-lock-lock-server-backend.js +30 -24
- package/build/database/advisory-lock-lock-server-embedded-client.js +74 -0
- package/build/database/advisory-lock-lock-server-engine.js +544 -0
- package/build/database/advisory-lock-lock-server-local-client.js +260 -0
- package/build/database/advisory-lock-lock-server-socket-path.js +17 -0
- package/build/database/advisory-lock-lock-server-transport.js +30 -0
- package/build/database/record/auditing.js +89 -11
- package/build/database/record/index.js +1 -1
- package/build/environment-handlers/base.js +50 -9
- package/build/environment-handlers/node/advisory-lock-lock-server-local-server.js +206 -0
- package/build/environment-handlers/node.js +30 -3
- package/build/http-server/server-lock.js +29 -16
- package/build/src/background-jobs/adapter.d.ts +23 -0
- package/build/src/background-jobs/adapter.d.ts.map +1 -1
- package/build/src/background-jobs/adapter.js +15 -1
- package/build/src/background-jobs/job-record.d.ts +3 -1
- package/build/src/background-jobs/job-record.d.ts.map +1 -1
- package/build/src/background-jobs/job-record.js +4 -2
- package/build/src/background-jobs/main.d.ts +33 -0
- package/build/src/background-jobs/main.d.ts.map +1 -1
- package/build/src/background-jobs/main.js +59 -3
- package/build/src/background-jobs/owner-liveness.d.ts +25 -3
- package/build/src/background-jobs/owner-liveness.d.ts.map +1 -1
- package/build/src/background-jobs/owner-liveness.js +26 -7
- package/build/src/background-jobs/store.d.ts +39 -0
- package/build/src/background-jobs/store.d.ts.map +1 -1
- package/build/src/background-jobs/store.js +66 -1
- package/build/src/background-jobs/types.d.ts +6 -0
- package/build/src/background-jobs/types.d.ts.map +1 -1
- package/build/src/background-jobs/types.js +4 -1
- package/build/src/configuration-types.d.ts +3 -1
- package/build/src/configuration-types.d.ts.map +1 -1
- package/build/src/configuration-types.js +2 -2
- package/build/src/configuration.d.ts +25 -1
- package/build/src/configuration.d.ts.map +1 -1
- package/build/src/configuration.js +47 -4
- package/build/src/database/advisory-lock-lock-server-backend.d.ts +10 -3
- package/build/src/database/advisory-lock-lock-server-backend.d.ts.map +1 -1
- package/build/src/database/advisory-lock-lock-server-backend.js +25 -22
- package/build/src/database/advisory-lock-lock-server-embedded-client.d.ts +24 -0
- package/build/src/database/advisory-lock-lock-server-embedded-client.d.ts.map +1 -0
- package/build/src/database/advisory-lock-lock-server-embedded-client.js +64 -0
- package/build/src/database/advisory-lock-lock-server-engine.d.ts +285 -0
- package/build/src/database/advisory-lock-lock-server-engine.d.ts.map +1 -0
- package/build/src/database/advisory-lock-lock-server-engine.js +460 -0
- package/build/src/database/advisory-lock-lock-server-local-client.d.ts +114 -0
- package/build/src/database/advisory-lock-lock-server-local-client.d.ts.map +1 -0
- package/build/src/database/advisory-lock-lock-server-local-client.js +227 -0
- package/build/src/database/advisory-lock-lock-server-socket-path.d.ts +9 -0
- package/build/src/database/advisory-lock-lock-server-socket-path.d.ts.map +1 -0
- package/build/src/database/advisory-lock-lock-server-socket-path.js +16 -0
- package/build/src/database/advisory-lock-lock-server-transport.d.ts +18 -0
- package/build/src/database/advisory-lock-lock-server-transport.d.ts.map +1 -0
- package/build/src/database/advisory-lock-lock-server-transport.js +27 -0
- package/build/src/database/record/auditing.d.ts +14 -4
- package/build/src/database/record/auditing.d.ts.map +1 -1
- package/build/src/database/record/auditing.js +77 -12
- package/build/src/database/record/index.d.ts +4 -2
- package/build/src/database/record/index.d.ts.map +1 -1
- package/build/src/database/record/index.js +2 -2
- package/build/src/environment-handlers/base.d.ts +86 -7
- package/build/src/environment-handlers/base.d.ts.map +1 -1
- package/build/src/environment-handlers/base.js +48 -9
- package/build/src/environment-handlers/node/advisory-lock-lock-server-local-server.d.ts +2 -0
- package/build/src/environment-handlers/node/advisory-lock-lock-server-local-server.d.ts.map +1 -0
- package/build/src/environment-handlers/node/advisory-lock-lock-server-local-server.js +179 -0
- package/build/src/environment-handlers/node.d.ts +10 -4
- package/build/src/environment-handlers/node.d.ts.map +1 -1
- package/build/src/environment-handlers/node.js +30 -4
- package/build/src/http-server/server-lock.d.ts +22 -11
- package/build/src/http-server/server-lock.d.ts.map +1 -1
- package/build/src/http-server/server-lock.js +30 -17
- package/package.json +1 -1
- package/src/background-jobs/adapter.js +16 -0
- package/src/background-jobs/job-record.js +3 -1
- package/src/background-jobs/main.js +62 -2
- package/src/background-jobs/owner-liveness.js +27 -5
- package/src/background-jobs/store.js +74 -0
- package/src/background-jobs/types.js +3 -0
- package/src/configuration-types.js +1 -1
- package/src/configuration.js +53 -3
- package/src/database/advisory-lock-lock-server-backend.js +30 -24
- package/src/database/advisory-lock-lock-server-embedded-client.js +74 -0
- package/src/database/advisory-lock-lock-server-engine.js +544 -0
- package/src/database/advisory-lock-lock-server-local-client.js +260 -0
- package/src/database/advisory-lock-lock-server-socket-path.js +17 -0
- package/src/database/advisory-lock-lock-server-transport.js +30 -0
- package/src/database/record/auditing.js +89 -11
- package/src/database/record/index.js +1 -1
- package/src/environment-handlers/base.js +50 -9
- package/src/environment-handlers/node/advisory-lock-lock-server-local-server.js +206 -0
- package/src/environment-handlers/node.js +30 -3
- package/src/http-server/server-lock.js +29 -16
|
@@ -1,21 +1,31 @@
|
|
|
1
1
|
import AdvisoryLockFileBackend from "../database/advisory-lock-file-backend.js";
|
|
2
2
|
/**
|
|
3
|
-
* Directory-based lock that allows only one HTTP server process
|
|
4
|
-
* directory. Thin wrapper over the shared
|
|
5
|
-
* (`src/database/advisory-lock-file-backend.js` — the
|
|
6
|
-
* primitive generalized from this lock's original
|
|
7
|
-
* protocol), so the server lock and the advisory-lock
|
|
8
|
-
* acquisition, owner-metadata, lease-renewal, and
|
|
3
|
+
* Directory + environment-based lock that allows only one HTTP server process
|
|
4
|
+
* per (app directory, environment) pair. Thin wrapper over the shared
|
|
5
|
+
* file-lock backend (`src/database/advisory-lock-file-backend.js` — the
|
|
6
|
+
* STC-210 `file` tier primitive generalized from this lock's original
|
|
7
|
+
* hand-rolled directory protocol), so the server lock and the advisory-lock
|
|
8
|
+
* file tier share one acquisition, owner-metadata, lease-renewal, and
|
|
9
|
+
* staleness implementation.
|
|
10
|
+
*
|
|
11
|
+
* The lock name is scoped by the configuration's environment
|
|
12
|
+
* (`server.<environment>` → `tmp/server.<environment>.lock`) so a development
|
|
13
|
+
* server and a test-spawned backend in the same app directory coexist, while
|
|
14
|
+
* the single-server-per-(directory, environment) invariant still fails loudly
|
|
15
|
+
* for a duplicate within one environment (STC-697).
|
|
9
16
|
*/
|
|
10
17
|
export default class VelociousHttpServerLock {
|
|
11
18
|
configuration: import("../configuration.js").default;
|
|
12
19
|
host: string;
|
|
13
20
|
port: number;
|
|
21
|
+
environment: string;
|
|
22
|
+
lockName: string;
|
|
14
23
|
lockPath: string;
|
|
15
24
|
backend: AdvisoryLockFileBackend;
|
|
16
25
|
acquired: boolean;
|
|
17
26
|
/**
|
|
18
|
-
* Build a lock for the configured application directory and
|
|
27
|
+
* Build a lock for the configured application directory, environment and
|
|
28
|
+
* server endpoint.
|
|
19
29
|
* @param {object} args - Options object.
|
|
20
30
|
* @param {import("../configuration.js").default} args.configuration - Configuration that owns the application directory.
|
|
21
31
|
* @param {string} args.host - Configured HTTP host.
|
|
@@ -27,14 +37,15 @@ export default class VelociousHttpServerLock {
|
|
|
27
37
|
port: number;
|
|
28
38
|
});
|
|
29
39
|
/**
|
|
30
|
-
* Acquires the app-directory HTTP server lock before startup
|
|
31
|
-
* run. A stale lock (dead local owner or expired lease) is
|
|
32
|
-
* backend; a live foreign owner
|
|
40
|
+
* Acquires the app-directory + environment HTTP server lock before startup
|
|
41
|
+
* side effects run. A stale lock (dead local owner or expired lease) is
|
|
42
|
+
* reclaimed by the backend; a live foreign owner in the same environment
|
|
43
|
+
* fails loudly with the owner details.
|
|
33
44
|
* @returns {Promise<void>} - Resolves after the lock has been acquired.
|
|
34
45
|
*/
|
|
35
46
|
acquire(): Promise<void>;
|
|
36
47
|
/**
|
|
37
|
-
* Releases the held HTTP server lock directory.
|
|
48
|
+
* Releases the held environment-scoped HTTP server lock directory.
|
|
38
49
|
* @returns {Promise<void>} - Resolves after best-effort lock release.
|
|
39
50
|
*/
|
|
40
51
|
release(): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server-lock.d.ts","sourceRoot":"","sources":["../../../src/http-server/server-lock.js"],"names":[],"mappings":"AAEA,OAAO,uBAAuB,MAAM,2CAA2C,CAAA;AAG/E
|
|
1
|
+
{"version":3,"file":"server-lock.d.ts","sourceRoot":"","sources":["../../../src/http-server/server-lock.js"],"names":[],"mappings":"AAEA,OAAO,uBAAuB,MAAM,2CAA2C,CAAA;AAG/E;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,OAAO,OAAO,uBAAuB;IAUnC,aAAa;IACb,IAAI;IACJ,IAAI;IACJ,WAAW;IAGX,QAAQ;IACR,QAAQ;IACR,OAAO;IAKP,QAAQ;IAtBf;;;;;;;OAOG;IACH,YAAY,EAAC,aAAa,EAAE,IAAI,EAAE,IAAI,EAAC,EAJpC;QAAoD,aAAa,EAAzD,OAAO,qBAAqB,EAAE,OAAO,CAC7C;QAAqB,IAAI,EAAjB,MAAM,CACd;QAAqB,IAAI,EAAjB,MAAM,CAChB;KACsC,EAetC;IAED;;;;;;OAMG;IACG,OAAO,IAFA,OAAO,CAAC,IAAI,CAAC,CAQzB;IAED;;;OAGG;IACG,OAAO,IAFA,OAAO,CAAC,IAAI,CAAC,CAOzB;IAED;;;OAGG;IACG,iBAAiB,IAFV,OAAO,CAAC,OAAO,2CAA2C,EAAE,qBAAqB,GAAG,IAAI,CAAC,CAIrG;IAED;;;OAGG;IACG,eAAe,IAFR,OAAO,CAAC,MAAM,CAAC,CAS3B;CACF"}
|
|
@@ -2,16 +2,24 @@
|
|
|
2
2
|
import AdvisoryLockFileBackend from "../database/advisory-lock-file-backend.js";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
/**
|
|
5
|
-
* Directory-based lock that allows only one HTTP server process
|
|
6
|
-
* directory. Thin wrapper over the shared
|
|
7
|
-
* (`src/database/advisory-lock-file-backend.js` — the
|
|
8
|
-
* primitive generalized from this lock's original
|
|
9
|
-
* protocol), so the server lock and the advisory-lock
|
|
10
|
-
* acquisition, owner-metadata, lease-renewal, and
|
|
5
|
+
* Directory + environment-based lock that allows only one HTTP server process
|
|
6
|
+
* per (app directory, environment) pair. Thin wrapper over the shared
|
|
7
|
+
* file-lock backend (`src/database/advisory-lock-file-backend.js` — the
|
|
8
|
+
* STC-210 `file` tier primitive generalized from this lock's original
|
|
9
|
+
* hand-rolled directory protocol), so the server lock and the advisory-lock
|
|
10
|
+
* file tier share one acquisition, owner-metadata, lease-renewal, and
|
|
11
|
+
* staleness implementation.
|
|
12
|
+
*
|
|
13
|
+
* The lock name is scoped by the configuration's environment
|
|
14
|
+
* (`server.<environment>` → `tmp/server.<environment>.lock`) so a development
|
|
15
|
+
* server and a test-spawned backend in the same app directory coexist, while
|
|
16
|
+
* the single-server-per-(directory, environment) invariant still fails loudly
|
|
17
|
+
* for a duplicate within one environment (STC-697).
|
|
11
18
|
*/
|
|
12
19
|
export default class VelociousHttpServerLock {
|
|
13
20
|
/**
|
|
14
|
-
* Build a lock for the configured application directory and
|
|
21
|
+
* Build a lock for the configured application directory, environment and
|
|
22
|
+
* server endpoint.
|
|
15
23
|
* @param {object} args - Options object.
|
|
16
24
|
* @param {import("../configuration.js").default} args.configuration - Configuration that owns the application directory.
|
|
17
25
|
* @param {string} args.host - Configured HTTP host.
|
|
@@ -21,7 +29,11 @@ export default class VelociousHttpServerLock {
|
|
|
21
29
|
this.configuration = configuration;
|
|
22
30
|
this.host = host;
|
|
23
31
|
this.port = port;
|
|
24
|
-
this.
|
|
32
|
+
this.environment = String(configuration.getEnvironment());
|
|
33
|
+
// Scope the lock name by environment and keep the segment filesystem-safe,
|
|
34
|
+
// so odd environment names can never escape tmp/ or collide with each other.
|
|
35
|
+
this.lockName = `server.${this.environment.replace(/[^a-zA-Z0-9._-]/g, "_")}`;
|
|
36
|
+
this.lockPath = path.join(configuration.getDirectory(), "tmp", `${this.lockName}.lock`);
|
|
25
37
|
this.backend = new AdvisoryLockFileBackend({
|
|
26
38
|
directory: path.join(configuration.getDirectory(), "tmp"),
|
|
27
39
|
extraOwnerFields: { host: this.host, port: this.port },
|
|
@@ -30,33 +42,34 @@ export default class VelociousHttpServerLock {
|
|
|
30
42
|
this.acquired = false;
|
|
31
43
|
}
|
|
32
44
|
/**
|
|
33
|
-
* Acquires the app-directory HTTP server lock before startup
|
|
34
|
-
* run. A stale lock (dead local owner or expired lease) is
|
|
35
|
-
* backend; a live foreign owner
|
|
45
|
+
* Acquires the app-directory + environment HTTP server lock before startup
|
|
46
|
+
* side effects run. A stale lock (dead local owner or expired lease) is
|
|
47
|
+
* reclaimed by the backend; a live foreign owner in the same environment
|
|
48
|
+
* fails loudly with the owner details.
|
|
36
49
|
* @returns {Promise<void>} - Resolves after the lock has been acquired.
|
|
37
50
|
*/
|
|
38
51
|
async acquire() {
|
|
39
|
-
const acquired = await this.backend.acquire(
|
|
52
|
+
const acquired = await this.backend.acquire(this.lockName, { timeoutMs: 0 });
|
|
40
53
|
if (!acquired)
|
|
41
54
|
throw new Error(await this.lockHeldMessage());
|
|
42
55
|
this.acquired = true;
|
|
43
56
|
}
|
|
44
57
|
/**
|
|
45
|
-
* Releases the held HTTP server lock directory.
|
|
58
|
+
* Releases the held environment-scoped HTTP server lock directory.
|
|
46
59
|
* @returns {Promise<void>} - Resolves after best-effort lock release.
|
|
47
60
|
*/
|
|
48
61
|
async release() {
|
|
49
62
|
if (!this.acquired)
|
|
50
63
|
return;
|
|
51
64
|
this.acquired = false;
|
|
52
|
-
await this.backend.release(
|
|
65
|
+
await this.backend.release(this.lockName);
|
|
53
66
|
}
|
|
54
67
|
/**
|
|
55
68
|
* Reads owner metadata from an existing server lock directory.
|
|
56
69
|
* @returns {Promise<import("../database/advisory-lock-file-backend.js").AdvisoryLockFileOwner | null>} - Parsed owner metadata, when readable.
|
|
57
70
|
*/
|
|
58
71
|
async readOwnerMetadata() {
|
|
59
|
-
return await this.backend.holderInfo(
|
|
72
|
+
return await this.backend.holderInfo(this.lockName);
|
|
60
73
|
}
|
|
61
74
|
/**
|
|
62
75
|
* Builds a duplicate-server error message with owner details when available.
|
|
@@ -67,7 +80,7 @@ export default class VelociousHttpServerLock {
|
|
|
67
80
|
const details = owner
|
|
68
81
|
? `PID ${String(owner.pid)} on ${String(owner.hostname)} (${String(owner.host)}:${String(owner.port)})`
|
|
69
82
|
: `lock directory ${this.lockPath}`;
|
|
70
|
-
return `A Velocious HTTP server is already running for this application (${details}). Remove ${this.lockPath} if the server is no longer running.`;
|
|
83
|
+
return `A Velocious HTTP server is already running for this application in the ${this.environment} environment (${details}). Remove ${this.lockPath} if the server is no longer running.`;
|
|
71
84
|
}
|
|
72
85
|
}
|
|
73
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
86
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2VydmVyLWxvY2suanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvaHR0cC1zZXJ2ZXIvc2VydmVyLWxvY2suanMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsWUFBWTtBQUVaLE9BQU8sdUJBQXVCLE1BQU0sMkNBQTJDLENBQUE7QUFDL0UsT0FBTyxJQUFJLE1BQU0sV0FBVyxDQUFBO0FBRTVCOzs7Ozs7Ozs7Ozs7OztHQWNHO0FBQ0gsTUFBTSxDQUFDLE9BQU8sT0FBTyx1QkFBdUI7SUFDMUM7Ozs7Ozs7T0FPRztJQUNILFlBQVksRUFBQyxhQUFhLEVBQUUsSUFBSSxFQUFFLElBQUksRUFBQztRQUNyQyxJQUFJLENBQUMsYUFBYSxHQUFHLGFBQWEsQ0FBQTtRQUNsQyxJQUFJLENBQUMsSUFBSSxHQUFHLElBQUksQ0FBQTtRQUNoQixJQUFJLENBQUMsSUFBSSxHQUFHLElBQUksQ0FBQTtRQUNoQixJQUFJLENBQUMsV0FBVyxHQUFHLE1BQU0sQ0FBQyxhQUFhLENBQUMsY0FBYyxFQUFFLENBQUMsQ0FBQTtRQUN6RCwyRUFBMkU7UUFDM0UsNkVBQTZFO1FBQzdFLElBQUksQ0FBQyxRQUFRLEdBQUcsVUFBVSxJQUFJLENBQUMsV0FBVyxDQUFDLE9BQU8sQ0FBQyxrQkFBa0IsRUFBRSxHQUFHLENBQUMsRUFBRSxDQUFBO1FBQzdFLElBQUksQ0FBQyxRQUFRLEdBQUcsSUFBSSxDQUFDLElBQUksQ0FBQyxhQUFhLENBQUMsWUFBWSxFQUFFLEVBQUUsS0FBSyxFQUFFLEdBQUcsSUFBSSxDQUFDLFFBQVEsT0FBTyxDQUFDLENBQUE7UUFDdkYsSUFBSSxDQUFDLE9BQU8sR0FBRyxJQUFJLHVCQUF1QixDQUFDO1lBQ3pDLFNBQVMsRUFBRSxJQUFJLENBQUMsSUFBSSxDQUFDLGFBQWEsQ0FBQyxZQUFZLEVBQUUsRUFBRSxLQUFLLENBQUM7WUFDekQsZ0JBQWdCLEVBQUUsRUFBQyxJQUFJLEVBQUUsSUFBSSxDQUFDLElBQUksRUFBRSxJQUFJLEVBQUUsSUFBSSxDQUFDLElBQUksRUFBQztZQUNwRCxTQUFTLEVBQUUsT0FBTztTQUNuQixDQUFDLENBQUE7UUFDRixJQUFJLENBQUMsUUFBUSxHQUFHLEtBQUssQ0FBQTtJQUN2QixDQUFDO0lBRUQ7Ozs7OztPQU1HO0lBQ0gsS0FBSyxDQUFDLE9BQU87UUFDWCxNQUFNLFFBQVEsR0FBRyxNQUFNLElBQUksQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxRQUFRLEVBQUUsRUFBQyxTQUFTLEVBQUUsQ0FBQyxFQUFDLENBQUMsQ0FBQTtRQUUxRSxJQUFJLENBQUMsUUFBUTtZQUFFLE1BQU0sSUFBSSxLQUFLLENBQUMsTUFBTSxJQUFJLENBQUMsZUFBZSxFQUFFLENBQUMsQ0FBQTtRQUU1RCxJQUFJLENBQUMsUUFBUSxHQUFHLElBQUksQ0FBQTtJQUN0QixDQUFDO0lBRUQ7OztPQUdHO0lBQ0gsS0FBSyxDQUFDLE9BQU87UUFDWCxJQUFJLENBQUMsSUFBSSxDQUFDLFFBQVE7WUFBRSxPQUFNO1FBRTFCLElBQUksQ0FBQyxRQUFRLEdBQUcsS0FBSyxDQUFBO1FBQ3JCLE1BQU0sSUFBSSxDQUFDLE9BQU8sQ0FBQyxPQUFPLENBQUMsSUFBSSxDQUFDLFFBQVEsQ0FBQyxDQUFBO0lBQzNDLENBQUM7SUFFRDs7O09BR0c7SUFDSCxLQUFLLENBQUMsaUJBQWlCO1FBQ3JCLE9BQU8sTUFBTSxJQUFJLENBQUMsT0FBTyxDQUFDLFVBQVUsQ0FBQyxJQUFJLENBQUMsUUFBUSxDQUFDLENBQUE7SUFDckQsQ0FBQztJQUVEOzs7T0FHRztJQUNILEtBQUssQ0FBQyxlQUFlO1FBQ25CLE1BQU0sS0FBSyxHQUFHLE1BQU0sSUFBSSxDQUFDLGlCQUFpQixFQUFFLENBQUE7UUFDNUMsTUFBTSxPQUFPLEdBQUcsS0FBSztZQUNuQixDQUFDLENBQUMsT0FBTyxNQUFNLENBQUMsS0FBSyxDQUFDLEdBQUcsQ0FBQyxPQUFPLE1BQU0sQ0FBQyxLQUFLLENBQUMsUUFBUSxDQUFDLEtBQUssTUFBTSxDQUFDLEtBQUssQ0FBQyxJQUFJLENBQUMsSUFBSSxNQUFNLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxHQUFHO1lBQ3ZHLENBQUMsQ0FBQyxrQkFBa0IsSUFBSSxDQUFDLFFBQVEsRUFBRSxDQUFBO1FBRXJDLE9BQU8sMEVBQTBFLElBQUksQ0FBQyxXQUFXLGlCQUFpQixPQUFPLGFBQWEsSUFBSSxDQUFDLFFBQVEsc0NBQXNDLENBQUE7SUFDM0wsQ0FBQztDQUNGIiwic291cmNlc0NvbnRlbnQiOlsiLy8gQHRzLWNoZWNrXG5cbmltcG9ydCBBZHZpc29yeUxvY2tGaWxlQmFja2VuZCBmcm9tIFwiLi4vZGF0YWJhc2UvYWR2aXNvcnktbG9jay1maWxlLWJhY2tlbmQuanNcIlxuaW1wb3J0IHBhdGggZnJvbSBcIm5vZGU6cGF0aFwiXG5cbi8qKlxuICogRGlyZWN0b3J5ICsgZW52aXJvbm1lbnQtYmFzZWQgbG9jayB0aGF0IGFsbG93cyBvbmx5IG9uZSBIVFRQIHNlcnZlciBwcm9jZXNzXG4gKiBwZXIgKGFwcCBkaXJlY3RvcnksIGVudmlyb25tZW50KSBwYWlyLiBUaGluIHdyYXBwZXIgb3ZlciB0aGUgc2hhcmVkXG4gKiBmaWxlLWxvY2sgYmFja2VuZCAoYHNyYy9kYXRhYmFzZS9hZHZpc29yeS1sb2NrLWZpbGUtYmFja2VuZC5qc2Ag4oCUIHRoZVxuICogU1RDLTIxMCBgZmlsZWAgdGllciBwcmltaXRpdmUgZ2VuZXJhbGl6ZWQgZnJvbSB0aGlzIGxvY2sncyBvcmlnaW5hbFxuICogaGFuZC1yb2xsZWQgZGlyZWN0b3J5IHByb3RvY29sKSwgc28gdGhlIHNlcnZlciBsb2NrIGFuZCB0aGUgYWR2aXNvcnktbG9ja1xuICogZmlsZSB0aWVyIHNoYXJlIG9uZSBhY3F1aXNpdGlvbiwgb3duZXItbWV0YWRhdGEsIGxlYXNlLXJlbmV3YWwsIGFuZFxuICogc3RhbGVuZXNzIGltcGxlbWVudGF0aW9uLlxuICpcbiAqIFRoZSBsb2NrIG5hbWUgaXMgc2NvcGVkIGJ5IHRoZSBjb25maWd1cmF0aW9uJ3MgZW52aXJvbm1lbnRcbiAqIChgc2VydmVyLjxlbnZpcm9ubWVudD5gIOKGkiBgdG1wL3NlcnZlci48ZW52aXJvbm1lbnQ+LmxvY2tgKSBzbyBhIGRldmVsb3BtZW50XG4gKiBzZXJ2ZXIgYW5kIGEgdGVzdC1zcGF3bmVkIGJhY2tlbmQgaW4gdGhlIHNhbWUgYXBwIGRpcmVjdG9yeSBjb2V4aXN0LCB3aGlsZVxuICogdGhlIHNpbmdsZS1zZXJ2ZXItcGVyLShkaXJlY3RvcnksIGVudmlyb25tZW50KSBpbnZhcmlhbnQgc3RpbGwgZmFpbHMgbG91ZGx5XG4gKiBmb3IgYSBkdXBsaWNhdGUgd2l0aGluIG9uZSBlbnZpcm9ubWVudCAoU1RDLTY5NykuXG4gKi9cbmV4cG9ydCBkZWZhdWx0IGNsYXNzIFZlbG9jaW91c0h0dHBTZXJ2ZXJMb2NrIHtcbiAgLyoqXG4gICAqIEJ1aWxkIGEgbG9jayBmb3IgdGhlIGNvbmZpZ3VyZWQgYXBwbGljYXRpb24gZGlyZWN0b3J5LCBlbnZpcm9ubWVudCBhbmRcbiAgICogc2VydmVyIGVuZHBvaW50LlxuICAgKiBAcGFyYW0ge29iamVjdH0gYXJncyAtIE9wdGlvbnMgb2JqZWN0LlxuICAgKiBAcGFyYW0ge2ltcG9ydChcIi4uL2NvbmZpZ3VyYXRpb24uanNcIikuZGVmYXVsdH0gYXJncy5jb25maWd1cmF0aW9uIC0gQ29uZmlndXJhdGlvbiB0aGF0IG93bnMgdGhlIGFwcGxpY2F0aW9uIGRpcmVjdG9yeS5cbiAgICogQHBhcmFtIHtzdHJpbmd9IGFyZ3MuaG9zdCAtIENvbmZpZ3VyZWQgSFRUUCBob3N0LlxuICAgKiBAcGFyYW0ge251bWJlcn0gYXJncy5wb3J0IC0gQ29uZmlndXJlZCBIVFRQIHBvcnQuXG4gICAqL1xuICBjb25zdHJ1Y3Rvcih7Y29uZmlndXJhdGlvbiwgaG9zdCwgcG9ydH0pIHtcbiAgICB0aGlzLmNvbmZpZ3VyYXRpb24gPSBjb25maWd1cmF0aW9uXG4gICAgdGhpcy5ob3N0ID0gaG9zdFxuICAgIHRoaXMucG9ydCA9IHBvcnRcbiAgICB0aGlzLmVudmlyb25tZW50ID0gU3RyaW5nKGNvbmZpZ3VyYXRpb24uZ2V0RW52aXJvbm1lbnQoKSlcbiAgICAvLyBTY29wZSB0aGUgbG9jayBuYW1lIGJ5IGVudmlyb25tZW50IGFuZCBrZWVwIHRoZSBzZWdtZW50IGZpbGVzeXN0ZW0tc2FmZSxcbiAgICAvLyBzbyBvZGQgZW52aXJvbm1lbnQgbmFtZXMgY2FuIG5ldmVyIGVzY2FwZSB0bXAvIG9yIGNvbGxpZGUgd2l0aCBlYWNoIG90aGVyLlxuICAgIHRoaXMubG9ja05hbWUgPSBgc2VydmVyLiR7dGhpcy5lbnZpcm9ubWVudC5yZXBsYWNlKC9bXmEtekEtWjAtOS5fLV0vZywgXCJfXCIpfWBcbiAgICB0aGlzLmxvY2tQYXRoID0gcGF0aC5qb2luKGNvbmZpZ3VyYXRpb24uZ2V0RGlyZWN0b3J5KCksIFwidG1wXCIsIGAke3RoaXMubG9ja05hbWV9LmxvY2tgKVxuICAgIHRoaXMuYmFja2VuZCA9IG5ldyBBZHZpc29yeUxvY2tGaWxlQmFja2VuZCh7XG4gICAgICBkaXJlY3Rvcnk6IHBhdGguam9pbihjb25maWd1cmF0aW9uLmdldERpcmVjdG9yeSgpLCBcInRtcFwiKSxcbiAgICAgIGV4dHJhT3duZXJGaWVsZHM6IHtob3N0OiB0aGlzLmhvc3QsIHBvcnQ6IHRoaXMucG9ydH0sXG4gICAgICBwYXRoU3R5bGU6IFwicGxhaW5cIlxuICAgIH0pXG4gICAgdGhpcy5hY3F1aXJlZCA9IGZhbHNlXG4gIH1cblxuICAvKipcbiAgICogQWNxdWlyZXMgdGhlIGFwcC1kaXJlY3RvcnkgKyBlbnZpcm9ubWVudCBIVFRQIHNlcnZlciBsb2NrIGJlZm9yZSBzdGFydHVwXG4gICAqIHNpZGUgZWZmZWN0cyBydW4uIEEgc3RhbGUgbG9jayAoZGVhZCBsb2NhbCBvd25lciBvciBleHBpcmVkIGxlYXNlKSBpc1xuICAgKiByZWNsYWltZWQgYnkgdGhlIGJhY2tlbmQ7IGEgbGl2ZSBmb3JlaWduIG93bmVyIGluIHRoZSBzYW1lIGVudmlyb25tZW50XG4gICAqIGZhaWxzIGxvdWRseSB3aXRoIHRoZSBvd25lciBkZXRhaWxzLlxuICAgKiBAcmV0dXJucyB7UHJvbWlzZTx2b2lkPn0gLSBSZXNvbHZlcyBhZnRlciB0aGUgbG9jayBoYXMgYmVlbiBhY3F1aXJlZC5cbiAgICovXG4gIGFzeW5jIGFjcXVpcmUoKSB7XG4gICAgY29uc3QgYWNxdWlyZWQgPSBhd2FpdCB0aGlzLmJhY2tlbmQuYWNxdWlyZSh0aGlzLmxvY2tOYW1lLCB7dGltZW91dE1zOiAwfSlcblxuICAgIGlmICghYWNxdWlyZWQpIHRocm93IG5ldyBFcnJvcihhd2FpdCB0aGlzLmxvY2tIZWxkTWVzc2FnZSgpKVxuXG4gICAgdGhpcy5hY3F1aXJlZCA9IHRydWVcbiAgfVxuXG4gIC8qKlxuICAgKiBSZWxlYXNlcyB0aGUgaGVsZCBlbnZpcm9ubWVudC1zY29wZWQgSFRUUCBzZXJ2ZXIgbG9jayBkaXJlY3RvcnkuXG4gICAqIEByZXR1cm5zIHtQcm9taXNlPHZvaWQ+fSAtIFJlc29sdmVzIGFmdGVyIGJlc3QtZWZmb3J0IGxvY2sgcmVsZWFzZS5cbiAgICovXG4gIGFzeW5jIHJlbGVhc2UoKSB7XG4gICAgaWYgKCF0aGlzLmFjcXVpcmVkKSByZXR1cm5cblxuICAgIHRoaXMuYWNxdWlyZWQgPSBmYWxzZVxuICAgIGF3YWl0IHRoaXMuYmFja2VuZC5yZWxlYXNlKHRoaXMubG9ja05hbWUpXG4gIH1cblxuICAvKipcbiAgICogUmVhZHMgb3duZXIgbWV0YWRhdGEgZnJvbSBhbiBleGlzdGluZyBzZXJ2ZXIgbG9jayBkaXJlY3RvcnkuXG4gICAqIEByZXR1cm5zIHtQcm9taXNlPGltcG9ydChcIi4uL2RhdGFiYXNlL2Fkdmlzb3J5LWxvY2stZmlsZS1iYWNrZW5kLmpzXCIpLkFkdmlzb3J5TG9ja0ZpbGVPd25lciB8IG51bGw+fSAtIFBhcnNlZCBvd25lciBtZXRhZGF0YSwgd2hlbiByZWFkYWJsZS5cbiAgICovXG4gIGFzeW5jIHJlYWRPd25lck1ldGFkYXRhKCkge1xuICAgIHJldHVybiBhd2FpdCB0aGlzLmJhY2tlbmQuaG9sZGVySW5mbyh0aGlzLmxvY2tOYW1lKVxuICB9XG5cbiAgLyoqXG4gICAqIEJ1aWxkcyBhIGR1cGxpY2F0ZS1zZXJ2ZXIgZXJyb3IgbWVzc2FnZSB3aXRoIG93bmVyIGRldGFpbHMgd2hlbiBhdmFpbGFibGUuXG4gICAqIEByZXR1cm5zIHtQcm9taXNlPHN0cmluZz59IC0gRXJyb3IgbWVzc2FnZSBleHBsYWluaW5nIHdoaWNoIHNlcnZlciBvd25zIHRoZSBsb2NrLlxuICAgKi9cbiAgYXN5bmMgbG9ja0hlbGRNZXNzYWdlKCkge1xuICAgIGNvbnN0IG93bmVyID0gYXdhaXQgdGhpcy5yZWFkT3duZXJNZXRhZGF0YSgpXG4gICAgY29uc3QgZGV0YWlscyA9IG93bmVyXG4gICAgICA/IGBQSUQgJHtTdHJpbmcob3duZXIucGlkKX0gb24gJHtTdHJpbmcob3duZXIuaG9zdG5hbWUpfSAoJHtTdHJpbmcob3duZXIuaG9zdCl9OiR7U3RyaW5nKG93bmVyLnBvcnQpfSlgXG4gICAgICA6IGBsb2NrIGRpcmVjdG9yeSAke3RoaXMubG9ja1BhdGh9YFxuXG4gICAgcmV0dXJuIGBBIFZlbG9jaW91cyBIVFRQIHNlcnZlciBpcyBhbHJlYWR5IHJ1bm5pbmcgZm9yIHRoaXMgYXBwbGljYXRpb24gaW4gdGhlICR7dGhpcy5lbnZpcm9ubWVudH0gZW52aXJvbm1lbnQgKCR7ZGV0YWlsc30pLiBSZW1vdmUgJHt0aGlzLmxvY2tQYXRofSBpZiB0aGUgc2VydmVyIGlzIG5vIGxvbmdlciBydW5uaW5nLmBcbiAgfVxufVxuIl19
|
package/package.json
CHANGED
|
@@ -190,6 +190,22 @@ export default class BackgroundJobsAdapter {
|
|
|
190
190
|
*/
|
|
191
191
|
async markOrphanedHandoffs(_args) { return [] }
|
|
192
192
|
|
|
193
|
+
/**
|
|
194
|
+
* Evaluates recorded runner-process liveness for exact active handoffs.
|
|
195
|
+
* Adapters without durable runner identity may return none.
|
|
196
|
+
* @param {{hostname?: string, liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[], probe?: (pid: number) => boolean}} [_args] - Probe controls.
|
|
197
|
+
* @returns {Promise<import("./types.js").BackgroundJobHandoffOwnershipVerdict[]>} - One verdict per exact active handoff.
|
|
198
|
+
*/
|
|
199
|
+
async snapshotHandedOffJobOwnership(_args) { return [] }
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* Settles exact active handoffs whose recorded runner process is provably
|
|
203
|
+
* dead. Adapters without durable runner identity settle none.
|
|
204
|
+
* @param {{error?: ReturnType<typeof JSON.parse>, hostname?: string, liveChildren?: import("./owner-liveness.js").BackgroundJobLiveChildIdentity[], probe?: (pid: number) => boolean}} [_args] - Orphan reason and probe controls.
|
|
205
|
+
* @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted orphan transitions.
|
|
206
|
+
*/
|
|
207
|
+
async markOrphanedHandoffsByDeadOwner(_args) { return [] }
|
|
208
|
+
|
|
193
209
|
/**
|
|
194
210
|
* Marks a handed-off job failed or retryable.
|
|
195
211
|
* @param {{jobId: string, error: ReturnType<typeof JSON.parse>, handoffId?: string, workerId?: string, handedOffAtMs?: number}} _args - Failure report.
|
|
@@ -21,14 +21,16 @@ export default class BackgroundJobRecord extends DatabaseRecord {
|
|
|
21
21
|
* registry, which this helper deliberately does not invent.
|
|
22
22
|
* @param {object} [args] - Options.
|
|
23
23
|
* @param {string} [args.hostname] - Hostname to compare against, defaults to this process's hostname.
|
|
24
|
+
* @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities observed as live, for the pid-recycling check.
|
|
24
25
|
* @param {(pid: number) => boolean} [args.probe] - Process probe, injectable for tests.
|
|
25
26
|
* @returns {import("./owner-liveness.js").BackgroundJobOwnerLivenessResult} - Liveness verdict.
|
|
26
27
|
*/
|
|
27
|
-
ownerIsAlive({hostname, probe} = {}) {
|
|
28
|
+
ownerIsAlive({hostname, liveChildren, probe} = {}) {
|
|
28
29
|
return evaluateBackgroundJobOwnerLiveness({
|
|
29
30
|
childInstanceId: /** @type {string | null} */ (this.readAttribute("childInstanceId")),
|
|
30
31
|
childPid: /** @type {number | null} */ (this.readAttribute("childPid")),
|
|
31
32
|
currentHostname: hostname,
|
|
33
|
+
liveChildren,
|
|
32
34
|
probe,
|
|
33
35
|
runnerHostname: /** @type {string | null} */ (this.readAttribute("runnerHostname"))
|
|
34
36
|
})
|
|
@@ -190,6 +190,13 @@ export default class BackgroundJobsMain {
|
|
|
190
190
|
* @type {Set<string>}
|
|
191
191
|
*/
|
|
192
192
|
this.reconnectedWorkerIds = new Set()
|
|
193
|
+
/**
|
|
194
|
+
* Pooled-runner identities this main observed accepting jobs, keyed by
|
|
195
|
+
* child instance id. A dead-owner sweep uses them to disprove a recorded
|
|
196
|
+
* pid that the OS recycled onto a newer runner before orphaning its rows.
|
|
197
|
+
* @type {Map<string, import("./owner-liveness.js").BackgroundJobLiveChildIdentity>}
|
|
198
|
+
*/
|
|
199
|
+
this.liveChildIdentities = new Map()
|
|
193
200
|
/** @type {import("./types.js").BackgroundJobHandoffSnapshot[]} */
|
|
194
201
|
this.startupHandoffSnapshot = []
|
|
195
202
|
/** @type {Promise<void>[]} */
|
|
@@ -637,6 +644,7 @@ export default class BackgroundJobsMain {
|
|
|
637
644
|
this.lifecycleState = "active"
|
|
638
645
|
this._creditReadyWorkers()
|
|
639
646
|
await this._settleForeignGenerationHandoffs()
|
|
647
|
+
await this._settleDeadOwnerHandoffs()
|
|
640
648
|
this.logger.info(() => ["Background jobs generation activation acknowledged", {generationId: this.generationId}])
|
|
641
649
|
void this._drain().catch((error) => {
|
|
642
650
|
this.logger.error(() => ["Background jobs generation post-activation drain failed", {error, generationId: this.generationId}])
|
|
@@ -912,6 +920,23 @@ export default class BackgroundJobsMain {
|
|
|
912
920
|
}
|
|
913
921
|
}
|
|
914
922
|
|
|
923
|
+
/**
|
|
924
|
+
* One dead-owner settlement pass: evaluate every exact active handoff's
|
|
925
|
+
* recorded runner process through the owner-liveness helper and settle the
|
|
926
|
+
* provably-dead ones regardless of which generation created the lease.
|
|
927
|
+
* Owners answering `alive` (the runner is still running the job) or `unknown`
|
|
928
|
+
* (remote host, or identity recorded before pids/hostnames existed) stay with
|
|
929
|
+
* their existing recovery paths. Live child identities disprove recorded pids
|
|
930
|
+
* the OS recycled onto a newer runner before anything is orphaned.
|
|
931
|
+
* @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted orphan transitions.
|
|
932
|
+
*/
|
|
933
|
+
async _settleDeadOwnerHandoffs() {
|
|
934
|
+
return await this.store.markOrphanedHandoffsByDeadOwner({
|
|
935
|
+
error: "Job orphaned when its recorded runner process was found dead",
|
|
936
|
+
liveChildren: [...this.liveChildIdentities.values()]
|
|
937
|
+
})
|
|
938
|
+
}
|
|
939
|
+
|
|
915
940
|
_setupStartupHandoffReclaim() {
|
|
916
941
|
if (this.startupHandoffSnapshot.length === 0) return
|
|
917
942
|
if (this._startupHandoffReclaimTimer || this._startupHandoffReclaimPromise || this._startupHandoffGraceElapsed) return
|
|
@@ -1445,6 +1470,9 @@ export default class BackgroundJobsMain {
|
|
|
1445
1470
|
startedAtMs: message.startedAtMs,
|
|
1446
1471
|
workerId: message.workerId
|
|
1447
1472
|
})
|
|
1473
|
+
if (typeof message.childInstanceId === "string" && typeof message.childPid === "number") {
|
|
1474
|
+
this._recordLiveChildIdentity({childInstanceId: message.childInstanceId, childPid: message.childPid})
|
|
1475
|
+
}
|
|
1448
1476
|
jsonSocket.send({type: "job-updated", jobId: message.jobId})
|
|
1449
1477
|
} catch (error) {
|
|
1450
1478
|
this._reportJobUpdateFailure({error, jobId: message.jobId, stage: "background-job-accepted"})
|
|
@@ -1452,6 +1480,25 @@ export default class BackgroundJobsMain {
|
|
|
1452
1480
|
}
|
|
1453
1481
|
}
|
|
1454
1482
|
|
|
1483
|
+
/**
|
|
1484
|
+
* Records one pooled-runner identity as live. A pid belongs to a single live
|
|
1485
|
+
* process, so an identity whose pid is now reported by a different child
|
|
1486
|
+
* instance proves that the older runner already exited: keeping its entry
|
|
1487
|
+
* would make the dead-owner pass answer `dead` for the newer, still-running
|
|
1488
|
+
* child that inherited the pid.
|
|
1489
|
+
* @param {object} args - Observed runner identity.
|
|
1490
|
+
* @param {string} args.childInstanceId - Stable pooled-child identity of the live runner.
|
|
1491
|
+
* @param {number} args.childPid - OS pid of the live runner.
|
|
1492
|
+
* @returns {void} - Nothing.
|
|
1493
|
+
*/
|
|
1494
|
+
_recordLiveChildIdentity({childInstanceId, childPid}) {
|
|
1495
|
+
for (const [recordedInstanceId, identity] of this.liveChildIdentities) {
|
|
1496
|
+
if (recordedInstanceId !== childInstanceId && identity.childPid === childPid) this.liveChildIdentities.delete(recordedInstanceId)
|
|
1497
|
+
}
|
|
1498
|
+
|
|
1499
|
+
this.liveChildIdentities.set(childInstanceId, {childInstanceId, childPid})
|
|
1500
|
+
}
|
|
1501
|
+
|
|
1455
1502
|
/**
|
|
1456
1503
|
* Requires the complete durable lease identity before a generation-mode
|
|
1457
1504
|
* reporter can mutate a job. Legacy reporters keep their permissive protocol.
|
|
@@ -2669,11 +2716,24 @@ export default class BackgroundJobsMain {
|
|
|
2669
2716
|
const handoffs = (await this._generationOwnedHandoffSnapshot()).filter((handoff) => {
|
|
2670
2717
|
return handoff.handedOffAtMs <= cutoff && !connectedWorkerIds.has(handoff.workerId)
|
|
2671
2718
|
})
|
|
2672
|
-
|
|
2719
|
+
const generationOrphanedJobs = handoffs.length === 0
|
|
2673
2720
|
? []
|
|
2674
2721
|
: await this.store.markOrphanedHandoffs({handoffs, error: "Job orphaned after its generation owner disappeared"})
|
|
2722
|
+
// Cross-generation rows are invisible to the fenced snapshot above; the
|
|
2723
|
+
// dead-owner pass settles them from the recorded runner process instead
|
|
2724
|
+
// of an age rule, so a retired generation can never strand a lease. The
|
|
2725
|
+
// fenced pass already settled its own rows, so only ids it did not
|
|
2726
|
+
// already report are appended.
|
|
2727
|
+
const deadOwnerJobs = await this._settleDeadOwnerHandoffs()
|
|
2728
|
+
const generationOrphanedJobIds = new Set(generationOrphanedJobs.map((job) => job.id))
|
|
2729
|
+
|
|
2730
|
+
orphanedJobs = [...generationOrphanedJobs, ...deadOwnerJobs.filter((job) => !generationOrphanedJobIds.has(job.id))]
|
|
2675
2731
|
} else {
|
|
2676
|
-
|
|
2732
|
+
const legacyOrphanedJobs = await this.store.markOrphanedJobs()
|
|
2733
|
+
const deadOwnerJobs = await this._settleDeadOwnerHandoffs()
|
|
2734
|
+
const legacyJobIds = new Set(legacyOrphanedJobs.map((job) => job.id))
|
|
2735
|
+
|
|
2736
|
+
orphanedJobs = [...legacyOrphanedJobs, ...deadOwnerJobs.filter((job) => !legacyJobIds.has(job.id))]
|
|
2677
2737
|
}
|
|
2678
2738
|
|
|
2679
2739
|
await this._handleOrphanedJobs({jobs: orphanedJobs, warning: "Marked orphaned background jobs"})
|
|
@@ -18,6 +18,14 @@ import os from "node:os"
|
|
|
18
18
|
* @property {BackgroundJobOwnerLivenessStatus} status - Liveness verdict.
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
+
/**
|
|
22
|
+
* A runner process this main currently observes as live, used to disprove a
|
|
23
|
+
* recorded pid that the OS has recycled onto a different runner.
|
|
24
|
+
* @typedef {object} BackgroundJobLiveChildIdentity
|
|
25
|
+
* @property {string | null} childInstanceId - Stable pooled-child identity of the live runner.
|
|
26
|
+
* @property {number} childPid - OS pid of the live runner.
|
|
27
|
+
*/
|
|
28
|
+
|
|
21
29
|
/**
|
|
22
30
|
* Probes a local pid without signalling the target process. `EPERM` means the
|
|
23
31
|
* process exists but belongs to another user, so it counts as alive; `ESRCH`
|
|
@@ -49,12 +57,16 @@ export function isProcessAlive(pid) {
|
|
|
49
57
|
* another machine the result is `unknown`, because liveness there needs a
|
|
50
58
|
* heartbeat or instance registry rather than a local probe. A pid alone can be
|
|
51
59
|
* recycled by the OS after a process exits, so treat `alive` as "a process with
|
|
52
|
-
* that pid exists now", not proof that it is the same process
|
|
53
|
-
* `
|
|
60
|
+
* that pid exists now", not proof that it is the same process. `child_instance_id`
|
|
61
|
+
* is a UUID and cannot be recycled: when `liveChildren` shows the recorded pid
|
|
62
|
+
* currently belongs to a runner with a different instance identity, the recorded
|
|
63
|
+
* runner is provably gone and the verdict is `dead` (`local_pid_recycled`)
|
|
64
|
+
* instead of a false `alive`.
|
|
54
65
|
* @param {object} [args] - Options.
|
|
55
66
|
* @param {string | null} [args.childInstanceId] - Recorded pooled-child identity.
|
|
56
67
|
* @param {number | null} [args.childPid] - Recorded OS pid.
|
|
57
68
|
* @param {string} [args.currentHostname] - Hostname of this process, defaults to `os.hostname()`.
|
|
69
|
+
* @param {BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this main observes as live, for the pid-recycling check.
|
|
58
70
|
* @param {(pid: number) => boolean} [args.probe] - Process probe, injectable for tests.
|
|
59
71
|
* @param {string | null} [args.runnerHostname] - Hostname recorded with the job.
|
|
60
72
|
* @returns {BackgroundJobOwnerLivenessResult} - Liveness verdict with its reason.
|
|
@@ -63,6 +75,7 @@ export function evaluateBackgroundJobOwnerLiveness({
|
|
|
63
75
|
childInstanceId = null,
|
|
64
76
|
childPid = null,
|
|
65
77
|
currentHostname = os.hostname(),
|
|
78
|
+
liveChildren = [],
|
|
66
79
|
probe = isProcessAlive,
|
|
67
80
|
runnerHostname = null
|
|
68
81
|
} = {}) {
|
|
@@ -86,7 +99,16 @@ export function evaluateBackgroundJobOwnerLiveness({
|
|
|
86
99
|
|
|
87
100
|
const probedPid = /** @type {number} */ (owner.childPid)
|
|
88
101
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
102
|
+
if (!probe(probedPid)) return {...owner, reason: "local_pid_missing", status: "dead"}
|
|
103
|
+
|
|
104
|
+
// PID-recycling guard: a live pid whose instance identity differs from the
|
|
105
|
+
// recorded one proves the recorded runner exited and the OS handed its pid
|
|
106
|
+
// to a new runner — the honest verdict is dead, not a false alive.
|
|
107
|
+
if (owner.childInstanceId) {
|
|
108
|
+
const recycledBy = liveChildren.find((child) => child.childPid === probedPid && child.childInstanceId !== null && child.childInstanceId !== owner.childInstanceId)
|
|
109
|
+
|
|
110
|
+
if (recycledBy) return {...owner, reason: "local_pid_recycled", status: "dead"}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
return {...owner, reason: "local_pid_alive", status: "alive"}
|
|
92
114
|
}
|
|
@@ -1691,6 +1691,80 @@ export default class BackgroundJobsStore extends BackgroundJobsAdapter {
|
|
|
1691
1691
|
})
|
|
1692
1692
|
}
|
|
1693
1693
|
|
|
1694
|
+
/**
|
|
1695
|
+
* Evaluates the recorded runner-process liveness of every exact, lease-aware
|
|
1696
|
+
* active handoff through {@link BackgroundJobRecord#ownerIsAlive}. Rows without
|
|
1697
|
+
* a complete lease identity stay out of the verdicts — they belong to the
|
|
1698
|
+
* age-based legacy sweep. The probe and live-child list are injectable so
|
|
1699
|
+
* specs can prove dead/recycled verdicts deterministically.
|
|
1700
|
+
* @param {object} [args] - Options.
|
|
1701
|
+
* @param {string} [args.hostname] - Hostname whose local probe answers ownership, defaults to this process's hostname.
|
|
1702
|
+
* @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this main observes as live, for the pid-recycling check.
|
|
1703
|
+
* @param {(pid: number) => boolean} [args.probe] - Process probe, injectable for tests.
|
|
1704
|
+
* @returns {Promise<import("./types.js").BackgroundJobHandoffOwnershipVerdict[]>} - One verdict per exact active handoff.
|
|
1705
|
+
*/
|
|
1706
|
+
async snapshotHandedOffJobOwnership({hostname, liveChildren = [], probe} = {}) {
|
|
1707
|
+
await this.ensureReady()
|
|
1708
|
+
|
|
1709
|
+
return await this._withDb(async (db) => {
|
|
1710
|
+
const rows = await db
|
|
1711
|
+
.newQuery()
|
|
1712
|
+
.from(JOBS_TABLE)
|
|
1713
|
+
.where({status: "handed_off"})
|
|
1714
|
+
.order("created_at_ms ASC")
|
|
1715
|
+
.order("id ASC")
|
|
1716
|
+
.results()
|
|
1717
|
+
/** @type {import("./types.js").BackgroundJobHandoffOwnershipVerdict[]} */
|
|
1718
|
+
const verdicts = []
|
|
1719
|
+
|
|
1720
|
+
for (const row of rows) {
|
|
1721
|
+
const job = this._normalizeJobRow(row)
|
|
1722
|
+
|
|
1723
|
+
if (!job.handoffId || !job.workerId || typeof job.handedOffAtMs !== "number") continue
|
|
1724
|
+
|
|
1725
|
+
const record = new BackgroundJobRecord()
|
|
1726
|
+
|
|
1727
|
+
record.loadExistingRecord(row)
|
|
1728
|
+
verdicts.push({
|
|
1729
|
+
handedOffAtMs: job.handedOffAtMs,
|
|
1730
|
+
handoffId: job.handoffId,
|
|
1731
|
+
jobId: job.id,
|
|
1732
|
+
liveness: record.ownerIsAlive({hostname, liveChildren, probe}),
|
|
1733
|
+
workerId: job.workerId
|
|
1734
|
+
})
|
|
1735
|
+
}
|
|
1736
|
+
|
|
1737
|
+
return verdicts
|
|
1738
|
+
})
|
|
1739
|
+
}
|
|
1740
|
+
|
|
1741
|
+
/**
|
|
1742
|
+
* Settles every exact active handoff whose recorded runner process is
|
|
1743
|
+
* provably dead on this host, regardless of which generation created the
|
|
1744
|
+
* lease. Owners that answer `alive` (still running) or `unknown` (remote
|
|
1745
|
+
* host, identity recorded before hostnames/pids existed) stay untouched for
|
|
1746
|
+
* their existing recovery paths. Each settled row goes through the same
|
|
1747
|
+
* fenced orphan transition as the orphan sweep, so retries, terminal status,
|
|
1748
|
+
* `orphaned_at_ms`/`last_error` stamping, count transitions, schedule
|
|
1749
|
+
* ownership, and concurrency release all follow.
|
|
1750
|
+
* @param {object} [args] - Options.
|
|
1751
|
+
* @param {ReturnType<typeof JSON.parse>} [args.error] - Orphan reason recorded on settled rows.
|
|
1752
|
+
* @param {string} [args.hostname] - Hostname whose local probe answers ownership, defaults to this process's hostname.
|
|
1753
|
+
* @param {import("./owner-liveness.js").BackgroundJobLiveChildIdentity[]} [args.liveChildren] - Runner identities this main observes as live, for the pid-recycling check.
|
|
1754
|
+
* @param {(pid: number) => boolean} [args.probe] - Process probe, injectable for tests.
|
|
1755
|
+
* @returns {Promise<import("./types.js").BackgroundJobRow[]>} - Accepted orphan transitions.
|
|
1756
|
+
*/
|
|
1757
|
+
async markOrphanedHandoffsByDeadOwner({error = "Job orphaned when its recorded runner process was found dead", hostname, liveChildren = [], probe} = {}) {
|
|
1758
|
+
await this.ensureReady()
|
|
1759
|
+
|
|
1760
|
+
const verdicts = await this.snapshotHandedOffJobOwnership({hostname, liveChildren, probe})
|
|
1761
|
+
const deadHandoffs = verdicts.filter((verdict) => verdict.liveness.status === "dead")
|
|
1762
|
+
|
|
1763
|
+
if (deadHandoffs.length === 0) return []
|
|
1764
|
+
|
|
1765
|
+
return await this.markOrphanedHandoffs({error, handoffs: deadHandoffs})
|
|
1766
|
+
}
|
|
1767
|
+
|
|
1694
1768
|
/**
|
|
1695
1769
|
* Runs mark failed.
|
|
1696
1770
|
* @param {object} args - Options.
|
|
@@ -159,6 +159,9 @@
|
|
|
159
159
|
* @property {string} workerId - Stable worker id that received the lease.
|
|
160
160
|
* @property {number} handedOffAtMs - Time handed to the worker in ms.
|
|
161
161
|
*/
|
|
162
|
+
/**
|
|
163
|
+
* @typedef {BackgroundJobHandoffSnapshot & {liveness: import("./owner-liveness.js").BackgroundJobOwnerLivenessResult}} BackgroundJobHandoffOwnershipVerdict
|
|
164
|
+
*/
|
|
162
165
|
/**
|
|
163
166
|
* @typedef {object} BackgroundJobHandoffRequest
|
|
164
167
|
* @property {string} jobId - Job to claim.
|
|
@@ -882,7 +882,7 @@
|
|
|
882
882
|
* behavior; the file backend is disabled by default and only selectable by
|
|
883
883
|
* explicit configuration.
|
|
884
884
|
* @typedef {object} AdvisoryLocksConfiguration
|
|
885
|
-
* @property {{file?: {directory?: string, enabled?: boolean, leaseMs?: number, renewIntervalMs?: number}, lockServer?: {authToken?: string, connectTimeoutMs?: number, enabled?: boolean, host?: string, leaseMs?: number, port?: number, reprobeCooldownMs?: number, url?: string}}} [backends] - Backend enablement. `file.enabled` defaults to false; enabling it is the deployment's explicit single-host declaration. `lockServer` defaults to enabled when a host/port/url or the `VELOCIOUS_LOCK_SERVER_HOST`/`_PORT` env pair is configured.
|
|
885
|
+
* @property {{file?: {directory?: string, enabled?: boolean, leaseMs?: number, renewIntervalMs?: number}, lockServer?: {authToken?: string, connectTimeoutMs?: number, enabled?: boolean, host?: string, leaseMs?: number, mode?: "embedded" | "external" | "local", port?: number, reprobeCooldownMs?: number, socketPath?: string, url?: string}}} [backends] - Backend enablement. `file.enabled` defaults to false; enabling it is the deployment's explicit single-host declaration. `lockServer` defaults to enabled when a host/port/url or the `VELOCIOUS_LOCK_SERVER_HOST`/`_PORT` env pair is configured.
|
|
886
886
|
* @property {import("./database/advisory-lock-backend-policy.js").AdvisoryLockBackendPolicyEntry[]} [policy] - Per-lock-name minimum/allowed/preferred tier rules (design §5).
|
|
887
887
|
*/
|
|
888
888
|
|
package/src/configuration.js
CHANGED
|
@@ -275,6 +275,14 @@ export default class VelociousConfiguration {
|
|
|
275
275
|
_advisoryLockAcquisitions = new Map()
|
|
276
276
|
_advisoryLockReentries = new Map()
|
|
277
277
|
|
|
278
|
+
/**
|
|
279
|
+
* Per-tier advisory-lock acquisition counters — the "how many locks are
|
|
280
|
+
* served by which tier" attribution surface (STC-218 scope 1). Keyed by
|
|
281
|
+
* tier so production reads tier attribution directly instead of ad-hoc
|
|
282
|
+
* processlist probing.
|
|
283
|
+
* @type {Map<import("./database/advisory-lock-backend-policy.js").AdvisoryLockTier, {count: number, lastServedAt: string}>} */
|
|
284
|
+
_advisoryLockTierCounts = new Map()
|
|
285
|
+
|
|
278
286
|
/** @type {Map<string, number>} */
|
|
279
287
|
_schemaCacheGenerationsByReuseKey = new Map()
|
|
280
288
|
|
|
@@ -4226,7 +4234,7 @@ export default class VelociousConfiguration {
|
|
|
4226
4234
|
* Full backward compatibility: an app that configures nothing gets today's
|
|
4227
4235
|
* driver-native-only behavior — the file backend is **disabled by default**
|
|
4228
4236
|
* and only ever selectable by explicit configuration.
|
|
4229
|
-
* @returns {{backends: {file: {directory?: string, enabled: boolean, leaseMs: number, renewIntervalMs: number}, lockServer: {authToken: string, connectTimeoutMs: number, enabled: boolean, leaseMs: number, reprobeCooldownMs: number, url: string}}, policy: import("./database/advisory-lock-backend-policy.js").AdvisoryLockBackendPolicyEntry[]}} - Advisory-lock configuration with defaults applied.
|
|
4237
|
+
* @returns {{backends: {file: {directory?: string, enabled: boolean, leaseMs: number, renewIntervalMs: number}, lockServer: {authToken: string, connectTimeoutMs: number, enabled: boolean, leaseMs: number, mode: "embedded" | "external" | "local", reprobeCooldownMs: number, socketPath: string, url: string}}, policy: import("./database/advisory-lock-backend-policy.js").AdvisoryLockBackendPolicyEntry[]}} - Advisory-lock configuration with defaults applied.
|
|
4230
4238
|
*/
|
|
4231
4239
|
getAdvisoryLockConfig() {
|
|
4232
4240
|
const configured = this._advisoryLocks || {}
|
|
@@ -4241,6 +4249,15 @@ export default class VelociousConfiguration {
|
|
|
4241
4249
|
: (typeof envPort === "number" && Number.isFinite(envPort) ? envPort : 3006)
|
|
4242
4250
|
const lockServerUrl = lockServerConfigured.url
|
|
4243
4251
|
|| `ws://${lockServerHost || "127.0.0.1"}:${lockServerPort}/websocket`
|
|
4252
|
+
// STC-715: `embedded` runs the lock-server engine in this process, `local`
|
|
4253
|
+
// spawns it as a child over stdio JSON-lines. Both default `enabled` to true
|
|
4254
|
+
// when the mode is named explicitly — no host/port/url required. `external`
|
|
4255
|
+
// (the default) keeps today's rule: enabled only via url/host/port config
|
|
4256
|
+
// or the env pair, so an app that configures nothing is unchanged.
|
|
4257
|
+
const configuredMode = lockServerConfigured.mode
|
|
4258
|
+
const lockServerMode = configuredMode === "embedded" || configuredMode === "local" || configuredMode === "external"
|
|
4259
|
+
? configuredMode
|
|
4260
|
+
: "external"
|
|
4244
4261
|
|
|
4245
4262
|
return {
|
|
4246
4263
|
backends: {
|
|
@@ -4257,9 +4274,13 @@ export default class VelociousConfiguration {
|
|
|
4257
4274
|
connectTimeoutMs: typeof lockServerConfigured.connectTimeoutMs === "number" ? lockServerConfigured.connectTimeoutMs : 3000,
|
|
4258
4275
|
enabled: typeof lockServerConfigured.enabled === "boolean"
|
|
4259
4276
|
? lockServerConfigured.enabled
|
|
4260
|
-
:
|
|
4277
|
+
: lockServerMode === "external"
|
|
4278
|
+
? Boolean(lockServerConfigured.url || lockServerConfigured.host || envHost || typeof lockServerConfigured.port === "number" || envPortRaw)
|
|
4279
|
+
: true,
|
|
4261
4280
|
leaseMs: typeof lockServerConfigured.leaseMs === "number" ? lockServerConfigured.leaseMs : 30_000,
|
|
4281
|
+
mode: lockServerMode,
|
|
4262
4282
|
reprobeCooldownMs: typeof lockServerConfigured.reprobeCooldownMs === "number" ? lockServerConfigured.reprobeCooldownMs : 5000,
|
|
4283
|
+
socketPath: typeof lockServerConfigured.socketPath === "string" ? lockServerConfigured.socketPath : "",
|
|
4263
4284
|
url: lockServerUrl
|
|
4264
4285
|
}
|
|
4265
4286
|
},
|
|
@@ -4329,11 +4350,13 @@ export default class VelociousConfiguration {
|
|
|
4329
4350
|
authToken: config.backends.lockServer.authToken,
|
|
4330
4351
|
connectTimeoutMs: config.backends.lockServer.connectTimeoutMs,
|
|
4331
4352
|
leaseMs: config.backends.lockServer.leaseMs,
|
|
4353
|
+
mode: config.backends.lockServer.mode,
|
|
4332
4354
|
onError: (/** @type {Error} */ error) => {
|
|
4333
4355
|
this.getErrorEvents().emit("framework-error", {context: "advisory-lock-lock-server", error})
|
|
4334
4356
|
this.getErrorEvents().emit("all-error", {context: "advisory-lock-lock-server", error, errorType: "framework-error"})
|
|
4335
4357
|
},
|
|
4336
4358
|
reprobeCooldownMs: config.backends.lockServer.reprobeCooldownMs,
|
|
4359
|
+
socketPath: config.backends.lockServer.socketPath,
|
|
4337
4360
|
url: config.backends.lockServer.url
|
|
4338
4361
|
})
|
|
4339
4362
|
|
|
@@ -4353,7 +4376,34 @@ export default class VelociousConfiguration {
|
|
|
4353
4376
|
* @returns {void}
|
|
4354
4377
|
*/
|
|
4355
4378
|
recordAdvisoryLockAcquisition({lockName, backend, fencingSupported}) {
|
|
4356
|
-
|
|
4379
|
+
const servedAt = new Date().toISOString()
|
|
4380
|
+
|
|
4381
|
+
this._advisoryLockAcquisitions.set(lockName, {backend, fencingSupported, servedAt})
|
|
4382
|
+
|
|
4383
|
+
const tierCount = this._advisoryLockTierCounts.get(backend) ?? {count: 0, lastServedAt: servedAt}
|
|
4384
|
+
|
|
4385
|
+
tierCount.count += 1
|
|
4386
|
+
tierCount.lastServedAt = servedAt
|
|
4387
|
+
this._advisoryLockTierCounts.set(backend, tierCount)
|
|
4388
|
+
}
|
|
4389
|
+
|
|
4390
|
+
/**
|
|
4391
|
+
* The "how many locks are served by which tier" attribution surface
|
|
4392
|
+
* (STC-218 scope 1): per tier, the total acquisitions this process served
|
|
4393
|
+
* and when the tier last served one. Together with
|
|
4394
|
+
* `getAdvisoryLockBackendStatus(lockName)` this is what production reads to
|
|
4395
|
+
* attribute lock traffic to backends without ad-hoc probing.
|
|
4396
|
+
* @returns {Record<string, {count: number, lastServedAt: string}>} - Snapshot of the per-tier counters.
|
|
4397
|
+
*/
|
|
4398
|
+
getAdvisoryLockBackendStats() {
|
|
4399
|
+
/** @type {Record<string, {count: number, lastServedAt: string}>} */
|
|
4400
|
+
const snapshot = {}
|
|
4401
|
+
|
|
4402
|
+
for (const [tier, counter] of this._advisoryLockTierCounts) {
|
|
4403
|
+
snapshot[tier] = {count: counter.count, lastServedAt: counter.lastServedAt}
|
|
4404
|
+
}
|
|
4405
|
+
|
|
4406
|
+
return snapshot
|
|
4357
4407
|
}
|
|
4358
4408
|
|
|
4359
4409
|
/**
|