tickmarkr 2.7.4 → 2.7.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/init.js +1 -1
- package/dist/cli/commands/report.js +4 -2
- package/dist/cli/commands/status.js +8 -5
- package/dist/cli/commands/verify.js +4 -4
- package/dist/run/daemon.js +6 -2
- package/dist/run/journal.d.ts +12 -0
- package/dist/run/journal.js +41 -3
- package/dist/run/lease.d.ts +19 -2
- package/dist/run/lease.js +56 -11
- package/dist/run/suite-census.d.ts +3 -0
- package/dist/run/suite-census.js +37 -28
- package/dist/run/verification-job.d.ts +4 -2
- package/dist/run/verification-job.js +1 -1
- package/package.json +1 -1
- package/skills/tickmarkr-overseer/scripts/watch-journal.sh +27 -6
- package/skills/tickmarkr-overseer/scripts/watch-launch.sh +11 -3
|
@@ -111,7 +111,7 @@ Before \`tickmarkr compile\` or \`tickmarkr run\`: run \`tickmarkr version\`, re
|
|
|
111
111
|
|
|
112
112
|
### Tip-verify-before-green
|
|
113
113
|
|
|
114
|
-
A run is green only when ALL of these hold: the run-end event exists in the journal, tip verify is not "failed", the run-end summary's \`failed\`, \`human\`, \`blocked\` and \`pending\` buckets are every one of them empty, AND the owed checks in CURRENT \`tickmarkr status <runId>\` are outstanding empty AND known (\`outstanding 0\`; \`outstanding unknown\` is never empty). The shorter "run-end plus tip verify" form is NOT sufficient: a run ending \`done=[T1,T3,T4] human=[T2] tipVerify=passed\` is a closed PARTIAL run — three of four delivered, one parked — and calling it green is how a park becomes invisible. Empty execution buckets are not sufficient either (D-660): a run ending with every execution bucket empty and \`tipVerify=passed\` after an operator waived one review still owes that accepted-risk review check — status reads \`outstanding 1 (T7 review)\`, and \`tickmarkr run\` and \`resume\` exit 0 on execution alone — so it is execution complete, not green. Read the debt from CURRENT status, never from the run-end record: a \`tickmarkr verify --record <runId>\` discharge that lands after run-end moves current status to \`outstanding 0\` while the historical run-end row still carries 1, and only then is that run green; a legacy waiver without a recorded obligation reads \`outstanding unknown\` and is never green. Never report green to the operator, tab titles, or records until every clause holds; when one does not, name the bucket that is not empty, the outstanding checks, or the unknown debt.
|
|
114
|
+
A run is green only when ALL of these hold: the run-end event exists in the journal and is not fatal, tip verify is not "failed", the run-end summary's \`failed\`, \`human\`, \`blocked\` and \`pending\` buckets are every one of them empty, AND the owed checks in CURRENT \`tickmarkr status <runId>\` are outstanding empty AND known (\`outstanding 0\`; \`outstanding unknown\` is never empty). The shorter "run-end plus tip verify" form is NOT sufficient: a run ending \`done=[T1,T3,T4] human=[T2] tipVerify=passed\` is a closed PARTIAL run — three of four delivered, one parked — and calling it green is how a park becomes invisible. Empty execution buckets are not sufficient either (D-660): a run ending with every execution bucket empty and \`tipVerify=passed\` after an operator waived one review still owes that accepted-risk review check — status reads \`outstanding 1 (T7 review)\`, and \`tickmarkr run\` and \`resume\` exit 0 on execution alone — so it is execution complete, not green. Read the debt from CURRENT status, never from the run-end record: a \`tickmarkr verify --record <runId>\` discharge that lands after run-end moves current status to \`outstanding 0\` while the historical run-end row still carries 1, and only then is that run green; a legacy waiver without a recorded obligation reads \`outstanding unknown\` and is never green. A fatal run-end is never green either: the crash record (\`fatal: true\`) names its phase and error, every task it left running or gated sits in \`pending\` or \`blocked\`, and even with every bucket empty status and report lead with the crash and read \`outstanding unknown\` until the run is resumed; from then on the resumed engagement owns the debt. Never report green to the operator, tab titles, or records until every clause holds; when one does not, name the bucket that is not empty, the outstanding checks, or the unknown debt.
|
|
115
115
|
|
|
116
116
|
### Verified handoffs
|
|
117
117
|
|
|
@@ -11,7 +11,7 @@ import { compareRuns } from "../../report/compare.js";
|
|
|
11
11
|
import { estimateCosts } from "../../report/cost.js";
|
|
12
12
|
import { assignmentChannel, buildOperatorRecord, formatChannelMoney, formatChannelTokens, formatTokenUsage as fields, totalTokens as total, } from "../../report/operator-record.js";
|
|
13
13
|
import { cellsOf, cellSummary } from "../../route/profile.js";
|
|
14
|
-
import { effectiveEvents, foldOwedChecks, Journal, loadRoutingProfile, OWED_DISCHARGE_EVENT, } from "../../run/journal.js";
|
|
14
|
+
import { effectiveEvents, fatalRunEndCause, foldOwedChecks, Journal, loadRoutingProfile, OWED_DISCHARGE_EVENT, } from "../../run/journal.js";
|
|
15
15
|
import { baselineProvenanceOf, fingerprintsOf, forgivenFingerprints, forgivenGateRow, formatBaselineProvenance, formatFingerprints, formatForgiven, formatTipProof, runEndTipProof, } from "../../run/daemon.js";
|
|
16
16
|
import { formatSpan, WALL_PRIORITY, WALL_PRIORITY_TEXT, wallBudget, wallBudgetFacts } from "../../run/wall-budget.js";
|
|
17
17
|
import { deriveRunCockpitData } from "../../tui/cockpit/derive.js";
|
|
@@ -241,8 +241,10 @@ const timeLead = (events) => {
|
|
|
241
241
|
* comparability or supersession note); the rest is folded here from the journal and the current fold.
|
|
242
242
|
*/
|
|
243
243
|
export function leadLines(events, owed, finished) {
|
|
244
|
+
// A fatal run-end leads with its crash: no tally after it may read as a finished run.
|
|
245
|
+
const crash = fatalRunEndCause(events);
|
|
244
246
|
return [
|
|
245
|
-
`finished ${[finished.tally, finished.tip, firstPassText(endToEndFirstPass(events)), ...(finished.notes ?? [])].join(" · ")}`,
|
|
247
|
+
`finished ${[...(crash ? [`run crashed — ${crash}`] : []), finished.tally, finished.tip, firstPassText(endToEndFirstPass(events)), ...(finished.notes ?? [])].join(" · ")}`,
|
|
246
248
|
timeLead(events),
|
|
247
249
|
needsYouText(events, owed),
|
|
248
250
|
];
|
|
@@ -13,7 +13,7 @@ import { projectOperatorSummary } from "../../run/operator-summary.js";
|
|
|
13
13
|
import { trackJournalRows } from "../../run/protocol.js";
|
|
14
14
|
import { newestPark, permittedDecisionVerbs } from "./approve.js";
|
|
15
15
|
import { currentOwed, endToEndFirstPass, firstPassText, leadLines, leadRows, needsYouText, wallText } from "./report.js";
|
|
16
|
-
import { Journal, bindingToken, effectiveEvents, formatJournalNarration, parseJournalText, physicalLine, engagementComparable, isQualityFailureParkKind, parseRunId, preservedRefsByTask, recordedTaskFailureKind, runHasEnded, upheldFeedbackByTask, } from "../../run/journal.js";
|
|
16
|
+
import { Journal, bindingToken, effectiveEvents, formatJournalNarration, parseJournalText, physicalLine, engagementComparable, isQualityFailureParkKind, parseRunId, preservedRefsByTask, fatalRunEndCause, recordedTaskFailureKind, runHasEnded, upheldFeedbackByTask, visibleControls, } from "../../run/journal.js";
|
|
17
17
|
import { isPidLive, runLockRunId, runStatusLine } from "../../run/lock.js";
|
|
18
18
|
import { normalizeGateOutcome } from "../../run/outcome.js";
|
|
19
19
|
import { desiredPanes } from "../../run/reconcile.js";
|
|
@@ -649,10 +649,11 @@ const terminalFailureCause = (events) => {
|
|
|
649
649
|
const e = events[i];
|
|
650
650
|
if (e.event !== "run-end" || typeof e.data.error !== "string")
|
|
651
651
|
continue;
|
|
652
|
-
|
|
653
|
-
|
|
652
|
+
// the recorded error is data: its terminal controls print escaped, never executed (the journal keeps them)
|
|
653
|
+
const phase = typeof e.data.phase === "string" && visibleControls(e.data.phase)
|
|
654
|
+
? `${visibleControls(e.data.phase)} failed`
|
|
654
655
|
: "run failed";
|
|
655
|
-
return `${phase}: ${e.data.error
|
|
656
|
+
return `${phase}: ${visibleControls(e.data.error)}`;
|
|
656
657
|
}
|
|
657
658
|
return undefined;
|
|
658
659
|
};
|
|
@@ -1429,8 +1430,10 @@ const oneLine = (cwd, namedRunId) => {
|
|
|
1429
1430
|
// CG1: the lead's facts in one line — lineage first pass, the wall window, and what needs you from
|
|
1430
1431
|
// the CURRENT fold, so a discharge after run-end moves this line too.
|
|
1431
1432
|
const { events } = record;
|
|
1433
|
+
// A fatal run-end leads right after the run id: no completion fraction or verify state reads first.
|
|
1434
|
+
const crash = fatalRunEndCause(events);
|
|
1432
1435
|
return sanitizeTaskText([
|
|
1433
|
-
record.runId, ...claims, firstPassText(endToEndFirstPass(events)), wallText(events), needsYouText(events, currentOwed(events, cwd)),
|
|
1436
|
+
record.runId, ...(crash ? [`run crashed — ${crash}`] : []), ...claims, firstPassText(endToEndFirstPass(events)), wallText(events), needsYouText(events, currentOwed(events, cwd)),
|
|
1434
1437
|
].join(" · "));
|
|
1435
1438
|
};
|
|
1436
1439
|
export async function status(argv, cwd = process.cwd(), opts = {}) {
|
|
@@ -15,7 +15,7 @@ import { GATE_NAMES } from "../../graph/schema.js";
|
|
|
15
15
|
import { executionSignal } from "../../run/execution-budget.js";
|
|
16
16
|
import { addDetachedWorktree, linkNodeModules, removeWorktree, shGit, shGitOk } from "../../run/git.js";
|
|
17
17
|
import { foldOwedChecks, integrationMapped, owedCriteria, Journal, OWED_DISCHARGE_EVENT } from "../../run/journal.js";
|
|
18
|
-
import { withRepositoryLease } from "../../run/lease.js";
|
|
18
|
+
import { describeLeaseWait, withRepositoryLease } from "../../run/lease.js";
|
|
19
19
|
import { isPidLive } from "../../run/lock.js";
|
|
20
20
|
import { recordLaunchServices } from "../../run/launchservices-check.js";
|
|
21
21
|
/**
|
|
@@ -529,9 +529,9 @@ export async function verify(argv, cwd = process.cwd(), options = {}) {
|
|
|
529
529
|
return after ? rows.map((r) => ({ ...r, pass: false, details: `${refused(after)}\n${r.details}`, meta: { ...r.meta, subjectChanged: after } })) : rows;
|
|
530
530
|
}, {
|
|
531
531
|
signal: executionSignal(),
|
|
532
|
-
onWait: (holder) => {
|
|
533
|
-
console.error(`verify: waiting for the repository's runner lease
|
|
534
|
-
recordJournal?.append("suite-wait", values.task ?? "VERIFY", { holderPid: holder.pid, holderCwd: holder.cwd });
|
|
532
|
+
onWait: (holder, blockers) => {
|
|
533
|
+
console.error(`verify: waiting for the repository's runner lease ${describeLeaseWait(holder, blockers)}`);
|
|
534
|
+
recordJournal?.append("suite-wait", values.task ?? "VERIFY", { holderPid: holder.pid, holderCwd: holder.cwd, ...(blockers ? { blockers } : {}) });
|
|
535
535
|
},
|
|
536
536
|
});
|
|
537
537
|
results = [...results, ...suite].sort((a, b) => GATE_NAMES.indexOf(a.gate) - GATE_NAMES.indexOf(b.gate));
|
package/dist/run/daemon.js
CHANGED
|
@@ -1858,14 +1858,18 @@ export function recordFatalRunEnd(journal, runId, branch, err, graph, phase = "s
|
|
|
1858
1858
|
catch (readErr) {
|
|
1859
1859
|
console.error(`tickmarkr ${runId}: journal read failed while recording the fatal run-end (${readErr instanceof Error ? readErr.message : String(readErr)}) — original error: ${original}`);
|
|
1860
1860
|
}
|
|
1861
|
+
// A crash can land mid-dispatch. Resume re-pends every dispatched-but-unfinished task, so a running or
|
|
1862
|
+
// gated task is bucketed as the pending task resume will make it — pending, or blocked under a parked
|
|
1863
|
+
// dependency — and no task falls outside the five buckets every reader knows.
|
|
1864
|
+
const repended = graph && { ...graph, tasks: graph.tasks.map((t) => t.status === "running" || t.status === "gated" ? { ...t, status: "pending" } : t) };
|
|
1861
1865
|
const record = {
|
|
1862
1866
|
runId,
|
|
1863
1867
|
branch,
|
|
1864
1868
|
done: graph?.tasks.filter((t) => t.status === "done").map((t) => t.id) ?? [],
|
|
1865
1869
|
failed: graph?.tasks.filter((t) => t.status === "failed").map((t) => t.id) ?? [],
|
|
1866
1870
|
human: graph?.tasks.filter((t) => t.status === "human").map((t) => t.id) ?? [],
|
|
1867
|
-
blocked:
|
|
1868
|
-
pending:
|
|
1871
|
+
blocked: repended ? blockedTasks(repended).map((t) => t.id) : [],
|
|
1872
|
+
pending: repended ? pendingTasks(repended).map((t) => t.id) : [],
|
|
1869
1873
|
phase,
|
|
1870
1874
|
fatal: true,
|
|
1871
1875
|
error: original,
|
package/dist/run/journal.d.ts
CHANGED
|
@@ -191,6 +191,13 @@ export declare const GATE_FINGERPRINT_CAP = 2;
|
|
|
191
191
|
export declare const receiptOrigin: (data: Record<string, unknown>) => string | undefined;
|
|
192
192
|
export declare function identicalGateFailures(events: JournalEvent[], taskId: string, gate: string, normalized: string): number;
|
|
193
193
|
export declare const readableExcerpt: (text: string) => string;
|
|
194
|
+
/**
|
|
195
|
+
* One-line display text for a recorded crash: every C0 control but tab and newline, DEL and every C1
|
|
196
|
+
* control prints as its visible escape (`\x1b`, `\u009b`), so no 7-bit or 8-bit OSC/CSI/ST sequence
|
|
197
|
+
* reaches a terminal and the reader still sees the text carried one; tabs and line breaks collapse to
|
|
198
|
+
* one space. The journal row keeps the raw bytes.
|
|
199
|
+
*/
|
|
200
|
+
export declare const visibleControls: (text: string) => string;
|
|
194
201
|
/** What each funded repair's next battery actually reached. A repair spends its budget only
|
|
195
202
|
* when that battery reaches one of the gates it was funded to fix; an earlier red is evidence that
|
|
196
203
|
* this repair never got its funded turn, not a charge against the repair ladder. */
|
|
@@ -379,6 +386,11 @@ export interface InterruptedAttempt {
|
|
|
379
386
|
}
|
|
380
387
|
export declare function resumeHarvestAuthor(events: JournalEvent[], taskId: string): Assignment | undefined;
|
|
381
388
|
export declare function interruptedAttempt(events: JournalEvent[], taskId: string): InterruptedAttempt | undefined;
|
|
389
|
+
/**
|
|
390
|
+
* The crash a FATAL run-end records, when it closes the newest engagement: `<phase> failed: <error>`.
|
|
391
|
+
* A later run-start or run-resume supersedes it; a normal run-end is no crash.
|
|
392
|
+
*/
|
|
393
|
+
export declare function fatalRunEndCause(events: readonly JournalEvent[]): string | undefined;
|
|
382
394
|
export declare function runHasEnded(events: JournalEvent[]): boolean;
|
|
383
395
|
/** OBS-53: classify worker-result failures so retries and routing see the true signal, not one lumped bucket. */
|
|
384
396
|
export declare function classifyWorkerResultCause(opts: {
|
package/dist/run/journal.js
CHANGED
|
@@ -763,6 +763,16 @@ export function identicalGateFailures(events, taskId, gate, normalized) {
|
|
|
763
763
|
// the failure fingerprint keep the raw text.
|
|
764
764
|
const TERMINAL_CONTROL_RE = /\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)?|\x1b\[[0-?]*[ -/]*[@-~]|\x1b[ -/]*[0-~]?|[\x00-\x08\x0b-\x1f\x7f]/g;
|
|
765
765
|
export const readableExcerpt = (text) => text.replace(TERMINAL_CONTROL_RE, "");
|
|
766
|
+
/**
|
|
767
|
+
* One-line display text for a recorded crash: every C0 control but tab and newline, DEL and every C1
|
|
768
|
+
* control prints as its visible escape (`\x1b`, `\u009b`), so no 7-bit or 8-bit OSC/CSI/ST sequence
|
|
769
|
+
* reaches a terminal and the reader still sees the text carried one; tabs and line breaks collapse to
|
|
770
|
+
* one space. The journal row keeps the raw bytes.
|
|
771
|
+
*/
|
|
772
|
+
export const visibleControls = (text) => text.replace(/[\x00-\x08\x0b-\x1f\x7f-\x9f]/gu, (c) => {
|
|
773
|
+
const code = c.charCodeAt(0).toString(16);
|
|
774
|
+
return c < "\x80" ? `\\x${code.padStart(2, "0")}` : `\\u${code.padStart(4, "0")}`;
|
|
775
|
+
}).replace(/\s+/gu, " ").trim();
|
|
766
776
|
export function repairReachSinceApproval(events, taskId) {
|
|
767
777
|
const repairs = [];
|
|
768
778
|
for (const e of effectiveEvents(events)) { // OBS-1178: a refused or unsound approval resets no repair history
|
|
@@ -1367,6 +1377,28 @@ export function interruptedAttempt(events, taskId) {
|
|
|
1367
1377
|
}
|
|
1368
1378
|
// Runs can end and later resume in the same journal. The newest lifecycle marker decides whether
|
|
1369
1379
|
// an unresolved task is still recoverable by this live daemon or belongs to an ended run.
|
|
1380
|
+
/**
|
|
1381
|
+
* The crash a FATAL run-end records, when it closes the newest engagement: `<phase> failed: <error>`.
|
|
1382
|
+
* A later run-start or run-resume supersedes it; a normal run-end is no crash.
|
|
1383
|
+
*/
|
|
1384
|
+
export function fatalRunEndCause(events) {
|
|
1385
|
+
for (let i = events.length - 1; i >= 0; i--) {
|
|
1386
|
+
const e = events[i];
|
|
1387
|
+
if (e.event === "run-start" || e.event === "run-resume")
|
|
1388
|
+
return undefined;
|
|
1389
|
+
if (e.event !== "run-end")
|
|
1390
|
+
continue;
|
|
1391
|
+
if (e.data.fatal !== true)
|
|
1392
|
+
return undefined;
|
|
1393
|
+
// Display text only: terminal controls in a recorded error (OSC title, CSI clear) print escaped,
|
|
1394
|
+
// never executed, and the journal row keeps the raw bytes.
|
|
1395
|
+
const shown = (v) => typeof v === "string" ? visibleControls(v) : "";
|
|
1396
|
+
const phase = shown(e.data.phase) || "run";
|
|
1397
|
+
const error = shown(e.data.error);
|
|
1398
|
+
return `${phase} failed: ${error || "unrecorded error"}`;
|
|
1399
|
+
}
|
|
1400
|
+
return undefined;
|
|
1401
|
+
}
|
|
1370
1402
|
export function runHasEnded(events) {
|
|
1371
1403
|
for (let i = events.length - 1; i >= 0; i--) {
|
|
1372
1404
|
const event = events[i].event;
|
|
@@ -1943,7 +1975,9 @@ export class Journal {
|
|
|
1943
1975
|
// C1: the terminal record carries the owed-check fold as of this append, revalidated from
|
|
1944
1976
|
// each discharge's artifact — a reopened journal re-reduces, it never copies a prior total.
|
|
1945
1977
|
// Canonical (validated) rows are strict and keep their schema; the live producer is the tuple.
|
|
1946
|
-
|
|
1978
|
+
// A fatal record folds over itself, so its own owedChecks already read unknown with the crash.
|
|
1979
|
+
const folded = !decisionRow && data.fatal === true ? [...priorEvents, { ts: new Date().toISOString(), event, data }] : priorEvents;
|
|
1980
|
+
return decisionRow ? reduced : { ...reduced, owedChecks: foldOwedChecks(folded, join(this.dir, "..", "..", "..")) };
|
|
1947
1981
|
})()
|
|
1948
1982
|
: event === "resume-restore" && rowTaskId && upheldFeedbackByTask(effectiveEvents(priorEvents)).has(rowTaskId)
|
|
1949
1983
|
? {
|
|
@@ -2701,12 +2735,16 @@ function dischargeProblem(row, o, events, cwd, memo) {
|
|
|
2701
2735
|
* across folds; every artifact, hash, criteria and reviewer condition is still re-read on every fold.
|
|
2702
2736
|
*/
|
|
2703
2737
|
export function foldOwedChecks(events, cwd, memo) {
|
|
2738
|
+
let fold;
|
|
2704
2739
|
try {
|
|
2705
|
-
|
|
2740
|
+
fold = foldOwedChecksOrThrow(events, cwd, memo);
|
|
2706
2741
|
}
|
|
2707
2742
|
catch (error) {
|
|
2708
|
-
|
|
2743
|
+
fold = { known: false, debt: "unknown", outstanding: [], acceptedRisk: [], discharged: [], unknown: [{ reason: `owed-check fold failed: ${String(error).split("\n")[0]}` }] };
|
|
2709
2744
|
}
|
|
2745
|
+
// A fatal run-end never finished its checks: its debt is unknown and the crash leads the reasons, never 0.
|
|
2746
|
+
const fatal = fatalRunEndCause(events);
|
|
2747
|
+
return fatal ? { ...fold, known: false, debt: "unknown", unknown: [{ reason: `fatal run-end — ${fatal}` }, ...fold.unknown] } : fold;
|
|
2710
2748
|
}
|
|
2711
2749
|
function foldOwedChecksOrThrow(events, cwd, memo) {
|
|
2712
2750
|
// v2.6.7 T4 (G2): checkout trust is re-derived at the fold boundary, before a memoised proof can answer —
|
package/dist/run/lease.d.ts
CHANGED
|
@@ -35,11 +35,19 @@ export interface RepositoryLeaseHolder {
|
|
|
35
35
|
roots?: number[];
|
|
36
36
|
rootBirths?: Record<string, string>;
|
|
37
37
|
}
|
|
38
|
+
/** Queue row 120: a live process keeping a dead or released holder's lease, as the wait names it: pid, executable and
|
|
39
|
+
* cwd. Its arguments and environment are never printed or read (`ps -o comm=`, the census's cwd-only probe). */
|
|
40
|
+
export interface LeaseBlocker {
|
|
41
|
+
pid: number;
|
|
42
|
+
executable: string;
|
|
43
|
+
cwd?: string;
|
|
44
|
+
}
|
|
38
45
|
export interface RepositoryLeaseOptions {
|
|
39
46
|
pollMs?: number;
|
|
40
47
|
signal?: AbortSignal;
|
|
41
|
-
/** Called once per distinct holder the waiter is queued behind
|
|
42
|
-
|
|
48
|
+
/** Called once per distinct holder the waiter is queued behind, and again whenever the set of processes keeping a
|
|
49
|
+
* dead holder's lease changes; `blockers` names that set (queue row 120). */
|
|
50
|
+
onWait?: (holder: RepositoryLeaseHolder, blockers?: readonly LeaseBlocker[]) => void;
|
|
43
51
|
/** The token a descendant inherited; defaults to REPOSITORY_LEASE_TOKEN_ENV. */
|
|
44
52
|
inherited?: string;
|
|
45
53
|
/** Manifest jobs bridge capabilities through explicit child environments, never process.env. */
|
|
@@ -82,6 +90,15 @@ export declare const readHolder: (path: string) => RepositoryLeaseHolder | undef
|
|
|
82
90
|
export declare function tryReserve(path: string, mine: RepositoryLeaseHolder): boolean;
|
|
83
91
|
/** Exposed for recovery regressions; the standalone verify uses the combined critical section. */
|
|
84
92
|
export declare const reclaimDead: (path: string, identity: string, pollMs?: number) => Promise<boolean>;
|
|
93
|
+
/** Each blocker's pid, executable and working directory, and NO argument: `ps -o comm=` (a command line joins argv with
|
|
94
|
+
* spaces, so no redaction could tell where a secret value ends, D-1710). The cwd comes from the census's cwd-only probe
|
|
95
|
+
* (async, batched, never rejecting, no environment read; queue row 103: no synchronous lsof on a waiting loop), imported
|
|
96
|
+
* lazily because suite-census imports this module. Diagnostics only: an unreadable process reads "(executable
|
|
97
|
+
* unreadable)", and this never throws into the wait. */
|
|
98
|
+
export declare function describeLeaseBlockers(pids: readonly number[]): Promise<LeaseBlocker[]>;
|
|
99
|
+
/** The wait's own words: the holder alone while it lives; once it has exited or released the lease, every process still
|
|
100
|
+
* keeping it. */
|
|
101
|
+
export declare function describeLeaseWait(holder: RepositoryLeaseHolder, blockers?: readonly LeaseBlocker[]): string;
|
|
85
102
|
export declare function protectedProcesses(holder: RepositoryLeaseHolder, tokenEnv?: string, ownedOnly?: boolean): Promise<number[]>;
|
|
86
103
|
export declare const repositoryLeaseEnvironment: (env: NodeJS.ProcessEnv) => NodeJS.ProcessEnv;
|
|
87
104
|
/** Called synchronously at spawn, before any user callback, and only on an owned generation. */
|
package/dist/run/lease.js
CHANGED
|
@@ -305,14 +305,47 @@ const reclaimDeadLocked = (path, identity) => {
|
|
|
305
305
|
export const reclaimDead = async (path, identity, pollMs = 10) => withMutationLock(path, pollMs, undefined, () => reclaimDeadLocked(path, identity));
|
|
306
306
|
const inspectAndReserve = async (path, mine) => {
|
|
307
307
|
let occupant = readOccupant(path);
|
|
308
|
-
|
|
308
|
+
const dead = occupant?.holder !== undefined && !holderAlive(occupant.holder);
|
|
309
|
+
// A dead holder's protected tree is what keeps the lease; the waiter is told which processes those are (queue row 120).
|
|
310
|
+
const blocking = dead && occupant.holder.protectOrphans ? await protectedProcesses(occupant.holder) : [];
|
|
311
|
+
if (occupant && (!occupant.holder || (dead && !blocking.length))) {
|
|
309
312
|
reclaimDeadLocked(path, occupant.identity);
|
|
310
313
|
occupant = readOccupant(path);
|
|
311
314
|
}
|
|
312
315
|
if (!occupant && tryReserve(path, mine))
|
|
313
316
|
return { acquired: true };
|
|
314
|
-
return { acquired: false, holder: occupant?.holder };
|
|
317
|
+
return { acquired: false, holder: occupant?.holder, ...(blocking.length ? { blocking } : {}) };
|
|
315
318
|
};
|
|
319
|
+
/** Each blocker's pid, executable and working directory, and NO argument: `ps -o comm=` (a command line joins argv with
|
|
320
|
+
* spaces, so no redaction could tell where a secret value ends, D-1710). The cwd comes from the census's cwd-only probe
|
|
321
|
+
* (async, batched, never rejecting, no environment read; queue row 103: no synchronous lsof on a waiting loop), imported
|
|
322
|
+
* lazily because suite-census imports this module. Diagnostics only: an unreadable process reads "(executable
|
|
323
|
+
* unreadable)", and this never throws into the wait. */
|
|
324
|
+
export async function describeLeaseBlockers(pids) {
|
|
325
|
+
const rows = await new Promise((done) => {
|
|
326
|
+
try {
|
|
327
|
+
execFile("ps", ["-o", "pid=,comm=", "-p", pids.join(",")], { encoding: "utf8", timeout: 5_000 }, (_error, stdout) => done(stdout ?? ""));
|
|
328
|
+
}
|
|
329
|
+
catch {
|
|
330
|
+
done("");
|
|
331
|
+
}
|
|
332
|
+
});
|
|
333
|
+
const executables = new Map(rows.split("\n").flatMap((line) => { const row = /^\s*(\d+)\s+(.*)$/.exec(line); return row ? [[Number(row[1]), row[2].trim()]] : []; }));
|
|
334
|
+
const cwds = await import("./suite-census.js").then(({ batchedProcessCwds }) => batchedProcessCwds(pids)).catch(() => undefined);
|
|
335
|
+
return pids.map((pid) => {
|
|
336
|
+
const cwd = cwds?.get(pid);
|
|
337
|
+
return { pid, executable: executables.get(pid) || "(executable unreadable)", ...(cwd ? { cwd } : {}) };
|
|
338
|
+
});
|
|
339
|
+
}
|
|
340
|
+
/** The wait's own words: the holder alone while it lives; once it has exited or released the lease, every process still
|
|
341
|
+
* keeping it. */
|
|
342
|
+
export function describeLeaseWait(holder, blockers) {
|
|
343
|
+
const held = `held by pid ${holder.pid} in ${holder.cwd}`;
|
|
344
|
+
if (!blockers?.length)
|
|
345
|
+
return held;
|
|
346
|
+
const named = blockers.map((b) => `pid ${b.pid} ${b.executable}${b.cwd ? ` (cwd ${b.cwd})` : ""}`).join("; ");
|
|
347
|
+
return `${held}, which has exited or released it; ${blockers.length} process(es) it started still run and keep the lease until they exit: ${named}`;
|
|
348
|
+
}
|
|
316
349
|
// A durable capability is also a crash owner: census checks the exact generation's inherited
|
|
317
350
|
// environment and recorded process groups, including live orphans reparented after owner death.
|
|
318
351
|
// An unreadable census refuses reclamation/release. No dead-pid shortcut admits over a live tree.
|
|
@@ -386,6 +419,8 @@ export function registerRepositoryChild(pid) {
|
|
|
386
419
|
}
|
|
387
420
|
export const withFreshCommandLease = (command, run) => ownership.run({ token: "", active: false }, () => withCommandLease(command, run));
|
|
388
421
|
const repositoryQueues = new Map();
|
|
422
|
+
/** Queue row 120: the queue head's last view of what keeps a dead holder's lease, so in-process followers name it too. */
|
|
423
|
+
const repositoryBlocking = new Map();
|
|
389
424
|
/** The standalone wrapper uses the same command-then-repository order when a scheduler exists.
|
|
390
425
|
* Without one (CLI/global setup), this remains exactly the outer file reservation API. */
|
|
391
426
|
export function withRepositoryLease(cwd, run, opts = {}) {
|
|
@@ -426,10 +461,11 @@ async function repositoryLease(cwd, run, opts) {
|
|
|
426
461
|
const ticket = Symbol("repository job");
|
|
427
462
|
queue.add(ticket);
|
|
428
463
|
let waitingOn;
|
|
429
|
-
const waitOn = (holder) => {
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
464
|
+
const waitOn = async (holder, blocking = []) => {
|
|
465
|
+
const key = holder && `${holder.token}\0${[...blocking].sort((a, b) => a - b).join(",")}`;
|
|
466
|
+
if (holder && key !== waitingOn) {
|
|
467
|
+
waitingOn = key;
|
|
468
|
+
opts.onWait?.(holder, blocking.length ? await describeLeaseBlockers(blocking) : undefined);
|
|
433
469
|
}
|
|
434
470
|
};
|
|
435
471
|
try {
|
|
@@ -437,12 +473,19 @@ async function repositoryLease(cwd, run, opts) {
|
|
|
437
473
|
opts.signal?.throwIfAborted();
|
|
438
474
|
if (queue.values().next().value === ticket) {
|
|
439
475
|
const attempt = await withMutationLock(path, pollMs, opts.signal, () => inspectAndReserve(path, mine));
|
|
440
|
-
if (attempt.acquired)
|
|
476
|
+
if (attempt.acquired) {
|
|
477
|
+
repositoryBlocking.delete(path);
|
|
441
478
|
break;
|
|
442
|
-
|
|
479
|
+
}
|
|
480
|
+
if (attempt.holder)
|
|
481
|
+
repositoryBlocking.set(path, { token: attempt.holder.token, blocking: attempt.blocking ?? [] });
|
|
482
|
+
await waitOn(attempt.holder, attempt.blocking);
|
|
483
|
+
}
|
|
484
|
+
else {
|
|
485
|
+
const holder = readHolder(path);
|
|
486
|
+
const seen = repositoryBlocking.get(path);
|
|
487
|
+
await waitOn(holder, holder && seen?.token === holder.token ? seen.blocking : []);
|
|
443
488
|
}
|
|
444
|
-
else
|
|
445
|
-
waitOn(readHolder(path));
|
|
446
489
|
await new Promise(wake => setTimeout(wake, pollMs));
|
|
447
490
|
}
|
|
448
491
|
const exported = process.env[REPOSITORY_LEASE_TOKEN_ENV];
|
|
@@ -495,8 +538,10 @@ async function repositoryLease(cwd, run, opts) {
|
|
|
495
538
|
}
|
|
496
539
|
finally {
|
|
497
540
|
queue.delete(ticket);
|
|
498
|
-
if (!queue.size)
|
|
541
|
+
if (!queue.size) {
|
|
499
542
|
repositoryQueues.delete(path);
|
|
543
|
+
repositoryBlocking.delete(path);
|
|
544
|
+
}
|
|
500
545
|
// A waiter removes no repository generation, including cancellation before acquisition.
|
|
501
546
|
}
|
|
502
547
|
}
|
|
@@ -6,6 +6,9 @@
|
|
|
6
6
|
* not already probed — the census asks for a parent's cwd too. The answers feed countLiveSuites' existing lookups.
|
|
7
7
|
*/
|
|
8
8
|
export type ProbeExec = (file: string, args: readonly string[]) => Promise<string>;
|
|
9
|
+
/** Each pid's working directory: /proc first, then ONE batched lsof for the rest; never rejects, and reads no process
|
|
10
|
+
* environment (the runner-lease wait names blockers through it, queue row 120). Answers land in `cwds`. */
|
|
11
|
+
export declare function batchedProcessCwds(pids: readonly number[], exec?: ProbeExec, proc?: ProcReader, cwds?: Map<number, string>): Promise<Map<number, string>>;
|
|
9
12
|
/** The pids of every live (non-zombie) row of a `ps -o pid=,ppid=,state=,command=` snapshot whose command is a runner. */
|
|
10
13
|
export declare function runnerPids(snapshot: string): number[];
|
|
11
14
|
/** The /proc reads, injectable so the Linux branch is testable anywhere; each throws where /proc has no entry. */
|
package/dist/run/suite-census.js
CHANGED
|
@@ -16,6 +16,42 @@ const probeExec = (file, args) => new Promise((resolve) => {
|
|
|
16
16
|
resolve("");
|
|
17
17
|
}
|
|
18
18
|
});
|
|
19
|
+
// D-1567 batch poisoning: lsof and ps fail the WHOLE command for one -p member they cannot take (a marker of 22 digits
|
|
20
|
+
// prints as 1e+21, 310 nines as Infinity; ps refuses one above int32), erasing every other pid's answer. The base probed
|
|
21
|
+
// per pid, so such a pid lost only its own answer: it stays unanswered here and never joins a batch.
|
|
22
|
+
const probeable = (pid) => Number.isSafeInteger(pid) && pid > 0 && pid <= 2_147_483_647;
|
|
23
|
+
/** Each pid's working directory: /proc first, then ONE batched lsof for the rest; never rejects, and reads no process
|
|
24
|
+
* environment (the runner-lease wait names blockers through it, queue row 120). Answers land in `cwds`. */
|
|
25
|
+
export async function batchedProcessCwds(pids, exec = probeExec, proc = procReader, cwds = new Map()) {
|
|
26
|
+
const rest = [];
|
|
27
|
+
for (const pid of pids.filter(probeable)) {
|
|
28
|
+
try {
|
|
29
|
+
cwds.set(pid, proc.cwd(pid));
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
rest.push(pid);
|
|
33
|
+
} // Darwin has no /proc
|
|
34
|
+
}
|
|
35
|
+
if (rest.length === 0)
|
|
36
|
+
return cwds;
|
|
37
|
+
let listing = "";
|
|
38
|
+
try {
|
|
39
|
+
listing = await exec("lsof", ["-a", "-p", rest.join(","), "-d", "cwd", "-Fn"]);
|
|
40
|
+
}
|
|
41
|
+
catch { /* the census never rejects */ }
|
|
42
|
+
let pid;
|
|
43
|
+
for (const line of listing.split("\n")) {
|
|
44
|
+
if (line.startsWith("p"))
|
|
45
|
+
pid = Number(line.slice(1));
|
|
46
|
+
else if (line.startsWith("n") && pid !== undefined) {
|
|
47
|
+
try {
|
|
48
|
+
cwds.set(pid, realpathSync(line.slice(1)));
|
|
49
|
+
}
|
|
50
|
+
catch { /* the directory is gone */ }
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return cwds;
|
|
54
|
+
}
|
|
19
55
|
/** The pids of every live (non-zombie) row of a `ps -o pid=,ppid=,state=,command=` snapshot whose command is a runner. */
|
|
20
56
|
export function runnerPids(snapshot) {
|
|
21
57
|
const pids = [];
|
|
@@ -41,34 +77,7 @@ export async function batchedProcessProbes(pids, exec = probeExec, proc = procRe
|
|
|
41
77
|
const cwds = new Map();
|
|
42
78
|
const parents = new Map();
|
|
43
79
|
const parentIn = (text) => new RegExp(`(?:^|\\s)${SUITE_PARENT_ENV}=(\\d+)(?:\\s|$)`).exec(text)?.[1];
|
|
44
|
-
|
|
45
|
-
// prints as 1e+21, 310 nines as Infinity; ps refuses one above int32), erasing every other pid's answer. The base probed
|
|
46
|
-
// per pid, so such a pid lost only its own answer: it stays unanswered here and never joins a batch.
|
|
47
|
-
const probeable = (pid) => Number.isSafeInteger(pid) && pid > 0 && pid <= 2_147_483_647;
|
|
48
|
-
const probeCwds = async (wanted) => {
|
|
49
|
-
const rest = [];
|
|
50
|
-
for (const pid of wanted.filter(probeable)) {
|
|
51
|
-
try {
|
|
52
|
-
cwds.set(pid, proc.cwd(pid));
|
|
53
|
-
}
|
|
54
|
-
catch {
|
|
55
|
-
rest.push(pid);
|
|
56
|
-
} // Darwin has no /proc
|
|
57
|
-
}
|
|
58
|
-
if (rest.length === 0)
|
|
59
|
-
return;
|
|
60
|
-
let pid;
|
|
61
|
-
for (const line of (await ask("lsof", ["-a", "-p", rest.join(","), "-d", "cwd", "-Fn"])).split("\n")) {
|
|
62
|
-
if (line.startsWith("p"))
|
|
63
|
-
pid = Number(line.slice(1));
|
|
64
|
-
else if (line.startsWith("n") && pid !== undefined) {
|
|
65
|
-
try {
|
|
66
|
-
cwds.set(pid, realpathSync(line.slice(1)));
|
|
67
|
-
}
|
|
68
|
-
catch { /* the directory is gone */ }
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
};
|
|
80
|
+
const probeCwds = (wanted) => batchedProcessCwds(wanted, ask, proc, cwds);
|
|
72
81
|
const probeParents = async (wanted) => {
|
|
73
82
|
const rest = [];
|
|
74
83
|
for (const pid of wanted.filter(probeable)) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ShellReceipt } from "./protocol.js";
|
|
2
2
|
import type { GateEvidenceOptions } from "../gates/baseline.js";
|
|
3
|
-
import { type RepositoryLeaseHolder } from "./lease.js";
|
|
3
|
+
import { type LeaseBlocker, type RepositoryLeaseHolder } from "./lease.js";
|
|
4
4
|
export interface VerificationPhase {
|
|
5
5
|
phase: "discovery" | "first-pass" | "continuation-discovery" | "continuation";
|
|
6
6
|
nonce: string;
|
|
@@ -33,7 +33,9 @@ export interface VerificationJobReport {
|
|
|
33
33
|
releasedAt?: number;
|
|
34
34
|
reservationReused?: boolean;
|
|
35
35
|
reservation?: RepositoryLeaseHolder;
|
|
36
|
-
waitingOn?: RepositoryLeaseHolder
|
|
36
|
+
waitingOn?: RepositoryLeaseHolder & {
|
|
37
|
+
blockers?: readonly LeaseBlocker[];
|
|
38
|
+
};
|
|
37
39
|
state: "queued" | "running" | "completed" | "failed" | "cancelled";
|
|
38
40
|
phases: VerificationPhase[];
|
|
39
41
|
}
|
|
@@ -132,7 +132,7 @@ export async function withVerificationJob(command, cwd, reportPath, evidence, ru
|
|
|
132
132
|
}
|
|
133
133
|
}));
|
|
134
134
|
}, { isolated: true, protectOrphans: true, independent: !!jobs.getStore(), signal, pollMs: 20,
|
|
135
|
-
onWait: holder => { report.waitingOn = { ...holder }; job.persist(); },
|
|
135
|
+
onWait: (holder, blockers) => { report.waitingOn = { ...holder, ...(blockers ? { blockers } : {}) }; job.persist(); },
|
|
136
136
|
onAdmission: holder => { owner = holder; report.repositoryAdmittedAt = clock(); report.reservation = { ...holder }; job.persist(); },
|
|
137
137
|
onRelease: () => { report.reapedAt ??= clock(); report.repositoryReleasedAt = clock(); job.persist(); } }));
|
|
138
138
|
report.releasedAt = clock();
|
package/package.json
CHANGED
|
@@ -70,6 +70,18 @@ since_arm() { tail -n +$((base + 1)) "$1" 2>/dev/null; }
|
|
|
70
70
|
|
|
71
71
|
field() { printf '%s' "$2" | sed -n "s/.*\"$1\":\"\([^\"]*\)\".*/\1/p" | head -1; }
|
|
72
72
|
bucket() { printf '%s' "$2" | sed -n "s/.*\"$1\":\[\([^]]*\)\].*/\1/p" | head -1; }
|
|
73
|
+
# A row's data string value DECODED by JSON.parse (an escaped `\"` never cuts a quoted ref in a git error
|
|
74
|
+
# short), printed on one line as the product's own crash reason is (visibleControls): every C0 control
|
|
75
|
+
# but tab and newline, DEL and every C1 control prints as its visible escape (`\x1b`, `\u009b`), so a
|
|
76
|
+
# recorded 7-bit or 8-bit OSC/CSI/ST never reaches the terminal; tabs and line breaks collapse to a space.
|
|
77
|
+
jstr() {
|
|
78
|
+
printf '%s' "$2" | node -e '
|
|
79
|
+
let v; try { const row = JSON.parse(require("fs").readFileSync(0, "utf8")); v = (row.data ?? row)[process.argv[1]]; } catch {}
|
|
80
|
+
if (typeof v === "string") process.stdout.write(v.replace(/[\x00-\x08\x0b-\x1f\x7f-\x9f]/g, (c) => {
|
|
81
|
+
const h = c.charCodeAt(0).toString(16);
|
|
82
|
+
return c < "\x80" ? "\\x" + h.padStart(2, "0") : "\\u" + h.padStart(4, "0");
|
|
83
|
+
}).replace(/\s+/g, " ").trim());' "$1" 2>/dev/null
|
|
84
|
+
}
|
|
73
85
|
|
|
74
86
|
report() {
|
|
75
87
|
# $3 is the row's physical journal line: with its ts it is the token a decision binds to (OBS-1178)
|
|
@@ -77,8 +89,13 @@ report() {
|
|
|
77
89
|
ev=$(field event "$line")
|
|
78
90
|
case "$ev" in
|
|
79
91
|
run-end)
|
|
80
|
-
local tv done failed human blocked pending verdict
|
|
92
|
+
local tv done failed human blocked pending verdict fatal=""
|
|
81
93
|
tv=$(field tipVerify "$line")
|
|
94
|
+
# A FATAL run-end is a crash, never green — even with every bucket empty (its tasks never finished).
|
|
95
|
+
case "$line" in *'"fatal":true'*)
|
|
96
|
+
local err; err=$(jstr error "$line")
|
|
97
|
+
fatal="$(jstr phase "$line") failed: ${err:0:160}" ;;
|
|
98
|
+
esac
|
|
82
99
|
done=$(bucket done "$line"); failed=$(bucket failed "$line")
|
|
83
100
|
human=$(bucket human "$line"); blocked=$(bucket blocked "$line")
|
|
84
101
|
pending=$(bucket pending "$line")
|
|
@@ -87,16 +104,20 @@ report() {
|
|
|
87
104
|
# clause open: a waived review is still owed (D-660), and a `verify --record` discharge that lands
|
|
88
105
|
# AFTER run-end moves only CURRENT status. This row is history, so the best it can say is
|
|
89
106
|
# EXECUTION COMPLETE; green is read from CURRENT `tickmarkr status <runId>` at `outstanding 0`.
|
|
90
|
-
if [ "$tv" != "failed" ] && [ -z "$failed$human$blocked$pending" ]; then
|
|
107
|
+
if [ -z "$fatal" ] && [ "$tv" != "failed" ] && [ -z "$failed$human$blocked$pending" ]; then
|
|
91
108
|
verdict="EXECUTION COMPLETE"
|
|
92
109
|
else
|
|
93
110
|
verdict="NOT GREEN"
|
|
94
111
|
fi
|
|
95
|
-
echo "RUN_END $run — $verdict (tipVerify=${tv:-unknown})"
|
|
112
|
+
echo "RUN_END $run — $verdict (${fatal:+fatal: $fatal; }tipVerify=${tv:-unknown})"
|
|
96
113
|
echo " done=[${done}] failed=[${failed}] human=[${human}] blocked=[${blocked}] pending=[${pending}]"
|
|
97
|
-
[ "$
|
|
98
|
-
|
|
99
|
-
|
|
114
|
+
if [ -n "$fatal" ]; then
|
|
115
|
+
echo " the run CRASHED ($fatal) — a fatal run-end is never green; its debt reads unknown until the run is resumed, and from then on the resumed engagement owns it"
|
|
116
|
+
elif [ "$verdict" = "EXECUTION COMPLETE" ]; then
|
|
117
|
+
echo " execution buckets empty and tip verify is not failed — not yet green: read CURRENT \`tickmarkr status $run\`; only \`outstanding 0\` is green, \`outstanding N (...)\` and \`outstanding unknown\` are not, and this run-end record's own count is historical"
|
|
118
|
+
else
|
|
119
|
+
echo " a non-empty bucket or a failed tip above is the reason; name it, never report this run as green"
|
|
120
|
+
fi
|
|
100
121
|
;;
|
|
101
122
|
task-human)
|
|
102
123
|
echo "TASK_HUMAN $(field taskId "$line") — $run"
|
|
@@ -15,15 +15,23 @@
|
|
|
15
15
|
# never hours; 900 is a generous default for a 7-task spec)
|
|
16
16
|
# <overseer-pane> the pane that must hear about it (herdr pane id), e.g. wZ:p18S
|
|
17
17
|
# [poll-s] poll interval, default 15
|
|
18
|
-
# exit 0 LAUNCH_OK (lock
|
|
18
|
+
# exit 0 LAUNCH_OK (a lock held by a LIVE pid; prints its contents) · exit 3 LAUNCH_OVERDUE (delivered) · exit 64 usage
|
|
19
|
+
#
|
|
20
|
+
# A lock is a launch only while its holder lives: a lock left by a crashed daemon (dead pid, or no pid
|
|
21
|
+
# at all) is stale, proves nothing about the new GO, and the watch keeps waiting for a live holder.
|
|
22
|
+
# `ps -p` answers for another user's process too, so EPERM never reads as death (lock.ts isPidLive).
|
|
19
23
|
set -u
|
|
20
24
|
LOCK="${1:-}"; DEADLINE="${2:-}"; PANE="${3:-}"; POLL="${4:-15}"
|
|
21
25
|
[ -n "$LOCK" ] && [ -n "$DEADLINE" ] && [ -n "$PANE" ] || { echo "usage: watch-launch.sh <lock-path> <deadline-s> <overseer-pane> [poll-s]" >&2; exit 64; }
|
|
22
26
|
start=$(date +%s)
|
|
23
27
|
while :; do
|
|
24
28
|
if [ -f "$LOCK" ]; then
|
|
25
|
-
|
|
26
|
-
|
|
29
|
+
body=$(tr -d '\n' < "$LOCK" 2>/dev/null)
|
|
30
|
+
pid=$(printf '%s' "$body" | sed -n 's/.*"pid":\([0-9][0-9]*\).*/\1/p')
|
|
31
|
+
if [ -n "$pid" ] && ps -p "$pid" >/dev/null 2>&1; then
|
|
32
|
+
printf 'LAUNCH_OK %s %s\n' "$(date -u +%H:%M:%SZ)" "$body"
|
|
33
|
+
exit 0
|
|
34
|
+
fi
|
|
27
35
|
fi
|
|
28
36
|
now=$(date +%s)
|
|
29
37
|
if [ $((now - start)) -ge "$DEADLINE" ]; then
|