sortie-dogs 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -5
- package/dist/asset-version.d.ts +1 -1
- package/dist/asset-version.js +1 -1
- package/dist/core/retained-state.d.ts +54 -0
- package/dist/core/retained-state.js +81 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +1 -0
- package/dist/plugin/continuation.js +15 -4
- package/dist/plugin/index.js +44 -4
- package/dist/plugin/run-metrics.d.ts +19 -0
- package/dist/plugin/run-metrics.js +136 -15
- package/dist/runtime-assets.d.ts +9 -9
- package/dist/runtime-assets.js +95 -31
- package/package.json +7 -3
package/README.md
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
# Sortie-dogs
|
|
2
2
|
|
|
3
|
-
**Give OpenCode a task; get a bounded, validated implementation loop instead of an open-ended agent run.**
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
**Give OpenCode a task; get a bounded, validated implementation loop instead of an open-ended agent run.**
|
|
4
|
+
|
|
5
|
+
Sortie-dogs is an execution harness for OpenCode, applying harness-engineering
|
|
6
|
+
principles through mechanical write boundaries, validation, review, and bounded
|
|
7
|
+
orchestration.
|
|
8
|
+
|
|
9
|
+
> **Project status: Beta.** Stable in regular use. As a pre-1.0 release,
|
|
10
|
+
> configuration and runtime assets may still change between releases.
|
|
7
11
|
|
|
8
12
|
[](https://www.npmjs.com/package/sortie-dogs)
|
|
9
13
|
[](LICENSE)
|
|
@@ -20,7 +24,7 @@ Requirements: Node.js 22.6 or newer, npm, and OpenCode.
|
|
|
20
24
|
|
|
21
25
|
Guides: [日本語](docs/guide-ja.md) · [简体中文](docs/guide-zh-CN.md) · [CLI testing](docs/cli-testing.md)
|
|
22
26
|
|
|
23
|
-
Release: [v0.
|
|
27
|
+
Release: [v0.7.0](https://github.com/zufall-upon/Sortie-dogs/releases/tag/v0.7.0)
|
|
24
28
|
|
|
25
29
|
## Quick start
|
|
26
30
|
|
package/dist/asset-version.d.ts
CHANGED
|
@@ -2,5 +2,5 @@
|
|
|
2
2
|
* Version of the installable runtime assets. Kept in its own module so the plugin can compare an
|
|
3
3
|
* installed project marker without importing every asset body.
|
|
4
4
|
*/
|
|
5
|
-
export declare const RUNTIME_ASSET_VERSION = "0.3.
|
|
5
|
+
export declare const RUNTIME_ASSET_VERSION = "0.3.51-state-observability-v1";
|
|
6
6
|
export type RuntimeAssetVersion = typeof RUNTIME_ASSET_VERSION;
|
package/dist/asset-version.js
CHANGED
|
@@ -2,4 +2,4 @@
|
|
|
2
2
|
* Version of the installable runtime assets. Kept in its own module so the plugin can compare an
|
|
3
3
|
* installed project marker without importing every asset body.
|
|
4
4
|
*/
|
|
5
|
-
export const RUNTIME_ASSET_VERSION = "0.3.
|
|
5
|
+
export const RUNTIME_ASSET_VERSION = "0.3.51-state-observability-v1";
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
export declare const RETAINED_STATE_EXTENSION: "sortie-dogs/retained-state";
|
|
2
|
+
export declare const RETAINED_STATE_SCHEMA_VERSION: "0.1";
|
|
3
|
+
export declare const RETAINED_STATE_AUTHORITY: "shadow";
|
|
4
|
+
export declare const MAX_RETAINED_STATE_BYTES: number;
|
|
5
|
+
export declare const MAX_RETAINED_STATE_ITEMS = 12;
|
|
6
|
+
export declare const MAX_RETAINED_STATE_WARNINGS = 8;
|
|
7
|
+
export interface RetainedValidationAttempt {
|
|
8
|
+
readonly command: string;
|
|
9
|
+
readonly exit: number | null;
|
|
10
|
+
readonly fingerprint: string;
|
|
11
|
+
}
|
|
12
|
+
export interface NextEvidenceDecision {
|
|
13
|
+
readonly schema_version: "0.1";
|
|
14
|
+
readonly authority: "shadow";
|
|
15
|
+
readonly gap_id: string;
|
|
16
|
+
readonly blocked_acceptance: string;
|
|
17
|
+
readonly question: string;
|
|
18
|
+
readonly expected_discrimination: string;
|
|
19
|
+
readonly action: string;
|
|
20
|
+
readonly stop_condition: string;
|
|
21
|
+
}
|
|
22
|
+
export interface AdmissionReceipt {
|
|
23
|
+
readonly evidence_id: string;
|
|
24
|
+
readonly source_agent: string;
|
|
25
|
+
readonly source_revision: string;
|
|
26
|
+
readonly evidence_fingerprint: string;
|
|
27
|
+
readonly supports: readonly string[];
|
|
28
|
+
readonly contradicts: readonly string[];
|
|
29
|
+
readonly freshness_basis: string;
|
|
30
|
+
readonly status: "recorded" | "recorded_with_warnings";
|
|
31
|
+
readonly warnings: readonly string[];
|
|
32
|
+
}
|
|
33
|
+
export interface RetainedStateCapsule {
|
|
34
|
+
readonly schema_version: "0.1";
|
|
35
|
+
readonly authority: "shadow";
|
|
36
|
+
readonly task_id: string;
|
|
37
|
+
readonly acceptance_fingerprint: string;
|
|
38
|
+
readonly source_manifest: "none" | readonly string[];
|
|
39
|
+
readonly operation_manifest: "none" | string;
|
|
40
|
+
readonly validation_history: readonly RetainedValidationAttempt[];
|
|
41
|
+
readonly blockers: readonly string[];
|
|
42
|
+
readonly next_action: string;
|
|
43
|
+
readonly next_evidence_decision?: NextEvidenceDecision;
|
|
44
|
+
readonly admissions?: readonly AdmissionReceipt[];
|
|
45
|
+
}
|
|
46
|
+
export interface RetainedStateWarning {
|
|
47
|
+
readonly code: "malformed" | "oversize" | "optional_invalid";
|
|
48
|
+
readonly message: string;
|
|
49
|
+
}
|
|
50
|
+
export interface RetainedStateInspection {
|
|
51
|
+
readonly capsule: RetainedStateCapsule | undefined;
|
|
52
|
+
readonly warnings: readonly RetainedStateWarning[];
|
|
53
|
+
}
|
|
54
|
+
export declare function inspectRetainedStateCapsule(handoff: unknown): RetainedStateInspection;
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
export const RETAINED_STATE_EXTENSION = "sortie-dogs/retained-state";
|
|
2
|
+
export const RETAINED_STATE_SCHEMA_VERSION = "0.1";
|
|
3
|
+
export const RETAINED_STATE_AUTHORITY = "shadow";
|
|
4
|
+
export const MAX_RETAINED_STATE_BYTES = 16 * 1024;
|
|
5
|
+
export const MAX_RETAINED_STATE_ITEMS = 12;
|
|
6
|
+
export const MAX_RETAINED_STATE_WARNINGS = 8;
|
|
7
|
+
const MAX_SHORT = 256;
|
|
8
|
+
const MAX_TEXT = 1000;
|
|
9
|
+
const isObject = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
10
|
+
const isText = (value, max = MAX_TEXT) => typeof value === "string" && value.length > 0 && value.length <= max;
|
|
11
|
+
const isBoundedList = (value, max = MAX_RETAINED_STATE_ITEMS) => Array.isArray(value) && value.length <= max;
|
|
12
|
+
const isSourceManifest = (value) => value === "none" || (isBoundedList(value) && value.length === new Set(value).size && value.every((item) => isText(item, 512)));
|
|
13
|
+
function validAttempt(value) {
|
|
14
|
+
const exit = isObject(value) ? value.exit : undefined;
|
|
15
|
+
return isObject(value) && isText(value.command, 1000) && (exit === null || (typeof exit === "number" && Number.isInteger(exit) && exit >= -1 && exit <= 255)) && isText(value.fingerprint, MAX_SHORT);
|
|
16
|
+
}
|
|
17
|
+
function validDecision(value) {
|
|
18
|
+
return isObject(value) && value.schema_version === "0.1" && value.authority === "shadow" &&
|
|
19
|
+
isText(value.gap_id, MAX_SHORT) && isText(value.blocked_acceptance) && isText(value.question) &&
|
|
20
|
+
isText(value.expected_discrimination) && isText(value.action) && isText(value.stop_condition);
|
|
21
|
+
}
|
|
22
|
+
function validAdmission(value) {
|
|
23
|
+
return isObject(value) && isText(value.evidence_id, MAX_SHORT) && isText(value.source_agent, MAX_SHORT) &&
|
|
24
|
+
isText(value.source_revision, MAX_SHORT) && isText(value.evidence_fingerprint, MAX_SHORT) &&
|
|
25
|
+
isBoundedList(value.supports) && value.supports.every((item) => isText(item, MAX_SHORT)) &&
|
|
26
|
+
isBoundedList(value.contradicts) && value.contradicts.every((item) => isText(item, MAX_SHORT)) &&
|
|
27
|
+
isText(value.freshness_basis) && (value.status === "recorded" || value.status === "recorded_with_warnings") &&
|
|
28
|
+
isBoundedList(value.warnings, MAX_RETAINED_STATE_WARNINGS) && value.warnings.every((item) => isText(item, MAX_SHORT));
|
|
29
|
+
}
|
|
30
|
+
export function inspectRetainedStateCapsule(handoff) {
|
|
31
|
+
if (!isObject(handoff) || !isObject(handoff.ext) || !(RETAINED_STATE_EXTENSION in handoff.ext)) {
|
|
32
|
+
return { capsule: undefined, warnings: [] };
|
|
33
|
+
}
|
|
34
|
+
const raw = handoff.ext[RETAINED_STATE_EXTENSION];
|
|
35
|
+
let bytes;
|
|
36
|
+
try {
|
|
37
|
+
bytes = new TextEncoder().encode(JSON.stringify(raw)).byteLength;
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
return { capsule: undefined, warnings: [{ code: "malformed", message: "Retained state was ignored because it is malformed." }] };
|
|
41
|
+
}
|
|
42
|
+
if (bytes > MAX_RETAINED_STATE_BYTES)
|
|
43
|
+
return { capsule: undefined, warnings: [{ code: "oversize", message: "Retained state was ignored because it exceeds its bound." }] };
|
|
44
|
+
if (!isObject(raw) || raw.schema_version !== "0.1" || raw.authority !== "shadow" || !isText(raw.task_id, MAX_SHORT) ||
|
|
45
|
+
!isText(raw.acceptance_fingerprint, MAX_SHORT) || !isSourceManifest(raw.source_manifest) ||
|
|
46
|
+
!(raw.operation_manifest === "none" || isText(raw.operation_manifest, 512)) ||
|
|
47
|
+
!isBoundedList(raw.validation_history) || !raw.validation_history.every(validAttempt) ||
|
|
48
|
+
!isBoundedList(raw.blockers) || !raw.blockers.every((item) => isText(item)) || !isText(raw.next_action)) {
|
|
49
|
+
return { capsule: undefined, warnings: [{ code: "malformed", message: "Retained state was ignored because it is malformed." }] };
|
|
50
|
+
}
|
|
51
|
+
const capsule = {
|
|
52
|
+
schema_version: "0.1",
|
|
53
|
+
authority: "shadow",
|
|
54
|
+
task_id: raw.task_id,
|
|
55
|
+
acceptance_fingerprint: raw.acceptance_fingerprint,
|
|
56
|
+
source_manifest: structuredClone(raw.source_manifest),
|
|
57
|
+
operation_manifest: raw.operation_manifest,
|
|
58
|
+
validation_history: structuredClone(raw.validation_history),
|
|
59
|
+
blockers: structuredClone(raw.blockers),
|
|
60
|
+
next_action: raw.next_action,
|
|
61
|
+
};
|
|
62
|
+
const warnings = [];
|
|
63
|
+
if ("next_evidence_decision" in raw) {
|
|
64
|
+
if (validDecision(raw.next_evidence_decision))
|
|
65
|
+
capsule.next_evidence_decision = structuredClone(raw.next_evidence_decision);
|
|
66
|
+
else
|
|
67
|
+
warnings.push({ code: "optional_invalid", message: "An optional retained-state decision was ignored." });
|
|
68
|
+
}
|
|
69
|
+
if ("admissions" in raw) {
|
|
70
|
+
if (isBoundedList(raw.admissions)) {
|
|
71
|
+
const validAdmissions = raw.admissions.filter(validAdmission);
|
|
72
|
+
if (validAdmissions.length > 0 || raw.admissions.length === 0)
|
|
73
|
+
capsule.admissions = structuredClone(validAdmissions);
|
|
74
|
+
if (validAdmissions.length !== raw.admissions.length)
|
|
75
|
+
warnings.push({ code: "optional_invalid", message: "Invalid retained-state admission receipts were ignored." });
|
|
76
|
+
}
|
|
77
|
+
else
|
|
78
|
+
warnings.push({ code: "optional_invalid", message: "Optional retained-state admission receipts were ignored." });
|
|
79
|
+
}
|
|
80
|
+
return { capsule, warnings };
|
|
81
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -19,3 +19,5 @@ export type { HandoffDenialReason, OpenCodeEvent, OpenCodeHooks, OpenCodePlugin,
|
|
|
19
19
|
export { CONSULTATION_CAPABILITIES, CONSULTATION_ROLE_POLICY, MAX_REVIEW_ARTIFACT_BYTES, SOURCE_REVIEW_RISK_TAGS, STRATEGY_TRIGGERS, evaluateReviewAvailability, evaluateReviewGate, evaluateSourceReviewRequirement, isSourceReviewRiskTag, requiresSourceReview, shouldConsultStrategy, validateReviewArtifact, validateReviewVerdict, } from "./core/consultation.js";
|
|
20
20
|
export type { ConsultationAdapter, ConsultationCapability, ConsultationRequest, ConsultationResult, ReviewArtifact, ReviewAvailability, ReviewFinding, ReviewFindingSeverity, ReviewGateInput, ReviewGateResult, ReviewVerdict, ReviewVerdictKind, SourceReviewConsultationRequest, SourceReviewConsultationResult, SourceReviewRequirement, SourceReviewRequirementInput, SourceReviewRiskTag, StrategyConsultationRequest, StrategyConsultationResult, StrategyTrigger, StrategyTriggerInput, UnavailableConsultationResult, ValidationResult, } from "./core/consultation.js";
|
|
21
21
|
export type * from "./core/types.js";
|
|
22
|
+
export { inspectRetainedStateCapsule, MAX_RETAINED_STATE_BYTES, MAX_RETAINED_STATE_ITEMS, MAX_RETAINED_STATE_WARNINGS, RETAINED_STATE_AUTHORITY, RETAINED_STATE_EXTENSION, RETAINED_STATE_SCHEMA_VERSION, } from "./core/retained-state.js";
|
|
23
|
+
export type { AdmissionReceipt, NextEvidenceDecision, RetainedStateCapsule, RetainedStateInspection, RetainedStateWarning, RetainedValidationAttempt, } from "./core/retained-state.js";
|
package/dist/index.js
CHANGED
|
@@ -15,3 +15,4 @@ export { SortieDogsPlugin } from "./plugin/index.js";
|
|
|
15
15
|
*/
|
|
16
16
|
export { HandoffDeniedError, InvalidModelTargetError, ModelRoutingDeniedError, isExplicitTaskHandoff, } from "./plugin/index.js";
|
|
17
17
|
export { CONSULTATION_CAPABILITIES, CONSULTATION_ROLE_POLICY, MAX_REVIEW_ARTIFACT_BYTES, SOURCE_REVIEW_RISK_TAGS, STRATEGY_TRIGGERS, evaluateReviewAvailability, evaluateReviewGate, evaluateSourceReviewRequirement, isSourceReviewRiskTag, requiresSourceReview, shouldConsultStrategy, validateReviewArtifact, validateReviewVerdict, } from "./core/consultation.js";
|
|
18
|
+
export { inspectRetainedStateCapsule, MAX_RETAINED_STATE_BYTES, MAX_RETAINED_STATE_ITEMS, MAX_RETAINED_STATE_WARNINGS, RETAINED_STATE_AUTHORITY, RETAINED_STATE_EXTENSION, RETAINED_STATE_SCHEMA_VERSION, } from "./core/retained-state.js";
|
|
@@ -403,7 +403,12 @@ export function createContinuationHooks(client, directory, policySource, timings
|
|
|
403
403
|
const entries = [];
|
|
404
404
|
for (const { index, line } of topLevelProtocolLines(text)) {
|
|
405
405
|
const explicit = /^status:\s*(DONE|BLOCKED|NEED_DECISION)\b/iu.exec(line)?.[1]?.toUpperCase();
|
|
406
|
-
const
|
|
406
|
+
const aliasMatch = /^([✅⛔❓])[ \t]+conclusion:\s*status:\s*(DONE|BLOCKED|NEED_DECISION)\b/iu.exec(line);
|
|
407
|
+
const aliasStatus = aliasMatch?.[2]?.toUpperCase();
|
|
408
|
+
const alias = aliasStatus === "DONE" && aliasMatch?.[1] === "✅" ? "DONE" :
|
|
409
|
+
aliasStatus === "BLOCKED" && aliasMatch?.[1] === "⛔" ? "BLOCKED" :
|
|
410
|
+
aliasStatus === "NEED_DECISION" && aliasMatch?.[1] === "❓" ? "NEED_DECISION" : undefined;
|
|
411
|
+
const status = explicit ?? alias ??
|
|
407
412
|
(/^✅[ \t]+\*\*DONE\*\*/u.test(line) ? "DONE" :
|
|
408
413
|
/^⛔[ \t]+\*\*BLOCKED\*\*/u.test(line) ? "BLOCKED" :
|
|
409
414
|
/^❓[ \t]+\*\*NEED_DECISION\*\*/u.test(line) ? "NEED_DECISION" : undefined);
|
|
@@ -413,15 +418,21 @@ export function createContinuationHooks(client, directory, policySource, timings
|
|
|
413
418
|
}
|
|
414
419
|
return entries;
|
|
415
420
|
}
|
|
421
|
+
function firstCheckpoint(text) {
|
|
422
|
+
const firstLine = topLevelProtocolLines(text).find(({ line }) => line.trim().length > 0);
|
|
423
|
+
return firstLine === undefined
|
|
424
|
+
? undefined
|
|
425
|
+
: checkpointEntries(text).find(({ index }) => index === firstLine.index);
|
|
426
|
+
}
|
|
416
427
|
function checkpointStatus(text) {
|
|
417
|
-
return
|
|
428
|
+
return firstCheckpoint(text)?.status;
|
|
418
429
|
}
|
|
419
430
|
function terminalCheckpoint(text) {
|
|
420
431
|
const status = checkpointStatus(text);
|
|
421
432
|
return status === "DONE" || status === "NEED_DECISION" || (status === "BLOCKED" && trueBlockerReport(text));
|
|
422
433
|
}
|
|
423
434
|
function trueBlockerReport(text) {
|
|
424
|
-
const checkpoint =
|
|
435
|
+
const checkpoint = firstCheckpoint(text);
|
|
425
436
|
if (checkpoint?.status !== "BLOCKED")
|
|
426
437
|
return false;
|
|
427
438
|
return topLevelProtocolLines(text).some(({ line, index }) => index > checkpoint.index &&
|
|
@@ -472,7 +483,7 @@ export function createContinuationHooks(client, directory, policySource, timings
|
|
|
472
483
|
function observeRecoveryReport(state, report) {
|
|
473
484
|
if (state.recoveryObservedRevision === state.turnRevision)
|
|
474
485
|
return state.recoveryRepeatCount;
|
|
475
|
-
const checkpoint =
|
|
486
|
+
const checkpoint = firstCheckpoint(report);
|
|
476
487
|
const checkpointLine = checkpoint === undefined
|
|
477
488
|
? undefined
|
|
478
489
|
: topLevelProtocolLines(report).find(({ index }) => index === checkpoint.index)?.line;
|
package/dist/plugin/index.js
CHANGED
|
@@ -18,7 +18,7 @@ import { BACKLOG_DRAIN_CAPABILITY, FastLaneController } from "./fast-lane.js";
|
|
|
18
18
|
import { createModelRoutingHook, } from "./model-routing-hook.js";
|
|
19
19
|
import { createTaskResultRepairHook, lastAssistantText, markConsultationFallbackRetry, taskChildSessionID, } from "./task-result-repair.js";
|
|
20
20
|
import { configRoot, nearestPackageVersion, REFLECTION_POLICY, reflectionEnabled, ReflectionError, ReflectionStore } from "../reflection/index.js";
|
|
21
|
-
import { collectRunMetrics, insertRunMetrics,
|
|
21
|
+
import { collectRunMetrics, insertRunMetrics, terminalRunOutcome } from "./run-metrics.js";
|
|
22
22
|
const INPUT_LIMITS = { config: 64 * 1024, manifest: 512 * 1024, handoff: 2 * 1024 * 1024, parallel: 512 * 1024 };
|
|
23
23
|
const INSPECTION_CACHE = { maximum: 256, ttlMilliseconds: 30 * 60 * 1000 };
|
|
24
24
|
const ACTIVE_SESSION_CACHE = { maximum: 256, ttlMilliseconds: 30 * 60 * 1000 };
|
|
@@ -854,6 +854,13 @@ export const SortieDogsPlugin = async (input, options) => {
|
|
|
854
854
|
bootstrapControlState: { count: 0, elapsedMilliseconds: 0 },
|
|
855
855
|
collectRunMetrics: { count: 0, elapsedMilliseconds: 0 },
|
|
856
856
|
},
|
|
857
|
+
compactionPolicy: {
|
|
858
|
+
count: 0,
|
|
859
|
+
contextInputBytes: 0,
|
|
860
|
+
contextOutputBytes: 0,
|
|
861
|
+
promptInputBytes: 0,
|
|
862
|
+
promptOutputBytes: 0,
|
|
863
|
+
},
|
|
857
864
|
};
|
|
858
865
|
sessionOperationMetrics.set(sessionID, created);
|
|
859
866
|
return created;
|
|
@@ -870,7 +877,9 @@ export const SortieDogsPlugin = async (input, options) => {
|
|
|
870
877
|
}
|
|
871
878
|
}
|
|
872
879
|
function operationMetricsSnapshot(sessionID) {
|
|
873
|
-
const
|
|
880
|
+
const measurements = sessionOperationMetrics.get(sessionID);
|
|
881
|
+
const operations = measurements?.operations;
|
|
882
|
+
const compaction = measurements?.compactionPolicy;
|
|
874
883
|
return {
|
|
875
884
|
hostSessionIdentityCount: operations?.hostSessionIdentity.count ?? 0,
|
|
876
885
|
hostSessionIdentityElapsedMilliseconds: Math.round(operations?.hostSessionIdentity.elapsedMilliseconds ?? 0),
|
|
@@ -878,8 +887,19 @@ export const SortieDogsPlugin = async (input, options) => {
|
|
|
878
887
|
bootstrapControlStateElapsedMilliseconds: Math.round(operations?.bootstrapControlState.elapsedMilliseconds ?? 0),
|
|
879
888
|
collectRunMetricsCount: operations?.collectRunMetrics.count ?? 0,
|
|
880
889
|
collectRunMetricsElapsedMilliseconds: Math.round(operations?.collectRunMetrics.elapsedMilliseconds ?? 0),
|
|
890
|
+
compactionPolicyCount: compaction?.count ?? 0,
|
|
891
|
+
compactionContextInputBytes: compaction?.contextInputBytes ?? 0,
|
|
892
|
+
compactionContextOutputBytes: compaction?.contextOutputBytes ?? 0,
|
|
893
|
+
compactionPromptInputBytes: compaction?.promptInputBytes ?? 0,
|
|
894
|
+
compactionPromptOutputBytes: compaction?.promptOutputBytes ?? 0,
|
|
881
895
|
};
|
|
882
896
|
}
|
|
897
|
+
function utf8Bytes(value) {
|
|
898
|
+
return value === undefined ? 0 : Buffer.byteLength(value, "utf8");
|
|
899
|
+
}
|
|
900
|
+
function contextBytes(value) {
|
|
901
|
+
return value?.reduce((total, entry) => total + utf8Bytes(entry), 0) ?? 0;
|
|
902
|
+
}
|
|
883
903
|
// Project config read is required discovery for its opt-in; no reflection storage/version read
|
|
884
904
|
// occurs unless that resolved config enables reflection. It stays isolated from write-gate load.
|
|
885
905
|
try {
|
|
@@ -2988,20 +3008,40 @@ export const SortieDogsPlugin = async (input, options) => {
|
|
|
2988
3008
|
.replaceAll(CONTINUATION_MARKER, "")
|
|
2989
3009
|
.trimEnd();
|
|
2990
3010
|
}
|
|
3011
|
+
const runOutcome = terminalRunOutcome(textOutput.text);
|
|
2991
3012
|
if ((isCoordinatorSession(textInput.sessionID) || await recoverCoordinatorRoot(textInput.sessionID)) &&
|
|
2992
|
-
|
|
3013
|
+
runOutcome !== undefined) {
|
|
2993
3014
|
const metrics = await measureSessionOperation(textInput.sessionID, "collectRunMetrics", () => collectRunMetrics(input.client, textInput.sessionID, input.directory).catch(() => undefined));
|
|
2994
|
-
if (metrics !== undefined)
|
|
3015
|
+
if (metrics !== undefined && runOutcome === "DONE")
|
|
2995
3016
|
textOutput.text = insertRunMetrics(textOutput.text, metrics);
|
|
2996
3017
|
appLogInfo("run-metrics.snapshot", textInput.sessionID, {
|
|
2997
3018
|
available: metrics !== undefined,
|
|
3019
|
+
outcome: runOutcome,
|
|
3020
|
+
runtimeAssetVersion: RUNTIME_ASSET_VERSION,
|
|
3021
|
+
...(metrics ?? {}),
|
|
2998
3022
|
...operationMetricsSnapshot(textInput.sessionID),
|
|
2999
3023
|
});
|
|
3000
3024
|
}
|
|
3001
3025
|
await completeContinuationText(textInput.sessionID, textOutput.text, false);
|
|
3002
3026
|
},
|
|
3003
3027
|
"experimental.session.compacting": async (compactInput, compactOutput) => {
|
|
3028
|
+
const before = {
|
|
3029
|
+
context: contextBytes(compactOutput.context),
|
|
3030
|
+
prompt: utf8Bytes(compactOutput.prompt),
|
|
3031
|
+
};
|
|
3004
3032
|
await continuation.sessionCompacting(compactInput, compactOutput);
|
|
3033
|
+
const after = {
|
|
3034
|
+
context: contextBytes(compactOutput.context),
|
|
3035
|
+
prompt: utf8Bytes(compactOutput.prompt),
|
|
3036
|
+
};
|
|
3037
|
+
if (before.context !== after.context || before.prompt !== after.prompt) {
|
|
3038
|
+
const measurement = operationMetricsFor(compactInput.sessionID).compactionPolicy;
|
|
3039
|
+
measurement.count += 1;
|
|
3040
|
+
measurement.contextInputBytes += before.context;
|
|
3041
|
+
measurement.contextOutputBytes += after.context;
|
|
3042
|
+
measurement.promptInputBytes += before.prompt;
|
|
3043
|
+
measurement.promptOutputBytes += after.prompt;
|
|
3044
|
+
}
|
|
3005
3045
|
},
|
|
3006
3046
|
"experimental.compaction.autocontinue": async (autoInput, autoOutput) => {
|
|
3007
3047
|
await continuation.compactionAutoContinue(autoInput, autoOutput);
|
|
@@ -29,12 +29,31 @@ export interface RunMetricsClient {
|
|
|
29
29
|
export interface RunMetrics {
|
|
30
30
|
readonly durationMilliseconds: number | undefined;
|
|
31
31
|
readonly tokens: number | undefined;
|
|
32
|
+
readonly inputTokens: number | undefined;
|
|
33
|
+
readonly outputTokens: number | undefined;
|
|
34
|
+
readonly reasoningTokens: number | undefined;
|
|
35
|
+
readonly cacheReadTokens: number | undefined;
|
|
36
|
+
readonly cacheWriteTokens: number | undefined;
|
|
32
37
|
readonly cost: number | undefined;
|
|
33
38
|
readonly steps: number | undefined;
|
|
34
39
|
readonly sessions: number | undefined;
|
|
35
40
|
readonly cacheRatio: number | undefined;
|
|
41
|
+
readonly roles: Readonly<Record<string, RunRoleMetrics>> | undefined;
|
|
36
42
|
}
|
|
43
|
+
export interface RunRoleMetrics {
|
|
44
|
+
readonly tokens: number;
|
|
45
|
+
readonly inputTokens: number;
|
|
46
|
+
readonly outputTokens: number;
|
|
47
|
+
readonly reasoningTokens: number;
|
|
48
|
+
readonly cacheReadTokens: number;
|
|
49
|
+
readonly cacheWriteTokens: number;
|
|
50
|
+
readonly cost: number | undefined;
|
|
51
|
+
readonly steps: number;
|
|
52
|
+
readonly cacheRatio: number | undefined;
|
|
53
|
+
}
|
|
54
|
+
export type RunTerminalOutcome = "DONE" | "BLOCKED" | "NEED_DECISION";
|
|
37
55
|
export declare function collectRunMetrics(client: RunMetricsClient | undefined, rootSessionID: string, directory?: string, now?: number): Promise<RunMetrics | undefined>;
|
|
38
56
|
export declare function formatRunMetrics(metrics: RunMetrics): string;
|
|
39
57
|
export declare function isDoneTerminalText(text: string): boolean;
|
|
58
|
+
export declare function terminalRunOutcome(text: string): RunTerminalOutcome | undefined;
|
|
40
59
|
export declare function insertRunMetrics(text: string, metrics: RunMetrics): string;
|
|
@@ -22,7 +22,19 @@ function messageTokens(message) {
|
|
|
22
22
|
const cacheWrite = number(cache?.write) ?? number(tokens.cacheWrite) ?? number(tokens.cache_write);
|
|
23
23
|
if (input === undefined || output === undefined || reasoning === undefined || cacheRead === undefined || cacheWrite === undefined)
|
|
24
24
|
return undefined;
|
|
25
|
-
return
|
|
25
|
+
return {
|
|
26
|
+
total: input + output + reasoning + cacheRead + cacheWrite,
|
|
27
|
+
input,
|
|
28
|
+
output,
|
|
29
|
+
reasoning,
|
|
30
|
+
cacheRead,
|
|
31
|
+
cacheWrite,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
function messageAgent(message) {
|
|
35
|
+
const info = record(message.info) ?? message;
|
|
36
|
+
const agent = info.agent ?? message.agent;
|
|
37
|
+
return typeof agent === "string" && agent.trim().length > 0 ? agent.slice(0, 128) : "unknown";
|
|
26
38
|
}
|
|
27
39
|
function assistantMessages(value) {
|
|
28
40
|
const payload = unwrap(value);
|
|
@@ -34,6 +46,14 @@ function assistantMessages(value) {
|
|
|
34
46
|
return item !== undefined && (info?.role ?? item.role) === "assistant";
|
|
35
47
|
});
|
|
36
48
|
}
|
|
49
|
+
function conclusionStatusAlias(line) {
|
|
50
|
+
const match = /^([✅⛔❓])[ \t]+conclusion:\s*status:\s*(DONE|BLOCKED|NEED_DECISION)\b/iu.exec(line);
|
|
51
|
+
const outcome = match?.[2]?.toUpperCase();
|
|
52
|
+
if (outcome !== "DONE" && outcome !== "BLOCKED" && outcome !== "NEED_DECISION")
|
|
53
|
+
return undefined;
|
|
54
|
+
const expectedIcon = outcome === "DONE" ? "✅" : outcome === "BLOCKED" ? "⛔" : "❓";
|
|
55
|
+
return match?.[1] === expectedIcon ? outcome : undefined;
|
|
56
|
+
}
|
|
37
57
|
export async function collectRunMetrics(client, rootSessionID, directory, now = Date.now()) {
|
|
38
58
|
const session = client?.session;
|
|
39
59
|
if (session?.messages === undefined)
|
|
@@ -74,12 +94,17 @@ export async function collectRunMetrics(client, rootSessionID, directory, now =
|
|
|
74
94
|
hierarchyComplete = false;
|
|
75
95
|
const uniqueMessages = new Set();
|
|
76
96
|
let totalTokens = 0;
|
|
97
|
+
let inputTokens = 0;
|
|
98
|
+
let outputTokens = 0;
|
|
99
|
+
let reasoningTokens = 0;
|
|
77
100
|
let cacheRead = 0;
|
|
101
|
+
let cacheWrite = 0;
|
|
78
102
|
let tokensAvailable = true;
|
|
79
103
|
let messagesComplete = true;
|
|
80
104
|
let steps = 0;
|
|
81
105
|
let cost = 0;
|
|
82
106
|
let costAvailable = true;
|
|
107
|
+
const roleMetrics = new Map();
|
|
83
108
|
for (const id of ids) {
|
|
84
109
|
try {
|
|
85
110
|
const messages = assistantMessages(await session.messages.call(session, { path: { id }, query: { directory } }));
|
|
@@ -99,18 +124,46 @@ export async function collectRunMetrics(client, rootSessionID, directory, now =
|
|
|
99
124
|
continue;
|
|
100
125
|
uniqueMessages.add(messageID);
|
|
101
126
|
steps += 1;
|
|
127
|
+
const agent = messageAgent(message);
|
|
128
|
+
const role = roleMetrics.get(agent) ?? {
|
|
129
|
+
tokens: 0,
|
|
130
|
+
inputTokens: 0,
|
|
131
|
+
outputTokens: 0,
|
|
132
|
+
reasoningTokens: 0,
|
|
133
|
+
cacheReadTokens: 0,
|
|
134
|
+
cacheWriteTokens: 0,
|
|
135
|
+
cost: 0,
|
|
136
|
+
costAvailable: true,
|
|
137
|
+
steps: 0,
|
|
138
|
+
};
|
|
139
|
+
role.steps += 1;
|
|
140
|
+
roleMetrics.set(agent, role);
|
|
102
141
|
const tokens = messageTokens(message);
|
|
103
142
|
if (tokens !== undefined) {
|
|
104
|
-
totalTokens += tokens
|
|
105
|
-
|
|
143
|
+
totalTokens += tokens.total;
|
|
144
|
+
inputTokens += tokens.input;
|
|
145
|
+
outputTokens += tokens.output;
|
|
146
|
+
reasoningTokens += tokens.reasoning;
|
|
147
|
+
cacheRead += tokens.cacheRead;
|
|
148
|
+
cacheWrite += tokens.cacheWrite;
|
|
149
|
+
role.tokens += tokens.total;
|
|
150
|
+
role.inputTokens += tokens.input;
|
|
151
|
+
role.outputTokens += tokens.output;
|
|
152
|
+
role.reasoningTokens += tokens.reasoning;
|
|
153
|
+
role.cacheReadTokens += tokens.cacheRead;
|
|
154
|
+
role.cacheWriteTokens += tokens.cacheWrite;
|
|
106
155
|
}
|
|
107
156
|
else
|
|
108
157
|
tokensAvailable = false;
|
|
109
158
|
const reportedCost = number(info.cost) ?? number(message.cost);
|
|
110
|
-
if (reportedCost === undefined)
|
|
159
|
+
if (reportedCost === undefined) {
|
|
111
160
|
costAvailable = false;
|
|
112
|
-
|
|
161
|
+
role.costAvailable = false;
|
|
162
|
+
}
|
|
163
|
+
else {
|
|
113
164
|
cost += reportedCost;
|
|
165
|
+
role.cost += reportedCost;
|
|
166
|
+
}
|
|
114
167
|
}
|
|
115
168
|
}
|
|
116
169
|
catch {
|
|
@@ -129,10 +182,28 @@ export async function collectRunMetrics(client, rootSessionID, directory, now =
|
|
|
129
182
|
return {
|
|
130
183
|
durationMilliseconds: created === undefined ? undefined : Math.max(0, now - created),
|
|
131
184
|
tokens: hierarchyComplete && messagesComplete && tokensAvailable ? totalTokens : undefined,
|
|
185
|
+
inputTokens: hierarchyComplete && messagesComplete && tokensAvailable ? inputTokens : undefined,
|
|
186
|
+
outputTokens: hierarchyComplete && messagesComplete && tokensAvailable ? outputTokens : undefined,
|
|
187
|
+
reasoningTokens: hierarchyComplete && messagesComplete && tokensAvailable ? reasoningTokens : undefined,
|
|
188
|
+
cacheReadTokens: hierarchyComplete && messagesComplete && tokensAvailable ? cacheRead : undefined,
|
|
189
|
+
cacheWriteTokens: hierarchyComplete && messagesComplete && tokensAvailable ? cacheWrite : undefined,
|
|
132
190
|
cost: hierarchyComplete && messagesComplete && costAvailable ? cost : undefined,
|
|
133
191
|
steps: hierarchyComplete && messagesComplete ? steps : undefined,
|
|
134
192
|
sessions: hierarchyComplete ? ids.length : undefined,
|
|
135
193
|
cacheRatio: hierarchyComplete && messagesComplete && tokensAvailable && totalTokens > 0 ? cacheRead / totalTokens : undefined,
|
|
194
|
+
roles: hierarchyComplete && messagesComplete && tokensAvailable
|
|
195
|
+
? Object.fromEntries([...roleMetrics].map(([agent, role]) => [agent, {
|
|
196
|
+
tokens: role.tokens,
|
|
197
|
+
inputTokens: role.inputTokens,
|
|
198
|
+
outputTokens: role.outputTokens,
|
|
199
|
+
reasoningTokens: role.reasoningTokens,
|
|
200
|
+
cacheReadTokens: role.cacheReadTokens,
|
|
201
|
+
cacheWriteTokens: role.cacheWriteTokens,
|
|
202
|
+
cost: role.costAvailable ? role.cost : undefined,
|
|
203
|
+
steps: role.steps,
|
|
204
|
+
cacheRatio: role.tokens > 0 ? role.cacheReadTokens / role.tokens : undefined,
|
|
205
|
+
}]))
|
|
206
|
+
: undefined,
|
|
136
207
|
};
|
|
137
208
|
}
|
|
138
209
|
function duration(milliseconds) {
|
|
@@ -151,19 +222,69 @@ export function formatRunMetrics(metrics) {
|
|
|
151
222
|
const cache = metrics.cacheRatio === undefined ? "cache ratio unavailable" : `${(metrics.cacheRatio * 100).toFixed(1)}% cache ratio`;
|
|
152
223
|
return `**Run:** pre-terminal host snapshot · ${elapsed} · ${tokens} · ${cost} · ${steps} · ${sessions} · ${cache}`;
|
|
153
224
|
}
|
|
225
|
+
function topLevelLines(text) {
|
|
226
|
+
const lines = [];
|
|
227
|
+
let fence;
|
|
228
|
+
for (const [index, line] of text.split(/\r?\n/u).entries()) {
|
|
229
|
+
if (fence === undefined) {
|
|
230
|
+
const opener = /^[ \t]*(`{3,}|~{3,})/u.exec(line)?.[1];
|
|
231
|
+
if (opener === undefined) {
|
|
232
|
+
if (!/^[ \t]*>/u.test(line))
|
|
233
|
+
lines.push({ index, line });
|
|
234
|
+
continue;
|
|
235
|
+
}
|
|
236
|
+
fence = { character: opener[0], length: opener.length };
|
|
237
|
+
continue;
|
|
238
|
+
}
|
|
239
|
+
const closer = /^[ \t]*(`{3,}|~{3,})[ \t]*$/u.exec(line)?.[1];
|
|
240
|
+
if (closer?.[0] === fence.character && closer.length >= fence.length)
|
|
241
|
+
fence = undefined;
|
|
242
|
+
}
|
|
243
|
+
return lines;
|
|
244
|
+
}
|
|
245
|
+
function terminalCheckpoint(text) {
|
|
246
|
+
const lines = topLevelLines(text);
|
|
247
|
+
const first = lines.find(({ line }) => line.trim().length > 0);
|
|
248
|
+
if (first === undefined)
|
|
249
|
+
return undefined;
|
|
250
|
+
const checkpoint = (() => {
|
|
251
|
+
const { index, line } = first;
|
|
252
|
+
const normalized = /^status:\s*(DONE|BLOCKED|NEED_DECISION)\b/iu.exec(line)?.[1]?.toUpperCase();
|
|
253
|
+
const explicit = normalized === "DONE" || normalized === "BLOCKED" || normalized === "NEED_DECISION"
|
|
254
|
+
? normalized
|
|
255
|
+
: undefined;
|
|
256
|
+
const outcome = explicit ?? conclusionStatusAlias(line) ??
|
|
257
|
+
(/^✅[ \t]+\*\*DONE\*\*/u.test(line) ? "DONE" :
|
|
258
|
+
/^⛔[ \t]+\*\*BLOCKED\*\*/u.test(line) ? "BLOCKED" :
|
|
259
|
+
/^❓[ \t]+\*\*NEED_DECISION\*\*/u.test(line) ? "NEED_DECISION" : undefined);
|
|
260
|
+
return outcome === "DONE" || outcome === "BLOCKED" || outcome === "NEED_DECISION"
|
|
261
|
+
? { index, outcome }
|
|
262
|
+
: undefined;
|
|
263
|
+
})();
|
|
264
|
+
return checkpoint;
|
|
265
|
+
}
|
|
154
266
|
export function isDoneTerminalText(text) {
|
|
155
|
-
|
|
156
|
-
|
|
267
|
+
return terminalCheckpoint(text)?.outcome === "DONE";
|
|
268
|
+
}
|
|
269
|
+
export function terminalRunOutcome(text) {
|
|
270
|
+
const checkpoint = terminalCheckpoint(text);
|
|
271
|
+
if (checkpoint === undefined)
|
|
272
|
+
return undefined;
|
|
273
|
+
if (checkpoint.outcome !== "BLOCKED")
|
|
274
|
+
return checkpoint.outcome;
|
|
275
|
+
return topLevelLines(text).some(({ index, line }) => index > checkpoint.index &&
|
|
276
|
+
/^TRUE_BLOCKER\s*:\s*(?:external|user-decision)\s*:\s*\S.*$/u.test(line))
|
|
277
|
+
? "BLOCKED"
|
|
278
|
+
: undefined;
|
|
157
279
|
}
|
|
158
280
|
export function insertRunMetrics(text, metrics) {
|
|
159
|
-
|
|
281
|
+
const checkpoint = terminalCheckpoint(text);
|
|
282
|
+
if (checkpoint?.outcome !== "DONE")
|
|
160
283
|
return text;
|
|
161
|
-
|
|
162
|
-
const statusStart = text.search(/^(?:✅\s+\*\*DONE\*\*|status:\s*DONE).*$/mu);
|
|
163
|
-
if (statusStart < 0)
|
|
284
|
+
if (topLevelLines(text).some(({ index, line }) => index > checkpoint.index && /^\*\*Run:\*\*/u.test(line)))
|
|
164
285
|
return text;
|
|
165
|
-
const
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
return
|
|
286
|
+
const newline = text.includes("\r\n") ? "\r\n" : "\n";
|
|
287
|
+
const lines = text.split(/\r?\n/u);
|
|
288
|
+
lines.splice(checkpoint.index + 1, 0, "", formatRunMetrics(metrics));
|
|
289
|
+
return lines.join(newline);
|
|
169
290
|
}
|