pluribus-context 0.3.21 → 0.3.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -3
- package/README.md +2 -2
- package/docs/community-review-packet.md +4 -2
- package/docs/context-budget-receipts.md +150 -0
- package/docs/context-input-evidence.md +397 -0
- package/docs/context-receipts-for-agent-observability.md +177 -0
- package/docs/orchestration-search-receipts.md +102 -0
- package/docs/portability-fidelity-report.md +10 -6
- package/examples/context-input-evidence/AGENTS.md +12 -0
- package/examples/context-input-evidence/agent-overlay-log.jsonl +4 -0
- package/examples/context-input-evidence/agent-overlay-otel-trace.json +548 -0
- package/examples/context-input-evidence/agent-overlay-receipt.ndjson +3 -0
- package/examples/context-input-evidence/agentgateway-progressive-disclosure-otel-trace.json +393 -0
- package/examples/context-input-evidence/agentgateway-progressive-disclosure-receipt.ndjson +4 -0
- package/examples/context-input-evidence/brain-remediation-otel-trace.json +645 -0
- package/examples/context-input-evidence/brain-remediation-receipt.ndjson +7 -0
- package/examples/context-input-evidence/claudekit-mcp-manager-otel-trace.json +417 -0
- package/examples/context-input-evidence/claudekit-mcp-manager-receipt.ndjson +5 -0
- package/examples/context-input-evidence/cli-progressive-disclosure-otel-trace.json +399 -0
- package/examples/context-input-evidence/cli-progressive-disclosure-receipt.ndjson +4 -0
- package/examples/context-input-evidence/compaction-otel-trace.json +711 -0
- package/examples/context-input-evidence/compaction-receipt.ndjson +6 -0
- package/examples/context-input-evidence/context-selection-otel-trace.json +627 -0
- package/examples/context-input-evidence/context-selection-receipt.ndjson +7 -0
- package/examples/context-input-evidence/convert-agent-overlay-log.mjs +156 -0
- package/examples/context-input-evidence/convert-agentgateway-progressive-disclosure-log.mjs +251 -0
- package/examples/context-input-evidence/convert-brain-remediation-log.mjs +241 -0
- package/examples/context-input-evidence/convert-claudekit-mcp-manager-log.mjs +253 -0
- package/examples/context-input-evidence/convert-cli-progressive-disclosure-log.mjs +251 -0
- package/examples/context-input-evidence/convert-compaction-log.mjs +224 -0
- package/examples/context-input-evidence/convert-context-selection-log.mjs +247 -0
- package/examples/context-input-evidence/convert-mcp-tool-search-log.mjs +242 -0
- package/examples/context-input-evidence/convert-memory-consolidation-log.mjs +240 -0
- package/examples/context-input-evidence/convert-memory-governance-delete-log.mjs +223 -0
- package/examples/context-input-evidence/convert-memory-log.mjs +226 -0
- package/examples/context-input-evidence/convert-memory-provenance-log.mjs +263 -0
- package/examples/context-input-evidence/convert-secret-scanning-log.mjs +233 -0
- package/examples/context-input-evidence/convert-session-log.mjs +186 -0
- package/examples/context-input-evidence/convert-skill-log.mjs +161 -0
- package/examples/context-input-evidence/convert-skill-registry-log.mjs +246 -0
- package/examples/context-input-evidence/convert-skill-routing-log.mjs +253 -0
- package/examples/context-input-evidence/convert-subagent-context-budget-log.mjs +267 -0
- package/examples/context-input-evidence/convert-subagent-delegation-log.mjs +264 -0
- package/examples/context-input-evidence/export-otel-trace.mjs +128 -0
- package/examples/context-input-evidence/generate-receipt.mjs +188 -0
- package/examples/context-input-evidence/mcp-tool-search-otel-trace.json +477 -0
- package/examples/context-input-evidence/mcp-tool-search-receipt.ndjson +5 -0
- package/examples/context-input-evidence/memory-consolidation-otel-trace.json +492 -0
- package/examples/context-input-evidence/memory-consolidation-receipt.ndjson +4 -0
- package/examples/context-input-evidence/memory-governance-delete-otel-trace.json +614 -0
- package/examples/context-input-evidence/memory-governance-delete-receipt.ndjson +5 -0
- package/examples/context-input-evidence/memory-otel-trace.json +645 -0
- package/examples/context-input-evidence/memory-provenance-otel-trace.json +711 -0
- package/examples/context-input-evidence/memory-provenance-receipt.ndjson +5 -0
- package/examples/context-input-evidence/memory-receipt.ndjson +4 -0
- package/examples/context-input-evidence/otel-trace.json +1119 -0
- package/examples/context-input-evidence/receipt.ndjson +6 -0
- package/examples/context-input-evidence/sample-agentgateway-progressive-disclosure-log.jsonl +5 -0
- package/examples/context-input-evidence/sample-brain-remediation-log.jsonl +9 -0
- package/examples/context-input-evidence/sample-claudekit-mcp-manager-log.jsonl +6 -0
- package/examples/context-input-evidence/sample-cli-progressive-disclosure-log.jsonl +5 -0
- package/examples/context-input-evidence/sample-compaction-log.jsonl +7 -0
- package/examples/context-input-evidence/sample-context-selection-log.jsonl +7 -0
- package/examples/context-input-evidence/sample-mcp-tool-search-log.jsonl +6 -0
- package/examples/context-input-evidence/sample-memory-consolidation-log.jsonl +5 -0
- package/examples/context-input-evidence/sample-memory-governance-delete-log.jsonl +6 -0
- package/examples/context-input-evidence/sample-memory-provenance-log.jsonl +6 -0
- package/examples/context-input-evidence/sample-memory-retrieval-log.jsonl +6 -0
- package/examples/context-input-evidence/sample-secret-scanning-log.jsonl +7 -0
- package/examples/context-input-evidence/sample-session-log.jsonl +6 -0
- package/examples/context-input-evidence/sample-skill-registry-log.jsonl +5 -0
- package/examples/context-input-evidence/sample-skill-routing-log.jsonl +7 -0
- package/examples/context-input-evidence/sample-subagent-context-budget-log.jsonl +6 -0
- package/examples/context-input-evidence/sample-subagent-delegation-log.jsonl +5 -0
- package/examples/context-input-evidence/secret-scanning-otel-trace.json +794 -0
- package/examples/context-input-evidence/secret-scanning-receipt.ndjson +6 -0
- package/examples/context-input-evidence/session-otel-trace.json +411 -0
- package/examples/context-input-evidence/session-receipt.ndjson +2 -0
- package/examples/context-input-evidence/skill-invocation-log.jsonl +4 -0
- package/examples/context-input-evidence/skill-otel-trace.json +548 -0
- package/examples/context-input-evidence/skill-receipt.ndjson +3 -0
- package/examples/context-input-evidence/skill-registry-otel-trace.json +471 -0
- package/examples/context-input-evidence/skill-registry-receipt.ndjson +5 -0
- package/examples/context-input-evidence/skill-routing-otel-trace.json +567 -0
- package/examples/context-input-evidence/skill-routing-receipt.ndjson +6 -0
- package/examples/context-input-evidence/subagent-context-budget-otel-trace.json +507 -0
- package/examples/context-input-evidence/subagent-context-budget-receipt.ndjson +5 -0
- package/examples/context-input-evidence/subagent-delegation-otel-trace.json +388 -0
- package/examples/context-input-evidence/subagent-delegation-receipt.ndjson +4 -0
- package/package.json +6 -2
- package/schemas/audit-result.schema.json +468 -43
- package/src/commands/audit.js +105 -5
- package/src/utils/version.js +1 -1
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
4
|
+
import { dirname, join } from 'node:path';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
6
|
+
|
|
7
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
8
|
+
const inputPath = join(here, 'skill-invocation-log.jsonl');
|
|
9
|
+
const receiptPath = join(here, 'skill-receipt.ndjson');
|
|
10
|
+
const tracePath = join(here, 'skill-otel-trace.json');
|
|
11
|
+
|
|
12
|
+
function sha256(value) {
|
|
13
|
+
return `sha256:${createHash('sha256').update(value).digest('hex')}`;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function parseJsonl(filePath) {
|
|
17
|
+
return readFileSync(filePath, 'utf8')
|
|
18
|
+
.trim()
|
|
19
|
+
.split('\n')
|
|
20
|
+
.filter(Boolean)
|
|
21
|
+
.map((line) => JSON.parse(line));
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function attributeValue(value) {
|
|
25
|
+
if (typeof value === 'boolean') return { boolValue: value };
|
|
26
|
+
if (typeof value === 'number') return { intValue: value };
|
|
27
|
+
return { stringValue: String(value ?? '') };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function toOtelAttributes(attributes) {
|
|
31
|
+
return Object.entries(attributes).map(([key, value]) => ({
|
|
32
|
+
key,
|
|
33
|
+
value: attributeValue(value)
|
|
34
|
+
}));
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const records = parseJsonl(inputPath);
|
|
38
|
+
const session = records.find((record) => record.type === 'session.start');
|
|
39
|
+
if (!session) {
|
|
40
|
+
throw new Error('skill-invocation-log.jsonl must contain a session.start record');
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const skillRecords = records.filter((record) => record.type === 'skill.invoked');
|
|
44
|
+
const duplicateGroups = new Map();
|
|
45
|
+
|
|
46
|
+
for (const record of skillRecords) {
|
|
47
|
+
const deliveredHash = sha256(record.delivered_text);
|
|
48
|
+
const key = `${record.session_id ?? session.session_id}:${record.skill_id}:${record.activation}:${record.hook_event}:${deliveredHash}`;
|
|
49
|
+
duplicateGroups.set(key, (duplicateGroups.get(key) ?? 0) + 1);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const events = skillRecords.map((record) => {
|
|
53
|
+
const sourceBytesHash = sha256(record.source_text);
|
|
54
|
+
const deliveredHash = sha256(record.delivered_text);
|
|
55
|
+
const dedupeScope = 'conversation';
|
|
56
|
+
const dedupeKey = `${dedupeScope}:${record.skill_id}:${record.activation}:${record.hook_event}:${deliveredHash}`;
|
|
57
|
+
const duplicateCount = duplicateGroups.get(`${session.session_id}:${record.skill_id}:${record.activation}:${record.hook_event}:${deliveredHash}`) ?? 1;
|
|
58
|
+
|
|
59
|
+
return {
|
|
60
|
+
trace_id: 'demo-trace-skill-context-receipts',
|
|
61
|
+
span_id: session.session_id,
|
|
62
|
+
name: 'context.skill.invoked',
|
|
63
|
+
time: record.time,
|
|
64
|
+
attributes: {
|
|
65
|
+
'context.input.kind': 'skill',
|
|
66
|
+
'context.skill.id': record.skill_id,
|
|
67
|
+
'context.skill.name': record.skill_name,
|
|
68
|
+
'context.skill.plugin': record.plugin,
|
|
69
|
+
'context.input.source.path': record.source_path,
|
|
70
|
+
'context.input.source.bytes_hash': sourceBytesHash,
|
|
71
|
+
'context.input.delivered.hash': deliveredHash,
|
|
72
|
+
'session.id': session.session_id,
|
|
73
|
+
'gen_ai.conversation.id': session.conversation_id,
|
|
74
|
+
'context.input.loaded_by': record.activation.startsWith('hook_') ? 'hook' : 'skill-runtime',
|
|
75
|
+
'context.input.activation': record.activation,
|
|
76
|
+
'context.input.hook_event': record.hook_event,
|
|
77
|
+
'context.input.scope': record.plugin === 'repo-local-skills' ? 'repo' : 'plugin',
|
|
78
|
+
'context.input.applies_to': session.agent,
|
|
79
|
+
'context.input.why_loaded': record.trigger,
|
|
80
|
+
'context.input.expected_benefit': record.expected_benefit,
|
|
81
|
+
'context.input.eval_gap': record.eval_gap,
|
|
82
|
+
'context.input.duplicate.dedupe_key': dedupeKey,
|
|
83
|
+
'context.input.duplicate.dedupe_scope': dedupeScope,
|
|
84
|
+
'context.input.duplicate.suppression_policy': record.suppression_policy,
|
|
85
|
+
'context.input.duplicate.role': record.duplicate_role,
|
|
86
|
+
'context.input.duplicate.candidate_count': duplicateCount,
|
|
87
|
+
'privacy.raw_context_recorded': false,
|
|
88
|
+
'privacy.raw_prompt_recorded': false,
|
|
89
|
+
'privacy.raw_tool_args_recorded': false
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
writeFileSync(receiptPath, `${events.map((event) => JSON.stringify(event)).join('\n')}\n`);
|
|
95
|
+
|
|
96
|
+
const trace = {
|
|
97
|
+
resourceSpans: [
|
|
98
|
+
{
|
|
99
|
+
resource: {
|
|
100
|
+
attributes: toOtelAttributes({
|
|
101
|
+
'service.name': 'pluribus-context-input-evidence',
|
|
102
|
+
'telemetry.sdk.language': 'javascript',
|
|
103
|
+
'pluribus.demo': 'skill-context-receipts'
|
|
104
|
+
})
|
|
105
|
+
},
|
|
106
|
+
scopeSpans: [
|
|
107
|
+
{
|
|
108
|
+
scope: { name: 'pluribus.context-input-evidence', version: '0.0.0-demo' },
|
|
109
|
+
spans: [
|
|
110
|
+
{
|
|
111
|
+
traceId: 'demo-trace-skill-context-receipts',
|
|
112
|
+
spanId: session.session_id,
|
|
113
|
+
name: 'agent.session',
|
|
114
|
+
kind: 'SPAN_KIND_INTERNAL',
|
|
115
|
+
startTimeUnixNano: '0',
|
|
116
|
+
endTimeUnixNano: '0',
|
|
117
|
+
attributes: toOtelAttributes({
|
|
118
|
+
'session.id': session.session_id,
|
|
119
|
+
'gen_ai.conversation.id': session.conversation_id,
|
|
120
|
+
'agent.name': session.agent,
|
|
121
|
+
'workspace.name': session.workspace,
|
|
122
|
+
'privacy.raw_context_recorded': false
|
|
123
|
+
}),
|
|
124
|
+
events: events.map((event) => ({
|
|
125
|
+
name: event.name,
|
|
126
|
+
timeUnixNano: '0',
|
|
127
|
+
attributes: toOtelAttributes(event.attributes)
|
|
128
|
+
}))
|
|
129
|
+
}
|
|
130
|
+
]
|
|
131
|
+
}
|
|
132
|
+
]
|
|
133
|
+
}
|
|
134
|
+
]
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
writeFileSync(tracePath, `${JSON.stringify(trace, null, 2)}\n`);
|
|
138
|
+
|
|
139
|
+
const selected = events.filter((event) => event.attributes['context.input.duplicate.role'] === 'selected').length;
|
|
140
|
+
const suppressed = events.filter((event) => event.attributes['context.input.duplicate.role'] === 'suppressed').length;
|
|
141
|
+
const rawLeakStrings = ['Weekly brief\nSummarize', 'After a commit, check tests', 'Post-commit review checklist loaded'];
|
|
142
|
+
const traceText = JSON.stringify(trace);
|
|
143
|
+
const receiptText = events.map((event) => JSON.stringify(event)).join('\n');
|
|
144
|
+
const leaksRawText = rawLeakStrings.some((value) => traceText.includes(value) || receiptText.includes(value));
|
|
145
|
+
|
|
146
|
+
const summary = {
|
|
147
|
+
schema: 'pluribus.skillContextReceipt.demo.v0',
|
|
148
|
+
eventCount: events.length,
|
|
149
|
+
selectedSkillLoads: selected,
|
|
150
|
+
suppressedDuplicateLoads: suppressed,
|
|
151
|
+
activations: [...new Set(events.map((event) => event.attributes['context.input.activation']))],
|
|
152
|
+
hookEvents: [...new Set(events.map((event) => event.attributes['context.input.hook_event']))],
|
|
153
|
+
includesExpectedBenefit: events.every((event) => Boolean(event.attributes['context.input.expected_benefit'])),
|
|
154
|
+
includesEvalGap: events.every((event) => Boolean(event.attributes['context.input.eval_gap'])),
|
|
155
|
+
rawTextCopiedToReceipt: leaksRawText,
|
|
156
|
+
receiptPath: 'examples/context-input-evidence/skill-receipt.ndjson',
|
|
157
|
+
tracePath: 'examples/context-input-evidence/skill-otel-trace.json',
|
|
158
|
+
lesson: 'Skill telemetry should prove invocation, activation, delivered identity, duplicate policy, expected benefit, and eval gaps without logging raw skill prompts.'
|
|
159
|
+
};
|
|
160
|
+
|
|
161
|
+
console.log(JSON.stringify(summary, null, 2));
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
4
|
+
import { dirname, join, resolve } from 'node:path';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
6
|
+
|
|
7
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
8
|
+
const inputPath = process.argv[2] ? resolve(process.argv[2]) : join(here, 'sample-skill-registry-log.jsonl');
|
|
9
|
+
const receiptPath = process.argv[3] ? resolve(process.argv[3]) : join(here, 'skill-registry-receipt.ndjson');
|
|
10
|
+
const tracePath = process.argv[4] ? resolve(process.argv[4]) : join(here, 'skill-registry-otel-trace.json');
|
|
11
|
+
|
|
12
|
+
function sha256(value) {
|
|
13
|
+
return `sha256:${createHash('sha256').update(String(value)).digest('hex')}`;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function readJsonl(path) {
|
|
17
|
+
return readFileSync(path, 'utf8')
|
|
18
|
+
.trim()
|
|
19
|
+
.split('\n')
|
|
20
|
+
.filter(Boolean)
|
|
21
|
+
.map((line, index) => {
|
|
22
|
+
try {
|
|
23
|
+
return JSON.parse(line);
|
|
24
|
+
} catch (error) {
|
|
25
|
+
throw new Error(`Invalid JSONL at ${path}:${index + 1}: ${error.message}`);
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function unixNano(isoTimestamp) {
|
|
31
|
+
return `${BigInt(Date.parse(isoTimestamp)) * 1_000_000n}`;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function otelValue(value) {
|
|
35
|
+
if (Array.isArray(value)) return { arrayValue: { values: value.map((item) => otelValue(item)) } };
|
|
36
|
+
if (typeof value === 'boolean') return { boolValue: value };
|
|
37
|
+
if (typeof value === 'number' && Number.isInteger(value)) return { intValue: String(value) };
|
|
38
|
+
if (typeof value === 'number') return { doubleValue: value };
|
|
39
|
+
if (typeof value === 'string') {
|
|
40
|
+
if (value === 'true' || value === 'false') return { boolValue: value === 'true' };
|
|
41
|
+
if (/^-?\d+$/.test(value)) return { intValue: value };
|
|
42
|
+
return { stringValue: value };
|
|
43
|
+
}
|
|
44
|
+
if (value == null) return { stringValue: '' };
|
|
45
|
+
return { stringValue: JSON.stringify(value) };
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function attributesToOtel(attributes) {
|
|
49
|
+
return Object.entries(attributes).map(([key, value]) => ({ key, value: otelValue(value) }));
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const records = readJsonl(inputPath);
|
|
53
|
+
const index = records.find((record) => record.type === 'skill.index');
|
|
54
|
+
if (!index) throw new Error(`No skill.index record found in ${inputPath}`);
|
|
55
|
+
|
|
56
|
+
const sessionId = index.session_id ?? 'demo-session-skill-registry';
|
|
57
|
+
const conversationId = index.conversation_id ?? sessionId;
|
|
58
|
+
const traceId = sha256(`${sessionId}:trace`).replace('sha256:', '').slice(0, 32);
|
|
59
|
+
const spanId = sha256(`${sessionId}:span`).replace('sha256:', '').slice(0, 16);
|
|
60
|
+
|
|
61
|
+
const events = records.map((record) => {
|
|
62
|
+
const skillIdHash = record.skill_id ? sha256(record.skill_id) : undefined;
|
|
63
|
+
const baseAttributes = {
|
|
64
|
+
'session.id': sessionId,
|
|
65
|
+
'gen_ai.conversation.id': record.conversation_id ?? conversationId,
|
|
66
|
+
'context.skill.registry.scope': record.registry_scope ?? index.registry_scope ?? 'unknown',
|
|
67
|
+
'context.skill.privacy.raw_body_exported': false,
|
|
68
|
+
'context.skill.audit_gap': 'receipt proves registry/index/read/injection boundary, not semantic skill quality'
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
if (record.type === 'skill.index') {
|
|
72
|
+
return {
|
|
73
|
+
trace_id: traceId,
|
|
74
|
+
span_id: spanId,
|
|
75
|
+
name: 'context.skill.registry.index.loaded',
|
|
76
|
+
time: record.time,
|
|
77
|
+
attributes: {
|
|
78
|
+
...baseAttributes,
|
|
79
|
+
'context.skill.index.strategy': record.index_strategy ?? 'unknown',
|
|
80
|
+
'context.skill.candidate_count': record.candidate_skill_count ?? 0,
|
|
81
|
+
'context.skill.indexed_count': record.indexed_skill_count ?? 0,
|
|
82
|
+
'context.skill.index.hash': sha256(`${sessionId}:skill-index:${record.indexed_skill_count ?? 0}:${record.index_strategy ?? 'unknown'}`),
|
|
83
|
+
'context.skill.index.token_bucket': record.index_token_bucket ?? 'unknown',
|
|
84
|
+
'context.skill.body.token_bucket': record.body_token_bucket ?? 'not_loaded',
|
|
85
|
+
'context.skill.cache.status': record.cache_status ?? 'unknown',
|
|
86
|
+
'context.skill.injection.policy': record.injection_policy ?? 'unknown'
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
if (record.type === 'skill.store') {
|
|
92
|
+
return {
|
|
93
|
+
trace_id: traceId,
|
|
94
|
+
span_id: spanId,
|
|
95
|
+
name: 'context.skill.registry.skill.stored',
|
|
96
|
+
time: record.time,
|
|
97
|
+
attributes: {
|
|
98
|
+
...baseAttributes,
|
|
99
|
+
'context.skill.id_hash': skillIdHash,
|
|
100
|
+
'context.skill.source': record.source ?? 'unknown',
|
|
101
|
+
'context.skill.store.reason': record.store_reason ?? 'unknown',
|
|
102
|
+
'context.skill.body.hash': sha256(`${record.skill_id}:body:${record.body_token_bucket ?? 'unknown'}`),
|
|
103
|
+
'context.skill.body.token_bucket': record.body_token_bucket ?? 'unknown',
|
|
104
|
+
'context.skill.write.status': record.write_status ?? 'unknown'
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
if (record.type === 'skill.read') {
|
|
110
|
+
return {
|
|
111
|
+
trace_id: traceId,
|
|
112
|
+
span_id: spanId,
|
|
113
|
+
name: 'context.skill.registry.skill.read',
|
|
114
|
+
time: record.time,
|
|
115
|
+
attributes: {
|
|
116
|
+
...baseAttributes,
|
|
117
|
+
'context.skill.id_hash': skillIdHash,
|
|
118
|
+
'context.skill.read.reason': record.read_reason ?? 'unknown',
|
|
119
|
+
'context.skill.body.hash': sha256(`${record.skill_id}:body:${record.body_token_bucket ?? 'unknown'}`),
|
|
120
|
+
'context.skill.body.token_bucket': record.body_token_bucket ?? 'unknown',
|
|
121
|
+
'context.skill.read.status': record.read_status ?? 'unknown'
|
|
122
|
+
}
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
if (record.type === 'skill.inject') {
|
|
127
|
+
return {
|
|
128
|
+
trace_id: traceId,
|
|
129
|
+
span_id: spanId,
|
|
130
|
+
name: 'context.skill.registry.skill.injected',
|
|
131
|
+
time: record.time,
|
|
132
|
+
attributes: {
|
|
133
|
+
...baseAttributes,
|
|
134
|
+
'context.skill.id_hash': skillIdHash,
|
|
135
|
+
'context.skill.injection.reason': record.injection_reason ?? 'unknown',
|
|
136
|
+
'context.skill.delivered.hash': sha256(`${record.skill_id}:delivered:${record.delivered_token_bucket ?? 'unknown'}`),
|
|
137
|
+
'context.skill.delivered.token_bucket': record.delivered_token_bucket ?? 'unknown',
|
|
138
|
+
'context.skill.delivery.status': record.delivery_status ?? 'unknown',
|
|
139
|
+
'context.skill.suppressed_count': record.suppressed_count ?? 0
|
|
140
|
+
}
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
if (record.type === 'skill.reuse') {
|
|
145
|
+
const selectedCount = record.selected_count ?? 0;
|
|
146
|
+
const decisiveCount = record.decisive_count ?? 0;
|
|
147
|
+
const supportingCount = record.supporting_count ?? 0;
|
|
148
|
+
const unusedCount = record.unused_count ?? 0;
|
|
149
|
+
const unknownCount = record.unknown_count ?? 0;
|
|
150
|
+
const accountedCount = decisiveCount + supportingCount + unusedCount + unknownCount;
|
|
151
|
+
if (accountedCount !== selectedCount) {
|
|
152
|
+
throw new Error(`Invalid skill reuse accounting: selected_count (${selectedCount}) must equal decisive + supporting + unused + unknown (${accountedCount})`);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
return {
|
|
156
|
+
trace_id: traceId,
|
|
157
|
+
span_id: spanId,
|
|
158
|
+
name: 'context.skill.registry.reuse.evaluated',
|
|
159
|
+
time: record.time,
|
|
160
|
+
attributes: {
|
|
161
|
+
...baseAttributes,
|
|
162
|
+
'decision.id_hash': sha256(record.decision_id ?? 'unknown-decision'),
|
|
163
|
+
'context.skill.id_hash': skillIdHash,
|
|
164
|
+
'context.skill.selected_count': selectedCount,
|
|
165
|
+
'context.skill.suppressed_count': record.suppressed_count ?? 0,
|
|
166
|
+
'context.skill.relevance.decisive_count': decisiveCount,
|
|
167
|
+
'context.skill.relevance.supporting_count': supportingCount,
|
|
168
|
+
'context.skill.relevance.unused_count': unusedCount,
|
|
169
|
+
'context.skill.relevance.unknown_count': unknownCount,
|
|
170
|
+
'context.skill.relevance.accounted_count': accountedCount,
|
|
171
|
+
'context.skill.relevance.invariant': 'selected_count == decisive_count + supporting_count + unused_count + unknown_count',
|
|
172
|
+
'context.skill.relevance.outcome': record.outcome ?? 'unknown'
|
|
173
|
+
}
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
throw new Error(`Unsupported record type: ${record.type}`);
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
writeFileSync(receiptPath, `${events.map((event) => JSON.stringify(event)).join('\n')}\n`);
|
|
181
|
+
|
|
182
|
+
const eventTimes = events.map((event) => Date.parse(event.time)).filter(Number.isFinite);
|
|
183
|
+
const startTimeMs = Math.min(...eventTimes);
|
|
184
|
+
const endTimeMs = Math.max(...eventTimes) + 1;
|
|
185
|
+
|
|
186
|
+
const otlpTrace = {
|
|
187
|
+
resourceSpans: [
|
|
188
|
+
{
|
|
189
|
+
resource: {
|
|
190
|
+
attributes: attributesToOtel({
|
|
191
|
+
'service.name': 'pluribus-skill-registry-demo',
|
|
192
|
+
'service.version': '0.0.0-fixture',
|
|
193
|
+
'deployment.environment.name': 'local-fixture'
|
|
194
|
+
})
|
|
195
|
+
},
|
|
196
|
+
scopeSpans: [
|
|
197
|
+
{
|
|
198
|
+
scope: { name: 'pluribus.skill_registry.demo', version: '0.0.0-fixture' },
|
|
199
|
+
spans: [
|
|
200
|
+
{
|
|
201
|
+
traceId,
|
|
202
|
+
spanId,
|
|
203
|
+
parentSpanId: '',
|
|
204
|
+
name: 'agent.session',
|
|
205
|
+
kind: 1,
|
|
206
|
+
startTimeUnixNano: `${BigInt(startTimeMs) * 1_000_000n}`,
|
|
207
|
+
endTimeUnixNano: `${BigInt(endTimeMs) * 1_000_000n}`,
|
|
208
|
+
attributes: attributesToOtel({
|
|
209
|
+
'session.id': sessionId,
|
|
210
|
+
'gen_ai.conversation.id': conversationId,
|
|
211
|
+
'gen_ai.agent.name': index.agent ?? 'unknown',
|
|
212
|
+
'gen_ai.operation.name': 'agent_session'
|
|
213
|
+
}),
|
|
214
|
+
events: events.map((event) => ({
|
|
215
|
+
name: event.name,
|
|
216
|
+
timeUnixNano: unixNano(event.time),
|
|
217
|
+
attributes: attributesToOtel(event.attributes)
|
|
218
|
+
}))
|
|
219
|
+
}
|
|
220
|
+
]
|
|
221
|
+
}
|
|
222
|
+
]
|
|
223
|
+
}
|
|
224
|
+
]
|
|
225
|
+
};
|
|
226
|
+
|
|
227
|
+
writeFileSync(tracePath, `${JSON.stringify(otlpTrace, null, 2)}\n`);
|
|
228
|
+
|
|
229
|
+
const rawLeakNeedles = [
|
|
230
|
+
'Acme-Co',
|
|
231
|
+
'Stripe prod incident',
|
|
232
|
+
'webhook secret',
|
|
233
|
+
'sk_live_private_demo',
|
|
234
|
+
'/private/work/acme',
|
|
235
|
+
'customer payload'
|
|
236
|
+
];
|
|
237
|
+
const receiptText = readFileSync(receiptPath, 'utf8');
|
|
238
|
+
const traceText = readFileSync(tracePath, 'utf8');
|
|
239
|
+
for (const needle of rawLeakNeedles) {
|
|
240
|
+
if (receiptText.includes(needle) || traceText.includes(needle)) {
|
|
241
|
+
throw new Error(`Raw/private fixture content leaked into receipt or trace: ${needle}`);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
console.log(`Wrote ${events.length} skill registry receipt events to ${receiptPath}`);
|
|
246
|
+
console.log(`Wrote OTLP-style trace to ${tracePath}`);
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
4
|
+
import { dirname, join, resolve } from 'node:path';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
6
|
+
|
|
7
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
8
|
+
const inputPath = process.argv[2] ? resolve(process.argv[2]) : join(here, 'sample-skill-routing-log.jsonl');
|
|
9
|
+
const receiptPath = process.argv[3] ? resolve(process.argv[3]) : join(here, 'skill-routing-receipt.ndjson');
|
|
10
|
+
const tracePath = process.argv[4] ? resolve(process.argv[4]) : join(here, 'skill-routing-otel-trace.json');
|
|
11
|
+
|
|
12
|
+
function sha256(value) {
|
|
13
|
+
return `sha256:${createHash('sha256').update(value ?? '').digest('hex')}`;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function hashRef(value) {
|
|
17
|
+
return sha256(value ?? '').slice(0, 19);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function readJsonl(path) {
|
|
21
|
+
return readFileSync(path, 'utf8')
|
|
22
|
+
.trim()
|
|
23
|
+
.split('\n')
|
|
24
|
+
.filter(Boolean)
|
|
25
|
+
.map((line, index) => {
|
|
26
|
+
try {
|
|
27
|
+
return JSON.parse(line);
|
|
28
|
+
} catch (error) {
|
|
29
|
+
throw new Error(`Invalid JSONL at ${path}:${index + 1}: ${error.message}`);
|
|
30
|
+
}
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function unixNano(isoTimestamp) {
|
|
35
|
+
return `${BigInt(Date.parse(isoTimestamp)) * 1_000_000n}`;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function otelValue(value) {
|
|
39
|
+
if (typeof value === 'boolean') return { boolValue: value };
|
|
40
|
+
if (typeof value === 'number' && Number.isInteger(value)) return { intValue: String(value) };
|
|
41
|
+
if (typeof value === 'number') return { doubleValue: value };
|
|
42
|
+
if (value == null) return { stringValue: '' };
|
|
43
|
+
return { stringValue: String(value) };
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function attributesToOtel(attributes) {
|
|
47
|
+
return Object.entries(attributes).map(([key, value]) => ({ key, value: otelValue(value) }));
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function tokenBucket(value) {
|
|
51
|
+
if (value < 1_000) return 'under_1k';
|
|
52
|
+
if (value < 10_000) return 'under_10k';
|
|
53
|
+
if (value < 50_000) return 'under_50k';
|
|
54
|
+
return 'over_50k';
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function scoreBucket(value) {
|
|
58
|
+
if (value >= 0.9) return 'gte_0_90';
|
|
59
|
+
if (value >= 0.75) return 'gte_0_75';
|
|
60
|
+
if (value >= 0.5) return 'gte_0_50';
|
|
61
|
+
return 'lt_0_50';
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function rateBucket(value) {
|
|
65
|
+
if (value >= 0.95) return 'gte_0_95';
|
|
66
|
+
if (value >= 0.75) return 'gte_0_75';
|
|
67
|
+
if (value >= 0.5) return 'gte_0_50';
|
|
68
|
+
return 'lt_0_50';
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const records = readJsonl(inputPath);
|
|
72
|
+
const benchmark = records.find((record) => record.type === 'benchmark.start');
|
|
73
|
+
const index = records.find((record) => record.type === 'skill.router.index.loaded');
|
|
74
|
+
const cases = records.filter((record) => record.type === 'skill.router.case.evaluated');
|
|
75
|
+
const bodyLoads = records.filter((record) => record.type === 'skill.body.loaded');
|
|
76
|
+
const completed = records.find((record) => record.type === 'skill.router.benchmark.completed');
|
|
77
|
+
|
|
78
|
+
if (!benchmark || !index || cases.length === 0 || !completed) {
|
|
79
|
+
throw new Error(`Expected benchmark.start, skill.router.index.loaded, skill.router.case.evaluated, and skill.router.benchmark.completed records in ${inputPath}`);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const traceSeed = `${benchmark.session_id}:${benchmark.benchmark_id}:skill-routing`;
|
|
83
|
+
const traceId = sha256(traceSeed).replace('sha256:', '').slice(0, 32);
|
|
84
|
+
const spanId = sha256(`${traceSeed}:span`).replace('sha256:', '').slice(0, 16);
|
|
85
|
+
|
|
86
|
+
const indexEvent = {
|
|
87
|
+
trace_id: traceId,
|
|
88
|
+
span_id: spanId,
|
|
89
|
+
name: 'skill.router.index.loaded',
|
|
90
|
+
time: index.time,
|
|
91
|
+
attributes: {
|
|
92
|
+
'session.id': benchmark.session_id,
|
|
93
|
+
'gen_ai.conversation.id': benchmark.conversation_id,
|
|
94
|
+
'agent.name': benchmark.agent,
|
|
95
|
+
'skill.catalog.id_hash': hashRef(index.catalog_id),
|
|
96
|
+
'skill.catalog.skill_count': index.skill_names.length,
|
|
97
|
+
'skill.catalog.names_hash': sha256(index.skill_names.join('\n')),
|
|
98
|
+
'skill.router.startup_strategy': index.startup_strategy,
|
|
99
|
+
'skill.router.description_token_count_bucket': tokenBucket(index.description_token_count),
|
|
100
|
+
'skill.router.full_body_token_count_bucket': tokenBucket(index.full_body_token_count),
|
|
101
|
+
'skill.router.full_bodies_loaded_at_startup': false,
|
|
102
|
+
'privacy.raw_skill_descriptions_recorded': false,
|
|
103
|
+
'privacy.raw_skill_bodies_recorded': false
|
|
104
|
+
}
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
const caseEvents = cases.map((record) => ({
|
|
108
|
+
trace_id: traceId,
|
|
109
|
+
span_id: spanId,
|
|
110
|
+
name: 'skill.router.case.evaluated',
|
|
111
|
+
time: record.time,
|
|
112
|
+
attributes: {
|
|
113
|
+
'session.id': benchmark.session_id,
|
|
114
|
+
'gen_ai.conversation.id': benchmark.conversation_id,
|
|
115
|
+
'skill.router.benchmark.id_hash': hashRef(benchmark.benchmark_id),
|
|
116
|
+
'skill.router.case.id_hash': hashRef(record.case_id),
|
|
117
|
+
'skill.router.prompt_hash': sha256(record.raw_prompt),
|
|
118
|
+
'skill.router.expected_skill_hash': hashRef(record.expected_skill),
|
|
119
|
+
'skill.router.selected_skill_hash': hashRef(record.selected_skill),
|
|
120
|
+
'skill.router.top_k_hash': sha256(record.top_k.join('\n')),
|
|
121
|
+
'skill.router.match': record.match,
|
|
122
|
+
'skill.router.confidence_bucket': scoreBucket(record.confidence),
|
|
123
|
+
'skill.router.reason_hash': sha256(record.reason),
|
|
124
|
+
'privacy.raw_prompt_recorded': false,
|
|
125
|
+
'privacy.raw_selection_reason_recorded': false
|
|
126
|
+
}
|
|
127
|
+
}));
|
|
128
|
+
|
|
129
|
+
const bodyLoadEvents = bodyLoads.map((record) => ({
|
|
130
|
+
trace_id: traceId,
|
|
131
|
+
span_id: spanId,
|
|
132
|
+
name: 'skill.body.loaded',
|
|
133
|
+
time: record.time,
|
|
134
|
+
attributes: {
|
|
135
|
+
'session.id': benchmark.session_id,
|
|
136
|
+
'gen_ai.conversation.id': benchmark.conversation_id,
|
|
137
|
+
'skill.router.case.id_hash': hashRef(record.case_id),
|
|
138
|
+
'skill.name_hash': hashRef(record.skill),
|
|
139
|
+
'skill.body.hash': sha256(record.raw_skill_body),
|
|
140
|
+
'skill.body.load_reason': record.load_reason,
|
|
141
|
+
'skill.body.loaded_after_route': true,
|
|
142
|
+
'privacy.raw_skill_body_recorded': false
|
|
143
|
+
}
|
|
144
|
+
}));
|
|
145
|
+
|
|
146
|
+
const completedEvent = {
|
|
147
|
+
trace_id: traceId,
|
|
148
|
+
span_id: spanId,
|
|
149
|
+
name: 'skill.router.benchmark.completed',
|
|
150
|
+
time: completed.time,
|
|
151
|
+
attributes: {
|
|
152
|
+
'session.id': benchmark.session_id,
|
|
153
|
+
'gen_ai.conversation.id': benchmark.conversation_id,
|
|
154
|
+
'skill.router.benchmark.id_hash': hashRef(benchmark.benchmark_id),
|
|
155
|
+
'skill.router.golden_set_hash': hashRef(benchmark.golden_set),
|
|
156
|
+
'skill.router.records_total': completed.records_total,
|
|
157
|
+
'skill.router.usable_records': completed.usable_records,
|
|
158
|
+
'skill.router.format_failures': completed.format_failures,
|
|
159
|
+
'skill.router.top1_rate_bucket': rateBucket(completed.top1_rate),
|
|
160
|
+
'skill.router.top2_rate_bucket': rateBucket(completed.top2_rate),
|
|
161
|
+
'skill.router.model_results_hash': sha256(JSON.stringify(completed.model_results)),
|
|
162
|
+
'skill.router.next_action_hash': sha256(completed.next_action),
|
|
163
|
+
'skill.router.audit_gap': 'receipt_proves_routing_boundary_not_task_effectiveness',
|
|
164
|
+
'privacy.raw_benchmark_prompts_recorded': false,
|
|
165
|
+
'privacy.raw_skill_text_recorded': false
|
|
166
|
+
}
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
const events = [indexEvent, ...caseEvents, ...bodyLoadEvents, completedEvent]
|
|
170
|
+
.sort((left, right) => Date.parse(left.time) - Date.parse(right.time));
|
|
171
|
+
|
|
172
|
+
writeFileSync(receiptPath, `${events.map((event) => JSON.stringify(event)).join('\n')}\n`);
|
|
173
|
+
|
|
174
|
+
const trace = {
|
|
175
|
+
resourceSpans: [
|
|
176
|
+
{
|
|
177
|
+
resource: {
|
|
178
|
+
attributes: attributesToOtel({
|
|
179
|
+
'service.name': 'pluribus-skill-routing-receipt-demo',
|
|
180
|
+
'service.version': '0.0.0-fixture',
|
|
181
|
+
'deployment.environment.name': 'local-fixture'
|
|
182
|
+
})
|
|
183
|
+
},
|
|
184
|
+
scopeSpans: [
|
|
185
|
+
{
|
|
186
|
+
scope: {
|
|
187
|
+
name: 'pluribus.context_input_evidence.skill_routing_demo',
|
|
188
|
+
version: '0.0.0-fixture'
|
|
189
|
+
},
|
|
190
|
+
spans: [
|
|
191
|
+
{
|
|
192
|
+
traceId,
|
|
193
|
+
spanId,
|
|
194
|
+
parentSpanId: '',
|
|
195
|
+
name: 'agent.session.skill.routing_benchmark',
|
|
196
|
+
kind: 1,
|
|
197
|
+
startTimeUnixNano: unixNano(benchmark.time),
|
|
198
|
+
endTimeUnixNano: unixNano(completed.time),
|
|
199
|
+
attributes: attributesToOtel({
|
|
200
|
+
'session.id': benchmark.session_id,
|
|
201
|
+
'gen_ai.conversation.id': benchmark.conversation_id,
|
|
202
|
+
'agent.name': benchmark.agent,
|
|
203
|
+
'workspace.name': benchmark.workspace,
|
|
204
|
+
'gen_ai.request.model': benchmark.model,
|
|
205
|
+
'skill.router.version_hash': hashRef(benchmark.router_version),
|
|
206
|
+
'skill.router.case_count': cases.length
|
|
207
|
+
}),
|
|
208
|
+
events: events.map((event) => ({
|
|
209
|
+
name: event.name,
|
|
210
|
+
timeUnixNano: unixNano(event.time),
|
|
211
|
+
attributes: attributesToOtel(event.attributes)
|
|
212
|
+
}))
|
|
213
|
+
}
|
|
214
|
+
]
|
|
215
|
+
}
|
|
216
|
+
]
|
|
217
|
+
}
|
|
218
|
+
]
|
|
219
|
+
};
|
|
220
|
+
|
|
221
|
+
writeFileSync(tracePath, `${JSON.stringify(trace, null, 2)}\n`);
|
|
222
|
+
|
|
223
|
+
const forbiddenRawStrings = [
|
|
224
|
+
'private activation prompts',
|
|
225
|
+
'internal corpus notes',
|
|
226
|
+
'unpublished skill body drafts',
|
|
227
|
+
'private draft description',
|
|
228
|
+
'private task: agent is losing the goal',
|
|
229
|
+
'private task: keep tool outputs out of prompt',
|
|
230
|
+
'private task: reconcile persistent agent memories',
|
|
231
|
+
'private SKILL.md body draft',
|
|
232
|
+
'operator notes',
|
|
233
|
+
'internal evaluation hints',
|
|
234
|
+
'split compression vs memory-consolidation activation examples'
|
|
235
|
+
];
|
|
236
|
+
const exportedText = `${events.map((event) => JSON.stringify(event)).join('\n')}\n${JSON.stringify(trace)}`;
|
|
237
|
+
const rawTextCopiedToReceipt = forbiddenRawStrings.some((value) => exportedText.includes(value));
|
|
238
|
+
|
|
239
|
+
const summary = {
|
|
240
|
+
schema: 'pluribus.skillRoutingReceipt.demo.v0',
|
|
241
|
+
eventCount: events.length,
|
|
242
|
+
evaluatedCases: cases.length,
|
|
243
|
+
loadedSkillBodies: bodyLoads.length,
|
|
244
|
+
indexOnlyAtStartup: indexEvent.attributes['skill.router.full_bodies_loaded_at_startup'] === false,
|
|
245
|
+
top1RateBucket: completedEvent.attributes['skill.router.top1_rate_bucket'],
|
|
246
|
+
includesAuditGap: completedEvent.attributes['skill.router.audit_gap'],
|
|
247
|
+
rawTextCopiedToReceipt,
|
|
248
|
+
receiptPath: 'examples/context-input-evidence/skill-routing-receipt.ndjson',
|
|
249
|
+
tracePath: 'examples/context-input-evidence/skill-routing-otel-trace.json',
|
|
250
|
+
lesson: 'Skill routing benchmarks need privacy-safe receipts: prove which description index loaded, which skill was selected for each activation case, which body expanded after routing, and where routing accuracy stops short of task effectiveness.'
|
|
251
|
+
};
|
|
252
|
+
|
|
253
|
+
console.log(JSON.stringify(summary, null, 2));
|