total-recall-brain 3.20.1 → 3.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/total-recall.mjs +3 -0
- package/frontend/src/api/system.ts +31 -0
- package/frontend/src/pages/UsagePage.tsx +87 -1
- package/package.json +1 -1
- package/src/cli/proposals.mjs +216 -0
- package/src/cli/rebuild.mjs +27 -2
- package/src/cli/secret.mjs +175 -1
- package/src/core/daemon-loop.mjs +13 -0
- package/src/core/dream.mjs +98 -7
- package/src/core/optimizer.mjs +222 -38
- package/src/core/proposal-applier.mjs +479 -0
- package/src/core/runtime.mjs +6 -2
- package/src/core/secrets-remote-deploy.mjs +249 -0
- package/src/core/surface.mjs +36 -6
- package/src/core/throttled-fetch.mjs +25 -1
- package/src/core/usage-fetcher.mjs +258 -0
- package/src/server/index.mjs +9 -9
- package/src/server/rest.mjs +2 -0
- package/src/server/route-manifest.json +28 -0
- package/src/server/routes/proposals.mjs +116 -0
- package/src/server/routes/system.mjs +16 -0
package/bin/total-recall.mjs
CHANGED
|
@@ -60,6 +60,8 @@ const COMMANDS = {
|
|
|
60
60
|
recall: 'recall.mjs',
|
|
61
61
|
search: 'recall.mjs',
|
|
62
62
|
research: 'research.mjs',
|
|
63
|
+
proposals: 'proposals.mjs',
|
|
64
|
+
proposal: 'proposals.mjs',
|
|
63
65
|
task: 'task.mjs',
|
|
64
66
|
share: 'share.mjs',
|
|
65
67
|
'generate-pat': 'generate-pat.mjs',
|
|
@@ -102,6 +104,7 @@ function printHelp() {
|
|
|
102
104
|
recall Read-path hybrid search
|
|
103
105
|
compile Rebuild instruction surfaces
|
|
104
106
|
dream Memory consolidation cycle
|
|
107
|
+
proposals <cmd> Review / apply optimizer proposals (list|apply|revert)
|
|
105
108
|
task <cmd> Enqueue daemon work (open envelope)
|
|
106
109
|
daemon <cmd> Background worker (start|stop|status)
|
|
107
110
|
skill <cmd> Skills registry, deploy, multi-repo sync
|
|
@@ -16,6 +16,37 @@ export async function fetchUsageStats(): Promise<UsageData> {
|
|
|
16
16
|
return res.json()
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
+
/** One provider's reported spend, or the reason we could not read it. */
|
|
20
|
+
export interface ProviderUsageEntry {
|
|
21
|
+
label: string
|
|
22
|
+
/** `ok` carries numbers; every other state carries a `reason` instead. */
|
|
23
|
+
state: 'ok' | 'no_key' | 'needs_admin_key' | 'unsupported' | 'error'
|
|
24
|
+
reason?: string
|
|
25
|
+
secret_name?: string
|
|
26
|
+
total_cost?: number
|
|
27
|
+
currency?: string
|
|
28
|
+
balance?: number
|
|
29
|
+
/** True when the figure is lifetime-to-date rather than the requested window. */
|
|
30
|
+
lifetime?: boolean
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface ProviderUsage {
|
|
34
|
+
fetched_at: string
|
|
35
|
+
window_days: number
|
|
36
|
+
from_cache?: boolean
|
|
37
|
+
providers: Record<string, ProviderUsageEntry>
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Spend as the providers report it, rather than estimated from local token counts.
|
|
42
|
+
* Served from an hourly cache unless `refresh` is set.
|
|
43
|
+
*/
|
|
44
|
+
export async function fetchProviderUsage(refresh = false): Promise<ProviderUsage> {
|
|
45
|
+
const res = await apiFetch(`${API_BASE}/api/usage/providers${refresh ? '?refresh=1' : ''}`)
|
|
46
|
+
if (!res.ok) throw new Error(`Provider usage API error: ${res.status}`)
|
|
47
|
+
return res.json()
|
|
48
|
+
}
|
|
49
|
+
|
|
19
50
|
export async function fetchLogs(type: 'server' | 'daemon'): Promise<{ content: string }> {
|
|
20
51
|
const res = await apiFetch(`${API_BASE}/api/logs/${type}`)
|
|
21
52
|
if (!res.ok) throw new Error(`Logs API error: ${res.status}`)
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { useState, useEffect, useMemo } from 'react';
|
|
2
|
-
import { fetchUsageStats, fetchConfigJson } from '../api';
|
|
2
|
+
import { fetchUsageStats, fetchConfigJson, fetchProviderUsage } from '../api';
|
|
3
|
+
import type { ProviderUsage } from '../api';
|
|
3
4
|
import type { ConfigJson, UsageData } from '../types';
|
|
4
5
|
import { AreaChart, Area, XAxis, YAxis, CartesianGrid, Tooltip, ResponsiveContainer, Legend } from 'recharts';
|
|
5
6
|
|
|
@@ -8,6 +9,9 @@ export default function UsagePage() {
|
|
|
8
9
|
const [config, setConfig] = useState<ConfigJson | null>(null);
|
|
9
10
|
const [loading, setLoading] = useState(true);
|
|
10
11
|
const [error, setError] = useState<string | null>(null);
|
|
12
|
+
const [providerUsage, setProviderUsage] = useState<ProviderUsage | null>(null);
|
|
13
|
+
const [providerLoading, setProviderLoading] = useState(false);
|
|
14
|
+
const [providerError, setProviderError] = useState<string | null>(null);
|
|
11
15
|
|
|
12
16
|
const fetchUsageAndConfig = async () => {
|
|
13
17
|
try {
|
|
@@ -24,6 +28,18 @@ export default function UsagePage() {
|
|
|
24
28
|
}
|
|
25
29
|
};
|
|
26
30
|
|
|
31
|
+
const loadProviderUsage = async (refresh = false) => {
|
|
32
|
+
setProviderLoading(true);
|
|
33
|
+
try {
|
|
34
|
+
setProviderUsage(await fetchProviderUsage(refresh));
|
|
35
|
+
setProviderError(null);
|
|
36
|
+
} catch (err: unknown) {
|
|
37
|
+
setProviderError((err as Error).message || 'Failed to read provider billing data.');
|
|
38
|
+
} finally {
|
|
39
|
+
setProviderLoading(false);
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
|
|
27
43
|
useEffect(() => {
|
|
28
44
|
// eslint-disable-next-line react-hooks/set-state-in-effect -- legitimate statistics sync on mount
|
|
29
45
|
void fetchUsageAndConfig();
|
|
@@ -31,6 +47,13 @@ export default function UsagePage() {
|
|
|
31
47
|
return () => clearInterval(interval);
|
|
32
48
|
}, []);
|
|
33
49
|
|
|
50
|
+
useEffect(() => {
|
|
51
|
+
// Deliberately not on the 5s ledger interval — this hits live provider
|
|
52
|
+
// billing APIs. The server caches hourly; refreshing is a manual action.
|
|
53
|
+
// eslint-disable-next-line react-hooks/set-state-in-effect -- initial fetch on mount
|
|
54
|
+
void loadProviderUsage();
|
|
55
|
+
}, []);
|
|
56
|
+
|
|
34
57
|
|
|
35
58
|
|
|
36
59
|
const dailyCap = config?.budget?.budget?.daily_cap_usd ?? 5.0;
|
|
@@ -611,6 +634,69 @@ export default function UsagePage() {
|
|
|
611
634
|
</div>
|
|
612
635
|
</div>
|
|
613
636
|
|
|
637
|
+
{/* ── Provider-Reported Spend ──
|
|
638
|
+
Everything above is estimated locally: observed tokens × a hardcoded
|
|
639
|
+
price table. This panel is what the providers themselves bill. */}
|
|
640
|
+
<div className="card" style={{ padding: 24, marginTop: 24, background: 'rgba(18, 18, 26, 0.6)', backdropFilter: 'blur(8px)', border: '1px solid var(--border)' }}>
|
|
641
|
+
<div style={{ display: 'flex', alignItems: 'baseline', justifyContent: 'space-between', borderBottom: '1px solid var(--border)', paddingBottom: 10, marginBottom: 16 }}>
|
|
642
|
+
<h3 style={{ fontSize: 14, fontWeight: 700, color: 'var(--text-tertiary)', textTransform: 'uppercase', letterSpacing: 0.5, margin: 0 }}>
|
|
643
|
+
Reported by Provider
|
|
644
|
+
</h3>
|
|
645
|
+
<div style={{ display: 'flex', alignItems: 'center', gap: 12 }}>
|
|
646
|
+
{providerUsage?.fetched_at && (
|
|
647
|
+
<span style={{ fontSize: 11, color: 'var(--text-tertiary)' }}>
|
|
648
|
+
{new Date(providerUsage.fetched_at).toLocaleString()}
|
|
649
|
+
{providerUsage.from_cache ? ' (cached)' : ''}
|
|
650
|
+
</span>
|
|
651
|
+
)}
|
|
652
|
+
<button
|
|
653
|
+
type="button"
|
|
654
|
+
onClick={() => { void loadProviderUsage(true); }}
|
|
655
|
+
disabled={providerLoading}
|
|
656
|
+
style={{ fontSize: 11, padding: '4px 10px', borderRadius: 6, border: '1px solid var(--border)', background: 'transparent', color: 'var(--text-secondary)', cursor: providerLoading ? 'wait' : 'pointer' }}
|
|
657
|
+
>
|
|
658
|
+
{providerLoading ? 'Refreshing…' : 'Refresh'}
|
|
659
|
+
</button>
|
|
660
|
+
</div>
|
|
661
|
+
</div>
|
|
662
|
+
|
|
663
|
+
{providerError && (
|
|
664
|
+
<div style={{ fontSize: 12, color: 'var(--error)', marginBottom: 12 }}>{providerError}</div>
|
|
665
|
+
)}
|
|
666
|
+
|
|
667
|
+
<div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(260px, 1fr))', gap: 16 }}>
|
|
668
|
+
{Object.entries(providerUsage?.providers || {}).map(([id, p]) => (
|
|
669
|
+
<div key={id} style={{
|
|
670
|
+
border: '1px solid var(--border)',
|
|
671
|
+
borderLeft: `3px solid ${p.state === 'ok' ? 'var(--success)' : p.state === 'error' ? 'var(--error)' : 'var(--warning)'}`,
|
|
672
|
+
borderRadius: 8,
|
|
673
|
+
padding: 16,
|
|
674
|
+
}}>
|
|
675
|
+
<div style={{ fontSize: 13, fontWeight: 700, color: '#fff', marginBottom: 8 }}>{p.label}</div>
|
|
676
|
+
{p.state === 'ok' ? (
|
|
677
|
+
<>
|
|
678
|
+
<div style={{ fontSize: 22, fontWeight: 700, color: '#fff' }}>
|
|
679
|
+
${(p.total_cost ?? 0).toFixed(2)}
|
|
680
|
+
</div>
|
|
681
|
+
<div style={{ fontSize: 11, color: 'var(--text-tertiary)', marginTop: 4 }}>
|
|
682
|
+
{/* Never label a lifetime figure as a windowed one. */}
|
|
683
|
+
{p.lifetime ? 'lifetime to date' : `last ${providerUsage?.window_days ?? 30} days`}
|
|
684
|
+
{typeof p.balance === 'number' ? ` · $${p.balance.toFixed(2)} remaining` : ''}
|
|
685
|
+
</div>
|
|
686
|
+
</>
|
|
687
|
+
) : (
|
|
688
|
+
<div style={{ fontSize: 11, color: 'var(--text-secondary)', lineHeight: 1.5 }}>
|
|
689
|
+
{p.reason}
|
|
690
|
+
</div>
|
|
691
|
+
)}
|
|
692
|
+
</div>
|
|
693
|
+
))}
|
|
694
|
+
{!providerUsage && !providerError && (
|
|
695
|
+
<div style={{ fontSize: 12, color: 'var(--text-tertiary)' }}>Loading provider billing data…</div>
|
|
696
|
+
)}
|
|
697
|
+
</div>
|
|
698
|
+
</div>
|
|
699
|
+
|
|
614
700
|
</div>
|
|
615
701
|
);
|
|
616
702
|
}
|
package/package.json
CHANGED
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { resolveBrainDir, parseLayerFlag } from './agent-dir.mjs';
|
|
3
|
+
import {
|
|
4
|
+
listProposals,
|
|
5
|
+
getProposal,
|
|
6
|
+
applyProposal,
|
|
7
|
+
setProposalStatus,
|
|
8
|
+
revertProposal,
|
|
9
|
+
hasHandler,
|
|
10
|
+
AUTO_APPLICABLE_TOPICS,
|
|
11
|
+
} from '../core/proposal-applier.mjs';
|
|
12
|
+
import { findStaleNodes } from '../core/optimizer.mjs';
|
|
13
|
+
|
|
14
|
+
function printHelp() {
|
|
15
|
+
console.log(`
|
|
16
|
+
total-recall proposals — Review and apply optimizer proposals
|
|
17
|
+
|
|
18
|
+
Usage: total-recall proposals <subcommand> [options]
|
|
19
|
+
|
|
20
|
+
Subcommands:
|
|
21
|
+
list Show open proposals (default)
|
|
22
|
+
show <id> Full detail for one proposal
|
|
23
|
+
apply <id> Perform the proposed work
|
|
24
|
+
reject <id> [reason] Refuse a proposal permanently
|
|
25
|
+
revert <id> Undo an applied proposal from its snapshot
|
|
26
|
+
supersede --topic <t> Bulk-close every open proposal of a retired topic
|
|
27
|
+
stale Nodes due for re-verification (read-only query)
|
|
28
|
+
|
|
29
|
+
Options:
|
|
30
|
+
--status <s> Filter: draft | accepted | applied | rejected | superseded | all
|
|
31
|
+
--topic <t> Filter by proposal topic
|
|
32
|
+
--dry-run With apply: show the change without writing
|
|
33
|
+
--global Use the global brain
|
|
34
|
+
--project Use the project brain
|
|
35
|
+
--help, -h Show this help
|
|
36
|
+
|
|
37
|
+
${AUTO_APPLICABLE_TOPICS.size > 0
|
|
38
|
+
? ` Applied automatically during the dream cycle: ${[...AUTO_APPLICABLE_TOPICS].join(', ')}.
|
|
39
|
+
Everything else waits here for review.`
|
|
40
|
+
: ` Nothing is applied automatically in this release — every proposal waits for
|
|
41
|
+
you to run \`apply\`. Applying is non-destructive (duplicates are superseded,
|
|
42
|
+
never deleted) and \`revert\` restores the exact prior bytes.`}
|
|
43
|
+
|
|
44
|
+
Examples:
|
|
45
|
+
npx total-recall proposals
|
|
46
|
+
npx total-recall proposals show prop_a1b2c3d4e5f6
|
|
47
|
+
npx total-recall proposals apply prop_a1b2c3d4e5f6 --dry-run
|
|
48
|
+
npx total-recall proposals reject prop_a1b2c3d4e5f6 "targets are not duplicates"
|
|
49
|
+
npx total-recall proposals revert prop_a1b2c3d4e5f6
|
|
50
|
+
`);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const STATUS_ICON = {
|
|
54
|
+
draft: '○',
|
|
55
|
+
accepted: '◆',
|
|
56
|
+
applied: '✓',
|
|
57
|
+
rejected: '✗',
|
|
58
|
+
superseded: '~',
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
function flagValue(args, name) {
|
|
62
|
+
const i = args.indexOf(name);
|
|
63
|
+
return i !== -1 ? args[i + 1] : null;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export default async function proposals(args) {
|
|
67
|
+
if (args.includes('--help') || args.includes('-h')) {
|
|
68
|
+
printHelp();
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const { layer, remainingArgs } = parseLayerFlag(args);
|
|
73
|
+
const brainDir = resolveBrainDir(layer);
|
|
74
|
+
const vaultDir = path.join(brainDir, 'memory-vault');
|
|
75
|
+
|
|
76
|
+
const sub = remainingArgs[0] && !remainingArgs[0].startsWith('--') ? remainingArgs[0] : 'list';
|
|
77
|
+
const id = remainingArgs[1] && !remainingArgs[1].startsWith('--') ? remainingArgs[1] : null;
|
|
78
|
+
|
|
79
|
+
switch (sub) {
|
|
80
|
+
case 'list': {
|
|
81
|
+
const statusFlag = flagValue(remainingArgs, '--status');
|
|
82
|
+
// Default view is the actionable queue. Terminal proposals are history, not
|
|
83
|
+
// work — `--status all` shows them when you actually want them.
|
|
84
|
+
const status = statusFlag === 'all' ? undefined : (statusFlag || ['draft', 'accepted']);
|
|
85
|
+
const items = listProposals(vaultDir, {
|
|
86
|
+
status,
|
|
87
|
+
topic: flagValue(remainingArgs, '--topic') || undefined,
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
if (items.length === 0) {
|
|
91
|
+
console.log('\n No proposals match.\n');
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
console.log(`\n ${items.length} proposal(s)\n`);
|
|
96
|
+
for (const p of items) {
|
|
97
|
+
const auto = AUTO_APPLICABLE_TOPICS.has(p.topic) && hasHandler(p.topic) ? ' [auto]' : '';
|
|
98
|
+
console.log(` ${STATUS_ICON[p.status] || '?'} ${p.proposal_id} ${p.topic}${auto}`);
|
|
99
|
+
console.log(` ${p.summary || p.title}`);
|
|
100
|
+
if (p.review_reason) console.log(` ↳ ${p.review_reason}`);
|
|
101
|
+
if (p.rejection_reason) console.log(` ↳ ${p.rejection_reason}`);
|
|
102
|
+
}
|
|
103
|
+
console.log('\n Apply one: npx total-recall proposals apply <id>\n');
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
case 'show': {
|
|
108
|
+
if (!id) { console.error(' ❌ Usage: proposals show <id>'); process.exitCode = 1; return; }
|
|
109
|
+
const p = getProposal(vaultDir, id);
|
|
110
|
+
if (!p) { console.error(` ❌ Proposal not found: ${id}`); process.exitCode = 1; return; }
|
|
111
|
+
console.log(`\n ${p.proposal_id} [${p.status}] ${p.topic}\n`);
|
|
112
|
+
console.log(` Summary: ${p.summary || p.title}`);
|
|
113
|
+
console.log(` Target: ${p.target_path || '(none)'}`);
|
|
114
|
+
console.log(` Proposed: ${p.proposed_at} by ${p.proposed_by}`);
|
|
115
|
+
if (p.reviewed_at) console.log(` Reviewed: ${p.reviewed_at} by ${p.reviewed_by}`);
|
|
116
|
+
if (p.applied_at) console.log(` Applied: ${p.applied_at} by ${p.applied_by}`);
|
|
117
|
+
if (p.review_reason) console.log(` Note: ${p.review_reason}`);
|
|
118
|
+
if (p.rejection_reason) console.log(` Rejected: ${p.rejection_reason}`);
|
|
119
|
+
console.log(`\n ${p.rationale || p.description || ''}\n`);
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
case 'apply': {
|
|
124
|
+
if (!id) { console.error(' ❌ Usage: proposals apply <id>'); process.exitCode = 1; return; }
|
|
125
|
+
const dryRun = remainingArgs.includes('--dry-run');
|
|
126
|
+
const result = await applyProposal(vaultDir, id, { dryRun, actor: 'cli' });
|
|
127
|
+
if (!result.ok) {
|
|
128
|
+
console.error(` ❌ Not applied: ${result.reason}`);
|
|
129
|
+
process.exitCode = 1;
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
if (dryRun) {
|
|
133
|
+
console.log(`\n Dry run — would keep ${result.canonical}, supersede ${result.merged.join(', ')}\n`);
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
console.log(`\n ✓ Applied ${id}`);
|
|
137
|
+
console.log(` Kept: ${result.canonical}`);
|
|
138
|
+
console.log(` Merged: ${result.merged.join(', ')}`);
|
|
139
|
+
console.log(`\n Undo: npx total-recall proposals revert ${id}\n`);
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
case 'reject': {
|
|
144
|
+
if (!id) { console.error(' ❌ Usage: proposals reject <id> [reason]'); process.exitCode = 1; return; }
|
|
145
|
+
const reason = remainingArgs.slice(2).filter(a => !a.startsWith('--')).join(' ') || 'Rejected by user.';
|
|
146
|
+
setProposalStatus(vaultDir, id, 'rejected', { rejection_reason: reason, reviewed_by: 'user' });
|
|
147
|
+
console.log(`\n ✗ Rejected ${id}: ${reason}\n`);
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
case 'supersede': {
|
|
152
|
+
// Bulk-resolve a whole topic. Needed when a generator is retired: its
|
|
153
|
+
// already-filed tickets are neither wrong nor refused, they just describe
|
|
154
|
+
// work that now flows somewhere else. Without this they sit in `list`
|
|
155
|
+
// forever, and the terminal-only pruner correctly refuses to touch them.
|
|
156
|
+
const topic = flagValue(remainingArgs, '--topic');
|
|
157
|
+
if (!topic) {
|
|
158
|
+
console.error(' ❌ Usage: proposals supersede --topic <topic> [--reason "..."]');
|
|
159
|
+
process.exitCode = 1;
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
const reason = flagValue(remainingArgs, '--reason') || `Topic "${topic}" is no longer generated.`;
|
|
163
|
+
const open = listProposals(vaultDir, { status: ['draft', 'accepted'], topic });
|
|
164
|
+
if (open.length === 0) {
|
|
165
|
+
console.log(`\n No open proposals under topic "${topic}".\n`);
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
if (!remainingArgs.includes('--yes')) {
|
|
169
|
+
console.log(`\n Would supersede ${open.length} open "${topic}" proposal(s).`);
|
|
170
|
+
console.log(` Reason: ${reason}`);
|
|
171
|
+
console.log('\n Re-run with --yes to apply.\n');
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
for (const p of open) {
|
|
175
|
+
setProposalStatus(vaultDir, p.proposal_id, 'superseded', { superseded_reason: reason });
|
|
176
|
+
}
|
|
177
|
+
console.log(`\n ~ Superseded ${open.length} "${topic}" proposal(s).\n`);
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
case 'revert': {
|
|
182
|
+
if (!id) { console.error(' ❌ Usage: proposals revert <id>'); process.exitCode = 1; return; }
|
|
183
|
+
try {
|
|
184
|
+
const { reverted, deleted } = revertProposal(vaultDir, id);
|
|
185
|
+
console.log(`\n ✓ Reverted ${id} — ${reverted} file(s) restored, ${deleted} removed.\n`);
|
|
186
|
+
} catch (err) {
|
|
187
|
+
console.error(` ❌ ${err.message}`);
|
|
188
|
+
process.exitCode = 1;
|
|
189
|
+
}
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
case 'stale': {
|
|
194
|
+
// Answers the question the old stale-knowledge-refresh generator used to
|
|
195
|
+
// answer by writing one file per node, every cycle, forever.
|
|
196
|
+
const stale = findStaleNodes(vaultDir);
|
|
197
|
+
if (stale.length === 0) {
|
|
198
|
+
console.log('\n No stale high-importance nodes.\n');
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
console.log(`\n ${stale.length} node(s) due for re-verification\n`);
|
|
202
|
+
for (const n of stale.slice(0, 40)) {
|
|
203
|
+
const days = Math.floor((Date.now() - new Date(n.last_accessed).getTime()) / 86400000);
|
|
204
|
+
console.log(` ${String(days).padStart(4)}d [i${n.importance}] ${n.slug}`);
|
|
205
|
+
}
|
|
206
|
+
if (stale.length > 40) console.log(`\n … and ${stale.length - 40} more.`);
|
|
207
|
+
console.log('\n The dream cycle queues these for research automatically.\n');
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
default:
|
|
212
|
+
console.error(` ❌ Unknown subcommand: ${sub}`);
|
|
213
|
+
printHelp();
|
|
214
|
+
process.exitCode = 1;
|
|
215
|
+
}
|
|
216
|
+
}
|
package/src/cli/rebuild.mjs
CHANGED
|
@@ -53,9 +53,22 @@ export async function runRebuild(options = {}) {
|
|
|
53
53
|
}
|
|
54
54
|
|
|
55
55
|
// Perform full rebuild
|
|
56
|
+
// Discard the cheap, fully-derivable indexes — but NOT embeddings.db.
|
|
57
|
+
//
|
|
58
|
+
// `fs.rmSync(derivedDir)` used to take the whole directory, which threw away
|
|
59
|
+
// every vector. Embeddings are not free to re-derive: each one is a provider
|
|
60
|
+
// call. Worse, the rebuild that followed was fire-and-forget, so the process
|
|
61
|
+
// exited before replacing them and the index stayed empty — that is how this
|
|
62
|
+
// brain ended up with 520 nodes and 0 embeddings while reporting success.
|
|
63
|
+
//
|
|
64
|
+
// The index is content-hash incremental and prunes slugs no longer in the
|
|
65
|
+
// vault, so keeping it across a rebuild is correct as well as cheap.
|
|
56
66
|
console.log('🗑️ Discarding existing derived indexes...');
|
|
57
67
|
if (fs.existsSync(derivedDir)) {
|
|
58
|
-
fs.
|
|
68
|
+
for (const entry of fs.readdirSync(derivedDir)) {
|
|
69
|
+
if (entry.startsWith('embeddings.db')) continue; // also skips -wal / -shm
|
|
70
|
+
fs.rmSync(path.join(derivedDir, entry), { recursive: true, force: true });
|
|
71
|
+
}
|
|
59
72
|
}
|
|
60
73
|
|
|
61
74
|
console.log('🏗️ Recompiling surface from canonical vault...');
|
|
@@ -64,7 +77,19 @@ export async function runRebuild(options = {}) {
|
|
|
64
77
|
console.log(`✅ Processed ${stats.nodesProcessed} canonical memory nodes.`);
|
|
65
78
|
console.log(`✅ Injected memory into ${stats.skillsInjected} skill files.`);
|
|
66
79
|
console.log(`✅ Rebuilt graph-index.jsonl, memory-layers.jsonl, and skill-routes.jsonl.`);
|
|
67
|
-
|
|
80
|
+
|
|
81
|
+
// Report vector coverage explicitly. "Post-build verification passed: 0 drift"
|
|
82
|
+
// checks the jsonl indexes only — it passed happily on a brain with 520 nodes
|
|
83
|
+
// and 0 embeddings, i.e. keyword-only recall that returns plausible-looking
|
|
84
|
+
// results from the wrong places.
|
|
85
|
+
if (stats.semanticUnavailable) {
|
|
86
|
+
console.log(`⚠️ Vector search is OFF — embeddings unavailable${stats.semanticError ? `: ${stats.semanticError}` : ''}.`);
|
|
87
|
+
console.log(` Recall will fall back to keyword matching.`);
|
|
88
|
+
} else {
|
|
89
|
+
console.log(`✅ Embeddings: ${stats.semanticIndexed} built, ${stats.semanticSkipped} unchanged${stats.semanticFailed ? `, ${stats.semanticFailed} FAILED` : ''}.`);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
|
|
68
93
|
// Verify
|
|
69
94
|
const drift = detectIndexDrift(vaultDir, derivedDir);
|
|
70
95
|
if (!drift.drifted) {
|
package/src/cli/secret.mjs
CHANGED
|
@@ -48,6 +48,13 @@ import {
|
|
|
48
48
|
enqueueRotationDueTasks,
|
|
49
49
|
getBrowserRotateAssist,
|
|
50
50
|
} from '../core/secrets-rotate.mjs';
|
|
51
|
+
import {
|
|
52
|
+
loadRemoteTargets,
|
|
53
|
+
addRemoteTarget,
|
|
54
|
+
removeRemoteTarget,
|
|
55
|
+
deployEnvToRemote,
|
|
56
|
+
deployKeyToRemotes,
|
|
57
|
+
} from '../core/secrets-remote-deploy.mjs';
|
|
51
58
|
import { listProviders, getProvider } from '../core/provider-catalog.mjs';
|
|
52
59
|
import { resolveBrainDir, parseLayerFlag } from './agent-dir.mjs';
|
|
53
60
|
import fs from 'node:fs';
|
|
@@ -77,7 +84,18 @@ function printHelp() {
|
|
|
77
84
|
--no-example Skip .env.example
|
|
78
85
|
--no-global Only keys bound to this repo (skip unbound)
|
|
79
86
|
--replace-all Overwrite entire .env (default: merge TR block only)
|
|
87
|
+
--remote <name> Also push the SSOT projection to a configured remote
|
|
88
|
+
target over SSH (see 'secret remote') instead of
|
|
89
|
+
(or in addition to) a local file
|
|
80
90
|
--dry-run Show keys that would be written
|
|
91
|
+
remote <cmd> Manage remote (production) deploy targets — generic,
|
|
92
|
+
per-repo config, no host/path ever hardcoded in TR
|
|
93
|
+
add <name> --host <h> --path </remote/dir> [--user root]
|
|
94
|
+
[--port 22] [--filename .env] [--restart-cmd "..."]
|
|
95
|
+
list Show configured targets (no secret values)
|
|
96
|
+
remove <name> Delete a target
|
|
97
|
+
deploy <name> Push the SSOT projection to that target now
|
|
98
|
+
[--dry-run] [--keys k1,k2]
|
|
81
99
|
import-env One-time migrate: scan local .env → store (not the steady-state path)
|
|
82
100
|
--all / --file / --overwrite / --dry-run
|
|
83
101
|
catalog Full catalog: keys, providers, usage, rotation, cost, tracking
|
|
@@ -116,6 +134,8 @@ function printHelp() {
|
|
|
116
134
|
rotate options:
|
|
117
135
|
--export-env After rotate, write .env to bound repos (and cwd if unbound)
|
|
118
136
|
--export-all After rotate, export-env to all project-registry paths
|
|
137
|
+
--remote <name> After rotate, also push to a configured remote target
|
|
138
|
+
(production, etc.) over SSH — see 'secret remote'
|
|
119
139
|
|
|
120
140
|
set / rotate options:
|
|
121
141
|
--provider <name> Optional provider tag (e.g. openai, anthropic)
|
|
@@ -186,6 +206,12 @@ function parseArgs(args) {
|
|
|
186
206
|
exempt: false,
|
|
187
207
|
tracking_exempt: null,
|
|
188
208
|
shared_ok: null,
|
|
209
|
+
subcommand: null,
|
|
210
|
+
remote: null,
|
|
211
|
+
remoteHost: null,
|
|
212
|
+
remoteUser: 'root',
|
|
213
|
+
remotePort: 22,
|
|
214
|
+
restartCmd: null,
|
|
189
215
|
};
|
|
190
216
|
if (!args.length || args[0] === '--help' || args[0] === '-h') {
|
|
191
217
|
out.help = true;
|
|
@@ -193,6 +219,12 @@ function parseArgs(args) {
|
|
|
193
219
|
}
|
|
194
220
|
out.command = args[0];
|
|
195
221
|
let i = 1;
|
|
222
|
+
if (out.command === 'remote') {
|
|
223
|
+
if (args[i] && !args[i].startsWith('--')) out.subcommand = args[i++];
|
|
224
|
+
if (['add', 'remove', 'rm', 'deploy'].includes(out.subcommand) && args[i] && !args[i].startsWith('--')) {
|
|
225
|
+
out.key = args[i++]; // reused as the remote target name
|
|
226
|
+
}
|
|
227
|
+
}
|
|
196
228
|
if (
|
|
197
229
|
['set', 'get', 'rotate', 'delete', 'meta', 'rotate-browser', 'rm', 'generate', 'account-sync'].includes(
|
|
198
230
|
out.command,
|
|
@@ -208,6 +240,21 @@ function parseArgs(args) {
|
|
|
208
240
|
while (i < args.length) {
|
|
209
241
|
const a = args[i];
|
|
210
242
|
switch (a) {
|
|
243
|
+
case '--remote':
|
|
244
|
+
out.remote = args[++i];
|
|
245
|
+
break;
|
|
246
|
+
case '--host':
|
|
247
|
+
out.remoteHost = args[++i];
|
|
248
|
+
break;
|
|
249
|
+
case '--user':
|
|
250
|
+
out.remoteUser = args[++i] || 'root';
|
|
251
|
+
break;
|
|
252
|
+
case '--port':
|
|
253
|
+
out.remotePort = parseInt(args[++i], 10) || 22;
|
|
254
|
+
break;
|
|
255
|
+
case '--restart-cmd':
|
|
256
|
+
out.restartCmd = args[++i];
|
|
257
|
+
break;
|
|
211
258
|
case '--provider':
|
|
212
259
|
out.provider = args[++i];
|
|
213
260
|
break;
|
|
@@ -724,11 +771,121 @@ export default async function secretCli(argv) {
|
|
|
724
771
|
return;
|
|
725
772
|
}
|
|
726
773
|
|
|
774
|
+
if (opts.command === 'remote') {
|
|
775
|
+
if (!opts.subcommand) {
|
|
776
|
+
console.error(
|
|
777
|
+
'Usage: total-recall secret remote <add|list|remove|deploy> [...]\n' +
|
|
778
|
+
' add <name> --host <h> --path </remote/dir> [--user root] [--port 22] [--filename .env] [--restart-cmd "..."]\n' +
|
|
779
|
+
' list\n' +
|
|
780
|
+
' remove <name>\n' +
|
|
781
|
+
' deploy <name> [--dry-run] [--keys k1,k2]',
|
|
782
|
+
);
|
|
783
|
+
process.exit(1);
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
if (opts.subcommand === 'add') {
|
|
787
|
+
if (!opts.key || !opts.remoteHost || !opts.path) {
|
|
788
|
+
console.error(
|
|
789
|
+
'Usage: total-recall secret remote add <name> --host <host> --path </remote/dir> [--user root] [--port 22] [--filename .env] [--restart-cmd "..."]',
|
|
790
|
+
);
|
|
791
|
+
process.exit(1);
|
|
792
|
+
}
|
|
793
|
+
const { path: targetsPath } = addRemoteTarget(brainDir, {
|
|
794
|
+
name: opts.key,
|
|
795
|
+
host: opts.remoteHost,
|
|
796
|
+
user: opts.remoteUser,
|
|
797
|
+
port: opts.remotePort,
|
|
798
|
+
remotePath: opts.path,
|
|
799
|
+
filename: opts.filename || '.env',
|
|
800
|
+
restartCommand: opts.restartCmd || null,
|
|
801
|
+
keys: opts.keys.length ? opts.keys : undefined,
|
|
802
|
+
});
|
|
803
|
+
console.log(`\n ✅ Remote target added: ${opts.key}`);
|
|
804
|
+
console.log(` ${opts.remoteUser}@${opts.remoteHost}:${opts.path}/${opts.filename || '.env'}`);
|
|
805
|
+
if (opts.restartCmd) console.log(` restart: ${opts.restartCmd}`);
|
|
806
|
+
console.log(` config: ${targetsPath}\n`);
|
|
807
|
+
return;
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
if (opts.subcommand === 'list') {
|
|
811
|
+
const targets = loadRemoteTargets(brainDir);
|
|
812
|
+
if (!targets.length) {
|
|
813
|
+
console.log(`\n No remote targets configured for this repo.\n`);
|
|
814
|
+
return;
|
|
815
|
+
}
|
|
816
|
+
console.log(`\n Remote deploy targets\n`);
|
|
817
|
+
for (const t of targets) {
|
|
818
|
+
console.log(
|
|
819
|
+
` • ${t.name.padEnd(20)} ${t.user}@${t.host}:${t.port} ${t.remotePath}/${t.filename}`,
|
|
820
|
+
);
|
|
821
|
+
if (t.restartCommand) console.log(` restart: ${t.restartCommand}`);
|
|
822
|
+
if (t.keys?.length) console.log(` keys: ${t.keys.join(', ')}`);
|
|
823
|
+
}
|
|
824
|
+
console.log('');
|
|
825
|
+
return;
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
if (opts.subcommand === 'remove' || opts.subcommand === 'rm') {
|
|
829
|
+
if (!opts.key) {
|
|
830
|
+
console.error('Usage: total-recall secret remote remove <name>');
|
|
831
|
+
process.exit(1);
|
|
832
|
+
}
|
|
833
|
+
const r = removeRemoteTarget(brainDir, opts.key);
|
|
834
|
+
console.log(r.removed ? ` ✅ Removed remote target: ${opts.key}` : ` Remote target not found: ${opts.key}`);
|
|
835
|
+
return;
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
if (opts.subcommand === 'deploy') {
|
|
839
|
+
if (!opts.key) {
|
|
840
|
+
console.error('Usage: total-recall secret remote deploy <name> [--dry-run] [--keys k1,k2]');
|
|
841
|
+
process.exit(1);
|
|
842
|
+
}
|
|
843
|
+
const r = await deployEnvToRemote(brainDir, opts.key, {
|
|
844
|
+
dryRun: opts.dryRun,
|
|
845
|
+
keys: opts.keys.length ? opts.keys : undefined,
|
|
846
|
+
});
|
|
847
|
+
if (r.dryRun) {
|
|
848
|
+
console.log(`\n 📤 dry-run remote deploy → ${r.target} (${r.host})`);
|
|
849
|
+
console.log(` would write ${r.count} keys → ${r.remoteFile}\n`);
|
|
850
|
+
return;
|
|
851
|
+
}
|
|
852
|
+
console.log(`\n ✅ Deployed ${r.count} secret(s) → ${r.target} (${r.host}:${r.remoteFile})`);
|
|
853
|
+
if (r.restarted) console.log(` restart: ok`);
|
|
854
|
+
else if (r.restartOutput) console.log(` restart: FAILED — ${r.restartOutput.slice(0, 300)}`);
|
|
855
|
+
console.log(` (values written with mode 0600 remotely; not printed)\n`);
|
|
856
|
+
return;
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
console.error(`Unknown remote subcommand: ${opts.subcommand}`);
|
|
860
|
+
process.exit(1);
|
|
861
|
+
}
|
|
862
|
+
|
|
727
863
|
if (opts.command === 'rotate') {
|
|
728
864
|
if (!opts.key || !opts.value) {
|
|
729
|
-
console.error('Usage: total-recall secret rotate <key> <new-value> [--export-env|--export-all]');
|
|
865
|
+
console.error('Usage: total-recall secret rotate <key> <new-value> [--export-env|--export-all|--remote <name>]');
|
|
730
866
|
process.exit(1);
|
|
731
867
|
}
|
|
868
|
+
if (opts.remote) {
|
|
869
|
+
await rotateSecret(brainDir, opts.key, opts.value, { provider: opts.provider, scope: opts.scope });
|
|
870
|
+
console.log(` ✅ Rotated: ${opts.key}`);
|
|
871
|
+
const r = await deployEnvToRemote(brainDir, opts.remote, {});
|
|
872
|
+
console.log(` ✅ Deployed to remote target: ${r.target} (${r.host}:${r.remoteFile}, ${r.count} keys)`);
|
|
873
|
+
if (r.restarted) console.log(` restart: ok`);
|
|
874
|
+
else if (r.restartOutput) console.log(` restart: FAILED — ${r.restartOutput.slice(0, 300)}`);
|
|
875
|
+
if (opts.exportEnv || opts.exportAll) {
|
|
876
|
+
const localR = await rotateSecretAndExport(brainDir, opts.key, opts.value, {
|
|
877
|
+
provider: opts.provider,
|
|
878
|
+
actor: 'cli',
|
|
879
|
+
exportEnv: true,
|
|
880
|
+
exportAllProjects: opts.exportAll,
|
|
881
|
+
exportCwd: true,
|
|
882
|
+
includeGlobal: true,
|
|
883
|
+
});
|
|
884
|
+
const ok = (localR.exports || []).filter((e) => e.ok !== false);
|
|
885
|
+
console.log(` local export-env: ${ok.length} target(s)`);
|
|
886
|
+
}
|
|
887
|
+
return;
|
|
888
|
+
}
|
|
732
889
|
if (opts.exportEnv || opts.exportAll) {
|
|
733
890
|
const r = await rotateSecretAndExport(brainDir, opts.key, opts.value, {
|
|
734
891
|
provider: opts.provider,
|
|
@@ -859,6 +1016,23 @@ export default async function secretCli(argv) {
|
|
|
859
1016
|
keys: opts.keys.length ? opts.keys : undefined,
|
|
860
1017
|
};
|
|
861
1018
|
|
|
1019
|
+
if (opts.remote) {
|
|
1020
|
+
const r = await deployEnvToRemote(brainDir, opts.remote, {
|
|
1021
|
+
dryRun: opts.dryRun,
|
|
1022
|
+
keys: opts.keys.length ? opts.keys : undefined,
|
|
1023
|
+
});
|
|
1024
|
+
if (r.dryRun) {
|
|
1025
|
+
console.log(`\n 📤 dry-run remote export → ${r.target} (${r.host})`);
|
|
1026
|
+
console.log(` would write ${r.count} keys → ${r.remoteFile}\n`);
|
|
1027
|
+
} else {
|
|
1028
|
+
console.log(`\n ✅ Exported ${r.count} secret(s) → ${r.target} (${r.host}:${r.remoteFile})`);
|
|
1029
|
+
if (r.restarted) console.log(` restart: ok`);
|
|
1030
|
+
else if (r.restartOutput) console.log(` restart: FAILED — ${r.restartOutput.slice(0, 300)}`);
|
|
1031
|
+
console.log(` (values written with mode 0600 remotely; not printed)\n`);
|
|
1032
|
+
}
|
|
1033
|
+
if (!opts.path && !opts.allProjects) return; // --remote alone doesn't also touch local disk
|
|
1034
|
+
}
|
|
1035
|
+
|
|
862
1036
|
if (opts.allProjects) {
|
|
863
1037
|
const results = await exportEnvToRegistry(brainDir, exportOpts);
|
|
864
1038
|
console.log(`\n 📤 export-env → ${results.length} project(s) from ${resolveSecretsPath(brainDir)}\n`);
|