tr-plugin-auth 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -2
- package/cli.mjs +384 -0
- package/package.json +40 -4
- package/plugin.json +266 -0
- package/src/access-keys.mjs +225 -0
- package/src/agent-dir.mjs +3 -0
- package/src/browser-session.mjs +240 -0
- package/src/cli-secret.mjs +1687 -0
- package/src/config.mjs +4 -0
- package/src/configuration.mjs +19 -0
- package/src/credential-preservation.mjs +28 -0
- package/src/crypto.mjs +195 -0
- package/src/env-import.mjs +502 -0
- package/src/fact-seeker.mjs +3 -0
- package/src/frontmatter.mjs +3 -0
- package/src/google-oauth.mjs +139 -0
- package/src/host-runtime.mjs +7 -0
- package/src/leader-election.mjs +9 -0
- package/src/logger.mjs +3 -0
- package/src/mesh-auth.mjs +7 -0
- package/src/mesh-identity.mjs +16 -0
- package/src/mesh-secrets.mjs +146 -0
- package/src/mesh.mjs +23 -0
- package/src/metadata-state.mjs +17 -0
- package/src/project-id.mjs +3 -0
- package/src/provider-account-sync.mjs +1145 -0
- package/src/provider-catalog.mjs +515 -0
- package/src/provider-rotation-recipes.mjs +299 -0
- package/src/research-gate.mjs +3 -0
- package/src/research-queue.mjs +3 -0
- package/src/rotation-capability.mjs +189 -0
- package/src/routes/auth.mjs +207 -0
- package/src/routes/keys.mjs +61 -0
- package/src/routes/secrets-runtime.mjs +5 -0
- package/src/routes/secrets.mjs +747 -0
- package/src/routes/webauthn.mjs +163 -0
- package/src/runtime-credentials.mjs +19 -0
- package/src/runtime.mjs +3 -0
- package/src/secret-integration-research.mjs +340 -0
- package/src/secrets-env-diff.mjs +132 -0
- package/src/secrets-env-export.mjs +322 -0
- package/src/secrets-gateway.mjs +253 -0
- package/src/secrets-keychain.mjs +83 -0
- package/src/secrets-rekey.mjs +366 -0
- package/src/secrets-remote-deploy.mjs +250 -0
- package/src/secrets-rotate.mjs +437 -0
- package/src/secrets-store.mjs +756 -0
- package/src/secrets-sync.mjs +81 -0
- package/src/secure-file.mjs +71 -0
- package/src/server-auth.mjs +500 -0
- package/src/session-credentials.mjs +71 -0
- package/src/ssss-store.mjs +13 -0
- package/src/stdin.mjs +31 -0
- package/src/throttled-fetch.mjs +3 -0
- package/src/vault.mjs +3 -0
- package/src/vfs-documents.mjs +3 -0
- package/src/webauthn-store.mjs +265 -0
- package/ui/DESIGN.md +19 -0
- package/ui/card.js +274 -0
- package/ui/panel.js +338 -0
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import express from 'express';
|
|
2
|
+
import { listPasskeys, hasPasskeys, beginRegistration, finishRegistration, beginAuthentication,
|
|
3
|
+
finishAuthentication, deletePasskey, resolveWebAuthnPath } from '../webauthn-store.mjs';
|
|
4
|
+
export function createWebauthnRouter({ auth, badRequest, serverError, BRAIN_DIR, logger }) {
|
|
5
|
+
const { requireAuth, requireScope, mintStepUpToken, verifyDashboardPassword } = auth;
|
|
6
|
+
const router = express.Router();
|
|
7
|
+
|
|
8
|
+
function brainDir() {
|
|
9
|
+
return process.env.TR_SECRETS_BRAIN || BRAIN_DIR;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
router.get('/api/webauthn/status', requireAuth, requireScope('keys:read', 'config:read'), (req, res) => {
|
|
13
|
+
try {
|
|
14
|
+
const dir = brainDir();
|
|
15
|
+
const passkeys = listPasskeys(dir);
|
|
16
|
+
res.json({
|
|
17
|
+
enabled: true,
|
|
18
|
+
has_passkeys: passkeys.length > 0,
|
|
19
|
+
count: passkeys.length,
|
|
20
|
+
passkeys,
|
|
21
|
+
store: resolveWebAuthnPath(dir),
|
|
22
|
+
// When false, UI may offer password re-entry as step-up fallback
|
|
23
|
+
password_step_up_allowed: true,
|
|
24
|
+
});
|
|
25
|
+
} catch (err) {
|
|
26
|
+
serverError(res, err);
|
|
27
|
+
}
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
router.post(
|
|
31
|
+
'/api/webauthn/register/options',
|
|
32
|
+
requireAuth,
|
|
33
|
+
requireScope('keys:write', 'config:write'),
|
|
34
|
+
async (req, res) => {
|
|
35
|
+
try {
|
|
36
|
+
const options = await beginRegistration(brainDir(), req);
|
|
37
|
+
res.json(options);
|
|
38
|
+
} catch (err) {
|
|
39
|
+
logger.warn('webauthn', `register options failed: ${err.message}`);
|
|
40
|
+
return badRequest(res, err.message);
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
);
|
|
44
|
+
|
|
45
|
+
router.post(
|
|
46
|
+
'/api/webauthn/register/verify',
|
|
47
|
+
requireAuth,
|
|
48
|
+
requireScope('keys:write', 'config:write'),
|
|
49
|
+
async (req, res) => {
|
|
50
|
+
try {
|
|
51
|
+
const body = req.body || {};
|
|
52
|
+
const response = body.response || body;
|
|
53
|
+
if (!response?.id) return badRequest(res, 'WebAuthn registration response required');
|
|
54
|
+
const result = await finishRegistration(brainDir(), req, response, {
|
|
55
|
+
label: body.label,
|
|
56
|
+
});
|
|
57
|
+
res.status(201).json({
|
|
58
|
+
...result,
|
|
59
|
+
passkeys: listPasskeys(brainDir()),
|
|
60
|
+
});
|
|
61
|
+
} catch (err) {
|
|
62
|
+
logger.warn('webauthn', `register verify failed: ${err.message}`);
|
|
63
|
+
return badRequest(res, err.message);
|
|
64
|
+
}
|
|
65
|
+
},
|
|
66
|
+
);
|
|
67
|
+
|
|
68
|
+
router.post(
|
|
69
|
+
'/api/webauthn/assert/options',
|
|
70
|
+
requireAuth,
|
|
71
|
+
requireScope('keys:read', 'config:read'),
|
|
72
|
+
async (req, res) => {
|
|
73
|
+
try {
|
|
74
|
+
const options = await beginAuthentication(brainDir(), req);
|
|
75
|
+
res.json(options);
|
|
76
|
+
} catch (err) {
|
|
77
|
+
return badRequest(res, err.message);
|
|
78
|
+
}
|
|
79
|
+
},
|
|
80
|
+
);
|
|
81
|
+
|
|
82
|
+
router.post(
|
|
83
|
+
'/api/webauthn/assert/verify',
|
|
84
|
+
requireAuth,
|
|
85
|
+
requireScope('keys:read', 'config:read'),
|
|
86
|
+
async (req, res) => {
|
|
87
|
+
try {
|
|
88
|
+
const body = req.body || {};
|
|
89
|
+
const response = body.response || body;
|
|
90
|
+
if (!response?.id) return badRequest(res, 'WebAuthn authentication response required');
|
|
91
|
+
const result = await finishAuthentication(brainDir(), req, response);
|
|
92
|
+
const step_up_token = mintStepUpToken({
|
|
93
|
+
purpose: body.purpose || 'secrets:reveal',
|
|
94
|
+
ttlSeconds: body.ttl_seconds || 60,
|
|
95
|
+
actor: 'passkey',
|
|
96
|
+
});
|
|
97
|
+
logger.info('webauthn', 'Passkey step-up granted', {
|
|
98
|
+
credentialId: result.credentialId,
|
|
99
|
+
purpose: body.purpose || 'secrets:reveal',
|
|
100
|
+
});
|
|
101
|
+
res.json({
|
|
102
|
+
verified: true,
|
|
103
|
+
step_up_token,
|
|
104
|
+
expires_in: body.ttl_seconds || 60,
|
|
105
|
+
});
|
|
106
|
+
} catch (err) {
|
|
107
|
+
logger.warn('webauthn', `assert verify failed: ${err.message}`);
|
|
108
|
+
return badRequest(res, err.message);
|
|
109
|
+
}
|
|
110
|
+
},
|
|
111
|
+
);
|
|
112
|
+
|
|
113
|
+
/** Password re-entry step-up (works always; preferred when no passkeys). */
|
|
114
|
+
router.post(
|
|
115
|
+
'/api/webauthn/step-up/password',
|
|
116
|
+
requireAuth,
|
|
117
|
+
requireScope('keys:read', 'config:read'),
|
|
118
|
+
async (req, res) => {
|
|
119
|
+
try {
|
|
120
|
+
const password = req.body?.password;
|
|
121
|
+
if (!password) return badRequest(res, 'password is required');
|
|
122
|
+
const ok = await verifyDashboardPassword(password);
|
|
123
|
+
if (!ok) {
|
|
124
|
+
return res.status(401).json({ error: 'Invalid password' });
|
|
125
|
+
}
|
|
126
|
+
// If passkeys exist, still allow password as fallback but log it
|
|
127
|
+
if (hasPasskeys(brainDir())) {
|
|
128
|
+
logger.info('webauthn', 'Password step-up used while passkeys registered');
|
|
129
|
+
}
|
|
130
|
+
const step_up_token = mintStepUpToken({
|
|
131
|
+
purpose: req.body?.purpose || 'secrets:reveal',
|
|
132
|
+
ttlSeconds: req.body?.ttl_seconds || 60,
|
|
133
|
+
actor: 'password',
|
|
134
|
+
});
|
|
135
|
+
res.json({
|
|
136
|
+
verified: true,
|
|
137
|
+
method: 'password',
|
|
138
|
+
step_up_token,
|
|
139
|
+
expires_in: req.body?.ttl_seconds || 60,
|
|
140
|
+
});
|
|
141
|
+
} catch (err) {
|
|
142
|
+
serverError(res, err);
|
|
143
|
+
}
|
|
144
|
+
},
|
|
145
|
+
);
|
|
146
|
+
|
|
147
|
+
router.delete(
|
|
148
|
+
'/api/webauthn/credentials/:id',
|
|
149
|
+
requireAuth,
|
|
150
|
+
requireScope('keys:write', 'config:write'),
|
|
151
|
+
(req, res) => {
|
|
152
|
+
try {
|
|
153
|
+
const r = deletePasskey(brainDir(), req.params.id);
|
|
154
|
+
if (!r.found) return res.status(404).json({ error: 'Passkey not found' });
|
|
155
|
+
res.json({ deleted: true, passkeys: listPasskeys(brainDir()) });
|
|
156
|
+
} catch (err) {
|
|
157
|
+
serverError(res, err);
|
|
158
|
+
}
|
|
159
|
+
},
|
|
160
|
+
);
|
|
161
|
+
|
|
162
|
+
return router;
|
|
163
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { loadSecretsSync, META_KEY } from './secrets-gateway.mjs';
|
|
2
|
+
const PROVIDERS = ['GOOGLE_API_KEY', 'GEMINI_API_KEY', 'ANTHROPIC_API_KEY', 'OPENAI_API_KEY', 'XAI_API_KEY', 'GROK_API_KEY', 'TAVILY_API_KEY', 'BRAVE_API_KEY', 'BRAVE_SEARCH_API_KEY', 'EXA_API_KEY', 'SERPER_API_KEY', 'GITHUB_TOKEN', 'OPENROUTER_API_KEY'];
|
|
3
|
+
/** Selected-store projection for harnesses; no session/password/master material. */
|
|
4
|
+
export function runtimeCredentials({ brainDir }) {
|
|
5
|
+
if (!brainDir) throw new Error('Explicit auth store required');
|
|
6
|
+
const stored = loadSecretsSync(brainDir), result = {};
|
|
7
|
+
for (const key of PROVIDERS) {
|
|
8
|
+
const item = stored[key] ?? stored[key.toLowerCase()];
|
|
9
|
+
const value = typeof item === 'object' && item !== null ? item.value : item;
|
|
10
|
+
if (typeof value === 'string' && value) result[key.toLowerCase()] = value;
|
|
11
|
+
}
|
|
12
|
+
delete result[META_KEY];
|
|
13
|
+
return result;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** Never inherit master/session/provider/password material from the host shell. */
|
|
17
|
+
export function sanitizeRuntimeEnvironment(env = process.env) {
|
|
18
|
+
return Object.fromEntries(Object.entries(env).filter(([name]) => !/(?:API_KEY|TOKEN|SECRET|PASSWORD|PASSWD|PRIVATE_KEY)$/i.test(name)));
|
|
19
|
+
}
|
package/src/runtime.mjs
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import { importHostModule } from './host-runtime.mjs';
|
|
2
|
+
const hostModule = await importHostModule('src/core/runtime.mjs');
|
|
3
|
+
export const { findBinaryInPath, loadRuntimeConfig, checkRuntimeHealth, loadDynamicSecrets, callLocalRuntime, callLocalRuntimeRaw, cleanAndParseJSON } = hostModule;
|
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AI-only secret → integration research.
|
|
3
|
+
*
|
|
4
|
+
* When a new secret is stored, a CLI agent (callLocalRuntime) decides:
|
|
5
|
+
* - whether this credential is worth product-API research
|
|
6
|
+
* - which product/provider it is
|
|
7
|
+
* - what research topic + notes to enqueue
|
|
8
|
+
*
|
|
9
|
+
* No hardcoded provider lists, no key-name regex classification, no catalog matching.
|
|
10
|
+
* Optional code-usage snippets are gathered as *evidence for the model*, not as rules.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import fs from 'node:fs';
|
|
14
|
+
import path from 'node:path';
|
|
15
|
+
import { spawnSync } from 'node:child_process';
|
|
16
|
+
import { fileURLToPath } from 'node:url';
|
|
17
|
+
import { logger } from './logger.mjs';
|
|
18
|
+
|
|
19
|
+
const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
|
|
20
|
+
|
|
21
|
+
const INFER_SYSTEM = `You are a senior integrations engineer for Total Recall (a personal memory / mesh OS).
|
|
22
|
+
You are given only:
|
|
23
|
+
(1) an environment/secret KEY NAME (never the secret value)
|
|
24
|
+
(2) optional code snippets where that key is referenced in THIS repo
|
|
25
|
+
|
|
26
|
+
Decide whether Total Recall should queue a *research* job to learn an *external third-party* product API that is NOT already part of this product.
|
|
27
|
+
|
|
28
|
+
Output ONLY valid JSON (no markdown fences):
|
|
29
|
+
{
|
|
30
|
+
"researchable": boolean,
|
|
31
|
+
"skip_reason": string|null,
|
|
32
|
+
"product_name": string|null,
|
|
33
|
+
"product_slug": string|null,
|
|
34
|
+
"kind": "api_key"|"password"|"oauth"|"webhook"|"internal"|"self_hosted"|"already_integrated"|"unknown",
|
|
35
|
+
"topic": string|null,
|
|
36
|
+
"notes": string|null,
|
|
37
|
+
"priority": "low"|"medium"|"high",
|
|
38
|
+
"confidence": number
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
Rules (strict):
|
|
42
|
+
- researchable=true ONLY for external SaaS/third-party APIs the product might need to learn about from the public internet.
|
|
43
|
+
- researchable=false when code shows the key is for:
|
|
44
|
+
- infrastructure we already own/run (mesh, Tailscale-compatible control planes, Headscale, local routers, self-hosted mail, etc.)
|
|
45
|
+
- features already implemented in this repo (routes, clients, adapters present)
|
|
46
|
+
- passwords, email logins, SSO/client secrets, webhook signing secrets, private keys, session/DB secrets
|
|
47
|
+
- Total Recall / TR_ / mesh-sync internal credentials
|
|
48
|
+
- If the key is used by existing modules (e.g. headscale routes, mesh auth, mail), set researchable=false with kind "already_integrated" or "self_hosted".
|
|
49
|
+
- Never queue research whose topic is the env var name itself.
|
|
50
|
+
- topic (when researchable) must name the external product and API surface; notes must say use official docs, not the raw key string.
|
|
51
|
+
- Prefer skip when uncertain. Do not invent product paths.`;
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Collect a small amount of code-usage context for the AI (not classification rules).
|
|
55
|
+
* @param {string} key
|
|
56
|
+
* @param {{ roots?: string[], maxHits?: number }} [opts]
|
|
57
|
+
* @returns {string}
|
|
58
|
+
*/
|
|
59
|
+
export function gatherCodeUsageContext(key, opts = {}) {
|
|
60
|
+
const roots = opts.roots || [path.join(REPO_ROOT, 'src')];
|
|
61
|
+
const maxHits = opts.maxHits ?? 12;
|
|
62
|
+
const snippets = [];
|
|
63
|
+
|
|
64
|
+
// Prefer ripgrep if available; fall back to a tiny walk.
|
|
65
|
+
const rg = spawnSync(
|
|
66
|
+
'rg',
|
|
67
|
+
[
|
|
68
|
+
'-n',
|
|
69
|
+
'--no-heading',
|
|
70
|
+
'-m',
|
|
71
|
+
'8',
|
|
72
|
+
'-g',
|
|
73
|
+
'!**/node_modules/**',
|
|
74
|
+
'-g',
|
|
75
|
+
'!**/*.spec.*',
|
|
76
|
+
'-e',
|
|
77
|
+
key,
|
|
78
|
+
...roots.filter((r) => fs.existsSync(r)),
|
|
79
|
+
],
|
|
80
|
+
{ encoding: 'utf8', timeout: 8000, maxBuffer: 512_000 },
|
|
81
|
+
);
|
|
82
|
+
|
|
83
|
+
if (rg.status === 0 && rg.stdout) {
|
|
84
|
+
for (const line of rg.stdout.split('\n').filter(Boolean).slice(0, maxHits)) {
|
|
85
|
+
snippets.push(line.slice(0, 400));
|
|
86
|
+
}
|
|
87
|
+
} else {
|
|
88
|
+
// Minimal fallback walk
|
|
89
|
+
for (const root of roots) {
|
|
90
|
+
if (!fs.existsSync(root)) continue;
|
|
91
|
+
walkForKey(root, key, snippets, maxHits, 0);
|
|
92
|
+
if (snippets.length >= maxHits) break;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
if (!snippets.length) {
|
|
97
|
+
return '(no code references found in local src tree)';
|
|
98
|
+
}
|
|
99
|
+
return snippets.join('\n');
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function walkForKey(dir, key, out, maxHits, depth) {
|
|
103
|
+
if (out.length >= maxHits || depth > 6) return;
|
|
104
|
+
let entries;
|
|
105
|
+
try {
|
|
106
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
107
|
+
} catch {
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
for (const ent of entries) {
|
|
111
|
+
if (out.length >= maxHits) return;
|
|
112
|
+
if (ent.name === 'node_modules' || ent.name === '.git' || ent.name.startsWith('.')) continue;
|
|
113
|
+
const full = path.join(dir, ent.name);
|
|
114
|
+
if (ent.isDirectory()) {
|
|
115
|
+
walkForKey(full, key, out, maxHits, depth + 1);
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
if (!/\.(mjs|js|ts|tsx|yml|yaml|json|md)$/i.test(ent.name)) continue;
|
|
119
|
+
if (/\.spec\./i.test(ent.name)) continue;
|
|
120
|
+
let text;
|
|
121
|
+
try {
|
|
122
|
+
text = fs.readFileSync(full, 'utf8');
|
|
123
|
+
} catch {
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
if (!text.includes(key)) continue;
|
|
127
|
+
const lines = text.split('\n');
|
|
128
|
+
for (let i = 0; i < lines.length && out.length < maxHits; i++) {
|
|
129
|
+
if (lines[i].includes(key)) {
|
|
130
|
+
const from = Math.max(0, i - 1);
|
|
131
|
+
const to = Math.min(lines.length, i + 2);
|
|
132
|
+
const chunk = lines
|
|
133
|
+
.slice(from, to)
|
|
134
|
+
.map((l, idx) => `${full}:${from + idx + 1}:${l.trim().slice(0, 200)}`)
|
|
135
|
+
.join('\n');
|
|
136
|
+
out.push(chunk);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Parse first JSON object from model output.
|
|
144
|
+
* @param {string} raw
|
|
145
|
+
*/
|
|
146
|
+
export function parseAiJson(raw) {
|
|
147
|
+
const text = String(raw || '');
|
|
148
|
+
const start = text.indexOf('{');
|
|
149
|
+
const end = text.lastIndexOf('}');
|
|
150
|
+
if (start < 0 || end <= start) return null;
|
|
151
|
+
try {
|
|
152
|
+
return JSON.parse(text.slice(start, end + 1));
|
|
153
|
+
} catch {
|
|
154
|
+
return null;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Ask a CLI agent to infer provider / research brief from key name + code usage.
|
|
160
|
+
*
|
|
161
|
+
* @param {string} key
|
|
162
|
+
* @param {{
|
|
163
|
+
* meta?: object,
|
|
164
|
+
* codeContext?: string,
|
|
165
|
+
* runtimeConfig?: object,
|
|
166
|
+
* callRuntime?: Function,
|
|
167
|
+
* }} [opts]
|
|
168
|
+
* @returns {Promise<{
|
|
169
|
+
* researchable: boolean,
|
|
170
|
+
* skip_reason: string|null,
|
|
171
|
+
* product_name: string|null,
|
|
172
|
+
* product_slug: string|null,
|
|
173
|
+
* kind: string,
|
|
174
|
+
* topic: string|null,
|
|
175
|
+
* notes: string|null,
|
|
176
|
+
* priority: string,
|
|
177
|
+
* confidence: number,
|
|
178
|
+
* source: 'ai',
|
|
179
|
+
* }>}
|
|
180
|
+
*/
|
|
181
|
+
export async function inferSecretIntegrationWithAi(key, opts = {}) {
|
|
182
|
+
const codeContext =
|
|
183
|
+
opts.codeContext !== undefined
|
|
184
|
+
? opts.codeContext
|
|
185
|
+
: gatherCodeUsageContext(key, { roots: opts.roots });
|
|
186
|
+
|
|
187
|
+
const prompt = [
|
|
188
|
+
`Secret key name: ${key}`,
|
|
189
|
+
opts.meta?.provider ? `Stored provider tag (may be wrong/empty): ${opts.meta.provider}` : null,
|
|
190
|
+
opts.meta?.label ? `Label: ${opts.meta.label}` : null,
|
|
191
|
+
opts.meta?.notes ? `User notes: ${String(opts.meta.notes).slice(0, 500)}` : null,
|
|
192
|
+
'',
|
|
193
|
+
'Code references (evidence only — reason about them):',
|
|
194
|
+
codeContext,
|
|
195
|
+
'',
|
|
196
|
+
'Return the JSON decision now.',
|
|
197
|
+
]
|
|
198
|
+
.filter((l) => l !== null)
|
|
199
|
+
.join('\n');
|
|
200
|
+
|
|
201
|
+
const callRuntime =
|
|
202
|
+
opts.callRuntime ||
|
|
203
|
+
(async (user, system, cfg) => {
|
|
204
|
+
const { callLocalRuntime, loadRuntimeConfig } = await import('./runtime.mjs');
|
|
205
|
+
return callLocalRuntime(user, system, cfg || opts.runtimeConfig || loadRuntimeConfig());
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
let raw;
|
|
209
|
+
try {
|
|
210
|
+
raw = await callRuntime(prompt, INFER_SYSTEM, opts.runtimeConfig);
|
|
211
|
+
} catch (err) {
|
|
212
|
+
logger.warn('secrets', 'AI secret integration inference failed', {
|
|
213
|
+
key,
|
|
214
|
+
error: err.message,
|
|
215
|
+
});
|
|
216
|
+
return {
|
|
217
|
+
researchable: false,
|
|
218
|
+
skip_reason: `AI inference unavailable: ${err.message}`,
|
|
219
|
+
product_name: null,
|
|
220
|
+
product_slug: null,
|
|
221
|
+
kind: 'unknown',
|
|
222
|
+
topic: null,
|
|
223
|
+
notes: null,
|
|
224
|
+
priority: 'low',
|
|
225
|
+
confidence: 0,
|
|
226
|
+
source: 'ai',
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
const parsed = parseAiJson(raw);
|
|
231
|
+
if (!parsed || typeof parsed !== 'object') {
|
|
232
|
+
return {
|
|
233
|
+
researchable: false,
|
|
234
|
+
skip_reason: 'AI returned non-JSON inference',
|
|
235
|
+
product_name: null,
|
|
236
|
+
product_slug: null,
|
|
237
|
+
kind: 'unknown',
|
|
238
|
+
topic: null,
|
|
239
|
+
notes: null,
|
|
240
|
+
priority: 'low',
|
|
241
|
+
confidence: 0,
|
|
242
|
+
source: 'ai',
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
const researchable = parsed.researchable === true;
|
|
247
|
+
return {
|
|
248
|
+
researchable,
|
|
249
|
+
skip_reason: researchable ? null : String(parsed.skip_reason || 'AI declined research'),
|
|
250
|
+
product_name: parsed.product_name ? String(parsed.product_name) : null,
|
|
251
|
+
product_slug: parsed.product_slug ? String(parsed.product_slug) : null,
|
|
252
|
+
kind: String(parsed.kind || 'unknown'),
|
|
253
|
+
topic: researchable && parsed.topic ? String(parsed.topic) : null,
|
|
254
|
+
notes: researchable && parsed.notes ? String(parsed.notes) : null,
|
|
255
|
+
priority: ['low', 'medium', 'high'].includes(parsed.priority) ? parsed.priority : 'low',
|
|
256
|
+
confidence: Number.isFinite(Number(parsed.confidence)) ? Number(parsed.confidence) : 0,
|
|
257
|
+
source: 'ai',
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* Build brief via AI only.
|
|
263
|
+
* @returns {Promise<{ topic: string, notes: string, priority: string, inference: object } | null>}
|
|
264
|
+
*/
|
|
265
|
+
export async function buildIntegrationResearchBrief(key, meta = {}, opts = {}) {
|
|
266
|
+
const inference = await inferSecretIntegrationWithAi(key, { meta, ...opts });
|
|
267
|
+
if (!inference.researchable || !inference.topic) {
|
|
268
|
+
return null;
|
|
269
|
+
}
|
|
270
|
+
return {
|
|
271
|
+
topic: inference.topic,
|
|
272
|
+
notes: inference.notes || `AI-inferred product research for secret \`${key}\`.`,
|
|
273
|
+
priority: inference.priority || 'low',
|
|
274
|
+
inference,
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* Enqueue integration research when the AI says so.
|
|
280
|
+
* @returns {Promise<{ enqueued: boolean, item?: object, skipped?: string, inference?: object }>}
|
|
281
|
+
*/
|
|
282
|
+
export async function maybeEnqueueIntegrationResearch(brainDir, key, meta = {}, opts = {}) {
|
|
283
|
+
const inference = await inferSecretIntegrationWithAi(key, { meta, ...opts });
|
|
284
|
+
if (!inference.researchable || !inference.topic) {
|
|
285
|
+
return {
|
|
286
|
+
enqueued: false,
|
|
287
|
+
skipped: inference.skip_reason || 'AI declined research',
|
|
288
|
+
inference,
|
|
289
|
+
};
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
// Nobody asked for this, so it goes through the autonomous gate and counts
|
|
293
|
+
// against the same daily budget as project gap research.
|
|
294
|
+
const { proposeAutonomousResearch } = await import('./research-gate.mjs');
|
|
295
|
+
const notes = inference.notes || `AI-inferred product research for secret \`${key}\`.`;
|
|
296
|
+
const { queued, skipped } = await proposeAutonomousResearch(
|
|
297
|
+
[{ topic: inference.topic, rationale: notes, priority: inference.priority || 'low' }],
|
|
298
|
+
{ project: null, via: 'secret', brainDir, coverageCheck: opts.coverageCheck },
|
|
299
|
+
);
|
|
300
|
+
if (!queued.length) {
|
|
301
|
+
return { enqueued: false, skipped: skipped[0]?.reason || 'gate declined', inference };
|
|
302
|
+
}
|
|
303
|
+
return { enqueued: true, item: queued[0], inference };
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* Retire obsolete key-name scrape queue items (old Automated API Integration Build: KEY topics).
|
|
308
|
+
*/
|
|
309
|
+
export async function cancelBogusApiIntegrationQueueItems(overrideBrainDir) {
|
|
310
|
+
const { loadQueue, updateQueueItem } = await import('./research-queue.mjs');
|
|
311
|
+
const items = loadQueue(overrideBrainDir);
|
|
312
|
+
let cancelled = 0;
|
|
313
|
+
for (const item of items) {
|
|
314
|
+
if (item.status !== 'pending' && item.status !== 'in_progress') continue;
|
|
315
|
+
if (!/^Automated API Integration Build:/i.test(String(item.topic || ''))) continue;
|
|
316
|
+
updateQueueItem(
|
|
317
|
+
item.id,
|
|
318
|
+
{
|
|
319
|
+
status: 'failed',
|
|
320
|
+
notes: `${item.notes || ''}\n\n[auto-cancelled] Obsolete key-name scrape job; re-add the secret to trigger AI provider inference.`,
|
|
321
|
+
research_phase: 'cancelled',
|
|
322
|
+
},
|
|
323
|
+
overrideBrainDir,
|
|
324
|
+
);
|
|
325
|
+
cancelled += 1;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
// The same fixture pattern also landed in the research AGENDA — a different
|
|
329
|
+
// file with a different owner. Walking only the queue is why the designated
|
|
330
|
+
// remediation never removed the 29 agenda topics it was written for.
|
|
331
|
+
let cancelledAgendaTopics = 0;
|
|
332
|
+
try {
|
|
333
|
+
const { cancelBogusAgendaTopics } = await import('./fact-seeker.mjs');
|
|
334
|
+
({ cancelled: cancelledAgendaTopics } = cancelBogusAgendaTopics());
|
|
335
|
+
} catch {
|
|
336
|
+
// Best-effort: a missing or unreadable agenda must not block the queue half.
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
return { cancelled, cancelledAgendaTopics };
|
|
340
|
+
}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Compare repo .env files against the secrets store.
|
|
3
|
+
*
|
|
4
|
+
* The store is meant to be the SSOT and .env files its projections, but nothing
|
|
5
|
+
* ever checked that they agreed. Drift is silent and consequential: the same
|
|
6
|
+
* key can hold a rotated credential in one place and a retired one in the
|
|
7
|
+
* other, and whichever the process happens to read decides whether a service
|
|
8
|
+
* works. Length-equal drift is the dangerous kind — same provider, same format,
|
|
9
|
+
* different key — because it looks like a match in every summary that counts
|
|
10
|
+
* keys rather than comparing values.
|
|
11
|
+
*
|
|
12
|
+
* Reports only shapes and lengths. Secret values never leave this module.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import fs from 'node:fs';
|
|
16
|
+
import path from 'node:path';
|
|
17
|
+
import os from 'node:os';
|
|
18
|
+
import { loadSecrets, listSecretsMeta } from './secrets-store.mjs';
|
|
19
|
+
import { parseEnvText, isCandidateKey } from './env-import.mjs';
|
|
20
|
+
|
|
21
|
+
/** Values that are obviously templates rather than credentials. */
|
|
22
|
+
const PLACEHOLDER = /^(your[_-]?|xxx+|changeme|<|placeholder|todo|sk-your|example)/i;
|
|
23
|
+
|
|
24
|
+
export function isPlaceholderValue(value) {
|
|
25
|
+
const v = String(value || '').trim();
|
|
26
|
+
if (!v) return true;
|
|
27
|
+
if (PLACEHOLDER.test(v)) return true;
|
|
28
|
+
// `..._HERE`, `PUT_KEY_HERE` and friends
|
|
29
|
+
return /(_|-)?here$/i.test(v);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Classify one key across the two sources.
|
|
34
|
+
* @returns {'match'|'drift'|'only_env'|'only_store'|'placeholder'}
|
|
35
|
+
*/
|
|
36
|
+
export function classify(envValue, storeValue, keyInStore) {
|
|
37
|
+
if (envValue !== undefined && isPlaceholderValue(envValue)) return 'placeholder';
|
|
38
|
+
if (envValue === undefined) return 'only_store';
|
|
39
|
+
if (!keyInStore) return 'only_env';
|
|
40
|
+
return String(storeValue) === String(envValue) ? 'match' : 'drift';
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Locate the .env files worth comparing, from the project registry plus any
|
|
45
|
+
* explicit paths. Never hardcodes a repo: registry entries and caller input only.
|
|
46
|
+
*/
|
|
47
|
+
export function resolveEnvFiles(roots = [], filenames = ['.env']) {
|
|
48
|
+
const files = [];
|
|
49
|
+
for (const root of roots) {
|
|
50
|
+
for (const name of filenames) {
|
|
51
|
+
const candidate = path.join(root, name);
|
|
52
|
+
if (fs.existsSync(candidate)) files.push(candidate);
|
|
53
|
+
// Sub-app env files (e.g. a server/ workspace) drift independently.
|
|
54
|
+
const nested = path.join(root, 'server', name);
|
|
55
|
+
if (fs.existsSync(nested)) files.push(nested);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return files;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Diff every candidate secret in the given .env files against the store.
|
|
63
|
+
*/
|
|
64
|
+
/**
|
|
65
|
+
* Which repo a .env belongs to, for comparison against a secret's binding.
|
|
66
|
+
* Walks up to the directory holding the file, ignoring a nested app dir.
|
|
67
|
+
*/
|
|
68
|
+
export function repoOfEnvFile(file) {
|
|
69
|
+
const parts = path.resolve(file).split(path.sep);
|
|
70
|
+
const idx = parts.lastIndexOf('Github');
|
|
71
|
+
if (idx !== -1 && parts[idx + 1]) return parts[idx + 1];
|
|
72
|
+
return path.basename(path.dirname(path.resolve(file)));
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export async function diffEnvAgainstStore(brainDir, envFiles) {
|
|
76
|
+
const store = await loadSecrets(brainDir);
|
|
77
|
+
const meta = await listSecretsMeta(brainDir).catch(() => []);
|
|
78
|
+
const bindingOf = new Map(meta.map((m) => [m.key, m.repos || m.repo || null]));
|
|
79
|
+
const rows = [];
|
|
80
|
+
const totals = { match: 0, drift: 0, only_env: 0, placeholder: 0, collision: 0 };
|
|
81
|
+
|
|
82
|
+
for (const file of envFiles) {
|
|
83
|
+
let parsed;
|
|
84
|
+
try {
|
|
85
|
+
parsed = parseEnvText(fs.readFileSync(file, 'utf8'));
|
|
86
|
+
} catch (err) {
|
|
87
|
+
rows.push({ key: '(unreadable)', file, status: 'error', detail: err.message });
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
for (const [key, value] of Object.entries(parsed)) {
|
|
92
|
+
if (!isCandidateKey(key)) continue;
|
|
93
|
+
let status = classify(value, store[key], key in store);
|
|
94
|
+
if (status === 'only_store') continue;
|
|
95
|
+
|
|
96
|
+
// A store entry is bound to exactly one repo. When the .env belongs to a
|
|
97
|
+
// different repo, the two values are not the same credential wearing one
|
|
98
|
+
// name — they are two products' credentials colliding on a shared
|
|
99
|
+
// variable name (STRIPE_SECRET_KEY, MAILCOW_API_KEY). Reporting that as
|
|
100
|
+
// drift invites "fixing" it by overwriting a live key with another
|
|
101
|
+
// product's.
|
|
102
|
+
const binding = bindingOf.get(key) || null;
|
|
103
|
+
const envRepo = repoOfEnvFile(file);
|
|
104
|
+
const crossProduct = Boolean(
|
|
105
|
+
status === 'drift' && binding && !String(binding).split(',').includes(envRepo),
|
|
106
|
+
);
|
|
107
|
+
if (crossProduct) status = 'collision';
|
|
108
|
+
|
|
109
|
+
totals[status] += 1;
|
|
110
|
+
if (status === 'match') continue;
|
|
111
|
+
rows.push({
|
|
112
|
+
key,
|
|
113
|
+
file,
|
|
114
|
+
status,
|
|
115
|
+
binding,
|
|
116
|
+
envRepo,
|
|
117
|
+
envLength: String(value ?? '').length,
|
|
118
|
+
storeLength: key in store ? String(store[key]).length : null,
|
|
119
|
+
// Equal lengths mean the same credential format — a rotation that
|
|
120
|
+
// landed on one side only, rather than two unrelated values.
|
|
121
|
+
sameShape: key in store && String(store[key]).length === String(value ?? '').length,
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
rows.sort((a, b) => (a.status === b.status ? a.key.localeCompare(b.key) : a.status.localeCompare(b.status)));
|
|
127
|
+
return { totals, rows, storeKeyCount: Object.keys(store).filter((k) => !k.startsWith('__')).length };
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export function defaultBrainDir() {
|
|
131
|
+
return path.join(os.homedir(), '.agent', 'skills', 'total-recall');
|
|
132
|
+
}
|