renor-engine 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -2
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli.js +26 -0
- package/dist/eval/indexSnapshots.js +1 -0
- package/dist/evidence/ollamaChat.js +1 -1
- package/dist/extraction/jevDecisionModel.js +307 -0
- package/dist/extraction/secretGuard.js +7 -3
- package/dist/learning/cli.js +113 -0
- package/dist/learning/schema.js +144 -0
- package/dist/learning/settings.js +45 -0
- package/dist/learning/store.js +267 -0
- package/dist/learning/tap.js +401 -0
- package/dist/memory/dreamer/adjudicate.js +215 -0
- package/dist/memory/dreamer/apply.js +120 -0
- package/dist/memory/dreamer/budget.js +64 -0
- package/dist/memory/dreamer/candidates.js +149 -0
- package/dist/memory/dreamer/collection.js +104 -0
- package/dist/memory/dreamer/dreamer.js +352 -0
- package/dist/memory/dreamer/review.js +111 -0
- package/dist/memory/factConflicts.js +2 -0
- package/dist/memory/lifecycle.js +28 -5
- package/dist/memory/writer.js +3 -2
- package/dist/mvp/agentMemory.js +3 -0
- package/dist/mvp/cli.js +11 -0
- package/dist/mvp/config.js +4 -0
- package/dist/mvp/exportAll.js +19 -0
- package/dist/mvp/server.js +118 -10
- package/dist/retrieval/orchestrator.js +69 -16
- package/dist/retrieval/rankingTrace.js +184 -0
- package/dist/sharing/cli.js +181 -0
- package/dist/sharing/examples.js +136 -0
- package/dist/sharing/fields.js +225 -0
- package/dist/sharing/loop.js +80 -0
- package/dist/sharing/project.js +327 -0
- package/dist/sharing/send.js +466 -0
- package/dist/sharing/state.js +222 -0
- package/dist/sharing/wording.js +48 -0
- package/package.json +1 -1
- package/dist/BUILD.json +0 -6
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.RankingTrace = exports.RANKING_TERMS = void 0;
|
|
7
|
+
exports.rankingTermIndex = rankingTermIndex;
|
|
8
|
+
exports.newRankingTerms = newRankingTerms;
|
|
9
|
+
exports.attachRankingTerms = attachRankingTerms;
|
|
10
|
+
exports.addRankingTerm = addRankingTerm;
|
|
11
|
+
exports.clientNotePath = clientNotePath;
|
|
12
|
+
exports.traceKey = traceKey;
|
|
13
|
+
exports.currentRankingTrace = currentRankingTrace;
|
|
14
|
+
exports.withRankingTrace = withRankingTrace;
|
|
15
|
+
const node_crypto_1 = require("node:crypto");
|
|
16
|
+
const node_async_hooks_1 = require("node:async_hooks");
|
|
17
|
+
const node_path_1 = __importDefault(require("node:path"));
|
|
18
|
+
const hiddenSpans_1 = require("../mvp/hiddenSpans");
|
|
19
|
+
/**
|
|
20
|
+
* What the ranker added up for each candidate of one read, for the learning log (src/learning, schema version 2).
|
|
21
|
+
*
|
|
22
|
+
* rankWindow (orchestrator.ts) gives every candidate a base score (a cosine for a semantic hit, 0.3 for a graph hit, the fact
|
|
23
|
+
* lane's own score for a fact hit, 0 for a session-expansion statement) and then adds or subtracts named terms. A trace keeps
|
|
24
|
+
* each term as it is applied, so base + the sum of the terms is the score the candidate was ranked by, and a learner can refit
|
|
25
|
+
* the weights from the log.
|
|
26
|
+
*
|
|
27
|
+
* Nothing here changes a score, an order or a reply. A trace is only written to while a read runs inside `withRankingTrace`,
|
|
28
|
+
* and only the learning log's tool wrapper opens that scope (server.ts), and only when the log is on. With the log off no scope
|
|
29
|
+
* is opened and every call here returns at once, so the ranking runs as it always did.
|
|
30
|
+
*
|
|
31
|
+
* Identity. A candidate is keyed by the note path the reply shows, its heading path and the text as the reply shows it
|
|
32
|
+
* (labelled by mvp/hiddenSpans.ts): the chunk id is not in the reply. The note path the reply shows is the path relative to the
|
|
33
|
+
* memory folder when the note is inside it (engine.ts clientPath); the ranker holds the stored path, so a trace made for a memory
|
|
34
|
+
* folder shows the same path the reply does. The key is hashed, so no text is kept in a map longer than the read that made it.
|
|
35
|
+
*/
|
|
36
|
+
/** The named additive terms rankWindow applies, in the order it applies them. Each is a signed number added to a score. */
|
|
37
|
+
exports.RANKING_TERMS = [
|
|
38
|
+
'stub', // a speaker's stub line: -0.5 (orchestrator.ts isSpeakerStub)
|
|
39
|
+
'content', // a contentful chunk: up to +0.12 (contentBoost)
|
|
40
|
+
'priority', // a path the caller prioritises: +0.1
|
|
41
|
+
'entity', // the query names the entity this note is the source of: +0.2
|
|
42
|
+
'entity_lane', // the optional entity lane's lexical coverage lift: up to +0.12
|
|
43
|
+
'recency', // time decay: up to +0.1 for a note written today, 0 after 30 days
|
|
44
|
+
'fact', // the note's own entity has a fact the query names: +0.15
|
|
45
|
+
'asof_year', // an explicit past-date query, and the text names its year: +0.08
|
|
46
|
+
'asof_month', // the same, and the text names its month: +0.04
|
|
47
|
+
'historical_intent', // past intent: a historical note sharing two of the query's words: +0.3
|
|
48
|
+
'stale_fact', // present query: the text carries a value the lifecycle has superseded: -0.15
|
|
49
|
+
'past_fact', // past intent: the same text is lifted instead: +0.1
|
|
50
|
+
'current_fact', // the text carries the current value of a superseded predicate: +0.1
|
|
51
|
+
];
|
|
52
|
+
function rankingTermIndex(term) {
|
|
53
|
+
return exports.RANKING_TERMS.indexOf(term);
|
|
54
|
+
}
|
|
55
|
+
// The terms and the pool rank travel with a candidate object as symbol-keyed properties: a spread copies them, JSON does not
|
|
56
|
+
// show them, and a reply never carries them (the reply copies named fields only).
|
|
57
|
+
const TERMS = Symbol('renor.rankingTerms');
|
|
58
|
+
const POOL_RANK = Symbol('renor.poolRank');
|
|
59
|
+
const zeros = () => new Array(exports.RANKING_TERMS.length).fill(0);
|
|
60
|
+
/** A fresh row of zeros for a candidate about to be scored. */
|
|
61
|
+
function newRankingTerms() {
|
|
62
|
+
return zeros();
|
|
63
|
+
}
|
|
64
|
+
/** Attaches a candidate's term row (the row rankWindow filled while it scored the candidate). */
|
|
65
|
+
function attachRankingTerms(hit, terms) {
|
|
66
|
+
hit[TERMS] = terms;
|
|
67
|
+
return hit;
|
|
68
|
+
}
|
|
69
|
+
/** Adds one term's delta to a candidate that is already scored; the score itself is changed by the caller, as before. */
|
|
70
|
+
function addRankingTerm(hit, term, delta) {
|
|
71
|
+
const carried = hit;
|
|
72
|
+
const row = carried[TERMS] ?? (carried[TERMS] = zeros());
|
|
73
|
+
row[rankingTermIndex(term)] += delta;
|
|
74
|
+
}
|
|
75
|
+
function termsOf(hit) {
|
|
76
|
+
return hit[TERMS];
|
|
77
|
+
}
|
|
78
|
+
function poolRankOf(hit) {
|
|
79
|
+
return hit[POOL_RANK] ?? null;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* The note path the reply shows for a stored path: relative to the memory folder when the note is inside it, else its file name.
|
|
83
|
+
* This is engine.ts `clientPath` without its cache, repeated here so the ranker does not import the engine (which imports it).
|
|
84
|
+
* A path that is not absolute is already in the form the reply shows (a candidate the engine's search has listed), so it is
|
|
85
|
+
* returned as it is; with no memory folder every path is.
|
|
86
|
+
*/
|
|
87
|
+
function clientNotePath(memoryDir, stored) {
|
|
88
|
+
if (memoryDir === undefined || !node_path_1.default.isAbsolute(stored))
|
|
89
|
+
return stored;
|
|
90
|
+
const resolved = node_path_1.default.resolve(stored);
|
|
91
|
+
const root = node_path_1.default.resolve(memoryDir);
|
|
92
|
+
return resolved === root || resolved.startsWith(root + node_path_1.default.sep) ? node_path_1.default.relative(root, resolved) || stored : node_path_1.default.basename(stored);
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* The key of a candidate, given the note path as the reply shows it. Hashed, over the path, the heading path and the text as the
|
|
96
|
+
* reply labels it (mvp/hiddenSpans.ts), so a candidate the ranker holds and the same candidate in a reply have one key.
|
|
97
|
+
*/
|
|
98
|
+
function traceKey(shownNotePath, headingPath, text) {
|
|
99
|
+
const labelled = (0, hiddenSpans_1.labelMemoryText)({ text }).text;
|
|
100
|
+
return (0, node_crypto_1.createHash)('sha256').update(JSON.stringify([shownNotePath, headingPath ?? '', labelled])).digest('hex');
|
|
101
|
+
}
|
|
102
|
+
class RankingTrace {
|
|
103
|
+
memoryDir;
|
|
104
|
+
entries = new Map();
|
|
105
|
+
/** `memoryDir`: the memory folder the reply's note paths are relative to. Without one, the stored paths are the shown ones. */
|
|
106
|
+
constructor(memoryDir) {
|
|
107
|
+
this.memoryDir = memoryDir;
|
|
108
|
+
}
|
|
109
|
+
/** Records a candidate as the ranker holds it now. Its terms are the ones addRankingTerm and attachRankingTerms put on it. */
|
|
110
|
+
touch(hit) {
|
|
111
|
+
const notePath = clientNotePath(this.memoryDir, hit.notePath);
|
|
112
|
+
const key = traceKey(notePath, hit.headingPath, hit.text);
|
|
113
|
+
const known = this.entries.get(key);
|
|
114
|
+
if (known)
|
|
115
|
+
return known;
|
|
116
|
+
const terms = termsOf(hit) ?? zeros();
|
|
117
|
+
const score = hit.score ?? 0;
|
|
118
|
+
const entry = {
|
|
119
|
+
key,
|
|
120
|
+
notePath,
|
|
121
|
+
headingPath: hit.headingPath ?? null,
|
|
122
|
+
reason: hit.reason ?? '',
|
|
123
|
+
base: score - terms.reduce((sum, term) => sum + term, 0),
|
|
124
|
+
terms: [...terms],
|
|
125
|
+
poolRank: poolRankOf(hit),
|
|
126
|
+
windowPos: null,
|
|
127
|
+
budgetKept: null,
|
|
128
|
+
};
|
|
129
|
+
this.entries.set(key, entry);
|
|
130
|
+
return entry;
|
|
131
|
+
}
|
|
132
|
+
/** Records a candidate's place in the scored pool (1 is the best score). */
|
|
133
|
+
pool(hit, rank) {
|
|
134
|
+
hit[POOL_RANK] = rank;
|
|
135
|
+
this.touch(hit);
|
|
136
|
+
}
|
|
137
|
+
/** Records the candidates in the order the reply's window presents them, 1-based, before any budget cut. */
|
|
138
|
+
presented(hits) {
|
|
139
|
+
hits.forEach((hit, index) => {
|
|
140
|
+
this.touch(hit).windowPos = index + 1;
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Records which candidates the budget kept. `considered` are the candidates it weighed and `kept` the ones it kept; a candidate
|
|
145
|
+
* considered and not kept is false. A ledger memory is listed with its note path and heading blanked (agentMemory.ts), so it
|
|
146
|
+
* cannot be told from the candidate the ranker scored: it gets no entry rather than a second, unmatched one.
|
|
147
|
+
*/
|
|
148
|
+
budgeted(considered, kept) {
|
|
149
|
+
const keptKeys = new Set(kept.filter(hit => hit.notePath !== '').map(hit => this.keyOfHit(hit)));
|
|
150
|
+
for (const hit of considered) {
|
|
151
|
+
if (hit.notePath === '')
|
|
152
|
+
continue;
|
|
153
|
+
const key = this.keyOfHit(hit);
|
|
154
|
+
const entry = this.entries.get(key) ?? this.touch(hit);
|
|
155
|
+
entry.budgetKept = keptKeys.has(key);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
/** The entry under a key from `traceKey`, when the trace has one. */
|
|
159
|
+
byKey(key) {
|
|
160
|
+
return this.entries.get(key);
|
|
161
|
+
}
|
|
162
|
+
/** Finds a unique chunk by fields that reply decoration preserves even when it changes the displayed text. */
|
|
163
|
+
byLocation(notePath, headingPath) {
|
|
164
|
+
const matches = [...this.entries.values()].filter(entry => entry.notePath === notePath && entry.headingPath === (headingPath ?? null));
|
|
165
|
+
return matches.length === 1 ? matches[0] : undefined;
|
|
166
|
+
}
|
|
167
|
+
/** Every candidate of the read, in the order it was first recorded. */
|
|
168
|
+
all() {
|
|
169
|
+
return [...this.entries.values()];
|
|
170
|
+
}
|
|
171
|
+
keyOfHit(hit) {
|
|
172
|
+
return traceKey(clientNotePath(this.memoryDir, hit.notePath), hit.headingPath, hit.text);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
exports.RankingTrace = RankingTrace;
|
|
176
|
+
const scope = new node_async_hooks_1.AsyncLocalStorage();
|
|
177
|
+
/** The trace of the read in progress, when one is being collected. */
|
|
178
|
+
function currentRankingTrace() {
|
|
179
|
+
return scope.getStore();
|
|
180
|
+
}
|
|
181
|
+
/** Runs `work` with `trace` collecting the terms of every read it makes. */
|
|
182
|
+
function withRankingTrace(trace, work) {
|
|
183
|
+
return scope.run(trace, work);
|
|
184
|
+
}
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.TELEMETRY_ACTIONS = void 0;
|
|
7
|
+
exports.runTelemetry = runTelemetry;
|
|
8
|
+
const node_fs_1 = __importDefault(require("node:fs"));
|
|
9
|
+
const settings_1 = require("../learning/settings");
|
|
10
|
+
const store_1 = require("../learning/store");
|
|
11
|
+
const fields_1 = require("./fields");
|
|
12
|
+
const send_1 = require("./send");
|
|
13
|
+
const state_1 = require("./state");
|
|
14
|
+
const wording_1 = require("./wording");
|
|
15
|
+
exports.TELEMETRY_ACTIONS = ['status', 'signals', 'data', 'off', 'delete', 'preview', 'send', 'fields'];
|
|
16
|
+
function memoryDirOf(options, env) {
|
|
17
|
+
const value = (options.memoryDir ?? env.MVP_MEMORY_DIR ?? '').trim();
|
|
18
|
+
return value === '' ? null : value;
|
|
19
|
+
}
|
|
20
|
+
/** Returns the exit code. Throws nothing the user needs to read: problems are printed on `err`. */
|
|
21
|
+
async function runTelemetry(action, argument, options, io) {
|
|
22
|
+
const env = io.env;
|
|
23
|
+
const memoryDir = memoryDirOf(options, env);
|
|
24
|
+
const dir = (0, state_1.sharingDirFor)(memoryDir, env, options.dir);
|
|
25
|
+
// Hosted or not as the service decides it, so send, off and delete reach the endpoint the way the service's sender does.
|
|
26
|
+
const common = { dir, env, fetch: io.fetch, now: io.now, hosted: (0, state_1.hostedEngine)(env) };
|
|
27
|
+
try {
|
|
28
|
+
switch (action) {
|
|
29
|
+
case 'status': {
|
|
30
|
+
const record = (0, state_1.readConsent)(dir);
|
|
31
|
+
const state = (0, state_1.readState)(dir);
|
|
32
|
+
const override = (0, state_1.envOverride)(env);
|
|
33
|
+
const level = (0, state_1.effectiveLevel)(dir, env);
|
|
34
|
+
io.out(`sharing: ${level}${override !== null && level !== (record?.level ?? 'off') ? ` (lowered by the environment from ${record?.level ?? 'off'})` : ''}`);
|
|
35
|
+
if (record === null)
|
|
36
|
+
io.out('choice: none made (the default is off; nothing is recorded for sharing or sent)');
|
|
37
|
+
else {
|
|
38
|
+
io.out(`choice: ${record.level}, chosen ${record.changedAt} via ${record.source}, wording ${record.wording}, 18 or older: ${record.adult ? 'yes' : 'no'}`);
|
|
39
|
+
}
|
|
40
|
+
io.out(`folder: ${dir}`);
|
|
41
|
+
io.out(`endpoint: ${(0, send_1.sharingEndpoint)(env)}`);
|
|
42
|
+
if (state?.paused?.code === 'sharing_closed')
|
|
43
|
+
io.out('sending: paused, Renor has closed sharing; nothing is sent until it is open again');
|
|
44
|
+
else if (state?.paused?.code === 'consent_required')
|
|
45
|
+
io.out('sending: paused, Renor refused the consent record; choose a level again to resume');
|
|
46
|
+
io.out(`debug mode: ${(0, send_1.debugOn)(env) ? 'on (RENOR_SHARING_DEBUG): nothing is sent, bodies are printed' : 'off'}`);
|
|
47
|
+
if (memoryDir === null)
|
|
48
|
+
io.out('learning log: not found (set MVP_MEMORY_DIR or pass --memory-dir; sharing sends rows made from it)');
|
|
49
|
+
else {
|
|
50
|
+
const file = (0, settings_1.learningDbPath)(memoryDir);
|
|
51
|
+
if (!node_fs_1.default.existsSync(file))
|
|
52
|
+
io.out(`learning log: none at ${file}`);
|
|
53
|
+
else {
|
|
54
|
+
const log = store_1.LearningLog.open(file, { retentionDays: (0, settings_1.learningRetentionDays)(env), rawQueries: false, create: false });
|
|
55
|
+
try {
|
|
56
|
+
io.out(`learning log: ${file}`);
|
|
57
|
+
if (log)
|
|
58
|
+
io.out(` rows inside the retention window: ${JSON.stringify(log.counts())}`);
|
|
59
|
+
}
|
|
60
|
+
finally {
|
|
61
|
+
log?.close();
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
if (state) {
|
|
66
|
+
io.out(`sent before: ${state.everSent ? 'yes' : 'no'}${state.deletePending ? '; a deletion is owed to the endpoint and will be retried' : ''}`);
|
|
67
|
+
}
|
|
68
|
+
return 0;
|
|
69
|
+
}
|
|
70
|
+
case 'fields': {
|
|
71
|
+
const wanted = argument === undefined ? ['signals', 'data'] : [argument];
|
|
72
|
+
for (const level of wanted) {
|
|
73
|
+
if (level !== 'signals' && level !== 'data') {
|
|
74
|
+
io.err('usage: renor-engine telemetry fields [signals|data]');
|
|
75
|
+
return 2;
|
|
76
|
+
}
|
|
77
|
+
io.out(`# fields sent at level ${level}`);
|
|
78
|
+
for (const field of (0, fields_1.fieldsAtLevel)(level))
|
|
79
|
+
io.out(field);
|
|
80
|
+
}
|
|
81
|
+
return 0;
|
|
82
|
+
}
|
|
83
|
+
case 'signals':
|
|
84
|
+
case 'data': {
|
|
85
|
+
const level = action;
|
|
86
|
+
io.out((0, wording_1.wordingFor)(level));
|
|
87
|
+
io.out('');
|
|
88
|
+
if (options.yes !== true) {
|
|
89
|
+
const answer = await io.ask(`${wording_1.ADULT_LINE} Share at this level? [y/N] `);
|
|
90
|
+
if (answer === null) {
|
|
91
|
+
io.err('No terminal to ask on: nothing was changed. Run it in a terminal, or add --yes if you have read the words above.');
|
|
92
|
+
return 1;
|
|
93
|
+
}
|
|
94
|
+
if (!answer) {
|
|
95
|
+
io.out('Nothing was changed.');
|
|
96
|
+
return 0;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
const result = await (0, send_1.chooseLevel)({ ...common, level, wording: wording_1.CLI_WORDING_VERSION, adult: true, source: 'cli' });
|
|
100
|
+
io.out(result.changed ? `Sharing is now: ${level}. Recorded ${result.record?.changedAt}.` : `Sharing was already: ${level}.`);
|
|
101
|
+
if (memoryDir === null)
|
|
102
|
+
io.out('Note: no learning log folder is set (MVP_MEMORY_DIR), so there is nothing to share yet.');
|
|
103
|
+
return 0;
|
|
104
|
+
}
|
|
105
|
+
case 'off': {
|
|
106
|
+
const result = await (0, send_1.chooseLevel)({ ...common, level: 'off', wording: wording_1.CLI_WORDING_VERSION, adult: true, source: 'cli' });
|
|
107
|
+
if (!result.changed && result.record === null)
|
|
108
|
+
io.out('Sharing was already off, and no choice had ever been made. Nothing was written.');
|
|
109
|
+
else
|
|
110
|
+
io.out('Sharing is now: off. Recording for sharing has stopped.');
|
|
111
|
+
if (result.remoteDeleted === true)
|
|
112
|
+
io.out('Renor was asked to delete what it holds, and has.');
|
|
113
|
+
else if (result.deleteOwedUnderDoNotTrack)
|
|
114
|
+
io.out('DO_NOT_TRACK=1 forbids any request, so Renor has not been asked yet to delete what it holds. The deletion is recorded here and sent the first time the engine runs without DO_NOT_TRACK=1.');
|
|
115
|
+
else if (result.remoteDeleted === false)
|
|
116
|
+
io.out(`Renor could not be reached to delete what it holds (${result.reason ?? 'unknown'}). The deletion is kept and retried when the service next runs.`);
|
|
117
|
+
return result.remoteDeleted === false ? 1 : 0;
|
|
118
|
+
}
|
|
119
|
+
case 'delete': {
|
|
120
|
+
const result = await (0, send_1.deleteAllShared)(common);
|
|
121
|
+
if (!result.remote.ok) {
|
|
122
|
+
io.err(result.remote.owedUnderDoNotTrack
|
|
123
|
+
? 'DO_NOT_TRACK=1 forbids any request, so Renor has not been asked yet to delete what it holds; nothing was removed here. The deletion is recorded and sent the first time the engine runs without DO_NOT_TRACK=1.'
|
|
124
|
+
: `Renor could not be reached (${result.remote.reason ?? 'unknown'}); nothing was removed here, and the deletion is kept to retry.`);
|
|
125
|
+
return 1;
|
|
126
|
+
}
|
|
127
|
+
io.out(result.remote.requested ? 'Renor deleted what it held for this install.' : 'Nothing had ever been sent, so Renor holds nothing.');
|
|
128
|
+
io.out('The local sharing files (state, saved example texts) were removed. Your choice is unchanged.');
|
|
129
|
+
if (options.local) {
|
|
130
|
+
if (memoryDir === null) {
|
|
131
|
+
io.err('--local needs MVP_MEMORY_DIR or --memory-dir: the learning log was not touched.');
|
|
132
|
+
return 1;
|
|
133
|
+
}
|
|
134
|
+
const file = (0, settings_1.learningDbPath)(memoryDir);
|
|
135
|
+
const log = node_fs_1.default.existsSync(file) ? store_1.LearningLog.open(file, { retentionDays: (0, settings_1.learningRetentionDays)(env), rawQueries: false, create: false }) : null;
|
|
136
|
+
if (log) {
|
|
137
|
+
try {
|
|
138
|
+
log.deleteAll();
|
|
139
|
+
}
|
|
140
|
+
finally {
|
|
141
|
+
log.close();
|
|
142
|
+
}
|
|
143
|
+
io.out('The local learning log was emptied.');
|
|
144
|
+
}
|
|
145
|
+
else
|
|
146
|
+
io.out('There was no local learning log.');
|
|
147
|
+
}
|
|
148
|
+
return 0;
|
|
149
|
+
}
|
|
150
|
+
case 'preview':
|
|
151
|
+
case 'send': {
|
|
152
|
+
if (action === 'preview' && argument !== undefined && argument !== 'signals' && argument !== 'data') {
|
|
153
|
+
io.err('usage: renor-engine telemetry preview [signals|data]');
|
|
154
|
+
return 2;
|
|
155
|
+
}
|
|
156
|
+
const result = await (0, send_1.shareNow)({
|
|
157
|
+
...common, memoryDir, debug: action === 'preview' ? true : undefined,
|
|
158
|
+
previewLevel: action === 'preview' && (argument === 'signals' || argument === 'data') ? argument : undefined,
|
|
159
|
+
});
|
|
160
|
+
if (action === 'send')
|
|
161
|
+
await (0, send_1.settlePendingDelete)(common);
|
|
162
|
+
if (result.status === 'debug') {
|
|
163
|
+
io.out(`# debug: nothing was sent and nothing was changed. These ${result.batches} request body(ies) are exactly what would be sent to ${(0, send_1.sharingEndpoint)(env)}:`);
|
|
164
|
+
for (const payload of result.payloads ?? [])
|
|
165
|
+
io.out(JSON.stringify(payload, null, 2));
|
|
166
|
+
}
|
|
167
|
+
else {
|
|
168
|
+
io.out(`status: ${result.status}; level: ${result.level}; rows: ${result.rows}; batches: ${result.batches}${result.skipped ? `; skipped (failed their own contract): ${result.skipped}` : ''}${result.reason ? `; ${result.reason}` : ''}`);
|
|
169
|
+
}
|
|
170
|
+
return result.status === 'failed' ? 1 : 0;
|
|
171
|
+
}
|
|
172
|
+
default:
|
|
173
|
+
io.err(`usage: renor-engine telemetry <${exports.TELEMETRY_ACTIONS.join(' | ')}>`);
|
|
174
|
+
return 2;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
catch (err) {
|
|
178
|
+
io.err(err instanceof Error ? err.message : String(err));
|
|
179
|
+
return 1;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.ExampleStore = void 0;
|
|
7
|
+
exports.closeExampleStores = closeExampleStores;
|
|
8
|
+
exports.sharingHook = sharingHook;
|
|
9
|
+
const sqlite_1 = __importDefault(require("../db/sqlite"));
|
|
10
|
+
const node_fs_1 = __importDefault(require("node:fs"));
|
|
11
|
+
const node_path_1 = __importDefault(require("node:path"));
|
|
12
|
+
const secretScanner_1 = require("../extraction/secretScanner");
|
|
13
|
+
const fields_1 = require("./fields");
|
|
14
|
+
const state_1 = require("./state");
|
|
15
|
+
class ExampleStore {
|
|
16
|
+
db;
|
|
17
|
+
constructor(db) {
|
|
18
|
+
this.db = db;
|
|
19
|
+
}
|
|
20
|
+
static open(dir, create) {
|
|
21
|
+
const file = node_path_1.default.join(dir, state_1.EXAMPLES_FILE);
|
|
22
|
+
if (!create && !node_fs_1.default.existsSync(file))
|
|
23
|
+
return null;
|
|
24
|
+
node_fs_1.default.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
25
|
+
node_fs_1.default.closeSync(node_fs_1.default.openSync(file, node_fs_1.default.constants.O_CREAT | node_fs_1.default.constants.O_WRONLY, 0o600));
|
|
26
|
+
const db = new sqlite_1.default(file);
|
|
27
|
+
try {
|
|
28
|
+
db.pragma('journal_mode = WAL');
|
|
29
|
+
db.pragma('secure_delete = ON');
|
|
30
|
+
db.pragma('busy_timeout = 5000');
|
|
31
|
+
db.exec(`
|
|
32
|
+
CREATE TABLE IF NOT EXISTS queries (query_key TEXT PRIMARY KEY, text TEXT NOT NULL, at INTEGER NOT NULL) WITHOUT ROWID;
|
|
33
|
+
CREATE TABLE IF NOT EXISTS memories (memory_key TEXT PRIMARY KEY, text TEXT NOT NULL, at INTEGER NOT NULL) WITHOUT ROWID;
|
|
34
|
+
`);
|
|
35
|
+
}
|
|
36
|
+
catch (err) {
|
|
37
|
+
db.close();
|
|
38
|
+
throw err;
|
|
39
|
+
}
|
|
40
|
+
return new ExampleStore(db);
|
|
41
|
+
}
|
|
42
|
+
putQuery(queryKey, text, at) {
|
|
43
|
+
const kept = keepable(text);
|
|
44
|
+
if (kept === null)
|
|
45
|
+
return;
|
|
46
|
+
this.db.prepare('INSERT INTO queries (query_key, text, at) VALUES (?, ?, ?) ON CONFLICT (query_key) DO UPDATE SET text = excluded.text, at = excluded.at')
|
|
47
|
+
.run(queryKey, kept, at);
|
|
48
|
+
}
|
|
49
|
+
putMemory(memoryKey, text, at) {
|
|
50
|
+
const kept = keepable(text);
|
|
51
|
+
if (kept === null)
|
|
52
|
+
return;
|
|
53
|
+
this.db.prepare('INSERT INTO memories (memory_key, text, at) VALUES (?, ?, ?) ON CONFLICT (memory_key) DO UPDATE SET text = excluded.text, at = excluded.at')
|
|
54
|
+
.run(memoryKey, kept, at);
|
|
55
|
+
}
|
|
56
|
+
query(queryKey) {
|
|
57
|
+
const row = this.db.prepare('SELECT text FROM queries WHERE query_key = ?').get(queryKey);
|
|
58
|
+
return row?.text ?? null;
|
|
59
|
+
}
|
|
60
|
+
memory(memoryKey) {
|
|
61
|
+
const row = this.db.prepare('SELECT text FROM memories WHERE memory_key = ?').get(memoryKey);
|
|
62
|
+
return row?.text ?? null;
|
|
63
|
+
}
|
|
64
|
+
count() {
|
|
65
|
+
const n = (table) => this.db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get().n;
|
|
66
|
+
return { queries: n('queries'), memories: n('memories') };
|
|
67
|
+
}
|
|
68
|
+
/** Deletes the texts last put at or before `cutoff` (ms): after a full send, no unsent row uses them. */
|
|
69
|
+
forgetUpTo(cutoff) {
|
|
70
|
+
this.db.prepare('DELETE FROM queries WHERE at <= ?').run(cutoff);
|
|
71
|
+
this.db.prepare('DELETE FROM memories WHERE at <= ?').run(cutoff);
|
|
72
|
+
}
|
|
73
|
+
close() {
|
|
74
|
+
if (this.db.open)
|
|
75
|
+
this.db.close();
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
exports.ExampleStore = ExampleStore;
|
|
79
|
+
/** A text worth keeping: not empty, not flagged by the secret scanner; cut at the limit. */
|
|
80
|
+
function keepable(text) {
|
|
81
|
+
const trimmed = text.trim();
|
|
82
|
+
if (trimmed === '')
|
|
83
|
+
return null;
|
|
84
|
+
const cut = trimmed.length > fields_1.MAX_EXAMPLE_TEXT ? trimmed.slice(0, fields_1.MAX_EXAMPLE_TEXT) : trimmed;
|
|
85
|
+
// The scan covers the whole text, not just the part kept: a secret past the cut still marks the text as one to leave out.
|
|
86
|
+
return (0, secretScanner_1.scanSecrets)(trimmed) === null ? cut : null;
|
|
87
|
+
}
|
|
88
|
+
const open = new Map();
|
|
89
|
+
function closeExampleStores() {
|
|
90
|
+
for (const store of open.values())
|
|
91
|
+
store.close();
|
|
92
|
+
open.clear();
|
|
93
|
+
}
|
|
94
|
+
/** The hook for one sharing folder. `now` is for tests. */
|
|
95
|
+
function sharingHook(dir, env = process.env, now = Date.now) {
|
|
96
|
+
const store = () => {
|
|
97
|
+
const key = node_path_1.default.resolve(dir, state_1.EXAMPLES_FILE);
|
|
98
|
+
let held = open.get(key);
|
|
99
|
+
// The file may have been removed by `off`/`delete` from another process: open a fresh handle if so.
|
|
100
|
+
if (held && !node_fs_1.default.existsSync(key)) {
|
|
101
|
+
held.close();
|
|
102
|
+
open.delete(key);
|
|
103
|
+
held = undefined;
|
|
104
|
+
}
|
|
105
|
+
if (!held) {
|
|
106
|
+
const made = ExampleStore.open(dir, true);
|
|
107
|
+
if (made === null)
|
|
108
|
+
throw new Error('the example store could not be created');
|
|
109
|
+
open.set(key, made);
|
|
110
|
+
held = made;
|
|
111
|
+
}
|
|
112
|
+
return held;
|
|
113
|
+
};
|
|
114
|
+
const dataOn = () => (0, state_1.effectiveLevel)(dir, env) === 'data';
|
|
115
|
+
return {
|
|
116
|
+
level: () => (0, state_1.effectiveLevel)(dir, env),
|
|
117
|
+
query(queryKey, text) {
|
|
118
|
+
try {
|
|
119
|
+
if (dataOn())
|
|
120
|
+
store().putQuery(queryKey, text, now());
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
// Keeping an example must never fail the call it describes.
|
|
124
|
+
}
|
|
125
|
+
},
|
|
126
|
+
memory(memoryKey, text) {
|
|
127
|
+
try {
|
|
128
|
+
if (dataOn())
|
|
129
|
+
store().putMemory(memoryKey, text, now());
|
|
130
|
+
}
|
|
131
|
+
catch {
|
|
132
|
+
// As above.
|
|
133
|
+
}
|
|
134
|
+
},
|
|
135
|
+
};
|
|
136
|
+
}
|