renor-engine 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -2
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli.js +26 -0
- package/dist/eval/indexSnapshots.js +1 -0
- package/dist/evidence/ollamaChat.js +1 -1
- package/dist/extraction/jevDecisionModel.js +307 -0
- package/dist/extraction/secretGuard.js +7 -3
- package/dist/learning/cli.js +113 -0
- package/dist/learning/schema.js +144 -0
- package/dist/learning/settings.js +45 -0
- package/dist/learning/store.js +267 -0
- package/dist/learning/tap.js +401 -0
- package/dist/memory/dreamer/adjudicate.js +215 -0
- package/dist/memory/dreamer/apply.js +120 -0
- package/dist/memory/dreamer/budget.js +64 -0
- package/dist/memory/dreamer/candidates.js +149 -0
- package/dist/memory/dreamer/collection.js +104 -0
- package/dist/memory/dreamer/dreamer.js +352 -0
- package/dist/memory/dreamer/review.js +111 -0
- package/dist/memory/factConflicts.js +2 -0
- package/dist/memory/lifecycle.js +28 -5
- package/dist/memory/writer.js +3 -2
- package/dist/mvp/agentMemory.js +3 -0
- package/dist/mvp/cli.js +11 -0
- package/dist/mvp/config.js +4 -0
- package/dist/mvp/exportAll.js +19 -0
- package/dist/mvp/server.js +118 -10
- package/dist/retrieval/orchestrator.js +69 -16
- package/dist/retrieval/rankingTrace.js +184 -0
- package/dist/sharing/cli.js +181 -0
- package/dist/sharing/examples.js +136 -0
- package/dist/sharing/fields.js +225 -0
- package/dist/sharing/loop.js +80 -0
- package/dist/sharing/project.js +327 -0
- package/dist/sharing/send.js +466 -0
- package/dist/sharing/state.js +222 -0
- package/dist/sharing/wording.js +48 -0
- package/package.json +1 -1
- package/dist/BUILD.json +0 -6
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.EXAMPLES_FILE = exports.STATE_FILE = exports.CONSENT_FILE = void 0;
|
|
7
|
+
exports.hostedEngine = hostedEngine;
|
|
8
|
+
exports.defaultSharingDir = defaultSharingDir;
|
|
9
|
+
exports.sharingDirFor = sharingDirFor;
|
|
10
|
+
exports.workspaceSharingDir = workspaceSharingDir;
|
|
11
|
+
exports.readConsent = readConsent;
|
|
12
|
+
exports.writeConsent = writeConsent;
|
|
13
|
+
exports.envOverride = envOverride;
|
|
14
|
+
exports.installLimit = installLimit;
|
|
15
|
+
exports.effectiveLevel = effectiveLevel;
|
|
16
|
+
exports.readState = readState;
|
|
17
|
+
exports.contributorId = contributorId;
|
|
18
|
+
exports.freshState = freshState;
|
|
19
|
+
exports.writeState = writeState;
|
|
20
|
+
exports.ensureState = ensureState;
|
|
21
|
+
exports.removeLocalSharingData = removeLocalSharingData;
|
|
22
|
+
const node_crypto_1 = require("node:crypto");
|
|
23
|
+
const node_fs_1 = __importDefault(require("node:fs"));
|
|
24
|
+
const node_os_1 = __importDefault(require("node:os"));
|
|
25
|
+
const node_path_1 = __importDefault(require("node:path"));
|
|
26
|
+
const fields_1 = require("./fields");
|
|
27
|
+
/**
|
|
28
|
+
* Where sharing keeps what it knows, and who may change it.
|
|
29
|
+
*
|
|
30
|
+
* Two small files in one owner-only folder (the sharing folder):
|
|
31
|
+
* sharing.json the consent record: the level, when it was chosen, the wording version the user saw, whether they
|
|
32
|
+
* attested to being an adult, and who set it. One per install (self-hosted) or per workspace (hosted).
|
|
33
|
+
* sharing-state.json what the sender needs between runs: the random secret that makes the contributor id, how far each row
|
|
34
|
+
* kind has been sent, and whether rows were ever sent (only then does `off` have anything to remove).
|
|
35
|
+
*
|
|
36
|
+
* No file exists until a user picks a level above off. A folder where the user never chose has neither file, and reading its
|
|
37
|
+
* level is `off`. Off writes nothing: the record of an off choice is made only when there was a level to turn off.
|
|
38
|
+
*/
|
|
39
|
+
exports.CONSENT_FILE = 'sharing.json';
|
|
40
|
+
exports.STATE_FILE = 'sharing-state.json';
|
|
41
|
+
/** The text of examples a `data` choice keeps until it is sent; it lives next to the two files. */
|
|
42
|
+
exports.EXAMPLES_FILE = 'sharing-examples.sqlite';
|
|
43
|
+
/**
|
|
44
|
+
* Whether this engine runs behind the hosted gateway: the same rule as the service (mvp/config.ts reads MVP_GATEWAY_SECRET as
|
|
45
|
+
* gatewaySecret, and mvp/cli.ts starts the sender with hosted: Boolean(config.gatewaySecret)), so the CLI sends as the service does.
|
|
46
|
+
*/
|
|
47
|
+
function hostedEngine(env = process.env) {
|
|
48
|
+
return (env.MVP_GATEWAY_SECRET ?? '').trim() !== '';
|
|
49
|
+
}
|
|
50
|
+
/** The folder of a self-hosted install: RENOR_SHARING_DIR, else the user's config folder. */
|
|
51
|
+
function defaultSharingDir(env = process.env) {
|
|
52
|
+
const set = (env.RENOR_SHARING_DIR ?? '').trim();
|
|
53
|
+
if (set !== '')
|
|
54
|
+
return node_path_1.default.resolve(set);
|
|
55
|
+
const xdg = (env.XDG_CONFIG_HOME ?? '').trim();
|
|
56
|
+
return node_path_1.default.join(xdg !== '' ? node_path_1.default.resolve(xdg) : node_path_1.default.join(node_os_1.default.homedir(), '.config'), 'renor');
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* The sharing folder for a run: an explicit folder, else RENOR_SHARING_DIR, else the one inside the memory folder when there is
|
|
60
|
+
* one (a service, or a command given its memory folder: the choice is per memory folder, so per workspace), else the user's
|
|
61
|
+
* config folder (one choice per install).
|
|
62
|
+
*/
|
|
63
|
+
function sharingDirFor(memoryDir, env = process.env, explicit) {
|
|
64
|
+
if (explicit !== undefined && explicit.trim() !== '')
|
|
65
|
+
return node_path_1.default.resolve(explicit);
|
|
66
|
+
if ((env.RENOR_SHARING_DIR ?? '').trim() !== '')
|
|
67
|
+
return defaultSharingDir(env);
|
|
68
|
+
return memoryDir !== null ? workspaceSharingDir(memoryDir) : defaultSharingDir(env);
|
|
69
|
+
}
|
|
70
|
+
/** The folder of one hosted workspace: inside its memory folder, so it goes with the memory on deletion. */
|
|
71
|
+
function workspaceSharingDir(memoryDir) {
|
|
72
|
+
return node_path_1.default.join(node_path_1.default.resolve(memoryDir), '.memory-index', 'sharing');
|
|
73
|
+
}
|
|
74
|
+
function readJson(file) {
|
|
75
|
+
try {
|
|
76
|
+
return JSON.parse(node_fs_1.default.readFileSync(file, 'utf8'));
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
function writeJson(dir, name, value) {
|
|
83
|
+
node_fs_1.default.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
84
|
+
const file = node_path_1.default.join(dir, name);
|
|
85
|
+
const temp = node_path_1.default.join(dir, `.${name}.${process.pid}.${(0, node_crypto_1.randomBytes)(4).toString('hex')}.tmp`);
|
|
86
|
+
node_fs_1.default.writeFileSync(temp, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });
|
|
87
|
+
node_fs_1.default.renameSync(temp, file);
|
|
88
|
+
}
|
|
89
|
+
const isRecord = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
90
|
+
/** The consent record, or null when none was made (or the file is unreadable: unreadable is never read as consent). */
|
|
91
|
+
function readConsent(dir) {
|
|
92
|
+
const raw = readJson(node_path_1.default.join(dir, exports.CONSENT_FILE));
|
|
93
|
+
if (!isRecord(raw) || raw.version !== 1 || !(0, fields_1.isSharingLevel)(raw.level))
|
|
94
|
+
return null;
|
|
95
|
+
const sources = ['cli', 'setup', 'dashboard', 'service'];
|
|
96
|
+
if (typeof raw.changedAt !== 'string' || typeof raw.wording !== 'string' || typeof raw.adult !== 'boolean')
|
|
97
|
+
return null;
|
|
98
|
+
if (typeof raw.source !== 'string' || !sources.includes(raw.source))
|
|
99
|
+
return null;
|
|
100
|
+
return {
|
|
101
|
+
version: 1,
|
|
102
|
+
level: raw.level,
|
|
103
|
+
changedAt: raw.changedAt,
|
|
104
|
+
wording: raw.wording,
|
|
105
|
+
adult: raw.adult,
|
|
106
|
+
source: raw.source,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
function writeConsent(dir, record) {
|
|
110
|
+
writeJson(dir, exports.CONSENT_FILE, record);
|
|
111
|
+
}
|
|
112
|
+
/** `RENOR_SHARING`: off, signals or data. It can only lower what the record says, never raise it: consent needs a record. */
|
|
113
|
+
function envOverride(env = process.env) {
|
|
114
|
+
if ((env.DO_NOT_TRACK ?? '').trim() === '1')
|
|
115
|
+
return 'off';
|
|
116
|
+
const raw = (env.RENOR_SHARING ?? '').trim().toLowerCase();
|
|
117
|
+
if (raw === '')
|
|
118
|
+
return null;
|
|
119
|
+
if (!(0, fields_1.isSharingLevel)(raw))
|
|
120
|
+
throw new Error('RENOR_SHARING must be off, signals or data');
|
|
121
|
+
return raw;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* The install's own record (the one `renor setup` and a command without a memory folder write, in defaultSharingDir) as a
|
|
125
|
+
* limit on another sharing folder on this install, a workspace's: null when there is no install record, or when `dir` is that
|
|
126
|
+
* folder itself. It only ever lowers: a workspace with no record of its own stays off whatever the install says.
|
|
127
|
+
*/
|
|
128
|
+
function installLimit(dir, env = process.env) {
|
|
129
|
+
const installDir = defaultSharingDir(env);
|
|
130
|
+
if (node_path_1.default.resolve(dir) === installDir)
|
|
131
|
+
return null;
|
|
132
|
+
const record = readConsentCached(installDir);
|
|
133
|
+
if (record === null)
|
|
134
|
+
return null;
|
|
135
|
+
return record.adult ? record.level : 'off';
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* The level in force right now: the recorded level, lowered by the install's own record (the stricter of the two wins) and by
|
|
139
|
+
* the environment. An unreadable record, an under-age user or a missing attestation is `off`. Read from the files at each call
|
|
140
|
+
* (a stat tells whether they changed), so a choice made in another process takes effect on the next call, with no restart.
|
|
141
|
+
*/
|
|
142
|
+
function effectiveLevel(dir, env = process.env) {
|
|
143
|
+
const record = readConsentCached(dir);
|
|
144
|
+
let level = record === null || !record.adult ? 'off' : record.level;
|
|
145
|
+
const limit = installLimit(dir, env);
|
|
146
|
+
if (limit !== null)
|
|
147
|
+
level = (0, fields_1.lowerLevel)(level, limit);
|
|
148
|
+
const override = envOverride(env);
|
|
149
|
+
if (override !== null)
|
|
150
|
+
level = (0, fields_1.lowerLevel)(level, override);
|
|
151
|
+
return level;
|
|
152
|
+
}
|
|
153
|
+
const cache = new Map();
|
|
154
|
+
function readConsentCached(dir) {
|
|
155
|
+
const file = node_path_1.default.join(dir, exports.CONSENT_FILE);
|
|
156
|
+
let stamp;
|
|
157
|
+
try {
|
|
158
|
+
const stat = node_fs_1.default.statSync(file);
|
|
159
|
+
stamp = `${stat.mtimeMs}:${stat.size}:${stat.ino}`;
|
|
160
|
+
}
|
|
161
|
+
catch {
|
|
162
|
+
cache.delete(file);
|
|
163
|
+
return null;
|
|
164
|
+
}
|
|
165
|
+
const hit = cache.get(file);
|
|
166
|
+
if (hit && hit.stamp === stamp)
|
|
167
|
+
return hit.record;
|
|
168
|
+
const record = readConsent(dir);
|
|
169
|
+
cache.set(file, { stamp, record });
|
|
170
|
+
return record;
|
|
171
|
+
}
|
|
172
|
+
function readState(dir) {
|
|
173
|
+
const raw = readJson(node_path_1.default.join(dir, exports.STATE_FILE));
|
|
174
|
+
if (!isRecord(raw) || raw.version !== 1 || typeof raw.secret !== 'string' || raw.secret.length < 32)
|
|
175
|
+
return null;
|
|
176
|
+
const marks = isRecord(raw.watermarks) ? raw.watermarks : {};
|
|
177
|
+
const num = (value) => (typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : 0);
|
|
178
|
+
return {
|
|
179
|
+
version: 1,
|
|
180
|
+
secret: raw.secret,
|
|
181
|
+
watermarks: { retrieval: num(marks.retrieval), save: num(marks.save), session: num(marks.session) },
|
|
182
|
+
everSent: raw.everSent === true,
|
|
183
|
+
deletePending: raw.deletePending === true,
|
|
184
|
+
...pausedOf(raw.paused),
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
function pausedOf(raw) {
|
|
188
|
+
if (!isRecord(raw) || (raw.code !== 'sharing_closed' && raw.code !== 'consent_required'))
|
|
189
|
+
return {};
|
|
190
|
+
return { paused: { code: raw.code, at: typeof raw.at === 'number' && Number.isFinite(raw.at) ? raw.at : 0 } };
|
|
191
|
+
}
|
|
192
|
+
/** The public name of a contributor: the SHA-256 of its secret, in hex. The endpoint knows only this, never the secret. */
|
|
193
|
+
function contributorId(secret) {
|
|
194
|
+
return (0, node_crypto_1.createHash)('sha256').update(secret).digest('hex');
|
|
195
|
+
}
|
|
196
|
+
function freshState() {
|
|
197
|
+
return {
|
|
198
|
+
version: 1,
|
|
199
|
+
secret: (0, node_crypto_1.randomBytes)(32).toString('base64url'),
|
|
200
|
+
watermarks: { retrieval: 0, save: 0, session: 0 },
|
|
201
|
+
everSent: false,
|
|
202
|
+
deletePending: false,
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
function writeState(dir, state) {
|
|
206
|
+
writeJson(dir, exports.STATE_FILE, state);
|
|
207
|
+
}
|
|
208
|
+
/** The state, made on first use. Only a sender that has a level above off calls this. */
|
|
209
|
+
function ensureState(dir) {
|
|
210
|
+
const existing = readState(dir);
|
|
211
|
+
if (existing !== null)
|
|
212
|
+
return existing;
|
|
213
|
+
const made = freshState();
|
|
214
|
+
writeState(dir, made);
|
|
215
|
+
return made;
|
|
216
|
+
}
|
|
217
|
+
/** Removes the files sharing keeps (the record stays: it is the proof of what was chosen, and holds no row). */
|
|
218
|
+
function removeLocalSharingData(dir) {
|
|
219
|
+
for (const name of [exports.STATE_FILE, exports.EXAMPLES_FILE, `${exports.EXAMPLES_FILE}-wal`, `${exports.EXAMPLES_FILE}-shm`]) {
|
|
220
|
+
node_fs_1.default.rmSync(node_path_1.default.join(dir, name), { force: true });
|
|
221
|
+
}
|
|
222
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.CLI_WORDING_VERSION = exports.ADULT_LINE = void 0;
|
|
4
|
+
exports.wordingFor = wordingFor;
|
|
5
|
+
exports.fieldList = fieldList;
|
|
6
|
+
const fields_1 = require("./fields");
|
|
7
|
+
Object.defineProperty(exports, "CLI_WORDING_VERSION", { enumerable: true, get: function () { return fields_1.CLI_WORDING_VERSION; } });
|
|
8
|
+
/**
|
|
9
|
+
* The words the command line shows before it records a choice. The version id (CLI_WORDING_VERSION) is stored with the
|
|
10
|
+
* choice, so the record shows which words the user saw. Change the words, change the id.
|
|
11
|
+
*
|
|
12
|
+
* These are the command line's own words. They are not the privacy policy: the public wording (policy, terms, website) is the
|
|
13
|
+
* owner's and a lawyer's to approve and is not part of this file. docs/TELEMETRY.md has the same facts for reading offline.
|
|
14
|
+
*/
|
|
15
|
+
function wordingFor(level) {
|
|
16
|
+
const common = [
|
|
17
|
+
'Renor can learn from how its memory is used, if you choose to help. Off is the default and stays off unless you pick a level here.',
|
|
18
|
+
'',
|
|
19
|
+
'Nothing is sent until you choose. You can change this at any time with `renor-engine telemetry off`, and `renor-engine telemetry delete`',
|
|
20
|
+
'removes what Renor holds. Turning it off stops recording at once and asks Renor to delete what it holds, at once (backups can keep a',
|
|
21
|
+
'copy for up to 7 more days). A ranking setting already fitted from rows is not recalled; the next fit leaves yours out.',
|
|
22
|
+
'Sharing never changes your plan, quota or features. Renor keeps signal rows up to 12 months and text examples up to 6 months.',
|
|
23
|
+
'',
|
|
24
|
+
];
|
|
25
|
+
if (level === 'signals') {
|
|
26
|
+
return [
|
|
27
|
+
...common,
|
|
28
|
+
'LEVEL: share signals',
|
|
29
|
+
' Sent: anonymous labelled numbers, worked out inside your own data folder from the learning log: which kind of memory was returned,',
|
|
30
|
+
' in what position, with what score, and whether your AI then opened, cited, updated or retired it, with times rounded to the hour.',
|
|
31
|
+
' Never sent: memory text, queries, titles, names, file paths, memory ids, keys or your IP address stored with the rows.',
|
|
32
|
+
' Every field is listed by `renor-engine telemetry fields signals`, and `renor-engine telemetry preview` prints exactly what would be sent.',
|
|
33
|
+
].join('\n');
|
|
34
|
+
}
|
|
35
|
+
return [
|
|
36
|
+
...common,
|
|
37
|
+
'LEVEL: share data for training',
|
|
38
|
+
' Sent: everything in "share signals", plus the TEXT of the examples: the query behind a retrieval, the text of the memories it returned,',
|
|
39
|
+
' and the statements that were saved. Renor may use that text to train and evaluate its own models.',
|
|
40
|
+
' Memories can contain other people\'s personal data and information about you. A text that looks like a secret (key, password, token)',
|
|
41
|
+
' is left out, but no scanner is perfect: choose this only if you accept that.',
|
|
42
|
+
' Every field is listed by `renor-engine telemetry fields data`, and `renor-engine telemetry preview` prints exactly what would be sent.',
|
|
43
|
+
].join('\n');
|
|
44
|
+
}
|
|
45
|
+
exports.ADULT_LINE = 'I am 18 or older and I choose this myself.';
|
|
46
|
+
function fieldList(level) {
|
|
47
|
+
return (0, fields_1.fieldsAtLevel)(level).join('\n');
|
|
48
|
+
}
|
package/package.json
CHANGED