sphica 0.5.6 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/README.md +4 -3
- package/db/migrations/0002.sql +145 -0
- package/db/schema.sql +54 -9
- package/dist/capture.js +61 -19
- package/dist/cli.js +95 -22
- package/dist/deliver.js +300 -258
- package/dist/mcp-record.js +450 -53
- package/dist/mcp.js +7 -8
- package/package.json +1 -1
- package/skills/forget/SKILL.md +53 -0
- package/skills/forget/agents/openai.yaml +2 -0
- package/skills/glean/SKILL.md +2 -2
- package/skills/harvest/SKILL.md +1 -1
- package/skills/trace/SKILL.md +1 -1
package/dist/deliver.js
CHANGED
|
@@ -21,6 +21,255 @@ import path7 from "node:path";
|
|
|
21
21
|
|
|
22
22
|
// server/src/anchors.ts
|
|
23
23
|
import path from "node:path";
|
|
24
|
+
|
|
25
|
+
// server/src/text.ts
|
|
26
|
+
import crypto from "node:crypto";
|
|
27
|
+
var segmenter = new Intl.Segmenter("ja", { granularity: "word" });
|
|
28
|
+
var HIRAGANA_ONLY = /^[\p{Script=Hiragana}ー]+$/u;
|
|
29
|
+
var STOP = new Set(["the", "a", "an", "of", "to", "in", "is", "and", "or", "for", "on", "it", "be"]);
|
|
30
|
+
var IDENT = /#\d+|[a-z0-9][a-z0-9_./#-]*[a-z0-9]/g;
|
|
31
|
+
var MAX_TERM = 100;
|
|
32
|
+
var OKURIGANA = /^(\p{Script=Han}+)[\p{Script=Hiragana}ー]+$/u;
|
|
33
|
+
function singular(w) {
|
|
34
|
+
if (!/^[a-z]{4,}$/.test(w))
|
|
35
|
+
return w;
|
|
36
|
+
if (w.endsWith("ies"))
|
|
37
|
+
return `${w.slice(0, -3)}y`;
|
|
38
|
+
if (w.endsWith("ie"))
|
|
39
|
+
return `${w.slice(0, -2)}y`;
|
|
40
|
+
if (/(?:ss|us|x|z|ch|sh)es$/.test(w))
|
|
41
|
+
return w.slice(0, -2);
|
|
42
|
+
if (/(?:ch|sh|us)e$/.test(w))
|
|
43
|
+
return w.slice(0, -1);
|
|
44
|
+
return w.endsWith("s") && !/(?:ss|us|is)$/.test(w) ? w.slice(0, -1) : w;
|
|
45
|
+
}
|
|
46
|
+
function terms(text) {
|
|
47
|
+
const norm = text.normalize("NFKC").toLowerCase();
|
|
48
|
+
const out = [];
|
|
49
|
+
const keep = (w) => {
|
|
50
|
+
if (w.length > MAX_TERM || STOP.has(w) || HIRAGANA_ONLY.test(w))
|
|
51
|
+
return;
|
|
52
|
+
out.push(OKURIGANA.exec(w)?.[1] ?? singular(w));
|
|
53
|
+
};
|
|
54
|
+
for (const s of segmenter.segment(norm))
|
|
55
|
+
if (s.isWordLike)
|
|
56
|
+
keep(s.segment.trim());
|
|
57
|
+
for (const m of norm.matchAll(IDENT))
|
|
58
|
+
if (m[0].length >= 3)
|
|
59
|
+
keep(m[0]);
|
|
60
|
+
return out.filter(Boolean);
|
|
61
|
+
}
|
|
62
|
+
var QUESTION = new Set([
|
|
63
|
+
...[
|
|
64
|
+
"why",
|
|
65
|
+
"what",
|
|
66
|
+
"which",
|
|
67
|
+
"when",
|
|
68
|
+
"where",
|
|
69
|
+
"who",
|
|
70
|
+
"how",
|
|
71
|
+
"do",
|
|
72
|
+
"does",
|
|
73
|
+
"did",
|
|
74
|
+
"we",
|
|
75
|
+
"our",
|
|
76
|
+
"us",
|
|
77
|
+
"you",
|
|
78
|
+
"i",
|
|
79
|
+
"are",
|
|
80
|
+
"was",
|
|
81
|
+
"were"
|
|
82
|
+
],
|
|
83
|
+
...[
|
|
84
|
+
"will",
|
|
85
|
+
"would",
|
|
86
|
+
"should",
|
|
87
|
+
"can",
|
|
88
|
+
"could",
|
|
89
|
+
"there",
|
|
90
|
+
"this",
|
|
91
|
+
"that",
|
|
92
|
+
"with",
|
|
93
|
+
"from",
|
|
94
|
+
"any",
|
|
95
|
+
"ever",
|
|
96
|
+
"long",
|
|
97
|
+
"use",
|
|
98
|
+
"used",
|
|
99
|
+
"using"
|
|
100
|
+
],
|
|
101
|
+
...["add", "get", "make", "have", "has", "not", "no", "yes", "reason"],
|
|
102
|
+
...["理由", "仕組", "何", "方", "場合", "今", "件"]
|
|
103
|
+
]);
|
|
104
|
+
var sha256 = (s) => crypto.createHash("sha256").update(s).digest();
|
|
105
|
+
function uuidFrom(...parts) {
|
|
106
|
+
const b = crypto.createHash("sha256").update(parts.join("\x00")).digest().subarray(0, 16);
|
|
107
|
+
b[6] = (b[6] ?? 0) & 15 | 128;
|
|
108
|
+
b[8] = (b[8] ?? 0) & 63 | 128;
|
|
109
|
+
const h = b.toString("hex");
|
|
110
|
+
return `${h.slice(0, 8)}-${h.slice(8, 12)}-${h.slice(12, 16)}-${h.slice(16, 20)}-${h.slice(20)}`;
|
|
111
|
+
}
|
|
112
|
+
var bytes = (s) => Buffer.byteLength(s, "utf8");
|
|
113
|
+
function head(s, n) {
|
|
114
|
+
if (bytes(s) <= n)
|
|
115
|
+
return s;
|
|
116
|
+
let out = "";
|
|
117
|
+
let used = 0;
|
|
118
|
+
for (const ch of s) {
|
|
119
|
+
const b = bytes(ch);
|
|
120
|
+
if (used + b > n)
|
|
121
|
+
break;
|
|
122
|
+
out += ch;
|
|
123
|
+
used += b;
|
|
124
|
+
}
|
|
125
|
+
return out;
|
|
126
|
+
}
|
|
127
|
+
function tail(s, n) {
|
|
128
|
+
if (bytes(s) <= n)
|
|
129
|
+
return s;
|
|
130
|
+
const chars = [...s];
|
|
131
|
+
let used = 0;
|
|
132
|
+
let i = chars.length;
|
|
133
|
+
while (i > 0) {
|
|
134
|
+
const b = bytes(chars[i - 1] ?? "");
|
|
135
|
+
if (used + b > n)
|
|
136
|
+
break;
|
|
137
|
+
used += b;
|
|
138
|
+
i--;
|
|
139
|
+
}
|
|
140
|
+
return chars.slice(i).join("");
|
|
141
|
+
}
|
|
142
|
+
var clean = (s) => s.replaceAll("\x00", "");
|
|
143
|
+
var visible = (s) => s.replace(/(?!\p{Join_Control}|\p{Variation_Selector})\p{Default_Ignorable_Code_Point}/gu, "");
|
|
144
|
+
var SECRETS = [
|
|
145
|
+
[/\bsk-(?:proj-|ant-)?[A-Za-z0-9_-]{20,}/g, "API key"],
|
|
146
|
+
[/\b[srp]k_(?:live|test)_[A-Za-z0-9]{16,}/g, "API key"],
|
|
147
|
+
[/\bwhsec_[A-Za-z0-9+/=]{16,}/g, "webhook signing secret"],
|
|
148
|
+
[/\bpa-[A-Za-z0-9_-]{20,}/g, "API key"],
|
|
149
|
+
[/\bAIza[0-9A-Za-z_-]{35}/g, "API key"],
|
|
150
|
+
[/\bnpg_[A-Za-z0-9]{12,}/g, "database password"],
|
|
151
|
+
[/\bnapi_[A-Za-z0-9]{30,}/g, "API key"],
|
|
152
|
+
[/\bnpm_[A-Za-z0-9]{36}\b/g, "npm token"],
|
|
153
|
+
[/\bglpat-[A-Za-z0-9_-]{20,}/g, "GitLab token"],
|
|
154
|
+
[/\bgh[pousr]_[A-Za-z0-9]{30,}/g, "GitHub token"],
|
|
155
|
+
[/\bgithub_pat_[A-Za-z0-9_]{40,}/g, "GitHub token"],
|
|
156
|
+
[/\bxox[abprs]-[A-Za-z0-9-]{10,}/g, "Slack token"],
|
|
157
|
+
[/https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/]+/g, "Slack webhook"],
|
|
158
|
+
[/\bAKIA[0-9A-Z]{16}\b/g, "AWS key"],
|
|
159
|
+
[/(?<![A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g, "JWT"],
|
|
160
|
+
[/\b(?:Bearer|BEARER)\s+(?=[A-Za-z0-9._~+/=-]{0,512}\d)[A-Za-z0-9._~+/=-]{16,}/g, "auth header value"]
|
|
161
|
+
];
|
|
162
|
+
var AUTH_HEADER = /(\bAuthorization["']?\s*[:=]\s*(?:["']\s*)?(?:Bearer|Basic|Token|Digest)\s+)[A-Za-z0-9._~+/=-]{8,}/gi;
|
|
163
|
+
var HEADER_BEARER = /(:[ \t]*(?:["'][ \t]*)?bearer[ \t]+)[A-Za-z0-9._~+/=-]{16,}/gi;
|
|
164
|
+
var ENV_ASSIGN = /\b((?:[A-Z][A-Z0-9_]*_)?(?:API|SECRET|MASTER|ENCRYPTION|PRIVATE|ACCESS|SIGNING|AUTH)?KEY|[A-Z][A-Z0-9_]*_(?:PASS|PWD)|(?:[A-Z][A-Z0-9_]*?)?(?:TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIALS?))(\s*=\s*)(?:"(?!\$)[^"\n]+"|'(?!\$)[^'\n]+'|(?![$"'])[^\s"']+)/g;
|
|
165
|
+
var FIELD_NAME = /(?:(?:api|account|access|private|secret)[-_]?key|secret|token|passw(?:or)?d)["']?\s*(?::=|=>|[:=])\s*/gi;
|
|
166
|
+
var MAX_QUOTED = 4096;
|
|
167
|
+
var BARE_HEAD = /[^\s"',;)]{1,256}/y;
|
|
168
|
+
var BARE_REST = /[^\s"',;)]*/y;
|
|
169
|
+
var NEXT_PARAM = /&[A-Za-z_][\w.-]*=/;
|
|
170
|
+
var bareAt = (re, text, at) => {
|
|
171
|
+
re.lastIndex = at;
|
|
172
|
+
const v = re.exec(text)?.[0] ?? "";
|
|
173
|
+
const cut = v.search(NEXT_PARAM);
|
|
174
|
+
return cut < 0 ? v : v.slice(0, cut);
|
|
175
|
+
};
|
|
176
|
+
function secretValue(quoted, v) {
|
|
177
|
+
if (v.length < 8 || /^\$(?:\{|[A-Za-z_])/.test(v))
|
|
178
|
+
return false;
|
|
179
|
+
if (!quoted)
|
|
180
|
+
return !/^[#$]/.test(v) && /\d/.test(v) && /[A-Za-z]/.test(v) && !/[()]/.test(v);
|
|
181
|
+
if (/^#[0-9a-f]{3,8}$/i.test(v) || !/[A-Za-z0-9]/.test(v))
|
|
182
|
+
return false;
|
|
183
|
+
return !/\s/.test(v) || v.split(/\s+/).some((w) => /\d/.test(w) && /[A-Za-z]/.test(w));
|
|
184
|
+
}
|
|
185
|
+
function maskFields(text) {
|
|
186
|
+
let out = "";
|
|
187
|
+
let last = 0;
|
|
188
|
+
FIELD_NAME.lastIndex = 0;
|
|
189
|
+
for (let m = FIELD_NAME.exec(text);m; m = FIELD_NAME.exec(text)) {
|
|
190
|
+
const at = m.index + m[0].length;
|
|
191
|
+
const q = text[at];
|
|
192
|
+
let quote = "";
|
|
193
|
+
let value;
|
|
194
|
+
if (q === '"' || q === "'") {
|
|
195
|
+
const close = text.indexOf(q, at + 1);
|
|
196
|
+
if (close < 0 || close - at - 1 > MAX_QUOTED)
|
|
197
|
+
continue;
|
|
198
|
+
value = text.slice(at + 1, close);
|
|
199
|
+
if (value.includes(`
|
|
200
|
+
`))
|
|
201
|
+
continue;
|
|
202
|
+
quote = q;
|
|
203
|
+
} else {
|
|
204
|
+
value = bareAt(BARE_HEAD, text, at);
|
|
205
|
+
}
|
|
206
|
+
if (!secretValue(quote !== "", value))
|
|
207
|
+
continue;
|
|
208
|
+
if (!quote && value.length === 256)
|
|
209
|
+
value += bareAt(BARE_REST, text, at + 256);
|
|
210
|
+
out += `${text.slice(last, at)}${quote}[redacted]`;
|
|
211
|
+
last = at + quote.length + value.length;
|
|
212
|
+
FIELD_NAME.lastIndex = last;
|
|
213
|
+
}
|
|
214
|
+
return out + text.slice(last);
|
|
215
|
+
}
|
|
216
|
+
var MYSQL_COMMAND = /\bmysql(?:dump|admin)?\b(?:'[^'\n]*'|"[^"\n]*"|[^\n;&|\\'"]|\\\r?\n|\\(?!\r?\n))*/g;
|
|
217
|
+
var MYSQL_PASSWORD = /(\s-p)(?:'[^'\n]*'|"[^"\n]*"|(?=[^\s-])\S+)/;
|
|
218
|
+
var URL_CREDENTIALS = /\b((?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|rediss?|amqps?|https?):\/\/[^:\s/@]*:)[^\s/]*@([^@\s/?#]+)/g;
|
|
219
|
+
var KEY_BEGIN = /-----BEGIN [A-Z ]*PRIVATE KEY-----/g;
|
|
220
|
+
var KEY_END = /-----END [A-Z ]*PRIVATE KEY-----/g;
|
|
221
|
+
function keyBlocks(text) {
|
|
222
|
+
const ends = [...text.matchAll(KEY_END)].map((m) => [m.index, m.index + m[0].length]);
|
|
223
|
+
const out = [];
|
|
224
|
+
let last = 0;
|
|
225
|
+
let e = 0;
|
|
226
|
+
if (ends.length > 0)
|
|
227
|
+
for (const m of text.matchAll(KEY_BEGIN)) {
|
|
228
|
+
const after = m.index + m[0].length;
|
|
229
|
+
if (m.index < last)
|
|
230
|
+
continue;
|
|
231
|
+
while (e < ends.length && (ends[e]?.[0] ?? 0) < after)
|
|
232
|
+
e++;
|
|
233
|
+
const end = ends[e];
|
|
234
|
+
if (!end)
|
|
235
|
+
break;
|
|
236
|
+
out.push([m.index, end[1]]);
|
|
237
|
+
last = end[1];
|
|
238
|
+
}
|
|
239
|
+
return out;
|
|
240
|
+
}
|
|
241
|
+
function maskPrivateKeys(text) {
|
|
242
|
+
let out = "";
|
|
243
|
+
let last = 0;
|
|
244
|
+
for (const [a, b] of keyBlocks(text)) {
|
|
245
|
+
out += `${text.slice(last, a)}[redacted: private key]`;
|
|
246
|
+
last = b;
|
|
247
|
+
}
|
|
248
|
+
return out + text.slice(last);
|
|
249
|
+
}
|
|
250
|
+
function mask(text) {
|
|
251
|
+
let out = maskFields(maskPrivateKeys(text).replace(URL_CREDENTIALS, "$1[redacted]@$2").replace(AUTH_HEADER, "$1[redacted]").replace(HEADER_BEARER, "$1[redacted]").replace(ENV_ASSIGN, "$1$2[redacted]")).replace(MYSQL_COMMAND, (command) => command.replace(MYSQL_PASSWORD, "$1[redacted]"));
|
|
252
|
+
for (const [re, what] of SECRETS)
|
|
253
|
+
out = out.replace(re, `[redacted: ${what}]`);
|
|
254
|
+
return out;
|
|
255
|
+
}
|
|
256
|
+
var reason = (e) => explain(e, 0) || "unknown failure";
|
|
257
|
+
var plural = (n, one, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
|
|
258
|
+
function explain(e, depth) {
|
|
259
|
+
if (!(e instanceof Error)) {
|
|
260
|
+
try {
|
|
261
|
+
return String(e);
|
|
262
|
+
} catch {
|
|
263
|
+
return "";
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
const own = e.message || (e.name === "Error" || e.name === "AggregateError" ? "" : e.name);
|
|
267
|
+
const parts = depth >= 3 ? [] : [...e instanceof AggregateError ? e.errors : [], ...e.cause === undefined ? [] : [e.cause]];
|
|
268
|
+
const inner = parts.map((x) => explain(x, depth + 1)).filter(Boolean).join(" / ");
|
|
269
|
+
return own && inner ? `${own} (${inner})` : own || inner;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
// server/src/anchors.ts
|
|
24
273
|
var MAX_BYTES = 2 * 1024 * 1024;
|
|
25
274
|
var leaves = (rel) => rel === ".." || rel.startsWith(`..${path.sep}`) || path.isAbsolute(rel);
|
|
26
275
|
|
|
@@ -4154,8 +4403,8 @@ class Deferred {
|
|
|
4154
4403
|
this.#resolve?.(value);
|
|
4155
4404
|
this.#resolve = this.#reject = undefined;
|
|
4156
4405
|
};
|
|
4157
|
-
reject = (
|
|
4158
|
-
this.#reject?.(
|
|
4406
|
+
reject = (reason2) => {
|
|
4407
|
+
this.#reject?.(reason2);
|
|
4159
4408
|
this.#reject = this.#resolve = undefined;
|
|
4160
4409
|
};
|
|
4161
4410
|
}
|
|
@@ -4207,9 +4456,9 @@ function assertNotAborted(signal, timing, beforeThrow) {
|
|
|
4207
4456
|
throwReasonWithTiming(signal.reason, timing);
|
|
4208
4457
|
}
|
|
4209
4458
|
}
|
|
4210
|
-
function throwReasonWithTiming(
|
|
4211
|
-
decorateWithTiming(
|
|
4212
|
-
throw
|
|
4459
|
+
function throwReasonWithTiming(reason2, timing) {
|
|
4460
|
+
decorateWithTiming(reason2, timing);
|
|
4461
|
+
throw reason2;
|
|
4213
4462
|
}
|
|
4214
4463
|
var ABORTED = {};
|
|
4215
4464
|
async function waitOrAbort(promise, signal, name, onAbort) {
|
|
@@ -4234,11 +4483,11 @@ async function waitOrAbort(promise, signal, name, onAbort) {
|
|
|
4234
4483
|
}
|
|
4235
4484
|
}
|
|
4236
4485
|
function printBackgroundFail(name) {
|
|
4237
|
-
return (
|
|
4486
|
+
return (reason2) => console.error(`\`${name}\` failed in the background after abortion: ${getMessage(reason2)}`);
|
|
4238
4487
|
}
|
|
4239
|
-
function decorateWithTiming(
|
|
4240
|
-
if (
|
|
4241
|
-
Object.defineProperty(
|
|
4488
|
+
function decorateWithTiming(reason2, timing) {
|
|
4489
|
+
if (reason2 !== null && typeof reason2 === "object" && !Object.isFrozen(reason2)) {
|
|
4490
|
+
Object.defineProperty(reason2, "__kysely_timing__", {
|
|
4242
4491
|
configurable: true,
|
|
4243
4492
|
enumerable: false,
|
|
4244
4493
|
value: timing,
|
|
@@ -8163,9 +8412,9 @@ class RuntimeDriver {
|
|
|
8163
8412
|
}
|
|
8164
8413
|
this.#initPromise ??= this.#driver.init(options).then(() => {
|
|
8165
8414
|
this.#initDone = true;
|
|
8166
|
-
}).catch((
|
|
8415
|
+
}).catch((reason2) => {
|
|
8167
8416
|
this.#initPromise = undefined;
|
|
8168
|
-
throw
|
|
8417
|
+
throw reason2;
|
|
8169
8418
|
});
|
|
8170
8419
|
await waitOrAbort(this.#initPromise, options?.signal, "init");
|
|
8171
8420
|
}
|
|
@@ -8226,9 +8475,9 @@ class RuntimeDriver {
|
|
|
8226
8475
|
return;
|
|
8227
8476
|
}
|
|
8228
8477
|
await waitOrAbort(this.#initPromise, options?.signal, "destroy:initPromise");
|
|
8229
|
-
this.#destroyPromise ??= this.#driver.destroy(options).catch((
|
|
8478
|
+
this.#destroyPromise ??= this.#driver.destroy(options).catch((reason2) => {
|
|
8230
8479
|
this.#destroyPromise = undefined;
|
|
8231
|
-
throw
|
|
8480
|
+
throw reason2;
|
|
8232
8481
|
});
|
|
8233
8482
|
await waitOrAbort(this.#destroyPromise, options?.signal, "destroy");
|
|
8234
8483
|
}
|
|
@@ -10837,7 +11086,7 @@ import os from "node:os";
|
|
|
10837
11086
|
import path2 from "node:path";
|
|
10838
11087
|
import { constants as C, DatabaseSync } from "node:sqlite";
|
|
10839
11088
|
var SCHEMA_GENERATION = 2;
|
|
10840
|
-
var SCHEMA_REVISION =
|
|
11089
|
+
var SCHEMA_REVISION = 2;
|
|
10841
11090
|
var sphicaHome = () => process.env.SPHICA_HOME || path2.join(os.homedir(), ".sphica");
|
|
10842
11091
|
var dbFile = () => process.env.SPHICA_DB || path2.join(sphicaHome(), "sphica.db");
|
|
10843
11092
|
function requireRuntime() {
|
|
@@ -10861,7 +11110,7 @@ function prepare(raw, check) {
|
|
|
10861
11110
|
const got = raw.prepare("pragma user_version").get()?.user_version;
|
|
10862
11111
|
if (got === SCHEMA_REVISION)
|
|
10863
11112
|
return;
|
|
10864
|
-
throw new Error(`The database schema is revision ${got}, but this Sphica expects revision ${SCHEMA_REVISION}. ` + ((got ?? 0) < SCHEMA_REVISION ? "
|
|
11113
|
+
throw new Error(`The database schema is revision ${got}, but this Sphica expects revision ${SCHEMA_REVISION}. ` + ((got ?? 0) < SCHEMA_REVISION ? "Update the sphica CLI (`npm i -g sphica`), then run `sphica init` to migrate it (records are kept)." : "Update sphica."));
|
|
10865
11114
|
}
|
|
10866
11115
|
function generationOf(raw) {
|
|
10867
11116
|
const has = raw.prepare("select 1 from sqlite_schema where type = 'table' and name = 'sphica_generation'").get();
|
|
@@ -10959,247 +11208,6 @@ function sqliteCode(e) {
|
|
|
10959
11208
|
|
|
10960
11209
|
// server/src/db-write.ts
|
|
10961
11210
|
import { constants as C2, DatabaseSync as DatabaseSync2 } from "node:sqlite";
|
|
10962
|
-
|
|
10963
|
-
// server/src/text.ts
|
|
10964
|
-
import crypto from "node:crypto";
|
|
10965
|
-
var segmenter = new Intl.Segmenter("ja", { granularity: "word" });
|
|
10966
|
-
var HIRAGANA_ONLY = /^[\p{Script=Hiragana}ー]+$/u;
|
|
10967
|
-
var STOP = new Set(["the", "a", "an", "of", "to", "in", "is", "and", "or", "for", "on", "it", "be"]);
|
|
10968
|
-
var IDENT = /#\d+|[a-z0-9][a-z0-9_./#-]*[a-z0-9]/g;
|
|
10969
|
-
var MAX_TERM = 100;
|
|
10970
|
-
var OKURIGANA = /^(\p{Script=Han}+)[\p{Script=Hiragana}ー]+$/u;
|
|
10971
|
-
function singular(w) {
|
|
10972
|
-
if (!/^[a-z]{4,}$/.test(w))
|
|
10973
|
-
return w;
|
|
10974
|
-
if (w.endsWith("ies"))
|
|
10975
|
-
return `${w.slice(0, -3)}y`;
|
|
10976
|
-
if (w.endsWith("ie"))
|
|
10977
|
-
return `${w.slice(0, -2)}y`;
|
|
10978
|
-
if (/(?:ss|us|x|z|ch|sh)es$/.test(w))
|
|
10979
|
-
return w.slice(0, -2);
|
|
10980
|
-
if (/(?:ch|sh|us)e$/.test(w))
|
|
10981
|
-
return w.slice(0, -1);
|
|
10982
|
-
return w.endsWith("s") && !/(?:ss|us|is)$/.test(w) ? w.slice(0, -1) : w;
|
|
10983
|
-
}
|
|
10984
|
-
function terms(text) {
|
|
10985
|
-
const norm = text.normalize("NFKC").toLowerCase();
|
|
10986
|
-
const out = [];
|
|
10987
|
-
const keep = (w) => {
|
|
10988
|
-
if (w.length > MAX_TERM || STOP.has(w) || HIRAGANA_ONLY.test(w))
|
|
10989
|
-
return;
|
|
10990
|
-
out.push(OKURIGANA.exec(w)?.[1] ?? singular(w));
|
|
10991
|
-
};
|
|
10992
|
-
for (const s of segmenter.segment(norm))
|
|
10993
|
-
if (s.isWordLike)
|
|
10994
|
-
keep(s.segment.trim());
|
|
10995
|
-
for (const m of norm.matchAll(IDENT))
|
|
10996
|
-
if (m[0].length >= 3)
|
|
10997
|
-
keep(m[0]);
|
|
10998
|
-
return out.filter(Boolean);
|
|
10999
|
-
}
|
|
11000
|
-
var QUESTION = new Set([
|
|
11001
|
-
...[
|
|
11002
|
-
"why",
|
|
11003
|
-
"what",
|
|
11004
|
-
"which",
|
|
11005
|
-
"when",
|
|
11006
|
-
"where",
|
|
11007
|
-
"who",
|
|
11008
|
-
"how",
|
|
11009
|
-
"do",
|
|
11010
|
-
"does",
|
|
11011
|
-
"did",
|
|
11012
|
-
"we",
|
|
11013
|
-
"our",
|
|
11014
|
-
"us",
|
|
11015
|
-
"you",
|
|
11016
|
-
"i",
|
|
11017
|
-
"are",
|
|
11018
|
-
"was",
|
|
11019
|
-
"were"
|
|
11020
|
-
],
|
|
11021
|
-
...[
|
|
11022
|
-
"will",
|
|
11023
|
-
"would",
|
|
11024
|
-
"should",
|
|
11025
|
-
"can",
|
|
11026
|
-
"could",
|
|
11027
|
-
"there",
|
|
11028
|
-
"this",
|
|
11029
|
-
"that",
|
|
11030
|
-
"with",
|
|
11031
|
-
"from",
|
|
11032
|
-
"any",
|
|
11033
|
-
"ever",
|
|
11034
|
-
"long",
|
|
11035
|
-
"use",
|
|
11036
|
-
"used",
|
|
11037
|
-
"using"
|
|
11038
|
-
],
|
|
11039
|
-
...["add", "get", "make", "have", "has", "not", "no", "yes", "reason"],
|
|
11040
|
-
...["理由", "仕組", "何", "方", "場合", "今", "件"]
|
|
11041
|
-
]);
|
|
11042
|
-
var sha256 = (s) => crypto.createHash("sha256").update(s).digest();
|
|
11043
|
-
function uuidFrom(...parts) {
|
|
11044
|
-
const b = crypto.createHash("sha256").update(parts.join("\x00")).digest().subarray(0, 16);
|
|
11045
|
-
b[6] = (b[6] ?? 0) & 15 | 128;
|
|
11046
|
-
b[8] = (b[8] ?? 0) & 63 | 128;
|
|
11047
|
-
const h = b.toString("hex");
|
|
11048
|
-
return `${h.slice(0, 8)}-${h.slice(8, 12)}-${h.slice(12, 16)}-${h.slice(16, 20)}-${h.slice(20)}`;
|
|
11049
|
-
}
|
|
11050
|
-
var bytes = (s) => Buffer.byteLength(s, "utf8");
|
|
11051
|
-
function head(s, n) {
|
|
11052
|
-
if (bytes(s) <= n)
|
|
11053
|
-
return s;
|
|
11054
|
-
let out = "";
|
|
11055
|
-
let used = 0;
|
|
11056
|
-
for (const ch of s) {
|
|
11057
|
-
const b = bytes(ch);
|
|
11058
|
-
if (used + b > n)
|
|
11059
|
-
break;
|
|
11060
|
-
out += ch;
|
|
11061
|
-
used += b;
|
|
11062
|
-
}
|
|
11063
|
-
return out;
|
|
11064
|
-
}
|
|
11065
|
-
function tail(s, n) {
|
|
11066
|
-
if (bytes(s) <= n)
|
|
11067
|
-
return s;
|
|
11068
|
-
const chars = [...s];
|
|
11069
|
-
let used = 0;
|
|
11070
|
-
let i = chars.length;
|
|
11071
|
-
while (i > 0) {
|
|
11072
|
-
const b = bytes(chars[i - 1] ?? "");
|
|
11073
|
-
if (used + b > n)
|
|
11074
|
-
break;
|
|
11075
|
-
used += b;
|
|
11076
|
-
i--;
|
|
11077
|
-
}
|
|
11078
|
-
return chars.slice(i).join("");
|
|
11079
|
-
}
|
|
11080
|
-
var clean = (s) => s.replaceAll("\x00", "");
|
|
11081
|
-
var visible = (s) => s.replace(/(?!\p{Join_Control}|\p{Variation_Selector})\p{Default_Ignorable_Code_Point}/gu, "");
|
|
11082
|
-
var SECRETS = [
|
|
11083
|
-
[/\bsk-(?:proj-|ant-)?[A-Za-z0-9_-]{20,}/g, "API key"],
|
|
11084
|
-
[/\b[srp]k_(?:live|test)_[A-Za-z0-9]{16,}/g, "API key"],
|
|
11085
|
-
[/\bwhsec_[A-Za-z0-9+/=]{16,}/g, "webhook signing secret"],
|
|
11086
|
-
[/\bpa-[A-Za-z0-9_-]{20,}/g, "API key"],
|
|
11087
|
-
[/\bAIza[0-9A-Za-z_-]{35}/g, "API key"],
|
|
11088
|
-
[/\bnpg_[A-Za-z0-9]{12,}/g, "database password"],
|
|
11089
|
-
[/\bnapi_[A-Za-z0-9]{30,}/g, "API key"],
|
|
11090
|
-
[/\bnpm_[A-Za-z0-9]{36}\b/g, "npm token"],
|
|
11091
|
-
[/\bglpat-[A-Za-z0-9_-]{20,}/g, "GitLab token"],
|
|
11092
|
-
[/\bgh[pousr]_[A-Za-z0-9]{30,}/g, "GitHub token"],
|
|
11093
|
-
[/\bgithub_pat_[A-Za-z0-9_]{40,}/g, "GitHub token"],
|
|
11094
|
-
[/\bxox[abprs]-[A-Za-z0-9-]{10,}/g, "Slack token"],
|
|
11095
|
-
[/https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9/]+/g, "Slack webhook"],
|
|
11096
|
-
[/\bAKIA[0-9A-Z]{16}\b/g, "AWS key"],
|
|
11097
|
-
[/(?<![A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g, "JWT"],
|
|
11098
|
-
[/\b(?:Bearer|BEARER)\s+(?=[A-Za-z0-9._~+/=-]{0,512}\d)[A-Za-z0-9._~+/=-]{16,}/g, "auth header value"]
|
|
11099
|
-
];
|
|
11100
|
-
var AUTH_HEADER = /(\bAuthorization["']?\s*[:=]\s*(?:["']\s*)?(?:Bearer|Basic|Token|Digest)\s+)[A-Za-z0-9._~+/=-]{8,}/gi;
|
|
11101
|
-
var HEADER_BEARER = /(:[ \t]*(?:["'][ \t]*)?bearer[ \t]+)[A-Za-z0-9._~+/=-]{16,}/gi;
|
|
11102
|
-
var ENV_ASSIGN = /\b((?:[A-Z][A-Z0-9_]*_)?(?:API|SECRET|MASTER|ENCRYPTION|PRIVATE|ACCESS|SIGNING|AUTH)?KEY|[A-Z][A-Z0-9_]*_(?:PASS|PWD)|(?:[A-Z][A-Z0-9_]*?)?(?:TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIALS?))(\s*=\s*)(?:"(?!\$)[^"\n]+"|'(?!\$)[^'\n]+'|(?![$"'])[^\s"']+)/g;
|
|
11103
|
-
var FIELD_NAME = /(?:(?:api|account|access|private|secret)[-_]?key|secret|token|passw(?:or)?d)["']?\s*(?::=|=>|[:=])\s*/gi;
|
|
11104
|
-
var MAX_QUOTED = 4096;
|
|
11105
|
-
var BARE_HEAD = /[^\s"',;)]{1,256}/y;
|
|
11106
|
-
var BARE_REST = /[^\s"',;)]*/y;
|
|
11107
|
-
var NEXT_PARAM = /&[A-Za-z_][\w.-]*=/;
|
|
11108
|
-
var bareAt = (re, text, at) => {
|
|
11109
|
-
re.lastIndex = at;
|
|
11110
|
-
const v = re.exec(text)?.[0] ?? "";
|
|
11111
|
-
const cut = v.search(NEXT_PARAM);
|
|
11112
|
-
return cut < 0 ? v : v.slice(0, cut);
|
|
11113
|
-
};
|
|
11114
|
-
function secretValue(quoted, v) {
|
|
11115
|
-
if (v.length < 8 || /^\$(?:\{|[A-Za-z_])/.test(v))
|
|
11116
|
-
return false;
|
|
11117
|
-
if (!quoted)
|
|
11118
|
-
return !/^[#$]/.test(v) && /\d/.test(v) && /[A-Za-z]/.test(v) && !/[()]/.test(v);
|
|
11119
|
-
if (/^#[0-9a-f]{3,8}$/i.test(v) || !/[A-Za-z0-9]/.test(v))
|
|
11120
|
-
return false;
|
|
11121
|
-
return !/\s/.test(v) || v.split(/\s+/).some((w) => /\d/.test(w) && /[A-Za-z]/.test(w));
|
|
11122
|
-
}
|
|
11123
|
-
function maskFields(text) {
|
|
11124
|
-
let out = "";
|
|
11125
|
-
let last = 0;
|
|
11126
|
-
FIELD_NAME.lastIndex = 0;
|
|
11127
|
-
for (let m = FIELD_NAME.exec(text);m; m = FIELD_NAME.exec(text)) {
|
|
11128
|
-
const at = m.index + m[0].length;
|
|
11129
|
-
const q = text[at];
|
|
11130
|
-
let quote = "";
|
|
11131
|
-
let value;
|
|
11132
|
-
if (q === '"' || q === "'") {
|
|
11133
|
-
const close = text.indexOf(q, at + 1);
|
|
11134
|
-
if (close < 0 || close - at - 1 > MAX_QUOTED)
|
|
11135
|
-
continue;
|
|
11136
|
-
value = text.slice(at + 1, close);
|
|
11137
|
-
if (value.includes(`
|
|
11138
|
-
`))
|
|
11139
|
-
continue;
|
|
11140
|
-
quote = q;
|
|
11141
|
-
} else {
|
|
11142
|
-
value = bareAt(BARE_HEAD, text, at);
|
|
11143
|
-
}
|
|
11144
|
-
if (!secretValue(quote !== "", value))
|
|
11145
|
-
continue;
|
|
11146
|
-
if (!quote && value.length === 256)
|
|
11147
|
-
value += bareAt(BARE_REST, text, at + 256);
|
|
11148
|
-
out += `${text.slice(last, at)}${quote}[redacted]`;
|
|
11149
|
-
last = at + quote.length + value.length;
|
|
11150
|
-
FIELD_NAME.lastIndex = last;
|
|
11151
|
-
}
|
|
11152
|
-
return out + text.slice(last);
|
|
11153
|
-
}
|
|
11154
|
-
var MYSQL_COMMAND = /\bmysql(?:dump|admin)?\b(?:'[^'\n]*'|"[^"\n]*"|[^\n;&|\\'"]|\\\r?\n|\\(?!\r?\n))*/g;
|
|
11155
|
-
var MYSQL_PASSWORD = /(\s-p)(?:'[^'\n]*'|"[^"\n]*"|(?=[^\s-])\S+)/;
|
|
11156
|
-
var URL_CREDENTIALS = /\b((?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|rediss?|amqps?|https?):\/\/[^:\s/@]*:)[^\s/]*@([^@\s/?#]+)/g;
|
|
11157
|
-
var KEY_BEGIN = /-----BEGIN [A-Z ]*PRIVATE KEY-----/g;
|
|
11158
|
-
var KEY_END = /-----END [A-Z ]*PRIVATE KEY-----/g;
|
|
11159
|
-
function maskPrivateKeys(text) {
|
|
11160
|
-
const ends = [...text.matchAll(KEY_END)].map((m) => [m.index, m.index + m[0].length]);
|
|
11161
|
-
if (ends.length === 0)
|
|
11162
|
-
return text;
|
|
11163
|
-
let out = "";
|
|
11164
|
-
let last = 0;
|
|
11165
|
-
let e = 0;
|
|
11166
|
-
for (const m of text.matchAll(KEY_BEGIN)) {
|
|
11167
|
-
const after = m.index + m[0].length;
|
|
11168
|
-
if (m.index < last)
|
|
11169
|
-
continue;
|
|
11170
|
-
while (e < ends.length && (ends[e]?.[0] ?? 0) < after)
|
|
11171
|
-
e++;
|
|
11172
|
-
const end = ends[e];
|
|
11173
|
-
if (!end)
|
|
11174
|
-
break;
|
|
11175
|
-
out += `${text.slice(last, m.index)}[redacted: private key]`;
|
|
11176
|
-
last = end[1];
|
|
11177
|
-
}
|
|
11178
|
-
return out + text.slice(last);
|
|
11179
|
-
}
|
|
11180
|
-
function mask(text) {
|
|
11181
|
-
let out = maskFields(maskPrivateKeys(text).replace(URL_CREDENTIALS, "$1[redacted]@$2").replace(AUTH_HEADER, "$1[redacted]").replace(HEADER_BEARER, "$1[redacted]").replace(ENV_ASSIGN, "$1$2[redacted]")).replace(MYSQL_COMMAND, (command) => command.replace(MYSQL_PASSWORD, "$1[redacted]"));
|
|
11182
|
-
for (const [re, what] of SECRETS)
|
|
11183
|
-
out = out.replace(re, `[redacted: ${what}]`);
|
|
11184
|
-
return out;
|
|
11185
|
-
}
|
|
11186
|
-
var reason = (e) => explain(e, 0) || "unknown failure";
|
|
11187
|
-
var plural = (n, one, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
|
|
11188
|
-
function explain(e, depth) {
|
|
11189
|
-
if (!(e instanceof Error)) {
|
|
11190
|
-
try {
|
|
11191
|
-
return String(e);
|
|
11192
|
-
} catch {
|
|
11193
|
-
return "";
|
|
11194
|
-
}
|
|
11195
|
-
}
|
|
11196
|
-
const own = e.message || (e.name === "Error" || e.name === "AggregateError" ? "" : e.name);
|
|
11197
|
-
const parts = depth >= 3 ? [] : [...e instanceof AggregateError ? e.errors : [], ...e.cause === undefined ? [] : [e.cause]];
|
|
11198
|
-
const inner = parts.map((x) => explain(x, depth + 1)).filter(Boolean).join(" / ");
|
|
11199
|
-
return own && inner ? `${own} (${inner})` : own || inner;
|
|
11200
|
-
}
|
|
11201
|
-
|
|
11202
|
-
// server/src/db-write.ts
|
|
11203
11211
|
var ddl = null;
|
|
11204
11212
|
var DDL = () => ddl ??= new Set([
|
|
11205
11213
|
"SQLITE_ALTER_TABLE",
|
|
@@ -11274,22 +11282,54 @@ function captureAuthorizer(own, action, p1, p2, triggerOrView) {
|
|
|
11274
11282
|
return C2.SQLITE_DENY;
|
|
11275
11283
|
}
|
|
11276
11284
|
var writes = (action) => action === C2.SQLITE_INSERT || action === C2.SQLITE_UPDATE || action === C2.SQLITE_DELETE;
|
|
11285
|
+
var FORGET_ONLY = new Set(["forget_batch", "source_forgotten"]);
|
|
11286
|
+
var KEPT_BY_INGEST = new Set(["source", "unit_evidence", "unit_adoption"]);
|
|
11277
11287
|
function ingestAuthorizer(action, p1, p2) {
|
|
11278
11288
|
if (DDL().has(action))
|
|
11279
11289
|
return C2.SQLITE_DENY;
|
|
11280
|
-
if (writes(action) && p1 === "owner_identity")
|
|
11290
|
+
if (writes(action) && (p1 === "owner_identity" || FORGET_ONLY.has(p1 ?? "")))
|
|
11291
|
+
return C2.SQLITE_DENY;
|
|
11292
|
+
if (action === C2.SQLITE_DELETE && KEPT_BY_INGEST.has(p1 ?? ""))
|
|
11281
11293
|
return C2.SQLITE_DENY;
|
|
11282
11294
|
if (action === C2.SQLITE_PRAGMA)
|
|
11283
11295
|
return readsDataVersion(p1, p2) ? C2.SQLITE_OK : C2.SQLITE_DENY;
|
|
11284
11296
|
return C2.SQLITE_OK;
|
|
11285
11297
|
}
|
|
11298
|
+
var FORGET_WRITES = {
|
|
11299
|
+
[C2.SQLITE_INSERT]: new Set(["forget_batch", "source_forgotten", "unit_state", "source_fts"]),
|
|
11300
|
+
[C2.SQLITE_DELETE]: new Set([
|
|
11301
|
+
"source",
|
|
11302
|
+
"external_reference",
|
|
11303
|
+
"unit_evidence",
|
|
11304
|
+
"unit_adoption",
|
|
11305
|
+
"source_processing",
|
|
11306
|
+
"source_fts"
|
|
11307
|
+
])
|
|
11308
|
+
};
|
|
11309
|
+
var FORGET_UPDATES = {
|
|
11310
|
+
unit: new Set(["lifecycle", "revision"]),
|
|
11311
|
+
unit_state: new Set(["source_id"])
|
|
11312
|
+
};
|
|
11313
|
+
function forgetAuthorizer(action, p1, p2) {
|
|
11314
|
+
if (SHADOW.test(p1 ?? "") && writes(action))
|
|
11315
|
+
return C2.SQLITE_OK;
|
|
11316
|
+
if (action === C2.SQLITE_UPDATE)
|
|
11317
|
+
return FORGET_UPDATES[p1 ?? ""]?.has(p2 ?? "") ? C2.SQLITE_OK : C2.SQLITE_DENY;
|
|
11318
|
+
if (action === C2.SQLITE_INSERT || action === C2.SQLITE_DELETE)
|
|
11319
|
+
return FORGET_WRITES[action]?.has(p1 ?? "") ? C2.SQLITE_OK : C2.SQLITE_DENY;
|
|
11320
|
+
if (action === C2.SQLITE_PRAGMA)
|
|
11321
|
+
return readsDataVersion(p1, p2) || p1 === "secure_delete" && ["on", "1", "true"].includes((p2 ?? "").toLowerCase()) || p1 === "wal_checkpoint" ? C2.SQLITE_OK : C2.SQLITE_DENY;
|
|
11322
|
+
if (action === C2.SQLITE_READ || action === C2.SQLITE_SELECT || action === C2.SQLITE_FUNCTION || action === C2.SQLITE_TRANSACTION || action === C2.SQLITE_SAVEPOINT)
|
|
11323
|
+
return C2.SQLITE_OK;
|
|
11324
|
+
return C2.SQLITE_DENY;
|
|
11325
|
+
}
|
|
11286
11326
|
function connectWriter(role, file = dbFile(), create = false) {
|
|
11287
11327
|
requireRuntime();
|
|
11288
11328
|
if (!create)
|
|
11289
11329
|
requireFile(file);
|
|
11290
11330
|
const raw = new DatabaseSync2(file);
|
|
11291
11331
|
try {
|
|
11292
|
-
prepare(raw, create ? "none" : role === "ingest" ? "revision" : "generation");
|
|
11332
|
+
prepare(raw, create ? "none" : role === "ingest" || role === "forget" ? "revision" : "generation");
|
|
11293
11333
|
raw.function("sphica_terms", { deterministic: true }, (text) => terms(String(text ?? "")).join(" "));
|
|
11294
11334
|
} catch (e) {
|
|
11295
11335
|
raw.close();
|
|
@@ -11297,6 +11337,8 @@ function connectWriter(role, file = dbFile(), create = false) {
|
|
|
11297
11337
|
}
|
|
11298
11338
|
if (role === "ingest")
|
|
11299
11339
|
raw.setAuthorizer(ingestAuthorizer);
|
|
11340
|
+
else if (role === "forget")
|
|
11341
|
+
raw.setAuthorizer(forgetAuthorizer);
|
|
11300
11342
|
else if (role === "capture") {
|
|
11301
11343
|
let own = false;
|
|
11302
11344
|
raw.setAuthorizer((action, p1, p2, _db, triggerOrView) => captureAuthorizer(own, action, p1, p2, triggerOrView));
|