rigjs 4.0.13 → 4.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/rig-wiki/SKILL.md +5 -3
- package/RIG_WIKI_SKILL.md +5 -3
- package/built/index.js +175 -173
- package/lib/wiki/index.ts +10 -2
- package/lib/wiki/indexCmd.ts +1 -1
- package/lib/wiki/ingest.ts +1 -1
- package/lib/wiki/init.ts +27 -26
- package/lib/wiki/prune.ts +145 -0
- package/lib/wiki/qmd.ts +64 -9
- package/lib/wiki/query.ts +1 -1
- package/lib/wiki/rebuild.ts +3 -3
- package/lib/wiki/scan.ts +0 -0
- package/lib/wiki/sync.ts +111 -0
- package/package.json +2 -2
package/lib/wiki/index.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import wikiInit from './init';
|
|
2
2
|
import wikiScan from './scan';
|
|
3
3
|
import wikiSurvey from './survey';
|
|
4
|
+
import wikiSync from './sync';
|
|
4
5
|
import wikiFetch from './fetch';
|
|
5
6
|
import wikiIngest from './ingest';
|
|
6
7
|
import wikiQuery from './query';
|
|
@@ -20,8 +21,8 @@ import { registerDaemonCommands } from './daemon';
|
|
|
20
21
|
export function registerWikiCommands(program: any): void {
|
|
21
22
|
const wiki = program.command('wiki').description('Karpathy-style LLM Wiki ops (macOS only in v1)');
|
|
22
23
|
|
|
23
|
-
wiki.command('init
|
|
24
|
-
.description('bootstrap a vault
|
|
24
|
+
wiki.command('init [scope]')
|
|
25
|
+
.description('bootstrap a vault under ./rig-wiki/. With no <scope>, the vault covers the whole CWD; with <scope>, only that subdir is scanned. Hidden/.gitignored/binary files are filtered at walk time regardless.')
|
|
25
26
|
.action(wikiInit);
|
|
26
27
|
|
|
27
28
|
wiki.command('scan')
|
|
@@ -38,6 +39,13 @@ export function registerWikiCommands(program: any): void {
|
|
|
38
39
|
.option('--json', 'machine-readable output')
|
|
39
40
|
.action(wikiSurvey);
|
|
40
41
|
|
|
42
|
+
wiki.command('sync')
|
|
43
|
+
.description('one-shot wiki update: scan → ingest NEW + MODIFIED → prune wiki pages for DELETED sources')
|
|
44
|
+
.option('--dry-run', 'print scan diff and what would change without writing anything')
|
|
45
|
+
.option('--no-prune', 'skip the delete step (only ingest NEW + MODIFIED)')
|
|
46
|
+
.option('--json', 'machine-readable output')
|
|
47
|
+
.action(wikiSync);
|
|
48
|
+
|
|
41
49
|
wiki.command('fetch <url>')
|
|
42
50
|
.description('verbatim download URL into raw/YYYY-MM-DD-<slug>.md')
|
|
43
51
|
.option('--slug <slug>', 'override the auto-derived slug')
|
package/lib/wiki/indexCmd.ts
CHANGED
|
@@ -7,7 +7,7 @@ interface IndexOpts { force?: boolean; }
|
|
|
7
7
|
export default async function wikiIndex(opts: IndexOpts): Promise<void> {
|
|
8
8
|
const t = requireVault();
|
|
9
9
|
print.start(`qmd embed: ${t.name}`);
|
|
10
|
-
const res = await qmdEmbed(t.name, t.path, { force: !!opts.force });
|
|
10
|
+
const res = await qmdEmbed(t.name, t.path, t.root, { force: !!opts.force });
|
|
11
11
|
if (res.ok) print.succeed(`qmd embed: ${t.name} done`);
|
|
12
12
|
else {
|
|
13
13
|
print.error(`qmd embed: ${t.name} failed: ${res.stderr.trim()}`);
|
package/lib/wiki/ingest.ts
CHANGED
|
@@ -112,7 +112,7 @@ export default async function wikiIngest(source: string, opts: IngestOpts): Prom
|
|
|
112
112
|
|
|
113
113
|
// Real ingest — trigger incremental embed.
|
|
114
114
|
print.info(`applied ${applied.length} file change${applied.length === 1 ? '' : 's'}; rejected ${rejected.length}.`);
|
|
115
|
-
const embedRes = await qmdEmbed(target.name, target.path);
|
|
115
|
+
const embedRes = await qmdEmbed(target.name, target.path, target.root);
|
|
116
116
|
if (!embedRes.ok) {
|
|
117
117
|
print.warn(`qmd embed failed after ingest: ${embedRes.stderr.trim().slice(0, 300)}`);
|
|
118
118
|
print.warn('your wiki content is committed to disk; only the vector index is stale.');
|
package/lib/wiki/init.ts
CHANGED
|
@@ -109,10 +109,7 @@ cases.
|
|
|
109
109
|
const SUBDIRS = ['sources', 'entities', 'concepts', 'synthesis', 'queries'];
|
|
110
110
|
|
|
111
111
|
const GITIGNORE_TMPL = `# rig wiki — local-only artifacts (do not commit)
|
|
112
|
-
#
|
|
113
|
-
.qmd/index.sqlite*
|
|
114
|
-
.qmd/*.sqlite-wal
|
|
115
|
-
.qmd/*.sqlite-shm
|
|
112
|
+
# (vector cache lives outside the vault at ~/.rig/<project>/wiki/)
|
|
116
113
|
# auto-generated reports
|
|
117
114
|
lint-report-*.md
|
|
118
115
|
# daemon proposal queue (per-machine)
|
|
@@ -154,28 +151,30 @@ function defaultVaultConfig(scope: string, rootRel: string): VaultConfig {
|
|
|
154
151
|
}
|
|
155
152
|
|
|
156
153
|
export default function wikiInit(scope?: string): void {
|
|
157
|
-
if (!scope || !scope.trim()) {
|
|
158
|
-
print.error('rig wiki init requires a scope.');
|
|
159
|
-
print.info('usage: rig wiki init <scope> (e.g. `rig wiki init personal` to ingest from ./personal/)');
|
|
160
|
-
print.info(`<scope> is an existing data subdir of the project. Vault metadata is auto-created at ./${VAULT_DIRNAME}/.`);
|
|
161
|
-
process.exit(1);
|
|
162
|
-
}
|
|
163
|
-
|
|
164
154
|
const cwd = process.cwd();
|
|
165
155
|
const vaultDir = path.join(cwd, VAULT_DIRNAME);
|
|
166
|
-
const scopeAbs = path.resolve(cwd, scope);
|
|
167
156
|
|
|
168
|
-
//
|
|
169
|
-
//
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
if (
|
|
177
|
-
|
|
178
|
-
|
|
157
|
+
// No-arg form: scope = the whole project (CWD). Vault scans up one level
|
|
158
|
+
// from `<CWD>/rig-wiki/` to reach CWD. Wiki name defaults to basename(CWD).
|
|
159
|
+
// Hidden segments, binary extensions, and .gitignored files are filtered
|
|
160
|
+
// by scan/survey at walk time — no scope restriction needed.
|
|
161
|
+
const hasScope = !!(scope && scope.trim());
|
|
162
|
+
const scopeAbs = hasScope ? path.resolve(cwd, scope!) : cwd;
|
|
163
|
+
const scopeName = hasScope ? scope!.trim() : path.basename(cwd);
|
|
164
|
+
|
|
165
|
+
if (hasScope) {
|
|
166
|
+
// The explicit scope must already exist — pointing the wiki at a missing
|
|
167
|
+
// dir would hide what is almost certainly a typo.
|
|
168
|
+
if (!fs.existsSync(scopeAbs) || !fs.statSync(scopeAbs).isDirectory()) {
|
|
169
|
+
print.error(`scope dir not found: ${scope}`);
|
|
170
|
+
print.info(`expected an existing data subdir at ${shortPath(scopeAbs)}`);
|
|
171
|
+
process.exit(1);
|
|
172
|
+
}
|
|
173
|
+
// The scope can't be (or contain) the vault dir itself.
|
|
174
|
+
if (scopeAbs === vaultDir || vaultDir.startsWith(scopeAbs + path.sep)) {
|
|
175
|
+
print.error(`scope cannot be or contain the vault dir (${VAULT_DIRNAME}/).`);
|
|
176
|
+
process.exit(1);
|
|
177
|
+
}
|
|
179
178
|
}
|
|
180
179
|
|
|
181
180
|
const guard = guardPath(vaultDir, cwd);
|
|
@@ -189,6 +188,7 @@ export default function wikiInit(scope?: string): void {
|
|
|
189
188
|
// If the vault already has a config, it must already be scoped to the
|
|
190
189
|
// same data dir — otherwise the user is trying to re-target an existing
|
|
191
190
|
// vault, which we won't do silently. Manual config edit only.
|
|
191
|
+
// No-arg init resolves to the CWD itself, which equals `path.dirname(vaultDir)`.
|
|
192
192
|
const cfgFile = vaultConfigPath(vaultDir);
|
|
193
193
|
if (fs.existsSync(cfgFile)) {
|
|
194
194
|
const existing = loadVaultConfig(vaultDir);
|
|
@@ -222,12 +222,13 @@ export default function wikiInit(scope?: string): void {
|
|
|
222
222
|
|
|
223
223
|
if (!fs.existsSync(cfgFile)) {
|
|
224
224
|
const rootRel = path.relative(vaultDir, scopeAbs);
|
|
225
|
-
saveVaultConfig(vaultDir, defaultVaultConfig(
|
|
225
|
+
saveVaultConfig(vaultDir, defaultVaultConfig(scopeName, rootRel));
|
|
226
226
|
}
|
|
227
227
|
|
|
228
|
-
|
|
228
|
+
const scopeLabel = hasScope ? `scope: ${scope}` : `scope: <project-wide, name "${scopeName}">`;
|
|
229
|
+
print.succeed(`vault initialized at ${shortPath(vaultDir)} (${scopeLabel})`);
|
|
229
230
|
print.info(`next: edit ${shortPath(path.join(vaultDir, 'purpose.md'))} to describe what this wiki is for.`);
|
|
230
|
-
print.info(`then run \`rig wiki
|
|
231
|
+
print.info(`then run \`rig wiki sync\` from anywhere inside ${shortPath(cwd)} to ingest, update, and prune in one shot.`);
|
|
231
232
|
}
|
|
232
233
|
|
|
233
234
|
function writeIfMissing(file: string, content: string) {
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
// Prune wiki pages whose underlying source file has been deleted from the
|
|
2
|
+
// scan root. Triggered by `rig wiki sync` (and exposed as a helper for
|
|
3
|
+
// `rig wiki prune` if/when we add a standalone command).
|
|
4
|
+
//
|
|
5
|
+
// Three actions per deleted source path:
|
|
6
|
+
//
|
|
7
|
+
// 1. Delete `sources/<slug>.md` whose `source-path:` frontmatter resolves
|
|
8
|
+
// to the deleted file. The slug = basename of that wiki source page.
|
|
9
|
+
//
|
|
10
|
+
// 2. Scrub the deleted slug from every derived page's `sources: [...]`
|
|
11
|
+
// frontmatter array (entities/concepts/synthesis/queries). Pages that
|
|
12
|
+
// end up with an empty `sources: []` are left in place for `rig wiki
|
|
13
|
+
// lint` to surface as orphans — we don't auto-delete derived pages
|
|
14
|
+
// since they may carry the user's intellectual content distinct from
|
|
15
|
+
// any single source.
|
|
16
|
+
//
|
|
17
|
+
// 3. Drop the row from `state.db.source_sha` so future scans don't keep
|
|
18
|
+
// reporting the file as DELETED.
|
|
19
|
+
|
|
20
|
+
import fs from 'fs';
|
|
21
|
+
import path from 'path';
|
|
22
|
+
import yaml from 'js-yaml';
|
|
23
|
+
import { WikiEntry } from './config';
|
|
24
|
+
import { deleteSourceSha } from './db';
|
|
25
|
+
|
|
26
|
+
const DERIVED_DIRS = ['entities', 'concepts', 'synthesis', 'queries'];
|
|
27
|
+
|
|
28
|
+
export interface PruneReport {
|
|
29
|
+
/** wiki-relative paths of source pages deleted */
|
|
30
|
+
deletedSourcePages: string[];
|
|
31
|
+
/** wiki-relative paths of derived pages whose sources[] lost the dropped slug */
|
|
32
|
+
scrubbedDerivedPages: string[];
|
|
33
|
+
/** root-relative paths of source_sha rows dropped */
|
|
34
|
+
shaRowsDropped: string[];
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function pruneDeletedSources(target: WikiEntry, deletedRelPaths: string[]): PruneReport {
|
|
38
|
+
const report: PruneReport = { deletedSourcePages: [], scrubbedDerivedPages: [], shaRowsDropped: [] };
|
|
39
|
+
if (deletedRelPaths.length === 0) return report;
|
|
40
|
+
|
|
41
|
+
const deletedSet = new Set(deletedRelPaths.map(normalizePath));
|
|
42
|
+
const sourcesDir = path.join(target.path, 'sources');
|
|
43
|
+
const droppedSlugs = new Set<string>();
|
|
44
|
+
|
|
45
|
+
// 1. sources/<slug>.md whose `source-path:` points at a deleted file
|
|
46
|
+
if (fs.existsSync(sourcesDir)) {
|
|
47
|
+
for (const file of fs.readdirSync(sourcesDir)) {
|
|
48
|
+
if (!file.endsWith('.md') || file === '.gitkeep') continue;
|
|
49
|
+
const abs = path.join(sourcesDir, file);
|
|
50
|
+
const fm = readFrontmatter(abs);
|
|
51
|
+
if (!fm) continue;
|
|
52
|
+
const rel = extractSourcePathRel(String(fm['source-path'] || ''));
|
|
53
|
+
if (rel && deletedSet.has(rel)) {
|
|
54
|
+
fs.rmSync(abs, { force: true });
|
|
55
|
+
report.deletedSourcePages.push(path.relative(target.path, abs));
|
|
56
|
+
droppedSlugs.add(path.basename(file, '.md'));
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// 2. derived pages — strip dropped slugs from `sources: [...]`
|
|
62
|
+
if (droppedSlugs.size > 0) {
|
|
63
|
+
for (const sub of DERIVED_DIRS) {
|
|
64
|
+
const dir = path.join(target.path, sub);
|
|
65
|
+
if (!fs.existsSync(dir)) continue;
|
|
66
|
+
for (const file of fs.readdirSync(dir)) {
|
|
67
|
+
if (!file.endsWith('.md') || file === '.gitkeep') continue;
|
|
68
|
+
const abs = path.join(dir, file);
|
|
69
|
+
const content = fs.readFileSync(abs, 'utf8');
|
|
70
|
+
const next = scrubSlugsFromSourcesArray(content, droppedSlugs);
|
|
71
|
+
if (next !== content) {
|
|
72
|
+
fs.writeFileSync(abs, next, 'utf8');
|
|
73
|
+
report.scrubbedDerivedPages.push(path.relative(target.path, abs));
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// 3. state.db cleanup
|
|
80
|
+
for (const p of deletedRelPaths) {
|
|
81
|
+
deleteSourceSha(target.name, p);
|
|
82
|
+
report.shaRowsDropped.push(p);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
return report;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// ─── helpers ────────────────────────────────────────────────────────────
|
|
89
|
+
|
|
90
|
+
function normalizePath(p: string): string {
|
|
91
|
+
return p.replace(/\\/g, '/');
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function readFrontmatter(file: string): Record<string, unknown> | null {
|
|
95
|
+
let content: string;
|
|
96
|
+
try { content = fs.readFileSync(file, 'utf8'); } catch { return null; }
|
|
97
|
+
if (!content.startsWith('---\n')) return null;
|
|
98
|
+
const end = content.indexOf('\n---', 4);
|
|
99
|
+
if (end < 0) return null;
|
|
100
|
+
try {
|
|
101
|
+
const parsed = yaml.load(content.slice(4, end));
|
|
102
|
+
return parsed && typeof parsed === 'object' ? parsed as Record<string, unknown> : null;
|
|
103
|
+
} catch { return null; }
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* `source-path` can be:
|
|
108
|
+
* - an obsidian:// URL: `obsidian://open?vault=name&file=<encoded-path>`
|
|
109
|
+
* - a plain root-relative path: `personal/work/foo.md`
|
|
110
|
+
* Returns the underlying root-relative path, or null if we can't extract one.
|
|
111
|
+
*/
|
|
112
|
+
function extractSourcePathRel(srcPath: string): string | null {
|
|
113
|
+
if (!srcPath) return null;
|
|
114
|
+
const m = srcPath.match(/[?&]file=([^&]+)/);
|
|
115
|
+
if (m) {
|
|
116
|
+
try { return normalizePath(decodeURIComponent(m[1])); }
|
|
117
|
+
catch { return normalizePath(m[1]); }
|
|
118
|
+
}
|
|
119
|
+
return normalizePath(srcPath);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Surgically rewrites the `sources: [a, b, c]` frontmatter line, removing
|
|
124
|
+
* any entries that appear in `slugs`. Leaves the rest of the frontmatter
|
|
125
|
+
* and body untouched. If no match, returns the content unchanged.
|
|
126
|
+
*/
|
|
127
|
+
function scrubSlugsFromSourcesArray(content: string, slugs: Set<string>): string {
|
|
128
|
+
if (!content.startsWith('---\n')) return content;
|
|
129
|
+
const end = content.indexOf('\n---', 4);
|
|
130
|
+
if (end < 0) return content;
|
|
131
|
+
const fm = content.slice(4, end);
|
|
132
|
+
const after = content.slice(end);
|
|
133
|
+
|
|
134
|
+
const m = fm.match(/^([ \t]*sources[ \t]*:[ \t]*)\[(.*?)\]/m);
|
|
135
|
+
if (!m) return content;
|
|
136
|
+
|
|
137
|
+
const items = m[2]
|
|
138
|
+
.split(',')
|
|
139
|
+
.map(s => s.trim().replace(/^['"]|['"]$/g, ''))
|
|
140
|
+
.filter(s => s && !slugs.has(s));
|
|
141
|
+
|
|
142
|
+
const newSources = `${m[1]}[${items.map(s => `'${s}'`).join(', ')}]`;
|
|
143
|
+
const newFm = fm.replace(m[0], newSources);
|
|
144
|
+
return '---\n' + newFm + after;
|
|
145
|
+
}
|
package/lib/wiki/qmd.ts
CHANGED
|
@@ -9,8 +9,13 @@
|
|
|
9
9
|
// - Embed: Qwen3-Embedding-0.6B (~610MB, sets QMD_EMBED_MODEL)
|
|
10
10
|
// - Rerank: Qwen3-Reranker-0.6B (~610MB, sets QMD_RERANK_MODEL)
|
|
11
11
|
//
|
|
12
|
-
// Per-wiki SQLite DB lives at `~/.rig
|
|
13
|
-
//
|
|
12
|
+
// Per-wiki SQLite DB lives at `~/.rig/<project>/wiki/<wiki>.sqlite`, where
|
|
13
|
+
// `<project>` is the `name` from the nearest `package.json` walking up from
|
|
14
|
+
// the vault root (fallback: vault-root basename). The extra `wiki/` segment
|
|
15
|
+
// reserves room for other per-project rig artifacts as siblings. Legacy
|
|
16
|
+
// `~/.rig/cache/qmd/<wiki>.sqlite` is migrated lazily on first open.
|
|
17
|
+
//
|
|
18
|
+
// Model GGUFs still cache in `~/.cache/qmd/models/` (hardcoded inside qmd).
|
|
14
19
|
//
|
|
15
20
|
// Concurrency note: qmd's `setConfigSource` is module-global; serialize
|
|
16
21
|
// store lifetimes by opening + closing inside each call.
|
|
@@ -71,10 +76,58 @@ function qmdVersion(): string {
|
|
|
71
76
|
return 'unknown';
|
|
72
77
|
}
|
|
73
78
|
|
|
74
|
-
|
|
75
|
-
|
|
79
|
+
/**
|
|
80
|
+
* Resolve the project name for a vault: read the `name` field from the
|
|
81
|
+
* nearest `package.json` walking up from `vaultRoot`. Falls back to
|
|
82
|
+
* `basename(vaultRoot)` if no package.json with a usable name is found.
|
|
83
|
+
* Scoped names (`@scope/foo`) are flattened to `scope_foo`.
|
|
84
|
+
*/
|
|
85
|
+
function resolveProjectName(vaultRoot: string): string {
|
|
86
|
+
let dir = path.resolve(vaultRoot);
|
|
87
|
+
while (true) {
|
|
88
|
+
const pkg = path.join(dir, 'package.json');
|
|
89
|
+
if (fs.existsSync(pkg)) {
|
|
90
|
+
try {
|
|
91
|
+
const parsed = JSON.parse(fs.readFileSync(pkg, 'utf8'));
|
|
92
|
+
if (typeof parsed.name === 'string' && parsed.name.trim()) {
|
|
93
|
+
return sanitizeSegment(parsed.name.trim());
|
|
94
|
+
}
|
|
95
|
+
} catch { /* malformed package.json — keep walking */ }
|
|
96
|
+
}
|
|
97
|
+
const parent = path.dirname(dir);
|
|
98
|
+
if (parent === dir) break;
|
|
99
|
+
dir = parent;
|
|
100
|
+
}
|
|
101
|
+
return sanitizeSegment(path.basename(vaultRoot)) || 'unknown';
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function sanitizeSegment(s: string): string {
|
|
105
|
+
// npm scoped names use `/`; flatten so we don't create unintended nesting.
|
|
106
|
+
// Also defang filesystem-hostile chars and leading dots.
|
|
107
|
+
return s
|
|
108
|
+
.replace(/^@/, '')
|
|
109
|
+
.replace(/[/\\]/g, '_')
|
|
110
|
+
.replace(/[<>:"|?*\x00-\x1f]/g, '_')
|
|
111
|
+
.replace(/^\.+/, '');
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function dbPathFor(wikiName: string, vaultRoot: string): string {
|
|
115
|
+
const projectName = resolveProjectName(vaultRoot);
|
|
116
|
+
const dir = path.join(paths.home, projectName, 'wiki');
|
|
76
117
|
fs.mkdirSync(dir, { recursive: true });
|
|
77
|
-
|
|
118
|
+
const target = path.join(dir, `${wikiName}.sqlite`);
|
|
119
|
+
// One-shot migration from the legacy flat layout.
|
|
120
|
+
const legacy = path.join(paths.cache, 'qmd', `${wikiName}.sqlite`);
|
|
121
|
+
if (!fs.existsSync(target) && fs.existsSync(legacy)) {
|
|
122
|
+
for (const suffix of ['', '-wal', '-shm']) {
|
|
123
|
+
const src = legacy + suffix;
|
|
124
|
+
if (fs.existsSync(src)) {
|
|
125
|
+
try { fs.renameSync(src, target + suffix); }
|
|
126
|
+
catch { /* cross-device? leave the legacy file in place */ }
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
return target;
|
|
78
131
|
}
|
|
79
132
|
|
|
80
133
|
async function loadQmd(): Promise<{ createStore: any }> {
|
|
@@ -90,12 +143,13 @@ async function loadQmd(): Promise<{ createStore: any }> {
|
|
|
90
143
|
export async function qmdEmbed(
|
|
91
144
|
wikiName: string,
|
|
92
145
|
dir: string,
|
|
146
|
+
vaultRoot: string,
|
|
93
147
|
opts: { force?: boolean } = {}
|
|
94
148
|
): Promise<{ ok: boolean; stderr: string }> {
|
|
95
149
|
try {
|
|
96
150
|
const { createStore } = await loadQmd();
|
|
97
151
|
const store = await createStore({
|
|
98
|
-
dbPath: dbPathFor(wikiName),
|
|
152
|
+
dbPath: dbPathFor(wikiName, vaultRoot),
|
|
99
153
|
config: {
|
|
100
154
|
collections: { [wikiName]: { path: dir, pattern: '**/*.md' } },
|
|
101
155
|
},
|
|
@@ -134,6 +188,7 @@ export interface QmdHit {
|
|
|
134
188
|
export async function qmdQuery(
|
|
135
189
|
q: string,
|
|
136
190
|
wikiName: string,
|
|
191
|
+
vaultRoot: string,
|
|
137
192
|
opts: { limit?: number; candidateLimit?: number; rerank?: boolean } = {}
|
|
138
193
|
): Promise<QmdHit[] | null> {
|
|
139
194
|
const limit = opts.limit ?? 10;
|
|
@@ -142,7 +197,7 @@ export async function qmdQuery(
|
|
|
142
197
|
|
|
143
198
|
try {
|
|
144
199
|
const { createStore } = await loadQmd();
|
|
145
|
-
const store = await createStore({ dbPath: dbPathFor(wikiName) });
|
|
200
|
+
const store = await createStore({ dbPath: dbPathFor(wikiName, vaultRoot) });
|
|
146
201
|
try {
|
|
147
202
|
const raw = await store.searchVector(q, { limit: candidateLimit, collection: wikiName });
|
|
148
203
|
const candidates: QmdHit[] = Array.isArray(raw) ? raw.map(normalizeHit) : [];
|
|
@@ -191,8 +246,8 @@ function normalizeHit(h: any): QmdHit {
|
|
|
191
246
|
}
|
|
192
247
|
|
|
193
248
|
/** Wipe the per-wiki SQLite store on disk. Caller should then qmdEmbed. */
|
|
194
|
-
export function qmdResetStore(wikiName: string): void {
|
|
195
|
-
const p = dbPathFor(wikiName);
|
|
249
|
+
export function qmdResetStore(wikiName: string, vaultRoot: string): void {
|
|
250
|
+
const p = dbPathFor(wikiName, vaultRoot);
|
|
196
251
|
for (const suffix of ['', '-wal', '-shm']) {
|
|
197
252
|
const f = p + suffix;
|
|
198
253
|
if (fs.existsSync(f)) fs.rmSync(f, { force: true });
|
package/lib/wiki/query.ts
CHANGED
|
@@ -30,7 +30,7 @@ export default async function wikiQuery(q: string, opts: QueryOpts): Promise<voi
|
|
|
30
30
|
const target = requireVault();
|
|
31
31
|
|
|
32
32
|
const limit = Math.max(1, Math.min(50, opts.limit || 10));
|
|
33
|
-
const hits = await qmdQuery(q, target.name, { limit, rerank: opts.rerank !== false });
|
|
33
|
+
const hits = await qmdQuery(q, target.name, target.root, { limit, rerank: opts.rerank !== false });
|
|
34
34
|
if (hits === null) {
|
|
35
35
|
print.error('qmd query failed. Run `rig wiki index` first to (re)build the vector store.');
|
|
36
36
|
process.exit(1);
|
package/lib/wiki/rebuild.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
//
|
|
3
3
|
// Use cases:
|
|
4
4
|
// 1. New device: source markdown is checked out, but ~/.rig/state.db and
|
|
5
|
-
// ~/.rig
|
|
5
|
+
// ~/.rig/<project>/wiki/<wiki>.sqlite are empty. Rebuild populates both.
|
|
6
6
|
// 2. Switched embedding model: old vectors are now meaningless. Rebuild
|
|
7
7
|
// re-embeds against the current QMD_EMBED_MODEL.
|
|
8
8
|
// 3. Local cache corruption: nuke and start over.
|
|
@@ -27,10 +27,10 @@ export default async function wikiRebuild(opts: RebuildOpts): Promise<void> {
|
|
|
27
27
|
const del = db.prepare('DELETE FROM source_sha WHERE wiki = ?').run(t.name);
|
|
28
28
|
print.info(` cleared ${del.changes} source_sha rows for ${t.name}`);
|
|
29
29
|
|
|
30
|
-
qmdResetStore(t.name);
|
|
30
|
+
qmdResetStore(t.name, t.root);
|
|
31
31
|
|
|
32
32
|
if (!opts.skipEmbed) {
|
|
33
|
-
const res = await qmdEmbed(t.name, t.path, { force: true });
|
|
33
|
+
const res = await qmdEmbed(t.name, t.path, t.root, { force: true });
|
|
34
34
|
if (res.ok) print.info(` qmd embed: ${t.name} done`);
|
|
35
35
|
else {
|
|
36
36
|
print.error(` qmd embed: ${t.name} failed: ${res.stderr.trim()}`);
|
package/lib/wiki/scan.ts
CHANGED
|
Binary file
|
package/lib/wiki/sync.ts
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
// `rig wiki sync` — one-shot wiki update.
|
|
2
|
+
//
|
|
3
|
+
// 1. scan: compute NEW / MODIFIED / DELETED against state.db.source_sha
|
|
4
|
+
// (hidden + .gitignored + binary-extension filters apply at walk time,
|
|
5
|
+
// and the gitignore check is multi-repo-aware so git submodules are
|
|
6
|
+
// respected — see `lib/wiki/gitignore.ts`).
|
|
7
|
+
// 2. ingest each NEW + MODIFIED source via the existing `wikiIngest`
|
|
8
|
+
// code path. Sequential so Claude doesn't double-write the same wiki
|
|
9
|
+
// page from concurrent runs, and so token spend is bounded.
|
|
10
|
+
// 3. prune the wiki pages whose underlying source files have been
|
|
11
|
+
// deleted: removes `sources/<slug>.md`, strips that slug from derived
|
|
12
|
+
// pages' `sources: [...]` frontmatter, drops the source_sha row.
|
|
13
|
+
//
|
|
14
|
+
// RAW DRIFT (a `raw/` file's bytes changed in place) is surfaced as an
|
|
15
|
+
// error — we do NOT try to "fix" it via re-ingest. raw/ is immutable by
|
|
16
|
+
// design; drift means something else (a copy-on-write filesystem,
|
|
17
|
+
// accidental edit, etc.) and the user needs to look.
|
|
18
|
+
|
|
19
|
+
import path from 'path';
|
|
20
|
+
import print from '../print';
|
|
21
|
+
import { requireVault } from './config';
|
|
22
|
+
import { recordLastRun } from './db';
|
|
23
|
+
import { scanOne, ScanReport } from './scan';
|
|
24
|
+
import { pruneDeletedSources, PruneReport } from './prune';
|
|
25
|
+
import { default as wikiIngest } from './ingest';
|
|
26
|
+
|
|
27
|
+
interface SyncOpts {
|
|
28
|
+
json?: boolean;
|
|
29
|
+
dryRun?: boolean;
|
|
30
|
+
noPrune?: boolean;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export default async function wikiSync(opts: SyncOpts): Promise<void> {
|
|
34
|
+
const target = requireVault();
|
|
35
|
+
|
|
36
|
+
// Step 1: scan. We don't baseline yet — ingest+prune are the ground truth
|
|
37
|
+
// for what gets recorded, and baselining before would mark unprocessed
|
|
38
|
+
// files as "known" prematurely.
|
|
39
|
+
const scan: ScanReport = scanOne(target, false);
|
|
40
|
+
|
|
41
|
+
if (scan.rawDrift.length > 0) {
|
|
42
|
+
print.error(`RAW DRIFT detected — ${scan.rawDrift.length} file(s) in raw/ changed. raw/ is immutable; resolve manually before syncing.`);
|
|
43
|
+
for (const p of scan.rawDrift) {
|
|
44
|
+
// eslint-disable-next-line no-console
|
|
45
|
+
console.log(` ${p}`);
|
|
46
|
+
}
|
|
47
|
+
recordLastRun(target.name, 'sync', 10);
|
|
48
|
+
process.exit(10);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const toIngest = [...scan.new, ...scan.modified];
|
|
52
|
+
print.info(`sync: ${target.name} NEW ${scan.new.length} MODIFIED ${scan.modified.length} DELETED ${scan.deleted.length} UNCHANGED ${scan.unchanged}`);
|
|
53
|
+
|
|
54
|
+
// Step 2: ingest NEW + MODIFIED (one Claude call per file).
|
|
55
|
+
let ingestOk = 0, ingestFail = 0;
|
|
56
|
+
for (const rel of toIngest) {
|
|
57
|
+
const abs = path.resolve(target.root, rel);
|
|
58
|
+
print.start(`ingest ${rel}`);
|
|
59
|
+
try {
|
|
60
|
+
await wikiIngest(abs, { dryRun: !!opts.dryRun });
|
|
61
|
+
ingestOk++;
|
|
62
|
+
} catch (e) {
|
|
63
|
+
ingestFail++;
|
|
64
|
+
print.error(`ingest ${rel} failed: ${(e as Error).message}`);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// Step 3: prune DELETED unless --no-prune / --dry-run.
|
|
69
|
+
let prune: PruneReport = { deletedSourcePages: [], scrubbedDerivedPages: [], shaRowsDropped: [] };
|
|
70
|
+
if (scan.deleted.length > 0 && !opts.noPrune && !opts.dryRun) {
|
|
71
|
+
prune = pruneDeletedSources(target, scan.deleted);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Step 4: report.
|
|
75
|
+
if (opts.json) {
|
|
76
|
+
// eslint-disable-next-line no-console
|
|
77
|
+
console.log(JSON.stringify({
|
|
78
|
+
ok: ingestFail === 0,
|
|
79
|
+
code: ingestFail === 0 ? 0 : 1,
|
|
80
|
+
data: {
|
|
81
|
+
wiki: target.name,
|
|
82
|
+
scan,
|
|
83
|
+
ingested: { ok: ingestOk, failed: ingestFail },
|
|
84
|
+
pruned: prune,
|
|
85
|
+
dryRun: !!opts.dryRun,
|
|
86
|
+
},
|
|
87
|
+
}, null, 2));
|
|
88
|
+
} else {
|
|
89
|
+
print.succeed(`sync: ingested ${ingestOk} ok, ${ingestFail} failed.`);
|
|
90
|
+
if (prune.deletedSourcePages.length > 0) {
|
|
91
|
+
print.info(`pruned ${prune.deletedSourcePages.length} source page(s):`);
|
|
92
|
+
for (const p of prune.deletedSourcePages) {
|
|
93
|
+
// eslint-disable-next-line no-console
|
|
94
|
+
console.log(` − ${p}`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
if (prune.scrubbedDerivedPages.length > 0) {
|
|
98
|
+
print.info(`scrubbed deleted-source refs from ${prune.scrubbedDerivedPages.length} derived page(s):`);
|
|
99
|
+
for (const p of prune.scrubbedDerivedPages) {
|
|
100
|
+
// eslint-disable-next-line no-console
|
|
101
|
+
console.log(` ~ ${p}`);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
if (opts.dryRun && scan.deleted.length > 0) {
|
|
105
|
+
print.info(`dry-run: ${scan.deleted.length} delete(s) NOT applied. Re-run without --dry-run to prune.`);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
recordLastRun(target.name, 'sync', ingestFail === 0 ? 0 : 1);
|
|
110
|
+
if (ingestFail > 0) process.exit(1);
|
|
111
|
+
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "rigjs",
|
|
3
|
-
"version": "4.0.
|
|
4
|
-
"versionCode":
|
|
3
|
+
"version": "4.0.15",
|
|
4
|
+
"versionCode": 26052422,
|
|
5
5
|
"description": "A multi-repos dev tool based on yarn and git.Rigjs is intended to be the simplest way to develop,share and deliver codes between different developers or different projects.",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"modular",
|