sensemaking 0.25.1 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -9
- package/dist/cjs/cli/index.d.cts +1 -1
- package/dist/cjs/cli/index.d.ts +1 -1
- package/dist/cjs/cli/index.js +1 -1
- package/dist/cjs/cli/index.js.map +1 -1
- package/dist/cjs/cli/path.js +5 -29
- package/dist/cjs/cli/path.js.map +1 -1
- package/dist/cjs/cli/related.js +4 -3
- package/dist/cjs/cli/related.js.map +1 -1
- package/dist/cjs/commands/index.d.cts +6 -3
- package/dist/cjs/commands/index.d.ts +6 -3
- package/dist/cjs/commands/index.js +4 -0
- package/dist/cjs/commands/index.js.map +1 -1
- package/dist/cjs/commands/map.d.cts +14 -5
- package/dist/cjs/commands/map.d.ts +14 -5
- package/dist/cjs/commands/map.js +42 -35
- package/dist/cjs/commands/map.js.map +1 -1
- package/dist/cjs/commands/path.d.cts +14 -0
- package/dist/cjs/commands/path.d.ts +14 -0
- package/dist/cjs/commands/path.js +209 -0
- package/dist/cjs/commands/path.js.map +1 -0
- package/dist/cjs/commands/peek.d.cts +16 -1
- package/dist/cjs/commands/peek.d.ts +16 -1
- package/dist/cjs/commands/peek.js +71 -33
- package/dist/cjs/commands/peek.js.map +1 -1
- package/dist/cjs/commands/related.d.cts +16 -3
- package/dist/cjs/commands/related.d.ts +16 -3
- package/dist/cjs/commands/related.js +131 -98
- package/dist/cjs/commands/related.js.map +1 -1
- package/dist/cjs/commands/scope.d.cts +1 -0
- package/dist/cjs/commands/scope.d.ts +1 -0
- package/dist/cjs/commands/scope.js +149 -0
- package/dist/cjs/commands/scope.js.map +1 -1
- package/dist/cjs/commands/search.d.cts +26 -3
- package/dist/cjs/commands/search.d.ts +26 -3
- package/dist/cjs/commands/search.js +211 -71
- package/dist/cjs/commands/search.js.map +1 -1
- package/dist/cjs/commands/signals.d.cts +4 -3
- package/dist/cjs/commands/signals.d.ts +4 -3
- package/dist/cjs/commands/signals.js +13 -4
- package/dist/cjs/commands/signals.js.map +1 -1
- package/dist/cjs/commands/snapshot.d.cts +3 -0
- package/dist/cjs/commands/snapshot.d.ts +3 -0
- package/dist/cjs/commands/snapshot.js +180 -0
- package/dist/cjs/commands/snapshot.js.map +1 -0
- package/dist/cjs/embed/cohere.js +41 -16
- package/dist/cjs/embed/cohere.js.map +1 -1
- package/dist/cjs/embed/endpoint.d.cts +2 -0
- package/dist/cjs/embed/endpoint.d.ts +2 -0
- package/dist/cjs/embed/endpoint.js +30 -0
- package/dist/cjs/embed/endpoint.js.map +1 -0
- package/dist/cjs/embed/handoff.d.cts +7 -2
- package/dist/cjs/embed/handoff.d.ts +7 -2
- package/dist/cjs/embed/handoff.js +10 -0
- package/dist/cjs/embed/handoff.js.map +1 -1
- package/dist/cjs/embed/http.d.cts +18 -1
- package/dist/cjs/embed/http.d.ts +18 -1
- package/dist/cjs/embed/http.js +239 -40
- package/dist/cjs/embed/http.js.map +1 -1
- package/dist/cjs/embed/openai.js +39 -17
- package/dist/cjs/embed/openai.js.map +1 -1
- package/dist/cjs/embed/query.d.cts +9 -3
- package/dist/cjs/embed/query.d.ts +9 -3
- package/dist/cjs/embed/query.js +257 -90
- package/dist/cjs/embed/query.js.map +1 -1
- package/dist/cjs/embed/registry.d.cts +2 -2
- package/dist/cjs/embed/registry.d.ts +2 -2
- package/dist/cjs/embed/registry.js +36 -4
- package/dist/cjs/embed/registry.js.map +1 -1
- package/dist/cjs/embed/static.js +6 -2
- package/dist/cjs/embed/static.js.map +1 -1
- package/dist/cjs/embed/store.js +42 -77
- package/dist/cjs/embed/store.js.map +1 -1
- package/dist/cjs/embed/types.d.cts +5 -2
- package/dist/cjs/embed/types.d.ts +5 -2
- package/dist/cjs/embed/types.js.map +1 -1
- package/dist/cjs/features/embed.js +290 -43
- package/dist/cjs/features/embed.js.map +1 -1
- package/dist/cjs/graph/traverse.d.cts +2 -2
- package/dist/cjs/graph/traverse.d.ts +2 -2
- package/dist/cjs/graph/traverse.js +15 -5
- package/dist/cjs/graph/traverse.js.map +1 -1
- package/dist/cjs/index.d.cts +2 -2
- package/dist/cjs/index.d.ts +2 -2
- package/dist/cjs/index.js +6 -0
- package/dist/cjs/index.js.map +1 -1
- package/dist/cjs/output/output.d.cts +4 -36
- package/dist/cjs/output/output.d.ts +4 -36
- package/dist/cjs/output/output.js +1 -2
- package/dist/cjs/output/output.js.map +1 -1
- package/dist/cjs/scan/pool.d.cts +2 -1
- package/dist/cjs/scan/pool.d.ts +2 -1
- package/dist/cjs/scan/pool.js +116 -27
- package/dist/cjs/scan/pool.js.map +1 -1
- package/dist/cjs/scan/reparse.d.cts +1 -0
- package/dist/cjs/scan/reparse.d.ts +1 -0
- package/dist/cjs/scan/reparse.js +75 -21
- package/dist/cjs/scan/reparse.js.map +1 -1
- package/dist/cjs/store/builder.d.cts +11 -21
- package/dist/cjs/store/builder.d.ts +11 -21
- package/dist/cjs/store/builder.js +1077 -438
- package/dist/cjs/store/builder.js.map +1 -1
- package/dist/cjs/store/duckdb/connection.js +16 -2
- package/dist/cjs/store/duckdb/connection.js.map +1 -1
- package/dist/cjs/store/duckdb/open.d.cts +1 -1
- package/dist/cjs/store/duckdb/open.d.ts +1 -1
- package/dist/cjs/store/duckdb/open.js +73 -65
- package/dist/cjs/store/duckdb/open.js.map +1 -1
- package/dist/cjs/store/duckdb/reconcile.js +533 -15
- package/dist/cjs/store/duckdb/reconcile.js.map +1 -1
- package/dist/cjs/store/embed-scope.js +19 -9
- package/dist/cjs/store/embed-scope.js.map +1 -1
- package/dist/cjs/store/index.d.cts +6 -0
- package/dist/cjs/store/index.d.ts +6 -0
- package/dist/cjs/store/index.js +8 -4
- package/dist/cjs/store/index.js.map +1 -1
- package/dist/cjs/store/open.d.cts +6 -1
- package/dist/cjs/store/open.d.ts +6 -1
- package/dist/cjs/store/open.js +230 -307
- package/dist/cjs/store/open.js.map +1 -1
- package/dist/cjs/store/readiness.d.cts +5 -0
- package/dist/cjs/store/readiness.d.ts +5 -0
- package/dist/cjs/store/readiness.js +39 -0
- package/dist/cjs/store/readiness.js.map +1 -0
- package/dist/cjs/store/reconcile.d.cts +26 -1
- package/dist/cjs/store/reconcile.d.ts +26 -1
- package/dist/cjs/store/reconcile.js +670 -576
- package/dist/cjs/store/reconcile.js.map +1 -1
- package/dist/cjs/store/signature.d.cts +1 -0
- package/dist/cjs/store/signature.d.ts +1 -0
- package/dist/cjs/store/signature.js +9 -0
- package/dist/cjs/store/signature.js.map +1 -1
- package/dist/cjs/store/sqlite/open.d.cts +1 -1
- package/dist/cjs/store/sqlite/open.d.ts +1 -1
- package/dist/cjs/store/sqlite/open.js +84 -90
- package/dist/cjs/store/sqlite/open.js.map +1 -1
- package/dist/cjs/store/sqlite/reconcile.js.map +1 -1
- package/dist/cjs/store/sqlite/store.js +63 -60
- package/dist/cjs/store/sqlite/store.js.map +1 -1
- package/dist/cjs/store/sqlite/vectors.d.cts +2 -2
- package/dist/cjs/store/sqlite/vectors.d.ts +2 -2
- package/dist/cjs/store/sqlite/vectors.js +8 -4
- package/dist/cjs/store/sqlite/vectors.js.map +1 -1
- package/dist/cjs/store/stages.js.map +1 -1
- package/dist/cjs/store/turso/connection.d.cts +1 -0
- package/dist/cjs/store/turso/connection.d.ts +1 -0
- package/dist/cjs/store/turso/connection.js +319 -94
- package/dist/cjs/store/turso/connection.js.map +1 -1
- package/dist/cjs/store/turso/open.d.cts +1 -1
- package/dist/cjs/store/turso/open.d.ts +1 -1
- package/dist/cjs/store/turso/open.js +70 -74
- package/dist/cjs/store/turso/open.js.map +1 -1
- package/dist/cjs/store/turso/store.js +25 -23
- package/dist/cjs/store/turso/store.js.map +1 -1
- package/dist/cjs/store/turso/vectors.d.cts +2 -2
- package/dist/cjs/store/turso/vectors.d.ts +2 -2
- package/dist/cjs/store/turso/vectors.js +26 -16
- package/dist/cjs/store/turso/vectors.js.map +1 -1
- package/dist/cjs/store/types.d.cts +4 -1
- package/dist/cjs/store/types.d.ts +4 -1
- package/dist/cjs/store/types.js.map +1 -1
- package/dist/cjs/store/vectors.d.cts +14 -1
- package/dist/cjs/store/vectors.d.ts +14 -1
- package/dist/cjs/store/vectors.js +61 -0
- package/dist/cjs/store/vectors.js.map +1 -1
- package/dist/cjs/watch.d.cts +6 -0
- package/dist/cjs/watch.d.ts +6 -0
- package/dist/cjs/watch.js +42 -10
- package/dist/cjs/watch.js.map +1 -1
- package/dist/cjs/workers/parse.d.cts +4 -3
- package/dist/cjs/workers/parse.d.ts +4 -3
- package/dist/cjs/workers/parse.js +19 -17
- package/dist/cjs/workers/parse.js.map +1 -1
- package/dist/esm/cli/index.d.ts +1 -1
- package/dist/esm/cli/index.js +1 -1
- package/dist/esm/cli/index.js.map +1 -1
- package/dist/esm/cli/path.js +4 -10
- package/dist/esm/cli/path.js.map +1 -1
- package/dist/esm/cli/related.js +5 -3
- package/dist/esm/cli/related.js.map +1 -1
- package/dist/esm/commands/index.d.ts +6 -3
- package/dist/esm/commands/index.js +1 -0
- package/dist/esm/commands/index.js.map +1 -1
- package/dist/esm/commands/map.d.ts +14 -5
- package/dist/esm/commands/map.js +2 -0
- package/dist/esm/commands/map.js.map +1 -1
- package/dist/esm/commands/path.d.ts +14 -0
- package/dist/esm/commands/path.js +33 -0
- package/dist/esm/commands/path.js.map +1 -0
- package/dist/esm/commands/peek.d.ts +16 -1
- package/dist/esm/commands/peek.js +35 -9
- package/dist/esm/commands/peek.js.map +1 -1
- package/dist/esm/commands/related.d.ts +16 -3
- package/dist/esm/commands/related.js +45 -28
- package/dist/esm/commands/related.js.map +1 -1
- package/dist/esm/commands/scope.d.ts +1 -0
- package/dist/esm/commands/scope.js +58 -0
- package/dist/esm/commands/scope.js.map +1 -1
- package/dist/esm/commands/search.d.ts +26 -3
- package/dist/esm/commands/search.js +85 -18
- package/dist/esm/commands/search.js.map +1 -1
- package/dist/esm/commands/signals.d.ts +4 -3
- package/dist/esm/commands/signals.js +11 -4
- package/dist/esm/commands/signals.js.map +1 -1
- package/dist/esm/commands/snapshot.d.ts +3 -0
- package/dist/esm/commands/snapshot.js +25 -0
- package/dist/esm/commands/snapshot.js.map +1 -0
- package/dist/esm/embed/cohere.js +35 -10
- package/dist/esm/embed/cohere.js.map +1 -1
- package/dist/esm/embed/endpoint.d.ts +2 -0
- package/dist/esm/embed/endpoint.js +11 -0
- package/dist/esm/embed/endpoint.js.map +1 -0
- package/dist/esm/embed/handoff.d.ts +7 -2
- package/dist/esm/embed/handoff.js +7 -0
- package/dist/esm/embed/handoff.js.map +1 -1
- package/dist/esm/embed/http.d.ts +18 -1
- package/dist/esm/embed/http.js +137 -17
- package/dist/esm/embed/http.js.map +1 -1
- package/dist/esm/embed/openai.js +34 -12
- package/dist/esm/embed/openai.js.map +1 -1
- package/dist/esm/embed/query.d.ts +9 -3
- package/dist/esm/embed/query.js +98 -26
- package/dist/esm/embed/query.js.map +1 -1
- package/dist/esm/embed/registry.d.ts +2 -2
- package/dist/esm/embed/registry.js +34 -5
- package/dist/esm/embed/registry.js.map +1 -1
- package/dist/esm/embed/static.js +10 -2
- package/dist/esm/embed/static.js.map +1 -1
- package/dist/esm/embed/store.js +30 -21
- package/dist/esm/embed/store.js.map +1 -1
- package/dist/esm/embed/types.d.ts +5 -2
- package/dist/esm/embed/types.js.map +1 -1
- package/dist/esm/features/embed.js +82 -21
- package/dist/esm/features/embed.js.map +1 -1
- package/dist/esm/graph/traverse.d.ts +2 -2
- package/dist/esm/graph/traverse.js +4 -2
- package/dist/esm/graph/traverse.js.map +1 -1
- package/dist/esm/index.d.ts +2 -2
- package/dist/esm/index.js +1 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/output/output.d.ts +4 -36
- package/dist/esm/output/output.js +1 -2
- package/dist/esm/output/output.js.map +1 -1
- package/dist/esm/scan/pool.d.ts +2 -1
- package/dist/esm/scan/pool.js +80 -23
- package/dist/esm/scan/pool.js.map +1 -1
- package/dist/esm/scan/reparse.d.ts +1 -0
- package/dist/esm/scan/reparse.js +40 -12
- package/dist/esm/scan/reparse.js.map +1 -1
- package/dist/esm/store/builder.d.ts +11 -21
- package/dist/esm/store/builder.js +337 -133
- package/dist/esm/store/builder.js.map +1 -1
- package/dist/esm/store/duckdb/connection.js +9 -1
- package/dist/esm/store/duckdb/connection.js.map +1 -1
- package/dist/esm/store/duckdb/open.d.ts +1 -1
- package/dist/esm/store/duckdb/open.js +13 -9
- package/dist/esm/store/duckdb/open.js.map +1 -1
- package/dist/esm/store/duckdb/reconcile.js +145 -10
- package/dist/esm/store/duckdb/reconcile.js.map +1 -1
- package/dist/esm/store/embed-scope.js +16 -6
- package/dist/esm/store/embed-scope.js.map +1 -1
- package/dist/esm/store/index.d.ts +6 -0
- package/dist/esm/store/index.js +10 -4
- package/dist/esm/store/index.js.map +1 -1
- package/dist/esm/store/open.d.ts +6 -1
- package/dist/esm/store/open.js +77 -87
- package/dist/esm/store/open.js.map +1 -1
- package/dist/esm/store/readiness.d.ts +5 -0
- package/dist/esm/store/readiness.js +11 -0
- package/dist/esm/store/readiness.js.map +1 -0
- package/dist/esm/store/reconcile.d.ts +26 -1
- package/dist/esm/store/reconcile.js +142 -120
- package/dist/esm/store/reconcile.js.map +1 -1
- package/dist/esm/store/signature.d.ts +1 -0
- package/dist/esm/store/signature.js +8 -0
- package/dist/esm/store/signature.js.map +1 -1
- package/dist/esm/store/sqlite/open.d.ts +1 -1
- package/dist/esm/store/sqlite/open.js +12 -12
- package/dist/esm/store/sqlite/open.js.map +1 -1
- package/dist/esm/store/sqlite/reconcile.js +2 -2
- package/dist/esm/store/sqlite/reconcile.js.map +1 -1
- package/dist/esm/store/sqlite/store.js +21 -18
- package/dist/esm/store/sqlite/store.js.map +1 -1
- package/dist/esm/store/sqlite/vectors.d.ts +2 -2
- package/dist/esm/store/sqlite/vectors.js +7 -5
- package/dist/esm/store/sqlite/vectors.js.map +1 -1
- package/dist/esm/store/stages.js.map +1 -1
- package/dist/esm/store/turso/connection.d.ts +1 -0
- package/dist/esm/store/turso/connection.js +97 -28
- package/dist/esm/store/turso/connection.js.map +1 -1
- package/dist/esm/store/turso/open.d.ts +1 -1
- package/dist/esm/store/turso/open.js +9 -8
- package/dist/esm/store/turso/open.js.map +1 -1
- package/dist/esm/store/turso/store.js +16 -14
- package/dist/esm/store/turso/store.js.map +1 -1
- package/dist/esm/store/turso/vectors.d.ts +2 -2
- package/dist/esm/store/turso/vectors.js +27 -17
- package/dist/esm/store/turso/vectors.js.map +1 -1
- package/dist/esm/store/types.d.ts +4 -1
- package/dist/esm/store/types.js.map +1 -1
- package/dist/esm/store/vectors.d.ts +14 -1
- package/dist/esm/store/vectors.js +55 -0
- package/dist/esm/store/vectors.js.map +1 -1
- package/dist/esm/watch.d.ts +6 -0
- package/dist/esm/watch.js +40 -14
- package/dist/esm/watch.js.map +1 -1
- package/dist/esm/workers/parse.d.ts +4 -3
- package/dist/esm/workers/parse.js +19 -17
- package/dist/esm/workers/parse.js.map +1 -1
- package/package.json +1 -1
- package/skills/sense/SKILL.md +4 -2
- package/skills/sense/references/stores/duckdb.md +1 -1
- package/skills/sense-setup/SKILL.md +2 -2
- package/skills/sense-setup/references/embeddings.md +1 -1
- package/skills/sense-setup/references/store-benchmarks.md +5 -5
- package/skills/sense-setup/references/stores/duckdb.md +1 -1
package/dist/esm/scan/reparse.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { availableParallelism } from 'node:os';
|
|
2
2
|
import { parseFile } from './index.js';
|
|
3
3
|
import { ParsePool } from './pool.js';
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
const
|
|
4
|
+
// The 2026-09-26 comparison kept ~326 KiB updates serial and 5.86 MiB work pooled.
|
|
5
|
+
// Companion byte shares prevent one dominant file from selecting workers for a tiny remainder.
|
|
6
|
+
export const PARSE_BYTES_PER_WORKER = 512 * 1024;
|
|
7
7
|
const DEFAULT_MAX_WORKERS = 8;
|
|
8
8
|
// Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and
|
|
9
9
|
// the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).
|
|
@@ -25,24 +25,52 @@ function reparseSerial(files, features, cfg, onParsed) {
|
|
|
25
25
|
async function reparsePooled(files, features, cfg, onParsed, maxWorkers, pool) {
|
|
26
26
|
if (pool) return pool.run(files, features, cfg, onParsed, maxWorkers);
|
|
27
27
|
const ephemeral = new ParsePool();
|
|
28
|
+
let results;
|
|
29
|
+
let dispatchError;
|
|
30
|
+
let dispatchFailed = false;
|
|
28
31
|
try {
|
|
29
|
-
|
|
30
|
-
await ephemeral.close();
|
|
31
|
-
return results;
|
|
32
|
+
results = await ephemeral.run(files, features, cfg, onParsed, maxWorkers);
|
|
32
33
|
} catch (err) {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
34
|
+
dispatchFailed = true;
|
|
35
|
+
dispatchError = err;
|
|
36
|
+
}
|
|
37
|
+
try {
|
|
38
|
+
await ephemeral.close();
|
|
39
|
+
} catch (closeError) {
|
|
40
|
+
if (dispatchFailed) throw new AggregateError([
|
|
41
|
+
dispatchError,
|
|
42
|
+
closeError
|
|
43
|
+
], 'parse dispatch and pool close failed');
|
|
44
|
+
throw closeError;
|
|
45
|
+
}
|
|
46
|
+
if (dispatchFailed) throw dispatchError;
|
|
47
|
+
if (results === undefined) throw new Error('parse dispatch finished without results');
|
|
48
|
+
return results;
|
|
49
|
+
}
|
|
50
|
+
function parseWorkerCount(files, bytesPerWorker, maxWorkers) {
|
|
51
|
+
if (files.length === 0) return 0;
|
|
52
|
+
const available = Math.max(1, Math.min(maxWorkers, files.length));
|
|
53
|
+
if (bytesPerWorker === 0) return available;
|
|
54
|
+
let totalBytes = 0;
|
|
55
|
+
let largestBytes = 0;
|
|
56
|
+
for (const file of files){
|
|
57
|
+
totalBytes += file.size;
|
|
58
|
+
largestBytes = Math.max(largestBytes, file.size);
|
|
37
59
|
}
|
|
60
|
+
const aggregateWorkers = Math.floor(totalBytes / bytesPerWorker);
|
|
61
|
+
// The largest file is indivisible, so extra workers need full shares among the remainder.
|
|
62
|
+
const companionWorkers = 1 + Math.floor((totalBytes - largestBytes) / bytesPerWorker);
|
|
63
|
+
return Math.min(available, aggregateWorkers, companionWorkers);
|
|
38
64
|
}
|
|
39
65
|
// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick
|
|
40
66
|
// call; pass one in to keep progress reporting working without this module owning a reporter.
|
|
41
67
|
export async function reparseFiles(files, features, cfg, knownColumns, onParsed, options = {}) {
|
|
42
68
|
var _options_threshold, _options_maxWorkers;
|
|
43
|
-
const threshold = (_options_threshold = options.threshold) !== null && _options_threshold !== void 0 ? _options_threshold :
|
|
69
|
+
const threshold = (_options_threshold = options.threshold) !== null && _options_threshold !== void 0 ? _options_threshold : PARSE_BYTES_PER_WORKER;
|
|
44
70
|
const maxWorkers = (_options_maxWorkers = options.maxWorkers) !== null && _options_maxWorkers !== void 0 ? _options_maxWorkers : Math.min(DEFAULT_MAX_WORKERS, availableParallelism());
|
|
45
|
-
const
|
|
71
|
+
const workerCount = parseWorkerCount(files, threshold, maxWorkers);
|
|
72
|
+
const pooled = files.length > 0 && (threshold === 0 || workerCount >= 2);
|
|
73
|
+
const results = pooled ? await reparsePooled(files, features, cfg, onParsed, workerCount, options.pool) : reparseSerial(files, features, cfg, onParsed);
|
|
46
74
|
const docs = [];
|
|
47
75
|
const warnings = [];
|
|
48
76
|
const newColumns = [];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { availableParallelism } from 'node:os';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport type { FileResult } from './pool.ts';\nimport { ParsePool } from './pool.ts';\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving:\n// one pass over `files`, pushed in order.\nexport interface ReparseResult {\n docs: ParsedDoc[];\n warnings: string[];\n // Frontmatter keys not in `knownColumns`, first-seen order -- the order callers ALTER TABLE ADD COLUMN in.\n newColumns: string[];\n // Sum of worker-side parseFile time (workers/parse.ts) across files; 0 on the serial path.\n workerParseMs: number;\n}\n\nexport interface ReparseOptions {\n // Overrides
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { availableParallelism } from 'node:os';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport type { FileResult } from './pool.ts';\nimport { ParsePool } from './pool.ts';\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving:\n// one pass over `files`, pushed in order.\nexport interface ReparseResult {\n docs: ParsedDoc[];\n warnings: string[];\n // Frontmatter keys not in `knownColumns`, first-seen order -- the order callers ALTER TABLE ADD COLUMN in.\n newColumns: string[];\n // Sum of worker-side parseFile time (workers/parse.ts) across files; 0 on the serial path.\n workerParseMs: number;\n}\n\nexport interface ReparseOptions {\n // Overrides PARSE_BYTES_PER_WORKER below. Zero forces worker dispatch in tests.\n // Internal: no caller in src/ passes it.\n threshold?: number;\n // Overrides DEFAULT_MAX_WORKERS below.\n maxWorkers?: number;\n // A builder-owned pool to dispatch through instead of creating and destroying one for this\n // call alone (see store/builder.ts). Absent, reparseFiles manages its own ephemeral pool.\n pool?: ParsePool;\n}\n\n// The 2026-09-26 comparison kept ~326 KiB updates serial and 5.86 MiB work pooled.\n// Companion byte shares prevent one dominant file from selecting workers for a tiny remainder.\nexport const PARSE_BYTES_PER_WORKER = 512 * 1024;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and\n// the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n const results: FileResult[] = [];\n let done = 0;\n for (const file of files) {\n const fileFeatures = featuresForFile(features, cfg, file);\n results.push(parseFile(file, fileFeatures, cfg));\n onParsed?.(++done);\n }\n return results;\n}\n\n// Dispatches through `pool` when the caller owns one (a builder, kept alive across calls); with\n// none given, opens an ephemeral ParsePool for this call alone and always closes it.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number, pool: ParsePool | undefined): Promise<FileResult[]> {\n if (pool) return pool.run(files, features, cfg, onParsed, maxWorkers);\n const ephemeral = new ParsePool();\n let results: FileResult[] | undefined;\n let dispatchError: unknown;\n let dispatchFailed = false;\n try {\n results = await ephemeral.run(files, features, cfg, onParsed, maxWorkers);\n } catch (err) {\n dispatchFailed = true;\n dispatchError = err;\n }\n try {\n await ephemeral.close();\n } catch (closeError) {\n if (dispatchFailed) throw new AggregateError([dispatchError, closeError], 'parse dispatch and pool close failed');\n throw closeError;\n }\n if (dispatchFailed) throw dispatchError;\n if (results === undefined) throw new Error('parse dispatch finished without results');\n return results;\n}\n\nfunction parseWorkerCount(files: FileStat[], bytesPerWorker: number, maxWorkers: number): number {\n if (files.length === 0) return 0;\n const available = Math.max(1, Math.min(maxWorkers, files.length));\n if (bytesPerWorker === 0) return available;\n let totalBytes = 0;\n let largestBytes = 0;\n for (const file of files) {\n totalBytes += file.size;\n largestBytes = Math.max(largestBytes, file.size);\n }\n const aggregateWorkers = Math.floor(totalBytes / bytesPerWorker);\n // The largest file is indivisible, so extra workers need full shares among the remainder.\n const companionWorkers = 1 + Math.floor((totalBytes - largestBytes) / bytesPerWorker);\n return Math.min(available, aggregateWorkers, companionWorkers);\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n const threshold = options.threshold ?? PARSE_BYTES_PER_WORKER;\n const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n const workerCount = parseWorkerCount(files, threshold, maxWorkers);\n const pooled = files.length > 0 && (threshold === 0 || workerCount >= 2);\n const results = pooled ? await reparsePooled(files, features, cfg, onParsed, workerCount, options.pool) : reparseSerial(files, features, cfg, onParsed);\n\n const docs: ParsedDoc[] = [];\n const warnings: string[] = [];\n const newColumns: string[] = [];\n const seen = new Set(knownColumns);\n let workerParseMs = 0;\n for (const { doc, warnings: fileWarnings, parseMs } of results) {\n warnings.push(...fileWarnings);\n workerParseMs += parseMs ?? 0;\n for (const key of Object.keys(doc.data)) {\n if (!seen.has(key)) {\n seen.add(key);\n newColumns.push(key);\n }\n }\n docs.push(doc);\n }\n\n return { docs, warnings, newColumns, workerParseMs };\n}\n"],"names":["availableParallelism","parseFile","ParsePool","PARSE_BYTES_PER_WORKER","DEFAULT_MAX_WORKERS","featuresForFile","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","reparsePooled","maxWorkers","pool","run","ephemeral","dispatchError","dispatchFailed","err","close","closeError","AggregateError","undefined","Error","parseWorkerCount","bytesPerWorker","length","available","Math","max","min","totalBytes","largestBytes","size","aggregateWorkers","floor","companionWorkers","reparseFiles","knownColumns","options","threshold","workerCount","pooled","docs","warnings","newColumns","seen","Set","workerParseMs","doc","fileWarnings","parseMs","key","Object","keys","data","has","add"],"mappings":"AAAA,SAASA,oBAAoB,QAAQ,UAAU;AAI/C,SAASC,SAAS,QAAQ,aAAa;AAGvC,SAASC,SAAS,QAAQ,YAAY;AAwBtC,mFAAmF;AACnF,+FAA+F;AAC/F,OAAO,MAAMC,yBAAyB,MAAM,KAAK;AACjD,MAAMC,sBAAsB;AAE5B,+FAA+F;AAC/F,kGAAkG;AAClG,OAAO,SAASC,gBAAgBC,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,CAACC,UAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,MAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;IACX,KAAK,MAAMR,QAAQK,MAAO;QACxB,MAAMI,eAAeZ,gBAAgBC,UAAUC,KAAKC;QACpDO,QAAQG,IAAI,CAACjB,UAAUO,MAAMS,cAAcV;QAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;IACf;IACA,OAAOD;AACT;AAEA,gGAAgG;AAChG,qFAAqF;AACrF,eAAeI,cAAcN,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEM,UAAkB,EAAEC,IAA2B;IAC/K,IAAIA,MAAM,OAAOA,KAAKC,GAAG,CAACT,OAAOP,UAAUC,KAAKO,UAAUM;IAC1D,MAAMG,YAAY,IAAIrB;IACtB,IAAIa;IACJ,IAAIS;IACJ,IAAIC,iBAAiB;IACrB,IAAI;QACFV,UAAU,MAAMQ,UAAUD,GAAG,CAACT,OAAOP,UAAUC,KAAKO,UAAUM;IAChE,EAAE,OAAOM,KAAK;QACZD,iBAAiB;QACjBD,gBAAgBE;IAClB;IACA,IAAI;QACF,MAAMH,UAAUI,KAAK;IACvB,EAAE,OAAOC,YAAY;QACnB,IAAIH,gBAAgB,MAAM,IAAII,eAAe;YAACL;YAAeI;SAAW,EAAE;QAC1E,MAAMA;IACR;IACA,IAAIH,gBAAgB,MAAMD;IAC1B,IAAIT,YAAYe,WAAW,MAAM,IAAIC,MAAM;IAC3C,OAAOhB;AACT;AAEA,SAASiB,iBAAiBnB,KAAiB,EAAEoB,cAAsB,EAAEb,UAAkB;IACrF,IAAIP,MAAMqB,MAAM,KAAK,GAAG,OAAO;IAC/B,MAAMC,YAAYC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,GAAG,CAAClB,YAAYP,MAAMqB,MAAM;IAC/D,IAAID,mBAAmB,GAAG,OAAOE;IACjC,IAAII,aAAa;IACjB,IAAIC,eAAe;IACnB,KAAK,MAAMhC,QAAQK,MAAO;QACxB0B,cAAc/B,KAAKiC,IAAI;QACvBD,eAAeJ,KAAKC,GAAG,CAACG,cAAchC,KAAKiC,IAAI;IACjD;IACA,MAAMC,mBAAmBN,KAAKO,KAAK,CAACJ,aAAaN;IACjD,0FAA0F;IAC1F,MAAMW,mBAAmB,IAAIR,KAAKO,KAAK,CAAC,AAACJ,CAAAA,aAAaC,YAAW,IAAKP;IACtE,OAAOG,KAAKE,GAAG,CAACH,WAAWO,kBAAkBE;AAC/C;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAeC,aAAahC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEuC,YAAiC,EAAEhC,QAAiC,EAAEiC,UAA0B,CAAC,CAAC;QACtKA,oBACCA;IADnB,MAAMC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqB5C;IACvC,MAAMiB,cAAa2B,sBAAAA,QAAQ3B,UAAU,cAAlB2B,iCAAAA,sBAAsBX,KAAKE,GAAG,CAAClC,qBAAqBJ;IACvE,MAAMiD,cAAcjB,iBAAiBnB,OAAOmC,WAAW5B;IACvD,MAAM8B,SAASrC,MAAMqB,MAAM,GAAG,KAAMc,CAAAA,cAAc,KAAKC,eAAe,CAAA;IACtE,MAAMlC,UAAUmC,SAAS,MAAM/B,cAAcN,OAAOP,UAAUC,KAAKO,UAAUmC,aAAaF,QAAQ1B,IAAI,IAAIT,cAAcC,OAAOP,UAAUC,KAAKO;IAE9I,MAAMqC,OAAoB,EAAE;IAC5B,MAAMC,WAAqB,EAAE;IAC7B,MAAMC,aAAuB,EAAE;IAC/B,MAAMC,OAAO,IAAIC,IAAIT;IACrB,IAAIU,gBAAgB;IACpB,KAAK,MAAM,EAAEC,GAAG,EAAEL,UAAUM,YAAY,EAAEC,OAAO,EAAE,IAAI5C,QAAS;QAC9DqC,SAASlC,IAAI,IAAIwC;QACjBF,iBAAiBG,oBAAAA,qBAAAA,UAAW;QAC5B,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACL,IAAIM,IAAI,EAAG;YACvC,IAAI,CAACT,KAAKU,GAAG,CAACJ,MAAM;gBAClBN,KAAKW,GAAG,CAACL;gBACTP,WAAWnC,IAAI,CAAC0C;YAClB;QACF;QACAT,KAAKjC,IAAI,CAACuC;IACZ;IAEA,OAAO;QAAEN;QAAMC;QAAUC;QAAYG;IAAc;AACrD"}
|
|
@@ -1,27 +1,17 @@
|
|
|
1
|
-
import { type
|
|
2
|
-
import type { EmbedChangeKind } from './embed-scope.js';
|
|
3
|
-
import type { FeatureToggle } from './feature-scope.js';
|
|
4
|
-
import { type ReconcilePaths } from './reconcile.js';
|
|
1
|
+
import { type ResolvedConfig } from '../config/index.js';
|
|
5
2
|
import type { Stages } from './stages.js';
|
|
6
3
|
import type { Connection, ReconcileDialect } from './types.js';
|
|
4
|
+
interface BuildResult {
|
|
5
|
+
parsed: number;
|
|
6
|
+
warnings: string[];
|
|
7
|
+
stages: Stages;
|
|
8
|
+
messages: string[];
|
|
9
|
+
}
|
|
7
10
|
export interface Builder {
|
|
8
|
-
build(
|
|
9
|
-
parsed: number;
|
|
10
|
-
warnings: string[];
|
|
11
|
-
stages: Stages;
|
|
12
|
-
}>;
|
|
13
|
-
recover(): Promise<{
|
|
14
|
-
parsed: number;
|
|
15
|
-
warnings: string[];
|
|
16
|
-
stages: Stages;
|
|
17
|
-
}>;
|
|
18
|
-
invalidate(kind: EmbedChangeKind): Promise<{
|
|
19
|
-
parsed: number;
|
|
20
|
-
}>;
|
|
21
|
-
invalidateFeatures(toggles: FeatureToggle[]): Promise<{
|
|
22
|
-
parsed: number;
|
|
23
|
-
}>;
|
|
11
|
+
build(): Promise<BuildResult>;
|
|
24
12
|
close(): Promise<void>;
|
|
25
13
|
readonly poolsCreated: number;
|
|
26
14
|
}
|
|
27
|
-
|
|
15
|
+
type EnsureFeatureSchema = (cfg: ResolvedConfig) => Promise<void>;
|
|
16
|
+
export declare function createBuilder(conn: Connection, cfg: ResolvedConfig, dialect: ReconcileDialect, ensureFeatureSchema: EnsureFeatureSchema): Builder;
|
|
17
|
+
export {};
|