sensemaking 0.15.4 → 0.17.0

package/dist/esm/chunk/group.js

@@ -0,0 +1,253 @@
+import { UNSPACED_SCRIPTS } from '../text/segment.js';
+import { extractText } from './extract.js';
+import { parse } from './parse.js';
+export const DEFAULT_TARGET_TOKENS = 500;
+const PGC_GROUP_SIZE = 2;
+const OVERSIZE_TRIGGER_MULTIPLE = 2;
+// D5: segment.ts's unspaced-script set packs close to one token per character; Hangul is
+// spaced but still token-dense, added here as a conservative bound (one Unicode set, one home).
+const DENSE_SCRIPT = new RegExp(`[${UNSPACED_SCRIPTS}\\p{scx=Hangul}]`, 'u');
+// D5's size estimate: dense-script graphemes 1:1, everything else at 4 chars/token.
+export function estimateTokens(text) {
+    let dense = 0;
+    let other = 0;
+    for (const ch of text){
+        if (DENSE_SCRIPT.test(ch)) dense++;
+        else other++;
+    }
+    return dense + other / 4;
+}
+function resolveOptions(opts) {
+    var _ref, _ref1;
+    return {
+        targetTokens: (_ref = opts === null || opts === void 0 ? void 0 : opts.targetTokens) !== null && _ref !== void 0 ? _ref : DEFAULT_TARGET_TOKENS,
+        text: (_ref1 = opts === null || opts === void 0 ? void 0 : opts.text) !== null && _ref1 !== void 0 ? _ref1 : 'raw'
+    };
+}
+// A heading of any depth ends the current scope and starts a new one (D1); the heading block
+// itself is carried into the new scope, where it joins that scope's first group (F7/F10).
+function splitScopes(blocks) {
+    const scopes = [];
+    let current = [];
+    for (const block of blocks){
+        if (block.type === 'heading' && current.length > 0) {
+            scopes.push(current);
+            current = [];
+        }
+        current.push(block);
+    }
+    if (current.length > 0) scopes.push(current);
+    return scopes;
+}
+function finalize(parts) {
+    if (parts.length === 0) return undefined;
+    const first = parts[0];
+    const last = parts[parts.length - 1];
+    return {
+        startLine: first.startLine,
+        endLine: last.endLine,
+        text: parts.map((p)=>p.text).join('\n'),
+        final: parts.some((p)=>p.final)
+    };
+}
+const NEWLINE_TOKENS = estimateTokens('\n');
+const SENTENCE_SEGMENTER = new Intl.Segmenter(undefined, {
+    granularity: 'sentence'
+});
+const WORD_SEGMENTER = new Intl.Segmenter(undefined, {
+    granularity: 'word'
+});
+function segmentsOf(text, segmenter) {
+    return Array.from(segmenter.segment(text), (s)=>s.segment);
+}
+// Greedily packs segments (already contiguous, tiling the source text with no gaps) into groups
+// of at most `working` estimated tokens; a lone segment over `working` still stands alone.
+function pack(segments, working) {
+    const groups = [];
+    let current = '';
+    let tokens = 0;
+    for (const segment of segments){
+        const segmentTokens = estimateTokens(segment);
+        if (current.length > 0 && tokens + segmentTokens > working) {
+            groups.push(current);
+            current = '';
+            tokens = 0;
+        }
+        current += segment;
+        tokens += segmentTokens;
+    }
+    if (current.length > 0) groups.push(current);
+    return groups;
+}
+// A single line's text, still over working after rule 5's line-split can't reduce a lone line
+// (the CJK case, one sentence-dense line with no newlines), falls back to Intl.Segmenter sentence
+// boundaries, then word boundaries for any one sentence still over working. Never splits inside a
+// grapheme either way (both are ECMA-402, the same proven component segment.ts uses for
+// graphemes). Mode-agnostic: the caller decides whether `text` is the raw line or its extracted
+// flavor -- this only packs whatever text it is given.
+function splitLineText(text, working) {
+    const sentences = segmentsOf(text, SENTENCE_SEGMENTER);
+    const out = [];
+    let current = '';
+    let tokens = 0;
+    const flush = ()=>{
+        if (current.length > 0) {
+            out.push(current);
+            current = '';
+            tokens = 0;
+        }
+    };
+    for (const sentence of sentences){
+        const sentenceTokens = estimateTokens(sentence);
+        if (sentenceTokens > working) {
+            flush();
+            out.push(...pack(segmentsOf(sentence, WORD_SEGMENTER), working));
+            continue;
+        }
+        if (current.length > 0 && tokens + sentenceTokens > working) flush();
+        current += sentence;
+        tokens += sentenceTokens;
+    }
+    flush();
+    return out;
+}
+const ATOMIC_TYPES = new Set([
+    'code',
+    'table',
+    'list'
+]);
+// An atomic block's (code/table/list) piece is always a raw line slice: re-parsing a table's
+// later pieces without their header/delimiter rows demotes them to paragraph text. A non-atomic
+// piece is a raw line slice too when raw mode requested it; extracted mode re-parses and resolves
+// flavor as before.
+function piece(pieceLines, startLine, endLine, blockType, textMode) {
+    const text = ATOMIC_TYPES.has(blockType) || textMode === 'raw' ? pieceLines.join('\n') : parse(pieceLines.join('\n')).map((b)=>extractText(b.node)).join('\n');
+    return {
+        startLine,
+        endLine,
+        text
+    };
+}
+// A piece over working size that came from exactly one source line cannot be reduced by another
+// line-boundary pass (rule 5's gap); it is sliced instead, at sentence then word boundaries, over
+// the mode-resolved text piece() already produced -- so raw mode packs the raw line and extracted
+// mode packs its flavor-resolved text. Extents collapse to that one line for every sub-piece (F5:
+// re-derivation is by index over chunk()'s own deterministic output, not by a unique line range
+// per piece); `final` flags each sub-piece so group() knows not to re-derive its text from that
+// shared extent.
+function finalizePiece(pieceLines, startLine, endLine, working, blockType, textMode) {
+    const p = piece(pieceLines, startLine, endLine, blockType, textMode);
+    if (pieceLines.length === 1 && estimateTokens(p.text) > working) {
+        return splitLineText(p.text, working).map((text)=>({
+                startLine,
+                endLine,
+                text,
+                final: true
+            }));
+    }
+    return [
+        p
+    ];
+}
+// A block over 2x working size splits at line boundaries into pieces each <= working size,
+// never mid-line. `seed`: pending tokens (a heading) the first piece must join, checked against
+// the limit too.
+function splitOversizeBlock(lines, startLine, endLine, working, blockType, textMode, seed = 0) {
+    const pieces = [];
+    let pieceLines = [];
+    let pieceStart = startLine;
+    let tokens = seed;
+    for(let line = startLine; line <= endLine; line++){
+        const lineText = lines[line - 1];
+        const sep = pieceLines.length > 0 || tokens > 0 ? NEWLINE_TOKENS : 0;
+        const lineTokens = estimateTokens(lineText);
+        if (pieceLines.length > 0 && tokens + sep + lineTokens > working) {
+            pieces.push(...finalizePiece(pieceLines, pieceStart, line - 1, working, blockType, textMode));
+            pieceLines = [];
+            tokens = 0;
+            pieceStart = line;
+            pieceLines.push(lineText);
+            tokens += lineTokens;
+            continue;
+        }
+        pieceLines.push(lineText);
+        tokens += sep + lineTokens;
+    }
+    if (pieceLines.length > 0) pieces.push(...finalizePiece(pieceLines, pieceStart, endLine, working, blockType, textMode));
+    return pieces;
+}
+// One heading scope's groups (D1): a heading opens the first group, and an oversize block
+// (rule 5, including an oversize heading) splits into pieces that each close their own group.
+function groupScope(scopeBlocks, lines, resolved) {
+    const working = resolved.targetTokens;
+    const trigger = working * OVERSIZE_TRIGGER_MULTIPLE;
+    const finished = [];
+    let parts = [];
+    let paragraphCount = 0;
+    let tokens = 0;
+    function close() {
+        const group = finalize(parts);
+        if (group) finished.push(group);
+        parts = [];
+        paragraphCount = 0;
+        tokens = 0;
+    }
+    // tokens tracks the active text mode's own estimate (a newline between parts costs
+    // NEWLINE_TOKENS too), so packing decisions size the text the chunk will actually ship as.
+    function addPart(text, sizeText, startLine, endLine) {
+        tokens += (parts.length > 0 ? NEWLINE_TOKENS : 0) + estimateTokens(sizeText);
+        parts.push({
+            startLine,
+            endLine,
+            text
+        });
+    }
+    for (const block of scopeBlocks){
+        const raw = lines.slice(block.startLine - 1, block.endLine).join('\n');
+        const blockTokens = estimateTokens(raw);
+        if (blockTokens > trigger) {
+            const seed = parts.length > 0 ? tokens : 0;
+            for (const p of splitOversizeBlock(lines, block.startLine, block.endLine, working, block.type, resolved.text, seed)){
+                parts.push(p);
+                close();
+            }
+            continue;
+        }
+        const extracted = extractText(block.node);
+        const sizeText = resolved.text === 'raw' ? raw : extracted;
+        if (block.type === 'heading') {
+            addPart(extracted, sizeText, block.startLine, block.endLine);
+            continue;
+        }
+        // The 2x-working invariant holds even under pgc's paper-faithful 2-paragraph pairing --
+        // close first if the pair about to form would cross it.
+        const pairOversize = parts.length > 0 && tokens + NEWLINE_TOKENS + blockTokens > trigger;
+        if (pairOversize) close();
+        addPart(extracted, sizeText, block.startLine, block.endLine);
+        paragraphCount++;
+        if (paragraphCount >= PGC_GROUP_SIZE) close();
+    }
+    close();
+    return finished;
+}
+// Groups already-parsed blocks per opts (D1/D3), against the same body the blocks were parsed
+// from (line lookups for oversize splitting).
+export function group(blocks, body, opts) {
+    const resolved = resolveOptions(opts);
+    const lines = body.split('\n');
+    const chunks = [];
+    for (const scope of splitScopes(blocks))chunks.push(...groupScope(scope, lines, resolved));
+    // 'raw': the chunk's own source lines verbatim, replacing the flavor-resolved join above (D9).
+    // A `final` chunk (a sub-line split) already carries its own slice's raw text -- re-slicing by
+    // extent would return the whole shared source line instead, once per sub-piece.
+    const texted = resolved.text === 'raw' ? chunks.map((c)=>c.final ? c : {
+            ...c,
+            text: lines.slice(c.startLine - 1, c.endLine).join('\n').trim()
+        }) : chunks;
+    // A group can be all-blank (flavor-stripped to nothing, or a raw slice of pure syntax); it never produces a chunk.
+    return texted.filter((c)=>c.text.trim().length > 0).map((c)=>({
+            startLine: c.startLine,
+            endLine: c.endLine,
+            text: c.text
+        }));
+}

package/dist/esm/chunk/group.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/group.ts"],"sourcesContent":["import { UNSPACED_SCRIPTS } from '../text/segment.ts';\nimport { extractText } from './extract.ts';\nimport { parse } from './parse.ts';\nimport type { Block, BlockType, Chunk, ChunkOptions } from './types.ts';\n\nexport const DEFAULT_TARGET_TOKENS = 500;\nconst PGC_GROUP_SIZE = 2;\nconst OVERSIZE_TRIGGER_MULTIPLE = 2;\n\n// D5: segment.ts's unspaced-script set packs close to one token per character; Hangul is\n// spaced but still token-dense, added here as a conservative bound (one Unicode set, one home).\nconst DENSE_SCRIPT = new RegExp(`[${UNSPACED_SCRIPTS}\\\\p{scx=Hangul}]`, 'u');\n\n// D5's size estimate: dense-script graphemes 1:1, everything else at 4 chars/token.\nexport function estimateTokens(text: string): number {\n  let dense = 0;\n  let other = 0;\n  for (const ch of text) {\n    if (DENSE_SCRIPT.test(ch)) dense++;\n    else other++;\n  }\n  return dense + other / 4;\n}\n\ninterface ResolvedOptions {\n  targetTokens: number;\n  text: 'extracted' | 'raw';\n}\n\nfunction resolveOptions(opts?: ChunkOptions): ResolvedOptions {\n  return {\n    targetTokens: opts?.targetTokens ?? DEFAULT_TARGET_TOKENS,\n    text: opts?.text ?? 'raw',\n  };\n}\n\n// A heading of any depth ends the current scope and starts a new one (D1); the heading block\n// itself is carried into the new scope, where it joins that scope's first group (F7/F10).\nfunction splitScopes(blocks: Block[]): Block[][] {\n  const scopes: Block[][] = [];\n  let current: Block[] = [];\n  for (const block of blocks) {\n    if (block.type === 'heading' && current.length > 0) {\n      scopes.push(current);\n      current = [];\n    }\n    current.push(block);\n  }\n  if (current.length > 0) scopes.push(current);\n  return scopes;\n}\n\ninterface Part {\n  startLine: number;\n  endLine: number;\n  text: string;\n  // true for a sub-line split piece (finalizePiece's splitLineText branch): its text is already\n  // the active mode's final text for its own slice of the line, not the whole line, so group()'s\n  // extent-based raw re-slice (startLine === endLine for every sibling sub-piece) must not touch it.\n  final?: boolean;\n}\n\nfunction finalize(parts: Part[]): (Chunk & { final?: boolean }) | undefined {\n  if (parts.length === 0) return undefined;\n  const first = parts[0];\n  const last = parts[parts.length - 1];\n  return { startLine: first.startLine, endLine: last.endLine, text: parts.map((p) => p.text).join('\\n'), final: parts.some((p) => p.final) };\n}\n\nconst NEWLINE_TOKENS = estimateTokens('\\n');\nconst SENTENCE_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'sentence' });\nconst WORD_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'word' });\n\nfunction segmentsOf(text: string, segmenter: Intl.Segmenter): string[] {\n  return Array.from(segmenter.segment(text), (s) => s.segment);\n}\n\n// Greedily packs segments (already contiguous, tiling the source text with no gaps) into groups\n// of at most `working` estimated tokens; a lone segment over `working` still stands alone.\nfunction pack(segments: string[], working: number): string[] {\n  const groups: string[] = [];\n  let current = '';\n  let tokens = 0;\n  for (const segment of segments) {\n    const segmentTokens = estimateTokens(segment);\n    if (current.length > 0 && tokens + segmentTokens > working) {\n      groups.push(current);\n      current = '';\n      tokens = 0;\n    }\n    current += segment;\n    tokens += segmentTokens;\n  }\n  if (current.length > 0) groups.push(current);\n  return groups;\n}\n\n// A single line's text, still over working after rule 5's line-split can't reduce a lone line\n// (the CJK case, one sentence-dense line with no newlines), falls back to Intl.Segmenter sentence\n// boundaries, then word boundaries for any one sentence still over working. Never splits inside a\n// grapheme either way (both are ECMA-402, the same proven component segment.ts uses for\n// graphemes). Mode-agnostic: the caller decides whether `text` is the raw line or its extracted\n// flavor -- this only packs whatever text it is given.\nfunction splitLineText(text: string, working: number): string[] {\n  const sentences = segmentsOf(text, SENTENCE_SEGMENTER);\n  const out: string[] = [];\n  let current = '';\n  let tokens = 0;\n  const flush = () => {\n    if (current.length > 0) {\n      out.push(current);\n      current = '';\n      tokens = 0;\n    }\n  };\n  for (const sentence of sentences) {\n    const sentenceTokens = estimateTokens(sentence);\n    if (sentenceTokens > working) {\n      flush();\n      out.push(...pack(segmentsOf(sentence, WORD_SEGMENTER), working));\n      continue;\n    }\n    if (current.length > 0 && tokens + sentenceTokens > working) flush();\n    current += sentence;\n    tokens += sentenceTokens;\n  }\n  flush();\n  return out;\n}\n\nconst ATOMIC_TYPES: ReadonlySet<BlockType> = new Set(['code', 'table', 'list']);\n\n// An atomic block's (code/table/list) piece is always a raw line slice: re-parsing a table's\n// later pieces without their header/delimiter rows demotes them to paragraph text. A non-atomic\n// piece is a raw line slice too when raw mode requested it; extracted mode re-parses and resolves\n// flavor as before.\nfunction piece(pieceLines: string[], startLine: number, endLine: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part {\n  const text =\n    ATOMIC_TYPES.has(blockType) || textMode === 'raw'\n      ? pieceLines.join('\\n')\n      : parse(pieceLines.join('\\n'))\n          .map((b) => extractText(b.node))\n          .join('\\n');\n  return { startLine, endLine, text };\n}\n\n// A piece over working size that came from exactly one source line cannot be reduced by another\n// line-boundary pass (rule 5's gap); it is sliced instead, at sentence then word boundaries, over\n// the mode-resolved text piece() already produced -- so raw mode packs the raw line and extracted\n// mode packs its flavor-resolved text. Extents collapse to that one line for every sub-piece (F5:\n// re-derivation is by index over chunk()'s own deterministic output, not by a unique line range\n// per piece); `final` flags each sub-piece so group() knows not to re-derive its text from that\n// shared extent.\nfunction finalizePiece(pieceLines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part[] {\n  const p = piece(pieceLines, startLine, endLine, blockType, textMode);\n  if (pieceLines.length === 1 && estimateTokens(p.text) > working) {\n    return splitLineText(p.text, working).map((text) => ({ startLine, endLine, text, final: true }));\n  }\n  return [p];\n}\n\n// A block over 2x working size splits at line boundaries into pieces each <= working size,\n// never mid-line. `seed`: pending tokens (a heading) the first piece must join, checked against\n// the limit too.\nfunction splitOversizeBlock(lines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw', seed = 0): Part[] {\n  const pieces: Part[] = [];\n  let pieceLines: string[] = [];\n  let pieceStart = startLine;\n  let tokens = seed;\n  for (let line = startLine; line <= endLine; line++) {\n    const lineText = lines[line - 1];\n    const sep = pieceLines.length > 0 || tokens > 0 ? NEWLINE_TOKENS : 0;\n    const lineTokens = estimateTokens(lineText);\n    if (pieceLines.length > 0 && tokens + sep + lineTokens > working) {\n      pieces.push(...finalizePiece(pieceLines, pieceStart, line - 1, working, blockType, textMode));\n      pieceLines = [];\n      tokens = 0;\n      pieceStart = line;\n      pieceLines.push(lineText);\n      tokens += lineTokens;\n      continue;\n    }\n    pieceLines.push(lineText);\n    tokens += sep + lineTokens;\n  }\n  if (pieceLines.length > 0) pieces.push(...finalizePiece(pieceLines, pieceStart, endLine, working, blockType, textMode));\n  return pieces;\n}\n\n// One heading scope's groups (D1): a heading opens the first group, and an oversize block\n// (rule 5, including an oversize heading) splits into pieces that each close their own group.\nfunction groupScope(scopeBlocks: Block[], lines: string[], resolved: ResolvedOptions): (Chunk & { final?: boolean })[] {\n  const working = resolved.targetTokens;\n  const trigger = working * OVERSIZE_TRIGGER_MULTIPLE;\n  const finished: (Chunk & { final?: boolean })[] = [];\n  let parts: Part[] = [];\n  let paragraphCount = 0;\n  let tokens = 0;\n\n  function close(): void {\n    const group = finalize(parts);\n    if (group) finished.push(group);\n    parts = [];\n    paragraphCount = 0;\n    tokens = 0;\n  }\n\n  // tokens tracks the active text mode's own estimate (a newline between parts costs\n  // NEWLINE_TOKENS too), so packing decisions size the text the chunk will actually ship as.\n  function addPart(text: string, sizeText: string, startLine: number, endLine: number): void {\n    tokens += (parts.length > 0 ? NEWLINE_TOKENS : 0) + estimateTokens(sizeText);\n    parts.push({ startLine, endLine, text });\n  }\n\n  for (const block of scopeBlocks) {\n    const raw = lines.slice(block.startLine - 1, block.endLine).join('\\n');\n    const blockTokens = estimateTokens(raw);\n\n    if (blockTokens > trigger) {\n      const seed = parts.length > 0 ? tokens : 0;\n      for (const p of splitOversizeBlock(lines, block.startLine, block.endLine, working, block.type, resolved.text, seed)) {\n        parts.push(p);\n        close();\n      }\n      continue;\n    }\n\n    const extracted = extractText(block.node);\n    const sizeText = resolved.text === 'raw' ? raw : extracted;\n\n    if (block.type === 'heading') {\n      addPart(extracted, sizeText, block.startLine, block.endLine);\n      continue;\n    }\n\n    // The 2x-working invariant holds even under pgc's paper-faithful 2-paragraph pairing --\n    // close first if the pair about to form would cross it.\n    const pairOversize = parts.length > 0 && tokens + NEWLINE_TOKENS + blockTokens > trigger;\n    if (pairOversize) close();\n\n    addPart(extracted, sizeText, block.startLine, block.endLine);\n    paragraphCount++;\n\n    if (paragraphCount >= PGC_GROUP_SIZE) close();\n  }\n  close();\n\n  return finished;\n}\n\n// Groups already-parsed blocks per opts (D1/D3), against the same body the blocks were parsed\n// from (line lookups for oversize splitting).\nexport function group(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[] {\n  const resolved = resolveOptions(opts);\n  const lines = body.split('\\n');\n  const chunks: (Chunk & { final?: boolean })[] = [];\n  for (const scope of splitScopes(blocks)) chunks.push(...groupScope(scope, lines, resolved));\n  // 'raw': the chunk's own source lines verbatim, replacing the flavor-resolved join above (D9).\n  // A `final` chunk (a sub-line split) already carries its own slice's raw text -- re-slicing by\n  // extent would return the whole shared source line instead, once per sub-piece.\n  const texted =\n    resolved.text === 'raw'\n      ? chunks.map((c) =>\n          c.final\n            ? c\n            : {\n                ...c,\n                text: lines\n                  .slice(c.startLine - 1, c.endLine)\n                  .join('\\n')\n                  .trim(),\n              }\n        )\n      : chunks;\n  // A group can be all-blank (flavor-stripped to nothing, or a raw slice of pure syntax); it never produces a chunk.\n  return texted.filter((c) => c.text.trim().length > 0).map((c) => ({ startLine: c.startLine, endLine: c.endLine, text: c.text }));\n}\n"],"names":["UNSPACED_SCRIPTS","extractText","parse","DEFAULT_TARGET_TOKENS","PGC_GROUP_SIZE","OVERSIZE_TRIGGER_MULTIPLE","DENSE_SCRIPT","RegExp","estimateTokens","text","dense","other","ch","test","resolveOptions","opts","targetTokens","splitScopes","blocks","scopes","current","block","type","length","push","finalize","parts","undefined","first","last","startLine","endLine","map","p","join","final","some","NEWLINE_TOKENS","SENTENCE_SEGMENTER","Intl","Segmenter","granularity","WORD_SEGMENTER","segmentsOf","segmenter","Array","from","segment","s","pack","segments","working","groups","tokens","segmentTokens","splitLineText","sentences","out","flush","sentence","sentenceTokens","ATOMIC_TYPES","Set","piece","pieceLines","blockType","textMode","has","b","node","finalizePiece","splitOversizeBlock","lines","seed","pieces","pieceStart","line","lineText","sep","lineTokens","groupScope","scopeBlocks","resolved","trigger","finished","paragraphCount","close","group","addPart","sizeText","raw","slice","blockTokens","extracted","pairOversize","body","split","chunks","scope","texted","c","trim","filter"],"mappings":"AAAA,SAASA,gBAAgB,QAAQ,qBAAqB;AACtD,SAASC,WAAW,QAAQ,eAAe;AAC3C,SAASC,KAAK,QAAQ,aAAa;AAGnC,OAAO,MAAMC,wBAAwB,IAAI;AACzC,MAAMC,iBAAiB;AACvB,MAAMC,4BAA4B;AAElC,yFAAyF;AACzF,gGAAgG;AAChG,MAAMC,eAAe,IAAIC,OAAO,CAAC,CAAC,EAAEP,iBAAiB,gBAAgB,CAAC,EAAE;AAExE,oFAAoF;AACpF,OAAO,SAASQ,eAAeC,IAAY;IACzC,IAAIC,QAAQ;IACZ,IAAIC,QAAQ;IACZ,KAAK,MAAMC,MAAMH,KAAM;QACrB,IAAIH,aAAaO,IAAI,CAACD,KAAKF;aACtBC;IACP;IACA,OAAOD,QAAQC,QAAQ;AACzB;AAOA,SAASG,eAAeC,IAAmB;;IACzC,OAAO;QACLC,YAAY,UAAED,iBAAAA,2BAAAA,KAAMC,YAAY,uCAAIb;QACpCM,IAAI,WAAEM,iBAAAA,2BAAAA,KAAMN,IAAI,yCAAI;IACtB;AACF;AAEA,6FAA6F;AAC7F,0FAA0F;AAC1F,SAASQ,YAAYC,MAAe;IAClC,MAAMC,SAAoB,EAAE;IAC5B,IAAIC,UAAmB,EAAE;IACzB,KAAK,MAAMC,SAASH,OAAQ;QAC1B,IAAIG,MAAMC,IAAI,KAAK,aAAaF,QAAQG,MAAM,GAAG,GAAG;YAClDJ,OAAOK,IAAI,CAACJ;YACZA,UAAU,EAAE;QACd;QACAA,QAAQI,IAAI,CAACH;IACf;IACA,IAAID,QAAQG,MAAM,GAAG,GAAGJ,OAAOK,IAAI,CAACJ;IACpC,OAAOD;AACT;AAYA,SAASM,SAASC,KAAa;IAC7B,IAAIA,MAAMH,MAAM,KAAK,GAAG,OAAOI;IAC/B,MAAMC,QAAQF,KAAK,CAAC,EAAE;IACtB,MAAMG,OAAOH,KAAK,CAACA,MAAMH,MAAM,GAAG,EAAE;IACpC,OAAO;QAAEO,WAAWF,MAAME,SAAS;QAAEC,SAASF,KAAKE,OAAO;QAAEtB,MAAMiB,MAAMM,GAAG,CAAC,CAACC,IAAMA,EAAExB,IAAI,EAAEyB,IAAI,CAAC;QAAOC,OAAOT,MAAMU,IAAI,CAAC,CAACH,IAAMA,EAAEE,KAAK;IAAE;AAC3I;AAEA,MAAME,iBAAiB7B,eAAe;AACtC,MAAM8B,qBAAqB,IAAIC,KAAKC,SAAS,CAACb,WAAW;IAAEc,aAAa;AAAW;AACnF,MAAMC,iBAAiB,IAAIH,KAAKC,SAAS,CAACb,WAAW;IAAEc,aAAa;AAAO;AAE3E,SAASE,WAAWlC,IAAY,EAAEmC,SAAyB;IACzD,OAAOC,MAAMC,IAAI,CAACF,UAAUG,OAAO,CAACtC,OAAO,CAACuC,IAAMA,EAAED,OAAO;AAC7D;AAEA,gGAAgG;AAChG,2FAA2F;AAC3F,SAASE,KAAKC,QAAkB,EAAEC,OAAe;IAC/C,MAAMC,SAAmB,EAAE;IAC3B,IAAIhC,UAAU;IACd,IAAIiC,SAAS;IACb,KAAK,MAAMN,WAAWG,SAAU;QAC9B,MAAMI,gBAAgB9C,eAAeuC;QACrC,IAAI3B,QAAQG,MAAM,GAAG,KAAK8B,SAASC,gBAAgBH,SAAS;YAC1DC,OAAO5B,IAAI,CAACJ;YACZA,UAAU;YACViC,SAAS;QACX;QACAjC,WAAW2B;QACXM,UAAUC;IACZ;IACA,IAAIlC,QAAQG,MAAM,GAAG,GAAG6B,OAAO5B,IAAI,CAACJ;IACpC,OAAOgC;AACT;AAEA,8FAA8F;AAC9F,kGAAkG;AAClG,kGAAkG;AAClG,wFAAwF;AACxF,gGAAgG;AAChG,uDAAuD;AACvD,SAASG,cAAc9C,IAAY,EAAE0C,OAAe;IAClD,MAAMK,YAAYb,WAAWlC,MAAM6B;IACnC,MAAMmB,MAAgB,EAAE;IACxB,IAAIrC,UAAU;IACd,IAAIiC,SAAS;IACb,MAAMK,QAAQ;QACZ,IAAItC,QAAQG,MAAM,GAAG,GAAG;YACtBkC,IAAIjC,IAAI,CAACJ;YACTA,UAAU;YACViC,SAAS;QACX;IACF;IACA,KAAK,MAAMM,YAAYH,UAAW;QAChC,MAAMI,iBAAiBpD,eAAemD;QACtC,IAAIC,iBAAiBT,SAAS;YAC5BO;YACAD,IAAIjC,IAAI,IAAIyB,KAAKN,WAAWgB,UAAUjB,iBAAiBS;YACvD;QACF;QACA,IAAI/B,QAAQG,MAAM,GAAG,KAAK8B,SAASO,iBAAiBT,SAASO;QAC7DtC,WAAWuC;QACXN,UAAUO;IACZ;IACAF;IACA,OAAOD;AACT;AAEA,MAAMI,eAAuC,IAAIC,IAAI;IAAC;IAAQ;IAAS;CAAO;AAE9E,6FAA6F;AAC7F,gGAAgG;AAChG,kGAAkG;AAClG,oBAAoB;AACpB,SAASC,MAAMC,UAAoB,EAAElC,SAAiB,EAAEC,OAAe,EAAEkC,SAAoB,EAAEC,QAA6B;IAC1H,MAAMzD,OACJoD,aAAaM,GAAG,CAACF,cAAcC,aAAa,QACxCF,WAAW9B,IAAI,CAAC,QAChBhC,MAAM8D,WAAW9B,IAAI,CAAC,OACnBF,GAAG,CAAC,CAACoC,IAAMnE,YAAYmE,EAAEC,IAAI,GAC7BnC,IAAI,CAAC;IACd,OAAO;QAAEJ;QAAWC;QAAStB;IAAK;AACpC;AAEA,gGAAgG;AAChG,kGAAkG;AAClG,kGAAkG;AAClG,kGAAkG;AAClG,gGAAgG;AAChG,gGAAgG;AAChG,iBAAiB;AACjB,SAAS6D,cAAcN,UAAoB,EAAElC,SAAiB,EAAEC,OAAe,EAAEoB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B;IACnJ,MAAMjC,IAAI8B,MAAMC,YAAYlC,WAAWC,SAASkC,WAAWC;IAC3D,IAAIF,WAAWzC,MAAM,KAAK,KAAKf,eAAeyB,EAAExB,IAAI,IAAI0C,SAAS;QAC/D,OAAOI,cAActB,EAAExB,IAAI,EAAE0C,SAASnB,GAAG,CAAC,CAACvB,OAAU,CAAA;gBAAEqB;gBAAWC;gBAAStB;gBAAM0B,OAAO;YAAK,CAAA;IAC/F;IACA,OAAO;QAACF;KAAE;AACZ;AAEA,2FAA2F;AAC3F,gGAAgG;AAChG,iBAAiB;AACjB,SAASsC,mBAAmBC,KAAe,EAAE1C,SAAiB,EAAEC,OAAe,EAAEoB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B,EAAEO,OAAO,CAAC;IAC7J,MAAMC,SAAiB,EAAE;IACzB,IAAIV,aAAuB,EAAE;IAC7B,IAAIW,aAAa7C;IACjB,IAAIuB,SAASoB;IACb,IAAK,IAAIG,OAAO9C,WAAW8C,QAAQ7C,SAAS6C,OAAQ;QAClD,MAAMC,WAAWL,KAAK,CAACI,OAAO,EAAE;QAChC,MAAME,MAAMd,WAAWzC,MAAM,GAAG,KAAK8B,SAAS,IAAIhB,iBAAiB;QACnE,MAAM0C,aAAavE,eAAeqE;QAClC,IAAIb,WAAWzC,MAAM,GAAG,KAAK8B,SAASyB,MAAMC,aAAa5B,SAAS;YAChEuB,OAAOlD,IAAI,IAAI8C,cAAcN,YAAYW,YAAYC,OAAO,GAAGzB,SAASc,WAAWC;YACnFF,aAAa,EAAE;YACfX,SAAS;YACTsB,aAAaC;YACbZ,WAAWxC,IAAI,CAACqD;YAChBxB,UAAU0B;YACV;QACF;QACAf,WAAWxC,IAAI,CAACqD;QAChBxB,UAAUyB,MAAMC;IAClB;IACA,IAAIf,WAAWzC,MAAM,GAAG,GAAGmD,OAAOlD,IAAI,IAAI8C,cAAcN,YAAYW,YAAY5C,SAASoB,SAASc,WAAWC;IAC7G,OAAOQ;AACT;AAEA,0FAA0F;AAC1F,8FAA8F;AAC9F,SAASM,WAAWC,WAAoB,EAAET,KAAe,EAAEU,QAAyB;IAClF,MAAM/B,UAAU+B,SAASlE,YAAY;IACrC,MAAMmE,UAAUhC,UAAU9C;IAC1B,MAAM+E,WAA4C,EAAE;IACpD,IAAI1D,QAAgB,EAAE;IACtB,IAAI2D,iBAAiB;IACrB,IAAIhC,SAAS;IAEb,SAASiC;QACP,MAAMC,QAAQ9D,SAASC;QACvB,IAAI6D,OAAOH,SAAS5D,IAAI,CAAC+D;QACzB7D,QAAQ,EAAE;QACV2D,iBAAiB;QACjBhC,SAAS;IACX;IAEA,mFAAmF;IACnF,2FAA2F;IAC3F,SAASmC,QAAQ/E,IAAY,EAAEgF,QAAgB,EAAE3D,SAAiB,EAAEC,OAAe;QACjFsB,UAAU,AAAC3B,CAAAA,MAAMH,MAAM,GAAG,IAAIc,iBAAiB,CAAA,IAAK7B,eAAeiF;QACnE/D,MAAMF,IAAI,CAAC;YAAEM;YAAWC;YAAStB;QAAK;IACxC;IAEA,KAAK,MAAMY,SAAS4D,YAAa;QAC/B,MAAMS,MAAMlB,MAAMmB,KAAK,CAACtE,MAAMS,SAAS,GAAG,GAAGT,MAAMU,OAAO,EAAEG,IAAI,CAAC;QACjE,MAAM0D,cAAcpF,eAAekF;QAEnC,IAAIE,cAAcT,SAAS;YACzB,MAAMV,OAAO/C,MAAMH,MAAM,GAAG,IAAI8B,SAAS;YACzC,KAAK,MAAMpB,KAAKsC,mBAAmBC,OAAOnD,MAAMS,SAAS,EAAET,MAAMU,OAAO,EAAEoB,SAAS9B,MAAMC,IAAI,EAAE4D,SAASzE,IAAI,EAAEgE,MAAO;gBACnH/C,MAAMF,IAAI,CAACS;gBACXqD;YACF;YACA;QACF;QAEA,MAAMO,YAAY5F,YAAYoB,MAAMgD,IAAI;QACxC,MAAMoB,WAAWP,SAASzE,IAAI,KAAK,QAAQiF,MAAMG;QAEjD,IAAIxE,MAAMC,IAAI,KAAK,WAAW;YAC5BkE,QAAQK,WAAWJ,UAAUpE,MAAMS,SAAS,EAAET,MAAMU,OAAO;YAC3D;QACF;QAEA,wFAAwF;QACxF,wDAAwD;QACxD,MAAM+D,eAAepE,MAAMH,MAAM,GAAG,KAAK8B,SAAShB,iBAAiBuD,cAAcT;QACjF,IAAIW,cAAcR;QAElBE,QAAQK,WAAWJ,UAAUpE,MAAMS,SAAS,EAAET,MAAMU,OAAO;QAC3DsD;QAEA,IAAIA,kBAAkBjF,gBAAgBkF;IACxC;IACAA;IAEA,OAAOF;AACT;AAEA,8FAA8F;AAC9F,8CAA8C;AAC9C,OAAO,SAASG,MAAMrE,MAAe,EAAE6E,IAAY,EAAEhF,IAAmB;IACtE,MAAMmE,WAAWpE,eAAeC;IAChC,MAAMyD,QAAQuB,KAAKC,KAAK,CAAC;IACzB,MAAMC,SAA0C,EAAE;IAClD,KAAK,MAAMC,SAASjF,YAAYC,QAAS+E,OAAOzE,IAAI,IAAIwD,WAAWkB,OAAO1B,OAAOU;IACjF,+FAA+F;IAC/F,+FAA+F;IAC/F,gFAAgF;IAChF,MAAMiB,SACJjB,SAASzE,IAAI,KAAK,QACdwF,OAAOjE,GAAG,CAAC,CAACoE,IACVA,EAAEjE,KAAK,GACHiE,IACA;YACE,GAAGA,CAAC;YACJ3F,MAAM+D,MACHmB,KAAK,CAACS,EAAEtE,SAAS,GAAG,GAAGsE,EAAErE,OAAO,EAChCG,IAAI,CAAC,MACLmE,IAAI;QACT,KAENJ;IACN,mHAAmH;IACnH,OAAOE,OAAOG,MAAM,CAAC,CAACF,IAAMA,EAAE3F,IAAI,CAAC4F,IAAI,GAAG9E,MAAM,GAAG,GAAGS,GAAG,CAAC,CAACoE,IAAO,CAAA;YAAEtE,WAAWsE,EAAEtE,SAAS;YAAEC,SAASqE,EAAErE,OAAO;YAAEtB,MAAM2F,EAAE3F,IAAI;QAAC,CAAA;AAC/H"}

package/dist/esm/chunk/index.d.ts

@@ -0,0 +1,8 @@
+import type { Block, Chunk, ChunkOptions } from './types.js';
+export declare function chunkFromBlocks(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[];
+export declare function chunk(body: string, opts?: ChunkOptions): Chunk[];
+export { extractText } from './extract.js';
+export { DEFAULT_TARGET_TOKENS, estimateTokens } from './group.js';
+export { parse } from './parse.js';
+export type { Block, BlockType, Chunk, ChunkOptions } from './types.js';
+export { CHUNK_VERSION } from './version.js';

package/dist/esm/chunk/index.js

@@ -0,0 +1,16 @@
+import { group } from './group.js';
+import { parse } from './parse.js';
+// Groups blocks a caller already parsed (e.g. scan/index.ts, sharing one parse with the FTS
+// text path) against the same body they came from, per opts (D1/D3).
+export function chunkFromBlocks(blocks, body, opts) {
+    return group(blocks, body, opts);
+}
+// Pure, deterministic function of file content; chunk semantics are version-stamped via
+// CHUNK_VERSION (./version.ts). Algorithm and evidence: BENCHMARKING.md, "The chunking algorithm".
+export function chunk(body, opts) {
+    return chunkFromBlocks(parse(body), body, opts);
+}
+export { extractText } from './extract.js';
+export { DEFAULT_TARGET_TOKENS, estimateTokens } from './group.js';
+export { parse } from './parse.js';
+export { CHUNK_VERSION } from './version.js';

package/dist/esm/chunk/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/index.ts"],"sourcesContent":["import { group } from './group.ts';\nimport { parse } from './parse.ts';\nimport type { Block, Chunk, ChunkOptions } from './types.ts';\n\n// Groups blocks a caller already parsed (e.g. scan/index.ts, sharing one parse with the FTS\n// text path) against the same body they came from, per opts (D1/D3).\nexport function chunkFromBlocks(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[] {\n  return group(blocks, body, opts);\n}\n\n// Pure, deterministic function of file content; chunk semantics are version-stamped via\n// CHUNK_VERSION (./version.ts). Algorithm and evidence: BENCHMARKING.md, \"The chunking algorithm\".\nexport function chunk(body: string, opts?: ChunkOptions): Chunk[] {\n  return chunkFromBlocks(parse(body), body, opts);\n}\n\nexport { extractText } from './extract.ts';\nexport { DEFAULT_TARGET_TOKENS, estimateTokens } from './group.ts';\nexport { parse } from './parse.ts';\nexport type { Block, BlockType, Chunk, ChunkOptions } from './types.ts';\nexport { CHUNK_VERSION } from './version.ts';\n"],"names":["group","parse","chunkFromBlocks","blocks","body","opts","chunk","extractText","DEFAULT_TARGET_TOKENS","estimateTokens","CHUNK_VERSION"],"mappings":"AAAA,SAASA,KAAK,QAAQ,aAAa;AACnC,SAASC,KAAK,QAAQ,aAAa;AAGnC,4FAA4F;AAC5F,qEAAqE;AACrE,OAAO,SAASC,gBAAgBC,MAAe,EAAEC,IAAY,EAAEC,IAAmB;IAChF,OAAOL,MAAMG,QAAQC,MAAMC;AAC7B;AAEA,wFAAwF;AACxF,mGAAmG;AACnG,OAAO,SAASC,MAAMF,IAAY,EAAEC,IAAmB;IACrD,OAAOH,gBAAgBD,MAAMG,OAAOA,MAAMC;AAC5C;AAEA,SAASE,WAAW,QAAQ,eAAe;AAC3C,SAASC,qBAAqB,EAAEC,cAAc,QAAQ,aAAa;AACnE,SAASR,KAAK,QAAQ,aAAa;AAEnC,SAASS,aAAa,QAAQ,eAAe"}

package/dist/esm/chunk/parse.d.ts

@@ -0,0 +1,2 @@
+import type { Block } from './types.js';
+export declare function parse(body: string): Block[];

package/dist/esm/chunk/parse.js

@@ -0,0 +1,63 @@
+import { fromMarkdown } from 'mdast-util-from-markdown';
+import { gfmAutolinkLiteralFromMarkdown } from 'mdast-util-gfm-autolink-literal';
+import { gfmFootnoteFromMarkdown } from 'mdast-util-gfm-footnote';
+import { gfmStrikethroughFromMarkdown } from 'mdast-util-gfm-strikethrough';
+import { gfmTableFromMarkdown } from 'mdast-util-gfm-table';
+import { gfmTaskListItemFromMarkdown } from 'mdast-util-gfm-task-list-item';
+import { gfmAutolinkLiteral } from 'micromark-extension-gfm-autolink-literal';
+import { gfmFootnote } from 'micromark-extension-gfm-footnote';
+import { gfmStrikethrough } from 'micromark-extension-gfm-strikethrough';
+import { gfmTable } from 'micromark-extension-gfm-table';
+import { gfmTaskListItem } from 'micromark-extension-gfm-task-list-item';
+import { extractText } from './extract.js';
+const BLOCK_TYPES = {
+    heading: 'heading',
+    paragraph: 'paragraph',
+    code: 'code',
+    table: 'table',
+    list: 'list',
+    blockquote: 'blockquote'
+};
+// The five extensions extract.ts/group.ts actually consume, imported individually rather than
+// through micromark-extension-gfm/mdast-util-gfm: those bundles also pull in gfm-tagfilter,
+// an HTML-serialization sanitizer this library never uses (no htmlExtensions call anywhere).
+// micromark and mdast-util-from-markdown each already flatten an extensions array themselves,
+// so no combine-extensions helper is needed here.
+const EXTENSIONS = [
+    gfmAutolinkLiteral(),
+    gfmFootnote(),
+    gfmStrikethrough(),
+    gfmTable(),
+    gfmTaskListItem()
+];
+const MDAST_EXTENSIONS = [
+    gfmAutolinkLiteralFromMarkdown(),
+    gfmFootnoteFromMarkdown(),
+    gfmStrikethroughFromMarkdown(),
+    gfmTableFromMarkdown(),
+    gfmTaskListItemFromMarkdown()
+];
+// Top-level blocks of a markdown body, typed and line-extent bounded. Extents come from
+// mdast's own node.position, never a regex guess at where a block starts or ends; GFM adds
+// tables, task lists, footnotes and strikethrough to the CommonMark set fromMarkdown parses.
+export function parse(body) {
+    const tree = fromMarkdown(body, {
+        extensions: EXTENSIONS,
+        mdastExtensions: MDAST_EXTENSIONS
+    });
+    return tree.children.map((node)=>{
+        var _BLOCK_TYPES_node_type;
+        const position = node.position;
+        const block = {
+            type: (_BLOCK_TYPES_node_type = BLOCK_TYPES[node.type]) !== null && _BLOCK_TYPES_node_type !== void 0 ? _BLOCK_TYPES_node_type : 'other',
+            startLine: position ? position.start.line : 1,
+            endLine: position ? position.end.line : 1,
+            node
+        };
+        if (node.type === 'heading') {
+            block.depth = node.depth;
+            block.text = extractText(node);
+        }
+        return block;
+    });
+}

package/dist/esm/chunk/parse.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/parse.ts"],"sourcesContent":["import type { RootContent } from 'mdast';\nimport { fromMarkdown } from 'mdast-util-from-markdown';\nimport { gfmAutolinkLiteralFromMarkdown } from 'mdast-util-gfm-autolink-literal';\nimport { gfmFootnoteFromMarkdown } from 'mdast-util-gfm-footnote';\nimport { gfmStrikethroughFromMarkdown } from 'mdast-util-gfm-strikethrough';\nimport { gfmTableFromMarkdown } from 'mdast-util-gfm-table';\nimport { gfmTaskListItemFromMarkdown } from 'mdast-util-gfm-task-list-item';\nimport { gfmAutolinkLiteral } from 'micromark-extension-gfm-autolink-literal';\nimport { gfmFootnote } from 'micromark-extension-gfm-footnote';\nimport { gfmStrikethrough } from 'micromark-extension-gfm-strikethrough';\nimport { gfmTable } from 'micromark-extension-gfm-table';\nimport { gfmTaskListItem } from 'micromark-extension-gfm-task-list-item';\nimport { extractText } from './extract.ts';\nimport type { Block, BlockType } from './types.ts';\n\nconst BLOCK_TYPES: Partial<Record<RootContent['type'], BlockType>> = {\n  heading: 'heading',\n  paragraph: 'paragraph',\n  code: 'code',\n  table: 'table',\n  list: 'list',\n  blockquote: 'blockquote',\n};\n\n// The five extensions extract.ts/group.ts actually consume, imported individually rather than\n// through micromark-extension-gfm/mdast-util-gfm: those bundles also pull in gfm-tagfilter,\n// an HTML-serialization sanitizer this library never uses (no htmlExtensions call anywhere).\n// micromark and mdast-util-from-markdown each already flatten an extensions array themselves,\n// so no combine-extensions helper is needed here.\nconst EXTENSIONS = [gfmAutolinkLiteral(), gfmFootnote(), gfmStrikethrough(), gfmTable(), gfmTaskListItem()];\nconst MDAST_EXTENSIONS = [gfmAutolinkLiteralFromMarkdown(), gfmFootnoteFromMarkdown(), gfmStrikethroughFromMarkdown(), gfmTableFromMarkdown(), gfmTaskListItemFromMarkdown()];\n\n// Top-level blocks of a markdown body, typed and line-extent bounded. Extents come from\n// mdast's own node.position, never a regex guess at where a block starts or ends; GFM adds\n// tables, task lists, footnotes and strikethrough to the CommonMark set fromMarkdown parses.\nexport function parse(body: string): Block[] {\n  const tree = fromMarkdown(body, { extensions: EXTENSIONS, mdastExtensions: MDAST_EXTENSIONS });\n  return tree.children.map((node) => {\n    const position = node.position;\n    const block: Block = {\n      type: BLOCK_TYPES[node.type] ?? 'other',\n      startLine: position ? position.start.line : 1,\n      endLine: position ? position.end.line : 1,\n      node,\n    };\n    if (node.type === 'heading') {\n      block.depth = node.depth;\n      block.text = extractText(node);\n    }\n    return block;\n  });\n}\n"],"names":["fromMarkdown","gfmAutolinkLiteralFromMarkdown","gfmFootnoteFromMarkdown","gfmStrikethroughFromMarkdown","gfmTableFromMarkdown","gfmTaskListItemFromMarkdown","gfmAutolinkLiteral","gfmFootnote","gfmStrikethrough","gfmTable","gfmTaskListItem","extractText","BLOCK_TYPES","heading","paragraph","code","table","list","blockquote","EXTENSIONS","MDAST_EXTENSIONS","parse","body","tree","extensions","mdastExtensions","children","map","node","position","block","type","startLine","start","line","endLine","end","depth","text"],"mappings":"AACA,SAASA,YAAY,QAAQ,2BAA2B;AACxD,SAASC,8BAA8B,QAAQ,kCAAkC;AACjF,SAASC,uBAAuB,QAAQ,0BAA0B;AAClE,SAASC,4BAA4B,QAAQ,+BAA+B;AAC5E,SAASC,oBAAoB,QAAQ,uBAAuB;AAC5D,SAASC,2BAA2B,QAAQ,gCAAgC;AAC5E,SAASC,kBAAkB,QAAQ,2CAA2C;AAC9E,SAASC,WAAW,QAAQ,mCAAmC;AAC/D,SAASC,gBAAgB,QAAQ,wCAAwC;AACzE,SAASC,QAAQ,QAAQ,gCAAgC;AACzD,SAASC,eAAe,QAAQ,yCAAyC;AACzE,SAASC,WAAW,QAAQ,eAAe;AAG3C,MAAMC,cAA+D;IACnEC,SAAS;IACTC,WAAW;IACXC,MAAM;IACNC,OAAO;IACPC,MAAM;IACNC,YAAY;AACd;AAEA,8FAA8F;AAC9F,4FAA4F;AAC5F,6FAA6F;AAC7F,8FAA8F;AAC9F,kDAAkD;AAClD,MAAMC,aAAa;IAACb;IAAsBC;IAAeC;IAAoBC;IAAYC;CAAkB;AAC3G,MAAMU,mBAAmB;IAACnB;IAAkCC;IAA2BC;IAAgCC;IAAwBC;CAA8B;AAE7K,wFAAwF;AACxF,2FAA2F;AAC3F,6FAA6F;AAC7F,OAAO,SAASgB,MAAMC,IAAY;IAChC,MAAMC,OAAOvB,aAAasB,MAAM;QAAEE,YAAYL;QAAYM,iBAAiBL;IAAiB;IAC5F,OAAOG,KAAKG,QAAQ,CAACC,GAAG,CAAC,CAACC;YAGhBhB;QAFR,MAAMiB,WAAWD,KAAKC,QAAQ;QAC9B,MAAMC,QAAe;YACnBC,IAAI,GAAEnB,yBAAAA,WAAW,CAACgB,KAAKG,IAAI,CAAC,cAAtBnB,oCAAAA,yBAA0B;YAChCoB,WAAWH,WAAWA,SAASI,KAAK,CAACC,IAAI,GAAG;YAC5CC,SAASN,WAAWA,SAASO,GAAG,CAACF,IAAI,GAAG;YACxCN;QACF;QACA,IAAIA,KAAKG,IAAI,KAAK,WAAW;YAC3BD,MAAMO,KAAK,GAAGT,KAAKS,KAAK;YACxBP,MAAMQ,IAAI,GAAG3B,YAAYiB;QAC3B;QACA,OAAOE;IACT;AACF"}

package/dist/esm/chunk/types.d.ts

@@ -0,0 +1,19 @@
+import type { RootContent } from 'mdast';
+export type BlockType = 'heading' | 'paragraph' | 'code' | 'table' | 'list' | 'blockquote' | 'other';
+export interface Block {
+    type: BlockType;
+    startLine: number;
+    endLine: number;
+    depth?: number;
+    text?: string;
+    node: RootContent;
+}
+export interface ChunkOptions {
+    targetTokens?: number;
+    text?: 'extracted' | 'raw';
+}
+export interface Chunk {
+    startLine: number;
+    endLine: number;
+    text: string;
+}

package/dist/esm/chunk/types.js

@@ -0,0 +1,2 @@
+// One grouped, extracted unit: 1-indexed line extent over the body, flavor-resolved text.
+export { };

package/dist/esm/chunk/types.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/types.ts"],"sourcesContent":["import type { RootContent } from 'mdast';\n\nexport type BlockType = 'heading' | 'paragraph' | 'code' | 'table' | 'list' | 'blockquote' | 'other';\n\n// One top-level mdast node plus its 1-indexed source line extent, heading depth/text if any.\n// `node` is the mdast node itself, so extract.ts and group.ts (W2) read the same parse.\nexport interface Block {\n  type: BlockType;\n  startLine: number;\n  endLine: number;\n  depth?: number;\n  text?: string;\n  node: RootContent;\n}\n\nexport interface ChunkOptions {\n  // D3/W3b: the pgc pairing cap and the sole owner lever (embed.chunkTokens); default 500.\n  targetTokens?: number;\n  // D9: 'raw' (source lines verbatim) is the shipped default; 'extracted' (flavor-resolved)\n  // stays available for a future measured revisit, not currently shipped.\n  text?: 'extracted' | 'raw';\n}\n\n// One grouped, extracted unit: 1-indexed line extent over the body, flavor-resolved text.\nexport interface Chunk {\n  startLine: number;\n  endLine: number;\n  text: string;\n}\n"],"names":[],"mappings":"AAuBA,0FAA0F;AAC1F,WAIC"}

package/dist/esm/chunk/version.d.ts

@@ -0,0 +1 @@
+export declare const CHUNK_VERSION = "chunk:v5";

package/dist/esm/chunk/version.js

@@ -0,0 +1,3 @@
+// D8: bump whenever chunk semantics change (grouping, sizing, splitting); the digest in
+// test/unit/chunk/version.test.ts tracks EXTRACTION changes only, so the two move independently.
+export const CHUNK_VERSION = 'chunk:v5';

package/dist/esm/chunk/version.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/version.ts"],"sourcesContent":["// D8: bump whenever chunk semantics change (grouping, sizing, splitting); the digest in\n// test/unit/chunk/version.test.ts tracks EXTRACTION changes only, so the two move independently.\nexport const CHUNK_VERSION = 'chunk:v5';\n"],"names":["CHUNK_VERSION"],"mappings":"AAAA,wFAAwF;AACxF,iGAAiG;AACjG,OAAO,MAAMA,gBAAgB,WAAW"}

package/dist/esm/cli/download.js

@@ -1,11 +1,11 @@
+import { basename } from 'node:path';
 import { embedConfig } from '../config/index.js';
+import { downloadModel, isDownloadable, modelDir, modelPresent } from '../embed/store.js';
 import { SenseError } from '../errors.js';
-import { downloadModel, modelDir, modelPresent } from '../features/embed.js';
 import { USAGE } from './index.js';
 import { CONFIG, parse } from './shared.js';
-// The one command that fetches weights. Nothing else touches the network for them, so a
-// query that reads like a query never spends minutes pulling 124 MB. Asks nothing, so there
-// is no prompt for CI to bypass; running it twice is a no-op.
+// A prefetch over the store's own fetch path: a search would fetch the same weights
+// lazily, so this only moves the wait earlier. Idempotent; asks nothing, so CI can run it unattended.
 const download = async (ctx)=>{
     const { values } = parse(ctx.argv, `usage: ${ctx.name} ${USAGE.download}`, {
         ...CONFIG
@@ -13,15 +13,16 @@ const download = async (ctx)=>{
     const cfg = ctx.resolveConfig(values.config);
     const e = embedConfig(cfg);
     if (!e) throw new SenseError('EMBED_DISABLED', 'no preset asks for vectors, so there is no model to download');
-    if (e.type === 'api') {
-        console.log(`embed type is "api" (${e.url}), so there is nothing to download; ${ctx.name} search reaches it over the network`);
+    if (e.provider !== 'static') {
+        console.log(`embed provider is "${e.provider}" (${e.url}), so there is nothing to download; ${ctx.name} search reaches it over the network`);
         return;
     }
     if (modelPresent(cfg)) {
-        console.log(`${e.model} is already available in ${modelDir(e.model)}`);
+        const at = isDownloadable(e.model) ? `@${basename(modelDir(e.model))} ` : ' ';
+        console.log(`${e.model}${at}is already available in ${modelDir(e.model)}`);
         return;
     }
     const dir = await downloadModel(e.model, (file, into)=>console.error(`fetching ${e.model}/${file} into ${into}`));
-    console.log(`downloaded ${e.model} into ${dir}`);
+    console.log(`downloaded ${e.model}@${basename(dir)} into ${dir}`);
 };
 export default download;

package/dist/esm/cli/download.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/download.ts"],"sourcesContent":["import { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, modelDir, modelPresent } from '../features/embed.ts';\nimport { USAGE } from './index.ts';\nimport { CONFIG, parse } from './shared.ts';\nimport type { Command } from './types.ts';\n\n// The one command that fetches weights. Nothing else touches the network for them, so a\n// query that reads like a query never spends minutes pulling 124 MB. Asks nothing, so there\n// is no prompt for CI to bypass; running it twice is a no-op.\nconst download: Command = async (ctx) => {\n  const { values } = parse(ctx.argv, `usage: ${ctx.name} ${USAGE.download}`, { ...CONFIG });\n  const cfg = ctx.resolveConfig(values.config as string | undefined);\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'no preset asks for vectors, so there is no model to download');\n\n  if (e.type === 'api') {\n    console.log(`embed type is \"api\" (${e.url}), so there is nothing to download; ${ctx.name} search reaches it over the network`);\n    return;\n  }\n  if (modelPresent(cfg)) {\n    console.log(`${e.model} is already available in ${modelDir(e.model)}`);\n    return;\n  }\n  const dir = await downloadModel(e.model, (file, into) => console.error(`fetching ${e.model}/${file} into ${into}`));\n  console.log(`downloaded ${e.model} into ${dir}`);\n};\nexport default download;\n"],"names":["embedConfig","SenseError","downloadModel","modelDir","modelPresent","USAGE","CONFIG","parse","download","ctx","values","argv","name","cfg","resolveConfig","config","e","type","console","log","url","model","dir","file","into","error"],"mappings":"AAAA,SAASA,WAAW,QAAQ,qBAAqB;AACjD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,aAAa,EAAEC,QAAQ,EAAEC,YAAY,QAAQ,uBAAuB;AAC7E,SAASC,KAAK,QAAQ,aAAa;AACnC,SAASC,MAAM,EAAEC,KAAK,QAAQ,cAAc;AAG5C,wFAAwF;AACxF,4FAA4F;AAC5F,8DAA8D;AAC9D,MAAMC,WAAoB,OAAOC;IAC/B,MAAM,EAAEC,MAAM,EAAE,GAAGH,MAAME,IAAIE,IAAI,EAAE,CAAC,OAAO,EAAEF,IAAIG,IAAI,CAAC,CAAC,EAAEP,MAAMG,QAAQ,EAAE,EAAE;QAAE,GAAGF,MAAM;IAAC;IACvF,MAAMO,MAAMJ,IAAIK,aAAa,CAACJ,OAAOK,MAAM;IAC3C,MAAMC,IAAIhB,YAAYa;IACtB,IAAI,CAACG,GAAG,MAAM,IAAIf,WAAW,kBAAkB;IAE/C,IAAIe,EAAEC,IAAI,KAAK,OAAO;QACpBC,QAAQC,GAAG,CAAC,CAAC,qBAAqB,EAAEH,EAAEI,GAAG,CAAC,oCAAoC,EAAEX,IAAIG,IAAI,CAAC,mCAAmC,CAAC;QAC7H;IACF;IACA,IAAIR,aAAaS,MAAM;QACrBK,QAAQC,GAAG,CAAC,GAAGH,EAAEK,KAAK,CAAC,yBAAyB,EAAElB,SAASa,EAAEK,KAAK,GAAG;QACrE;IACF;IACA,MAAMC,MAAM,MAAMpB,cAAcc,EAAEK,KAAK,EAAE,CAACE,MAAMC,OAASN,QAAQO,KAAK,CAAC,CAAC,SAAS,EAAET,EAAEK,KAAK,CAAC,CAAC,EAAEE,KAAK,MAAM,EAAEC,MAAM;IACjHN,QAAQC,GAAG,CAAC,CAAC,WAAW,EAAEH,EAAEK,KAAK,CAAC,MAAM,EAAEC,KAAK;AACjD;AACA,eAAed,SAAS"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/download.ts"],"sourcesContent":["import { basename } from 'node:path';\nimport { embedConfig } from '../config/index.ts';\nimport { downloadModel, isDownloadable, modelDir, modelPresent } from '../embed/store.ts';\nimport { SenseError } from '../errors.ts';\nimport { USAGE } from './index.ts';\nimport { CONFIG, parse } from './shared.ts';\nimport type { Command } from './types.ts';\n\n// A prefetch over the store's own fetch path: a search would fetch the same weights\n// lazily, so this only moves the wait earlier. Idempotent; asks nothing, so CI can run it unattended.\nconst download: Command = async (ctx) => {\n  const { values } = parse(ctx.argv, `usage: ${ctx.name} ${USAGE.download}`, { ...CONFIG });\n  const cfg = ctx.resolveConfig(values.config as string | undefined);\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'no preset asks for vectors, so there is no model to download');\n\n  if (e.provider !== 'static') {\n    console.log(`embed provider is \"${e.provider}\" (${e.url}), so there is nothing to download; ${ctx.name} search reaches it over the network`);\n    return;\n  }\n  if (modelPresent(cfg)) {\n    const at = isDownloadable(e.model) ? `@${basename(modelDir(e.model))} ` : ' ';\n    console.log(`${e.model}${at}is already available in ${modelDir(e.model)}`);\n    return;\n  }\n  const dir = await downloadModel(e.model, (file, into) => console.error(`fetching ${e.model}/${file} into ${into}`));\n  console.log(`downloaded ${e.model}@${basename(dir)} into ${dir}`);\n};\nexport default download;\n"],"names":["basename","embedConfig","downloadModel","isDownloadable","modelDir","modelPresent","SenseError","USAGE","CONFIG","parse","download","ctx","values","argv","name","cfg","resolveConfig","config","e","provider","console","log","url","at","model","dir","file","into","error"],"mappings":"AAAA,SAASA,QAAQ,QAAQ,YAAY;AACrC,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,aAAa,EAAEC,cAAc,EAAEC,QAAQ,EAAEC,YAAY,QAAQ,oBAAoB;AAC1F,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,KAAK,QAAQ,aAAa;AACnC,SAASC,MAAM,EAAEC,KAAK,QAAQ,cAAc;AAG5C,oFAAoF;AACpF,sGAAsG;AACtG,MAAMC,WAAoB,OAAOC;IAC/B,MAAM,EAAEC,MAAM,EAAE,GAAGH,MAAME,IAAIE,IAAI,EAAE,CAAC,OAAO,EAAEF,IAAIG,IAAI,CAAC,CAAC,EAAEP,MAAMG,QAAQ,EAAE,EAAE;QAAE,GAAGF,MAAM;IAAC;IACvF,MAAMO,MAAMJ,IAAIK,aAAa,CAACJ,OAAOK,MAAM;IAC3C,MAAMC,IAAIjB,YAAYc;IACtB,IAAI,CAACG,GAAG,MAAM,IAAIZ,WAAW,kBAAkB;IAE/C,IAAIY,EAAEC,QAAQ,KAAK,UAAU;QAC3BC,QAAQC,GAAG,CAAC,CAAC,mBAAmB,EAAEH,EAAEC,QAAQ,CAAC,GAAG,EAAED,EAAEI,GAAG,CAAC,oCAAoC,EAAEX,IAAIG,IAAI,CAAC,mCAAmC,CAAC;QAC3I;IACF;IACA,IAAIT,aAAaU,MAAM;QACrB,MAAMQ,KAAKpB,eAAee,EAAEM,KAAK,IAAI,CAAC,CAAC,EAAExB,SAASI,SAASc,EAAEM,KAAK,GAAG,CAAC,CAAC,GAAG;QAC1EJ,QAAQC,GAAG,CAAC,GAAGH,EAAEM,KAAK,GAAGD,GAAG,wBAAwB,EAAEnB,SAASc,EAAEM,KAAK,GAAG;QACzE;IACF;IACA,MAAMC,MAAM,MAAMvB,cAAcgB,EAAEM,KAAK,EAAE,CAACE,MAAMC,OAASP,QAAQQ,KAAK,CAAC,CAAC,SAAS,EAAEV,EAAEM,KAAK,CAAC,CAAC,EAAEE,KAAK,MAAM,EAAEC,MAAM;IACjHP,QAAQC,GAAG,CAAC,CAAC,WAAW,EAAEH,EAAEM,KAAK,CAAC,CAAC,EAAExB,SAASyB,KAAK,MAAM,EAAEA,KAAK;AAClE;AACA,eAAef,SAAS"}

package/dist/esm/cli/index.d.ts

@@ -1,6 +1,6 @@
 import type { Command } from './types.js';
 export declare const USAGE: {
-    readonly init: "init";
+    readonly init: "init [--model id-or-path] [--provider static|openai|cohere] [--url url]";
     readonly watch: "watch [--force] [--config path]";
     readonly status: "status [--format table|json] [--config path]";
     readonly download: "download [--config path]";

package/dist/esm/cli/index.js

@@ -2,7 +2,7 @@
 // top-level usage() and the command's own parse() error/--help output, so there is one
 // copy of each line, not two that can drift.
 export const USAGE = {
-    init: 'init',
+    init: 'init [--model id-or-path] [--provider static|openai|cohere] [--url url]',
     watch: 'watch [--force] [--config path]',
     status: 'status [--format table|json] [--config path]',
     download: 'download [--config path]',

package/dist/esm/cli/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/index.ts"],"sourcesContent":["import type { Command } from './types.ts';\n\n// Each command's usage line, minus the leading \"usage: <name> \" -- shared by cli.ts's\n// top-level usage() and the command's own parse() error/--help output, so there is one\n// copy of each line, not two that can drift.\nexport const USAGE = {\n  init: 'init',\n  watch: 'watch [--force] [--config path]',\n  status: 'status [--format table|json] [--config path]',\n  download: 'download [--config path]',\n  sql: 'sql \"<statement>\" [params...] [--preset name] [--format table|json|csv] [--config path]',\n  search: 'search \"<terms>\" [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json|csv] [--config path]',\n  map: 'map [--format table|json] [--config path]',\n  peek: 'peek <path> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--format table|json] [--config path]',\n  path: 'path <a> <b> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--max-depth n] [--format table|json] [--config path]',\n  related: 'related <note> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json|csv] [--config path]',\n} as const;\n\n// Lazy registry: a command's imports load only when it runs, so a heavy dependency in one\n// command never taxes the others (or --version/--help). These names are reserved: a saved\n// entry in config can never shadow them.\nexport const COMMANDS: Record<string, () => Promise<{ default: Command }>> = {\n  init: () => import('./init.ts'),\n  watch: () => import('./watch.ts'),\n  status: () => import('./status.ts'),\n  download: () => import('./download.ts'),\n  sql: () => import('./sql.ts'),\n  search: () => import('./search.ts'),\n  map: () => import('./map.ts'),\n  peek: () => import('./peek.ts'),\n  path: () => import('./path.ts'),\n  related: () => import('./related.ts'),\n};\n\nexport type { Command, Ctx } from './types.ts';\n"],"names":["USAGE","init","watch","status","download","sql","search","map","peek","path","related","COMMANDS"],"mappings":"AAEA,sFAAsF;AACtF,uFAAuF;AACvF,6CAA6C;AAC7C,OAAO,MAAMA,QAAQ;IACnBC,MAAM;IACNC,OAAO;IACPC,QAAQ;IACRC,UAAU;IACVC,KAAK;IACLC,QAAQ;IACRC,KAAK;IACLC,MAAM;IACNC,MAAM;IACNC,SAAS;AACX,EAAW;AAEX,0FAA0F;AAC1F,0FAA0F;AAC1F,yCAAyC;AACzC,OAAO,MAAMC,WAAgE;IAC3EV,MAAM,IAAM,MAAM,CAAC;IACnBC,OAAO,IAAM,MAAM,CAAC;IACpBC,QAAQ,IAAM,MAAM,CAAC;IACrBC,UAAU,IAAM,MAAM,CAAC;IACvBC,KAAK,IAAM,MAAM,CAAC;IAClBC,QAAQ,IAAM,MAAM,CAAC;IACrBC,KAAK,IAAM,MAAM,CAAC;IAClBC,MAAM,IAAM,MAAM,CAAC;IACnBC,MAAM,IAAM,MAAM,CAAC;IACnBC,SAAS,IAAM,MAAM,CAAC;AACxB,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/index.ts"],"sourcesContent":["import type { Command } from './types.ts';\n\n// Each command's usage line, minus the leading \"usage: <name> \" -- shared by cli.ts's\n// top-level usage() and the command's own parse() error/--help output, so there is one\n// copy of each line, not two that can drift.\nexport const USAGE = {\n  init: 'init [--model id-or-path] [--provider static|openai|cohere] [--url url]',\n  watch: 'watch [--force] [--config path]',\n  status: 'status [--format table|json] [--config path]',\n  download: 'download [--config path]',\n  sql: 'sql \"<statement>\" [params...] [--preset name] [--format table|json|csv] [--config path]',\n  search: 'search \"<terms>\" [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json|csv] [--config path]',\n  map: 'map [--format table|json] [--config path]',\n  peek: 'peek <path> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--format table|json] [--config path]',\n  path: 'path <a> <b> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--max-depth n] [--format table|json] [--config path]',\n  related: 'related <note> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json|csv] [--config path]',\n} as const;\n\n// Lazy registry: a command's imports load only when it runs, so a heavy dependency in one\n// command never taxes the others (or --version/--help). These names are reserved: a saved\n// entry in config can never shadow them.\nexport const COMMANDS: Record<string, () => Promise<{ default: Command }>> = {\n  init: () => import('./init.ts'),\n  watch: () => import('./watch.ts'),\n  status: () => import('./status.ts'),\n  download: () => import('./download.ts'),\n  sql: () => import('./sql.ts'),\n  search: () => import('./search.ts'),\n  map: () => import('./map.ts'),\n  peek: () => import('./peek.ts'),\n  path: () => import('./path.ts'),\n  related: () => import('./related.ts'),\n};\n\nexport type { Command, Ctx } from './types.ts';\n"],"names":["USAGE","init","watch","status","download","sql","search","map","peek","path","related","COMMANDS"],"mappings":"AAEA,sFAAsF;AACtF,uFAAuF;AACvF,6CAA6C;AAC7C,OAAO,MAAMA,QAAQ;IACnBC,MAAM;IACNC,OAAO;IACPC,QAAQ;IACRC,UAAU;IACVC,KAAK;IACLC,QAAQ;IACRC,KAAK;IACLC,MAAM;IACNC,MAAM;IACNC,SAAS;AACX,EAAW;AAEX,0FAA0F;AAC1F,0FAA0F;AAC1F,yCAAyC;AACzC,OAAO,MAAMC,WAAgE;IAC3EV,MAAM,IAAM,MAAM,CAAC;IACnBC,OAAO,IAAM,MAAM,CAAC;IACpBC,QAAQ,IAAM,MAAM,CAAC;IACrBC,UAAU,IAAM,MAAM,CAAC;IACvBC,KAAK,IAAM,MAAM,CAAC;IAClBC,QAAQ,IAAM,MAAM,CAAC;IACrBC,KAAK,IAAM,MAAM,CAAC;IAClBC,MAAM,IAAM,MAAM,CAAC;IACnBC,MAAM,IAAM,MAAM,CAAC;IACnBC,SAAS,IAAM,MAAM,CAAC;AACxB,EAAE"}

package/dist/esm/cli/init.js

@@ -1,14 +1,40 @@
-import { initConfig } from '../config/index.js';
+import { DEFAULT_EMBED_MODEL, initConfig } from '../config/index.js';
 import { USAGE } from './index.js';
 import { parse } from './shared.js';
 const init = (ctx)=>{
-    parse(ctx.argv, `usage: ${ctx.name} ${USAGE.init}`, {});
-    const configPath = initConfig(process.cwd());
+    const { values } = parse(ctx.argv, `usage: ${ctx.name} ${USAGE.init}`, {
+        model: {
+            type: 'string'
+        },
+        provider: {
+            type: 'string'
+        },
+        url: {
+            type: 'string'
+        }
+    });
+    const modelFlag = values.model;
+    const providerFlag = values.provider;
+    const model = modelFlag !== null && modelFlag !== void 0 ? modelFlag : DEFAULT_EMBED_MODEL;
+    const provider = providerFlag !== null && providerFlag !== void 0 ? providerFlag : 'static';
+    const url = values.url;
+    // An invalid --provider throws here (validateConfig's own shape error), before anything is
+    // written -- one validator, not a second copy of its rules.
+    const configPath = initConfig(process.cwd(), {
+        model: modelFlag,
+        provider: providerFlag,
+        url
+    });
     console.log(`created ${configPath}`);
     console.log(`query away: ${ctx.name} sql "SELECT path FROM frontmatter LIMIT 10"`);
-    // Semantic search needs a 124 MB model that nothing downloads implicitly; name the step
-    // here so the first search meets it as a choice rather than as a degraded result.
-    console.log(`for meaning-based search: ${ctx.name} download (one 124 MB model, once)`);
+    // Naming a model is consent to fetch it, so writing one into the config says so loudly,
+    // with the download consequence and the prefetch command, right where it was just written.
+    if (provider === 'static') {
+        console.log(`embed model: ${model} (static); nothing downloads until the first vector search -- ${ctx.name} download fetches it now instead`);
+    } else {
+        console.log(`embed model: ${model} (${provider}${url ? ` at ${url}` : ''}); reached over the network at search time, nothing to download`);
+    }
+    console.log('a non-English tree should pick a model for its languages: see the sense-setup skill or INTEGRATIONS.md');
     // The decisions a new tree faces -- features, frontmatter conventions, note size -- are not
     // in this output; name both places that hold them, since an agent may have neither.
     console.log('features and tree design: the sense-setup skill, or schema.json');

package/dist/esm/cli/init.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/init.ts"],"sourcesContent":["import { initConfig } from '../config/index.ts';\nimport { USAGE } from './index.ts';\nimport { parse } from './shared.ts';\nimport type { Command } from './types.ts';\n\nconst init: Command = (ctx) => {\n  parse(ctx.argv, `usage: ${ctx.name} ${USAGE.init}`, {});\n  const configPath = initConfig(process.cwd());\n  console.log(`created ${configPath}`);\n  console.log(`query away: ${ctx.name} sql \"SELECT path FROM frontmatter LIMIT 10\"`);\n  // Semantic search needs a 124 MB model that nothing downloads implicitly; name the step\n  // here so the first search meets it as a choice rather than as a degraded result.\n  console.log(`for meaning-based search: ${ctx.name} download (one 124 MB model, once)`);\n  // The decisions a new tree faces -- features, frontmatter conventions, note size -- are not\n  // in this output; name both places that hold them, since an agent may have neither.\n  console.log('features and tree design: the sense-setup skill, or schema.json');\n};\nexport default init;\n"],"names":["initConfig","USAGE","parse","init","ctx","argv","name","configPath","process","cwd","console","log"],"mappings":"AAAA,SAASA,UAAU,QAAQ,qBAAqB;AAChD,SAASC,KAAK,QAAQ,aAAa;AACnC,SAASC,KAAK,QAAQ,cAAc;AAGpC,MAAMC,OAAgB,CAACC;IACrBF,MAAME,IAAIC,IAAI,EAAE,CAAC,OAAO,EAAED,IAAIE,IAAI,CAAC,CAAC,EAAEL,MAAME,IAAI,EAAE,EAAE,CAAC;IACrD,MAAMI,aAAaP,WAAWQ,QAAQC,GAAG;IACzCC,QAAQC,GAAG,CAAC,CAAC,QAAQ,EAAEJ,YAAY;IACnCG,QAAQC,GAAG,CAAC,CAAC,YAAY,EAAEP,IAAIE,IAAI,CAAC,4CAA4C,CAAC;IACjF,wFAAwF;IACxF,kFAAkF;IAClFI,QAAQC,GAAG,CAAC,CAAC,0BAA0B,EAAEP,IAAIE,IAAI,CAAC,kCAAkC,CAAC;IACrF,4FAA4F;IAC5F,oFAAoF;IACpFI,QAAQC,GAAG,CAAC;AACd;AACA,eAAeR,KAAK"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/init.ts"],"sourcesContent":["import { DEFAULT_EMBED_MODEL, initConfig } from '../config/index.ts';\nimport { USAGE } from './index.ts';\nimport { parse } from './shared.ts';\nimport type { Command } from './types.ts';\n\nconst init: Command = (ctx) => {\n  const { values } = parse(ctx.argv, `usage: ${ctx.name} ${USAGE.init}`, {\n    model: { type: 'string' },\n    provider: { type: 'string' },\n    url: { type: 'string' },\n  });\n  const modelFlag = values.model as string | undefined;\n  const providerFlag = values.provider as string | undefined;\n  const model = modelFlag ?? DEFAULT_EMBED_MODEL;\n  const provider = providerFlag ?? 'static';\n  const url = values.url as string | undefined;\n  // An invalid --provider throws here (validateConfig's own shape error), before anything is\n  // written -- one validator, not a second copy of its rules.\n  const configPath = initConfig(process.cwd(), { model: modelFlag, provider: providerFlag, url });\n  console.log(`created ${configPath}`);\n  console.log(`query away: ${ctx.name} sql \"SELECT path FROM frontmatter LIMIT 10\"`);\n  // Naming a model is consent to fetch it, so writing one into the config says so loudly,\n  // with the download consequence and the prefetch command, right where it was just written.\n  if (provider === 'static') {\n    console.log(`embed model: ${model} (static); nothing downloads until the first vector search -- ${ctx.name} download fetches it now instead`);\n  } else {\n    console.log(`embed model: ${model} (${provider}${url ? ` at ${url}` : ''}); reached over the network at search time, nothing to download`);\n  }\n  console.log('a non-English tree should pick a model for its languages: see the sense-setup skill or INTEGRATIONS.md');\n  // The decisions a new tree faces -- features, frontmatter conventions, note size -- are not\n  // in this output; name both places that hold them, since an agent may have neither.\n  console.log('features and tree design: the sense-setup skill, or schema.json');\n};\nexport default init;\n"],"names":["DEFAULT_EMBED_MODEL","initConfig","USAGE","parse","init","ctx","values","argv","name","model","type","provider","url","modelFlag","providerFlag","configPath","process","cwd","console","log"],"mappings":"AAAA,SAASA,mBAAmB,EAAEC,UAAU,QAAQ,qBAAqB;AACrE,SAASC,KAAK,QAAQ,aAAa;AACnC,SAASC,KAAK,QAAQ,cAAc;AAGpC,MAAMC,OAAgB,CAACC;IACrB,MAAM,EAAEC,MAAM,EAAE,GAAGH,MAAME,IAAIE,IAAI,EAAE,CAAC,OAAO,EAAEF,IAAIG,IAAI,CAAC,CAAC,EAAEN,MAAME,IAAI,EAAE,EAAE;QACrEK,OAAO;YAAEC,MAAM;QAAS;QACxBC,UAAU;YAAED,MAAM;QAAS;QAC3BE,KAAK;YAAEF,MAAM;QAAS;IACxB;IACA,MAAMG,YAAYP,OAAOG,KAAK;IAC9B,MAAMK,eAAeR,OAAOK,QAAQ;IACpC,MAAMF,QAAQI,sBAAAA,uBAAAA,YAAab;IAC3B,MAAMW,WAAWG,yBAAAA,0BAAAA,eAAgB;IACjC,MAAMF,MAAMN,OAAOM,GAAG;IACtB,2FAA2F;IAC3F,4DAA4D;IAC5D,MAAMG,aAAad,WAAWe,QAAQC,GAAG,IAAI;QAAER,OAAOI;QAAWF,UAAUG;QAAcF;IAAI;IAC7FM,QAAQC,GAAG,CAAC,CAAC,QAAQ,EAAEJ,YAAY;IACnCG,QAAQC,GAAG,CAAC,CAAC,YAAY,EAAEd,IAAIG,IAAI,CAAC,4CAA4C,CAAC;IACjF,wFAAwF;IACxF,2FAA2F;IAC3F,IAAIG,aAAa,UAAU;QACzBO,QAAQC,GAAG,CAAC,CAAC,aAAa,EAAEV,MAAM,8DAA8D,EAAEJ,IAAIG,IAAI,CAAC,gCAAgC,CAAC;IAC9I,OAAO;QACLU,QAAQC,GAAG,CAAC,CAAC,aAAa,EAAEV,MAAM,EAAE,EAAEE,WAAWC,MAAM,CAAC,IAAI,EAAEA,KAAK,GAAG,GAAG,+DAA+D,CAAC;IAC3I;IACAM,QAAQC,GAAG,CAAC;IACZ,4FAA4F;IAC5F,oFAAoF;IACpFD,QAAQC,GAAG,CAAC;AACd;AACA,eAAef,KAAK"}