sensemaking 0.16.0 → 0.17.0

package/dist/cjs/chunk/group.js

@@ -0,0 +1,489 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get DEFAULT_TARGET_TOKENS () {
+        return DEFAULT_TARGET_TOKENS;
+    },
+    get estimateTokens () {
+        return estimateTokens;
+    },
+    get group () {
+        return group;
+    }
+});
+var _segmentts = require("../text/segment.js");
+var _extractts = require("./extract.js");
+var _parsets = require("./parse.js");
+function _array_like_to_array(arr, len) {
+    if (len == null || len > arr.length) len = arr.length;
+    for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
+    return arr2;
+}
+function _array_without_holes(arr) {
+    if (Array.isArray(arr)) return _array_like_to_array(arr);
+}
+function _define_property(obj, key, value) {
+    if (key in obj) {
+        Object.defineProperty(obj, key, {
+            value: value,
+            enumerable: true,
+            configurable: true,
+            writable: true
+        });
+    } else obj[key] = value;
+    return obj;
+}
+function _iterable_to_array(iter) {
+    if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
+        return Array.from(iter);
+    }
+}
+function _non_iterable_spread() {
+    throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
+}
+function _object_spread(target) {
+    for(var i = 1; i < arguments.length; i++){
+        var source = arguments[i] != null ? arguments[i] : {};
+        var ownKeys = Object.keys(source);
+        if (typeof Object.getOwnPropertySymbols === "function") {
+            ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function(sym) {
+                return Object.getOwnPropertyDescriptor(source, sym).enumerable;
+            }));
+        }
+        ownKeys.forEach(function(key) {
+            _define_property(target, key, source[key]);
+        });
+    }
+    return target;
+}
+function ownKeys(object, enumerableOnly) {
+    var keys = Object.keys(object);
+    if (Object.getOwnPropertySymbols) {
+        var symbols = Object.getOwnPropertySymbols(object);
+        if (enumerableOnly) {
+            symbols = symbols.filter(function(sym) {
+                return Object.getOwnPropertyDescriptor(object, sym).enumerable;
+            });
+        }
+        keys.push.apply(keys, symbols);
+    }
+    return keys;
+}
+function _object_spread_props(target, source) {
+    source = source != null ? source : {};
+    if (Object.getOwnPropertyDescriptors) Object.defineProperties(target, Object.getOwnPropertyDescriptors(source));
+    else {
+        ownKeys(Object(source)).forEach(function(key) {
+            Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key));
+        });
+    }
+    return target;
+}
+function _to_consumable_array(arr) {
+    return _array_without_holes(arr) || _iterable_to_array(arr) || _unsupported_iterable_to_array(arr) || _non_iterable_spread();
+}
+function _unsupported_iterable_to_array(o, minLen) {
+    if (!o) return;
+    if (typeof o === "string") return _array_like_to_array(o, minLen);
+    var n = Object.prototype.toString.call(o).slice(8, -1);
+    if (n === "Object" && o.constructor) n = o.constructor.name;
+    if (n === "Map" || n === "Set") return Array.from(n);
+    if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
+}
+var DEFAULT_TARGET_TOKENS = 500;
+var PGC_GROUP_SIZE = 2;
+var OVERSIZE_TRIGGER_MULTIPLE = 2;
+// D5: segment.ts's unspaced-script set packs close to one token per character; Hangul is
+// spaced but still token-dense, added here as a conservative bound (one Unicode set, one home).
+var DENSE_SCRIPT = new RegExp("[".concat(_segmentts.UNSPACED_SCRIPTS, "\\p{scx=Hangul}]"), 'u');
+function estimateTokens(text) {
+    var dense = 0;
+    var other = 0;
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = text[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var ch = _step.value;
+            if (DENSE_SCRIPT.test(ch)) dense++;
+            else other++;
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    return dense + other / 4;
+}
+function resolveOptions(opts) {
+    var _ref, _ref1;
+    return {
+        targetTokens: (_ref = opts === null || opts === void 0 ? void 0 : opts.targetTokens) !== null && _ref !== void 0 ? _ref : DEFAULT_TARGET_TOKENS,
+        text: (_ref1 = opts === null || opts === void 0 ? void 0 : opts.text) !== null && _ref1 !== void 0 ? _ref1 : 'raw'
+    };
+}
+// A heading of any depth ends the current scope and starts a new one (D1); the heading block
+// itself is carried into the new scope, where it joins that scope's first group (F7/F10).
+function splitScopes(blocks) {
+    var scopes = [];
+    var current = [];
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = blocks[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var block = _step.value;
+            if (block.type === 'heading' && current.length > 0) {
+                scopes.push(current);
+                current = [];
+            }
+            current.push(block);
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    if (current.length > 0) scopes.push(current);
+    return scopes;
+}
+function finalize(parts) {
+    if (parts.length === 0) return undefined;
+    var first = parts[0];
+    var last = parts[parts.length - 1];
+    return {
+        startLine: first.startLine,
+        endLine: last.endLine,
+        text: parts.map(function(p) {
+            return p.text;
+        }).join('\n'),
+        final: parts.some(function(p) {
+            return p.final;
+        })
+    };
+}
+var NEWLINE_TOKENS = estimateTokens('\n');
+var SENTENCE_SEGMENTER = new Intl.Segmenter(undefined, {
+    granularity: 'sentence'
+});
+var WORD_SEGMENTER = new Intl.Segmenter(undefined, {
+    granularity: 'word'
+});
+function segmentsOf(text, segmenter) {
+    return Array.from(segmenter.segment(text), function(s) {
+        return s.segment;
+    });
+}
+// Greedily packs segments (already contiguous, tiling the source text with no gaps) into groups
+// of at most `working` estimated tokens; a lone segment over `working` still stands alone.
+function pack(segments, working) {
+    var groups = [];
+    var current = '';
+    var tokens = 0;
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = segments[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var segment = _step.value;
+            var segmentTokens = estimateTokens(segment);
+            if (current.length > 0 && tokens + segmentTokens > working) {
+                groups.push(current);
+                current = '';
+                tokens = 0;
+            }
+            current += segment;
+            tokens += segmentTokens;
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    if (current.length > 0) groups.push(current);
+    return groups;
+}
+// A single line's text, still over working after rule 5's line-split can't reduce a lone line
+// (the CJK case, one sentence-dense line with no newlines), falls back to Intl.Segmenter sentence
+// boundaries, then word boundaries for any one sentence still over working. Never splits inside a
+// grapheme either way (both are ECMA-402, the same proven component segment.ts uses for
+// graphemes). Mode-agnostic: the caller decides whether `text` is the raw line or its extracted
+// flavor -- this only packs whatever text it is given.
+function splitLineText(text, working) {
+    var sentences = segmentsOf(text, SENTENCE_SEGMENTER);
+    var out = [];
+    var current = '';
+    var tokens = 0;
+    var flush = function flush() {
+        if (current.length > 0) {
+            out.push(current);
+            current = '';
+            tokens = 0;
+        }
+    };
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = sentences[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var sentence = _step.value;
+            var sentenceTokens = estimateTokens(sentence);
+            if (sentenceTokens > working) {
+                var _out;
+                flush();
+                (_out = out).push.apply(_out, _to_consumable_array(pack(segmentsOf(sentence, WORD_SEGMENTER), working)));
+                continue;
+            }
+            if (current.length > 0 && tokens + sentenceTokens > working) flush();
+            current += sentence;
+            tokens += sentenceTokens;
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    flush();
+    return out;
+}
+var ATOMIC_TYPES = new Set([
+    'code',
+    'table',
+    'list'
+]);
+// An atomic block's (code/table/list) piece is always a raw line slice: re-parsing a table's
+// later pieces without their header/delimiter rows demotes them to paragraph text. A non-atomic
+// piece is a raw line slice too when raw mode requested it; extracted mode re-parses and resolves
+// flavor as before.
+function piece(pieceLines, startLine, endLine, blockType, textMode) {
+    var text = ATOMIC_TYPES.has(blockType) || textMode === 'raw' ? pieceLines.join('\n') : (0, _parsets.parse)(pieceLines.join('\n')).map(function(b) {
+        return (0, _extractts.extractText)(b.node);
+    }).join('\n');
+    return {
+        startLine: startLine,
+        endLine: endLine,
+        text: text
+    };
+}
+// A piece over working size that came from exactly one source line cannot be reduced by another
+// line-boundary pass (rule 5's gap); it is sliced instead, at sentence then word boundaries, over
+// the mode-resolved text piece() already produced -- so raw mode packs the raw line and extracted
+// mode packs its flavor-resolved text. Extents collapse to that one line for every sub-piece (F5:
+// re-derivation is by index over chunk()'s own deterministic output, not by a unique line range
+// per piece); `final` flags each sub-piece so group() knows not to re-derive its text from that
+// shared extent.
+function finalizePiece(pieceLines, startLine, endLine, working, blockType, textMode) {
+    var p = piece(pieceLines, startLine, endLine, blockType, textMode);
+    if (pieceLines.length === 1 && estimateTokens(p.text) > working) {
+        return splitLineText(p.text, working).map(function(text) {
+            return {
+                startLine: startLine,
+                endLine: endLine,
+                text: text,
+                final: true
+            };
+        });
+    }
+    return [
+        p
+    ];
+}
+// A block over 2x working size splits at line boundaries into pieces each <= working size,
+// never mid-line. `seed`: pending tokens (a heading) the first piece must join, checked against
+// the limit too.
+function splitOversizeBlock(lines, startLine, endLine, working, blockType, textMode) {
+    var seed = arguments.length > 6 && arguments[6] !== void 0 ? arguments[6] : 0;
+    var _pieces;
+    var pieces = [];
+    var pieceLines = [];
+    var pieceStart = startLine;
+    var tokens = seed;
+    for(var line = startLine; line <= endLine; line++){
+        var lineText = lines[line - 1];
+        var sep = pieceLines.length > 0 || tokens > 0 ? NEWLINE_TOKENS : 0;
+        var lineTokens = estimateTokens(lineText);
+        if (pieceLines.length > 0 && tokens + sep + lineTokens > working) {
+            var _pieces1;
+            (_pieces1 = pieces).push.apply(_pieces1, _to_consumable_array(finalizePiece(pieceLines, pieceStart, line - 1, working, blockType, textMode)));
+            pieceLines = [];
+            tokens = 0;
+            pieceStart = line;
+            pieceLines.push(lineText);
+            tokens += lineTokens;
+            continue;
+        }
+        pieceLines.push(lineText);
+        tokens += sep + lineTokens;
+    }
+    if (pieceLines.length > 0) (_pieces = pieces).push.apply(_pieces, _to_consumable_array(finalizePiece(pieceLines, pieceStart, endLine, working, blockType, textMode)));
+    return pieces;
+}
+// One heading scope's groups (D1): a heading opens the first group, and an oversize block
+// (rule 5, including an oversize heading) splits into pieces that each close their own group.
+function groupScope(scopeBlocks, lines, resolved) {
+    var working = resolved.targetTokens;
+    var trigger = working * OVERSIZE_TRIGGER_MULTIPLE;
+    var finished = [];
+    var parts = [];
+    var paragraphCount = 0;
+    var tokens = 0;
+    function close() {
+        var group = finalize(parts);
+        if (group) finished.push(group);
+        parts = [];
+        paragraphCount = 0;
+        tokens = 0;
+    }
+    // tokens tracks the active text mode's own estimate (a newline between parts costs
+    // NEWLINE_TOKENS too), so packing decisions size the text the chunk will actually ship as.
+    function addPart(text, sizeText, startLine, endLine) {
+        tokens += (parts.length > 0 ? NEWLINE_TOKENS : 0) + estimateTokens(sizeText);
+        parts.push({
+            startLine: startLine,
+            endLine: endLine,
+            text: text
+        });
+    }
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = scopeBlocks[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var block = _step.value;
+            var raw = lines.slice(block.startLine - 1, block.endLine).join('\n');
+            var blockTokens = estimateTokens(raw);
+            if (blockTokens > trigger) {
+                var seed = parts.length > 0 ? tokens : 0;
+                var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
+                try {
+                    for(var _iterator1 = splitOversizeBlock(lines, block.startLine, block.endLine, working, block.type, resolved.text, seed)[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
+                        var p = _step1.value;
+                        parts.push(p);
+                        close();
+                    }
+                } catch (err) {
+                    _didIteratorError1 = true;
+                    _iteratorError1 = err;
+                } finally{
+                    try {
+                        if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
+                            _iterator1.return();
+                        }
+                    } finally{
+                        if (_didIteratorError1) {
+                            throw _iteratorError1;
+                        }
+                    }
+                }
+                continue;
+            }
+            var extracted = (0, _extractts.extractText)(block.node);
+            var sizeText = resolved.text === 'raw' ? raw : extracted;
+            if (block.type === 'heading') {
+                addPart(extracted, sizeText, block.startLine, block.endLine);
+                continue;
+            }
+            // The 2x-working invariant holds even under pgc's paper-faithful 2-paragraph pairing --
+            // close first if the pair about to form would cross it.
+            var pairOversize = parts.length > 0 && tokens + NEWLINE_TOKENS + blockTokens > trigger;
+            if (pairOversize) close();
+            addPart(extracted, sizeText, block.startLine, block.endLine);
+            paragraphCount++;
+            if (paragraphCount >= PGC_GROUP_SIZE) close();
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    close();
+    return finished;
+}
+function group(blocks, body, opts) {
+    var _chunks;
+    var resolved = resolveOptions(opts);
+    var lines = body.split('\n');
+    var chunks = [];
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = splitScopes(blocks)[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var scope = _step.value;
+            (_chunks = chunks).push.apply(_chunks, _to_consumable_array(groupScope(scope, lines, resolved)));
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    // 'raw': the chunk's own source lines verbatim, replacing the flavor-resolved join above (D9).
+    // A `final` chunk (a sub-line split) already carries its own slice's raw text -- re-slicing by
+    // extent would return the whole shared source line instead, once per sub-piece.
+    var texted = resolved.text === 'raw' ? chunks.map(function(c) {
+        return c.final ? c : _object_spread_props(_object_spread({}, c), {
+            text: lines.slice(c.startLine - 1, c.endLine).join('\n').trim()
+        });
+    }) : chunks;
+    // A group can be all-blank (flavor-stripped to nothing, or a raw slice of pure syntax); it never produces a chunk.
+    return texted.filter(function(c) {
+        return c.text.trim().length > 0;
+    }).map(function(c) {
+        return {
+            startLine: c.startLine,
+            endLine: c.endLine,
+            text: c.text
+        };
+    });
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/chunk/group.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/group.ts"],"sourcesContent":["import { UNSPACED_SCRIPTS } from '../text/segment.ts';\nimport { extractText } from './extract.ts';\nimport { parse } from './parse.ts';\nimport type { Block, BlockType, Chunk, ChunkOptions } from './types.ts';\n\nexport const DEFAULT_TARGET_TOKENS = 500;\nconst PGC_GROUP_SIZE = 2;\nconst OVERSIZE_TRIGGER_MULTIPLE = 2;\n\n// D5: segment.ts's unspaced-script set packs close to one token per character; Hangul is\n// spaced but still token-dense, added here as a conservative bound (one Unicode set, one home).\nconst DENSE_SCRIPT = new RegExp(`[${UNSPACED_SCRIPTS}\\\\p{scx=Hangul}]`, 'u');\n\n// D5's size estimate: dense-script graphemes 1:1, everything else at 4 chars/token.\nexport function estimateTokens(text: string): number {\n  let dense = 0;\n  let other = 0;\n  for (const ch of text) {\n    if (DENSE_SCRIPT.test(ch)) dense++;\n    else other++;\n  }\n  return dense + other / 4;\n}\n\ninterface ResolvedOptions {\n  targetTokens: number;\n  text: 'extracted' | 'raw';\n}\n\nfunction resolveOptions(opts?: ChunkOptions): ResolvedOptions {\n  return {\n    targetTokens: opts?.targetTokens ?? DEFAULT_TARGET_TOKENS,\n    text: opts?.text ?? 'raw',\n  };\n}\n\n// A heading of any depth ends the current scope and starts a new one (D1); the heading block\n// itself is carried into the new scope, where it joins that scope's first group (F7/F10).\nfunction splitScopes(blocks: Block[]): Block[][] {\n  const scopes: Block[][] = [];\n  let current: Block[] = [];\n  for (const block of blocks) {\n    if (block.type === 'heading' && current.length > 0) {\n      scopes.push(current);\n      current = [];\n    }\n    current.push(block);\n  }\n  if (current.length > 0) scopes.push(current);\n  return scopes;\n}\n\ninterface Part {\n  startLine: number;\n  endLine: number;\n  text: string;\n  // true for a sub-line split piece (finalizePiece's splitLineText branch): its text is already\n  // the active mode's final text for its own slice of the line, not the whole line, so group()'s\n  // extent-based raw re-slice (startLine === endLine for every sibling sub-piece) must not touch it.\n  final?: boolean;\n}\n\nfunction finalize(parts: Part[]): (Chunk & { final?: boolean }) | undefined {\n  if (parts.length === 0) return undefined;\n  const first = parts[0];\n  const last = parts[parts.length - 1];\n  return { startLine: first.startLine, endLine: last.endLine, text: parts.map((p) => p.text).join('\\n'), final: parts.some((p) => p.final) };\n}\n\nconst NEWLINE_TOKENS = estimateTokens('\\n');\nconst SENTENCE_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'sentence' });\nconst WORD_SEGMENTER = new Intl.Segmenter(undefined, { granularity: 'word' });\n\nfunction segmentsOf(text: string, segmenter: Intl.Segmenter): string[] {\n  return Array.from(segmenter.segment(text), (s) => s.segment);\n}\n\n// Greedily packs segments (already contiguous, tiling the source text with no gaps) into groups\n// of at most `working` estimated tokens; a lone segment over `working` still stands alone.\nfunction pack(segments: string[], working: number): string[] {\n  const groups: string[] = [];\n  let current = '';\n  let tokens = 0;\n  for (const segment of segments) {\n    const segmentTokens = estimateTokens(segment);\n    if (current.length > 0 && tokens + segmentTokens > working) {\n      groups.push(current);\n      current = '';\n      tokens = 0;\n    }\n    current += segment;\n    tokens += segmentTokens;\n  }\n  if (current.length > 0) groups.push(current);\n  return groups;\n}\n\n// A single line's text, still over working after rule 5's line-split can't reduce a lone line\n// (the CJK case, one sentence-dense line with no newlines), falls back to Intl.Segmenter sentence\n// boundaries, then word boundaries for any one sentence still over working. Never splits inside a\n// grapheme either way (both are ECMA-402, the same proven component segment.ts uses for\n// graphemes). Mode-agnostic: the caller decides whether `text` is the raw line or its extracted\n// flavor -- this only packs whatever text it is given.\nfunction splitLineText(text: string, working: number): string[] {\n  const sentences = segmentsOf(text, SENTENCE_SEGMENTER);\n  const out: string[] = [];\n  let current = '';\n  let tokens = 0;\n  const flush = () => {\n    if (current.length > 0) {\n      out.push(current);\n      current = '';\n      tokens = 0;\n    }\n  };\n  for (const sentence of sentences) {\n    const sentenceTokens = estimateTokens(sentence);\n    if (sentenceTokens > working) {\n      flush();\n      out.push(...pack(segmentsOf(sentence, WORD_SEGMENTER), working));\n      continue;\n    }\n    if (current.length > 0 && tokens + sentenceTokens > working) flush();\n    current += sentence;\n    tokens += sentenceTokens;\n  }\n  flush();\n  return out;\n}\n\nconst ATOMIC_TYPES: ReadonlySet<BlockType> = new Set(['code', 'table', 'list']);\n\n// An atomic block's (code/table/list) piece is always a raw line slice: re-parsing a table's\n// later pieces without their header/delimiter rows demotes them to paragraph text. A non-atomic\n// piece is a raw line slice too when raw mode requested it; extracted mode re-parses and resolves\n// flavor as before.\nfunction piece(pieceLines: string[], startLine: number, endLine: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part {\n  const text =\n    ATOMIC_TYPES.has(blockType) || textMode === 'raw'\n      ? pieceLines.join('\\n')\n      : parse(pieceLines.join('\\n'))\n          .map((b) => extractText(b.node))\n          .join('\\n');\n  return { startLine, endLine, text };\n}\n\n// A piece over working size that came from exactly one source line cannot be reduced by another\n// line-boundary pass (rule 5's gap); it is sliced instead, at sentence then word boundaries, over\n// the mode-resolved text piece() already produced -- so raw mode packs the raw line and extracted\n// mode packs its flavor-resolved text. Extents collapse to that one line for every sub-piece (F5:\n// re-derivation is by index over chunk()'s own deterministic output, not by a unique line range\n// per piece); `final` flags each sub-piece so group() knows not to re-derive its text from that\n// shared extent.\nfunction finalizePiece(pieceLines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw'): Part[] {\n  const p = piece(pieceLines, startLine, endLine, blockType, textMode);\n  if (pieceLines.length === 1 && estimateTokens(p.text) > working) {\n    return splitLineText(p.text, working).map((text) => ({ startLine, endLine, text, final: true }));\n  }\n  return [p];\n}\n\n// A block over 2x working size splits at line boundaries into pieces each <= working size,\n// never mid-line. `seed`: pending tokens (a heading) the first piece must join, checked against\n// the limit too.\nfunction splitOversizeBlock(lines: string[], startLine: number, endLine: number, working: number, blockType: BlockType, textMode: 'extracted' | 'raw', seed = 0): Part[] {\n  const pieces: Part[] = [];\n  let pieceLines: string[] = [];\n  let pieceStart = startLine;\n  let tokens = seed;\n  for (let line = startLine; line <= endLine; line++) {\n    const lineText = lines[line - 1];\n    const sep = pieceLines.length > 0 || tokens > 0 ? NEWLINE_TOKENS : 0;\n    const lineTokens = estimateTokens(lineText);\n    if (pieceLines.length > 0 && tokens + sep + lineTokens > working) {\n      pieces.push(...finalizePiece(pieceLines, pieceStart, line - 1, working, blockType, textMode));\n      pieceLines = [];\n      tokens = 0;\n      pieceStart = line;\n      pieceLines.push(lineText);\n      tokens += lineTokens;\n      continue;\n    }\n    pieceLines.push(lineText);\n    tokens += sep + lineTokens;\n  }\n  if (pieceLines.length > 0) pieces.push(...finalizePiece(pieceLines, pieceStart, endLine, working, blockType, textMode));\n  return pieces;\n}\n\n// One heading scope's groups (D1): a heading opens the first group, and an oversize block\n// (rule 5, including an oversize heading) splits into pieces that each close their own group.\nfunction groupScope(scopeBlocks: Block[], lines: string[], resolved: ResolvedOptions): (Chunk & { final?: boolean })[] {\n  const working = resolved.targetTokens;\n  const trigger = working * OVERSIZE_TRIGGER_MULTIPLE;\n  const finished: (Chunk & { final?: boolean })[] = [];\n  let parts: Part[] = [];\n  let paragraphCount = 0;\n  let tokens = 0;\n\n  function close(): void {\n    const group = finalize(parts);\n    if (group) finished.push(group);\n    parts = [];\n    paragraphCount = 0;\n    tokens = 0;\n  }\n\n  // tokens tracks the active text mode's own estimate (a newline between parts costs\n  // NEWLINE_TOKENS too), so packing decisions size the text the chunk will actually ship as.\n  function addPart(text: string, sizeText: string, startLine: number, endLine: number): void {\n    tokens += (parts.length > 0 ? NEWLINE_TOKENS : 0) + estimateTokens(sizeText);\n    parts.push({ startLine, endLine, text });\n  }\n\n  for (const block of scopeBlocks) {\n    const raw = lines.slice(block.startLine - 1, block.endLine).join('\\n');\n    const blockTokens = estimateTokens(raw);\n\n    if (blockTokens > trigger) {\n      const seed = parts.length > 0 ? tokens : 0;\n      for (const p of splitOversizeBlock(lines, block.startLine, block.endLine, working, block.type, resolved.text, seed)) {\n        parts.push(p);\n        close();\n      }\n      continue;\n    }\n\n    const extracted = extractText(block.node);\n    const sizeText = resolved.text === 'raw' ? raw : extracted;\n\n    if (block.type === 'heading') {\n      addPart(extracted, sizeText, block.startLine, block.endLine);\n      continue;\n    }\n\n    // The 2x-working invariant holds even under pgc's paper-faithful 2-paragraph pairing --\n    // close first if the pair about to form would cross it.\n    const pairOversize = parts.length > 0 && tokens + NEWLINE_TOKENS + blockTokens > trigger;\n    if (pairOversize) close();\n\n    addPart(extracted, sizeText, block.startLine, block.endLine);\n    paragraphCount++;\n\n    if (paragraphCount >= PGC_GROUP_SIZE) close();\n  }\n  close();\n\n  return finished;\n}\n\n// Groups already-parsed blocks per opts (D1/D3), against the same body the blocks were parsed\n// from (line lookups for oversize splitting).\nexport function group(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[] {\n  const resolved = resolveOptions(opts);\n  const lines = body.split('\\n');\n  const chunks: (Chunk & { final?: boolean })[] = [];\n  for (const scope of splitScopes(blocks)) chunks.push(...groupScope(scope, lines, resolved));\n  // 'raw': the chunk's own source lines verbatim, replacing the flavor-resolved join above (D9).\n  // A `final` chunk (a sub-line split) already carries its own slice's raw text -- re-slicing by\n  // extent would return the whole shared source line instead, once per sub-piece.\n  const texted =\n    resolved.text === 'raw'\n      ? chunks.map((c) =>\n          c.final\n            ? c\n            : {\n                ...c,\n                text: lines\n                  .slice(c.startLine - 1, c.endLine)\n                  .join('\\n')\n                  .trim(),\n              }\n        )\n      : chunks;\n  // A group can be all-blank (flavor-stripped to nothing, or a raw slice of pure syntax); it never produces a chunk.\n  return texted.filter((c) => c.text.trim().length > 0).map((c) => ({ startLine: c.startLine, endLine: c.endLine, text: c.text }));\n}\n"],"names":["DEFAULT_TARGET_TOKENS","estimateTokens","group","PGC_GROUP_SIZE","OVERSIZE_TRIGGER_MULTIPLE","DENSE_SCRIPT","RegExp","UNSPACED_SCRIPTS","text","dense","other","ch","test","resolveOptions","opts","targetTokens","splitScopes","blocks","scopes","current","block","type","length","push","finalize","parts","undefined","first","last","startLine","endLine","map","p","join","final","some","NEWLINE_TOKENS","SENTENCE_SEGMENTER","Intl","Segmenter","granularity","WORD_SEGMENTER","segmentsOf","segmenter","Array","from","segment","s","pack","segments","working","groups","tokens","segmentTokens","splitLineText","sentences","out","flush","sentence","sentenceTokens","ATOMIC_TYPES","Set","piece","pieceLines","blockType","textMode","has","parse","b","extractText","node","finalizePiece","splitOversizeBlock","lines","seed","pieces","pieceStart","line","lineText","sep","lineTokens","groupScope","scopeBlocks","resolved","trigger","finished","paragraphCount","close","addPart","sizeText","raw","slice","blockTokens","extracted","pairOversize","body","chunks","split","scope","texted","c","trim","filter"],"mappings":";;;;;;;;;;;QAKaA;eAAAA;;QASGC;eAAAA;;QA8OAC;eAAAA;;;yBA5PiB;yBACL;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGf,IAAMF,wBAAwB;AACrC,IAAMG,iBAAiB;AACvB,IAAMC,4BAA4B;AAElC,yFAAyF;AACzF,gGAAgG;AAChG,IAAMC,eAAe,IAAIC,OAAO,AAAC,IAAoB,OAAjBC,2BAAgB,EAAC,qBAAmB;AAGjE,SAASN,eAAeO,IAAY;IACzC,IAAIC,QAAQ;IACZ,IAAIC,QAAQ;QACP,kCAAA,2BAAA;;QAAL,QAAK,YAAYF,yBAAZ,SAAA,6BAAA,QAAA,yBAAA,iCAAkB;YAAlB,IAAMG,KAAN;YACH,IAAIN,aAAaO,IAAI,CAACD,KAAKF;iBACtBC;QACP;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAOD,QAAQC,QAAQ;AACzB;AAOA,SAASG,eAAeC,IAAmB;;IACzC,OAAO;QACLC,YAAY,UAAED,iBAAAA,2BAAAA,KAAMC,YAAY,uCAAIf;QACpCQ,IAAI,WAAEM,iBAAAA,2BAAAA,KAAMN,IAAI,yCAAI;IACtB;AACF;AAEA,6FAA6F;AAC7F,0FAA0F;AAC1F,SAASQ,YAAYC,MAAe;IAClC,IAAMC,SAAoB,EAAE;IAC5B,IAAIC,UAAmB,EAAE;QACpB,kCAAA,2BAAA;;QAAL,QAAK,YAAeF,2BAAf,SAAA,6BAAA,QAAA,yBAAA,iCAAuB;YAAvB,IAAMG,QAAN;YACH,IAAIA,MAAMC,IAAI,KAAK,aAAaF,QAAQG,MAAM,GAAG,GAAG;gBAClDJ,OAAOK,IAAI,CAACJ;gBACZA,UAAU,EAAE;YACd;YACAA,QAAQI,IAAI,CAACH;QACf;;QANK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAOL,IAAID,QAAQG,MAAM,GAAG,GAAGJ,OAAOK,IAAI,CAACJ;IACpC,OAAOD;AACT;AAYA,SAASM,SAASC,KAAa;IAC7B,IAAIA,MAAMH,MAAM,KAAK,GAAG,OAAOI;IAC/B,IAAMC,QAAQF,KAAK,CAAC,EAAE;IACtB,IAAMG,OAAOH,KAAK,CAACA,MAAMH,MAAM,GAAG,EAAE;IACpC,OAAO;QAAEO,WAAWF,MAAME,SAAS;QAAEC,SAASF,KAAKE,OAAO;QAAEtB,MAAMiB,MAAMM,GAAG,CAAC,SAACC;mBAAMA,EAAExB,IAAI;WAAEyB,IAAI,CAAC;QAAOC,OAAOT,MAAMU,IAAI,CAAC,SAACH;mBAAMA,EAAEE,KAAK;;IAAE;AAC3I;AAEA,IAAME,iBAAiBnC,eAAe;AACtC,IAAMoC,qBAAqB,IAAIC,KAAKC,SAAS,CAACb,WAAW;IAAEc,aAAa;AAAW;AACnF,IAAMC,iBAAiB,IAAIH,KAAKC,SAAS,CAACb,WAAW;IAAEc,aAAa;AAAO;AAE3E,SAASE,WAAWlC,IAAY,EAAEmC,SAAyB;IACzD,OAAOC,MAAMC,IAAI,CAACF,UAAUG,OAAO,CAACtC,OAAO,SAACuC;eAAMA,EAAED,OAAO;;AAC7D;AAEA,gGAAgG;AAChG,2FAA2F;AAC3F,SAASE,KAAKC,QAAkB,EAAEC,OAAe;IAC/C,IAAMC,SAAmB,EAAE;IAC3B,IAAIhC,UAAU;IACd,IAAIiC,SAAS;QACR,kCAAA,2BAAA;;QAAL,QAAK,YAAiBH,6BAAjB,SAAA,6BAAA,QAAA,yBAAA,iCAA2B;YAA3B,IAAMH,UAAN;YACH,IAAMO,gBAAgBpD,eAAe6C;YACrC,IAAI3B,QAAQG,MAAM,GAAG,KAAK8B,SAASC,gBAAgBH,SAAS;gBAC1DC,OAAO5B,IAAI,CAACJ;gBACZA,UAAU;gBACViC,SAAS;YACX;YACAjC,WAAW2B;YACXM,UAAUC;QACZ;;QATK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAUL,IAAIlC,QAAQG,MAAM,GAAG,GAAG6B,OAAO5B,IAAI,CAACJ;IACpC,OAAOgC;AACT;AAEA,8FAA8F;AAC9F,kGAAkG;AAClG,kGAAkG;AAClG,wFAAwF;AACxF,gGAAgG;AAChG,uDAAuD;AACvD,SAASG,cAAc9C,IAAY,EAAE0C,OAAe;IAClD,IAAMK,YAAYb,WAAWlC,MAAM6B;IACnC,IAAMmB,MAAgB,EAAE;IACxB,IAAIrC,UAAU;IACd,IAAIiC,SAAS;IACb,IAAMK,QAAQ;QACZ,IAAItC,QAAQG,MAAM,GAAG,GAAG;YACtBkC,IAAIjC,IAAI,CAACJ;YACTA,UAAU;YACViC,SAAS;QACX;IACF;QACK,kCAAA,2BAAA;;QAAL,QAAK,YAAkBG,8BAAlB,SAAA,6BAAA,QAAA,yBAAA,iCAA6B;YAA7B,IAAMG,WAAN;YACH,IAAMC,iBAAiB1D,eAAeyD;YACtC,IAAIC,iBAAiBT,SAAS;oBAE5BM;gBADAC;gBACAD,CAAAA,OAAAA,KAAIjC,IAAI,OAARiC,MAAS,qBAAGR,KAAKN,WAAWgB,UAAUjB,iBAAiBS;gBACvD;YACF;YACA,IAAI/B,QAAQG,MAAM,GAAG,KAAK8B,SAASO,iBAAiBT,SAASO;YAC7DtC,WAAWuC;YACXN,UAAUO;QACZ;;QAVK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAWLF;IACA,OAAOD;AACT;AAEA,IAAMI,eAAuC,IAAIC,IAAI;IAAC;IAAQ;IAAS;CAAO;AAE9E,6FAA6F;AAC7F,gGAAgG;AAChG,kGAAkG;AAClG,oBAAoB;AACpB,SAASC,MAAMC,UAAoB,EAAElC,SAAiB,EAAEC,OAAe,EAAEkC,SAAoB,EAAEC,QAA6B;IAC1H,IAAMzD,OACJoD,aAAaM,GAAG,CAACF,cAAcC,aAAa,QACxCF,WAAW9B,IAAI,CAAC,QAChBkC,IAAAA,cAAK,EAACJ,WAAW9B,IAAI,CAAC,OACnBF,GAAG,CAAC,SAACqC;eAAMC,IAAAA,sBAAW,EAACD,EAAEE,IAAI;OAC7BrC,IAAI,CAAC;IACd,OAAO;QAAEJ,WAAAA;QAAWC,SAAAA;QAAStB,MAAAA;IAAK;AACpC;AAEA,gGAAgG;AAChG,kGAAkG;AAClG,kGAAkG;AAClG,kGAAkG;AAClG,gGAAgG;AAChG,gGAAgG;AAChG,iBAAiB;AACjB,SAAS+D,cAAcR,UAAoB,EAAElC,SAAiB,EAAEC,OAAe,EAAEoB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B;IACnJ,IAAMjC,IAAI8B,MAAMC,YAAYlC,WAAWC,SAASkC,WAAWC;IAC3D,IAAIF,WAAWzC,MAAM,KAAK,KAAKrB,eAAe+B,EAAExB,IAAI,IAAI0C,SAAS;QAC/D,OAAOI,cAActB,EAAExB,IAAI,EAAE0C,SAASnB,GAAG,CAAC,SAACvB;mBAAU;gBAAEqB,WAAAA;gBAAWC,SAAAA;gBAAStB,MAAAA;gBAAM0B,OAAO;YAAK;;IAC/F;IACA,OAAO;QAACF;KAAE;AACZ;AAEA,2FAA2F;AAC3F,gGAAgG;AAChG,iBAAiB;AACjB,SAASwC,mBAAmBC,KAAe,EAAE5C,SAAiB,EAAEC,OAAe,EAAEoB,OAAe,EAAEc,SAAoB,EAAEC,QAA6B;QAAES,OAAAA,iEAAO;QAqBjIC;IApB3B,IAAMA,SAAiB,EAAE;IACzB,IAAIZ,aAAuB,EAAE;IAC7B,IAAIa,aAAa/C;IACjB,IAAIuB,SAASsB;IACb,IAAK,IAAIG,OAAOhD,WAAWgD,QAAQ/C,SAAS+C,OAAQ;QAClD,IAAMC,WAAWL,KAAK,CAACI,OAAO,EAAE;QAChC,IAAME,MAAMhB,WAAWzC,MAAM,GAAG,KAAK8B,SAAS,IAAIhB,iBAAiB;QACnE,IAAM4C,aAAa/E,eAAe6E;QAClC,IAAIf,WAAWzC,MAAM,GAAG,KAAK8B,SAAS2B,MAAMC,aAAa9B,SAAS;gBAChEyB;YAAAA,CAAAA,WAAAA,QAAOpD,IAAI,OAAXoD,UAAY,qBAAGJ,cAAcR,YAAYa,YAAYC,OAAO,GAAG3B,SAASc,WAAWC;YACnFF,aAAa,EAAE;YACfX,SAAS;YACTwB,aAAaC;YACbd,WAAWxC,IAAI,CAACuD;YAChB1B,UAAU4B;YACV;QACF;QACAjB,WAAWxC,IAAI,CAACuD;QAChB1B,UAAU2B,MAAMC;IAClB;IACA,IAAIjB,WAAWzC,MAAM,GAAG,GAAGqD,CAAAA,UAAAA,QAAOpD,IAAI,OAAXoD,SAAY,qBAAGJ,cAAcR,YAAYa,YAAY9C,SAASoB,SAASc,WAAWC;IAC7G,OAAOU;AACT;AAEA,0FAA0F;AAC1F,8FAA8F;AAC9F,SAASM,WAAWC,WAAoB,EAAET,KAAe,EAAEU,QAAyB;IAClF,IAAMjC,UAAUiC,SAASpE,YAAY;IACrC,IAAMqE,UAAUlC,UAAU9C;IAC1B,IAAMiF,WAA4C,EAAE;IACpD,IAAI5D,QAAgB,EAAE;IACtB,IAAI6D,iBAAiB;IACrB,IAAIlC,SAAS;IAEb,SAASmC;QACP,IAAMrF,QAAQsB,SAASC;QACvB,IAAIvB,OAAOmF,SAAS9D,IAAI,CAACrB;QACzBuB,QAAQ,EAAE;QACV6D,iBAAiB;QACjBlC,SAAS;IACX;IAEA,mFAAmF;IACnF,2FAA2F;IAC3F,SAASoC,QAAQhF,IAAY,EAAEiF,QAAgB,EAAE5D,SAAiB,EAAEC,OAAe;QACjFsB,UAAU,AAAC3B,CAAAA,MAAMH,MAAM,GAAG,IAAIc,iBAAiB,CAAA,IAAKnC,eAAewF;QACnEhE,MAAMF,IAAI,CAAC;YAAEM,WAAAA;YAAWC,SAAAA;YAAStB,MAAAA;QAAK;IACxC;QAEK,kCAAA,2BAAA;;QAAL,QAAK,YAAe0E,gCAAf,SAAA,6BAAA,QAAA,yBAAA,iCAA4B;YAA5B,IAAM9D,QAAN;YACH,IAAMsE,MAAMjB,MAAMkB,KAAK,CAACvE,MAAMS,SAAS,GAAG,GAAGT,MAAMU,OAAO,EAAEG,IAAI,CAAC;YACjE,IAAM2D,cAAc3F,eAAeyF;YAEnC,IAAIE,cAAcR,SAAS;gBACzB,IAAMV,OAAOjD,MAAMH,MAAM,GAAG,IAAI8B,SAAS;oBACpC,mCAAA,4BAAA;;oBAAL,QAAK,aAAWoB,mBAAmBC,OAAOrD,MAAMS,SAAS,EAAET,MAAMU,OAAO,EAAEoB,SAAS9B,MAAMC,IAAI,EAAE8D,SAAS3E,IAAI,EAAEkE,0BAAzG,UAAA,8BAAA,SAAA,0BAAA,kCAAgH;wBAAhH,IAAM1C,IAAN;wBACHP,MAAMF,IAAI,CAACS;wBACXuD;oBACF;;oBAHK;oBAAA;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;gBAIL;YACF;YAEA,IAAMM,YAAYxB,IAAAA,sBAAW,EAACjD,MAAMkD,IAAI;YACxC,IAAMmB,WAAWN,SAAS3E,IAAI,KAAK,QAAQkF,MAAMG;YAEjD,IAAIzE,MAAMC,IAAI,KAAK,WAAW;gBAC5BmE,QAAQK,WAAWJ,UAAUrE,MAAMS,SAAS,EAAET,MAAMU,OAAO;gBAC3D;YACF;YAEA,wFAAwF;YACxF,wDAAwD;YACxD,IAAMgE,eAAerE,MAAMH,MAAM,GAAG,KAAK8B,SAAShB,iBAAiBwD,cAAcR;YACjF,IAAIU,cAAcP;YAElBC,QAAQK,WAAWJ,UAAUrE,MAAMS,SAAS,EAAET,MAAMU,OAAO;YAC3DwD;YAEA,IAAIA,kBAAkBnF,gBAAgBoF;QACxC;;QA9BK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IA+BLA;IAEA,OAAOF;AACT;AAIO,SAASnF,MAAMe,MAAe,EAAE8E,IAAY,EAAEjF,IAAmB;QAI7BkF;IAHzC,IAAMb,WAAWtE,eAAeC;IAChC,IAAM2D,QAAQsB,KAAKE,KAAK,CAAC;IACzB,IAAMD,SAA0C,EAAE;QAC7C,kCAAA,2BAAA;;QAAL,QAAK,YAAehF,YAAYC,4BAA3B,SAAA,6BAAA,QAAA,yBAAA;YAAA,IAAMiF,QAAN;YAAoCF,CAAAA,UAAAA,QAAOzE,IAAI,OAAXyE,SAAY,qBAAGf,WAAWiB,OAAOzB,OAAOU;;;QAA5E;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,+FAA+F;IAC/F,+FAA+F;IAC/F,gFAAgF;IAChF,IAAMgB,SACJhB,SAAS3E,IAAI,KAAK,QACdwF,OAAOjE,GAAG,CAAC,SAACqE;eACVA,EAAElE,KAAK,GACHkE,IACA,wCACKA;YACH5F,MAAMiE,MACHkB,KAAK,CAACS,EAAEvE,SAAS,GAAG,GAAGuE,EAAEtE,OAAO,EAChCG,IAAI,CAAC,MACLoE,IAAI;;SAGfL;IACN,mHAAmH;IACnH,OAAOG,OAAOG,MAAM,CAAC,SAACF;eAAMA,EAAE5F,IAAI,CAAC6F,IAAI,GAAG/E,MAAM,GAAG;OAAGS,GAAG,CAAC,SAACqE;eAAO;YAAEvE,WAAWuE,EAAEvE,SAAS;YAAEC,SAASsE,EAAEtE,OAAO;YAAEtB,MAAM4F,EAAE5F,IAAI;QAAC;;AAC/H"}

package/dist/cjs/chunk/index.d.cts

@@ -0,0 +1,8 @@
+import type { Block, Chunk, ChunkOptions } from './types.js';
+export declare function chunkFromBlocks(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[];
+export declare function chunk(body: string, opts?: ChunkOptions): Chunk[];
+export { extractText } from './extract.js';
+export { DEFAULT_TARGET_TOKENS, estimateTokens } from './group.js';
+export { parse } from './parse.js';
+export type { Block, BlockType, Chunk, ChunkOptions } from './types.js';
+export { CHUNK_VERSION } from './version.js';

package/dist/cjs/chunk/index.d.ts

@@ -0,0 +1,8 @@
+import type { Block, Chunk, ChunkOptions } from './types.js';
+export declare function chunkFromBlocks(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[];
+export declare function chunk(body: string, opts?: ChunkOptions): Chunk[];
+export { extractText } from './extract.js';
+export { DEFAULT_TARGET_TOKENS, estimateTokens } from './group.js';
+export { parse } from './parse.js';
+export type { Block, BlockType, Chunk, ChunkOptions } from './types.js';
+export { CHUNK_VERSION } from './version.js';

package/dist/cjs/chunk/index.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get CHUNK_VERSION () {
+        return _versionts.CHUNK_VERSION;
+    },
+    get DEFAULT_TARGET_TOKENS () {
+        return _groupts.DEFAULT_TARGET_TOKENS;
+    },
+    get chunk () {
+        return chunk;
+    },
+    get chunkFromBlocks () {
+        return chunkFromBlocks;
+    },
+    get estimateTokens () {
+        return _groupts.estimateTokens;
+    },
+    get extractText () {
+        return _extractts.extractText;
+    },
+    get parse () {
+        return _parsets.parse;
+    }
+});
+var _groupts = require("./group.js");
+var _parsets = require("./parse.js");
+var _extractts = require("./extract.js");
+var _versionts = require("./version.js");
+function chunkFromBlocks(blocks, body, opts) {
+    return (0, _groupts.group)(blocks, body, opts);
+}
+function chunk(body, opts) {
+    return chunkFromBlocks((0, _parsets.parse)(body), body, opts);
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/chunk/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/chunk/index.ts"],"sourcesContent":["import { group } from './group.ts';\nimport { parse } from './parse.ts';\nimport type { Block, Chunk, ChunkOptions } from './types.ts';\n\n// Groups blocks a caller already parsed (e.g. scan/index.ts, sharing one parse with the FTS\n// text path) against the same body they came from, per opts (D1/D3).\nexport function chunkFromBlocks(blocks: Block[], body: string, opts?: ChunkOptions): Chunk[] {\n  return group(blocks, body, opts);\n}\n\n// Pure, deterministic function of file content; chunk semantics are version-stamped via\n// CHUNK_VERSION (./version.ts). Algorithm and evidence: BENCHMARKING.md, \"The chunking algorithm\".\nexport function chunk(body: string, opts?: ChunkOptions): Chunk[] {\n  return chunkFromBlocks(parse(body), body, opts);\n}\n\nexport { extractText } from './extract.ts';\nexport { DEFAULT_TARGET_TOKENS, estimateTokens } from './group.ts';\nexport { parse } from './parse.ts';\nexport type { Block, BlockType, Chunk, ChunkOptions } from './types.ts';\nexport { CHUNK_VERSION } from './version.ts';\n"],"names":["CHUNK_VERSION","DEFAULT_TARGET_TOKENS","chunk","chunkFromBlocks","estimateTokens","extractText","parse","blocks","body","opts","group"],"mappings":";;;;;;;;;;;QAoBSA;eAAAA,wBAAa;;QAHbC;eAAAA,8BAAqB;;QALdC;eAAAA;;QANAC;eAAAA;;QAWgBC;eAAAA,uBAAc;;QADrCC;eAAAA,sBAAW;;QAEXC;eAAAA,cAAK;;;uBAlBQ;uBACA;yBAeM;yBAIE;AAdvB,SAASH,gBAAgBI,MAAe,EAAEC,IAAY,EAAEC,IAAmB;IAChF,OAAOC,IAAAA,cAAK,EAACH,QAAQC,MAAMC;AAC7B;AAIO,SAASP,MAAMM,IAAY,EAAEC,IAAmB;IACrD,OAAON,gBAAgBG,IAAAA,cAAK,EAACE,OAAOA,MAAMC;AAC5C"}

package/dist/cjs/chunk/parse.d.cts

@@ -0,0 +1,2 @@
+import type { Block } from './types.js';
+export declare function parse(body: string): Block[];

package/dist/cjs/chunk/parse.d.ts

@@ -0,0 +1,2 @@
+import type { Block } from './types.js';
+export declare function parse(body: string): Block[];