sensemaking 0.23.0 → 0.24.0

package/dist/cjs/store/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, ResolvedConfig, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'lexical'/'vectors': the store's LexicalIndex/VectorStore is functionally implemented, not\n// present-but-inert. 'sql-functions': the engine can register has/basename/segment as UDFs at all,\n// which turso's client cannot. The rest are finer FTS5-only behaviors; a missing one fails at open\n// or first use. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['phrases', 'snippets', 'lexical', 'vectors', 'sql-functions'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Bulk-inserts rows that cannot conflict, through a path that binds no per-value parameters.\n  // Optional: a store that has nothing faster than its own INSERT omits it and callers fall back\n  // (appendRows in shared.ts). `columns` names the values each row carries; a table column the\n  // caller does not write takes its default.\n  appendRows?(table: string, columns: string[], rows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Adds `names` to frontmatter, already filtered to columns this connection doesn't have yet.\n  // sqlite/turso loop (their ADD COLUMN is metadata-only); duckdb issues one statement per call.\n  addColumns(conn: Connection, names: string[]): Promise<void>;\n  // Deletes content rows for `touched`, inserts rows for `docs`; `delta` carries the tree state a\n  // strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n}\n\n// One open algorithm (src/store/open.ts), parameterised per engine. `Handle` is whatever this\n// store needs to close the connection and construct its Store (sqlite: {db}; duckdb:\n// {instance, duckdb}; turso: db) -- opaque to the shared orchestration, threaded through unchanged.\nexport interface OpenDialect<Handle> {\n  // Cache filename under STATE_DIR, e.g. 'cache.db'.\n  filename: string;\n  // Cache shape version, independent of the config's own `version`; bumping it rebuilds an existing tree.\n  schemaVersion: string;\n  reconcileDialect: ReconcileDialect;\n  // Opens the physical connection and applies pragmas due before any SQL runs (sqlite: busy_timeout\n  // + WAL; turso: connect-time timeout; duckdb: none).\n  connect(dbPath: string, cfg: ResolvedConfig): Promise<{ handle: Handle; conn: Connection }>;\n  // Releases the handle, for both the rebuild-and-reopen branch and error cleanup on this attempt.\n  close(handle: Handle): Promise<void>;\n  // True when connect() failed because another process holds the cache file, which the orchestration\n  // retries. Each engine words it differently, so the match is the dialect's own (`native-not-emulated`).\n  // Absent for sqlite, whose file lock is shared and whose concurrent-open failure is a different defect.\n  // Matching on message text is forced, not chosen: measured 2026-09-02, duckdb throws a plain Error\n  // whose only own properties are stack and message, and turso sets code to the constant\n  // 'GenericFailure' on every failure alike with rawCode undefined. Neither exposes anything a\n  // predicate could switch on, so each dialect pins its engine's wordings and unit-tests them.\n  isLocked?(err: Error): boolean;\n  // Schema DDL beyond frontmatter/preset_files/meta (content table, feature hooks); sole owner of\n  // whether it wraps itself in a write transaction (sqlite: yes, guards a cold-open ALTER race; duckdb/turso: no).\n  ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;\n  // Installs the derived busy_timeout PRAGMA right before reconcile (sqlite/turso); absent for\n  // duckdb, which has no such PRAGMA.\n  setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;\n  createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig): Store;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;;;;;+BAU/EA;;;eAAAA;;;AAAN,IAAMA,mBAAmB;IAAC;IAAW;IAAY;IAAW;IAAW;CAAgB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, ResolvedConfig, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'vectors': the store's VectorStore is functionally implemented, not present-but-inert; checked\n// at open (index.ts) before a config with an \"embed\" block is allowed onto a store lacking it.\n// 'segment': the engine can run segment() (Intl.Segmenter grapheme clustering) as a SQL function;\n// turso's client cannot register it and has no rewrite for it, unlike has()/basename(), which now\n// work everywhere. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['vectors', 'segment'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Bulk-inserts rows that cannot conflict, through a path that binds no per-value parameters.\n  // Optional: a store that has nothing faster than its own INSERT omits it and callers fall back\n  // (appendRows in shared.ts). `columns` names the values each row carries; a table column the\n  // caller does not write takes its default.\n  appendRows?(table: string, columns: string[], rows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Adds `names` to frontmatter, already filtered to columns this connection doesn't have yet.\n  // sqlite/turso loop (their ADD COLUMN is metadata-only); duckdb issues one statement per call.\n  addColumns(conn: Connection, names: string[]): Promise<void>;\n  // Deletes content rows for `touched`, inserts rows for `docs`; `delta` carries the tree state a\n  // strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n}\n\n// One open algorithm (src/store/open.ts), parameterised per engine. `Handle` is whatever this\n// store needs to close the connection and construct its Store (sqlite: {db}; duckdb:\n// {instance, duckdb}; turso: db) -- opaque to the shared orchestration, threaded through unchanged.\nexport interface OpenDialect<Handle> {\n  // Cache filename under STATE_DIR, e.g. 'cache.db'.\n  filename: string;\n  // Cache shape version, independent of the config's own `version`; bumping it rebuilds an existing tree.\n  schemaVersion: string;\n  reconcileDialect: ReconcileDialect;\n  // Opens the physical connection and applies pragmas due before any SQL runs (sqlite: busy_timeout\n  // + WAL; turso: connect-time timeout; duckdb: none).\n  connect(dbPath: string, cfg: ResolvedConfig): Promise<{ handle: Handle; conn: Connection }>;\n  // Releases the handle, for both the rebuild-and-reopen branch and error cleanup on this attempt.\n  close(handle: Handle): Promise<void>;\n  // True when connect() failed because another process holds the cache file, which the orchestration\n  // retries. Each engine words it differently, so the match is the dialect's own (`native-not-emulated`).\n  // Absent for sqlite, whose file lock is shared and whose concurrent-open failure is a different defect.\n  // Matching on message text is forced, not chosen: measured 2026-09-02, duckdb throws a plain Error\n  // whose only own properties are stack and message, and turso sets code to the constant\n  // 'GenericFailure' on every failure alike with rawCode undefined. Neither exposes anything a\n  // predicate could switch on, so each dialect pins its engine's wordings and unit-tests them.\n  isLocked?(err: Error): boolean;\n  // Schema DDL beyond frontmatter/preset_files/meta (content table, feature hooks); sole owner of\n  // whether it wraps itself in a write transaction (sqlite: yes, guards a cold-open ALTER race; duckdb/turso: no).\n  ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;\n  // Installs the derived busy_timeout PRAGMA right before reconcile (sqlite/turso); absent for\n  // duckdb, which has no such PRAGMA.\n  setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;\n  createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig): Store;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\n// A ranked match, nothing more. Snippets are cut from the document above the store\n// (commands/search.ts), identically for every engine, so no store carries them.\nexport interface LexicalHit {\n  path: string;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with snippet, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;;;;;+BAW/EA;;;eAAAA;;;AAAN,IAAMA,mBAAmB;IAAC;IAAW;CAAU"}

package/dist/cjs/store/vectors.d.cts

@@ -2,6 +2,13 @@ import type { Connection } from './types.js';
 export declare const TARGET_CHUNK_CAP = 16;
 export declare function sampleEvenly<T>(rows: T[], cap?: number): T[];
 export declare function asCosine(score: number): number;
+export declare function compareVectorScores(a: {
+    path: string;
+    score: number;
+}, b: {
+    path: string;
+    score: number;
+}): number;
 export declare function pendingRows(conn: Connection): Promise<Array<{
     path: string;
     chunk: number;

package/dist/cjs/store/vectors.d.ts

@@ -2,6 +2,13 @@ import type { Connection } from './types.js';
 export declare const TARGET_CHUNK_CAP = 16;
 export declare function sampleEvenly<T>(rows: T[], cap?: number): T[];
 export declare function asCosine(score: number): number;
+export declare function compareVectorScores(a: {
+    path: string;
+    score: number;
+}, b: {
+    path: string;
+    score: number;
+}): number;
 export declare function pendingRows(conn: Connection): Promise<Array<{
     path: string;
     chunk: number;

package/dist/cjs/store/vectors.js

@@ -15,6 +15,9 @@ _export(exports, {
     get asCosine () {
         return asCosine;
     },
+    get compareVectorScores () {
+        return compareVectorScores;
+    },
     get hasVectorRow () {
         return hasVectorRow;
     },
@@ -161,6 +164,11 @@ function sampleEvenly(rows) {
 function asCosine(score) {
     return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;
 }
+function compareVectorScores(a, b) {
+    var byScore = b.score - a.score;
+    if (byScore !== 0) return byScore;
+    return Buffer.compare(Buffer.from(a.path), Buffer.from(b.path));
+}
 function pendingRows(conn) {
     return _async_to_generator(function() {
         var stmt;

package/dist/cjs/store/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/vectors.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// Seed chunks that participate in a `related` scan. Cost is target_chunks x stored_chunks, so a\n// heading-dense seed multiplies a full-corpus scan (12.7s at 201 chunks/note unsampled).\nexport const TARGET_CHUNK_CAP = 16;\n\n// Evenly samples down to at most `cap` rows, so late sections of a long note still get a vote\n// instead of being cut off by a fixed prefix.\nexport function sampleEvenly<T>(rows: T[], cap: number = TARGET_CHUNK_CAP): T[] {\n  const step = Math.max(1, Math.ceil(rows.length / cap));\n  return rows.filter((_, i) => i % step === 0);\n}\n\n// Dequantised int8 dot products (sqlite) land a little either side of a true cosine (an identical\n// pair can print 1.001); array_cosine_similarity (duckdb) has no such error but is rounded the same way, so both stores print the same bounded number.\nexport function asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\nexport async function pendingRows(conn: Connection): Promise<Array<{ path: string; chunk: number }>> {\n  const stmt = await conn.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk');\n  return (await stmt.all()) as Array<{ path: string; chunk: number }>;\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasVectorRow(conn: Connection, path: string): Promise<boolean> {\n  const stmt = await conn.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1');\n  const row = (await stmt.get(path)) as { ok: number } | undefined;\n  return row !== undefined;\n}\n"],"names":["TARGET_CHUNK_CAP","asCosine","hasVectorRow","pendingRows","sampleEvenly","rows","cap","step","Math","max","ceil","length","filter","_","i","score","round","min","conn","stmt","prepare","all","path","row","get","undefined"],"mappings":";;;;;;;;;;;QAIaA;eAAAA;;QAWGC;eAAAA;;QAWMC;eAAAA;;QAPAC;eAAAA;;QAXNC;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAJT,IAAMJ,mBAAmB;AAIzB,SAASI,aAAgBC,IAAS;QAAEC,MAAAA,iEAAcN;IACvD,IAAMO,OAAOC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,IAAI,CAACL,KAAKM,MAAM,GAAGL;IACjD,OAAOD,KAAKO,MAAM,CAAC,SAACC,GAAGC;eAAMA,IAAIP,SAAS;;AAC5C;AAIO,SAASN,SAASc,KAAa;IACpC,OAAOP,KAAKQ,KAAK,CAACR,KAAKS,GAAG,CAAC,GAAGT,KAAKC,GAAG,CAAC,CAAC,GAAGM,UAAU,QAAQ;AAC/D;AAEO,SAAeZ,YAAYe,IAAgB;;YAC1CC;;;;oBAAO;;wBAAMD,KAAKE,OAAO,CAAC;;;oBAA1BD,OAAO;oBACL;;wBAAMA,KAAKE,GAAG;;;oBAAtB;;wBAAQ;;;;IACV;;AAIO,SAAenB,aAAagB,IAAgB,EAAEI,IAAY;;YACzDH,MACAI;;;;oBADO;;wBAAML,KAAKE,OAAO,CAAC;;;oBAA1BD,OAAO;oBACA;;wBAAMA,KAAKK,GAAG,CAACF;;;oBAAtBC,MAAO;oBACb;;wBAAOA,QAAQE;;;;IACjB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/vectors.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// Seed chunks that participate in a `related` scan. Cost is target_chunks x stored_chunks, so a\n// heading-dense seed multiplies a full-corpus scan (12.7s at 201 chunks/note unsampled).\nexport const TARGET_CHUNK_CAP = 16;\n\n// Evenly samples down to at most `cap` rows, so late sections of a long note still get a vote\n// instead of being cut off by a fixed prefix.\nexport function sampleEvenly<T>(rows: T[], cap: number = TARGET_CHUNK_CAP): T[] {\n  const step = Math.max(1, Math.ceil(rows.length / cap));\n  return rows.filter((_, i) => i % step === 0);\n}\n\n// Every store computes a real cosine; clamp float error at the ends and round for the public score.\nexport function asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\n// Rank by the computed cosine before rounding for display. Bytewise path order resolves only an\n// exact score tie, so two distinguishable similarities never cross a caller's result boundary.\nexport function compareVectorScores(a: { path: string; score: number }, b: { path: string; score: number }): number {\n  const byScore = b.score - a.score;\n  if (byScore !== 0) return byScore;\n  return Buffer.compare(Buffer.from(a.path), Buffer.from(b.path));\n}\n\nexport async function pendingRows(conn: Connection): Promise<Array<{ path: string; chunk: number }>> {\n  const stmt = await conn.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk');\n  return (await stmt.all()) as Array<{ path: string; chunk: number }>;\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasVectorRow(conn: Connection, path: string): Promise<boolean> {\n  const stmt = await conn.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1');\n  const row = (await stmt.get(path)) as { ok: number } | undefined;\n  return row !== undefined;\n}\n"],"names":["TARGET_CHUNK_CAP","asCosine","compareVectorScores","hasVectorRow","pendingRows","sampleEvenly","rows","cap","step","Math","max","ceil","length","filter","_","i","score","round","min","a","b","byScore","Buffer","compare","from","path","conn","stmt","prepare","all","row","get","undefined"],"mappings":";;;;;;;;;;;QAIaA;eAAAA;;QAUGC;eAAAA;;QAMAC;eAAAA;;QAaMC;eAAAA;;QAPAC;eAAAA;;QAlBNC;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAJT,IAAML,mBAAmB;AAIzB,SAASK,aAAgBC,IAAS;QAAEC,MAAAA,iEAAcP;IACvD,IAAMQ,OAAOC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,IAAI,CAACL,KAAKM,MAAM,GAAGL;IACjD,OAAOD,KAAKO,MAAM,CAAC,SAACC,GAAGC;eAAMA,IAAIP,SAAS;;AAC5C;AAGO,SAASP,SAASe,KAAa;IACpC,OAAOP,KAAKQ,KAAK,CAACR,KAAKS,GAAG,CAAC,GAAGT,KAAKC,GAAG,CAAC,CAAC,GAAGM,UAAU,QAAQ;AAC/D;AAIO,SAASd,oBAAoBiB,CAAkC,EAAEC,CAAkC;IACxG,IAAMC,UAAUD,EAAEJ,KAAK,GAAGG,EAAEH,KAAK;IACjC,IAAIK,YAAY,GAAG,OAAOA;IAC1B,OAAOC,OAAOC,OAAO,CAACD,OAAOE,IAAI,CAACL,EAAEM,IAAI,GAAGH,OAAOE,IAAI,CAACJ,EAAEK,IAAI;AAC/D;AAEO,SAAerB,YAAYsB,IAAgB;;YAC1CC;;;;oBAAO;;wBAAMD,KAAKE,OAAO,CAAC;;;oBAA1BD,OAAO;oBACL;;wBAAMA,KAAKE,GAAG;;;oBAAtB;;wBAAQ;;;;IACV;;AAIO,SAAe1B,aAAauB,IAAgB,EAAED,IAAY;;YACzDE,MACAG;;;;oBADO;;wBAAMJ,KAAKE,OAAO,CAAC;;;oBAA1BD,OAAO;oBACA;;wBAAMA,KAAKI,GAAG,CAACN;;;oBAAtBK,MAAO;oBACb;;wBAAOA,QAAQE;;;;IACjB"}

package/dist/cjs/text/segment.d.cts

@@ -1,4 +1,17 @@
 export declare const UNSPACED_SCRIPTS = "\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}\\p{scx=Thai}\\p{scx=Khmer}\\p{scx=Lao}\\p{scx=Myanmar}";
 export declare function hasUnspacedRun(text: string): boolean;
+export interface SearchToken {
+    text: string;
+    start: number;
+    end: number;
+    unspaced: boolean;
+}
+export declare function searchTokens(text: string): SearchToken[];
+export declare function matchesSearchPhrase(text: string, phrase: SearchToken[]): boolean;
+export declare function unspacedRuns(text: string): Array<{
+    start: number;
+    end: number;
+}>;
 export declare function segmentField(text: string): string;
 export declare function segmentMatch(terms: string): string;
+export declare function foldForSearch(text: string): string;

package/dist/cjs/text/segment.d.ts

@@ -1,4 +1,17 @@
 export declare const UNSPACED_SCRIPTS = "\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}\\p{scx=Thai}\\p{scx=Khmer}\\p{scx=Lao}\\p{scx=Myanmar}";
 export declare function hasUnspacedRun(text: string): boolean;
+export interface SearchToken {
+    text: string;
+    start: number;
+    end: number;
+    unspaced: boolean;
+}
+export declare function searchTokens(text: string): SearchToken[];
+export declare function matchesSearchPhrase(text: string, phrase: SearchToken[]): boolean;
+export declare function unspacedRuns(text: string): Array<{
+    start: number;
+    end: number;
+}>;
 export declare function segmentField(text: string): string;
 export declare function segmentMatch(terms: string): string;
+export declare function foldForSearch(text: string): string;

package/dist/cjs/text/segment.js

@@ -1,7 +1,3 @@
-// Word boundaries FTS5's tokenizers cannot find on their own. Contract: seg(query) must appear in
-// seg(document) for any substring query; word mode is context-dependent and breaks that, so grapheme mode (UAX #29) is used.
-// Scripts written without word spaces: a closed set of writing systems.
-// Script_Extensions, not Script: the katakana long-vowel mark ー is Script=Common.
 "use strict";
 Object.defineProperty(exports, "__esModule", {
     value: true
@@ -16,16 +12,29 @@ _export(exports, {
     get UNSPACED_SCRIPTS () {
         return UNSPACED_SCRIPTS;
     },
+    get foldForSearch () {
+        return foldForSearch;
+    },
     get hasUnspacedRun () {
         return hasUnspacedRun;
     },
+    get matchesSearchPhrase () {
+        return matchesSearchPhrase;
+    },
+    get searchTokens () {
+        return searchTokens;
+    },
     get segmentField () {
         return segmentField;
     },
     get segmentMatch () {
         return segmentMatch;
+    },
+    get unspacedRuns () {
+        return unspacedRuns;
     }
 });
+var _stemmer = require("stemmer");
 var UNSPACED_SCRIPTS = '\\p{scx=Han}\\p{scx=Hiragana}\\p{scx=Katakana}\\p{scx=Thai}\\p{scx=Khmer}\\p{scx=Lao}\\p{scx=Myanmar}';
 // A run is script BASE characters with their combining marks attached; a bare mark after a
 // Latin letter (decomposed é) never starts one.
@@ -34,6 +43,99 @@ var HAS_RUN = new RegExp("[".concat(UNSPACED_SCRIPTS, "]"), 'u');
 function hasUnspacedRun(text) {
     return HAS_RUN.test(text);
 }
+function searchTokens(text) {
+    var tokens = [];
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = text.matchAll(RegExp("[\\u0041-\\u005A\\u0061-\\u007A\\u00AA\\u00B5\\u00BA\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02C1\\u02C6-\\u02D1\\u02E0-\\u02E4\\u02EC\\u02EE\\u0370-\\u0374\\u0376-\\u0377\\u037A-\\u037D\\u037F\\u0386\\u0388-\\u038A\\u038C\\u038E-\\u03A1\\u03A3-\\u03F5\\u03F7-\\u0481\\u048A-\\u052F\\u0531-\\u0556\\u0559\\u0560-\\u0588\\u05D0-\\u05EA\\u05EF-\\u05F2\\u0620-\\u064A\\u066E-\\u066F\\u0671-\\u06D3\\u06D5\\u06E5-\\u06E6\\u06EE-\\u06EF\\u06FA-\\u06FC\\u06FF\\u0710\\u0712-\\u072F\\u074D-\\u07A5\\u07B1\\u07CA-\\u07EA\\u07F4-\\u07F5\\u07FA\\u0800-\\u0815\\u081A\\u0824\\u0828\\u0840-\\u0858\\u0860-\\u086A\\u0870-\\u0887\\u0889-\\u088F\\u08A0-\\u08C9\\u0904-\\u0939\\u093D\\u0950\\u0958-\\u0961\\u0971-\\u0980\\u0985-\\u098C\\u098F-\\u0990\\u0993-\\u09A8\\u09AA-\\u09B0\\u09B2\\u09B6-\\u09B9\\u09BD\\u09CE\\u09DC-\\u09DD\\u09DF-\\u09E1\\u09F0-\\u09F1\\u09FC\\u0A05-\\u0A0A\\u0A0F-\\u0A10\\u0A13-\\u0A28\\u0A2A-\\u0A30\\u0A32-\\u0A33\\u0A35-\\u0A36\\u0A38-\\u0A39\\u0A59-\\u0A5C\\u0A5E\\u0A72-\\u0A74\\u0A85-\\u0A8D\\u0A8F-\\u0A91\\u0A93-\\u0AA8\\u0AAA-\\u0AB0\\u0AB2-\\u0AB3\\u0AB5-\\u0AB9\\u0ABD\\u0AD0\\u0AE0-\\u0AE1\\u0AF9\\u0B05-\\u0B0C\\u0B0F-\\u0B10\\u0B13-\\u0B28\\u0B2A-\\u0B30\\u0B32-\\u0B33\\u0B35-\\u0B39\\u0B3D\\u0B5C-\\u0B5D\\u0B5F-\\u0B61\\u0B71\\u0B83\\u0B85-\\u0B8A\\u0B8E-\\u0B90\\u0B92-\\u0B95\\u0B99-\\u0B9A\\u0B9C\\u0B9E-\\u0B9F\\u0BA3-\\u0BA4\\u0BA8-\\u0BAA\\u0BAE-\\u0BB9\\u0BD0\\u0C05-\\u0C0C\\u0C0E-\\u0C10\\u0C12-\\u0C28\\u0C2A-\\u0C39\\u0C3D\\u0C58-\\u0C5A\\u0C5C-\\u0C5D\\u0C60-\\u0C61\\u0C80\\u0C85-\\u0C8C\\u0C8E-\\u0C90\\u0C92-\\u0CA8\\u0CAA-\\u0CB3\\u0CB5-\\u0CB9\\u0CBD\\u0CDC-\\u0CDE\\u0CE0-\\u0CE1\\u0CF1-\\u0CF2\\u0D04-\\u0D0C\\u0D0E-\\u0D10\\u0D12-\\u0D3A\\u0D3D\\u0D4E\\u0D54-\\u0D56\\u0D5F-\\u0D61\\u0D7A-\\u0D7F\\u0D85-\\u0D96\\u0D9A-\\u0DB1\\u0DB3-\\u0DBB\\u0DBD\\u0DC0-\\u0DC6\\u0E01-\\u0E30\\u0E32-\\u0E33\\u0E40-\\u0E46\\u0E81-\\u0E82\\u0E84\\u0E86-\\u0E8A\\u0E8C-\\u0EA3\\u0EA5\\u0EA7-\\u0EB0\\u0EB2-\\u0EB3\\u0EBD\\u0EC0-\\u0EC4\\u0EC6\\u0EDC-\\u0EDF\\u0F00\\u0F40-\\u0F47\\u0F49-\\u0F6C\\u0F88-\\u0F8C\\u1000-\\u102A\\u103F\\u1050-\\u1055\\u105A-\\u105D\\u1061\\u1065-\\u1066\\u106E-\\u1070\\u1075-\\u1081\\u108E\\u10A0-\\u10C5\\u10C7\\u10CD\\u10D0-\\u10FA\\u10FC-\\u1248\\u124A-\\u124D\\u1250-\\u1256\\u1258\\u125A-\\u125D\\u1260-\\u1288\\u128A-\\u128D\\u1290-\\u12B0\\u12B2-\\u12B5\\u12B8-\\u12BE\\u12C0\\u12C2-\\u12C5\\u12C8-\\u12D6\\u12D8-\\u1310\\u1312-\\u1315\\u1318-\\u135A\\u1380-\\u138F\\u13A0-\\u13F5\\u13F8-\\u13FD\\u1401-\\u166C\\u166F-\\u167F\\u1681-\\u169A\\u16A0-\\u16EA\\u16F1-\\u16F8\\u1700-\\u1711\\u171F-\\u1731\\u1740-\\u1751\\u1760-\\u176C\\u176E-\\u1770\\u1780-\\u17B3\\u17D7\\u17DC\\u1820-\\u1878\\u1880-\\u1884\\u1887-\\u18A8\\u18AA\\u18B0-\\u18F5\\u1900-\\u191E\\u1950-\\u196D\\u1970-\\u1974\\u1980-\\u19AB\\u19B0-\\u19C9\\u1A00-\\u1A16\\u1A20-\\u1A54\\u1AA7\\u1B05-\\u1B33\\u1B45-\\u1B4C\\u1B83-\\u1BA0\\u1BAE-\\u1BAF\\u1BBA-\\u1BE5\\u1C00-\\u1C23\\u1C4D-\\u1C4F\\u1C5A-\\u1C7D\\u1C80-\\u1C8A\\u1C90-\\u1CBA\\u1CBD-\\u1CBF\\u1CE9-\\u1CEC\\u1CEE-\\u1CF3\\u1CF5-\\u1CF6\\u1CFA\\u1D00-\\u1DBF\\u1E00-\\u1F15\\u1F18-\\u1F1D\\u1F20-\\u1F45\\u1F48-\\u1F4D\\u1F50-\\u1F57\\u1F59\\u1F5B\\u1F5D\\u1F5F-\\u1F7D\\u1F80-\\u1FB4\\u1FB6-\\u1FBC\\u1FBE\\u1FC2-\\u1FC4\\u1FC6-\\u1FCC\\u1FD0-\\u1FD3\\u1FD6-\\u1FDB\\u1FE0-\\u1FEC\\u1FF2-\\u1FF4\\u1FF6-\\u1FFC\\u2071\\u207F\\u2090-\\u209C\\u2102\\u2107\\u210A-\\u2113\\u2115\\u2119-\\u211D\\u2124\\u2126\\u2128\\u212A-\\u212D\\u212F-\\u2139\\u213C-\\u213F\\u2145-\\u2149\\u214E\\u2183-\\u2184\\u2C00-\\u2CE4\\u2CEB-\\u2CEE\\u2CF2-\\u2CF3\\u2D00-\\u2D25\\u2D27\\u2D2D\\u2D30-\\u2D67\\u2D6F\\u2D80-\\u2D96\\u2DA0-\\u2DA6\\u2DA8-\\u2DAE\\u2DB0-\\u2DB6\\u2DB8-\\u2DBE\\u2DC0-\\u2DC6\\u2DC8-\\u2DCE\\u2DD0-\\u2DD6\\u2DD8-\\u2DDE\\u2E2F\\u3005-\\u3006\\u3031-\\u3035\\u303B-\\u303C\\u3041-\\u3096\\u309D-\\u309F\\u30A1-\\u30FA\\u30FC-\\u30FF\\u3105-\\u312F\\u3131-\\u318E\\u31A0-\\u31BF\\u31F0-\\u31FF\\u3400-\\u4DBF\\u4E00-\\uA48C\\uA4D0-\\uA4FD\\uA500-\\uA60C\\uA610-\\uA61F\\uA62A-\\uA62B\\uA640-\\uA66E\\uA67F-\\uA69D\\uA6A0-\\uA6E5\\uA717-\\uA71F\\uA722-\\uA788\\uA78B-\\uA7DC\\uA7F1-\\uA801\\uA803-\\uA805\\uA807-\\uA80A\\uA80C-\\uA822\\uA840-\\uA873\\uA882-\\uA8B3\\uA8F2-\\uA8F7\\uA8FB\\uA8FD-\\uA8FE\\uA90A-\\uA925\\uA930-\\uA946\\uA960-\\uA97C\\uA984-\\uA9B2\\uA9CF\\uA9E0-\\uA9E4\\uA9E6-\\uA9EF\\uA9FA-\\uA9FE\\uAA00-\\uAA28\\uAA40-\\uAA42\\uAA44-\\uAA4B\\uAA60-\\uAA76\\uAA7A\\uAA7E-\\uAAAF\\uAAB1\\uAAB5-\\uAAB6\\uAAB9-\\uAABD\\uAAC0\\uAAC2\\uAADB-\\uAADD\\uAAE0-\\uAAEA\\uAAF2-\\uAAF4\\uAB01-\\uAB06\\uAB09-\\uAB0E\\uAB11-\\uAB16\\uAB20-\\uAB26\\uAB28-\\uAB2E\\uAB30-\\uAB5A\\uAB5C-\\uAB69\\uAB70-\\uABE2\\uAC00-\\uD7A3\\uD7B0-\\uD7C6\\uD7CB-\\uD7FB\\uF900-\\uFA6D\\uFA70-\\uFAD9\\uFB00-\\uFB06\\uFB13-\\uFB17\\uFB1D\\uFB1F-\\uFB28\\uFB2A-\\uFB36\\uFB38-\\uFB3C\\uFB3E\\uFB40-\\uFB41\\uFB43-\\uFB44\\uFB46-\\uFBB1\\uFBD3-\\uFD3D\\uFD50-\\uFD8F\\uFD92-\\uFDC7\\uFDF0-\\uFDFB\\uFE70-\\uFE74\\uFE76-\\uFEFC\\uFF21-\\uFF3A\\uFF41-\\uFF5A\\uFF66-\\uFFBE\\uFFC2-\\uFFC7\\uFFCA-\\uFFCF\\uFFD2-\\uFFD7\\uFFDA-\\uFFDC\\u{10000}-\\u{1000B}\\u{1000D}-\\u{10026}\\u{10028}-\\u{1003A}\\u{1003C}-\\u{1003D}\\u{1003F}-\\u{1004D}\\u{10050}-\\u{1005D}\\u{10080}-\\u{100FA}\\u{10280}-\\u{1029C}\\u{102A0}-\\u{102D0}\\u{10300}-\\u{1031F}\\u{1032D}-\\u{10340}\\u{10342}-\\u{10349}\\u{10350}-\\u{10375}\\u{10380}-\\u{1039D}\\u{103A0}-\\u{103C3}\\u{103C8}-\\u{103CF}\\u{10400}-\\u{1049D}\\u{104B0}-\\u{104D3}\\u{104D8}-\\u{104FB}\\u{10500}-\\u{10527}\\u{10530}-\\u{10563}\\u{10570}-\\u{1057A}\\u{1057C}-\\u{1058A}\\u{1058C}-\\u{10592}\\u{10594}-\\u{10595}\\u{10597}-\\u{105A1}\\u{105A3}-\\u{105B1}\\u{105B3}-\\u{105B9}\\u{105BB}-\\u{105BC}\\u{105C0}-\\u{105F3}\\u{10600}-\\u{10736}\\u{10740}-\\u{10755}\\u{10760}-\\u{10767}\\u{10780}-\\u{10785}\\u{10787}-\\u{107B0}\\u{107B2}-\\u{107BA}\\u{10800}-\\u{10805}\\u{10808}\\u{1080A}-\\u{10835}\\u{10837}-\\u{10838}\\u{1083C}\\u{1083F}-\\u{10855}\\u{10860}-\\u{10876}\\u{10880}-\\u{1089E}\\u{108E0}-\\u{108F2}\\u{108F4}-\\u{108F5}\\u{10900}-\\u{10915}\\u{10920}-\\u{10939}\\u{10940}-\\u{10959}\\u{10980}-\\u{109B7}\\u{109BE}-\\u{109BF}\\u{10A00}\\u{10A10}-\\u{10A13}\\u{10A15}-\\u{10A17}\\u{10A19}-\\u{10A35}\\u{10A60}-\\u{10A7C}\\u{10A80}-\\u{10A9C}\\u{10AC0}-\\u{10AC7}\\u{10AC9}-\\u{10AE4}\\u{10B00}-\\u{10B35}\\u{10B40}-\\u{10B55}\\u{10B60}-\\u{10B72}\\u{10B80}-\\u{10B91}\\u{10C00}-\\u{10C48}\\u{10C80}-\\u{10CB2}\\u{10CC0}-\\u{10CF2}\\u{10D00}-\\u{10D23}\\u{10D4A}-\\u{10D65}\\u{10D6F}-\\u{10D85}\\u{10E80}-\\u{10EA9}\\u{10EB0}-\\u{10EB1}\\u{10EC2}-\\u{10EC7}\\u{10F00}-\\u{10F1C}\\u{10F27}\\u{10F30}-\\u{10F45}\\u{10F70}-\\u{10F81}\\u{10FB0}-\\u{10FC4}\\u{10FE0}-\\u{10FF6}\\u{11003}-\\u{11037}\\u{11071}-\\u{11072}\\u{11075}\\u{11083}-\\u{110AF}\\u{110D0}-\\u{110E8}\\u{11103}-\\u{11126}\\u{11144}\\u{11147}\\u{11150}-\\u{11172}\\u{11176}\\u{11183}-\\u{111B2}\\u{111C1}-\\u{111C4}\\u{111DA}\\u{111DC}\\u{11200}-\\u{11211}\\u{11213}-\\u{1122B}\\u{1123F}-\\u{11240}\\u{11280}-\\u{11286}\\u{11288}\\u{1128A}-\\u{1128D}\\u{1128F}-\\u{1129D}\\u{1129F}-\\u{112A8}\\u{112B0}-\\u{112DE}\\u{11305}-\\u{1130C}\\u{1130F}-\\u{11310}\\u{11313}-\\u{11328}\\u{1132A}-\\u{11330}\\u{11332}-\\u{11333}\\u{11335}-\\u{11339}\\u{1133D}\\u{11350}\\u{1135D}-\\u{11361}\\u{11380}-\\u{11389}\\u{1138B}\\u{1138E}\\u{11390}-\\u{113B5}\\u{113B7}\\u{113D1}\\u{113D3}\\u{11400}-\\u{11434}\\u{11447}-\\u{1144A}\\u{1145F}-\\u{11461}\\u{11480}-\\u{114AF}\\u{114C4}-\\u{114C5}\\u{114C7}\\u{11580}-\\u{115AE}\\u{115D8}-\\u{115DB}\\u{11600}-\\u{1162F}\\u{11644}\\u{11680}-\\u{116AA}\\u{116B8}\\u{11700}-\\u{1171A}\\u{11740}-\\u{11746}\\u{11800}-\\u{1182B}\\u{118A0}-\\u{118DF}\\u{118FF}-\\u{11906}\\u{11909}\\u{1190C}-\\u{11913}\\u{11915}-\\u{11916}\\u{11918}-\\u{1192F}\\u{1193F}\\u{11941}\\u{119A0}-\\u{119A7}\\u{119AA}-\\u{119D0}\\u{119E1}\\u{119E3}\\u{11A00}\\u{11A0B}-\\u{11A32}\\u{11A3A}\\u{11A50}\\u{11A5C}-\\u{11A89}\\u{11A9D}\\u{11AB0}-\\u{11AF8}\\u{11BC0}-\\u{11BE0}\\u{11C00}-\\u{11C08}\\u{11C0A}-\\u{11C2E}\\u{11C40}\\u{11C72}-\\u{11C8F}\\u{11D00}-\\u{11D06}\\u{11D08}-\\u{11D09}\\u{11D0B}-\\u{11D30}\\u{11D46}\\u{11D60}-\\u{11D65}\\u{11D67}-\\u{11D68}\\u{11D6A}-\\u{11D89}\\u{11D98}\\u{11DB0}-\\u{11DDB}\\u{11EE0}-\\u{11EF2}\\u{11F02}\\u{11F04}-\\u{11F10}\\u{11F12}-\\u{11F33}\\u{11FB0}\\u{12000}-\\u{12399}\\u{12480}-\\u{12543}\\u{12F90}-\\u{12FF0}\\u{13000}-\\u{1342F}\\u{13441}-\\u{13446}\\u{13460}-\\u{143FA}\\u{14400}-\\u{14646}\\u{16100}-\\u{1611D}\\u{16800}-\\u{16A38}\\u{16A40}-\\u{16A5E}\\u{16A70}-\\u{16ABE}\\u{16AD0}-\\u{16AED}\\u{16B00}-\\u{16B2F}\\u{16B40}-\\u{16B43}\\u{16B63}-\\u{16B77}\\u{16B7D}-\\u{16B8F}\\u{16D40}-\\u{16D6C}\\u{16E40}-\\u{16E7F}\\u{16EA0}-\\u{16EB8}\\u{16EBB}-\\u{16ED3}\\u{16F00}-\\u{16F4A}\\u{16F50}\\u{16F93}-\\u{16F9F}\\u{16FE0}-\\u{16FE1}\\u{16FE3}\\u{16FF2}-\\u{16FF3}\\u{17000}-\\u{18CD5}\\u{18CFF}-\\u{18D1E}\\u{18D80}-\\u{18DF2}\\u{1AFF0}-\\u{1AFF3}\\u{1AFF5}-\\u{1AFFB}\\u{1AFFD}-\\u{1AFFE}\\u{1B000}-\\u{1B122}\\u{1B132}\\u{1B150}-\\u{1B152}\\u{1B155}\\u{1B164}-\\u{1B167}\\u{1B170}-\\u{1B2FB}\\u{1BC00}-\\u{1BC6A}\\u{1BC70}-\\u{1BC7C}\\u{1BC80}-\\u{1BC88}\\u{1BC90}-\\u{1BC99}\\u{1D400}-\\u{1D454}\\u{1D456}-\\u{1D49C}\\u{1D49E}-\\u{1D49F}\\u{1D4A2}\\u{1D4A5}-\\u{1D4A6}\\u{1D4A9}-\\u{1D4AC}\\u{1D4AE}-\\u{1D4B9}\\u{1D4BB}\\u{1D4BD}-\\u{1D4C3}\\u{1D4C5}-\\u{1D505}\\u{1D507}-\\u{1D50A}\\u{1D50D}-\\u{1D514}\\u{1D516}-\\u{1D51C}\\u{1D51E}-\\u{1D539}\\u{1D53B}-\\u{1D53E}\\u{1D540}-\\u{1D544}\\u{1D546}\\u{1D54A}-\\u{1D550}\\u{1D552}-\\u{1D6A5}\\u{1D6A8}-\\u{1D6C0}\\u{1D6C2}-\\u{1D6DA}\\u{1D6DC}-\\u{1D6FA}\\u{1D6FC}-\\u{1D714}\\u{1D716}-\\u{1D734}\\u{1D736}-\\u{1D74E}\\u{1D750}-\\u{1D76E}\\u{1D770}-\\u{1D788}\\u{1D78A}-\\u{1D7A8}\\u{1D7AA}-\\u{1D7C2}\\u{1D7C4}-\\u{1D7CB}\\u{1DF00}-\\u{1DF1E}\\u{1DF25}-\\u{1DF2A}\\u{1E030}-\\u{1E06D}\\u{1E100}-\\u{1E12C}\\u{1E137}-\\u{1E13D}\\u{1E14E}\\u{1E290}-\\u{1E2AD}\\u{1E2C0}-\\u{1E2EB}\\u{1E4D0}-\\u{1E4EB}\\u{1E5D0}-\\u{1E5ED}\\u{1E5F0}\\u{1E6C0}-\\u{1E6DE}\\u{1E6E0}-\\u{1E6E2}\\u{1E6E4}-\\u{1E6E5}\\u{1E6E7}-\\u{1E6ED}\\u{1E6F0}-\\u{1E6F4}\\u{1E6FE}-\\u{1E6FF}\\u{1E7E0}-\\u{1E7E6}\\u{1E7E8}-\\u{1E7EB}\\u{1E7ED}-\\u{1E7EE}\\u{1E7F0}-\\u{1E7FE}\\u{1E800}-\\u{1E8C4}\\u{1E900}-\\u{1E943}\\u{1E94B}\\u{1EE00}-\\u{1EE03}\\u{1EE05}-\\u{1EE1F}\\u{1EE21}-\\u{1EE22}\\u{1EE24}\\u{1EE27}\\u{1EE29}-\\u{1EE32}\\u{1EE34}-\\u{1EE37}\\u{1EE39}\\u{1EE3B}\\u{1EE42}\\u{1EE47}\\u{1EE49}\\u{1EE4B}\\u{1EE4D}-\\u{1EE4F}\\u{1EE51}-\\u{1EE52}\\u{1EE54}\\u{1EE57}\\u{1EE59}\\u{1EE5B}\\u{1EE5D}\\u{1EE5F}\\u{1EE61}-\\u{1EE62}\\u{1EE64}\\u{1EE67}-\\u{1EE6A}\\u{1EE6C}-\\u{1EE72}\\u{1EE74}-\\u{1EE77}\\u{1EE79}-\\u{1EE7C}\\u{1EE7E}\\u{1EE80}-\\u{1EE89}\\u{1EE8B}-\\u{1EE9B}\\u{1EEA1}-\\u{1EEA3}\\u{1EEA5}-\\u{1EEA9}\\u{1EEAB}-\\u{1EEBB}\\u{20000}-\\u{2A6DF}\\u{2A700}-\\u{2B81D}\\u{2B820}-\\u{2CEAD}\\u{2CEB0}-\\u{2EBE0}\\u{2EBF0}-\\u{2EE5D}\\u{2F800}-\\u{2FA1D}\\u{30000}-\\u{3134A}\\u{31350}-\\u{33479}\\u0030-\\u0039\\u00B2-\\u00B3\\u00B9\\u00BC-\\u00BE\\u0660-\\u0669\\u06F0-\\u06F9\\u07C0-\\u07C9\\u0966-\\u096F\\u09E6-\\u09EF\\u09F4-\\u09F9\\u0A66-\\u0A6F\\u0AE6-\\u0AEF\\u0B66-\\u0B6F\\u0B72-\\u0B77\\u0BE6-\\u0BF2\\u0C66-\\u0C6F\\u0C78-\\u0C7E\\u0CE6-\\u0CEF\\u0D58-\\u0D5E\\u0D66-\\u0D78\\u0DE6-\\u0DEF\\u0E50-\\u0E59\\u0ED0-\\u0ED9\\u0F20-\\u0F33\\u1040-\\u1049\\u1090-\\u1099\\u1369-\\u137C\\u16EE-\\u16F0\\u17E0-\\u17E9\\u17F0-\\u17F9\\u1810-\\u1819\\u1946-\\u194F\\u19D0-\\u19DA\\u1A80-\\u1A89\\u1A90-\\u1A99\\u1B50-\\u1B59\\u1BB0-\\u1BB9\\u1C40-\\u1C49\\u1C50-\\u1C59\\u2070\\u2074-\\u2079\\u2080-\\u2089\\u2150-\\u2182\\u2185-\\u2189\\u2460-\\u249B\\u24EA-\\u24FF\\u2776-\\u2793\\u2CFD\\u3007\\u3021-\\u3029\\u3038-\\u303A\\u3192-\\u3195\\u3220-\\u3229\\u3248-\\u324F\\u3251-\\u325F\\u3280-\\u3289\\u32B1-\\u32BF\\uA620-\\uA629\\uA6E6-\\uA6EF\\uA830-\\uA835\\uA8D0-\\uA8D9\\uA900-\\uA909\\uA9D0-\\uA9D9\\uA9F0-\\uA9F9\\uAA50-\\uAA59\\uABF0-\\uABF9\\uFF10-\\uFF19\\u{10107}-\\u{10133}\\u{10140}-\\u{10178}\\u{1018A}-\\u{1018B}\\u{102E1}-\\u{102FB}\\u{10320}-\\u{10323}\\u{10341}\\u{1034A}\\u{103D1}-\\u{103D5}\\u{104A0}-\\u{104A9}\\u{10858}-\\u{1085F}\\u{10879}-\\u{1087F}\\u{108A7}-\\u{108AF}\\u{108FB}-\\u{108FF}\\u{10916}-\\u{1091B}\\u{109BC}-\\u{109BD}\\u{109C0}-\\u{109CF}\\u{109D2}-\\u{109FF}\\u{10A40}-\\u{10A48}\\u{10A7D}-\\u{10A7E}\\u{10A9D}-\\u{10A9F}\\u{10AEB}-\\u{10AEF}\\u{10B58}-\\u{10B5F}\\u{10B78}-\\u{10B7F}\\u{10BA9}-\\u{10BAF}\\u{10CFA}-\\u{10CFF}\\u{10D30}-\\u{10D39}\\u{10D40}-\\u{10D49}\\u{10E60}-\\u{10E7E}\\u{10F1D}-\\u{10F26}\\u{10F51}-\\u{10F54}\\u{10FC5}-\\u{10FCB}\\u{11052}-\\u{1106F}\\u{110F0}-\\u{110F9}\\u{11136}-\\u{1113F}\\u{111D0}-\\u{111D9}\\u{111E1}-\\u{111F4}\\u{112F0}-\\u{112F9}\\u{11450}-\\u{11459}\\u{114D0}-\\u{114D9}\\u{11650}-\\u{11659}\\u{116C0}-\\u{116C9}\\u{116D0}-\\u{116E3}\\u{11730}-\\u{1173B}\\u{118E0}-\\u{118F2}\\u{11950}-\\u{11959}\\u{11BF0}-\\u{11BF9}\\u{11C50}-\\u{11C6C}\\u{11D50}-\\u{11D59}\\u{11DA0}-\\u{11DA9}\\u{11DE0}-\\u{11DE9}\\u{11F50}-\\u{11F59}\\u{11FC0}-\\u{11FD4}\\u{12400}-\\u{1246E}\\u{16130}-\\u{16139}\\u{16A60}-\\u{16A69}\\u{16AC0}-\\u{16AC9}\\u{16B50}-\\u{16B59}\\u{16B5B}-\\u{16B61}\\u{16D70}-\\u{16D79}\\u{16E80}-\\u{16E96}\\u{16FF4}-\\u{16FF6}\\u{1CCF0}-\\u{1CCF9}\\u{1D2C0}-\\u{1D2D3}\\u{1D2E0}-\\u{1D2F3}\\u{1D360}-\\u{1D378}\\u{1D7CE}-\\u{1D7FF}\\u{1E140}-\\u{1E149}\\u{1E2F0}-\\u{1E2F9}\\u{1E4F0}-\\u{1E4F9}\\u{1E5F1}-\\u{1E5FA}\\u{1E8C7}-\\u{1E8CF}\\u{1E950}-\\u{1E959}\\u{1EC71}-\\u{1ECAB}\\u{1ECAD}-\\u{1ECAF}\\u{1ECB1}-\\u{1ECB4}\\u{1ED01}-\\u{1ED2D}\\u{1ED2F}-\\u{1ED3D}\\u{1F100}-\\u{1F10C}\\u{1FBF0}-\\u{1FBF9}][\\u0041-\\u005A\\u0061-\\u007A\\u00AA\\u00B5\\u00BA\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02C1\\u02C6-\\u02D1\\u02E0-\\u02E4\\u02EC\\u02EE\\u0370-\\u0374\\u0376-\\u0377\\u037A-\\u037D\\u037F\\u0386\\u0388-\\u038A\\u038C\\u038E-\\u03A1\\u03A3-\\u03F5\\u03F7-\\u0481\\u048A-\\u052F\\u0531-\\u0556\\u0559\\u0560-\\u0588\\u05D0-\\u05EA\\u05EF-\\u05F2\\u0620-\\u064A\\u066E-\\u066F\\u0671-\\u06D3\\u06D5\\u06E5-\\u06E6\\u06EE-\\u06EF\\u06FA-\\u06FC\\u06FF\\u0710\\u0712-\\u072F\\u074D-\\u07A5\\u07B1\\u07CA-\\u07EA\\u07F4-\\u07F5\\u07FA\\u0800-\\u0815\\u081A\\u0824\\u0828\\u0840-\\u0858\\u0860-\\u086A\\u0870-\\u0887\\u0889-\\u088F\\u08A0-\\u08C9\\u0904-\\u0939\\u093D\\u0950\\u0958-\\u0961\\u0971-\\u0980\\u0985-\\u098C\\u098F-\\u0990\\u0993-\\u09A8\\u09AA-\\u09B0\\u09B2\\u09B6-\\u09B9\\u09BD\\u09CE\\u09DC-\\u09DD\\u09DF-\\u09E1\\u09F0-\\u09F1\\u09FC\\u0A05-\\u0A0A\\u0A0F-\\u0A10\\u0A13-\\u0A28\\u0A2A-\\u0A30\\u0A32-\\u0A33\\u0A35-\\u0A36\\u0A38-\\u0A39\\u0A59-\\u0A5C\\u0A5E\\u0A72-\\u0A74\\u0A85-\\u0A8D\\u0A8F-\\u0A91\\u0A93-\\u0AA8\\u0AAA-\\u0AB0\\u0AB2-\\u0AB3\\u0AB5-\\u0AB9\\u0ABD\\u0AD0\\u0AE0-\\u0AE1\\u0AF9\\u0B05-\\u0B0C\\u0B0F-\\u0B10\\u0B13-\\u0B28\\u0B2A-\\u0B30\\u0B32-\\u0B33\\u0B35-\\u0B39\\u0B3D\\u0B5C-\\u0B5D\\u0B5F-\\u0B61\\u0B71\\u0B83\\u0B85-\\u0B8A\\u0B8E-\\u0B90\\u0B92-\\u0B95\\u0B99-\\u0B9A\\u0B9C\\u0B9E-\\u0B9F\\u0BA3-\\u0BA4\\u0BA8-\\u0BAA\\u0BAE-\\u0BB9\\u0BD0\\u0C05-\\u0C0C\\u0C0E-\\u0C10\\u0C12-\\u0C28\\u0C2A-\\u0C39\\u0C3D\\u0C58-\\u0C5A\\u0C5C-\\u0C5D\\u0C60-\\u0C61\\u0C80\\u0C85-\\u0C8C\\u0C8E-\\u0C90\\u0C92-\\u0CA8\\u0CAA-\\u0CB3\\u0CB5-\\u0CB9\\u0CBD\\u0CDC-\\u0CDE\\u0CE0-\\u0CE1\\u0CF1-\\u0CF2\\u0D04-\\u0D0C\\u0D0E-\\u0D10\\u0D12-\\u0D3A\\u0D3D\\u0D4E\\u0D54-\\u0D56\\u0D5F-\\u0D61\\u0D7A-\\u0D7F\\u0D85-\\u0D96\\u0D9A-\\u0DB1\\u0DB3-\\u0DBB\\u0DBD\\u0DC0-\\u0DC6\\u0E01-\\u0E30\\u0E32-\\u0E33\\u0E40-\\u0E46\\u0E81-\\u0E82\\u0E84\\u0E86-\\u0E8A\\u0E8C-\\u0EA3\\u0EA5\\u0EA7-\\u0EB0\\u0EB2-\\u0EB3\\u0EBD\\u0EC0-\\u0EC4\\u0EC6\\u0EDC-\\u0EDF\\u0F00\\u0F40-\\u0F47\\u0F49-\\u0F6C\\u0F88-\\u0F8C\\u1000-\\u102A\\u103F\\u1050-\\u1055\\u105A-\\u105D\\u1061\\u1065-\\u1066\\u106E-\\u1070\\u1075-\\u1081\\u108E\\u10A0-\\u10C5\\u10C7\\u10CD\\u10D0-\\u10FA\\u10FC-\\u1248\\u124A-\\u124D\\u1250-\\u1256\\u1258\\u125A-\\u125D\\u1260-\\u1288\\u128A-\\u128D\\u1290-\\u12B0\\u12B2-\\u12B5\\u12B8-\\u12BE\\u12C0\\u12C2-\\u12C5\\u12C8-\\u12D6\\u12D8-\\u1310\\u1312-\\u1315\\u1318-\\u135A\\u1380-\\u138F\\u13A0-\\u13F5\\u13F8-\\u13FD\\u1401-\\u166C\\u166F-\\u167F\\u1681-\\u169A\\u16A0-\\u16EA\\u16F1-\\u16F8\\u1700-\\u1711\\u171F-\\u1731\\u1740-\\u1751\\u1760-\\u176C\\u176E-\\u1770\\u1780-\\u17B3\\u17D7\\u17DC\\u1820-\\u1878\\u1880-\\u1884\\u1887-\\u18A8\\u18AA\\u18B0-\\u18F5\\u1900-\\u191E\\u1950-\\u196D\\u1970-\\u1974\\u1980-\\u19AB\\u19B0-\\u19C9\\u1A00-\\u1A16\\u1A20-\\u1A54\\u1AA7\\u1B05-\\u1B33\\u1B45-\\u1B4C\\u1B83-\\u1BA0\\u1BAE-\\u1BAF\\u1BBA-\\u1BE5\\u1C00-\\u1C23\\u1C4D-\\u1C4F\\u1C5A-\\u1C7D\\u1C80-\\u1C8A\\u1C90-\\u1CBA\\u1CBD-\\u1CBF\\u1CE9-\\u1CEC\\u1CEE-\\u1CF3\\u1CF5-\\u1CF6\\u1CFA\\u1D00-\\u1DBF\\u1E00-\\u1F15\\u1F18-\\u1F1D\\u1F20-\\u1F45\\u1F48-\\u1F4D\\u1F50-\\u1F57\\u1F59\\u1F5B\\u1F5D\\u1F5F-\\u1F7D\\u1F80-\\u1FB4\\u1FB6-\\u1FBC\\u1FBE\\u1FC2-\\u1FC4\\u1FC6-\\u1FCC\\u1FD0-\\u1FD3\\u1FD6-\\u1FDB\\u1FE0-\\u1FEC\\u1FF2-\\u1FF4\\u1FF6-\\u1FFC\\u2071\\u207F\\u2090-\\u209C\\u2102\\u2107\\u210A-\\u2113\\u2115\\u2119-\\u211D\\u2124\\u2126\\u2128\\u212A-\\u212D\\u212F-\\u2139\\u213C-\\u213F\\u2145-\\u2149\\u214E\\u2183-\\u2184\\u2C00-\\u2CE4\\u2CEB-\\u2CEE\\u2CF2-\\u2CF3\\u2D00-\\u2D25\\u2D27\\u2D2D\\u2D30-\\u2D67\\u2D6F\\u2D80-\\u2D96\\u2DA0-\\u2DA6\\u2DA8-\\u2DAE\\u2DB0-\\u2DB6\\u2DB8-\\u2DBE\\u2DC0-\\u2DC6\\u2DC8-\\u2DCE\\u2DD0-\\u2DD6\\u2DD8-\\u2DDE\\u2E2F\\u3005-\\u3006\\u3031-\\u3035\\u303B-\\u303C\\u3041-\\u3096\\u309D-\\u309F\\u30A1-\\u30FA\\u30FC-\\u30FF\\u3105-\\u312F\\u3131-\\u318E\\u31A0-\\u31BF\\u31F0-\\u31FF\\u3400-\\u4DBF\\u4E00-\\uA48C\\uA4D0-\\uA4FD\\uA500-\\uA60C\\uA610-\\uA61F\\uA62A-\\uA62B\\uA640-\\uA66E\\uA67F-\\uA69D\\uA6A0-\\uA6E5\\uA717-\\uA71F\\uA722-\\uA788\\uA78B-\\uA7DC\\uA7F1-\\uA801\\uA803-\\uA805\\uA807-\\uA80A\\uA80C-\\uA822\\uA840-\\uA873\\uA882-\\uA8B3\\uA8F2-\\uA8F7\\uA8FB\\uA8FD-\\uA8FE\\uA90A-\\uA925\\uA930-\\uA946\\uA960-\\uA97C\\uA984-\\uA9B2\\uA9CF\\uA9E0-\\uA9E4\\uA9E6-\\uA9EF\\uA9FA-\\uA9FE\\uAA00-\\uAA28\\uAA40-\\uAA42\\uAA44-\\uAA4B\\uAA60-\\uAA76\\uAA7A\\uAA7E-\\uAAAF\\uAAB1\\uAAB5-\\uAAB6\\uAAB9-\\uAABD\\uAAC0\\uAAC2\\uAADB-\\uAADD\\uAAE0-\\uAAEA\\uAAF2-\\uAAF4\\uAB01-\\uAB06\\uAB09-\\uAB0E\\uAB11-\\uAB16\\uAB20-\\uAB26\\uAB28-\\uAB2E\\uAB30-\\uAB5A\\uAB5C-\\uAB69\\uAB70-\\uABE2\\uAC00-\\uD7A3\\uD7B0-\\uD7C6\\uD7CB-\\uD7FB\\uF900-\\uFA6D\\uFA70-\\uFAD9\\uFB00-\\uFB06\\uFB13-\\uFB17\\uFB1D\\uFB1F-\\uFB28\\uFB2A-\\uFB36\\uFB38-\\uFB3C\\uFB3E\\uFB40-\\uFB41\\uFB43-\\uFB44\\uFB46-\\uFBB1\\uFBD3-\\uFD3D\\uFD50-\\uFD8F\\uFD92-\\uFDC7\\uFDF0-\\uFDFB\\uFE70-\\uFE74\\uFE76-\\uFEFC\\uFF21-\\uFF3A\\uFF41-\\uFF5A\\uFF66-\\uFFBE\\uFFC2-\\uFFC7\\uFFCA-\\uFFCF\\uFFD2-\\uFFD7\\uFFDA-\\uFFDC\\u{10000}-\\u{1000B}\\u{1000D}-\\u{10026}\\u{10028}-\\u{1003A}\\u{1003C}-\\u{1003D}\\u{1003F}-\\u{1004D}\\u{10050}-\\u{1005D}\\u{10080}-\\u{100FA}\\u{10280}-\\u{1029C}\\u{102A0}-\\u{102D0}\\u{10300}-\\u{1031F}\\u{1032D}-\\u{10340}\\u{10342}-\\u{10349}\\u{10350}-\\u{10375}\\u{10380}-\\u{1039D}\\u{103A0}-\\u{103C3}\\u{103C8}-\\u{103CF}\\u{10400}-\\u{1049D}\\u{104B0}-\\u{104D3}\\u{104D8}-\\u{104FB}\\u{10500}-\\u{10527}\\u{10530}-\\u{10563}\\u{10570}-\\u{1057A}\\u{1057C}-\\u{1058A}\\u{1058C}-\\u{10592}\\u{10594}-\\u{10595}\\u{10597}-\\u{105A1}\\u{105A3}-\\u{105B1}\\u{105B3}-\\u{105B9}\\u{105BB}-\\u{105BC}\\u{105C0}-\\u{105F3}\\u{10600}-\\u{10736}\\u{10740}-\\u{10755}\\u{10760}-\\u{10767}\\u{10780}-\\u{10785}\\u{10787}-\\u{107B0}\\u{107B2}-\\u{107BA}\\u{10800}-\\u{10805}\\u{10808}\\u{1080A}-\\u{10835}\\u{10837}-\\u{10838}\\u{1083C}\\u{1083F}-\\u{10855}\\u{10860}-\\u{10876}\\u{10880}-\\u{1089E}\\u{108E0}-\\u{108F2}\\u{108F4}-\\u{108F5}\\u{10900}-\\u{10915}\\u{10920}-\\u{10939}\\u{10940}-\\u{10959}\\u{10980}-\\u{109B7}\\u{109BE}-\\u{109BF}\\u{10A00}\\u{10A10}-\\u{10A13}\\u{10A15}-\\u{10A17}\\u{10A19}-\\u{10A35}\\u{10A60}-\\u{10A7C}\\u{10A80}-\\u{10A9C}\\u{10AC0}-\\u{10AC7}\\u{10AC9}-\\u{10AE4}\\u{10B00}-\\u{10B35}\\u{10B40}-\\u{10B55}\\u{10B60}-\\u{10B72}\\u{10B80}-\\u{10B91}\\u{10C00}-\\u{10C48}\\u{10C80}-\\u{10CB2}\\u{10CC0}-\\u{10CF2}\\u{10D00}-\\u{10D23}\\u{10D4A}-\\u{10D65}\\u{10D6F}-\\u{10D85}\\u{10E80}-\\u{10EA9}\\u{10EB0}-\\u{10EB1}\\u{10EC2}-\\u{10EC7}\\u{10F00}-\\u{10F1C}\\u{10F27}\\u{10F30}-\\u{10F45}\\u{10F70}-\\u{10F81}\\u{10FB0}-\\u{10FC4}\\u{10FE0}-\\u{10FF6}\\u{11003}-\\u{11037}\\u{11071}-\\u{11072}\\u{11075}\\u{11083}-\\u{110AF}\\u{110D0}-\\u{110E8}\\u{11103}-\\u{11126}\\u{11144}\\u{11147}\\u{11150}-\\u{11172}\\u{11176}\\u{11183}-\\u{111B2}\\u{111C1}-\\u{111C4}\\u{111DA}\\u{111DC}\\u{11200}-\\u{11211}\\u{11213}-\\u{1122B}\\u{1123F}-\\u{11240}\\u{11280}-\\u{11286}\\u{11288}\\u{1128A}-\\u{1128D}\\u{1128F}-\\u{1129D}\\u{1129F}-\\u{112A8}\\u{112B0}-\\u{112DE}\\u{11305}-\\u{1130C}\\u{1130F}-\\u{11310}\\u{11313}-\\u{11328}\\u{1132A}-\\u{11330}\\u{11332}-\\u{11333}\\u{11335}-\\u{11339}\\u{1133D}\\u{11350}\\u{1135D}-\\u{11361}\\u{11380}-\\u{11389}\\u{1138B}\\u{1138E}\\u{11390}-\\u{113B5}\\u{113B7}\\u{113D1}\\u{113D3}\\u{11400}-\\u{11434}\\u{11447}-\\u{1144A}\\u{1145F}-\\u{11461}\\u{11480}-\\u{114AF}\\u{114C4}-\\u{114C5}\\u{114C7}\\u{11580}-\\u{115AE}\\u{115D8}-\\u{115DB}\\u{11600}-\\u{1162F}\\u{11644}\\u{11680}-\\u{116AA}\\u{116B8}\\u{11700}-\\u{1171A}\\u{11740}-\\u{11746}\\u{11800}-\\u{1182B}\\u{118A0}-\\u{118DF}\\u{118FF}-\\u{11906}\\u{11909}\\u{1190C}-\\u{11913}\\u{11915}-\\u{11916}\\u{11918}-\\u{1192F}\\u{1193F}\\u{11941}\\u{119A0}-\\u{119A7}\\u{119AA}-\\u{119D0}\\u{119E1}\\u{119E3}\\u{11A00}\\u{11A0B}-\\u{11A32}\\u{11A3A}\\u{11A50}\\u{11A5C}-\\u{11A89}\\u{11A9D}\\u{11AB0}-\\u{11AF8}\\u{11BC0}-\\u{11BE0}\\u{11C00}-\\u{11C08}\\u{11C0A}-\\u{11C2E}\\u{11C40}\\u{11C72}-\\u{11C8F}\\u{11D00}-\\u{11D06}\\u{11D08}-\\u{11D09}\\u{11D0B}-\\u{11D30}\\u{11D46}\\u{11D60}-\\u{11D65}\\u{11D67}-\\u{11D68}\\u{11D6A}-\\u{11D89}\\u{11D98}\\u{11DB0}-\\u{11DDB}\\u{11EE0}-\\u{11EF2}\\u{11F02}\\u{11F04}-\\u{11F10}\\u{11F12}-\\u{11F33}\\u{11FB0}\\u{12000}-\\u{12399}\\u{12480}-\\u{12543}\\u{12F90}-\\u{12FF0}\\u{13000}-\\u{1342F}\\u{13441}-\\u{13446}\\u{13460}-\\u{143FA}\\u{14400}-\\u{14646}\\u{16100}-\\u{1611D}\\u{16800}-\\u{16A38}\\u{16A40}-\\u{16A5E}\\u{16A70}-\\u{16ABE}\\u{16AD0}-\\u{16AED}\\u{16B00}-\\u{16B2F}\\u{16B40}-\\u{16B43}\\u{16B63}-\\u{16B77}\\u{16B7D}-\\u{16B8F}\\u{16D40}-\\u{16D6C}\\u{16E40}-\\u{16E7F}\\u{16EA0}-\\u{16EB8}\\u{16EBB}-\\u{16ED3}\\u{16F00}-\\u{16F4A}\\u{16F50}\\u{16F93}-\\u{16F9F}\\u{16FE0}-\\u{16FE1}\\u{16FE3}\\u{16FF2}-\\u{16FF3}\\u{17000}-\\u{18CD5}\\u{18CFF}-\\u{18D1E}\\u{18D80}-\\u{18DF2}\\u{1AFF0}-\\u{1AFF3}\\u{1AFF5}-\\u{1AFFB}\\u{1AFFD}-\\u{1AFFE}\\u{1B000}-\\u{1B122}\\u{1B132}\\u{1B150}-\\u{1B152}\\u{1B155}\\u{1B164}-\\u{1B167}\\u{1B170}-\\u{1B2FB}\\u{1BC00}-\\u{1BC6A}\\u{1BC70}-\\u{1BC7C}\\u{1BC80}-\\u{1BC88}\\u{1BC90}-\\u{1BC99}\\u{1D400}-\\u{1D454}\\u{1D456}-\\u{1D49C}\\u{1D49E}-\\u{1D49F}\\u{1D4A2}\\u{1D4A5}-\\u{1D4A6}\\u{1D4A9}-\\u{1D4AC}\\u{1D4AE}-\\u{1D4B9}\\u{1D4BB}\\u{1D4BD}-\\u{1D4C3}\\u{1D4C5}-\\u{1D505}\\u{1D507}-\\u{1D50A}\\u{1D50D}-\\u{1D514}\\u{1D516}-\\u{1D51C}\\u{1D51E}-\\u{1D539}\\u{1D53B}-\\u{1D53E}\\u{1D540}-\\u{1D544}\\u{1D546}\\u{1D54A}-\\u{1D550}\\u{1D552}-\\u{1D6A5}\\u{1D6A8}-\\u{1D6C0}\\u{1D6C2}-\\u{1D6DA}\\u{1D6DC}-\\u{1D6FA}\\u{1D6FC}-\\u{1D714}\\u{1D716}-\\u{1D734}\\u{1D736}-\\u{1D74E}\\u{1D750}-\\u{1D76E}\\u{1D770}-\\u{1D788}\\u{1D78A}-\\u{1D7A8}\\u{1D7AA}-\\u{1D7C2}\\u{1D7C4}-\\u{1D7CB}\\u{1DF00}-\\u{1DF1E}\\u{1DF25}-\\u{1DF2A}\\u{1E030}-\\u{1E06D}\\u{1E100}-\\u{1E12C}\\u{1E137}-\\u{1E13D}\\u{1E14E}\\u{1E290}-\\u{1E2AD}\\u{1E2C0}-\\u{1E2EB}\\u{1E4D0}-\\u{1E4EB}\\u{1E5D0}-\\u{1E5ED}\\u{1E5F0}\\u{1E6C0}-\\u{1E6DE}\\u{1E6E0}-\\u{1E6E2}\\u{1E6E4}-\\u{1E6E5}\\u{1E6E7}-\\u{1E6ED}\\u{1E6F0}-\\u{1E6F4}\\u{1E6FE}-\\u{1E6FF}\\u{1E7E0}-\\u{1E7E6}\\u{1E7E8}-\\u{1E7EB}\\u{1E7ED}-\\u{1E7EE}\\u{1E7F0}-\\u{1E7FE}\\u{1E800}-\\u{1E8C4}\\u{1E900}-\\u{1E943}\\u{1E94B}\\u{1EE00}-\\u{1EE03}\\u{1EE05}-\\u{1EE1F}\\u{1EE21}-\\u{1EE22}\\u{1EE24}\\u{1EE27}\\u{1EE29}-\\u{1EE32}\\u{1EE34}-\\u{1EE37}\\u{1EE39}\\u{1EE3B}\\u{1EE42}\\u{1EE47}\\u{1EE49}\\u{1EE4B}\\u{1EE4D}-\\u{1EE4F}\\u{1EE51}-\\u{1EE52}\\u{1EE54}\\u{1EE57}\\u{1EE59}\\u{1EE5B}\\u{1EE5D}\\u{1EE5F}\\u{1EE61}-\\u{1EE62}\\u{1EE64}\\u{1EE67}-\\u{1EE6A}\\u{1EE6C}-\\u{1EE72}\\u{1EE74}-\\u{1EE77}\\u{1EE79}-\\u{1EE7C}\\u{1EE7E}\\u{1EE80}-\\u{1EE89}\\u{1EE8B}-\\u{1EE9B}\\u{1EEA1}-\\u{1EEA3}\\u{1EEA5}-\\u{1EEA9}\\u{1EEAB}-\\u{1EEBB}\\u{20000}-\\u{2A6DF}\\u{2A700}-\\u{2B81D}\\u{2B820}-\\u{2CEAD}\\u{2CEB0}-\\u{2EBE0}\\u{2EBF0}-\\u{2EE5D}\\u{2F800}-\\u{2FA1D}\\u{30000}-\\u{3134A}\\u{31350}-\\u{33479}\\u0300-\\u036F\\u0483-\\u0489\\u0591-\\u05BD\\u05BF\\u05C1-\\u05C2\\u05C4-\\u05C5\\u05C7\\u0610-\\u061A\\u064B-\\u065F\\u0670\\u06D6-\\u06DC\\u06DF-\\u06E4\\u06E7-\\u06E8\\u06EA-\\u06ED\\u0711\\u0730-\\u074A\\u07A6-\\u07B0\\u07EB-\\u07F3\\u07FD\\u0816-\\u0819\\u081B-\\u0823\\u0825-\\u0827\\u0829-\\u082D\\u0859-\\u085B\\u0897-\\u089F\\u08CA-\\u08E1\\u08E3-\\u0903\\u093A-\\u093C\\u093E-\\u094F\\u0951-\\u0957\\u0962-\\u0963\\u0981-\\u0983\\u09BC\\u09BE-\\u09C4\\u09C7-\\u09C8\\u09CB-\\u09CD\\u09D7\\u09E2-\\u09E3\\u09FE\\u0A01-\\u0A03\\u0A3C\\u0A3E-\\u0A42\\u0A47-\\u0A48\\u0A4B-\\u0A4D\\u0A51\\u0A70-\\u0A71\\u0A75\\u0A81-\\u0A83\\u0ABC\\u0ABE-\\u0AC5\\u0AC7-\\u0AC9\\u0ACB-\\u0ACD\\u0AE2-\\u0AE3\\u0AFA-\\u0AFF\\u0B01-\\u0B03\\u0B3C\\u0B3E-\\u0B44\\u0B47-\\u0B48\\u0B4B-\\u0B4D\\u0B55-\\u0B57\\u0B62-\\u0B63\\u0B82\\u0BBE-\\u0BC2\\u0BC6-\\u0BC8\\u0BCA-\\u0BCD\\u0BD7\\u0C00-\\u0C04\\u0C3C\\u0C3E-\\u0C44\\u0C46-\\u0C48\\u0C4A-\\u0C4D\\u0C55-\\u0C56\\u0C62-\\u0C63\\u0C81-\\u0C83\\u0CBC\\u0CBE-\\u0CC4\\u0CC6-\\u0CC8\\u0CCA-\\u0CCD\\u0CD5-\\u0CD6\\u0CE2-\\u0CE3\\u0CF3\\u0D00-\\u0D03\\u0D3B-\\u0D3C\\u0D3E-\\u0D44\\u0D46-\\u0D48\\u0D4A-\\u0D4D\\u0D57\\u0D62-\\u0D63\\u0D81-\\u0D83\\u0DCA\\u0DCF-\\u0DD4\\u0DD6\\u0DD8-\\u0DDF\\u0DF2-\\u0DF3\\u0E31\\u0E34-\\u0E3A\\u0E47-\\u0E4E\\u0EB1\\u0EB4-\\u0EBC\\u0EC8-\\u0ECE\\u0F18-\\u0F19\\u0F35\\u0F37\\u0F39\\u0F3E-\\u0F3F\\u0F71-\\u0F84\\u0F86-\\u0F87\\u0F8D-\\u0F97\\u0F99-\\u0FBC\\u0FC6\\u102B-\\u103E\\u1056-\\u1059\\u105E-\\u1060\\u1062-\\u1064\\u1067-\\u106D\\u1071-\\u1074\\u1082-\\u108D\\u108F\\u109A-\\u109D\\u135D-\\u135F\\u1712-\\u1715\\u1732-\\u1734\\u1752-\\u1753\\u1772-\\u1773\\u17B4-\\u17D3\\u17DD\\u180B-\\u180D\\u180F\\u1885-\\u1886\\u18A9\\u1920-\\u192B\\u1930-\\u193B\\u1A17-\\u1A1B\\u1A55-\\u1A5E\\u1A60-\\u1A7C\\u1A7F\\u1AB0-\\u1ADD\\u1AE0-\\u1AEB\\u1B00-\\u1B04\\u1B34-\\u1B44\\u1B6B-\\u1B73\\u1B80-\\u1B82\\u1BA1-\\u1BAD\\u1BE6-\\u1BF3\\u1C24-\\u1C37\\u1CD0-\\u1CD2\\u1CD4-\\u1CE8\\u1CED\\u1CF4\\u1CF7-\\u1CF9\\u1DC0-\\u1DFF\\u20D0-\\u20F0\\u2CEF-\\u2CF1\\u2D7F\\u2DE0-\\u2DFF\\u302A-\\u302F\\u3099-\\u309A\\uA66F-\\uA672\\uA674-\\uA67D\\uA69E-\\uA69F\\uA6F0-\\uA6F1\\uA802\\uA806\\uA80B\\uA823-\\uA827\\uA82C\\uA880-\\uA881\\uA8B4-\\uA8C5\\uA8E0-\\uA8F1\\uA8FF\\uA926-\\uA92D\\uA947-\\uA953\\uA980-\\uA983\\uA9B3-\\uA9C0\\uA9E5\\uAA29-\\uAA36\\uAA43\\uAA4C-\\uAA4D\\uAA7B-\\uAA7D\\uAAB0\\uAAB2-\\uAAB4\\uAAB7-\\uAAB8\\uAABE-\\uAABF\\uAAC1\\uAAEB-\\uAAEF\\uAAF5-\\uAAF6\\uABE3-\\uABEA\\uABEC-\\uABED\\uFB1E\\uFE00-\\uFE0F\\uFE20-\\uFE2F\\u{101FD}\\u{102E0}\\u{10376}-\\u{1037A}\\u{10A01}-\\u{10A03}\\u{10A05}-\\u{10A06}\\u{10A0C}-\\u{10A0F}\\u{10A38}-\\u{10A3A}\\u{10A3F}\\u{10AE5}-\\u{10AE6}\\u{10D24}-\\u{10D27}\\u{10D69}-\\u{10D6D}\\u{10EAB}-\\u{10EAC}\\u{10EFA}-\\u{10EFF}\\u{10F46}-\\u{10F50}\\u{10F82}-\\u{10F85}\\u{11000}-\\u{11002}\\u{11038}-\\u{11046}\\u{11070}\\u{11073}-\\u{11074}\\u{1107F}-\\u{11082}\\u{110B0}-\\u{110BA}\\u{110C2}\\u{11100}-\\u{11102}\\u{11127}-\\u{11134}\\u{11145}-\\u{11146}\\u{11173}\\u{11180}-\\u{11182}\\u{111B3}-\\u{111C0}\\u{111C9}-\\u{111CC}\\u{111CE}-\\u{111CF}\\u{1122C}-\\u{11237}\\u{1123E}\\u{11241}\\u{112DF}-\\u{112EA}\\u{11300}-\\u{11303}\\u{1133B}-\\u{1133C}\\u{1133E}-\\u{11344}\\u{11347}-\\u{11348}\\u{1134B}-\\u{1134D}\\u{11357}\\u{11362}-\\u{11363}\\u{11366}-\\u{1136C}\\u{11370}-\\u{11374}\\u{113B8}-\\u{113C0}\\u{113C2}\\u{113C5}\\u{113C7}-\\u{113CA}\\u{113CC}-\\u{113D0}\\u{113D2}\\u{113E1}-\\u{113E2}\\u{11435}-\\u{11446}\\u{1145E}\\u{114B0}-\\u{114C3}\\u{115AF}-\\u{115B5}\\u{115B8}-\\u{115C0}\\u{115DC}-\\u{115DD}\\u{11630}-\\u{11640}\\u{116AB}-\\u{116B7}\\u{1171D}-\\u{1172B}\\u{1182C}-\\u{1183A}\\u{11930}-\\u{11935}\\u{11937}-\\u{11938}\\u{1193B}-\\u{1193E}\\u{11940}\\u{11942}-\\u{11943}\\u{119D1}-\\u{119D7}\\u{119DA}-\\u{119E0}\\u{119E4}\\u{11A01}-\\u{11A0A}\\u{11A33}-\\u{11A39}\\u{11A3B}-\\u{11A3E}\\u{11A47}\\u{11A51}-\\u{11A5B}\\u{11A8A}-\\u{11A99}\\u{11B60}-\\u{11B67}\\u{11C2F}-\\u{11C36}\\u{11C38}-\\u{11C3F}\\u{11C92}-\\u{11CA7}\\u{11CA9}-\\u{11CB6}\\u{11D31}-\\u{11D36}\\u{11D3A}\\u{11D3C}-\\u{11D3D}\\u{11D3F}-\\u{11D45}\\u{11D47}\\u{11D8A}-\\u{11D8E}\\u{11D90}-\\u{11D91}\\u{11D93}-\\u{11D97}\\u{11EF3}-\\u{11EF6}\\u{11F00}-\\u{11F01}\\u{11F03}\\u{11F34}-\\u{11F3A}\\u{11F3E}-\\u{11F42}\\u{11F5A}\\u{13440}\\u{13447}-\\u{13455}\\u{1611E}-\\u{1612F}\\u{16AF0}-\\u{16AF4}\\u{16B30}-\\u{16B36}\\u{16F4F}\\u{16F51}-\\u{16F87}\\u{16F8F}-\\u{16F92}\\u{16FE4}\\u{16FF0}-\\u{16FF1}\\u{1BC9D}-\\u{1BC9E}\\u{1CF00}-\\u{1CF2D}\\u{1CF30}-\\u{1CF46}\\u{1D165}-\\u{1D169}\\u{1D16D}-\\u{1D172}\\u{1D17B}-\\u{1D182}\\u{1D185}-\\u{1D18B}\\u{1D1AA}-\\u{1D1AD}\\u{1D242}-\\u{1D244}\\u{1DA00}-\\u{1DA36}\\u{1DA3B}-\\u{1DA6C}\\u{1DA75}\\u{1DA84}\\u{1DA9B}-\\u{1DA9F}\\u{1DAA1}-\\u{1DAAF}\\u{1E000}-\\u{1E006}\\u{1E008}-\\u{1E018}\\u{1E01B}-\\u{1E021}\\u{1E023}-\\u{1E024}\\u{1E026}-\\u{1E02A}\\u{1E08F}\\u{1E130}-\\u{1E136}\\u{1E2AE}\\u{1E2EC}-\\u{1E2EF}\\u{1E4EC}-\\u{1E4EF}\\u{1E5EE}-\\u{1E5EF}\\u{1E6E3}\\u{1E6E6}\\u{1E6EE}-\\u{1E6EF}\\u{1E6F5}\\u{1E8D0}-\\u{1E8D6}\\u{1E944}-\\u{1E94A}\\u{E0100}-\\u{E01EF}\\u0030-\\u0039\\u00B2-\\u00B3\\u00B9\\u00BC-\\u00BE\\u0660-\\u0669\\u06F0-\\u06F9\\u07C0-\\u07C9\\u0966-\\u096F\\u09E6-\\u09EF\\u09F4-\\u09F9\\u0A66-\\u0A6F\\u0AE6-\\u0AEF\\u0B66-\\u0B6F\\u0B72-\\u0B77\\u0BE6-\\u0BF2\\u0C66-\\u0C6F\\u0C78-\\u0C7E\\u0CE6-\\u0CEF\\u0D58-\\u0D5E\\u0D66-\\u0D78\\u0DE6-\\u0DEF\\u0E50-\\u0E59\\u0ED0-\\u0ED9\\u0F20-\\u0F33\\u1040-\\u1049\\u1090-\\u1099\\u1369-\\u137C\\u16EE-\\u16F0\\u17E0-\\u17E9\\u17F0-\\u17F9\\u1810-\\u1819\\u1946-\\u194F\\u19D0-\\u19DA\\u1A80-\\u1A89\\u1A90-\\u1A99\\u1B50-\\u1B59\\u1BB0-\\u1BB9\\u1C40-\\u1C49\\u1C50-\\u1C59\\u2070\\u2074-\\u2079\\u2080-\\u2089\\u2150-\\u2182\\u2185-\\u2189\\u2460-\\u249B\\u24EA-\\u24FF\\u2776-\\u2793\\u2CFD\\u3007\\u3021-\\u3029\\u3038-\\u303A\\u3192-\\u3195\\u3220-\\u3229\\u3248-\\u324F\\u3251-\\u325F\\u3280-\\u3289\\u32B1-\\u32BF\\uA620-\\uA629\\uA6E6-\\uA6EF\\uA830-\\uA835\\uA8D0-\\uA8D9\\uA900-\\uA909\\uA9D0-\\uA9D9\\uA9F0-\\uA9F9\\uAA50-\\uAA59\\uABF0-\\uABF9\\uFF10-\\uFF19\\u{10107}-\\u{10133}\\u{10140}-\\u{10178}\\u{1018A}-\\u{1018B}\\u{102E1}-\\u{102FB}\\u{10320}-\\u{10323}\\u{10341}\\u{1034A}\\u{103D1}-\\u{103D5}\\u{104A0}-\\u{104A9}\\u{10858}-\\u{1085F}\\u{10879}-\\u{1087F}\\u{108A7}-\\u{108AF}\\u{108FB}-\\u{108FF}\\u{10916}-\\u{1091B}\\u{109BC}-\\u{109BD}\\u{109C0}-\\u{109CF}\\u{109D2}-\\u{109FF}\\u{10A40}-\\u{10A48}\\u{10A7D}-\\u{10A7E}\\u{10A9D}-\\u{10A9F}\\u{10AEB}-\\u{10AEF}\\u{10B58}-\\u{10B5F}\\u{10B78}-\\u{10B7F}\\u{10BA9}-\\u{10BAF}\\u{10CFA}-\\u{10CFF}\\u{10D30}-\\u{10D39}\\u{10D40}-\\u{10D49}\\u{10E60}-\\u{10E7E}\\u{10F1D}-\\u{10F26}\\u{10F51}-\\u{10F54}\\u{10FC5}-\\u{10FCB}\\u{11052}-\\u{1106F}\\u{110F0}-\\u{110F9}\\u{11136}-\\u{1113F}\\u{111D0}-\\u{111D9}\\u{111E1}-\\u{111F4}\\u{112F0}-\\u{112F9}\\u{11450}-\\u{11459}\\u{114D0}-\\u{114D9}\\u{11650}-\\u{11659}\\u{116C0}-\\u{116C9}\\u{116D0}-\\u{116E3}\\u{11730}-\\u{1173B}\\u{118E0}-\\u{118F2}\\u{11950}-\\u{11959}\\u{11BF0}-\\u{11BF9}\\u{11C50}-\\u{11C6C}\\u{11D50}-\\u{11D59}\\u{11DA0}-\\u{11DA9}\\u{11DE0}-\\u{11DE9}\\u{11F50}-\\u{11F59}\\u{11FC0}-\\u{11FD4}\\u{12400}-\\u{1246E}\\u{16130}-\\u{16139}\\u{16A60}-\\u{16A69}\\u{16AC0}-\\u{16AC9}\\u{16B50}-\\u{16B59}\\u{16B5B}-\\u{16B61}\\u{16D70}-\\u{16D79}\\u{16E80}-\\u{16E96}\\u{16FF4}-\\u{16FF6}\\u{1CCF0}-\\u{1CCF9}\\u{1D2C0}-\\u{1D2D3}\\u{1D2E0}-\\u{1D2F3}\\u{1D360}-\\u{1D378}\\u{1D7CE}-\\u{1D7FF}\\u{1E140}-\\u{1E149}\\u{1E2F0}-\\u{1E2F9}\\u{1E4F0}-\\u{1E4F9}\\u{1E5F1}-\\u{1E5FA}\\u{1E8C7}-\\u{1E8CF}\\u{1E950}-\\u{1E959}\\u{1EC71}-\\u{1ECAB}\\u{1ECAD}-\\u{1ECAF}\\u{1ECB1}-\\u{1ECB4}\\u{1ED01}-\\u{1ED2D}\\u{1ED2F}-\\u{1ED3D}\\u{1F100}-\\u{1F10C}\\u{1FBF0}-\\u{1FBF9}]*", "gu"))[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var match = _step.value;
+            var start = match.index;
+            var end = start + match[0].length;
+            var cursor = start;
+            var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
+            try {
+                for(var _iterator1 = text.slice(start, end).matchAll(RUN)[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
+                    var run = _step1.value;
+                    var runStart = start + run.index;
+                    if (runStart > cursor) tokens.push({
+                        text: text.slice(cursor, runStart),
+                        start: cursor,
+                        end: runStart,
+                        unspaced: false
+                    });
+                    var runEnd = runStart + run[0].length;
+                    tokens.push({
+                        text: run[0],
+                        start: runStart,
+                        end: runEnd,
+                        unspaced: true
+                    });
+                    cursor = runEnd;
+                }
+            } catch (err) {
+                _didIteratorError1 = true;
+                _iteratorError1 = err;
+            } finally{
+                try {
+                    if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
+                        _iterator1.return();
+                    }
+                } finally{
+                    if (_didIteratorError1) {
+                        throw _iteratorError1;
+                    }
+                }
+            }
+            if (cursor < end) tokens.push({
+                text: text.slice(cursor, end),
+                start: cursor,
+                end: end,
+                unspaced: false
+            });
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    return tokens;
+}
+function matchesSearchPhrase(text, phrase) {
+    var haystack = searchTokens(text);
+    if (phrase.length === 0 || haystack.length < phrase.length) return false;
+    for(var start = 0; start <= haystack.length - phrase.length; start++){
+        var matches = true;
+        for(var i = 0; i < phrase.length; i++){
+            var query = phrase[i];
+            var authored = haystack[start + i];
+            if (query.unspaced) {
+                if (!authored.unspaced || !authored.text.includes(query.text)) matches = false;
+            } else if (authored.unspaced || (0, _stemmer.stemmer)(foldForSearch(authored.text)) !== (0, _stemmer.stemmer)(foldForSearch(query.text))) {
+                matches = false;
+            }
+            if (!matches) break;
+        }
+        if (matches) return true;
+    }
+    return false;
+}
+function unspacedRuns(text) {
+    return Array.from(text.matchAll(RUN), function(m) {
+        return {
+            start: m.index,
+            end: m.index + m[0].length
+        };
+    });
+}
 // Grapheme clusters, ECMA-402/UAX #29: base char plus its marks, ZWJ sequences, Hangul jamo.
 // Built on first use and kept: construction is 6.6 ms, and a tree with no unspaced-script run
 // never segments at all, so no command pays it at module load.
@@ -149,4 +251,7 @@ function segmentMatch(terms) {
     }
     return out;
 }
+function foldForSearch(text) {
+    return text.normalize('NFD').replace(RegExp("[\\u0300-\\u036F\\u0483-\\u0489\\u0591-\\u05BD\\u05BF\\u05C1-\\u05C2\\u05C4-\\u05C5\\u05C7\\u0610-\\u061A\\u064B-\\u065F\\u0670\\u06D6-\\u06DC\\u06DF-\\u06E4\\u06E7-\\u06E8\\u06EA-\\u06ED\\u0711\\u0730-\\u074A\\u07A6-\\u07B0\\u07EB-\\u07F3\\u07FD\\u0816-\\u0819\\u081B-\\u0823\\u0825-\\u0827\\u0829-\\u082D\\u0859-\\u085B\\u0897-\\u089F\\u08CA-\\u08E1\\u08E3-\\u0903\\u093A-\\u093C\\u093E-\\u094F\\u0951-\\u0957\\u0962-\\u0963\\u0981-\\u0983\\u09BC\\u09BE-\\u09C4\\u09C7-\\u09C8\\u09CB-\\u09CD\\u09D7\\u09E2-\\u09E3\\u09FE\\u0A01-\\u0A03\\u0A3C\\u0A3E-\\u0A42\\u0A47-\\u0A48\\u0A4B-\\u0A4D\\u0A51\\u0A70-\\u0A71\\u0A75\\u0A81-\\u0A83\\u0ABC\\u0ABE-\\u0AC5\\u0AC7-\\u0AC9\\u0ACB-\\u0ACD\\u0AE2-\\u0AE3\\u0AFA-\\u0AFF\\u0B01-\\u0B03\\u0B3C\\u0B3E-\\u0B44\\u0B47-\\u0B48\\u0B4B-\\u0B4D\\u0B55-\\u0B57\\u0B62-\\u0B63\\u0B82\\u0BBE-\\u0BC2\\u0BC6-\\u0BC8\\u0BCA-\\u0BCD\\u0BD7\\u0C00-\\u0C04\\u0C3C\\u0C3E-\\u0C44\\u0C46-\\u0C48\\u0C4A-\\u0C4D\\u0C55-\\u0C56\\u0C62-\\u0C63\\u0C81-\\u0C83\\u0CBC\\u0CBE-\\u0CC4\\u0CC6-\\u0CC8\\u0CCA-\\u0CCD\\u0CD5-\\u0CD6\\u0CE2-\\u0CE3\\u0CF3\\u0D00-\\u0D03\\u0D3B-\\u0D3C\\u0D3E-\\u0D44\\u0D46-\\u0D48\\u0D4A-\\u0D4D\\u0D57\\u0D62-\\u0D63\\u0D81-\\u0D83\\u0DCA\\u0DCF-\\u0DD4\\u0DD6\\u0DD8-\\u0DDF\\u0DF2-\\u0DF3\\u0E31\\u0E34-\\u0E3A\\u0E47-\\u0E4E\\u0EB1\\u0EB4-\\u0EBC\\u0EC8-\\u0ECE\\u0F18-\\u0F19\\u0F35\\u0F37\\u0F39\\u0F3E-\\u0F3F\\u0F71-\\u0F84\\u0F86-\\u0F87\\u0F8D-\\u0F97\\u0F99-\\u0FBC\\u0FC6\\u102B-\\u103E\\u1056-\\u1059\\u105E-\\u1060\\u1062-\\u1064\\u1067-\\u106D\\u1071-\\u1074\\u1082-\\u108D\\u108F\\u109A-\\u109D\\u135D-\\u135F\\u1712-\\u1715\\u1732-\\u1734\\u1752-\\u1753\\u1772-\\u1773\\u17B4-\\u17D3\\u17DD\\u180B-\\u180D\\u180F\\u1885-\\u1886\\u18A9\\u1920-\\u192B\\u1930-\\u193B\\u1A17-\\u1A1B\\u1A55-\\u1A5E\\u1A60-\\u1A7C\\u1A7F\\u1AB0-\\u1ADD\\u1AE0-\\u1AEB\\u1B00-\\u1B04\\u1B34-\\u1B44\\u1B6B-\\u1B73\\u1B80-\\u1B82\\u1BA1-\\u1BAD\\u1BE6-\\u1BF3\\u1C24-\\u1C37\\u1CD0-\\u1CD2\\u1CD4-\\u1CE8\\u1CED\\u1CF4\\u1CF7-\\u1CF9\\u1DC0-\\u1DFF\\u20D0-\\u20F0\\u2CEF-\\u2CF1\\u2D7F\\u2DE0-\\u2DFF\\u302A-\\u302F\\u3099-\\u309A\\uA66F-\\uA672\\uA674-\\uA67D\\uA69E-\\uA69F\\uA6F0-\\uA6F1\\uA802\\uA806\\uA80B\\uA823-\\uA827\\uA82C\\uA880-\\uA881\\uA8B4-\\uA8C5\\uA8E0-\\uA8F1\\uA8FF\\uA926-\\uA92D\\uA947-\\uA953\\uA980-\\uA983\\uA9B3-\\uA9C0\\uA9E5\\uAA29-\\uAA36\\uAA43\\uAA4C-\\uAA4D\\uAA7B-\\uAA7D\\uAAB0\\uAAB2-\\uAAB4\\uAAB7-\\uAAB8\\uAABE-\\uAABF\\uAAC1\\uAAEB-\\uAAEF\\uAAF5-\\uAAF6\\uABE3-\\uABEA\\uABEC-\\uABED\\uFB1E\\uFE00-\\uFE0F\\uFE20-\\uFE2F\\u{101FD}\\u{102E0}\\u{10376}-\\u{1037A}\\u{10A01}-\\u{10A03}\\u{10A05}-\\u{10A06}\\u{10A0C}-\\u{10A0F}\\u{10A38}-\\u{10A3A}\\u{10A3F}\\u{10AE5}-\\u{10AE6}\\u{10D24}-\\u{10D27}\\u{10D69}-\\u{10D6D}\\u{10EAB}-\\u{10EAC}\\u{10EFA}-\\u{10EFF}\\u{10F46}-\\u{10F50}\\u{10F82}-\\u{10F85}\\u{11000}-\\u{11002}\\u{11038}-\\u{11046}\\u{11070}\\u{11073}-\\u{11074}\\u{1107F}-\\u{11082}\\u{110B0}-\\u{110BA}\\u{110C2}\\u{11100}-\\u{11102}\\u{11127}-\\u{11134}\\u{11145}-\\u{11146}\\u{11173}\\u{11180}-\\u{11182}\\u{111B3}-\\u{111C0}\\u{111C9}-\\u{111CC}\\u{111CE}-\\u{111CF}\\u{1122C}-\\u{11237}\\u{1123E}\\u{11241}\\u{112DF}-\\u{112EA}\\u{11300}-\\u{11303}\\u{1133B}-\\u{1133C}\\u{1133E}-\\u{11344}\\u{11347}-\\u{11348}\\u{1134B}-\\u{1134D}\\u{11357}\\u{11362}-\\u{11363}\\u{11366}-\\u{1136C}\\u{11370}-\\u{11374}\\u{113B8}-\\u{113C0}\\u{113C2}\\u{113C5}\\u{113C7}-\\u{113CA}\\u{113CC}-\\u{113D0}\\u{113D2}\\u{113E1}-\\u{113E2}\\u{11435}-\\u{11446}\\u{1145E}\\u{114B0}-\\u{114C3}\\u{115AF}-\\u{115B5}\\u{115B8}-\\u{115C0}\\u{115DC}-\\u{115DD}\\u{11630}-\\u{11640}\\u{116AB}-\\u{116B7}\\u{1171D}-\\u{1172B}\\u{1182C}-\\u{1183A}\\u{11930}-\\u{11935}\\u{11937}-\\u{11938}\\u{1193B}-\\u{1193E}\\u{11940}\\u{11942}-\\u{11943}\\u{119D1}-\\u{119D7}\\u{119DA}-\\u{119E0}\\u{119E4}\\u{11A01}-\\u{11A0A}\\u{11A33}-\\u{11A39}\\u{11A3B}-\\u{11A3E}\\u{11A47}\\u{11A51}-\\u{11A5B}\\u{11A8A}-\\u{11A99}\\u{11B60}-\\u{11B67}\\u{11C2F}-\\u{11C36}\\u{11C38}-\\u{11C3F}\\u{11C92}-\\u{11CA7}\\u{11CA9}-\\u{11CB6}\\u{11D31}-\\u{11D36}\\u{11D3A}\\u{11D3C}-\\u{11D3D}\\u{11D3F}-\\u{11D45}\\u{11D47}\\u{11D8A}-\\u{11D8E}\\u{11D90}-\\u{11D91}\\u{11D93}-\\u{11D97}\\u{11EF3}-\\u{11EF6}\\u{11F00}-\\u{11F01}\\u{11F03}\\u{11F34}-\\u{11F3A}\\u{11F3E}-\\u{11F42}\\u{11F5A}\\u{13440}\\u{13447}-\\u{13455}\\u{1611E}-\\u{1612F}\\u{16AF0}-\\u{16AF4}\\u{16B30}-\\u{16B36}\\u{16F4F}\\u{16F51}-\\u{16F87}\\u{16F8F}-\\u{16F92}\\u{16FE4}\\u{16FF0}-\\u{16FF1}\\u{1BC9D}-\\u{1BC9E}\\u{1CF00}-\\u{1CF2D}\\u{1CF30}-\\u{1CF46}\\u{1D165}-\\u{1D169}\\u{1D16D}-\\u{1D172}\\u{1D17B}-\\u{1D182}\\u{1D185}-\\u{1D18B}\\u{1D1AA}-\\u{1D1AD}\\u{1D242}-\\u{1D244}\\u{1DA00}-\\u{1DA36}\\u{1DA3B}-\\u{1DA6C}\\u{1DA75}\\u{1DA84}\\u{1DA9B}-\\u{1DA9F}\\u{1DAA1}-\\u{1DAAF}\\u{1E000}-\\u{1E006}\\u{1E008}-\\u{1E018}\\u{1E01B}-\\u{1E021}\\u{1E023}-\\u{1E024}\\u{1E026}-\\u{1E02A}\\u{1E08F}\\u{1E130}-\\u{1E136}\\u{1E2AE}\\u{1E2EC}-\\u{1E2EF}\\u{1E4EC}-\\u{1E4EF}\\u{1E5EE}-\\u{1E5EF}\\u{1E6E3}\\u{1E6E6}\\u{1E6EE}-\\u{1E6EF}\\u{1E6F5}\\u{1E8D0}-\\u{1E8D6}\\u{1E944}-\\u{1E94A}\\u{E0100}-\\u{E01EF}]", "gu"), '').toLowerCase();
+}
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/text/segment.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/segment.ts"],"sourcesContent":["// Word boundaries FTS5's tokenizers cannot find on their own. Contract: seg(query) must appear in\n// seg(document) for any substring query; word mode is context-dependent and breaks that, so grapheme mode (UAX #29) is used.\n\n// Scripts written without word spaces: a closed set of writing systems.\n// Script_Extensions, not Script: the katakana long-vowel mark ー is Script=Common.\nexport const UNSPACED_SCRIPTS = '\\\\p{scx=Han}\\\\p{scx=Hiragana}\\\\p{scx=Katakana}\\\\p{scx=Thai}\\\\p{scx=Khmer}\\\\p{scx=Lao}\\\\p{scx=Myanmar}';\n// A run is script BASE characters with their combining marks attached; a bare mark after a\n// Latin letter (decomposed é) never starts one.\nconst RUN = new RegExp(`((?:[${UNSPACED_SCRIPTS}]\\\\p{M}*)+)`, 'gu');\nconst HAS_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');\n\n// Whether text holds a script that marks no word boundaries, the predicate every store's\n// sidecar population and query split turns on.\nexport function hasUnspacedRun(text: string): boolean {\n  return HAS_RUN.test(text);\n}\n// Grapheme clusters, ECMA-402/UAX #29: base char plus its marks, ZWJ sequences, Hangul jamo.\n// Built on first use and kept: construction is 6.6 ms, and a tree with no unspaced-script run\n// never segments at all, so no command pays it at module load.\nlet graphemeSegmenter: Intl.Segmenter | undefined;\n// unicode61 drops Unicode punctuation as a separator. Some of it (。、「」) has Script_Extensions\n// into an unspaced script, so RUN keeps it -- a grapheme matching this becomes a split point.\nconst PUNCTUATION = /\\p{P}/u;\n// Token barrier between separate runs (or a punctuation split within one) so their graphemes\n// are never phrase-adjacent. U+A7F7: a letter (so FTS5 keeps it as a token) no one types.\nconst BARRIER = 'ꟷ';\n\nfunction graphemes(run: string): string[] {\n  graphemeSegmenter ??= new Intl.Segmenter(undefined, { granularity: 'grapheme' });\n  return Array.from(graphemeSegmenter.segment(run), (s) => s.segment);\n}\n\n// A run's graphemes, cut into punctuation-free groups at every punctuation grapheme (dropped,\n// matching unicode61). The one place punctuation is classified, so index and query agree.\nfunction splitOnPunctuation(run: string): string[][] {\n  const groups: string[][] = [[]];\n  for (const g of graphemes(run)) {\n    if (PUNCTUATION.test(g)) groups.push([]);\n    else groups[groups.length - 1].push(g);\n  }\n  return groups.filter((g) => g.length > 0);\n}\n\n// Index side: '' when the field has no unspaced-script run. Otherwise each run explodes into\n// its graphemes, barrier-delimited from its neighbors and from a punctuation split within itself.\nexport function segmentField(text: string): string {\n  if (!HAS_RUN.test(text)) return '';\n  const out = text.replace(RUN, (run) => {\n    const body = splitOnPunctuation(run)\n      .map((g) => g.join(' '))\n      .join(` ${BARRIER} `);\n    return ` ${BARRIER} ${body} ${BARRIER} `;\n  });\n  return out.replace(/\\s+/g, ' ').trim();\n}\n\n// A `title:`/`summary:`/`text:` qualifier directly before a run that is about to become a\n// quoted grapheme phrase, so the rewrite can retarget it at the matching `_seg` column.\nconst QUALIFIER = /(^|[\\s(])(-?)(title|summary|text)\\s*:\\s*$/;\n\n// Raw title/summary/text drop punctuation as unicode61's token separator, so `数数` and `数。数`\n// falsely match there; only the barriered `_seg` columns are safe from it, hence this fallback target.\nconst SIDECAR_COLUMNS = '{title_seg summary_seg text_seg}:';\n\n// A run's punctuation-free groups, each its own quoted phrase (bare token if one grapheme),\n// space-joined -- the same split points segmentField barriers, so query and index agree.\nfunction runQuery(run: string, columnPrefix: string): string {\n  return splitOnPunctuation(run)\n    .map((g) => `${columnPrefix}${g.length > 1 ? `\"${g.join(' ')}\"` : g[0]}`)\n    .join(' ');\n}\n\n// Query side: each unspaced run becomes phrases of its graphemes, matching segmentField. A\n// qualifier maps to its `_seg` column; unqualified maps to all three (SIDECAR_COLUMNS).\nexport function segmentMatch(terms: string): string {\n  if (!HAS_RUN.test(terms)) return terms;\n  let out = '';\n  let quoted = false;\n  const pieces = terms.split(RUN); // split keeps captured runs at odd indices\n  for (let i = 0; i < pieces.length; i++) {\n    if (i % 2 === 0) {\n      for (const ch of pieces[i]) if (ch === '\"') quoted = !quoted;\n      out += pieces[i];\n      continue;\n    }\n    if (quoted) {\n      out += pieces[i]; // an author's phrase is matched as written\n      continue;\n    }\n    const m = out.match(QUALIFIER);\n    if (m) {\n      out = `${out.slice(0, m.index)}${m[1]}${runQuery(pieces[i], `${m[2]}${m[3]}_seg:`)}`;\n    } else {\n      out += runQuery(pieces[i], SIDECAR_COLUMNS);\n    }\n  }\n  return out;\n}\n"],"names":["UNSPACED_SCRIPTS","hasUnspacedRun","segmentField","segmentMatch","RUN","RegExp","HAS_RUN","text","test","graphemeSegmenter","PUNCTUATION","BARRIER","graphemes","run","Intl","Segmenter","undefined","granularity","Array","from","segment","s","splitOnPunctuation","groups","g","push","length","filter","out","replace","body","map","join","trim","QUALIFIER","SIDECAR_COLUMNS","runQuery","columnPrefix","terms","quoted","pieces","split","i","ch","m","match","slice","index"],"mappings":"AAAA,kGAAkG;AAClG,6HAA6H;AAE7H,wEAAwE;AACxE,kFAAkF;;;;;;;;;;;;QACrEA;eAAAA;;QAQGC;eAAAA;;QAgCAC;eAAAA;;QA6BAC;eAAAA;;;AArET,IAAMH,mBAAmB;AAChC,2FAA2F;AAC3F,gDAAgD;AAChD,IAAMI,MAAM,IAAIC,OAAO,AAAC,QAAwB,OAAjBL,kBAAiB,gBAAc;AAC9D,IAAMM,UAAU,IAAID,OAAO,AAAC,IAAoB,OAAjBL,kBAAiB,MAAI;AAI7C,SAASC,eAAeM,IAAY;IACzC,OAAOD,QAAQE,IAAI,CAACD;AACtB;AACA,6FAA6F;AAC7F,8FAA8F;AAC9F,+DAA+D;AAC/D,IAAIE;AACJ,8FAA8F;AAC9F,8FAA8F;AAC9F,IAAMC,cAAc;AACpB,6FAA6F;AAC7F,0FAA0F;AAC1F,IAAMC,UAAU;AAEhB,SAASC,UAAUC,GAAW;IAC5BJ,8BAAAA,+BAAAA,oBAAAA,oBAAsB,IAAIK,KAAKC,SAAS,CAACC,WAAW;QAAEC,aAAa;IAAW;IAC9E,OAAOC,MAAMC,IAAI,CAACV,kBAAkBW,OAAO,CAACP,MAAM,SAACQ;eAAMA,EAAED,OAAO;;AACpE;AAEA,8FAA8F;AAC9F,0FAA0F;AAC1F,SAASE,mBAAmBT,GAAW;IACrC,IAAMU,SAAqB;QAAC,EAAE;KAAC;QAC1B,kCAAA,2BAAA;;QAAL,QAAK,YAAWX,UAAUC,yBAArB,SAAA,6BAAA,QAAA,yBAAA,iCAA2B;YAA3B,IAAMW,IAAN;YACH,IAAId,YAAYF,IAAI,CAACgB,IAAID,OAAOE,IAAI,CAAC,EAAE;iBAClCF,MAAM,CAACA,OAAOG,MAAM,GAAG,EAAE,CAACD,IAAI,CAACD;QACtC;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAOD,OAAOI,MAAM,CAAC,SAACH;eAAMA,EAAEE,MAAM,GAAG;;AACzC;AAIO,SAASxB,aAAaK,IAAY;IACvC,IAAI,CAACD,QAAQE,IAAI,CAACD,OAAO,OAAO;IAChC,IAAMqB,MAAMrB,KAAKsB,OAAO,CAACzB,KAAK,SAACS;QAC7B,IAAMiB,OAAOR,mBAAmBT,KAC7BkB,GAAG,CAAC,SAACP;mBAAMA,EAAEQ,IAAI,CAAC;WAClBA,IAAI,CAAC,AAAC,IAAW,OAARrB,SAAQ;QACpB,OAAO,AAAC,IAAcmB,OAAXnB,SAAQ,KAAWA,OAARmB,MAAK,KAAW,OAARnB,SAAQ;IACxC;IACA,OAAOiB,IAAIC,OAAO,CAAC,QAAQ,KAAKI,IAAI;AACtC;AAEA,0FAA0F;AAC1F,wFAAwF;AACxF,IAAMC,YAAY;AAElB,4FAA4F;AAC5F,uGAAuG;AACvG,IAAMC,kBAAkB;AAExB,4FAA4F;AAC5F,yFAAyF;AACzF,SAASC,SAASvB,GAAW,EAAEwB,YAAoB;IACjD,OAAOf,mBAAmBT,KACvBkB,GAAG,CAAC,SAACP;eAAM,AAAC,GAAiBA,OAAfa,cAAwD,OAAzCb,EAAEE,MAAM,GAAG,IAAI,AAAC,IAAe,OAAZF,EAAEQ,IAAI,CAAC,MAAK,OAAKR,CAAC,CAAC,EAAE;OACrEQ,IAAI,CAAC;AACV;AAIO,SAAS7B,aAAamC,KAAa;IACxC,IAAI,CAAChC,QAAQE,IAAI,CAAC8B,QAAQ,OAAOA;IACjC,IAAIV,MAAM;IACV,IAAIW,SAAS;IACb,IAAMC,SAASF,MAAMG,KAAK,CAACrC,MAAM,2CAA2C;IAC5E,IAAK,IAAIsC,IAAI,GAAGA,IAAIF,OAAOd,MAAM,EAAEgB,IAAK;QACtC,IAAIA,IAAI,MAAM,GAAG;gBACV,kCAAA,2BAAA;;gBAAL,QAAK,YAAYF,MAAM,CAACE,EAAE,qBAArB,SAAA,6BAAA,QAAA,yBAAA;oBAAA,IAAMC,KAAN;oBAAuB,IAAIA,OAAO,KAAKJ,SAAS,CAACA;;;gBAAjD;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YACLX,OAAOY,MAAM,CAACE,EAAE;YAChB;QACF;QACA,IAAIH,QAAQ;YACVX,OAAOY,MAAM,CAACE,EAAE,EAAE,2CAA2C;YAC7D;QACF;QACA,IAAME,IAAIhB,IAAIiB,KAAK,CAACX;QACpB,IAAIU,GAAG;YACLhB,MAAM,AAAC,GAA0BgB,OAAxBhB,IAAIkB,KAAK,CAAC,GAAGF,EAAEG,KAAK,GAAWX,OAAPQ,CAAC,CAAC,EAAE,EAA8C,OAA3CR,SAASI,MAAM,CAACE,EAAE,EAAE,AAAC,GAASE,OAAPA,CAAC,CAAC,EAAE,EAAQ,OAALA,CAAC,CAAC,EAAE,EAAC;QAC7E,OAAO;YACLhB,OAAOQ,SAASI,MAAM,CAACE,EAAE,EAAEP;QAC7B;IACF;IACA,OAAOP;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/segment.ts"],"sourcesContent":["import { stemmer } from 'stemmer';\n\n// Word boundaries FTS5's tokenizers cannot find on their own. Contract: seg(query) must appear in\n// seg(document) for any substring query; word mode is context-dependent and breaks that, so grapheme mode (UAX #29) is used.\n\n// Scripts written without word spaces: a closed set of writing systems.\n// Script_Extensions, not Script: the katakana long-vowel mark ー is Script=Common.\nexport const UNSPACED_SCRIPTS = '\\\\p{scx=Han}\\\\p{scx=Hiragana}\\\\p{scx=Katakana}\\\\p{scx=Thai}\\\\p{scx=Khmer}\\\\p{scx=Lao}\\\\p{scx=Myanmar}';\n// A run is script BASE characters with their combining marks attached; a bare mark after a\n// Latin letter (decomposed é) never starts one.\nconst RUN = new RegExp(`((?:[${UNSPACED_SCRIPTS}]\\\\p{M}*)+)`, 'gu');\nconst HAS_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');\n\n// Whether text holds a script that marks no word boundaries, the predicate every store's\n// sidecar population and query split turns on.\nexport function hasUnspacedRun(text: string): boolean {\n  return HAS_RUN.test(text);\n}\n\nexport interface SearchToken {\n  text: string;\n  start: number;\n  end: number;\n  unspaced: boolean;\n}\n\n// Search tokens share the native separator contract: letters, marks, and numbers form tokens;\n// underscore, apostrophe, and hyphen separate them. Unspaced-script runs stay whole so callers\n// can apply substring semantics, while adjacent Latin text still gets its own token.\nexport function searchTokens(text: string): SearchToken[] {\n  const tokens: SearchToken[] = [];\n  for (const match of text.matchAll(/[\\p{L}\\p{N}][\\p{L}\\p{M}\\p{N}]*/gu)) {\n    const start = match.index;\n    const end = start + match[0].length;\n    let cursor = start;\n    for (const run of text.slice(start, end).matchAll(RUN)) {\n      const runStart = start + run.index;\n      if (runStart > cursor) tokens.push({ text: text.slice(cursor, runStart), start: cursor, end: runStart, unspaced: false });\n      const runEnd = runStart + run[0].length;\n      tokens.push({ text: run[0], start: runStart, end: runEnd, unspaced: true });\n      cursor = runEnd;\n    }\n    if (cursor < end) tokens.push({ text: text.slice(cursor, end), start: cursor, end, unspaced: false });\n  }\n  return tokens;\n}\n\n// Phrase membership is checked after native ranking. A CJK token matches inside one authored\n// run; Latin tokens compare folded Porter stems. Each call scans one field once per candidate.\nexport function matchesSearchPhrase(text: string, phrase: SearchToken[]): boolean {\n  const haystack = searchTokens(text);\n  if (phrase.length === 0 || haystack.length < phrase.length) return false;\n  for (let start = 0; start <= haystack.length - phrase.length; start++) {\n    let matches = true;\n    for (let i = 0; i < phrase.length; i++) {\n      const query = phrase[i];\n      const authored = haystack[start + i];\n      if (query.unspaced) {\n        if (!authored.unspaced || !authored.text.includes(query.text)) matches = false;\n      } else if (authored.unspaced || stemmer(foldForSearch(authored.text)) !== stemmer(foldForSearch(query.text))) {\n        matches = false;\n      }\n      if (!matches) break;\n    }\n    if (matches) return true;\n  }\n  return false;\n}\n\n// Character spans of unspaced-script runs in text, the same boundary the index segments on --\n// the snippet marker uses this to keep substring semantics inside a run and word/stem matching outside it.\nexport function unspacedRuns(text: string): Array<{ start: number; end: number }> {\n  return Array.from(text.matchAll(RUN), (m) => ({ start: m.index, end: m.index + m[0].length }));\n}\n// Grapheme clusters, ECMA-402/UAX #29: base char plus its marks, ZWJ sequences, Hangul jamo.\n// Built on first use and kept: construction is 6.6 ms, and a tree with no unspaced-script run\n// never segments at all, so no command pays it at module load.\nlet graphemeSegmenter: Intl.Segmenter | undefined;\n// unicode61 drops Unicode punctuation as a separator. Some of it (。、「」) has Script_Extensions\n// into an unspaced script, so RUN keeps it -- a grapheme matching this becomes a split point.\nconst PUNCTUATION = /\\p{P}/u;\n// Token barrier between separate runs (or a punctuation split within one) so their graphemes\n// are never phrase-adjacent. U+A7F7: a letter (so FTS5 keeps it as a token) no one types.\nconst BARRIER = 'ꟷ';\n\nfunction graphemes(run: string): string[] {\n  graphemeSegmenter ??= new Intl.Segmenter(undefined, { granularity: 'grapheme' });\n  return Array.from(graphemeSegmenter.segment(run), (s) => s.segment);\n}\n\n// A run's graphemes, cut into punctuation-free groups at every punctuation grapheme (dropped,\n// matching unicode61). The one place punctuation is classified, so index and query agree.\nfunction splitOnPunctuation(run: string): string[][] {\n  const groups: string[][] = [[]];\n  for (const g of graphemes(run)) {\n    if (PUNCTUATION.test(g)) groups.push([]);\n    else groups[groups.length - 1].push(g);\n  }\n  return groups.filter((g) => g.length > 0);\n}\n\n// Index side: '' when the field has no unspaced-script run. Otherwise each run explodes into\n// its graphemes, barrier-delimited from its neighbors and from a punctuation split within itself.\nexport function segmentField(text: string): string {\n  if (!HAS_RUN.test(text)) return '';\n  const out = text.replace(RUN, (run) => {\n    const body = splitOnPunctuation(run)\n      .map((g) => g.join(' '))\n      .join(` ${BARRIER} `);\n    return ` ${BARRIER} ${body} ${BARRIER} `;\n  });\n  return out.replace(/\\s+/g, ' ').trim();\n}\n\n// A `title:`/`summary:`/`text:` qualifier directly before a run that is about to become a\n// quoted grapheme phrase, so the rewrite can retarget it at the matching `_seg` column.\nconst QUALIFIER = /(^|[\\s(])(-?)(title|summary|text)\\s*:\\s*$/;\n\n// Raw title/summary/text drop punctuation as unicode61's token separator, so `数数` and `数。数`\n// falsely match there; only the barriered `_seg` columns are safe from it, hence this fallback target.\nconst SIDECAR_COLUMNS = '{title_seg summary_seg text_seg}:';\n\n// A run's punctuation-free groups, each its own quoted phrase (bare token if one grapheme),\n// space-joined -- the same split points segmentField barriers, so query and index agree.\nfunction runQuery(run: string, columnPrefix: string): string {\n  return splitOnPunctuation(run)\n    .map((g) => `${columnPrefix}${g.length > 1 ? `\"${g.join(' ')}\"` : g[0]}`)\n    .join(' ');\n}\n\n// Query side: each unspaced run becomes phrases of its graphemes, matching segmentField. A\n// qualifier maps to its `_seg` column; unqualified maps to all three (SIDECAR_COLUMNS).\nexport function segmentMatch(terms: string): string {\n  if (!HAS_RUN.test(terms)) return terms;\n  let out = '';\n  let quoted = false;\n  const pieces = terms.split(RUN); // split keeps captured runs at odd indices\n  for (let i = 0; i < pieces.length; i++) {\n    if (i % 2 === 0) {\n      for (const ch of pieces[i]) if (ch === '\"') quoted = !quoted;\n      out += pieces[i];\n      continue;\n    }\n    if (quoted) {\n      out += pieces[i]; // an author's phrase is matched as written\n      continue;\n    }\n    const m = out.match(QUALIFIER);\n    if (m) {\n      out = `${out.slice(0, m.index)}${m[1]}${runQuery(pieces[i], `${m[2]}${m[3]}_seg:`)}`;\n    } else {\n      out += runQuery(pieces[i], SIDECAR_COLUMNS);\n    }\n  }\n  return out;\n}\n\n// FTS tokenizers compare letters without combining marks. Keep the authored string for output and\n// offsets, but use this key when JS needs to compare a matched word to its source segment.\nexport function foldForSearch(text: string): string {\n  return text.normalize('NFD').replace(/\\p{M}/gu, '').toLowerCase();\n}\n"],"names":["UNSPACED_SCRIPTS","foldForSearch","hasUnspacedRun","matchesSearchPhrase","searchTokens","segmentField","segmentMatch","unspacedRuns","RUN","RegExp","HAS_RUN","text","test","tokens","matchAll","match","start","index","end","length","cursor","slice","run","runStart","push","unspaced","runEnd","phrase","haystack","matches","i","query","authored","includes","stemmer","Array","from","m","graphemeSegmenter","PUNCTUATION","BARRIER","graphemes","Intl","Segmenter","undefined","granularity","segment","s","splitOnPunctuation","groups","g","filter","out","replace","body","map","join","trim","QUALIFIER","SIDECAR_COLUMNS","runQuery","columnPrefix","terms","quoted","pieces","split","ch","normalize","toLowerCase"],"mappings":";;;;;;;;;;;QAOaA;eAAAA;;QAwJGC;eAAAA;;QAhJAC;eAAAA;;QAkCAC;eAAAA;;QApBAC;eAAAA;;QA0EAC;eAAAA;;QA6BAC;eAAAA;;QA7DAC;eAAAA;;;uBAvEQ;AAOjB,IAAMP,mBAAmB;AAChC,2FAA2F;AAC3F,gDAAgD;AAChD,IAAMQ,MAAM,IAAIC,OAAO,AAAC,QAAwB,OAAjBT,kBAAiB,gBAAc;AAC9D,IAAMU,UAAU,IAAID,OAAO,AAAC,IAAoB,OAAjBT,kBAAiB,MAAI;AAI7C,SAASE,eAAeS,IAAY;IACzC,OAAOD,QAAQE,IAAI,CAACD;AACtB;AAYO,SAASP,aAAaO,IAAY;IACvC,IAAME,SAAwB,EAAE;QAC3B,kCAAA,2BAAA;;QAAL,QAAK,YAAeF,KAAKG,QAAQ,CAAC,oo+BAA7B,SAAA,6BAAA,QAAA,yBAAA,iCAAkE;YAAlE,IAAMC,QAAN;YACH,IAAMC,QAAQD,MAAME,KAAK;YACzB,IAAMC,MAAMF,QAAQD,KAAK,CAAC,EAAE,CAACI,MAAM;YACnC,IAAIC,SAASJ;gBACR,mCAAA,4BAAA;;gBAAL,QAAK,aAAaL,KAAKU,KAAK,CAACL,OAAOE,KAAKJ,QAAQ,CAACN,yBAA7C,UAAA,8BAAA,SAAA,0BAAA,kCAAmD;oBAAnD,IAAMc,MAAN;oBACH,IAAMC,WAAWP,QAAQM,IAAIL,KAAK;oBAClC,IAAIM,WAAWH,QAAQP,OAAOW,IAAI,CAAC;wBAAEb,MAAMA,KAAKU,KAAK,CAACD,QAAQG;wBAAWP,OAAOI;wBAAQF,KAAKK;wBAAUE,UAAU;oBAAM;oBACvH,IAAMC,SAASH,WAAWD,GAAG,CAAC,EAAE,CAACH,MAAM;oBACvCN,OAAOW,IAAI,CAAC;wBAAEb,MAAMW,GAAG,CAAC,EAAE;wBAAEN,OAAOO;wBAAUL,KAAKQ;wBAAQD,UAAU;oBAAK;oBACzEL,SAASM;gBACX;;gBANK;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAOL,IAAIN,SAASF,KAAKL,OAAOW,IAAI,CAAC;gBAAEb,MAAMA,KAAKU,KAAK,CAACD,QAAQF;gBAAMF,OAAOI;gBAAQF,KAAAA;gBAAKO,UAAU;YAAM;QACrG;;QAZK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAaL,OAAOZ;AACT;AAIO,SAASV,oBAAoBQ,IAAY,EAAEgB,MAAqB;IACrE,IAAMC,WAAWxB,aAAaO;IAC9B,IAAIgB,OAAOR,MAAM,KAAK,KAAKS,SAAST,MAAM,GAAGQ,OAAOR,MAAM,EAAE,OAAO;IACnE,IAAK,IAAIH,QAAQ,GAAGA,SAASY,SAAST,MAAM,GAAGQ,OAAOR,MAAM,EAAEH,QAAS;QACrE,IAAIa,UAAU;QACd,IAAK,IAAIC,IAAI,GAAGA,IAAIH,OAAOR,MAAM,EAAEW,IAAK;YACtC,IAAMC,QAAQJ,MAAM,CAACG,EAAE;YACvB,IAAME,WAAWJ,QAAQ,CAACZ,QAAQc,EAAE;YACpC,IAAIC,MAAMN,QAAQ,EAAE;gBAClB,IAAI,CAACO,SAASP,QAAQ,IAAI,CAACO,SAASrB,IAAI,CAACsB,QAAQ,CAACF,MAAMpB,IAAI,GAAGkB,UAAU;YAC3E,OAAO,IAAIG,SAASP,QAAQ,IAAIS,IAAAA,gBAAO,EAACjC,cAAc+B,SAASrB,IAAI,OAAOuB,IAAAA,gBAAO,EAACjC,cAAc8B,MAAMpB,IAAI,IAAI;gBAC5GkB,UAAU;YACZ;YACA,IAAI,CAACA,SAAS;QAChB;QACA,IAAIA,SAAS,OAAO;IACtB;IACA,OAAO;AACT;AAIO,SAAStB,aAAaI,IAAY;IACvC,OAAOwB,MAAMC,IAAI,CAACzB,KAAKG,QAAQ,CAACN,MAAM,SAAC6B;eAAO;YAAErB,OAAOqB,EAAEpB,KAAK;YAAEC,KAAKmB,EAAEpB,KAAK,GAAGoB,CAAC,CAAC,EAAE,CAAClB,MAAM;QAAC;;AAC7F;AACA,6FAA6F;AAC7F,8FAA8F;AAC9F,+DAA+D;AAC/D,IAAImB;AACJ,8FAA8F;AAC9F,8FAA8F;AAC9F,IAAMC,cAAc;AACpB,6FAA6F;AAC7F,0FAA0F;AAC1F,IAAMC,UAAU;AAEhB,SAASC,UAAUnB,GAAW;IAC5BgB,8BAAAA,+BAAAA,oBAAAA,oBAAsB,IAAII,KAAKC,SAAS,CAACC,WAAW;QAAEC,aAAa;IAAW;IAC9E,OAAOV,MAAMC,IAAI,CAACE,kBAAkBQ,OAAO,CAACxB,MAAM,SAACyB;eAAMA,EAAED,OAAO;;AACpE;AAEA,8FAA8F;AAC9F,0FAA0F;AAC1F,SAASE,mBAAmB1B,GAAW;IACrC,IAAM2B,SAAqB;QAAC,EAAE;KAAC;QAC1B,kCAAA,2BAAA;;QAAL,QAAK,YAAWR,UAAUnB,yBAArB,SAAA,6BAAA,QAAA,yBAAA,iCAA2B;YAA3B,IAAM4B,IAAN;YACH,IAAIX,YAAY3B,IAAI,CAACsC,IAAID,OAAOzB,IAAI,CAAC,EAAE;iBAClCyB,MAAM,CAACA,OAAO9B,MAAM,GAAG,EAAE,CAACK,IAAI,CAAC0B;QACtC;;QAHK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAIL,OAAOD,OAAOE,MAAM,CAAC,SAACD;eAAMA,EAAE/B,MAAM,GAAG;;AACzC;AAIO,SAASd,aAAaM,IAAY;IACvC,IAAI,CAACD,QAAQE,IAAI,CAACD,OAAO,OAAO;IAChC,IAAMyC,MAAMzC,KAAK0C,OAAO,CAAC7C,KAAK,SAACc;QAC7B,IAAMgC,OAAON,mBAAmB1B,KAC7BiC,GAAG,CAAC,SAACL;mBAAMA,EAAEM,IAAI,CAAC;WAClBA,IAAI,CAAC,AAAC,IAAW,OAARhB,SAAQ;QACpB,OAAO,AAAC,IAAcc,OAAXd,SAAQ,KAAWA,OAARc,MAAK,KAAW,OAARd,SAAQ;IACxC;IACA,OAAOY,IAAIC,OAAO,CAAC,QAAQ,KAAKI,IAAI;AACtC;AAEA,0FAA0F;AAC1F,wFAAwF;AACxF,IAAMC,YAAY;AAElB,4FAA4F;AAC5F,uGAAuG;AACvG,IAAMC,kBAAkB;AAExB,4FAA4F;AAC5F,yFAAyF;AACzF,SAASC,SAAStC,GAAW,EAAEuC,YAAoB;IACjD,OAAOb,mBAAmB1B,KACvBiC,GAAG,CAAC,SAACL;eAAM,AAAC,GAAiBA,OAAfW,cAAwD,OAAzCX,EAAE/B,MAAM,GAAG,IAAI,AAAC,IAAe,OAAZ+B,EAAEM,IAAI,CAAC,MAAK,OAAKN,CAAC,CAAC,EAAE;OACrEM,IAAI,CAAC;AACV;AAIO,SAASlD,aAAawD,KAAa;IACxC,IAAI,CAACpD,QAAQE,IAAI,CAACkD,QAAQ,OAAOA;IACjC,IAAIV,MAAM;IACV,IAAIW,SAAS;IACb,IAAMC,SAASF,MAAMG,KAAK,CAACzD,MAAM,2CAA2C;IAC5E,IAAK,IAAIsB,IAAI,GAAGA,IAAIkC,OAAO7C,MAAM,EAAEW,IAAK;QACtC,IAAIA,IAAI,MAAM,GAAG;gBACV,kCAAA,2BAAA;;gBAAL,QAAK,YAAYkC,MAAM,CAAClC,EAAE,qBAArB,SAAA,6BAAA,QAAA,yBAAA;oBAAA,IAAMoC,KAAN;oBAAuB,IAAIA,OAAO,KAAKH,SAAS,CAACA;;;gBAAjD;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YACLX,OAAOY,MAAM,CAAClC,EAAE;YAChB;QACF;QACA,IAAIiC,QAAQ;YACVX,OAAOY,MAAM,CAAClC,EAAE,EAAE,2CAA2C;YAC7D;QACF;QACA,IAAMO,IAAIe,IAAIrC,KAAK,CAAC2C;QACpB,IAAIrB,GAAG;YACLe,MAAM,AAAC,GAA0Bf,OAAxBe,IAAI/B,KAAK,CAAC,GAAGgB,EAAEpB,KAAK,GAAW2C,OAAPvB,CAAC,CAAC,EAAE,EAA8C,OAA3CuB,SAASI,MAAM,CAAClC,EAAE,EAAE,AAAC,GAASO,OAAPA,CAAC,CAAC,EAAE,EAAQ,OAALA,CAAC,CAAC,EAAE,EAAC;QAC7E,OAAO;YACLe,OAAOQ,SAASI,MAAM,CAAClC,EAAE,EAAE6B;QAC7B;IACF;IACA,OAAOP;AACT;AAIO,SAASnD,cAAcU,IAAY;IACxC,OAAOA,KAAKwD,SAAS,CAAC,OAAOd,OAAO,CAAC,27JAAW,IAAIe,WAAW;AACjE"}

package/dist/esm/cli/index.d.ts

@@ -5,7 +5,7 @@ export declare const USAGE: {
     readonly status: "status [--format table|json] [--config path]";
     readonly download: "download [--config path]";
     readonly sql: "sql \"<statement>\" [params...] [--preset name] [--format table|json|csv] [--config path]";
-    readonly search: "search \"<terms>\" [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json|csv] [--config path]";
+    readonly search: "search \"<terms>\" [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--snippet-char-limit n] [--snippet-count-limit n] [--format table|json|csv] [--config path]";
     readonly map: "map [--format table|json] [--config path]";
     readonly peek: "peek <path> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--format table|json] [--config path]";
     readonly path: "path <a> <b> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--max-depth n] [--format table|json] [--config path]";

package/dist/esm/cli/index.js

@@ -6,7 +6,7 @@ export const USAGE = {
     status: 'status [--format table|json] [--config path]',
     download: 'download [--config path]',
     sql: 'sql "<statement>" [params...] [--preset name] [--format table|json|csv] [--config path]',
-    search: 'search "<terms>" [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where "<sql>"] [--k n] [--format table|json|csv] [--config path]',
+    search: 'search "<terms>" [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where "<sql>"] [--k n] [--snippet-char-limit n] [--snippet-count-limit n] [--format table|json|csv] [--config path]',
     map: 'map [--format table|json] [--config path]',
     peek: 'peek <path> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where "<sql>"] [--format table|json] [--config path]',
     path: 'path <a> <b> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where "<sql>"] [--max-depth n] [--format table|json] [--config path]',

package/dist/esm/cli/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/index.ts"],"sourcesContent":["import type { Command } from './types.ts';\n\n// Each command's usage line, minus the leading \"usage: <name> \": shared by cli.ts's\n// usage() and the command's own parse() error/--help output so there is one copy, not two.\nexport const USAGE = {\n  init: 'init [--model id-or-path] [--provider static|openai|cohere] [--url url]',\n  watch: 'watch [--force] [--config path]',\n  status: 'status [--format table|json] [--config path]',\n  download: 'download [--config path]',\n  sql: 'sql \"<statement>\" [params...] [--preset name] [--format table|json|csv] [--config path]',\n  search: 'search \"<terms>\" [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json|csv] [--config path]',\n  map: 'map [--format table|json] [--config path]',\n  peek: 'peek <path> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--format table|json] [--config path]',\n  path: 'path <a> <b> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--max-depth n] [--format table|json] [--config path]',\n  related: 'related <note> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json|csv] [--config path]',\n} as const;\n\n// Lazy registry: a command's imports load only when it runs, so a heavy dependency in one\n// command never taxes the others (or --version/--help). These names are reserved from saved queries.\nexport const COMMANDS: Record<string, () => Promise<{ default: Command }>> = {\n  init: () => import('./init.ts'),\n  watch: () => import('./watch.ts'),\n  status: () => import('./status.ts'),\n  download: () => import('./download.ts'),\n  sql: () => import('./sql.ts'),\n  search: () => import('./search.ts'),\n  map: () => import('./map.ts'),\n  peek: () => import('./peek.ts'),\n  path: () => import('./path.ts'),\n  related: () => import('./related.ts'),\n};\n\nexport type { Command, Ctx } from './types.ts';\n"],"names":["USAGE","init","watch","status","download","sql","search","map","peek","path","related","COMMANDS"],"mappings":"AAEA,oFAAoF;AACpF,2FAA2F;AAC3F,OAAO,MAAMA,QAAQ;IACnBC,MAAM;IACNC,OAAO;IACPC,QAAQ;IACRC,UAAU;IACVC,KAAK;IACLC,QAAQ;IACRC,KAAK;IACLC,MAAM;IACNC,MAAM;IACNC,SAAS;AACX,EAAW;AAEX,0FAA0F;AAC1F,qGAAqG;AACrG,OAAO,MAAMC,WAAgE;IAC3EV,MAAM,IAAM,MAAM,CAAC;IACnBC,OAAO,IAAM,MAAM,CAAC;IACpBC,QAAQ,IAAM,MAAM,CAAC;IACrBC,UAAU,IAAM,MAAM,CAAC;IACvBC,KAAK,IAAM,MAAM,CAAC;IAClBC,QAAQ,IAAM,MAAM,CAAC;IACrBC,KAAK,IAAM,MAAM,CAAC;IAClBC,MAAM,IAAM,MAAM,CAAC;IACnBC,MAAM,IAAM,MAAM,CAAC;IACnBC,SAAS,IAAM,MAAM,CAAC;AACxB,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/index.ts"],"sourcesContent":["import type { Command } from './types.ts';\n\n// Each command's usage line, minus the leading \"usage: <name> \": shared by cli.ts's\n// usage() and the command's own parse() error/--help output so there is one copy, not two.\nexport const USAGE = {\n  init: 'init [--model id-or-path] [--provider static|openai|cohere] [--url url]',\n  watch: 'watch [--force] [--config path]',\n  status: 'status [--format table|json] [--config path]',\n  download: 'download [--config path]',\n  sql: 'sql \"<statement>\" [params...] [--preset name] [--format table|json|csv] [--config path]',\n  search: 'search \"<terms>\" [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--snippet-char-limit n] [--snippet-count-limit n] [--format table|json|csv] [--config path]',\n  map: 'map [--format table|json] [--config path]',\n  peek: 'peek <path> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--format table|json] [--config path]',\n  path: 'path <a> <b> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--max-depth n] [--format table|json] [--config path]',\n  related: 'related <note> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json|csv] [--config path]',\n} as const;\n\n// Lazy registry: a command's imports load only when it runs, so a heavy dependency in one\n// command never taxes the others (or --version/--help). These names are reserved from saved queries.\nexport const COMMANDS: Record<string, () => Promise<{ default: Command }>> = {\n  init: () => import('./init.ts'),\n  watch: () => import('./watch.ts'),\n  status: () => import('./status.ts'),\n  download: () => import('./download.ts'),\n  sql: () => import('./sql.ts'),\n  search: () => import('./search.ts'),\n  map: () => import('./map.ts'),\n  peek: () => import('./peek.ts'),\n  path: () => import('./path.ts'),\n  related: () => import('./related.ts'),\n};\n\nexport type { Command, Ctx } from './types.ts';\n"],"names":["USAGE","init","watch","status","download","sql","search","map","peek","path","related","COMMANDS"],"mappings":"AAEA,oFAAoF;AACpF,2FAA2F;AAC3F,OAAO,MAAMA,QAAQ;IACnBC,MAAM;IACNC,OAAO;IACPC,QAAQ;IACRC,UAAU;IACVC,KAAK;IACLC,QAAQ;IACRC,KAAK;IACLC,MAAM;IACNC,MAAM;IACNC,SAAS;AACX,EAAW;AAEX,0FAA0F;AAC1F,qGAAqG;AACrG,OAAO,MAAMC,WAAgE;IAC3EV,MAAM,IAAM,MAAM,CAAC;IACnBC,OAAO,IAAM,MAAM,CAAC;IACpBC,QAAQ,IAAM,MAAM,CAAC;IACrBC,UAAU,IAAM,MAAM,CAAC;IACvBC,KAAK,IAAM,MAAM,CAAC;IAClBC,QAAQ,IAAM,MAAM,CAAC;IACrBC,KAAK,IAAM,MAAM,CAAC;IAClBC,MAAM,IAAM,MAAM,CAAC;IACnBC,MAAM,IAAM,MAAM,CAAC;IACnBC,SAAS,IAAM,MAAM,CAAC;AACxB,EAAE"}

package/dist/esm/cli/named.js

@@ -1,11 +1,11 @@
 import { search } from '../commands/search.js';
 import { printRows } from '../output/output.js';
-import { CONFIG, FORMAT, parse, parseK, rowFormatOf, runSql, SEARCH_FLAGS, withDb } from './shared.js';
+import { CONFIG, FORMAT, parse, parseK, parseSnippetCharLimit, parseSnippetCountLimit, rowFormatOf, runSql, SEARCH_FLAGS, withDb } from './shared.js';
 // Fallback when the first positional is not a command: a saved query, { sql } or { search }.
 // SEARCH_FLAGS override a saved search's fields the same way they override a preset's.
 export default async function named(ctx, queryName) {
     var _parseK, _values_where, _values_preset, _values_include, _values_exclude;
-    const usage = `usage: ${ctx.name} ${queryName} [params...] [--format table|json|csv] [--config path] [--where "<sql>"] [--k n] [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude]`;
+    const usage = `usage: ${ctx.name} ${queryName} [params...] [--format table|json|csv] [--config path] [--where "<sql>"] [--k n] [--snippet-char-limit n] [--snippet-count-limit n] [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude]`;
     const { values, positionals: params } = parse(ctx.argv, usage, {
         ...SEARCH_FLAGS,
         ...FORMAT,
@@ -31,6 +31,8 @@ export default async function named(ctx, queryName) {
         ctx.usageError(`"${queryName}" is a saved search and takes no positional parameters; edit its "search" in sense.config.json, or use "${ctx.name} search" directly`);
     }
     const k = (_parseK = parseK(values.k, ctx.usageError)) !== null && _parseK !== void 0 ? _parseK : entry.k;
+    const snippetCharLimit = parseSnippetCharLimit(values['snippet-char-limit'], ctx.usageError);
+    const snippetCountLimit = parseSnippetCountLimit(values['snippet-count-limit'], ctx.usageError);
     const where = (_values_where = values.where) !== null && _values_where !== void 0 ? _values_where : entry.where;
     const preset = (_values_preset = values.preset) !== null && _values_preset !== void 0 ? _values_preset : entry.preset;
     const include = (_values_include = values.include) !== null && _values_include !== void 0 ? _values_include : entry.include;
@@ -38,6 +40,8 @@ export default async function named(ctx, queryName) {
     const noExclude = values['no-exclude'] === true;
     await withDb(ctx, configPath, async (store, resolvedCfg)=>printRows(await search(store, resolvedCfg, entry.search, {
             k,
+            snippetCharLimit,
+            snippetCountLimit,
             where,
             preset,
             include,

package/dist/esm/cli/named.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/named.ts"],"sourcesContent":["import { search } from '../commands/search.ts';\nimport { printRows } from '../output/output.ts';\nimport { CONFIG, FORMAT, parse, parseK, rowFormatOf, runSql, SEARCH_FLAGS, withDb } from './shared.ts';\nimport type { Ctx } from './types.ts';\n\n// Fallback when the first positional is not a command: a saved query, { sql } or { search }.\n// SEARCH_FLAGS override a saved search's fields the same way they override a preset's.\nexport default async function named(ctx: Ctx, queryName: string): Promise<void> {\n  const usage = `usage: ${ctx.name} ${queryName} [params...] [--format table|json|csv] [--config path] [--where \"<sql>\"] [--k n] [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude]`;\n  const { values, positionals: params } = parse(ctx.argv, usage, { ...SEARCH_FLAGS, ...FORMAT, ...CONFIG });\n  const format = rowFormatOf(values);\n  const configPath = values.config as string | undefined;\n\n  const cfg = ctx.resolveConfig(configPath);\n  const entry = cfg.queries[queryName];\n  if (entry === undefined) {\n    console.error(`unknown command or saved entry: \"${queryName}\"`);\n    console.error(`saved queries: ${Object.keys(cfg.queries).sort().join(', ')}`);\n    process.exit(2);\n  }\n\n  if ('sql' in entry) {\n    // A saved statement written against `scope` is preset-agnostic, so the same entry can be\n    // re-pointed at another layer from the command line instead of being copied per preset.\n    await runSql(cfg, entry.sql, params, format, `query \"${queryName}\"`, values.preset as string | undefined);\n    return;\n  }\n\n  // A saved search's text is fixed in config; there is nowhere for a positional to bind.\n  if (params.length > 0) {\n    ctx.usageError(`\"${queryName}\" is a saved search and takes no positional parameters; edit its \"search\" in sense.config.json, or use \"${ctx.name} search\" directly`);\n  }\n  const k = parseK(values.k as string | undefined, ctx.usageError) ?? entry.k;\n  const where = (values.where as string | undefined) ?? entry.where;\n  const preset = (values.preset as string | undefined) ?? entry.preset;\n  const include = (values.include as string[] | undefined) ?? entry.include;\n  const exclude = (values.exclude as string[] | undefined) ?? entry.exclude;\n  const noExclude = values['no-exclude'] === true;\n  await withDb(ctx, configPath, async (store, resolvedCfg) => printRows(await search(store, resolvedCfg, entry.search, { k, where, preset, include, exclude, noExclude }), format));\n}\n"],"names":["search","printRows","CONFIG","FORMAT","parse","parseK","rowFormatOf","runSql","SEARCH_FLAGS","withDb","named","ctx","queryName","values","usage","name","positionals","params","argv","format","configPath","config","cfg","resolveConfig","entry","queries","undefined","console","error","Object","keys","sort","join","process","exit","sql","preset","length","usageError","k","where","include","exclude","noExclude","store","resolvedCfg"],"mappings":"AAAA,SAASA,MAAM,QAAQ,wBAAwB;AAC/C,SAASC,SAAS,QAAQ,sBAAsB;AAChD,SAASC,MAAM,EAAEC,MAAM,EAAEC,KAAK,EAAEC,MAAM,EAAEC,WAAW,EAAEC,MAAM,EAAEC,YAAY,EAAEC,MAAM,QAAQ,cAAc;AAGvG,6FAA6F;AAC7F,uFAAuF;AACvF,eAAe,eAAeC,MAAMC,GAAQ,EAAEC,SAAiB;QAyBnDP,SACKQ,eACCA,gBACCA,iBACAA;IA5BjB,MAAMC,QAAQ,CAAC,OAAO,EAAEH,IAAII,IAAI,CAAC,CAAC,EAAEH,UAAU,yJAAyJ,CAAC;IACxM,MAAM,EAAEC,MAAM,EAAEG,aAAaC,MAAM,EAAE,GAAGb,MAAMO,IAAIO,IAAI,EAAEJ,OAAO;QAAE,GAAGN,YAAY;QAAE,GAAGL,MAAM;QAAE,GAAGD,MAAM;IAAC;IACvG,MAAMiB,SAASb,YAAYO;IAC3B,MAAMO,aAAaP,OAAOQ,MAAM;IAEhC,MAAMC,MAAMX,IAAIY,aAAa,CAACH;IAC9B,MAAMI,QAAQF,IAAIG,OAAO,CAACb,UAAU;IACpC,IAAIY,UAAUE,WAAW;QACvBC,QAAQC,KAAK,CAAC,CAAC,iCAAiC,EAAEhB,UAAU,CAAC,CAAC;QAC9De,QAAQC,KAAK,CAAC,CAAC,eAAe,EAAEC,OAAOC,IAAI,CAACR,IAAIG,OAAO,EAAEM,IAAI,GAAGC,IAAI,CAAC,OAAO;QAC5EC,QAAQC,IAAI,CAAC;IACf;IAEA,IAAI,SAASV,OAAO;QAClB,yFAAyF;QACzF,wFAAwF;QACxF,MAAMjB,OAAOe,KAAKE,MAAMW,GAAG,EAAElB,QAAQE,QAAQ,CAAC,OAAO,EAAEP,UAAU,CAAC,CAAC,EAAEC,OAAOuB,MAAM;QAClF;IACF;IAEA,uFAAuF;IACvF,IAAInB,OAAOoB,MAAM,GAAG,GAAG;QACrB1B,IAAI2B,UAAU,CAAC,CAAC,CAAC,EAAE1B,UAAU,wGAAwG,EAAED,IAAII,IAAI,CAAC,iBAAiB,CAAC;IACpK;IACA,MAAMwB,KAAIlC,UAAAA,OAAOQ,OAAO0B,CAAC,EAAwB5B,IAAI2B,UAAU,eAArDjC,qBAAAA,UAA0DmB,MAAMe,CAAC;IAC3E,MAAMC,SAAS3B,gBAAAA,OAAO2B,KAAK,cAAZ3B,2BAAAA,gBAAuCW,MAAMgB,KAAK;IACjE,MAAMJ,UAAUvB,iBAAAA,OAAOuB,MAAM,cAAbvB,4BAAAA,iBAAwCW,MAAMY,MAAM;IACpE,MAAMK,WAAW5B,kBAAAA,OAAO4B,OAAO,cAAd5B,6BAAAA,kBAA2CW,MAAMiB,OAAO;IACzE,MAAMC,WAAW7B,kBAAAA,OAAO6B,OAAO,cAAd7B,6BAAAA,kBAA2CW,MAAMkB,OAAO;IACzE,MAAMC,YAAY9B,MAAM,CAAC,aAAa,KAAK;IAC3C,MAAMJ,OAAOE,KAAKS,YAAY,OAAOwB,OAAOC,cAAgB5C,UAAU,MAAMD,OAAO4C,OAAOC,aAAarB,MAAMxB,MAAM,EAAE;YAAEuC;YAAGC;YAAOJ;YAAQK;YAASC;YAASC;QAAU,IAAIxB;AAC3K"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/named.ts"],"sourcesContent":["import { search } from '../commands/search.ts';\nimport { printRows } from '../output/output.ts';\nimport { CONFIG, FORMAT, parse, parseK, parseSnippetCharLimit, parseSnippetCountLimit, rowFormatOf, runSql, SEARCH_FLAGS, withDb } from './shared.ts';\nimport type { Ctx } from './types.ts';\n\n// Fallback when the first positional is not a command: a saved query, { sql } or { search }.\n// SEARCH_FLAGS override a saved search's fields the same way they override a preset's.\nexport default async function named(ctx: Ctx, queryName: string): Promise<void> {\n  const usage = `usage: ${ctx.name} ${queryName} [params...] [--format table|json|csv] [--config path] [--where \"<sql>\"] [--k n] [--snippet-char-limit n] [--snippet-count-limit n] [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude]`;\n  const { values, positionals: params } = parse(ctx.argv, usage, { ...SEARCH_FLAGS, ...FORMAT, ...CONFIG });\n  const format = rowFormatOf(values);\n  const configPath = values.config as string | undefined;\n\n  const cfg = ctx.resolveConfig(configPath);\n  const entry = cfg.queries[queryName];\n  if (entry === undefined) {\n    console.error(`unknown command or saved entry: \"${queryName}\"`);\n    console.error(`saved queries: ${Object.keys(cfg.queries).sort().join(', ')}`);\n    process.exit(2);\n  }\n\n  if ('sql' in entry) {\n    // A saved statement written against `scope` is preset-agnostic, so the same entry can be\n    // re-pointed at another layer from the command line instead of being copied per preset.\n    await runSql(cfg, entry.sql, params, format, `query \"${queryName}\"`, values.preset as string | undefined);\n    return;\n  }\n\n  // A saved search's text is fixed in config; there is nowhere for a positional to bind.\n  if (params.length > 0) {\n    ctx.usageError(`\"${queryName}\" is a saved search and takes no positional parameters; edit its \"search\" in sense.config.json, or use \"${ctx.name} search\" directly`);\n  }\n  const k = parseK(values.k as string | undefined, ctx.usageError) ?? entry.k;\n  const snippetCharLimit = parseSnippetCharLimit(values['snippet-char-limit'] as string | undefined, ctx.usageError);\n  const snippetCountLimit = parseSnippetCountLimit(values['snippet-count-limit'] as string | undefined, ctx.usageError);\n  const where = (values.where as string | undefined) ?? entry.where;\n  const preset = (values.preset as string | undefined) ?? entry.preset;\n  const include = (values.include as string[] | undefined) ?? entry.include;\n  const exclude = (values.exclude as string[] | undefined) ?? entry.exclude;\n  const noExclude = values['no-exclude'] === true;\n  await withDb(ctx, configPath, async (store, resolvedCfg) => printRows(await search(store, resolvedCfg, entry.search, { k, snippetCharLimit, snippetCountLimit, where, preset, include, exclude, noExclude }), format));\n}\n"],"names":["search","printRows","CONFIG","FORMAT","parse","parseK","parseSnippetCharLimit","parseSnippetCountLimit","rowFormatOf","runSql","SEARCH_FLAGS","withDb","named","ctx","queryName","values","usage","name","positionals","params","argv","format","configPath","config","cfg","resolveConfig","entry","queries","undefined","console","error","Object","keys","sort","join","process","exit","sql","preset","length","usageError","k","snippetCharLimit","snippetCountLimit","where","include","exclude","noExclude","store","resolvedCfg"],"mappings":"AAAA,SAASA,MAAM,QAAQ,wBAAwB;AAC/C,SAASC,SAAS,QAAQ,sBAAsB;AAChD,SAASC,MAAM,EAAEC,MAAM,EAAEC,KAAK,EAAEC,MAAM,EAAEC,qBAAqB,EAAEC,sBAAsB,EAAEC,WAAW,EAAEC,MAAM,EAAEC,YAAY,EAAEC,MAAM,QAAQ,cAAc;AAGtJ,6FAA6F;AAC7F,uFAAuF;AACvF,eAAe,eAAeC,MAAMC,GAAQ,EAAEC,SAAiB;QAyBnDT,SAGKU,eACCA,gBACCA,iBACAA;IA9BjB,MAAMC,QAAQ,CAAC,OAAO,EAAEH,IAAII,IAAI,CAAC,CAAC,EAAEH,UAAU,4MAA4M,CAAC;IAC3P,MAAM,EAAEC,MAAM,EAAEG,aAAaC,MAAM,EAAE,GAAGf,MAAMS,IAAIO,IAAI,EAAEJ,OAAO;QAAE,GAAGN,YAAY;QAAE,GAAGP,MAAM;QAAE,GAAGD,MAAM;IAAC;IACvG,MAAMmB,SAASb,YAAYO;IAC3B,MAAMO,aAAaP,OAAOQ,MAAM;IAEhC,MAAMC,MAAMX,IAAIY,aAAa,CAACH;IAC9B,MAAMI,QAAQF,IAAIG,OAAO,CAACb,UAAU;IACpC,IAAIY,UAAUE,WAAW;QACvBC,QAAQC,KAAK,CAAC,CAAC,iCAAiC,EAAEhB,UAAU,CAAC,CAAC;QAC9De,QAAQC,KAAK,CAAC,CAAC,eAAe,EAAEC,OAAOC,IAAI,CAACR,IAAIG,OAAO,EAAEM,IAAI,GAAGC,IAAI,CAAC,OAAO;QAC5EC,QAAQC,IAAI,CAAC;IACf;IAEA,IAAI,SAASV,OAAO;QAClB,yFAAyF;QACzF,wFAAwF;QACxF,MAAMjB,OAAOe,KAAKE,MAAMW,GAAG,EAAElB,QAAQE,QAAQ,CAAC,OAAO,EAAEP,UAAU,CAAC,CAAC,EAAEC,OAAOuB,MAAM;QAClF;IACF;IAEA,uFAAuF;IACvF,IAAInB,OAAOoB,MAAM,GAAG,GAAG;QACrB1B,IAAI2B,UAAU,CAAC,CAAC,CAAC,EAAE1B,UAAU,wGAAwG,EAAED,IAAII,IAAI,CAAC,iBAAiB,CAAC;IACpK;IACA,MAAMwB,KAAIpC,UAAAA,OAAOU,OAAO0B,CAAC,EAAwB5B,IAAI2B,UAAU,eAArDnC,qBAAAA,UAA0DqB,MAAMe,CAAC;IAC3E,MAAMC,mBAAmBpC,sBAAsBS,MAAM,CAAC,qBAAqB,EAAwBF,IAAI2B,UAAU;IACjH,MAAMG,oBAAoBpC,uBAAuBQ,MAAM,CAAC,sBAAsB,EAAwBF,IAAI2B,UAAU;IACpH,MAAMI,SAAS7B,gBAAAA,OAAO6B,KAAK,cAAZ7B,2BAAAA,gBAAuCW,MAAMkB,KAAK;IACjE,MAAMN,UAAUvB,iBAAAA,OAAOuB,MAAM,cAAbvB,4BAAAA,iBAAwCW,MAAMY,MAAM;IACpE,MAAMO,WAAW9B,kBAAAA,OAAO8B,OAAO,cAAd9B,6BAAAA,kBAA2CW,MAAMmB,OAAO;IACzE,MAAMC,WAAW/B,kBAAAA,OAAO+B,OAAO,cAAd/B,6BAAAA,kBAA2CW,MAAMoB,OAAO;IACzE,MAAMC,YAAYhC,MAAM,CAAC,aAAa,KAAK;IAC3C,MAAMJ,OAAOE,KAAKS,YAAY,OAAO0B,OAAOC,cAAgBhD,UAAU,MAAMD,OAAOgD,OAAOC,aAAavB,MAAM1B,MAAM,EAAE;YAAEyC;YAAGC;YAAkBC;YAAmBC;YAAON;YAAQO;YAASC;YAASC;QAAU,IAAI1B;AAChN"}

package/dist/esm/cli/search.js

@@ -1,7 +1,7 @@
 import { search } from '../commands/search.js';
 import { printRows } from '../output/output.js';
 import { USAGE } from './index.js';
-import { CONFIG, FORMAT, parse, parseK, rowFormatOf, SEARCH_FLAGS, scopeOf, withDb } from './shared.js';
+import { CONFIG, FORMAT, parse, parseK, parseSnippetCharLimit, parseSnippetCountLimit, rowFormatOf, SEARCH_FLAGS, scopeOf, withDb } from './shared.js';
 const searchCmd = async (ctx)=>{
     const usage = `usage: ${ctx.name} ${USAGE.search}`;
     const { values, positionals } = parse(ctx.argv, usage, {
@@ -12,9 +12,13 @@ const searchCmd = async (ctx)=>{
     const [terms] = positionals;
     if (!terms) ctx.usageError(usage);
     const k = parseK(values.k, ctx.usageError);
+    const snippetCharLimit = parseSnippetCharLimit(values['snippet-char-limit'], ctx.usageError);
+    const snippetCountLimit = parseSnippetCountLimit(values['snippet-count-limit'], ctx.usageError);
     const format = rowFormatOf(values);
     await withDb(ctx, values.config, async (store, cfg)=>printRows(await search(store, cfg, terms, {
             k,
+            snippetCharLimit,
+            snippetCountLimit,
             ...scopeOf(values)
         }), format));
 };

package/dist/esm/cli/search.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/search.ts"],"sourcesContent":["import { search } from '../commands/search.ts';\nimport { printRows } from '../output/output.ts';\nimport { USAGE } from './index.ts';\nimport { CONFIG, FORMAT, parse, parseK, rowFormatOf, SEARCH_FLAGS, scopeOf, withDb } from './shared.ts';\nimport type { Command } from './types.ts';\n\nconst searchCmd: Command = async (ctx) => {\n  const usage = `usage: ${ctx.name} ${USAGE.search}`;\n  const { values, positionals } = parse(ctx.argv, usage, { ...SEARCH_FLAGS, ...FORMAT, ...CONFIG });\n  const [terms] = positionals;\n  if (!terms) ctx.usageError(usage);\n  const k = parseK(values.k as string | undefined, ctx.usageError);\n  const format = rowFormatOf(values);\n  await withDb(ctx, values.config as string | undefined, async (store, cfg) => printRows(await search(store, cfg, terms, { k, ...scopeOf(values) }), format));\n};\nexport default searchCmd;\n"],"names":["search","printRows","USAGE","CONFIG","FORMAT","parse","parseK","rowFormatOf","SEARCH_FLAGS","scopeOf","withDb","searchCmd","ctx","usage","name","values","positionals","argv","terms","usageError","k","format","config","store","cfg"],"mappings":"AAAA,SAASA,MAAM,QAAQ,wBAAwB;AAC/C,SAASC,SAAS,QAAQ,sBAAsB;AAChD,SAASC,KAAK,QAAQ,aAAa;AACnC,SAASC,MAAM,EAAEC,MAAM,EAAEC,KAAK,EAAEC,MAAM,EAAEC,WAAW,EAAEC,YAAY,EAAEC,OAAO,EAAEC,MAAM,QAAQ,cAAc;AAGxG,MAAMC,YAAqB,OAAOC;IAChC,MAAMC,QAAQ,CAAC,OAAO,EAAED,IAAIE,IAAI,CAAC,CAAC,EAAEZ,MAAMF,MAAM,EAAE;IAClD,MAAM,EAAEe,MAAM,EAAEC,WAAW,EAAE,GAAGX,MAAMO,IAAIK,IAAI,EAAEJ,OAAO;QAAE,GAAGL,YAAY;QAAE,GAAGJ,MAAM;QAAE,GAAGD,MAAM;IAAC;IAC/F,MAAM,CAACe,MAAM,GAAGF;IAChB,IAAI,CAACE,OAAON,IAAIO,UAAU,CAACN;IAC3B,MAAMO,IAAId,OAAOS,OAAOK,CAAC,EAAwBR,IAAIO,UAAU;IAC/D,MAAME,SAASd,YAAYQ;IAC3B,MAAML,OAAOE,KAAKG,OAAOO,MAAM,EAAwB,OAAOC,OAAOC,MAAQvB,UAAU,MAAMD,OAAOuB,OAAOC,KAAKN,OAAO;YAAEE;YAAG,GAAGX,QAAQM,OAAO;QAAC,IAAIM;AACrJ;AACA,eAAeV,UAAU"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/search.ts"],"sourcesContent":["import { search } from '../commands/search.ts';\nimport { printRows } from '../output/output.ts';\nimport { USAGE } from './index.ts';\nimport { CONFIG, FORMAT, parse, parseK, parseSnippetCharLimit, parseSnippetCountLimit, rowFormatOf, SEARCH_FLAGS, scopeOf, withDb } from './shared.ts';\nimport type { Command } from './types.ts';\n\nconst searchCmd: Command = async (ctx) => {\n  const usage = `usage: ${ctx.name} ${USAGE.search}`;\n  const { values, positionals } = parse(ctx.argv, usage, { ...SEARCH_FLAGS, ...FORMAT, ...CONFIG });\n  const [terms] = positionals;\n  if (!terms) ctx.usageError(usage);\n  const k = parseK(values.k as string | undefined, ctx.usageError);\n  const snippetCharLimit = parseSnippetCharLimit(values['snippet-char-limit'] as string | undefined, ctx.usageError);\n  const snippetCountLimit = parseSnippetCountLimit(values['snippet-count-limit'] as string | undefined, ctx.usageError);\n  const format = rowFormatOf(values);\n  await withDb(ctx, values.config as string | undefined, async (store, cfg) => printRows(await search(store, cfg, terms, { k, snippetCharLimit, snippetCountLimit, ...scopeOf(values) }), format));\n};\nexport default searchCmd;\n"],"names":["search","printRows","USAGE","CONFIG","FORMAT","parse","parseK","parseSnippetCharLimit","parseSnippetCountLimit","rowFormatOf","SEARCH_FLAGS","scopeOf","withDb","searchCmd","ctx","usage","name","values","positionals","argv","terms","usageError","k","snippetCharLimit","snippetCountLimit","format","config","store","cfg"],"mappings":"AAAA,SAASA,MAAM,QAAQ,wBAAwB;AAC/C,SAASC,SAAS,QAAQ,sBAAsB;AAChD,SAASC,KAAK,QAAQ,aAAa;AACnC,SAASC,MAAM,EAAEC,MAAM,EAAEC,KAAK,EAAEC,MAAM,EAAEC,qBAAqB,EAAEC,sBAAsB,EAAEC,WAAW,EAAEC,YAAY,EAAEC,OAAO,EAAEC,MAAM,QAAQ,cAAc;AAGvJ,MAAMC,YAAqB,OAAOC;IAChC,MAAMC,QAAQ,CAAC,OAAO,EAAED,IAAIE,IAAI,CAAC,CAAC,EAAEd,MAAMF,MAAM,EAAE;IAClD,MAAM,EAAEiB,MAAM,EAAEC,WAAW,EAAE,GAAGb,MAAMS,IAAIK,IAAI,EAAEJ,OAAO;QAAE,GAAGL,YAAY;QAAE,GAAGN,MAAM;QAAE,GAAGD,MAAM;IAAC;IAC/F,MAAM,CAACiB,MAAM,GAAGF;IAChB,IAAI,CAACE,OAAON,IAAIO,UAAU,CAACN;IAC3B,MAAMO,IAAIhB,OAAOW,OAAOK,CAAC,EAAwBR,IAAIO,UAAU;IAC/D,MAAME,mBAAmBhB,sBAAsBU,MAAM,CAAC,qBAAqB,EAAwBH,IAAIO,UAAU;IACjH,MAAMG,oBAAoBhB,uBAAuBS,MAAM,CAAC,sBAAsB,EAAwBH,IAAIO,UAAU;IACpH,MAAMI,SAAShB,YAAYQ;IAC3B,MAAML,OAAOE,KAAKG,OAAOS,MAAM,EAAwB,OAAOC,OAAOC,MAAQ3B,UAAU,MAAMD,OAAO2B,OAAOC,KAAKR,OAAO;YAAEE;YAAGC;YAAkBC;YAAmB,GAAGb,QAAQM,OAAO;QAAC,IAAIQ;AAC1L;AACA,eAAeZ,UAAU"}

package/dist/esm/cli/shared.d.ts

@@ -16,6 +16,8 @@ export declare function parse(argv: string[], usage: string, options: ParseArgsO
     positionals: string[];
 };
 export declare function parseK(k: string | undefined, usageError: (message: string) => never): number | undefined;
+export declare function parseSnippetCharLimit(value: string | undefined, usageError: (message: string) => never): number | undefined;
+export declare function parseSnippetCountLimit(value: string | undefined, usageError: (message: string) => never): number | undefined;
 export declare function printWarnings(warnings: string[]): void;
 export declare function withDb(ctx: Ctx, configPath: string | undefined, fn: (store: OpenResult['store'], cfg: ResolvedConfig) => void | Promise<void>): Promise<void>;
 export declare function runSql(cfg: ResolvedConfig, sql: string, params: string[], format: RowFormat, label: string, preset?: string): Promise<void>;