rehydra 0.11.1 → 0.13.0

package/dist/ner/ner-model.js

@@ -6,6 +6,7 @@
 import { loadRuntime, getRuntimeType, } from "#onnx-runtime";
 import { WordPieceTokenizer, loadVocabFromFile, } from "./tokenizer.js";
 import { decodeBIOTags, convertToSpanMatches, cleanupSpanBoundaries, mergeAdjacentSpans, } from "./bio-decoder.js";
+import { createTokenWindows, mergeWindowSpans, validateWindowConfig, windowTokenBudget, } from "./token-window.js";
 import { getStorageProvider, isBrowser } from "#storage";
 /**
  * Default label map for common NER models (CoNLL-style)
@@ -88,35 +89,52 @@ export class NERModel {
         this.isLoaded = true;
     }
     /**
-     * Runs a full NER pass: tokenize into overlapping windows, infer per
-     * window, decode BIO tags, and reconcile window results.
+     * Runs a single NER pass: tokenize, infer, decode BIO tags.
      * @param inputText - Text to tokenize and feed to the model
      * @param originalText - Original text used for extracting entity text (may differ in casing)
      */
     async runNERPass(inputText, originalText, minConfidence) {
-        // Tokenize into overlapping model-sized windows so inputs longer than
-        // maxLength tokens are fully processed instead of silently truncated
-        const windows = this.tokenizer.tokenizeWindows(inputText);
+        // Tokenize without per-model truncation so windows retain global
+        // character offsets. Bound the token count to what maxWindowsPerInput
+        // windows can consume: only the leading windows are ever scanned, so
+        // materializing a multi-million-token array for a multi-MB value would
+        // be pure waste (and a memory hazard). maxTokens undefined = unbounded.
+        const maxTokens = this.config.maxWindowsPerInput === undefined
+            ? undefined
+            : windowTokenBudget(this.config.maxLength, this.config.windowOverlapTokens, this.config.maxWindowsPerInput);
+        const tokenization = this.tokenizer.tokenize(inputText, { truncate: false, maxTokens });
+        const windows = createTokenWindows(tokenization, this.config.maxLength, this.config.windowOverlapTokens);
+        if (tokenization.truncated) {
+            const info = { scannedWindows: windows.length, inputLength: inputText.length };
+            if (this.config.onWindowCap) {
+                this.config.onWindowCap(info);
+            }
+            else {
+                console.warn(`[rehydra] NER input (${info.inputLength} chars) exceeds the window budget; scanning only the first ${info.scannedWindows} windows (maxWindowsPerInput)`);
+            }
+        }
         const spans = [];
         for (const window of windows) {
-            // Run inference
             const { labels, confidences } = await this.runInference(window);
-            // Decode BIO tags using original text for entity text extraction
-            // (offsets are identical since casing doesn't change string length)
             const rawEntities = decodeBIOTags(window.tokens, labels, confidences, originalText);
-            // Keep entities that overlap this window's core range. Overlap
-            // (rather than strict start-anchoring) tolerates adjacent windows
-            // decoding slightly different boundaries for the same entity near a
-            // core boundary — otherwise both windows could reject it and the
-            // entity would be dropped entirely
-            const anchored = rawEntities.filter((e) => e.end > window.coreCharStart && e.start < window.coreCharEnd);
-            // Convert to SpanMatch format with confidence filtering
-            spans.push(...convertToSpanMatches(anchored, minConfidence));
+            spans.push(...convertToSpanMatches(rawEntities, minConfidence));
         }
-        // An entity straddling a core boundary can be kept by both adjacent
-        // windows; union overlapping same-type spans so it is reported once
-        // with its full detected extent
-        return mergeOverlappingWindowSpans(spans, originalText);
+        // Chars of input actually scanned = end of the last content token (the
+        // token before the trailing SEP). When there is no content token, coverage
+        // is 0 if we truncated (nothing was scanned — a caller must not treat this
+        // as "all covered") and the full length otherwise (empty/whitespace input).
+        const lastContentToken = tokenization.tokens[tokenization.tokens.length - 2];
+        const hasContentToken = lastContentToken !== undefined && !lastContentToken.isSpecial;
+        const coverageChars = hasContentToken
+            ? lastContentToken.end
+            : tokenization.truncated
+                ? 0
+                : inputText.length;
+        return {
+            spans: mergeWindowSpans(spans, originalText),
+            truncated: tokenization.truncated,
+            coverageChars,
+        };
     }
     /**
      * Predicts entities in text
@@ -128,21 +146,29 @@ export class NERModel {
         }
         const minConfidence = this.getMinConfidence(policy);
         // Primary NER pass on original text
-        let spans = await this.runNERPass(text, text, minConfidence);
+        const primary = await this.runNERPass(text, text, minConfidence);
+        let spans = primary.spans;
+        let truncated = primary.truncated;
+        // A caller cutting at coverageChars must stay within what BOTH passes
+        // scanned, so take the smaller offset. (Title-casing preserves length but
+        // changes tokenization, so the fallback pass can cap at a different point.)
+        let coverageChars = primary.coverageChars;
         // Case fallback: run a second pass on title-cased text to catch lowercase names
         const caseFallback = this.config.caseFallback ?? false;
         if (caseFallback) {
             const titleCased = titleCaseWords(text);
             if (titleCased !== text) {
                 const penalty = this.config.caseFallbackPenalty ?? 0.85;
-                const fallbackSpans = await this.runNERPass(titleCased, text, minConfidence);
+                const fallback = await this.runNERPass(titleCased, text, minConfidence);
+                truncated = truncated || fallback.truncated;
+                coverageChars = Math.min(coverageChars, fallback.coverageChars);
                 // Merge only non-overlapping fallback detections with a confidence penalty
-                for (const fallback of fallbackSpans) {
-                    const overlaps = spans.some((primary) => primary.start < fallback.end && fallback.start < primary.end);
+                for (const span of fallback.spans) {
+                    const overlaps = spans.some((existing) => existing.start < span.end && span.start < existing.end);
                     if (!overlaps) {
                         spans.push({
-                            ...fallback,
-                            confidence: fallback.confidence * penalty,
+                            ...span,
+                            confidence: span.confidence * penalty,
                         });
                     }
                 }
@@ -161,6 +187,8 @@ export class NERModel {
             spans,
             processingTimeMs: endTime - startTime,
             modelVersion: this.config.modelVersion,
+            truncated,
+            coverageChars,
         };
     }
     /**
@@ -268,33 +296,6 @@ export class NERModel {
         return Promise.resolve();
     }
 }
-/**
- * Merges overlapping same-type spans collected from adjacent windows into
- * their union. Windows can disagree on the exact boundaries of an entity
- * near a core boundary; keeping the union guarantees no part of a detected
- * entity is left unmasked.
- */
-function mergeOverlappingWindowSpans(spans, originalText) {
-    if (spans.length <= 1)
-        return spans;
-    const sorted = [...spans].sort((a, b) => a.start - b.start || b.end - a.end);
-    const merged = [{ ...sorted[0] }];
-    for (let i = 1; i < sorted.length; i++) {
-        const span = sorted[i];
-        const last = merged[merged.length - 1];
-        if (span.type === last.type && span.start < last.end) {
-            if (span.end > last.end) {
-                last.end = span.end;
-                last.text = originalText.slice(last.start, last.end);
-            }
-            last.confidence = Math.max(last.confidence, span.confidence);
-        }
-        else {
-            merged.push({ ...span });
-        }
-    }
-    return merged;
-}
 /**
  * Common function words / stopwords that should stay lowercase during case
  * fallback so the NER model can still use contrastive casing as a signal.
@@ -460,17 +461,24 @@ function softmax(logits) {
  * Creates a NER model instance with configuration
  */
 export function createNERModel(config) {
+    const maxLength = config.maxLength ?? 512;
     const fullConfig = {
         modelPath: config.modelPath,
         vocabPath: config.vocabPath,
         labelMap: config.labelMap ?? DEFAULT_LABEL_MAP,
-        maxLength: config.maxLength ?? 512,
+        maxLength,
+        windowOverlapTokens: config.windowOverlapTokens
+            ?? Math.min(64, Math.max(0, Math.floor((maxLength - 2) / 4))),
+        maxWindowsPerInput: config.maxWindowsPerInput,
+        onWindowCap: config.onWindowCap,
         doLowerCase: config.doLowerCase ?? false, // XLM-RoBERTa is cased
         modelVersion: config.modelVersion ?? "1.0.0",
         sessionOptions: config.sessionOptions,
         caseFallback: config.caseFallback,
         caseFallbackPenalty: config.caseFallbackPenalty,
     };
+    // Fail fast on bad windowing config instead of on the first predict().
+    validateWindowConfig(fullConfig.maxLength, fullConfig.windowOverlapTokens, fullConfig.maxWindowsPerInput);
     return new NERModel(fullConfig);
 }
 /**
@@ -488,6 +496,8 @@ export class NERModelStub {
             spans: [],
             processingTimeMs: 0,
             modelVersion: this.version,
+            truncated: false,
+            coverageChars: _text.length,
         });
     }
     dispose() {

package/dist/ner/ner-model.js.map

@@ -1 +1 @@
-{"version":3,"file":"ner-model.js","sourceRoot":"","sources":["../../src/ner/ner-model.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EACL,WAAW,EACX,cAAc,GAGf,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,kBAAkB,EAClB,iBAAiB,GAElB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,aAAa,EACb,oBAAoB,EACpB,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AA8CzD;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,GAAG;IACH,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,QAAQ;IACR,QAAQ;CACT,CAAC;AAEF;;;;GAIG;AACH,SAAS,mBAAmB,CAC1B,WAAkC,EAClC,aAAiC;IAEjC,MAAM,QAAQ,GAAsB;QAClC,sBAAsB,EAAE,KAAK;QAC7B,iBAAiB,EAAE,IAAI;QACvB,gBAAgB,EAAE,IAAI;QACtB,iBAAiB,EAAE,CAAC;QACpB,iBAAiB,EAAE,CAAC;KACrB,CAAC;IAEF,+CAA+C;IAC/C,IAAI,WAAW,KAAK,KAAK,EAAE,CAAC;QAC1B,wDAAwD;QACxD,QAAQ,CAAC,kBAAkB,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACnD,CAAC;IACD,wEAAwE;IAExE,+DAA+D;IAC/D,OAAO,EAAE,GAAG,QAAQ,EAAE,GAAG,aAAa,EAAE,CAAC;AAC3C,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,QAAQ;IACX,GAAG,GAAsB,IAAI,CAAC;IAC9B,OAAO,GAAY,IAAI,CAAC;IACxB,SAAS,GAA8B,IAAI,CAAC;IAC5C,MAAM,CAAiB;IACvB,QAAQ,GAAG,KAAK,CAAC;IAEzB,YAAY,MAAsB;QAChC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAE1B,gEAAgE;QAChE,IAAI,CAAC,GAAG,GAAG,MAAM,WAAW,EAAE,CAAC;QAE/B,mEAAmE;QACnE,MAAM,WAAW,GAAG,cAAc,EAAE,CAAC;QACrC,MAAM,cAAc,GAAG,mBAAmB,CACxC,WAAW,EACX,IAAI,CAAC,MAAM,CAAC,cAAc,CAC3B,CAAC;QAEF,kBAAkB;QAClB,oFAAoF;QACpF,4FAA4F;QAC5F,IAAI,SAAS,EAAE,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,MAAM,kBAAkB,EAAE,CAAC;YAC3C,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAChE,8CAA8C;YAC9C,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CACnD,SAAS,EACT,cAAc,CACf,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,gDAAgD;YAChD,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CACnD,IAAI,CAAC,MAAM,CAAC,SAAS,EACrB,cAAc,CACf,CAAC;QACJ,CAAC;QAED,0EAA0E;QAC1E,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,SAAS,GAAG,IAAI,kBAAkB,CAAC,KAAK,EAAE;YAC7C,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;YAChC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;SACrC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACvB,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,UAAU,CACtB,SAAiB,EACjB,YAAoB,EACpB,aAAqB;QAErB,sEAAsE;QACtE,qEAAqE;QACrE,MAAM,OAAO,GAAG,IAAI,CAAC,SAAU,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;QAE3D,MAAM,KAAK,GAAgB,EAAE,CAAC;QAC9B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,gBAAgB;YAChB,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YAEhE,iEAAiE;YACjE,oEAAoE;YACpE,MAAM,WAAW,GAAG,aAAa,CAC/B,MAAM,CAAC,MAAM,EACb,MAAM,EACN,WAAW,EACX,YAAY,CACb,CAAC;YAEF,+DAA+D;YAC/D,kEAAkE;YAClE,oEAAoE;YACpE,iEAAiE;YACjE,mCAAmC;YACnC,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CACjC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC,KAAK,GAAG,MAAM,CAAC,WAAW,CACpE,CAAC;YAEF,wDAAwD;YACxD,KAAK,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC;QAC/D,CAAC;QAED,oEAAoE;QACpE,oEAAoE;QACpE,gCAAgC;QAChC,OAAO,2BAA2B,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CACX,IAAY,EACZ,MAA4B;QAE5B,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAEpC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAEpD,oCAAoC;QACpC,IAAI,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,aAAa,CAAC,CAAC;QAE7D,gFAAgF;QAChF,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,KAAK,CAAC;QACvD,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;gBACxB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,IAAI,IAAI,CAAC;gBACxD,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,IAAI,EAAE,aAAa,CAAC,CAAC;gBAE7E,2EAA2E;gBAC3E,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;oBACrC,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CACzB,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC,GAAG,IAAI,QAAQ,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAC1E,CAAC;oBACF,IAAI,CAAC,QAAQ,EAAE,CAAC;wBACd,KAAK,CAAC,IAAI,CAAC;4BACT,GAAG,QAAQ;4BACX,UAAU,EAAE,QAAQ,CAAC,UAAU,GAAG,OAAO;yBAC1C,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,KAAK,GAAG,qBAAqB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC3C,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAExC,oCAAoC;QACpC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,KAAK,GAAG,KAAK,CAAC,MAAM,CAClB,CAAC,IAAI,EAAE,EAAE,CACP,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;gBAClC,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CACxC,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAElC,OAAO;YACL,KAAK;YACL,gBAAgB,EAAE,OAAO,GAAG,SAAS;YACrC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY;SACvC,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY,CACxB,YAAgC;QAEhC,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,IAAI,CAAC,GAAG,KAAK,IAAI,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAMpB,CAAC;QAEF,MAAM,SAAS,GAAG,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC;QAE/C,iBAAiB;QACjB,MAAM,cAAc,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CACxC,OAAO,EACP,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EACrD,CAAC,CAAC,EAAE,SAAS,CAAC,CACf,CAAC;QAEF,MAAM,mBAAmB,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAC7C,OAAO,EACP,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAC1D,CAAC,CAAC,EAAE,SAAS,CAAC,CACf,CAAC;QAEF,MAAM,kBAAkB,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAC5C,OAAO,EACP,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EACzD,CAAC,CAAC,EAAE,SAAS,CAAC,CACf,CAAC;QAEF,gBAAgB;QAChB,MAAM,KAAK,GAA4B;YACrC,SAAS,EAAE,cAAc;YACzB,cAAc,EAAE,mBAAmB;SACpC,CAAC;QAEF,uCAAuC;QACvC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACtC,IAAI,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAC1C,KAAK,CAAC,gBAAgB,CAAC,GAAG,kBAAkB,CAAC;QAC/C,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEzC,oBAAoB;QACpB,MAAM,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QACnC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QAED,+CAA+C;QAC/C,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACK,aAAa,CACnB,MAA8B,EAC9B,SAAiB;QAEjB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QAE9C,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,4BAA4B;YAC5B,MAAM,WAAW,GAAa,EAAE,CAAC;YACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YACjD,CAAC;YAED,gBAAgB;YAChB,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;YAEnC,aAAa;YACb,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,IAAI,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC;oBAC9B,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;oBACxB,MAAM,GAAG,CAAC,CAAC;gBACb,CAAC;YACH,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;YACjD,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC5B,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IACjC,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,MAA4B;QACnD,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,GAAG,CAAC;QAErC,yCAAyC;QACzC,IAAI,YAAY,GAAG,GAAG,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;YAC1C,MAAM,SAAS,GAAG,MAAM,CAAC,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC;YAC/D,IAAI,SAAS,GAAG,YAAY,EAAE,CAAC;gBAC7B,YAAY,GAAG,SAAS,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAED;;OAEG;IACH,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,OAAO;QACL,+EAA+E;QAC/E,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;CACF;AAED;;;;;GAKG;AACH,SAAS,2BAA2B,CAClC,KAAkB,EAClB,YAAoB;IAEpB,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAEpC,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7E,MAAM,MAAM,GAAgB,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC,CAAE,EAAE,CAAC,CAAC;IAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;QACxB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;QAExC,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACrD,IAAI,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACxB,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;gBACpB,IAAI,CAAC,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YACvD,CAAC;YACD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/D,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAC;IACtC,WAAW;IACX,GAAG,EAAE,IAAI,EAAE,KAAK;IAChB,WAAW;IACX,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ;IACjC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU;IAClC,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS;IAC7B,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS;IAC/B,IAAI,EAAE,KAAK,EAAE,QAAQ;IACrB,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW;IACtC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY;IAC/C,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;IAChC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM;IACvC,eAAe;IACf,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM;IACzD,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO;IAC7D,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK;IAC1D,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ;IAC1D,eAAe;IACf,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ;IACxD,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM;IACvD,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO;IAC3D,6BAA6B;IAC7B,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO;IACvD,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ;IAC9B,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO;IAC5B,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ;IAClC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM;IACtC,MAAM,EAAE,MAAM,EAAE,OAAO;IACvB,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS;IAC/B,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM;IAC5B,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;IACrC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ;IACjC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ;IACjC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ;IAC1C,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ;IAC1C,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS;IAClC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS;IAC3C,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU;IACxC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ;IACtC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS;IACpC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS;IACpC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO;IAC9B,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS;IACnC,KAAK,EAAE,MAAM,EAAE,SAAS;IACxB,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ;IACjC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ;IAChC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS;IACpC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS;IACpC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS;IAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS;IACpC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS;IACpC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS;IAClC,KAAK,EAAE,MAAM,EAAE,SAAS;IACxB,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS;IAC7C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS;IACpC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ;IAClC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS;IACpC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS;IAC/B,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS;IAClC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU;IACxC,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW;IAC5C,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS;IAChD,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS;IAC/B,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU;IACtC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ;IACjC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ;IAC/B,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS;IACjC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS;IAClC,MAAM,EAAE,OAAO,EAAE,SAAS;IAC1B,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU;IACtC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS;IAC3C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS;IACpC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS;IACpC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS;IAC/B,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU;IACxC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ;IACjC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU;IAChD,iBAAiB;IACjB,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS;IAC1D,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM;IAC3D,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK;IAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO;IACvD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ;IAClD,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ;IACxC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO;IAC1D,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO;IAC/C,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU;IACnD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK;IAChD,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK;IACvD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM;IACrD,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IACxC,2BAA2B;IAC3B,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS;IAClD,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO;IACzD,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;IAC9B,qDAAqD;IACrD,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO;IAC3C,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IACrC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM;IAC5C,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ;IACjC,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ;IACnC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ;IACpD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ;IAClC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK;IAC5C,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;IACvC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO;IAC3C,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK;IACtD,KAAK,EAAE,KAAK,EAAE,KAAK;IACnB,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO;IACrD,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI;IACpD,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI;IAC3C,IAAI,EAAE,MAAM,EAAE,MAAM;IACpB,mBAAmB;IACnB,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK;IACrC,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK;IACjC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO;IACxD,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM;IAC7C,IAAI,EAAE,OAAO,EAAE,KAAK;IACpB,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ;IACrC,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK;IACjC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO;IACrD,IAAI,EAAE,KAAK,EAAE,IAAI;IACjB,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;IACrC,oBAAoB;IACpB,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM;IAChD,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK;IAClD,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK;IAChC,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO;IAC7C,KAAK,EAAE,IAAI;IACX,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO;IACjC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ;CAC3C,CAAC,CAAC;AAEH;;;;;GAKG;AACH,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,IAAI,EAAE,EAAE;QAC/C,IAAI,uBAAuB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACtC,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,OAAO,CAAC,MAAgB;IAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC;IAC5D,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACpD,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAC5B,MAA0E;IAE1E,MAAM,UAAU,GAAmB;QACjC,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,iBAAiB;QAC9C,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,GAAG;QAClC,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,KAAK,EAAE,uBAAuB;QACjE,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,OAAO;QAC5C,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;KAChD,CAAC;IAEF,OAAO,IAAI,QAAQ,CAAC,UAAU,CAAC,CAAC;AAClC,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,YAAY;IACd,OAAO,GAAG,YAAY,CAAC;IACvB,MAAM,GAAG,IAAI,CAAC;IAEvB,KAAK,CAAC,IAAI;QACR,QAAQ;IACV,CAAC;IAED,OAAO,CACL,KAAa,EACb,OAA6B;QAE7B,OAAO,OAAO,CAAC,OAAO,CAAC;YACrB,KAAK,EAAE,EAAE;YACT,gBAAgB,EAAE,CAAC;YACnB,YAAY,EAAE,IAAI,CAAC,OAAO;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,QAAQ;QACR,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB;IAChC,OAAO,IAAI,YAAY,EAAE,CAAC;AAC5B,CAAC"}
+{"version":3,"file":"ner-model.js","sourceRoot":"","sources":["../../src/ner/ner-model.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EACL,WAAW,EACX,cAAc,GAGf,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,kBAAkB,EAClB,iBAAiB,GAElB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,aAAa,EACb,oBAAoB,EACpB,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,kBAAkB,EAClB,gBAAgB,EAChB,oBAAoB,EACpB,iBAAiB,GAClB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAgFzD;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,GAAG;IACH,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,OAAO;IACP,QAAQ;IACR,QAAQ;CACT,CAAC;AAEF;;;;GAIG;AACH,SAAS,mBAAmB,CAC1B,WAAkC,EAClC,aAAiC;IAEjC,MAAM,QAAQ,GAAsB;QAClC,sBAAsB,EAAE,KAAK;QAC7B,iBAAiB,EAAE,IAAI;QACvB,gBAAgB,EAAE,IAAI;QACtB,iBAAiB,EAAE,CAAC;QACpB,iBAAiB,EAAE,CAAC;KACrB,CAAC;IAEF,+CAA+C;IAC/C,IAAI,WAAW,KAAK,KAAK,EAAE,CAAC;QAC1B,wDAAwD;QACxD,QAAQ,CAAC,kBAAkB,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACnD,CAAC;IACD,wEAAwE;IAExE,+DAA+D;IAC/D,OAAO,EAAE,GAAG,QAAQ,EAAE,GAAG,aAAa,EAAE,CAAC;AAC3C,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,QAAQ;IACX,GAAG,GAAsB,IAAI,CAAC;IAC9B,OAAO,GAAY,IAAI,CAAC;IACxB,SAAS,GAA8B,IAAI,CAAC;IAC5C,MAAM,CAAiB;IACvB,QAAQ,GAAG,KAAK,CAAC;IAEzB,YAAY,MAAsB;QAChC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAE1B,gEAAgE;QAChE,IAAI,CAAC,GAAG,GAAG,MAAM,WAAW,EAAE,CAAC;QAE/B,mEAAmE;QACnE,MAAM,WAAW,GAAG,cAAc,EAAE,CAAC;QACrC,MAAM,cAAc,GAAG,mBAAmB,CACxC,WAAW,EACX,IAAI,CAAC,MAAM,CAAC,cAAc,CAC3B,CAAC;QAEF,kBAAkB;QAClB,oFAAoF;QACpF,4FAA4F;QAC5F,IAAI,SAAS,EAAE,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,MAAM,kBAAkB,EAAE,CAAC;YAC3C,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAChE,8CAA8C;YAC9C,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CACnD,SAAS,EACT,cAAc,CACf,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,gDAAgD;YAChD,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CACnD,IAAI,CAAC,MAAM,CAAC,SAAS,EACrB,cAAc,CACf,CAAC;QACJ,CAAC;QAED,0EAA0E;QAC1E,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC7D,IAAI,CAAC,SAAS,GAAG,IAAI,kBAAkB,CAAC,KAAK,EAAE;YAC7C,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;YAChC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;SACrC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACvB,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,UAAU,CACtB,SAAiB,EACjB,YAAoB,EACpB,aAAqB;QAErB,iEAAiE;QACjE,sEAAsE;QACtE,qEAAqE;QACrE,uEAAuE;QACvE,wEAAwE;QACxE,MAAM,SAAS,GACb,IAAI,CAAC,MAAM,CAAC,kBAAkB,KAAK,SAAS;YAC1C,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,iBAAiB,CACf,IAAI,CAAC,MAAM,CAAC,SAAS,EACrB,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAC/B,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAC/B,CAAC;QACR,MAAM,YAAY,GAAG,IAAI,CAAC,SAAU,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;QACzF,MAAM,OAAO,GAAG,kBAAkB,CAChC,YAAY,EACZ,IAAI,CAAC,MAAM,CAAC,SAAS,EACrB,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAChC,CAAC;QACF,IAAI,YAAY,CAAC,SAAS,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,EAAE,cAAc,EAAE,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC;YAC/E,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;gBAC5B,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YAChC,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CACV,wBAAwB,IAAI,CAAC,WAAW,8DAA8D,IAAI,CAAC,cAAc,+BAA+B,CACzJ,CAAC;YACJ,CAAC;QACH,CAAC;QACD,MAAM,KAAK,GAAgB,EAAE,CAAC;QAE9B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YAChE,MAAM,WAAW,GAAG,aAAa,CAC/B,MAAM,CAAC,MAAM,EACb,MAAM,EACN,WAAW,EACX,YAAY,CACb,CAAC;YACF,KAAK,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC;QAClE,CAAC;QAED,uEAAuE;QACvE,2EAA2E;QAC3E,2EAA2E;QAC3E,4EAA4E;QAC5E,MAAM,gBAAgB,GAAG,YAAY,CAAC,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC7E,MAAM,eAAe,GAAG,gBAAgB,KAAK,SAAS,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC;QACtF,MAAM,aAAa,GAAG,eAAe;YACnC,CAAC,CAAC,gBAAgB,CAAC,GAAG;YACtB,CAAC,CAAC,YAAY,CAAC,SAAS;gBACtB,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC;QAEvB,OAAO;YACL,KAAK,EAAE,gBAAgB,CAAC,KAAK,EAAE,YAAY,CAAC;YAC5C,SAAS,EAAE,YAAY,CAAC,SAAS;YACjC,aAAa;SACd,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CACX,IAAY,EACZ,MAA4B;QAE5B,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAEpC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,IAAI,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAEpD,oCAAoC;QACpC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,aAAa,CAAC,CAAC;QACjE,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC1B,IAAI,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QAClC,sEAAsE;QACtE,0EAA0E;QAC1E,4EAA4E;QAC5E,IAAI,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAE1C,gFAAgF;QAChF,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,KAAK,CAAC;QACvD,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;gBACxB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,IAAI,IAAI,CAAC;gBACxD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,IAAI,EAAE,aAAa,CAAC,CAAC;gBACxE,SAAS,GAAG,SAAS,IAAI,QAAQ,CAAC,SAAS,CAAC;gBAC5C,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAC;gBAEhE,2EAA2E;gBAC3E,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;oBAClC,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CACzB,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,GAAG,CACrE,CAAC;oBACF,IAAI,CAAC,QAAQ,EAAE,CAAC;wBACd,KAAK,CAAC,IAAI,CAAC;4BACT,GAAG,IAAI;4BACP,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,OAAO;yBACtC,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,KAAK,GAAG,qBAAqB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC3C,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAExC,oCAAoC;QACpC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,KAAK,GAAG,KAAK,CAAC,MAAM,CAClB,CAAC,IAAI,EAAE,EAAE,CACP,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;gBAClC,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CACxC,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAElC,OAAO;YACL,KAAK;YACL,gBAAgB,EAAE,OAAO,GAAG,SAAS;YACrC,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY;YACtC,SAAS;YACT,aAAa;SACd,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY,CACxB,YAAgC;QAEhC,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,IAAI,CAAC,GAAG,KAAK,IAAI,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAMpB,CAAC;QAEF,MAAM,SAAS,GAAG,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC;QAE/C,iBAAiB;QACjB,MAAM,cAAc,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CACxC,OAAO,EACP,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EACrD,CAAC,CAAC,EAAE,SAAS,CAAC,CACf,CAAC;QAEF,MAAM,mBAAmB,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAC7C,OAAO,EACP,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAC1D,CAAC,CAAC,EAAE,SAAS,CAAC,CACf,CAAC;QAEF,MAAM,kBAAkB,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAC5C,OAAO,EACP,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EACzD,CAAC,CAAC,EAAE,SAAS,CAAC,CACf,CAAC;QAEF,gBAAgB;QAChB,MAAM,KAAK,GAA4B;YACrC,SAAS,EAAE,cAAc;YACzB,cAAc,EAAE,mBAAmB;SACpC,CAAC;QAEF,uCAAuC;QACvC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACtC,IAAI,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAC1C,KAAK,CAAC,gBAAgB,CAAC,GAAG,kBAAkB,CAAC;QAC/C,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEzC,oBAAoB;QACpB,MAAM,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QACnC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QAED,+CAA+C;QAC/C,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACK,aAAa,CACnB,MAA8B,EAC9B,SAAiB;QAEjB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QAE9C,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,4BAA4B;YAC5B,MAAM,WAAW,GAAa,EAAE,CAAC;YACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YACjD,CAAC;YAED,gBAAgB;YAChB,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;YAEnC,aAAa;YACb,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,IAAI,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC;oBAC9B,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;oBACxB,MAAM,GAAG,CAAC,CAAC;gBACb,CAAC;YACH,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;YACjD,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC5B,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IACjC,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,MAA4B;QACnD,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,GAAG,CAAC;QAErC,yCAAyC;QACzC,IAAI,YAAY,GAAG,GAAG,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;YAC1C,MAAM,SAAS,GAAG,MAAM,CAAC,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC;YAC/D,IAAI,SAAS,GAAG,YAAY,EAAE,CAAC;gBAC7B,YAAY,GAAG,SAAS,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAED;;OAEG;IACH,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,OAAO;QACL,+EAA+E;QAC/E,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAC;IACtC,WAAW;IACX,GAAG,EAAE,IAAI,EAAE,KAAK;IAChB,WAAW;IACX,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ;IACjC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU;IAClC,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS;IAC7B,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS;IAC/B,IAAI,EAAE,KAAK,EAAE,QAAQ;IACrB,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW;IACtC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY;IAC/C,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;IAChC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM;IACvC,eAAe;IACf,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM;IACzD,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO;IAC7D,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK;IAC1D,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ;IAC1D,eAAe;IACf,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ;IACxD,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM;IACvD,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO;IAC3D,6BAA6B;IAC7B,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO;IACvD,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ;IAC9B,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO;IAC5B,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ;IAClC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM;IACtC,MAAM,EAAE,MAAM,EAAE,OAAO;IACvB,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS;IAC/B,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM;IAC5B,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;IACrC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ;IACjC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ;IACjC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ;IAC1C,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ;IAC1C,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS;IAClC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS;IAC3C,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU;IACxC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ;IACtC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS;IACpC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS;IACpC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO;IAC9B,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS;IACnC,KAAK,EAAE,MAAM,EAAE,SAAS;IACxB,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ;IACjC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ;IAChC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS;IACpC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS;IACpC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS;IAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS;IACpC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS;IACpC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS;IAClC,KAAK,EAAE,MAAM,EAAE,SAAS;IACxB,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS;IAC7C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS;IACpC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ;IAClC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS;IACpC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS;IAC/B,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS;IAClC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU;IACxC,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW;IAC5C,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS;IAChD,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS;IAC/B,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU;IACtC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ;IACjC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ;IAC/B,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS;IACjC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS;IAClC,MAAM,EAAE,OAAO,EAAE,SAAS;IAC1B,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU;IACtC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS;IAC3C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS;IACpC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS;IACpC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS;IAC/B,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU;IACxC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ;IACjC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU;IAChD,iBAAiB;IACjB,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS;IAC1D,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM;IAC3D,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK;IAC3D,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO;IACvD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ;IAClD,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ;IACxC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO;IAC1D,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO;IAC/C,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU;IACnD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK;IAChD,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK;IACvD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM;IACrD,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IACxC,2BAA2B;IAC3B,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS;IAClD,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO;IACzD,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;IAC9B,qDAAqD;IACrD,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO;IAC3C,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IACrC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM;IAC5C,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ;IACjC,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ;IACnC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ;IACpD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ;IAClC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK;IAC5C,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;IACvC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO;IAC3C,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK;IACtD,KAAK,EAAE,KAAK,EAAE,KAAK;IACnB,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO;IACrD,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI;IACpD,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI;IAC3C,IAAI,EAAE,MAAM,EAAE,MAAM;IACpB,mBAAmB;IACnB,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK;IACrC,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK;IACjC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO;IACxD,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM;IAC7C,IAAI,EAAE,OAAO,EAAE,KAAK;IACpB,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ;IACrC,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK;IACjC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO;IACrD,IAAI,EAAE,KAAK,EAAE,IAAI;IACjB,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;IACrC,oBAAoB;IACpB,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM;IAChD,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK;IAClD,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK;IAChC,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO;IAC7C,KAAK,EAAE,IAAI;IACX,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO;IACjC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ;CAC3C,CAAC,CAAC;AAEH;;;;;GAKG;AACH,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,IAAI,EAAE,EAAE;QAC/C,IAAI,uBAAuB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACtC,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,OAAO,CAAC,MAAgB;IAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC;IAC5D,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACpD,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAC5B,MAA0E;IAE1E,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,GAAG,CAAC;IAC1C,MAAM,UAAU,GAAmB;QACjC,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,iBAAiB;QAC9C,SAAS;QACT,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;eAC1C,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC/D,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;QAC7C,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,KAAK,EAAE,uBAAuB;QACjE,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,OAAO;QAC5C,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;KAChD,CAAC;IAEF,uEAAuE;IACvE,oBAAoB,CAClB,UAAU,CAAC,SAAS,EACpB,UAAU,CAAC,mBAAmB,EAC9B,UAAU,CAAC,kBAAkB,CAC9B,CAAC;IAEF,OAAO,IAAI,QAAQ,CAAC,UAAU,CAAC,CAAC;AAClC,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,YAAY;IACd,OAAO,GAAG,YAAY,CAAC;IACvB,MAAM,GAAG,IAAI,CAAC;IAEvB,KAAK,CAAC,IAAI;QACR,QAAQ;IACV,CAAC;IAED,OAAO,CACL,KAAa,EACb,OAA6B;QAE7B,OAAO,OAAO,CAAC,OAAO,CAAC;YACrB,KAAK,EAAE,EAAE;YACT,gBAAgB,EAAE,CAAC;YACnB,YAAY,EAAE,IAAI,CAAC,OAAO;YAC1B,SAAS,EAAE,KAAK;YAChB,aAAa,EAAE,KAAK,CAAC,MAAM;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,QAAQ;QACR,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB;IAChC,OAAO,IAAI,YAAY,EAAE,CAAC;AAC5B,CAAC"}

package/dist/ner/token-window.d.ts

@@ -0,0 +1,19 @@
+import type { SpanMatch } from "../types/index.js";
+import type { TokenizationResult } from "./tokenizer.js";
+/**
+ * Total token count (including the two special tokens) that `maxWindows`
+ * windows can consume. Used to bound tokenization up front so huge inputs
+ * are not fully tokenized just to discard everything past the cap.
+ */
+export declare function windowTokenBudget(maxLength: number, overlapTokens: number, maxWindows: number): number;
+export declare function validateWindowConfig(maxLength: number, overlapTokens: number, maxWindows?: number): void;
+/**
+ * Splits an already-tokenized input into overlapping model-sized windows.
+ * The window count is bounded by the caller pre-bounding tokenization via
+ * `tokenize`'s `maxTokens` (see `runNERPass`) — this just slices whatever it
+ * is handed, so a truncated tokenization yields a truncated set of windows,
+ * and the truncation signal lives on the parent `TokenizationResult`.
+ */
+export declare function createTokenWindows(tokenization: TokenizationResult, maxLength: number, overlapTokens: number): TokenizationResult[];
+export declare function mergeWindowSpans(spans: SpanMatch[], originalText: string): SpanMatch[];
+//# sourceMappingURL=token-window.d.ts.map

package/dist/ner/token-window.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-window.d.ts","sourceRoot":"","sources":["../../src/ner/token-window.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,KAAK,EAAS,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAiBhE;;;;GAIG;AACH,wBAAgB,iBAAiB,CAC/B,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,UAAU,EAAE,MAAM,GACjB,MAAM,CAKR;AAED,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,UAAU,CAAC,EAAE,MAAM,GAClB,IAAI,CAUN;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAChC,YAAY,EAAE,kBAAkB,EAChC,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,GACpB,kBAAkB,EAAE,CA+BtB;AAED,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,SAAS,EAAE,EAClB,YAAY,EAAE,MAAM,GACnB,SAAS,EAAE,CAwCb"}

package/dist/ner/token-window.js

@@ -0,0 +1,101 @@
+function buildTokenization(tokens) {
+    return {
+        tokens,
+        inputIds: tokens.map((token) => token.id),
+        attentionMask: tokens.map(() => 1),
+        tokenTypeIds: tokens.map(() => 0),
+        tokenToCharSpan: tokens.map((token) => token.isSpecial ? null : [token.start, token.end]),
+        // A window is a complete slice of an already-tokenized input; the
+        // truncated signal lives on the parent tokenization, not each window.
+        truncated: false,
+    };
+}
+/**
+ * Total token count (including the two special tokens) that `maxWindows`
+ * windows can consume. Used to bound tokenization up front so huge inputs
+ * are not fully tokenized just to discard everything past the cap.
+ */
+export function windowTokenBudget(maxLength, overlapTokens, maxWindows) {
+    validateWindowConfig(maxLength, overlapTokens, maxWindows);
+    const contentCapacity = maxLength - 2;
+    const stride = contentCapacity - overlapTokens;
+    return 2 + contentCapacity + (maxWindows - 1) * stride;
+}
+export function validateWindowConfig(maxLength, overlapTokens, maxWindows) {
+    if (maxLength - 2 < 1) {
+        throw new Error("NER maxLength must leave room for content and two special tokens");
+    }
+    if (overlapTokens < 0 || overlapTokens >= maxLength - 2) {
+        throw new Error("NER window overlap must be smaller than the content capacity");
+    }
+    if (maxWindows !== undefined && maxWindows < 1) {
+        throw new Error("NER maxWindowsPerInput must be at least 1");
+    }
+}
+/**
+ * Splits an already-tokenized input into overlapping model-sized windows.
+ * The window count is bounded by the caller pre-bounding tokenization via
+ * `tokenize`'s `maxTokens` (see `runNERPass`) — this just slices whatever it
+ * is handed, so a truncated tokenization yields a truncated set of windows,
+ * and the truncation signal lives on the parent `TokenizationResult`.
+ */
+export function createTokenWindows(tokenization, maxLength, overlapTokens) {
+    validateWindowConfig(maxLength, overlapTokens);
+    const contentCapacity = maxLength - 2;
+    if (tokenization.tokens.length <= maxLength) {
+        return [tokenization];
+    }
+    const firstToken = tokenization.tokens[0];
+    const lastToken = tokenization.tokens[tokenization.tokens.length - 1];
+    if (firstToken === undefined ||
+        firstToken.isSpecial !== true ||
+        lastToken === undefined ||
+        lastToken.isSpecial !== true) {
+        throw new Error("NER tokenization must include leading and trailing special tokens");
+    }
+    const contentTokens = tokenization.tokens.slice(1, -1);
+    const stride = contentCapacity - overlapTokens;
+    const windows = [];
+    for (let start = 0; start < contentTokens.length; start += stride) {
+        const end = Math.min(start + contentCapacity, contentTokens.length);
+        windows.push(buildTokenization([firstToken, ...contentTokens.slice(start, end), lastToken]));
+        if (end === contentTokens.length)
+            break;
+    }
+    return windows;
+}
+export function mergeWindowSpans(spans, originalText) {
+    const sorted = [...spans].sort((a, b) => a.start - b.start || b.end - a.end || b.confidence - a.confidence);
+    const merged = [];
+    for (const span of sorted) {
+        let combined = span;
+        for (let index = merged.length - 1; index >= 0; index--) {
+            const existing = merged[index];
+            // Only stitch spans that actually overlap (touching end-to-start is not
+            // an overlap). This relies on the window overlap being at least as long
+            // as the longest expected entity — otherwise an entity split across a
+            // window boundary produces two abutting fragments that never merge.
+            // The default overlap (64 tokens) is far longer than any name/entity.
+            if (existing.type !== combined.type ||
+                existing.start >= combined.end ||
+                combined.start >= existing.end) {
+                continue;
+            }
+            const preferred = combined.confidence > existing.confidence
+                ? combined
+                : existing;
+            const start = Math.min(existing.start, combined.start);
+            const end = Math.max(existing.end, combined.end);
+            combined = {
+                ...preferred,
+                start,
+                end,
+                text: originalText.slice(start, end),
+            };
+            merged.splice(index, 1);
+        }
+        merged.push(combined);
+    }
+    return merged.sort((a, b) => a.start - b.start || b.end - a.end);
+}
+//# sourceMappingURL=token-window.js.map

package/dist/ner/token-window.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-window.js","sourceRoot":"","sources":["../../src/ner/token-window.ts"],"names":[],"mappings":"AAGA,SAAS,iBAAiB,CAAC,MAAe;IACxC,OAAO;QACL,MAAM;QACN,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;QACzC,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAClC,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACjC,eAAe,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACpC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAClD;QACD,kEAAkE;QAClE,sEAAsE;QACtE,SAAS,EAAE,KAAK;KACjB,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAC/B,SAAiB,EACjB,aAAqB,EACrB,UAAkB;IAElB,oBAAoB,CAAC,SAAS,EAAE,aAAa,EAAE,UAAU,CAAC,CAAC;IAC3D,MAAM,eAAe,GAAG,SAAS,GAAG,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,eAAe,GAAG,aAAa,CAAC;IAC/C,OAAO,CAAC,GAAG,eAAe,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,SAAiB,EACjB,aAAqB,EACrB,UAAmB;IAEnB,IAAI,SAAS,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IACtF,CAAC;IACD,IAAI,aAAa,GAAG,CAAC,IAAI,aAAa,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAClF,CAAC;IACD,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAChC,YAAgC,EAChC,SAAiB,EACjB,aAAqB;IAErB,oBAAoB,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IAC/C,MAAM,eAAe,GAAG,SAAS,GAAG,CAAC,CAAC;IACtC,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC5C,OAAO,CAAC,YAAY,CAAC,CAAC;IACxB,CAAC;IAED,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACtE,IACE,UAAU,KAAK,SAAS;QACxB,UAAU,CAAC,SAAS,KAAK,IAAI;QAC7B,SAAS,KAAK,SAAS;QACvB,SAAS,CAAC,SAAS,KAAK,IAAI,EAC5B,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;IACvF,CAAC;IAED,MAAM,aAAa,GAAG,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,eAAe,GAAG,aAAa,CAAC;IAC/C,MAAM,OAAO,GAAyB,EAAE,CAAC;IAEzC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,aAAa,CAAC,MAAM,EAAE,KAAK,IAAI,MAAM,EAAE,CAAC;QAClE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,eAAe,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;QACpE,OAAO,CAAC,IAAI,CACV,iBAAiB,CAAC,CAAC,UAAU,EAAE,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,SAAS,CAAC,CAAC,CAC/E,CAAC;QACF,IAAI,GAAG,KAAK,aAAa,CAAC,MAAM;YAAE,MAAM;IAC1C,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,KAAkB,EAClB,YAAoB;IAEpB,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACtC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAClE,CAAC;IACF,MAAM,MAAM,GAAgB,EAAE,CAAC;IAE/B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,IAAI,QAAQ,GAAG,IAAI,CAAC;QACpB,KAAK,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;YACxD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAE,CAAC;YAChC,wEAAwE;YACxE,wEAAwE;YACxE,sEAAsE;YACtE,oEAAoE;YACpE,sEAAsE;YACtE,IACE,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI;gBAC/B,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,GAAG;gBAC9B,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,GAAG,EAC9B,CAAC;gBACD,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,QAAQ,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU;gBACzD,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,QAAQ,CAAC;YACb,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;YACvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;YACjD,QAAQ,GAAG;gBACT,GAAG,SAAS;gBACZ,KAAK;gBACL,GAAG;gBACH,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC;aACrC,CAAC;YACF,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC1B,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACxB,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;AACnE,CAAC"}

package/dist/ner/tokenizer.d.ts

@@ -34,27 +34,13 @@ export interface TokenizationResult {
     tokenTypeIds: number[];
     /** Mapping from token index to character span [start, end] */
     tokenToCharSpan: Array<[number, number] | null>;
+    /**
+     * True when tokenization stopped before consuming all input — either
+     * because `truncate` capped it at maxLength or because `maxTokens` bounded
+     * it. Lets callers know detection coverage is partial.
+     */
+    truncated: boolean;
 }
-/**
- * A model-sized window over the full token stream of a long input.
- * Consecutive windows overlap so entities near a window boundary are seen
- * with full context by at least one window. The core range delimits the
- * characters this window is authoritative for: cores tile the input exactly,
- * so an entity anchored (by start offset) in a core belongs to exactly one
- * window and cross-window duplicates cannot occur.
- */
-export interface TokenizationWindow extends TokenizationResult {
-    /** Start character offset (inclusive) of this window's core range */
-    coreCharStart: number;
-    /** End character offset (exclusive) of this window's core range */
-    coreCharEnd: number;
-}
-/**
- * Default token overlap between consecutive windows.
- * Entities up to half this length are guaranteed to fit entirely inside
- * the window that owns them.
- */
-export declare const DEFAULT_WINDOW_OVERLAP = 128;
 /**
  * Tokenizer configuration
  */
@@ -64,6 +50,18 @@ export interface TokenizerConfig {
     /** Whether to lowercase input */
     doLowerCase: boolean;
 }
+export interface TokenizeOptions {
+    /** Whether to truncate to the configured maximum sequence length. */
+    truncate?: boolean;
+    /**
+     * Stop after producing this many tokens (including the CLS/SEP special
+     * tokens). Bounds memory and CPU on very large inputs: the caller only
+     * needs enough tokens to build its windows, so there is no point
+     * materializing a multi-million-token array for a multi-MB value.
+     * Takes precedence over `truncate` when set.
+     */
+    maxTokens?: number;
+}
 /**
  * Default tokenizer configuration
  */
@@ -95,28 +93,9 @@ export declare class WordPieceTokenizer {
      */
     private detectSpecialTokens;
     /**
-     * Tokenizes text into tokens with offset tracking.
-     * Truncates to maxLength tokens — use tokenizeWindows() for inputs that
-     * may exceed the model's sequence length.
-     */
-    tokenize(text: string): TokenizationResult;
-    /**
-     * Tokenizes text into overlapping model-sized windows covering the full
-     * input, so no tokens are silently dropped for long texts.
-     * Each window carries a core character range; together the cores tile the
-     * input exactly. Callers should keep an entity only from the window whose
-     * core contains the entity's start offset.
-     */
-    tokenizeWindows(text: string, overlap?: number): TokenizationWindow[];
-    /**
-     * Tokenizes text into content tokens (no special tokens, no truncation)
-     * using greedy longest-match with character offset tracking
-     */
-    private tokenizeContent;
-    /**
-     * Wraps content tokens with CLS/SEP and builds the model input arrays
+     * Tokenizes text into tokens with offset tracking
      */
-    private buildResult;
+    tokenize(text: string, options?: TokenizeOptions): TokenizationResult;
     /**
      * Find the best matching token using trie-based greedy longest-match
      * O(max_token_length) instead of O(vocab_size)

package/dist/ner/tokenizer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../src/ner/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;GAEG;AACH,MAAM,WAAW,KAAK;IACpB,6BAA6B;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,mBAAmB;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,8CAA8C;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,4CAA4C;IAC5C,GAAG,EAAE,MAAM,CAAC;IACZ,2CAA2C;IAC3C,cAAc,EAAE,OAAO,CAAC;IACxB,sCAAsC;IACtC,SAAS,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,sBAAsB;IACtB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,qBAAqB;IACrB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,6CAA6C;IAC7C,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8DAA8D;IAC9D,eAAe,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;CACjD;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,kBAAmB,SAAQ,kBAAkB;IAC5D,qEAAqE;IACrE,aAAa,EAAE,MAAM,CAAC;IACtB,mEAAmE;IACnE,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;;;GAIG;AACH,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAsB1C;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,iCAAiC;IACjC,WAAW,EAAE,OAAO,CAAC;CACtB;AAED;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,eAGtC,CAAC;AAkBF;;;GAGG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,YAAY,CAAsB;IAC1C,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,IAAI,CAAW;IAGvB,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,KAAK,CAAa;IAG1B,OAAO,CAAC,QAAQ,CAAiB;IACjC,OAAO,CAAC,QAAQ,CAAkB;IAClC,OAAO,CAAC,QAAQ,CAAmB;IACnC,OAAO,CAAC,QAAQ,CAAmB;gBAEvB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAE,OAAO,CAAC,eAAe,CAAM;IAsB7E;;OAEG;IACH,OAAO,CAAC,cAAc;IAYtB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAyB3B;;;;OAIG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,kBAAkB;IAW1C;;;;;;OAMG;IACH,eAAe,CACb,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,MAA+B,GACvC,kBAAkB,EAAE;IA+DvB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAmCvB;;OAEG;IACH,OAAO,CAAC,WAAW;IAoCnB;;;OAGG;IACH,OAAO,CAAC,aAAa;IAyBrB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAsC5B;;OAEG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM;IAmBlC;;OAEG;IACH,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED;;OAEG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAI7C;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;CAGzC;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAWtF;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAmCzE;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAY/D;AAED;;GAEG;AACH,wBAAgB,eAAe,IAAI,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAuBrD"}
+{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../src/ner/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;GAEG;AACH,MAAM,WAAW,KAAK;IACpB,6BAA6B;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,mBAAmB;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,8CAA8C;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,4CAA4C;IAC5C,GAAG,EAAE,MAAM,CAAC;IACZ,2CAA2C;IAC3C,cAAc,EAAE,OAAO,CAAC;IACxB,sCAAsC;IACtC,SAAS,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,sBAAsB;IACtB,MAAM,EAAE,KAAK,EAAE,CAAC;IAChB,0BAA0B;IAC1B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,qBAAqB;IACrB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,6CAA6C;IAC7C,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8DAA8D;IAC9D,eAAe,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;IAChD;;;;OAIG;IACH,SAAS,EAAE,OAAO,CAAC;CACpB;AAsBD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,iCAAiC;IACjC,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,qEAAqE;IACrE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,eAGtC,CAAC;AAkBF;;;GAGG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,YAAY,CAAsB;IAC1C,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,IAAI,CAAW;IAGvB,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,KAAK,CAAa;IAG1B,OAAO,CAAC,QAAQ,CAAiB;IACjC,OAAO,CAAC,QAAQ,CAAkB;IAClC,OAAO,CAAC,QAAQ,CAAmB;IACnC,OAAO,CAAC,QAAQ,CAAmB;gBAEvB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAE,OAAO,CAAC,eAAe,CAAM;IAsB7E;;OAEG;IACH,OAAO,CAAC,cAAc;IAYtB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAyB3B;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,kBAAkB;IAqFzE;;;OAGG;IACH,OAAO,CAAC,aAAa;IAyBrB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAsC5B;;OAEG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM;IAmBlC;;OAEG;IACH,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED;;OAEG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAI7C;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;CAGzC;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAWtF;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAmCzE;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAY/D;AAED;;GAEG;AACH,wBAAgB,eAAe,IAAI,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAuBrD"}