wingbot 3.67.8 → 3.67.9

package/index.js

@@ -64,6 +64,9 @@ const {
 } = require('./src/analytics/consts');
 
 const { version: wingbotVersion } = require('./package.json');
+const { fuzzy } = require('./src/fuzzy');
+const prepareFuzzyIndex = require('./src/fuzzy/prepareFuzzyIndex');
+const factoryFuzzySearch = require('./src/fuzzy/factoryFuzzySearch');
 
 module.exports = {
 
@@ -109,6 +112,11 @@ module.exports = {
     plugins,
     vars,
 
+    // FUZZY
+    fuzzy,
+    prepareFuzzyIndex,
+    factoryFuzzySearch,
+
     // Notifications
     Notifications,
     NotificationsStorage,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wingbot",
-  "version": "3.67.8",
+  "version": "3.67.9",
   "description": "Enterprise Messaging Bot Conversation Engine",
   "main": "index.js",
   "scripts": {

package/src/Ai.js

@@ -59,7 +59,7 @@ let uq = 1;
 
 /**
  * @callback WordEntityDetectorFactory
- * @returns {Promise<WordEntityDetector|WordDetectorData>}
+ * @returns {Promise<WordDetectorData>}
  */
 
 /** @typedef {[string,EntityDetector|RegExp,DetectorOptions]} DetectorArgs */
@@ -241,11 +241,14 @@ class Ai {
      * @returns {T}
      * @memberOf Ai
      */
-    register (model, prefix = this.DEFAULT_PREFIX) {
+    register (model = null, prefix = this.DEFAULT_PREFIX) {
         /** @type {T} */
         let modelObj;
 
-        if (typeof model === 'string') {
+        if (!model) {
+            // @ts-ignore
+            modelObj = new CustomEntityDetectionModel({ prefix });
+        } else if (typeof model === 'string') {
             // @ts-ignore
             modelObj = new WingbotModel({
                 model,
@@ -803,7 +806,6 @@ class Ai {
         if (!req.isText()) {
             return;
         }
-
         if (this._keyworders.size !== 0) {
             const model = this._getModelForRequest(req);
             if (!model) {

package/src/fuzzy/factoryFuzzySearch.js

@@ -0,0 +1,243 @@
+/**
+ * @author David Menger
+ */
+'use strict';
+
+const { shortArrayIndex, splitToNgrams, cleanup } = require('./fuzzyUtils');
+const {
+    relativeLevenshtein, SEED_FUZZY, SEED_FUZZY_MULTIPLICATOR, WORD_HANDICAP_K_FUZZY
+} = require('./levenshtein');
+
+const LOWER_DUPLICATES = 0.9;
+
+function getIndexesToIterate (ngrams, tfEntry) {
+    if (tfEntry.length === 2) {
+        return [1, 1];
+    }
+    const min = Math.ceil(ngrams * 0.6);
+    const max = Math.floor(ngrams * 1.5);
+    return [shortArrayIndex(min), shortArrayIndex(max)];
+}
+
+/**
+ * @typedef {object} FuzzySearchOptions
+ * @prop {boolean} [keepMultipleValues]
+ * @prop {Stemmer} [stemmer]
+ * @prop {number} [threshold]
+ */
+
+/** @typedef {import('./prepareFuzzyIndex').FuzzyIndexData} FuzzyIndexData */
+/** @typedef {import('./prepareFuzzyIndex').Stemmer} Stemmer */
+/** @typedef {import('../Ai').WordEntityDetector} WordEntityDetector */
+/** @typedef {import('../Ai').WordDetectorData} WordDetectorData */
+
+/**
+ * @typedef {object} Entity
+ * @prop {string} entity
+ * @prop {string} value
+ * @prop {string[]} [synonyms]
+ */
+
+function searchFnFactory (indexMap, ngramCounts, entities, maxIdf, {
+    stemmer = null,
+    keepMultipleValues = false,
+    threshold = 0.5,
+    limit = undefined
+}, hasFuzzyMultiplier = false) {
+    /** @type {WordEntityDetector} */
+    const searchFn = (search) => {
+        const cleanQuery = cleanup(search, stemmer);
+        const tokens = splitToNgrams(cleanQuery);
+        const results = new Map();
+
+        tokens.forEach((token) => {
+            const entry = indexMap.get(token);
+            if (!entry) {
+                return;
+            }
+            const [idf] = entry;
+            const [startIndex, endIndex] = getIndexesToIterate(tokens.length, entry);
+
+            const maxIndex = Math.min(endIndex, entry.length - 1);
+            for (let i = startIndex; i <= maxIndex; i++) {
+                for (const id of entry[i]) {
+                    let res = results.get(id);
+                    if (!res) {
+                        res = { cnt: 0, idf: 0 };
+                        results.set(id, res);
+                    }
+                    res.cnt++;
+                    res.idf += idf;
+                }
+            }
+
+        });
+
+        let maxScore = 0;
+        let maxRelIdf = 0; // small but positive
+        const levenshteinSeed = hasFuzzyMultiplier
+            ? SEED_FUZZY_MULTIPLICATOR
+            : SEED_FUZZY;
+
+        const percentage = hasFuzzyMultiplier
+            ? 0.6
+            : 0.5;
+
+        const preprocessed = Array.from(results.entries())
+            .filter(([id, { cnt }]) => {
+                const [ngramCount] = ngramCounts[id];
+                const percentageOfMatchedNgrams = (cnt * 2) / (ngramCount + tokens.length);
+                return percentageOfMatchedNgrams >= percentage;
+            })
+            .map(([id, { cnt, idf }]) => {
+                const [, entityIndex, cleanText] = ngramCounts[id];
+                const [entity, value] = entities[entityIndex];
+                const relIdf = (idf / cnt) / maxIdf;
+                let score = relativeLevenshtein(
+                    cleanText,
+                    cleanQuery,
+                    levenshteinSeed,
+                    WORD_HANDICAP_K_FUZZY
+                );
+                let start = 0;
+
+                if (cleanQuery.match(/^[^\s]{1,3}\s+.{6,}$/)) {
+                    const without = cleanQuery.replace(/^[^\s]{1,3}\s+/, '');
+                    const altScore = relativeLevenshtein(
+                        cleanText,
+                        without,
+                        levenshteinSeed,
+                        WORD_HANDICAP_K_FUZZY
+                    );
+
+                    if (altScore > score) {
+                        score = altScore;
+                        start = cleanQuery.length - without.length;
+                    }
+                }
+
+                if (maxScore < score) maxScore = score;
+                if (maxRelIdf < relIdf) maxRelIdf = relIdf;
+
+                return {
+                    entity,
+                    value,
+                    _relIdf: relIdf,
+                    score,
+                    ...(start ? { start } : {})
+                };
+            });
+
+        const found = preprocessed.map((o) => {
+            const { _relIdf: relIdf } = o;
+            // eslint-disable-next-line no-param-reassign
+            delete o._relIdf;
+
+            const koef = maxRelIdf <= 0 ? relIdf : (relIdf / maxRelIdf);
+            const addToScore = ((1 - maxScore) / 2) * koef;
+
+            Object.assign(o, {
+                score: Math.round((o.score + addToScore) * 10000) / 10000
+            });
+
+            return o;
+        });
+
+        found.sort((a, z) => z.score - a.score);
+
+        const known = new Map();
+        const res = found
+            .filter((result) => {
+                const key = keepMultipleValues ? `${result.entity}|${result.value}` : result.entity;
+                if (result.score < threshold) {
+                    return false;
+                }
+                if (known.has(key)) {
+                    const { result: origResult, score, alts } = known.get(key);
+                    if (!keepMultipleValues
+                            && Math.abs(score - result.score) < (1 - LOWER_DUPLICATES)
+                            && origResult.value !== result.value) {
+
+                        if (!alts.some((a) => a.value === result.value)) {
+                            // five percent down for collisions
+                            origResult.score *= LOWER_DUPLICATES;
+                        }
+
+                        alts.push(result);
+
+                        Object.assign(origResult, {
+                            alternatives: alts
+                        });
+                    }
+                    return false;
+                }
+                known.set(key, { result, score: result.score, alts: [] });
+                return true;
+            })
+            .slice(0, limit);
+
+        res.forEach((entity) => {
+            if ('alternatives' in entity) {
+                // @ts-ignore
+                let { alternatives } = entity;
+
+                const kn = new Set([entity.value]);
+                alternatives = alternatives
+                    // @ts-ignore
+                    .sort((a, z) => z.score - a.score)
+                    .filter((e) => !known.has(e.value) && kn.add(e.value));
+
+                // @ts-ignore
+                for (let i = 0; i < alternatives.length; i++) {
+                    const alt = alternatives[i];
+                    // @ts-ignore
+                    Object.assign(alt, {
+                        // @ts-ignore
+                        score: alt.score * (LOWER_DUPLICATES ** alternatives.length)
+                    });
+                }
+
+                Object.assign(entity, { alternatives });
+            }
+        });
+
+        return res;
+    };
+
+    return searchFn;
+}
+
+/**
+ *
+ * @param {FuzzyIndexData} data
+ * @param {FuzzySearchOptions} [options]
+ * @returns {WordDetectorData}
+ */
+function factoryFuzzySearch (data, options = {}) {
+    const {
+        ngramCounts,
+        entities,
+        indexArray,
+        maxIdf,
+        hasFuzzyMultiplier,
+        maxWordCount
+    } = data;
+
+    const indexMap = new Map(indexArray);
+
+    const detector = searchFnFactory(
+        indexMap,
+        ngramCounts,
+        entities,
+        maxIdf,
+        options,
+        hasFuzzyMultiplier
+    );
+
+    return {
+        detector,
+        maxWordCount
+    };
+}
+
+module.exports = factoryFuzzySearch;

package/src/fuzzy/fuzzyUtils.js

@@ -0,0 +1,91 @@
+/**
+ * @author David Menger
+ */
+'use strict';
+
+const { normalize } = require('./normalize');
+
+const SHORTEN_BY = 2;
+const NGRAMS = 3;
+
+/**
+ *
+ * @param {string|number} word
+ * @returns {string}
+ */
+function preNormalize (word) {
+    return normalize(word)
+        .replace(/[^a-z0-9]+/g, ' ')
+        .trim();
+}
+
+function stem (normalized, stemmer) {
+    if (!stemmer) {
+        return normalized;
+    }
+
+    const stems = normalized
+        .split(/\s+/g)
+        .map((w) => stemmer(w) || w);
+
+    return `${normalized} ${stems.join(' ')}`;
+}
+
+/** @typedef {{ (word: string): string}} Stemmer */
+
+/**
+ *
+ * @param {string|number} word
+ * @param {Stemmer} stemmer
+ * @returns {string}
+ */
+function cleanup (word, stemmer) {
+    const normalized = preNormalize(word);
+    return stem(normalized, stemmer);
+}
+
+/**
+ *
+ * @param {string} normalized
+ * @param {Stemmer} stemmer
+ * @returns {string}
+ */
+function cleanupPreNormalized (normalized, stemmer) {
+    return stem(normalized, stemmer);
+}
+
+/**
+ *
+ * @param {number} ngramCount
+ * @returns {number}
+ */
+function shortArrayIndex (ngramCount) {
+    return Math.floor(ngramCount / SHORTEN_BY) + 1;
+}
+
+/**
+ *
+ * @param {string} word
+ * @returns {string[]}
+ */
+function splitToNgrams (word) {
+    const prolonged = ` ${word} `;
+    const len = prolonged.length - NGRAMS + 1;
+    if (len <= 0) {
+        return word.length > 0 ? [prolonged] : [];
+    }
+    const ret = new Array(len);
+    for (let i = 0; i < len; i++) {
+        const sub = prolonged.substring(i, i + NGRAMS);
+        ret[i] = sub;
+    }
+    return ret;
+}
+
+module.exports = {
+    cleanup,
+    shortArrayIndex,
+    splitToNgrams,
+    cleanupPreNormalized,
+    preNormalize
+};

package/src/fuzzy/index.js

@@ -0,0 +1,40 @@
+/**
+ * @author David Menger
+ */
+'use strict';
+
+const factoryFuzzySearch = require('./factoryFuzzySearch');
+const prepareFuzzyIndex = require('./prepareFuzzyIndex');
+
+/** @typedef {import('./factoryFuzzySearch').Entity} Entity */
+/** @typedef {import('./factoryFuzzySearch').FuzzySearchOptions} FuzzySearchOptions */
+/** @typedef {import('../Ai').WordEntityDetectorFactory} WordEntityDetectorFactory */
+
+/**
+ * @callback EntityFactory
+ * @returns {Promise<Entity[]>}
+ */
+
+/**
+ *
+ * @param {Entity[]|EntityFactory} entities
+ * @param {FuzzySearchOptions} options
+ * @returns {WordEntityDetectorFactory}
+ */
+function fuzzy (entities, options = {}) {
+
+    return async () => {
+        const data = typeof entities === 'function'
+            ? (await entities())
+            : entities;
+
+        const index = prepareFuzzyIndex(data, options);
+        return factoryFuzzySearch(index, options);
+    };
+}
+
+module.exports = {
+    fuzzy,
+    prepareFuzzyIndex,
+    factoryFuzzySearch
+};

package/src/fuzzy/levenshtein.js

@@ -0,0 +1,228 @@
+/**
+ * @author David Menger
+ */
+'use strict';
+
+const NUMERIC_KOEF = 4;
+const SUFFIX_WEIGHT = 0.055;
+
+const SEED_DEFAULT = 0.5;
+const SEED_FUZZY = 0.25;
+const SEED_FUZZY_MULTIPLICATOR = -0.25;
+
+const WORD_HANDICAP_K_DEFAULT = 0.9;
+const WORD_HANDICAP_K_FUZZY = 0.6;
+
+function _min (d0, d1, d2, bx, ay) {
+    if (d0 < d1 || d2 < d1) {
+        return d0 > d2
+            ? d2 + 1
+            : d0 + 1;
+    }
+    return bx === ay
+        ? d1
+        : d1 + 1;
+}
+
+/**
+ *
+ * @param {string} left
+ * @param {string} right
+ * @returns {number}
+ */
+function levenshtein (left, right) {
+    if (left === right) {
+        return 0;
+    }
+
+    let a = left;
+    let b = right;
+
+    if (a.length > b.length) {
+        const tmp = a;
+        a = b;
+        b = tmp;
+    }
+
+    let la = a.length;
+    let lb = b.length;
+
+    while (la > 0 && (a.charCodeAt(la - 1) === b.charCodeAt(lb - 1))) {
+        la--;
+        lb--;
+    }
+
+    let offset = 0;
+
+    while (offset < la && (a.charCodeAt(offset) === b.charCodeAt(offset))) {
+        offset++;
+    }
+
+    la -= offset;
+    lb -= offset;
+
+    if (la === 0 || lb < 3) {
+        return lb;
+    }
+
+    let x = 0;
+    let y;
+    let d0;
+    let d1;
+    let d2;
+    let d3;
+    let dd;
+    let dy;
+    let ay;
+    let bx0;
+    let bx1;
+    let bx2;
+    let bx3;
+
+    const vector = [];
+
+    for (y = 0; y < la; y++) {
+        vector.push(y + 1);
+        vector.push(a.charCodeAt(offset + y));
+    }
+
+    const len = vector.length - 1;
+
+    for (; x < lb - 3;) {
+        bx0 = b.charCodeAt(offset + (d0 = x));
+        bx1 = b.charCodeAt(offset + (d1 = x + 1));
+        bx2 = b.charCodeAt(offset + (d2 = x + 2));
+        bx3 = b.charCodeAt(offset + (d3 = x + 3));
+        x += 4;
+        dd = x;
+        for (y = 0; y < len; y += 2) {
+            dy = vector[y];
+            ay = vector[y + 1];
+            d0 = _min(dy, d0, d1, bx0, ay);
+            d1 = _min(d0, d1, d2, bx1, ay);
+            d2 = _min(d1, d2, d3, bx2, ay);
+            dd = _min(d2, d3, dd, bx3, ay);
+            vector[y] = dd;
+            d3 = d2;
+            d2 = d1;
+            d1 = d0;
+            d0 = dy;
+        }
+    }
+
+    for (; x < lb;) {
+        bx0 = b.charCodeAt(offset + (d0 = x));
+        dd = ++x;
+        for (y = 0; y < len; y += 2) {
+            dy = vector[y];
+            dd = _min(dy, d0, dd, bx0, vector[y + 1]);
+            vector[y] = dd;
+            d0 = dy;
+        }
+    }
+
+    return dd;
+}
+
+function addSeed (seed, len, value, base = seed) {
+    return base + (((len - value) / len) * (1 - seed));
+}
+
+/**
+ *
+ * @param {string} left - training data
+ * @param {string} right - query
+ * @param {number} [seed]
+ * @param {number} [wordKoef]
+ * @returns {number}
+ */
+function relativeLevenshtein (
+    left,
+    right,
+    seed = SEED_DEFAULT,
+    wordKoef = WORD_HANDICAP_K_DEFAULT
+) {
+    const len = Math.max(left.length, right.length);
+    if (!len) {
+        return 0;
+    }
+    let stemLen = Math.min(left.length, right.length);
+
+    const leftWordCount = (left.match(/[^\s]+/g) || ['']).length;
+    const rightWordCount = (right.match(/[^\s]+/g) || ['']).length;
+
+    const wordDiff = Math.max(0, rightWordCount - leftWordCount);
+    const wordHandicap = (wordKoef ** wordDiff);
+
+    const leftNum = left.replace(/[^0-9]+/g, '');
+    const rightNum = right.replace(/[^0-9]+/g, '');
+    const numLen = leftNum.length ? leftNum.length * NUMERIC_KOEF : rightNum.length;
+    const useNumK = leftNum.length ? NUMERIC_KOEF : 1;
+    const numLev = numLen ? levenshtein(leftNum, rightNum) * useNumK : 0;
+
+    if (stemLen < 3) {
+        return addSeed(seed, len + numLen, levenshtein(left, right) + numLev) * wordHandicap;
+    }
+
+    let diff = len - stemLen;
+
+    if (diff <= 2) {
+        diff += 2;
+        stemLen -= 2;
+    }
+
+    let diffWeight = diff * SUFFIX_WEIGHT;
+
+    const lStem = left.substring(0, stemLen);
+    const rStem = right.substring(0, stemLen);
+    const lSuff = left.substring(stemLen);
+    const rSuff = right.substring(stemLen);
+
+    const stemLev = levenshtein(lStem, rStem);
+    const suffLev = levenshtein(lSuff, rSuff);
+
+    if (suffLev === 1 && stemLev === 0) {
+        diffWeight = (diff - 1) * SUFFIX_WEIGHT;
+    }
+
+    const vStem = addSeed(seed, stemLen + numLen, stemLev + numLev, seed - diffWeight);
+    const vSuffix = addSeed(1 - diffWeight, diff, suffLev, 0);
+
+    const r = (vStem + vSuffix) * wordHandicap;
+
+    // console.log(`#levenshtein "${left}" <- ${right}: ${r.toFixed(3)}`);
+    return r;
+}
+
+/**
+ *
+ * @param {string} left
+ * @param {string} right
+ * @param {number} seed
+ * @param {number} [wordKoef]
+ * @returns {number}
+ */
+function multiwordLevenshtein (left, right, seed, wordKoef = undefined) {
+    const leftSplit = `${left}`.split(/\s+/g);
+    const rightSplit = `${right}`.split(/\s+/g);
+
+    let sum = 0;
+
+    const max = Math.max(leftSplit.length, rightSplit.length, 1);
+    for (let i = 0; i < max; i++) {
+        sum += relativeLevenshtein(leftSplit[i] || '', rightSplit[i] || '', seed, wordKoef);
+    }
+
+    return sum / max;
+}
+
+module.exports = {
+    levenshtein,
+    multiwordLevenshtein,
+    relativeLevenshtein,
+    SEED_DEFAULT,
+    SEED_FUZZY,
+    SEED_FUZZY_MULTIPLICATOR,
+    WORD_HANDICAP_K_FUZZY,
+    WORD_HANDICAP_K_DEFAULT
+};

package/src/fuzzy/normalize.js

@@ -0,0 +1,62 @@
+/*
+ * @author David Menger
+ */
+'use strict';
+
+const { normalize } = require('../utils/tokenizer');
+
+/**
+ * Preserves only letters (with or withour diacritics) and makes everything lowercased
+ *
+ * @param {string} str - input string
+ * @returns {string}
+ */
+function cleanup (str) {
+    return str
+        .replace(/[`']+(\s|$)|(\s|^)['`]+/g, ' ')
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+
+/**
+ *
+ * @param {string} str
+ * @param {boolean} strict
+ * @returns {string}
+ */
+function normalizeEntity (str, strict) {
+    if (strict) {
+        return `${str}`.toLocaleLowerCase()
+            .replace(/\s+/g, ' ')
+            .trim();
+    }
+    return cleanup(normalize(str));
+}
+
+/**
+ *
+ * @param {string} str
+ * @returns {string}
+ */
+function normalizePreserveEntities (str) {
+
+    let ret = normalize(str);
+
+    str.replace(/@[A-Z0-9-]+/g, (entity, start) => {
+        const begin = ret.substring(0, start);
+        const end = ret.substring(start + entity.length);
+
+        ret = `${begin}${entity}${end}`;
+
+        return entity;
+    });
+
+    return ret;
+}
+
+module.exports = {
+    normalize,
+    cleanup,
+    normalizePreserveEntities,
+    normalizeEntity
+};

package/src/fuzzy/prepareFuzzyIndex.js

@@ -0,0 +1,196 @@
+/**
+ * @author David Menger
+ */
+'use strict';
+
+const {
+    shortArrayIndex,
+    splitToNgrams,
+    cleanupPreNormalized,
+    preNormalize
+} = require('./fuzzyUtils');
+
+const SHORTEN_MIN = 5000;
+
+/**
+ *
+ * @param {number} idf
+ * @param {*} tfArray
+ * @param {NgramCount[]} ngramCounts
+ * @returns {IndexMapTuple}
+ */
+function divideTfArray (idf, tfArray, ngramCounts) {
+    // first index is ID, second tfArray
+    if (tfArray.length < SHORTEN_MIN) {
+        return [idf, tfArray];
+    }
+
+    /** @type {IndexMapTuple} */
+    const ret = [idf];
+    for (const id of tfArray) {
+        const [ngramCount] = ngramCounts[id];
+        const i = shortArrayIndex(ngramCount);
+        if (!ret[i]) {
+            ret[i] = [];
+        }
+        // @ts-ignore
+        ret[i].push(id);
+    }
+    for (let i = 1; i < ret.length; i++) {
+        if (!ret[i]) {
+            ret[i] = [];
+        }
+    }
+    return ret;
+}
+
+/**
+ * @typedef {object} Entity
+ * @prop {boolean} [id]
+ * @prop {string} entity
+ * @prop {string|number} value
+ * @prop {string[]} [synonyms]
+ */
+
+/** @typedef {[idf: number, ...index: number[][]]} IndexMapTuple */
+/** @typedef {[entity: string, value: string|number]} EntityIndex */
+/** @typedef {[ngramCount: number, index: number, cleanText: string]} NgramCount */
+/** @typedef {[ngram: string, index: IndexMapTuple]} IndexMapEntry */
+
+/** @typedef {Map<string, [number, Set<number>]>} IndexMap */
+
+/**
+ * @typedef {object} FuzzyIndexData
+ * @prop {NgramCount[]} ngramCounts,
+ * @prop {EntityIndex[]} entities,
+ * @prop {IndexMapEntry[]} indexArray,
+ * @prop {number} maxIdf,
+ * @prop {number} tfEntryMaxLen,
+ * @prop {number} tfTotal,
+ * @prop {number} avgIdf
+ * @prop {boolean} hasFuzzyMultiplier
+ * @prop {number} maxWordCount
+ */
+
+/** @typedef {import('./fuzzyUtils').Stemmer} Stemmer */
+
+const DEFAULT_MULTIPLIER = (w) => [w];
+
+/**
+ *
+ * @param {Entity[]} data
+ * @param {Object} [options]
+ * @param {Stemmer} [options.stemmer]
+ * @param {Function} [options.multiplier]
+ * @returns {FuzzyIndexData}
+ */
+function prepareFuzzyIndex (data, {
+    stemmer = null,
+    multiplier = DEFAULT_MULTIPLIER
+} = {}) {
+
+    /** @type {IndexMap} */
+    const indexMap = new Map();
+
+    function addToIndex (token, id) {
+        let entry = indexMap.get(token);
+        if (!entry) {
+            entry = [null, new Set()];
+            indexMap.set(token, entry);
+        }
+        entry[1].add(id);
+    }
+
+    function addItemToIndex (cleanText, id) {
+        const tokens = splitToNgrams(cleanText);
+
+        tokens
+            .forEach((token) => {
+                addToIndex(token, id);
+            });
+
+        return tokens.length;
+    }
+
+    function cleanForMultiples (text) {
+        return text.toLocaleLowerCase().replace(/[^a-z0-9\u00C0-\u017F]+/g, ' ');
+    }
+
+    let maxWordCount = 0;
+    const entities = new Array(data.length);
+    let overAllIndex = 0;
+    const ngramCounts = data
+        // flattern synonyms
+        .reduce((arr, {
+            entity, value, synonyms = [], id = null
+        }, index) => {
+            const known = new Set();
+            let texts = Array.isArray(synonyms) && synonyms.length && id === true
+                ? synonyms
+                : [value, ...synonyms];
+
+            texts = texts.map((text) => cleanForMultiples(text));
+
+            texts = texts
+                .map((text) => multiplier(text, texts[0]))
+                .reduce((a, multiplied) => [
+                    ...a,
+                    ...multiplied.filter((word) => {
+                        if (known.has(word)) {
+                            return false;
+                        }
+                        known.add(word);
+                        return true;
+                    })
+                ], []);
+
+            entities[index] = [entity, value];
+            const ngramsData = texts
+                .map((text, i) => {
+                    const normalized = preNormalize(text);
+                    const wordCount = normalized.split(/\s+/g).length;
+                    if (wordCount > maxWordCount) maxWordCount = wordCount;
+                    const cleanText = cleanupPreNormalized(normalized, stemmer);
+                    const ngramCount = addItemToIndex(cleanText, i + overAllIndex);
+                    return [ngramCount, index, cleanText];
+                });
+            overAllIndex += ngramsData.length;
+            arr.push(...ngramsData);
+            return arr;
+        }, []);
+
+    let totIdf = 0;
+    let maxIdf = 0;
+    let tfEntryMaxLen = 0;
+    let tfTotal = 0;
+    for (const [key, entry] of indexMap.entries()) {
+        const idf = Math.log10((indexMap.size / entry[1].size));
+        const tfArray = Array.from(entry[1].values());
+        const tfEntry = divideTfArray(idf, tfArray, ngramCounts);
+        // @ts-ignore
+        indexMap.set(key, tfEntry);
+
+        // stats
+        tfTotal++;
+        totIdf += idf;
+        if (maxIdf < idf) maxIdf = idf;
+        if (tfEntryMaxLen < tfEntry.length) tfEntryMaxLen = tfEntry.length;
+    }
+    const indexArray = Array.from(indexMap.entries());
+    const avgIdf = totIdf / indexArray.length;
+
+    return {
+        ngramCounts,
+        entities,
+        // @ts-ignore
+        indexArray,
+        maxIdf,
+        tfEntryMaxLen,
+        tfTotal,
+        avgIdf,
+        hasFuzzyMultiplier: multiplier !== DEFAULT_MULTIPLIER,
+        maxWordCount
+    };
+}
+
+module.exports = prepareFuzzyIndex;

package/src/utils/tokenizer.js

@@ -87,14 +87,51 @@ const DEFAULT_REMOVAL_MAP = [
     { base: 'w', letters: '\u0077\u24E6\uFF57\u1E81\u1E83\u0175\u1E87\u1E85\u1E98\u1E89\u2C73' },
     { base: 'x', letters: '\u0078\u24E7\uFF58\u1E8B\u1E8D' },
     { base: 'y', letters: '\u0079\u24E8\uFF59\u1EF3\u00FD\u0177\u1EF9\u0233\u1E8F\u00FF\u1EF7\u1E99\u1EF5\u01B4\u024F\u1EFF' },
-    { base: 'z', letters: '\u007A\u24E9\uFF5A\u017A\u1E91\u017C\u017E\u1E93\u1E95\u01B6\u0225\u0240\u2C6C\uA763' }
+    { base: 'z', letters: '\u007A\u24E9\uFF5A\u017A\u1E91\u017C\u017E\u1E93\u1E95\u01B6\u0225\u0240\u2C6C\uA763' },
+
+    { base: '\u0433', letters: '\u0403\u0490\u0491\u0492\u0493\u0413\u0494\u0495\u04F6\u04F7' }, // Г
+    { base: 'i', letters: '\u0406\u0456\u04c0\u0407\u0457\u04CF' },
+    { base: 'j', letters: '\u0408\u0458' },
+    { base: '\u0438', letters: '\u040d\u0419\u0439\u0418\u045D\u048B\u04E2\u04E3\u04E4\u04E5' }, // И
+    { base: 'a', letters: '\u0410\u0430\u04D0\u04D1\u04D2\u04D3' },
+    { base: 'b', letters: '\u0412\u0432' },
+    { base: 'e', letters: '\u0400\u0401\u0415\u0435\u0450\u0451\u0454\u04BC\u04BD\u04BE\u04BF\u04D6\u04D7' },
+    { base: 'h', letters: '\u04BA\u04BB\u04C7\u04C8\u04C9\u04CA' },
+    { base: 'k', letters: '\u040c\u041a\u043A\u045C\u049A\u049B\u049C\u049D\u049E\u049F\u04A0\u04A1\u04C3\u04C4' },
+    { base: 'm', letters: '\u041c\u043C\u04CD\u04CE' },
+    { base: 'h', letters: '\u041d\u043D\u045B\u04A2\u04A3\u04A4\u04A5' },
+    { base: 'o', letters: '\u041e\u043E\u04E6\u04E7\u04E8\u04E9\u04EA\u04EB' },
+    { base: 'p', letters: '\u0420\u0440\u048E\u048F' },
+    { base: 's', letters: '\u0405\u0455' },
+    { base: 'c', letters: '\u0421\u0441\u04AA\u04AB' },
+    { base: 't', letters: '\u0422\u0442\u04AC\u04AD' },
+    { base: 'y', letters: '\u0423\u040E\u0478\u04ee\u04f0\u04ef\u0443\u04f1\u04f2\u04f3\u045E\u04AE\u04AF\u04B0\u04B1' },
+    { base: 'x', letters: '\u0425\u0445\u04A8\u04A9\u04B2\u04B3' }, // Х (H)
+    { base: '\u044C', letters: '\u042C\u048C\u048D' }, // ь
+    { base: '\u0436', letters: '\u0496\u0497\u0416\u04C1\u04C2\u04DC\u04DD' }, // Ж (ZH)
+    { base: '\u0437', letters: '\u0417\u0498\u0499\u04DE\u04DF\u04E0\u04E1' }, // З (ZE)
+    { base: '\u043f', letters: '\u041f\u04A6\u04A7' }, // П (P)
+    { base: '\u0446', letters: '\u0426\u04B4\u04B5' }, // Ц (TSE)
+    { base: '\u0447', letters: '\u0427\u04B6\u04B7\u04B8\u04B9\u04CB\u04CC\u04F4\u04F5' }, // Ч (CHE)
+    { base: '\u0434', letters: '\u041B\u04C5\u04C6' }, // Л (L)
+    { base: '\u044D', letters: '\u042D\u04ED\u04EC' }, // Э (E)
+    { base: '\u044b', letters: '\u042b\u04F8\u04F9' }, // Ы (YER)
+
+    { base: 'nj', letters: '\u045A\u040A' }, // њ
+    { base: 'lj', letters: '\u0409\u0459' }, // Љ
+    { base: 'dz', letters: '\u045F\u040F' } // џ
+
 ];
 
-const diacriticsMap = {};
-for (let i = 0; i < DEFAULT_REMOVAL_MAP.length; i++) {
-    const { letters } = DEFAULT_REMOVAL_MAP[i];
-    for (let j = 0; j < letters.length; j++) {
-        diacriticsMap[letters[j]] = DEFAULT_REMOVAL_MAP[i].base;
+let diacriticsMap = null;
+
+function buildDiacriticsMap () {
+    diacriticsMap = {};
+    for (let i = 0; i < DEFAULT_REMOVAL_MAP.length; i++) {
+        const { letters } = DEFAULT_REMOVAL_MAP[i];
+        for (let j = 0; j < letters.length; j++) {
+            diacriticsMap[letters[j]] = DEFAULT_REMOVAL_MAP[i].base;
+        }
     }
 }
 
@@ -106,9 +143,28 @@ for (let i = 0; i < DEFAULT_REMOVAL_MAP.length; i++) {
  * @returns {string}
  */
 function replaceDiacritics (str) {
+    if (!diacriticsMap) {
+        buildDiacriticsMap();
+    }
     return str.replace(/[^\u0000-\u007E]/g, (a) => diacriticsMap[a] || a); // eslint-disable-line no-control-regex
 }
 
+/**
+ *
+ * @param {string|number} str
+ * @returns {string}
+ */
+function normalize (str) {
+    if (!diacriticsMap) {
+        buildDiacriticsMap();
+    }
+    //  U+0400–U+04FF - cyrillic
+    return `${str}`
+        .replace(/[\u0400-\u04ff]/g, (a) => (diacriticsMap[a] ? diacriticsMap[a] : a)) // cyrillic
+        .replace(/[^A-Za-z0-9\s'`\u0400-\u04ff]/g, (a) => (diacriticsMap[a] ? diacriticsMap[a] : ' '))
+        .toLowerCase();
+}
+
 /**
  *
  * @param {string} string
@@ -124,5 +180,6 @@ function tokenize (string) {
 
 module.exports = {
     replaceDiacritics,
-    tokenize
+    tokenize,
+    normalize
 };

package/src/wingbot/CustomEntityDetectionModel.js

@@ -65,9 +65,9 @@ const { iterateThroughWords } = require('../utils/ai');
 /**
  * @callback WordEntityDetector
  * @param {string} text
- * @param {DetectedEntity[]} entities
- * @param {number} startIndex
- * @param {string} prefix
+ * @param {DetectedEntity[]} [entities]
+ * @param {number} [startIndex]
+ * @param {string} [prefix]
  * @returns {DetectedEntity[]}
  */