wingbot 3.67.19 → 3.67.21

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wingbot",
-  "version": "3.67.19",
+  "version": "3.67.21",
   "description": "Enterprise Messaging Bot Conversation Engine",
   "main": "index.js",
   "type": "commonjs",

package/src/AiMatching.js

@@ -123,6 +123,8 @@ const COMPARE = {
  * @prop {number} confidence
  */
 
+const ENTITY_OK = 0.79; // 0.835 on NLP;
+
 /**
  * @class {AiMatching}
  *
@@ -650,6 +652,7 @@ class AiMatching {
 
         const allOptional = wantedEntities.every((e) => e.optional
             && (!e.op || useEntities.every((n) => n.entity !== e.entity)));
+
         if (entitiesScore <= 0 && !allOptional) {
             return { score: 0, entities: [] };
         }
@@ -698,6 +701,8 @@ class AiMatching {
         let fromState = 0;
         let metl = 0;
 
+        let optHandicap = 0;
+
         for (const wanted of wantedEntities) {
             const usedIndexes = occurences.has(wanted.entity)
                 ? occurences.get(wanted.entity)
@@ -706,7 +711,9 @@ class AiMatching {
             let entityExists = false;
             const index = requestEntities
                 .findIndex((e, i) => {
-                    if (e.entity !== wanted.entity || usedIndexes.includes(i)) {
+                    if (e.entity !== wanted.entity
+                        || usedIndexes.includes(i)
+                        || e.score < ENTITY_OK) {
                         return false;
                     }
                     entityExists = true;
@@ -754,14 +761,21 @@ class AiMatching {
             }
 
             if (!matching) { // && optional && !entityExists
-                handicap += this.redundantEntityHandicap;
+                if (optHandicap < this.redundantEntityHandicap) {
+                    handicap += this.redundantEntityHandicap;
+                } else {
+                    handicap += this.optionalHandicap;
+                }
+                optHandicap += this.redundantEntityHandicap;
                 continue;
             }
 
             if (wanted.optional) {
-                handicap += wanted.op
+                const oph = wanted.op
                     ? this.optionalEqualityHandicap
                     : this.optionalHandicap;
+
+                handicap += oph;
             }
 
             if (wanted.op === COMPARE.NOT_EQUAL) {

package/src/Request.js

@@ -1649,11 +1649,14 @@ It looks like the bot isn't connected to class BotApp or the Processor is used w
         entity,
         value,
         score = 1,
+        entityScore = Math.max(score, 0.8),
         timestamp = makeTimestamp()
     ) {
         const res = Request.text(senderId, text, timestamp);
 
-        return Request.addIntentToRequest(res, intent, [{ entity, value, score }], score);
+        return Request.addIntentToRequest(res, intent, [
+            { entity, value, score: entityScore }
+        ], score);
     }
 
     static quickReply (senderId, action, data = {}, timestamp = makeTimestamp()) {

package/src/fuzzy/factoryFuzzySearch.js

@@ -5,7 +5,7 @@
 
 const { shortArrayIndex, splitToNgrams, cleanup } = require('./fuzzyUtils');
 const {
-    relativeLevenshtein, SEED_FUZZY, SEED_FUZZY_MULTIPLICATOR, WORD_HANDICAP_K_FUZZY
+    multiwordLevenshtein, SEED_FUZZY, SEED_FUZZY_MULTIPLICATOR, WORD_HANDICAP_K_FUZZY
 } = require('./levenshtein');
 
 const LOWER_DUPLICATES = 0.9;
@@ -93,7 +93,7 @@ function searchFnFactory (indexMap, ngramCounts, entities, maxIdf, {
                 const [, entityIndex, cleanText] = ngramCounts[id];
                 const [entity, value] = entities[entityIndex];
                 const relIdf = (idf / cnt) / maxIdf;
-                let score = relativeLevenshtein(
+                let score = multiwordLevenshtein(
                     cleanText,
                     cleanQuery,
                     levenshteinSeed,
@@ -103,7 +103,7 @@ function searchFnFactory (indexMap, ngramCounts, entities, maxIdf, {
 
                 if (cleanQuery.match(/^[^\s]{1,3}\s+.{6,}$/)) {
                     const without = cleanQuery.replace(/^[^\s]{1,3}\s+/, '');
-                    const altScore = relativeLevenshtein(
+                    const altScore = multiwordLevenshtein(
                         cleanText,
                         without,
                         levenshteinSeed,

package/src/fuzzy/levenshtein.js

@@ -128,42 +128,68 @@ function addSeed (seed, len, value, base = seed) {
     return base + (((len - value) / len) * (1 - seed));
 }
 
+function numStats (left, right) {
+    const len = Math.max(left.length, right.length);
+    const leftNum = left.replace(/[^0-9]+/g, '');
+    const rightNum = right.replace(/[^0-9]+/g, '');
+    const numLen = Math.max(leftNum.length, rightNum.length);
+    const numLev = numLen ? levenshtein(leftNum, rightNum) * NUMERIC_KOEF : 0;
+    const numRelLen = len ? numLen / len : 0;
+
+    return {
+        len,
+        leftNum,
+        rightNum,
+        numLen,
+        numLev,
+        numRelLen
+    };
+}
+
+const STRING_REPLACER = /[^\s-]+/g;
+const STRING_SPLITTER = /\s+/g;
+
 /**
  *
  * @param {string} left - training data
  * @param {string} right - query
  * @param {number} [seed]
  * @param {number} [wordKoef]
+ * @param {object} [nums]
  * @returns {number}
  */
 function relativeLevenshtein (
     left,
     right,
     seed = SEED_DEFAULT,
-    wordKoef = WORD_HANDICAP_K_DEFAULT
+    wordKoef = WORD_HANDICAP_K_DEFAULT,
+    nums = numStats(left, right)
 ) {
-    const len = Math.max(left.length, right.length);
-    if (!len) {
+    if (nums.len === 0) {
         return 0;
     }
     let stemLen = Math.min(left.length, right.length);
 
-    const leftWordCount = (left.match(/[^\s]+/g) || ['']).length;
-    const rightWordCount = (right.match(/[^\s]+/g) || ['']).length;
+    const leftWordCount = (left.match(STRING_REPLACER) || ['']).length;
+    const rightWordCount = (right.match(STRING_REPLACER) || ['']).length;
 
     const wordDiff = Math.max(0, rightWordCount - leftWordCount);
     const wordHandicap = (wordKoef ** wordDiff);
 
-    const leftNum = left.replace(/[^0-9]+/g, '');
-    const rightNum = right.replace(/[^0-9]+/g, '');
-    const numLen = Math.max(leftNum.length, rightNum.length);
-    const numLev = numLen ? levenshtein(leftNum, rightNum) * NUMERIC_KOEF : 0;
+    const l = levenshtein(left, right);
+
+    if (nums.numRelLen >= 0.5) {
+        const max = nums.len * 0.2; // 1/5 allowed error
+        const s = (l / max) * 0.25;
+
+        return Math.max(0, 1 - s) * wordHandicap;
+    }
 
     if (stemLen < 3) {
-        return addSeed(seed, len + numLen, levenshtein(left, right) + numLev) * wordHandicap;
+        return addSeed(seed, nums.len + nums.numLen, l + nums.numLev) * wordHandicap;
     }
 
-    let diff = len - stemLen;
+    let diff = nums.len - stemLen;
 
     if (diff <= 2) {
         diff += 2;
@@ -184,7 +210,7 @@ function relativeLevenshtein (
         diffWeight = (diff - 1) * SUFFIX_WEIGHT;
     }
 
-    const vStem = addSeed(seed, stemLen + numLen, stemLev + numLev, seed - diffWeight);
+    const vStem = addSeed(seed, stemLen + nums.numLen, stemLev + nums.numLev, seed - diffWeight);
     const vSuffix = addSeed(1 - diffWeight, diff, suffLev, 0);
 
     const r = (vStem + vSuffix) * wordHandicap;
@@ -200,18 +226,35 @@ function relativeLevenshtein (
  * @param {number} [wordKoef]
  * @returns {number}
  */
-function multiwordLevenshtein (left, right, seed, wordKoef = undefined) {
-    const leftSplit = `${left}`.split(/\s+/g);
-    const rightSplit = `${right}`.split(/\s+/g);
+function multiwordLevenshtein (left, right, seed = SEED_DEFAULT, wordKoef = undefined) {
+    const leftSplit = `${left}`.split(STRING_SPLITTER);
+    const rightSplit = `${right}`.split(STRING_SPLITTER);
 
     let sum = 0;
+    let sumNums = 0;
+    let cntNums = 0;
 
     const max = Math.max(leftSplit.length, rightSplit.length, 1);
     for (let i = 0; i < max; i++) {
-        sum += relativeLevenshtein(leftSplit[i] || '', rightSplit[i] || '', seed, wordKoef);
+        const ls = leftSplit[i] || '';
+        const rs = rightSplit[i] || '';
+        const nums = numStats(ls, rs);
+
+        const l = relativeLevenshtein(ls, rs, seed, wordKoef, nums);
+
+        if (nums.numRelLen >= 0.25) {
+            cntNums++;
+            sumNums += l;
+            sum += Math.max(0.85, l);
+        } else {
+            sum += l;
+        }
     }
 
-    return sum / max;
+    const total = sum / max;
+    const numeric = cntNums ? (sumNums / cntNums) : 1;
+
+    return total * numeric;
 }
 
 module.exports = {