spamscanner 5.0.0 → 5.1.0

package/index.js

@@ -1,6 +1,7 @@
-const process = require('process');
 const dns = require('dns');
 const fs = require('fs');
+const path = require('path');
+const process = require('process');
 const { debuglog } = require('util');
 
 // eslint-disable-next-line n/no-deprecated-api
@@ -55,6 +56,19 @@ const { simpleParser } = require('mailparser');
 
 const debug = debuglog('spamscanner');
 
+//
+// NOTE: we periodically need to update this
+//
+// Source from: CC-CEDICT
+// Licensed under Creative Commons Attribution-ShareAlike 4.0 International License
+// <https://www.mdbg.net/chinese/dictionary?page=cc-cedict>
+//
+// <https://github.com/yishn/chinese-tokenizer>
+//
+const chineseTokenizer = require('chinese-tokenizer').loadFile(
+  path.join(__dirname, 'cedict_1_0_ts_utf-8_mdbg.txt')
+);
+
 const aggressiveTokenizer = new natural.AggressiveTokenizer();
 const orthographyTokenizer = new natural.OrthographyTokenizer({
   language: 'fi'
@@ -195,6 +209,8 @@ const VOCABULARY_LIMIT = require('./vocabulary-limit.js');
 // TODO: convert this into a Map
 const ISO_CODE_MAPPING = require('./iso-code-mapping.json');
 
+const ISO_CODE_MAPPING_KEYS = Object.keys(ISO_CODE_MAPPING);
+
 // <https://kb.smarshmail.com/Article/23567>
 const EXECUTABLES = new Set(require('./executables.json'));
 
@@ -426,12 +442,6 @@ class SpamScanner {
             : '/var/run/clamav/clamd.ctl'
         }
       },
-      franc: {
-        minLength: 100,
-        // we can only support languages available
-        // in stopwords and natural's tokenizer methods
-        only: Object.keys(ISO_CODE_MAPPING)
-      },
       hasha: {
         algorithm: 'sha256'
       },
@@ -446,6 +456,21 @@ class SpamScanner {
       client: false,
       cachePrefix: 'spamscanner',
       ttlMs: ms('1h'),
+      // franc
+      franc: {
+        // NOTE: if locale was passed and was valid
+        //       then we need to compare it against english
+        //       and if it was english detected (and not und)
+        //       then switch the detected locale to english
+        minLength: 5,
+        // we can only support languages available
+        // in stopwords and natural's tokenizer methods
+        // and if it was detected to be english, compare against all languages
+        // otherwise if not, then compare only against english
+        // (namely we need to check against JP/ZH, but perhaps _all_ in future)
+        // (the edge case is that someone could spoof a language and it go undetected and tokenization bugs occur)
+        only: ISO_CODE_MAPPING_KEYS
+      },
       ...config
     };
 
@@ -952,6 +977,7 @@ class SpamScanner {
     // <https://github.com/FGRibreau/node-language-detect> (not too accurate)
     //
     const detectedLanguage = franc(string, this.config.franc);
+
     if (
       detectedLanguage !== 'und' &&
       isSANB(ISO_CODE_MAPPING[detectedLanguage])
@@ -1110,8 +1136,12 @@ class SpamScanner {
         break;
       case 'zh':
         // cmn
-        // TODO: use this instead https://github.com/yishn/chinese-tokenizer
         // ISO 639-3 = zho (Chinese, Macrolanguage)
+        // https://github.com/yishn/chinese-tokenizer
+        tokenizer = {
+          tokenize: (str) =>
+            chineseTokenizer(str).map((results) => results.text)
+        };
         language = 'chinese';
         stopwords = stopwordsZh;
         stemword = false;
@@ -1214,6 +1244,9 @@ class SpamScanner {
     //
     const tokens = [];
     for (const token of tokenizer.tokenize(string.toLowerCase())) {
+      // zh tokenizr yields empty strings
+      if (token === '' || token === ' ') continue;
+
       // whitelist words from being stemmed (safeguard)
       if (
         this.WHITELISTED_WORDS.has(token) ||

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "spamscanner",
   "description": "Spam Scanner - The Best Anti-Spam Scanning Service and Anti-Spam API",
-  "version": "5.0.0",
+  "version": "5.1.0",
   "author": "Niftylettuce, LLC. <niftylettuce@gmail.com> (https://niftylettuce.com/)",
   "bugs": {
     "url": "https://github.com/spamscanner/spamscanner/issues",
@@ -14,6 +14,7 @@
   "dependencies": {
     "@ladjs/naivebayes": "^0.1.0",
     "bitcoin-regex": "^2.0.0",
+    "chinese-tokenizer": "^2.4.0",
     "clamscan": "^2.1.2",
     "credit-card-regex": "^3.0.0",
     "crypto-random-string": "3",
@@ -91,6 +92,7 @@
     "node": ">=14"
   },
   "files": [
+    "cedict_1_0_ts_utf-8_mdbg.txt",
     "package.json",
     "index.js",
     "vocabulary-limit.js",