xcraft-core-utils 3.1.1 → 4.1.0

package/index.js

@@ -16,10 +16,9 @@ exports.propTypes = require('./lib/prop-types.js');
 exports.reflect = require('./lib/reflect.js');
 exports.regex = require('./lib/regex.js');
 exports.string = require('./lib/string.js');
+exports.whereIs = require('./lib/whereIs.js');
 exports.yaml = require('./lib/yaml.js');
 
-exports.SQLite = require('./lib/sqlite.js');
 exports.RankedCache = require('./lib/ranked-cache.js');
 exports.JobQueue = require('./lib/job-queue.js');
-exports.PersistantJobQueue = require('./lib/persistant-job-queue.js');
 exports.CursorPump = require('./lib/cursorPump.js');

package/lib/.babelrc

@@ -0,0 +1,8 @@
+{
+  "plugins": [
+    "@babel/proposal-class-properties",
+    "@babel/proposal-object-rest-spread",
+    "@babel/proposal-function-bind",
+    "@babel/transform-modules-commonjs"
+  ]
+}

package/lib/arrayCollector.js

@@ -1,12 +1,13 @@
+'use strict';
+
 const {throttle} = require('lodash');
 const watt = require('gigawatts');
 
 class ArrayCollector {
-  constructor(wait = 20, onCollect, leading = true) {
+  constructor(resp, wait = 20, onCollect, leading = true) {
     this.onCollect = watt(onCollect);
     this.entries = {};
-    const busClient = require('xcraft-core-busclient').getGlobal();
-    this.resp = busClient.newResponse('collector', 'token');
+    this.resp = resp;
     this.release = throttle(this._release, wait, {leading});
   }
 

package/lib/async.js

@@ -0,0 +1,34 @@
+'use strict';
+
+const watt = require('gigawatts');
+
+class Async {
+  constructor() {
+    watt.wrapAll(this);
+  }
+
+  /**
+   * Reduce an array to a map with an async iteration.
+   *
+   * It's useful in the case where the main event loop must not
+   * be blocked and the array is very large. The purpose of this
+   * helper is only for task which be done with an usual sync
+   * map reduce.
+   *
+   * @param {*} keyFunc - Key for the item in the map.
+   * @param {*} valueFunc - Value for the item in the map.
+   * @param {*} list - The input array.
+   * @returns {Object} the map.
+   */
+  *mapReduce(keyFunc, valueFunc, list) {
+    const state = {};
+    for (const item of list) {
+      state[keyFunc(item)] = yield new Promise((resolve) =>
+        resolve(valueFunc(item))
+      );
+    }
+    return state;
+  }
+}
+
+module.exports = new Async();

package/lib/batch.js

@@ -9,7 +9,7 @@ exports.run = function (filter, location, callbackAction) {
 
   files.forEach(function (file) {
     var fullPath = path.join(location, file);
-    var st = fs.statSync(fullPath);
+    var st = fs.lstatSync(fullPath);
 
     if (st.isDirectory()) {
       exports.run(filter, fullPath, callbackAction);

package/lib/crypto.js

@@ -1,7 +1,7 @@
 'use strict';
 
 var crypto = require('crypto');
-const uuidV4 = require('uuid/v4');
+const {v4: uuidV4} = require('uuid');
 
 exports.md5 = function (data) {
   var md5 = crypto.createHash('md5');

package/lib/cursorPump.js

@@ -1,3 +1,5 @@
+'use strict';
+
 const watt = require('gigawatts');
 
 class CursorPump {

package/lib/eventDebouncer.js

@@ -1,19 +1,20 @@
+'use strict';
+
 const {debounce} = require('lodash');
 
 class EventDebouncer {
-  constructor(wait = 1000) {
+  constructor(resp, wait = 1000) {
     this.debouncers = {};
     this.wait = wait;
-    const busClient = require('xcraft-core-busclient').getGlobal();
-    this.resp = busClient.newResponse('event-debouncer', 'token');
+    this.resp = resp;
   }
 
-  publish(topic) {
+  publish(topic, data) {
     if (!this.debouncers[topic]) {
-      const send = (topic) => this.resp.events.send(topic);
+      const send = (topic, data) => this.resp.events.send(topic, data);
       this.debouncers[topic] = debounce(send, this.wait);
     }
-    this.debouncers[topic](topic);
+    this.debouncers[topic](topic, data);
   }
 }
 

package/lib/files.js

@@ -1,4 +1,5 @@
 'use strict';
+
 const path = require('path');
 
 const exts = {

package/lib/job-queue.js

@@ -1,35 +1,64 @@
 'use strict';
-var clc = require('cli-color');
 
 const moduleName = 'job-queue';
+const clc = require('cli-color');
 const watt = require('gigawatts');
 const {locks} = require('xcraft-core-utils');
 const runner = require('./runnerInstance.js');
 const throttle = require('lodash/throttle');
+const defaultOptions = {
+  priorityGroup: 'default',
+  waitOn: [], //groups to wait
+  waitDelay: 50,
+  maxAttempt: 100,
+  useLogger: false,
+};
 class JobQueue {
-  constructor(name, runner, parallelLimit, useLogger = true) {
-    this.log = useLogger && require('xcraft-core-log')(`${moduleName}`, null);
+  constructor(name, runner, parallelLimit, options) {
+    //override default options
+    if (!options) {
+      options = defaultOptions;
+    } else {
+      options = {...defaultOptions, ...options};
+    }
+
+    this.log =
+      options.useLogger && require('xcraft-core-log')(`${moduleName}`, null);
     this.name = name;
+    this.priorityGroup = options.priorityGroup;
+    this.waitOn = options.waitOn;
+    this.maxAttempt = options.maxAttempt;
+    this.attempt = 0;
+    this.waitDelay = options.waitDelay;
     this.channel = name.toLowerCase().replace(/\.|\[|\//g, '-');
-    this.runner = watt(runner);
+    this.runner = watt(function* (...args) {
+      this.running++;
+      this.runningSamples++;
+      yield* runner(...args);
+    });
     this.parallelLimit = parallelLimit;
     this.waiting = new Map();
     this.running = 0;
+    this.runningSamples = 0;
     this._mutex = new locks.Mutex();
     const busClient = require('xcraft-core-busclient').getGlobal();
     this.resp = busClient.newResponse('job-queue', 'token');
     watt.wrapAll(this);
-    this.notify = throttle(this._notify, 500);
+    this.notify = throttle(this._notify, 500).bind(this);
   }
 
-  _notify() {
-    const n = this.waiting.size;
+  get maxAttemptReached() {
+    return this.attempt >= this.maxAttempt;
+  }
+
+  _notify(runningSamples) {
     this.resp.events.send('<job-queue.sampled>', {
       channel: this.channel,
-      sample: n,
+      sample: runningSamples,
       current: 0,
       total: 0,
     });
+    this.runningSamples = 0;
   }
 
   _log() {
@@ -37,11 +66,10 @@ class JobQueue {
       return;
     }
 
-    this.log.info(
-      `«${clc.blackBright.bold(this.name)}» waiting:${
-        this.waiting.size
-      } running:${this.running}`
-    );
+    const msg = `«${clc.blackBright.bold(this.name)}» waiting:${
+      this.waiting.size
+    } running:${this.running}`;
+    this.log.info(msg);
   }
 
   _dbg(debugMessage) {
@@ -49,19 +77,34 @@ class JobQueue {
       return;
     }
 
-    this.log.dbg(`«${clc.blackBright.bold(this.name)}» ${debugMessage}`);
+    const msg = `«${clc.blackBright.bold(this.name)}» ${debugMessage}`;
+    this.log.dbg(msg);
+  }
+
+  err(ex) {
+    const msg = `«${clc.blackBright.bold(this.name)}» ${
+      ex.stack || ex.message || ex
+    }`;
+
+    if (!this.log) {
+      console.error(msg);
+      return;
+    }
+
+    this.log.err(msg);
   }
 
   push(job) {
     try {
       this.waiting.set(job.id, job);
     } finally {
-      setImmediate(this.notify.bind(this));
-      setImmediate(runner.instance.run, this);
+      //launch macro-task
+      setTimeout(runner.instance.run, 0, this);
     }
   }
 
   dispose() {
+    this.notify(0);
     this.resp.events.send('<job-queue.disposed>', {
       channel: this.channel,
     });

package/lib/log.js

@@ -64,13 +64,13 @@ exports.decorate = function (mode, prefix, mod, log, maxWidth, stripBegin) {
 
   var beginLength = begin.replace(ansiRegex(), '').length;
   var availableSpace = maxWidth - beginLength - 1;
-  if (availableSpace <= 0) {
+  if (availableSpace <= 1) {
     return util.format('%s%s', begin, text);
   }
 
   let beginEmbedded = '';
   let textNoColor = text.replace(ansiRegex(), '');
-  const embedded = /^[a-zA-Z]+ \[[a-zA-Z/.\-_]+\] (?:Verb|Info|Warn|Err):/.test(
+  const embedded = /^[a-zA-Z]+ \[[a-zA-Z/.\-_]+\] (?:Verb|Info|Warn|Err|Dbg):/.test(
     textNoColor
   );
 
@@ -90,16 +90,17 @@ exports.decorate = function (mode, prefix, mod, log, maxWidth, stripBegin) {
   }
 
   var output = '';
+  let _output = '';
   spaces = new Array(beginLength + 1).join(' ');
 
   if (embedded) {
     const beginEmbeddedNoCol = beginEmbedded.replace(ansiRegex(), '');
     const beginEmbeddedLength = beginEmbeddedNoCol.length;
-    const isSmall = /Err:$/.test(beginEmbeddedNoCol); // only 3 chars
+    const isSmall = /(Err|Dbg):$/.test(beginEmbeddedNoCol); // only 3 chars
     const length = beginLength - beginEmbeddedLength + (isSmall ? -1 : 0);
     const padding = length > 0 ? new Array(length).join(' ') : '';
     if (!stripBegin) {
-      output += `${begin}...\n`;
+      _output += `${begin}...\n`;
     }
     begin = `${padding}${beginEmbedded}` + (isSmall ? ' ' : '');
   }
@@ -141,7 +142,7 @@ exports.decorate = function (mode, prefix, mod, log, maxWidth, stripBegin) {
     begin = spaces;
   });
 
-  return output;
+  return _output + output;
 };
 
 exports.graffiti = function (text, callback) {

package/lib/runnerInstance.js

@@ -1,3 +1,5 @@
+'use strict';
+
 // create a unique, global symbol name
 // -----------------------------------
 const RUNNER_INSTANCE_KEY = Symbol.for('xcraft-core-utils.runnerInstance');
@@ -6,47 +8,99 @@ const watt = require('gigawatts');
 class Runner {
   constructor() {
     this.totalRunning = 0;
+    this.runningsGroups = [];
+    this.runnings = {};
+    this.run = this.run.bind(this);
     watt.wrapAll(this);
   }
 
-  *run(jobQueue, next) {
+  run(jobQueue) {
+    //must be postponed?
+    if (jobQueue.waitOn.length > 0) {
+      const mustWait = this.runningsGroups.some((q) =>
+        jobQueue.waitOn.includes(q)
+      );
+      if (mustWait) {
+        if (!jobQueue.maxAttemptReached) {
+          jobQueue.attempt++;
+          setTimeout(this.run, jobQueue.waitDelay, jobQueue);
+          return;
+        } else {
+          jobQueue.attempt = 0;
+        }
+      }
+    }
+
+    //queue is draining?
     if (jobQueue.running > 0 && jobQueue.waiting.size === 0) {
       return;
     }
-    yield jobQueue._mutex.lock();
-    for (let x = jobQueue.running; x < jobQueue.parallelLimit; x++) {
-      setTimeout(jobQueue._log.bind(jobQueue), 1);
 
+    //jobqueue releasing
+    for (let x = jobQueue.running; x < jobQueue.parallelLimit; x++) {
       const nextEntry = jobQueue.waiting.entries().next();
       if (nextEntry.done) {
+        //we can leave, nothing is waiting us
         break;
       }
+
+      //remove the jobEntry from queue
       const jobEntry = Object.assign({}, nextEntry.value);
       jobQueue.waiting.delete(jobEntry[0]);
 
+      //log for journal inspections
       if (this.totalRunning === 0 && jobQueue.parallelLimit > 1) {
-        jobQueue._dbg(`this queue is now running new jobs`);
+        jobQueue._dbg(`system are running new jobs`);
+      }
+
+      //priority tracking
+      if (jobQueue.priorityGroup !== 'default' && jobQueue.running === 0) {
+        if (!this.runnings[jobQueue.priorityGroup]) {
+          this.runnings[jobQueue.priorityGroup] = {};
+        }
+        this.runnings[jobQueue.priorityGroup][jobQueue.name] = true;
+        //update runnings group
+        this.runningsGroups = Object.entries(this.runnings)
+          .filter((e) => Object.values(e[1]).some((r) => r === true))
+          .map((e) => e[0]);
       }
-      jobQueue.running++;
+
+      //update counters
       this.totalRunning++;
-      setImmediate(() =>
-        jobQueue.runner(jobEntry[1], () => {
-          jobQueue.running--;
-          this.totalRunning--;
-          if (this.totalRunning === 0 && jobQueue.waiting.size === 0) {
-            jobQueue._dbg(
-              `this queue is now empty, and there is no more jobs running`
-            );
-          }
-          setTimeout(jobQueue._log.bind(jobQueue), 1);
-          if (jobQueue.waiting.size > 0) {
-            setImmediate(this.run, jobQueue);
-          }
-        })
-      );
-      yield setTimeout(next, this.totalRunning * 10);
+
+      //trigger job start on queue runner
+      jobQueue.runner(jobEntry[1], (err) => {
+        if (err) {
+          jobQueue.err(err);
+        }
+        //end callback
+
+        jobQueue.notify(jobQueue.runningSamples);
+
+        //update counter
+        jobQueue.running--;
+        this.totalRunning--;
+
+        //priority tracking
+        if (jobQueue.priorityGroup !== 'default' && jobQueue.running === 0) {
+          this.runnings[jobQueue.priorityGroup][jobQueue.name] = false;
+          //update runnings group
+          this.runningsGroups = Object.entries(this.runnings)
+            .filter((e) => Object.values(e[1]).some((r) => r === true))
+            .map((e) => e[0]);
+        }
+
+        //log
+        if (this.totalRunning === 0 && jobQueue.waiting.size === 0) {
+          jobQueue._dbg(`no more jobs running`);
+        }
+
+        //schedule a new run for this queue if needed
+        if (jobQueue.waiting.size > 0) {
+          setTimeout(this.run, 0, jobQueue);
+        }
+      });
     }
-    jobQueue._mutex.unlock();
   }
 }
 

package/lib/whereIs.js

@@ -0,0 +1,13 @@
+const path = require('path');
+const fse = require('fs-extra');
+
+module.exports = function whereIs(bin) {
+  var fullLocation = null;
+
+  var exists = process.env.PATH.split(path.delimiter).some(function (location) {
+    fullLocation = path.join(location, bin);
+    return fse.existsSync(fullLocation);
+  });
+
+  return exists ? fullLocation : null;
+};

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "xcraft-core-utils",
-  "version": "3.1.1",
+  "version": "4.1.0",
   "description": "Xcraft utils",
   "main": "index.js",
   "engines": {
-    "node": ">=6"
+    "node": ">=14"
   },
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
@@ -23,26 +23,28 @@
   "dependencies": {
     "ansi-regex": "^2.1.1",
     "cli-color": "^1.2.0",
-    "escape-regex": "^1.0.7",
+    "escape-regexp": "0.0.1",
     "escape-string-regexp": "^1.0.5",
     "figlet": "^1.2.1",
+    "fs-extra": "^9.1.0",
     "gigawatts": "^4.0.1",
     "js-yaml": "^3.9.0",
     "linked-list": "^1.0.4",
     "lodash": "^4.17.20",
     "locks": "^0.2.2",
     "prop-types": "^15.7.2",
-    "uuid": "^3.3.2",
-    "xcraft-core-fs": "^2.0.0"
-  },
-  "optionalDependencies": {
-    "better-sqlite3": "^5.2.1"
+    "uuid": "^8.3.2",
+    "xcraft-core-busclient": "^5.0.0",
+    "xcraft-core-fs": "^2.0.0",
+    "xcraft-core-log": "^2.0.0",
+    "xcraft-core-utils": "^4.0.0"
   },
   "bugs": {
     "url": "https://github.com/Xcraft-Inc/xcraft-core-utils/issues"
   },
   "homepage": "https://github.com/Xcraft-Inc/xcraft-core-utils#readme",
   "devDependencies": {
+    "should": "^11.2.1",
     "prettier": "2.0.4",
     "xcraft-dev-prettier": "^2.0.0",
     "xcraft-dev-rules": "^2.0.0"

package/lib/persistant-job-queue.js

@@ -1,144 +0,0 @@
-'use strict';
-
-var clc = require('cli-color');
-
-const moduleName = 'job-queue';
-const watt = require('gigawatts');
-const locks = require('xcraft-core-utils/lib/locks.js');
-
-class PersistantJobQueue {
-  constructor(
-    path,
-    name,
-    runner,
-    parallelLimit,
-    useLogger = true,
-    options = {}
-  ) {
-    // Status db
-    const Database = require('better-sqlite3');
-    this.db = new Database(path, options);
-
-    this.db.exec(`
-      CREATE TABLE IF NOT EXISTS JobQueue (
-        jobId TEXT PRIMARY KEY,
-        topic TEXT,
-        seq NUMBER,
-        status TEXT,
-        work TEXT
-      );
-    `);
-
-    this.insertJob = this.db.prepare(`
-      INSERT OR REPLACE INTO JobQueue (jobId,topic,seq,status, work)
-      VALUES ($jobId, $topic, $seq, $status, $work);
-    `);
-
-    this.getJob = this.db.prepare(`
-      SELECT * FROM JobQueue WHERE status='waiting' ORDER BY seq LIMIT 0,1;
-    `);
-
-    this.changeJobStatus = this.db.prepare(`
-      UPDATE JobQueue
-      SET status=$status
-      WHERE jobId=$jobId;
-    `);
-
-    this.deleteJobDone = this.db.prepare(`
-      DELETE FROM JobQueue WHERE jobId=$jobId;
-    `);
-
-    this.getJobsCount = () => {
-      const row = this.db.prepare(
-        `SELECT COUNT(*) as count FROM JobQueue WHERE status='waiting';`
-      );
-
-      const r = row.get();
-      return parseInt(r.count);
-    };
-
-    this.count = this.getJobsCount();
-
-    this.log = useLogger && require('xcraft-core-log')(`${moduleName}`, null);
-    this.name = name;
-    this.runner = runner;
-    this.parallelLimit = parallelLimit;
-    this.running = 0;
-    this.paused = false;
-    this._mutex = new locks.Mutex();
-    watt.wrapAll(this);
-  }
-
-  _log() {
-    if (!this.log) {
-      return;
-    }
-
-    const count = this.getJobsCount();
-
-    /* FIXME: change dbg to info when it's no longer necessary */
-    this.log.dbg(
-      `«${clc.blackBright.bold(this.name)}» waiting:${count} running:${
-        this.running
-      }`
-    );
-  }
-
-  run() {
-    if (this.paused) {
-      return;
-    }
-    if (this.getJobsCount() === 0) {
-      this._log();
-      return;
-    }
-
-    if (this.running >= this.parallelLimit) {
-      return;
-    }
-
-    // Change status running
-    const job = this.getJob.get();
-    this.changeJobStatus.run({jobId: job.jobId, status: 'running'});
-    this.running++;
-    this._log();
-    job.work = JSON.parse(job.work);
-    setImmediate(this.runner, job, () => {
-      this.deleteJobDone.run({jobId: job.jobId});
-      this.running--;
-      this.run();
-    });
-
-    this.run();
-  }
-
-  *push(job) {
-    yield this._mutex.lock();
-    try {
-      this.insertJob.run({
-        jobId: job.id,
-        topic: job.topic || this.name,
-        seq: this.count++,
-        status: 'waiting',
-        work: JSON.stringify(job.work || {}),
-      });
-    } catch (ex) {
-      throw ex;
-    } finally {
-      this._log();
-      this._mutex.unlock();
-      this.run();
-    }
-  }
-
-  resume() {
-    this.paused = false;
-    this.run();
-  }
-
-  pause() {
-    this.paused = true;
-  }
-}
-
-module.exports = PersistantJobQueue;

package/lib/sqlite.js

@@ -1,141 +0,0 @@
-'use strict';
-
-const path = require('path');
-const xFs = require('xcraft-core-fs');
-
-class SQLite {
-  constructor(location, skip) {
-    this._stmts = {};
-    this._db = {};
-    this._dir = location;
-    SQLite.prototype._init.call(this, skip);
-  }
-
-  _init(skip) {
-    try {
-      this.Database = skip ? null : require('better-sqlite3');
-    } catch (ex) {
-      /* ... */
-    }
-  }
-
-  _path(dbName) {
-    return path.join(this._dir, `${dbName}.db`);
-  }
-
-  _onError(resp) {
-    resp.log.info('sqlite3 is not supported on this platform');
-  }
-
-  _prepare(dbName, query, sql) {
-    this._stmts[dbName][query] = this._db[dbName].prepare(sql);
-  }
-
-  stmts(dbName) {
-    return this._stmts[dbName];
-  }
-
-  getLocation() {
-    return this._dir;
-  }
-
-  setEnable(en) {
-    SQLite.prototype._init.call(this, !en);
-
-    if (!en) {
-      Object.keys(this._db).forEach((db) => this.close(db));
-    }
-  }
-
-  /**
-   * Check if SQLite is usable.
-   *
-   * @return {Boolean} true if SQLite is available.
-   */
-  usable() {
-    return !!this.Database;
-  }
-
-  /**
-   *
-   * @param {Object} resp - Response object provided by busClient.
-   * @returns {Boolean} true if usable.
-   */
-  tryToUse(resp) {
-    if (!this.usable()) {
-      SQLite.prototype._onError.call(this, resp);
-      return false;
-    }
-    return true;
-  }
-
-  timestamp() {
-    return new Date().toISOString();
-  }
-
-  /**
-   * Open (and create if necessary) a SQLite database.
-   *
-   * @param {string} dbName - Database name used for the database file.
-   * @param {string} tables - Main queries for creating the tables.
-   * @param {Object} queries - Raw queries to prepare.
-   * @param {function} onOpen - Callback just after opening the database.
-   * @param {function} onMigrate - Callback for migrations.
-   * @return {Boolean} false if SQLite is not available.
-   */
-  open(dbName, tables, queries, onOpen, onMigrate) {
-    if (!this.usable()) {
-      return false;
-    }
-
-    if (this._db[dbName]) {
-      return true;
-    }
-
-    xFs.mkdir(this._dir);
-
-    const dbPath = this._path(dbName);
-    this._db[dbName] = new this.Database(dbPath, {timeout: 30000});
-    this._stmts[dbName] = {};
-
-    if (onOpen) {
-      onOpen();
-    }
-
-    this._db[dbName].exec(tables);
-
-    if (onMigrate) {
-      onMigrate();
-    }
-
-    for (const query in queries) {
-      SQLite.prototype._prepare.call(this, dbName, query, queries[query]);
-    }
-
-    return true;
-  }
-
-  close(dbName) {
-    if (!this._db[dbName]) {
-      return;
-    }
-    this._db[dbName].close();
-    delete this._db[dbName];
-  }
-
-  exec(dbName, query) {
-    if (!this.usable() || !this._db[dbName]) {
-      return false;
-    }
-    this._db[dbName].exec(query);
-  }
-
-  pragma(dbName, pragma) {
-    if (!this.usable() || !this._db[dbName]) {
-      return false;
-    }
-    return this._db[dbName].pragma(pragma, {simple: true});
-  }
-}
-
-module.exports = SQLite;