zsyp 1.1.0 → 1.2.2

package/index.js

@@ -1,15 +1,12 @@
 require('dotenv').config({ path: '/etc/default/zsyp' });
 
-const logger = require('./lib/logger');
 const makeApp = require('./lib/app');
 
 const {
   ZSYP_PORT: PORT = 3090,
 } = process.env;
 
-const app = makeApp({
-  logger
-});
+const app = makeApp();
 
 module.exports = app;
 

package/lib/app.js

@@ -1,20 +1,25 @@
 const connect = require('@pirxpilot/connect');
+const mniam = require('mniam');
 const { json } = require('body-parser');
 const router = require('./router');
+const event = require('./event');
 
 module.exports = makeApp;
 
 const {
   ZSYP_DOMAINS: domains,
+  ZSYP_DB: database = 'mongodb://localhost/zsyp'
 } = process.env;
 
-
-function makeApp(opts) {
+function makeApp(opts = {}) {
   const app = connect();
 
+  opts.db = mniam.db(database);
+  app.db = opts.db;
+
   app.use(json({ limit: 5000, type: ['*/json', 'application/csp-report'] }));
   app.use('/csp', router({ ...opts, name: 'csp', domains }));
-  app.use('/event', router({ ...opts, name: 'event' }));
+  app.use('/event', router({ ...opts, converter: event.converter }));
 
   return app;
 }

package/lib/converter.js

@@ -0,0 +1,23 @@
+module.exports = makeConverter;
+
+function makeConverter({ converter }) {
+
+  async function convert(req, res, next) {
+    try {
+      const { item, meta } = await converter(req.body);
+      req.item = item;
+      req.meta = meta;
+      return next();
+    } catch(e) {
+      return next(e);
+    }
+  }
+
+  return converter ? convert : keep;
+}
+
+function keep(req, res, next) {
+  req.item = req.body;
+  return next();
+}
+

package/lib/event.js

@@ -0,0 +1,75 @@
+const { ObjectId } = require('mongodb');
+const { createHash } = require('node:crypto');
+const stackParser = require('error-stack-parser');
+const { resolve } = require('./source-map');
+
+module.exports = {
+  converter
+};
+
+async function converter(event) {
+  const { type = 'event' } = event;
+  let item;
+  let name;
+  switch (type) {
+    case 'error':
+    case 'exception':
+      name = 'error';
+      item = await convertError(event);
+      break;
+    default:
+      name = 'event';
+      item = event;
+      break;
+  }
+  return {
+    item,
+    meta: { name }
+  };
+}
+
+async function convertError(error) {
+  const { an, av, stack } = error;
+  const frames = safeParseStack(error).map(mapFrame);
+  error.stack = await Promise.all(frames);
+  delete error.type;
+  if (stack) {
+    // save original stack if present
+    error.org_stack = stack;
+  }
+  return addHash(error);
+
+  function safeParseStack(error) {
+    try {
+      return stackParser.parse(error);
+    } catch {
+      return [];
+    }
+  }
+
+  async function mapFrame(frame) {
+    const { fileName, lineNumber, columnNumber = 0, functionName } = frame;
+    const resolved = await resolve({ an, av }, [normalizeFilename(fileName), lineNumber, columnNumber]);
+    resolved.splice(3, 0, functionName);
+    return resolved;
+  }
+}
+
+function normalizeFilename(fileName) {
+  const file = new URL(fileName, 'http://localhost');
+  // strip hostname and leading slashes
+  return file.pathname.replace(/^\/+/, '');
+}
+
+function addHash(error) {
+  if (error.stack.length < 1) {
+    return error;
+  }
+  const hasher = createHash('shake128', { outputLength: 12 });
+  error.stack.some(([file, line], index) => {
+    hasher.update(`${file}:${line}`);
+    return index > 9; // only hash at most 10 stack lines
+  });
+  error._hash = new ObjectId(hasher.digest());
+  return error;
+}

package/lib/logger.js

@@ -1,26 +1,28 @@
-const mniam = require('mniam');
 const debug = require('debug')('zsyp:logger');
 
 module.exports = makeLogger;
 
-const {
-  ZSYP_DB: database = 'mongodb://localhost/zsyp'
-} = process.env;
+function makeLogger({ db, name }) {
+  const cache = Object.create(null);
 
-
-const db = mniam.db(database);
-
-function makeLogger({ name }) {
-  const collection = db.collection({ name });
+  const getCollection =  name ?
+    () => collectionFromCache(name) :
+    ({ name }) => collectionFromCache(name);
 
   return log;
 
-  function log(report) {
-    debug('saving %j', report);
-    collection.insertOne(report, function(err) {
-      if (err) {
-        console.error(err);
-      }
-    });
+  function log({ from, item, meta }, res, next) {
+    debug('saving %j', item);
+    getCollection(meta)
+      .insertOne({ ...item, from })
+      .then(() => next())
+      .catch(err => { console.error(err); next(); });
+  }
+
+  function collectionFromCache(name) {
+    if (!cache[name]) {
+      cache[name] = db.collection({ name });
+    }
+    return cache[name];
   }
 }

package/lib/router.js

@@ -1,14 +1,9 @@
 const Router = require('router');
-const debug = require('debug')('zsyp:router');
 
 const from = require('./from');
 const makeFilter = require('./filter');
-
-function respond(req, res, next) {
-  res.statusCode = 204; // empty
-  res.end();
-  next();
-}
+const makeConverter = require('./converter');
+const makeLogger = require('./logger');
 
 module.exports = function (opts) {
   const router = new Router({
@@ -17,28 +12,21 @@ module.exports = function (opts) {
   });
 
   const stack = [
+    respond,
     from,
-    respond
-  ];
-
-  const filter = makeFilter(opts);
-  if (filter) {
-    stack.push(filter);
-  }
-
-  const logger = opts.logger(opts);
+    makeFilter(opts),
+    makeConverter(opts),
+    makeLogger(opts),
+    opts.finalMiddleware
+  ].filter(Boolean);
 
-  router.post('/', ...stack, log);
+  router.post('/', stack);
 
   return router;
-
-  function log({ from, body }) {
-    const report = {
-      ...body,
-      from
-    };
-
-    debug('Logging report %j', report);
-    logger(report);
-  }
 };
+
+function respond(req, res, next) {
+  res.statusCode = 204; // empty
+  res.end();
+  next();
+}

package/lib/source-map.js

@@ -0,0 +1,57 @@
+const { readFile } = require('fs').promises;
+const path = require('path');
+const { SourceMapConsumer } = require('source-map');
+const LRU = require('lru-cache');
+
+module.exports = {
+  resolve,
+  clear
+};
+
+const {
+  ZSYP_SOURCE_MAP_DIR = '/var/lib/zsyp',
+  ZSYP_SOURCE_MAP_CACHE_SIZE = 30
+} = process.env;
+
+
+const cache = new LRU({
+  max: ZSYP_SOURCE_MAP_CACHE_SIZE,
+  fetchMethod: fetchSourceMap,
+  dispose: smc => smc && smc.destroy()
+});
+
+async function resolve({ an, av }, frame) {
+  const [source, line, column ] = frame;
+  const smc = await loadSourceMap(an, av, source);
+  if (!smc) {
+    return frame;
+  }
+  const pos = smc.originalPositionFor({ line, column });
+  if (!pos) {
+    return frame;
+  }
+  const resolved = [pos.source, pos.line, pos.column];
+  if (pos.name) {
+    resolved.push(pos.name);
+  }
+  return resolved;
+}
+
+function clear() {
+  cache.clear();
+}
+
+function loadSourceMap(app, version, source) {
+  const filename = path.resolve(ZSYP_SOURCE_MAP_DIR, app, version, source);
+  return cache.fetch(filename);
+}
+
+async function fetchSourceMap(filename) {
+  try {
+    const txt = await readFile(filename + '.map');
+    const map = JSON.parse(txt);
+    return new SourceMapConsumer(map);
+  } catch {
+    // ignore errors during map parsing
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zsyp",
-  "version": "1.1.0",
+  "version": "1.2.2",
   "description": "CSP violation reports logger.",
   "author": {
     "name": "Damian Krzeminski",
@@ -20,8 +20,11 @@
     "body-parser": "~1",
     "debug": "~2 || ~3 || ~4",
     "dotenv": "~16",
-    "mniam": "^2.1.0",
+    "error-stack-parser": "^2.1.4",
+    "lru-cache": "^7.10.1",
+    "mniam": "^3.0.0",
     "router": "~1",
+    "source-map": "^0.7.4",
     "supertest": "~6",
     "useragent": "^2.3.0"
   },