zsyp 1.0.0 → 1.2.0

package/Readme.md

@@ -1,7 +1,6 @@
 [![NPM version][npm-image]][npm-url]
-[![Build Status][travis-image]][travis-url]
+[![Build Status][build-image]][build-url]
 [![Dependency Status][deps-image]][deps-url]
-[![Dev Dependency Status][deps-dev-image]][deps-dev-url]
 
 # zsyp
 
@@ -72,14 +71,11 @@ MIT © [Damian Krzeminski](https://pirxpilot.me)
 [capped collection]: https://docs.mongodb.com/manual/core/capped-collections/
 [dotenv]: https://www.npmjs.com/package/dotenv
 
-[npm-image]: https://img.shields.io/npm/v/zsyp.svg
+[npm-image]: https://img.shields.io/npm/v/zsyp
 [npm-url]: https://npmjs.org/package/zsyp
 
-[travis-url]: https://travis-ci.com/pirxpilot/zsyp
-[travis-image]: https://img.shields.io/travis/com/pirxpilot/zsyp.svg
+[build-url]: https://github.com/pirxpilot/zsyp/actions/workflows/check.yaml
+[build-image]: https://img.shields.io/github/workflow/status/pirxpilot/zsyp/check
 
-[deps-image]: https://img.shields.io/david/pirxpilot/zsyp.svg
-[deps-url]: https://david-dm.org/pirxpilot/zsyp
-
-[deps-dev-image]: https://img.shields.io/david/dev/pirxpilot/zsyp.svg
-[deps-dev-url]: https://david-dm.org/pirxpilot/zsyp?type=dev
+[deps-image]: https://img.shields.io/librariesio/release/npm/zsyp
+[deps-url]: https://libraries.io/npm/zsyp

package/index.js

@@ -1,24 +1,12 @@
 require('dotenv').config({ path: '/etc/default/zsyp' });
 
-const connect = require('@pirxpilot/connect');
-const router = require('./lib/csp');
-const makeLogger = require('./lib/logger');
+const makeApp = require('./lib/app');
 
 const {
   ZSYP_PORT: PORT = 3090,
-  ZSYP_DOMAINS: domains,
-  ZSYP_DB: database = 'mongodb://localhost/zsyp'
 } = process.env;
 
-const app = connect();
-const log = makeLogger({ database });
-
-app.use(function (req, res, next) {
-  req.log = log;
-  next();
-});
-app.use(router({ domains }));
-
+const app = makeApp();
 
 module.exports = app;
 

package/lib/app.js

@@ -0,0 +1,26 @@
+const connect = require('@pirxpilot/connect');
+const mniam = require('mniam');
+const { json } = require('body-parser');
+const router = require('./router');
+const event = require('./event');
+
+module.exports = makeApp;
+
+const {
+  ZSYP_DOMAINS: domains,
+  ZSYP_DB: database = 'mongodb://localhost/zsyp'
+} = process.env;
+
+function makeApp(opts = {}) {
+  const app = connect();
+
+  opts.db = mniam.db(database);
+  app.db = opts.db;
+
+  app.use(json({ limit: 5000, type: ['*/json', 'application/csp-report'] }));
+  app.use('/csp', router({ ...opts, name: 'csp', domains }));
+  app.use('/event', router({ ...opts, converter: event.converter }));
+
+  return app;
+}
+

package/lib/converter.js

@@ -0,0 +1,23 @@
+module.exports = makeConverter;
+
+function makeConverter({ converter }) {
+
+  async function convert(req, res, next) {
+    try {
+      const { item, meta } = await converter(req.body);
+      req.item = item;
+      req.meta = meta;
+      return next();
+    } catch(e) {
+      return next(e);
+    }
+  }
+
+  return converter ? convert : keep;
+}
+
+function keep(req, res, next) {
+  req.item = req.body;
+  return next();
+}
+

package/lib/event.js

@@ -0,0 +1,56 @@
+const stackParser = require('error-stack-parser');
+const { resolve } = require('./source-map');
+
+module.exports = {
+  converter
+};
+
+async function converter(event) {
+  const { type = 'event' } = event;
+  let item;
+  let name;
+  switch (type) {
+    case 'error':
+    case 'exception':
+      name = 'error';
+      item = await convertError(event);
+      break;
+    default:
+      name = 'event';
+      item = event;
+      break;
+  }
+  return {
+    item,
+    meta: { name }
+  };
+}
+
+async function convertError(error) {
+  const { an, av } = error;
+  const frames = safeParseStack(error).map(mapFrame);
+  error.stack = await Promise.all(frames);
+  delete error.type;
+  return error;
+
+  function safeParseStack(error) {
+    try {
+      return stackParser.parse(error);
+    } catch {
+      return [];
+    }
+  }
+
+  async function mapFrame(frame) {
+    const { fileName, lineNumber, columnNumber = 0, functionName } = frame;
+    const resolved = await resolve({ an, av }, [normalizeFilename(fileName), lineNumber, columnNumber]);
+    resolved.splice(3, 0, functionName);
+    return resolved;
+  }
+}
+
+function normalizeFilename(fileName) {
+  const file = new URL(fileName, 'http://localhost');
+  // strip hostname and leading slashes
+  return file.pathname.replace(/^\/+/, '');
+}

package/lib/filter.js

@@ -4,18 +4,17 @@ module.exports = makeFilter;
 
 function makeFilter({ domains }) {
 
-  const domainRe = domains && new RegExp(domains);
+  if (!domains) {
+    return;
+  }
 
+  const domainRe = domains && new RegExp(domains);
   return filter;
 
   function filter({ body: csp }, res, next) {
-    if (!domainRe) {
-      return next();
-    }
     const uri = csp['csp-report']['document-uri'];
     const { hostname } = new URL(uri);
     if (domainRe.test(hostname)) {
-      console.log('OK!!!!!!!!');
       return next();
     }
   }

package/lib/from.js

@@ -3,7 +3,8 @@ const { parse } = require('useragent');
 module.exports = from;
 
 function from(req, res, next) {
-  const ua = req.headers['user-agent'];
+  const { headers, body } = req;
+  const ua = body?.from?.ua ??  headers['user-agent'];
   const {
     family,
     major,
@@ -20,7 +21,7 @@ function from(req, res, next) {
       name: os.family,
       version: os.major
     },
-    ip: req.headers['x-forwarded-for'] || req.connection.remoteAddress
+    ip: body?.from?.ip ?? headers['x-forwarded-for'] ?? req.connection.remoteAddress
   };
   if (device.family !== 'Other') {
     data.device = device.family;

package/lib/logger.js

@@ -1,19 +1,28 @@
-const { db } = require('mniam');
 const debug = require('debug')('zsyp:logger');
 
 module.exports = makeLogger;
 
-function makeLogger({ database }) {
-  const csp = db(database).collection({ name: 'csp'});
+function makeLogger({ db, name }) {
+  const cache = Object.create(null);
+
+  const getCollection =  name ?
+    () => collectionFromCache(name) :
+    ({ name }) => collectionFromCache(name);
 
   return log;
 
-  function log(report) {
-    debug('saving %j', report);
-    csp.insertOne(report, function(err) {
-      if (err) {
-        console.error(err);
-      }
-    });
+  function log({ from, item, meta }, res, next) {
+    debug('saving %j', item);
+    getCollection(meta)
+      .insertOne({ from, ...item })
+      .then(() => next())
+      .catch(err => { console.error(err); next(); });
+  }
+
+  function collectionFromCache(name) {
+    if (!cache[name]) {
+      cache[name] = db.collection({ name });
+    }
+    return cache[name];
   }
 }

package/lib/router.js

@@ -0,0 +1,32 @@
+const Router = require('router');
+
+const from = require('./from');
+const makeFilter = require('./filter');
+const makeConverter = require('./converter');
+const makeLogger = require('./logger');
+
+module.exports = function (opts) {
+  const router = new Router({
+    strict: true,
+    caseSensitive: true
+  });
+
+  const stack = [
+    respond,
+    from,
+    makeFilter(opts),
+    makeConverter(opts),
+    makeLogger(opts),
+    opts.finalMiddleware
+  ].filter(Boolean);
+
+  router.post('/', stack);
+
+  return router;
+};
+
+function respond(req, res, next) {
+  res.statusCode = 204; // empty
+  res.end();
+  next();
+}

package/lib/source-map.js

@@ -0,0 +1,57 @@
+const { readFile } = require('fs').promises;
+const path = require('path');
+const { SourceMapConsumer } = require('source-map');
+const LRU = require('lru-cache');
+
+module.exports = {
+  resolve,
+  clear
+};
+
+const {
+  ZSYP_SOURCE_MAP_DIR = '/var/lib/zsyp',
+  ZSYP_SOURCE_MAP_CACHE_SIZE = 30
+} = process.env;
+
+
+const cache = new LRU({
+  max: ZSYP_SOURCE_MAP_CACHE_SIZE,
+  fetchMethod: fetchSourceMap,
+  dispose: smc => smc && smc.destroy()
+});
+
+async function resolve({ an, av }, frame) {
+  const [source, line, column ] = frame;
+  const smc = await loadSourceMap(an, av, source);
+  if (!smc) {
+    return frame;
+  }
+  const pos = smc.originalPositionFor({ line, column });
+  if (!pos) {
+    return frame;
+  }
+  const resolved = [pos.source, pos.line, pos.column];
+  if (pos.name) {
+    resolved.push(pos.name);
+  }
+  return resolved;
+}
+
+function clear() {
+  cache.clear();
+}
+
+function loadSourceMap(app, version, source) {
+  const filename = path.resolve(ZSYP_SOURCE_MAP_DIR, app, version, source);
+  return cache.fetch(filename);
+}
+
+async function fetchSourceMap(filename) {
+  try {
+    const txt = await readFile(filename + '.map');
+    const map = JSON.parse(txt);
+    return new SourceMapConsumer(map);
+  } catch {
+    // ignore errors during map parsing
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zsyp",
-  "version": "1.0.0",
+  "version": "1.2.0",
   "description": "CSP violation reports logger.",
   "author": {
     "name": "Damian Krzeminski",
@@ -19,14 +19,18 @@
     "@pirxpilot/connect": "^4.0.0",
     "body-parser": "~1",
     "debug": "~2 || ~3 || ~4",
-    "dotenv": "~8",
-    "mniam": "^2.1.0",
+    "dotenv": "~16",
+    "error-stack-parser": "^2.1.4",
+    "lru-cache": "^7.10.1",
+    "mniam": "^3.0.0",
     "router": "~1",
+    "source-map": "^0.7.4",
+    "supertest": "~6",
     "useragent": "^2.3.0"
   },
   "devDependencies": {
     "jshint": "~2",
-    "tape": "~4"
+    "tape": "~5"
   },
   "scripts": {
     "test": "make check"

package/History.md

@@ -1,17 +0,0 @@
-
-1.0.0 / 2020-04-18
-==================
-
- * update docs
- * update links to Travis build results
- * update dependencies
-
-0.0.2 / 2019-07-17
-==================
-
- * fix reports domain filtering
-
-0.0.1 / 2019-07-15
-==================
-
- * basic implementation: UA parsing, domain filtering, mongo backend

package/lib/csp.js

@@ -1,36 +0,0 @@
-const Router = require('router');
-const { json } = require('body-parser');
-
-const from = require('./from');
-const makeFilter = require('./filter');
-
-function respond(req, res, next) {
-  res.statusCode = 204; // empty
-  res.end();
-  next();
-}
-
-
-function log({ from, body: csp, log }) {
-  const report = {
-    from,
-    ...csp
-  };
-
-  log(report);
-}
-
-module.exports = function (opts) {
-  const router = new Router();
-  const filter = makeFilter(opts);
-
-  router.post('/csp',
-    from,
-    json({ limit: 5000, type: [ '*/json', 'application/csp-report' ] }),
-    respond,
-    filter,
-    log
-  );
-
-  return router;
-};