profoundjs-swagger-stats 1.1.3 → 2.0.0

package/examples/testapp/testapp.js

@@ -1,5 +1,3 @@
-'use strict';
-
 var http = require('http');
 var path = require('path');
 var debug = require('debug')('sws:testapp');
@@ -10,11 +8,8 @@ var server = null;
 // Express and middlewares
 var express = require('express');
 var expressBodyParser = require('body-parser');
-var expressFavicon = require('serve-favicon');
-var expressStatic = require('serve-static');
 
-var swaggerJSDoc = require('swagger-jsdoc');
-var swaggerParser = require('swagger-parser');
+const swaggerParser = require('swagger-parser');
 
 var swStats = require('../../lib');    // require('swagger-stats');
 
@@ -22,11 +17,6 @@ var swStats = require('../../lib');    // require('swagger-stats');
 var API = require('./api');
 
 var app = module.exports = express();
-app.use(expressFavicon(path.join(__dirname, '../../ui/favicon.png')));
-app.use('/ui',expressStatic(path.join(__dirname, '../../ui')));
-app.use('/ui/dist',expressStatic(path.join(__dirname, '../../dist')));
-app.use('/node_modules',expressStatic(path.join(__dirname, '../../node_modules')));
-app.use('/node_modules',expressStatic(path.join(__dirname, '../../node_modules')));
 app.use(expressBodyParser.json()); // for parsing application/json
 app.use(expressBodyParser.urlencoded({ extended: true })); // for parsing application/x-www-form-urlencoded
 
@@ -37,7 +27,7 @@ app.set('port', process.env.PORT || 3040);
 app.disable('etag');
 
 app.get('/', function(req,res) {
-    res.redirect('/ui');
+    res.redirect('/swagger-stats/');
 });
 
 app.get('/apidoc.json', function(req,res){
@@ -50,66 +40,20 @@ if( process.env.SWS_TEST_TIMEBUCKET ){
     tlBucket = parseInt(process.env.SWS_TEST_TIMEBUCKET);
 }
 
-// SWAGGER-JSDOC Initialization //
-var apifile = path.join(__dirname,'api.js');
-debug('initializing swagger spec from api file %s',apifile);
-var swOptions = {
-    swaggerDefinition: {
-        "info": {
-            "description": "This is a Petstore API implementation for swagger-stats sample app",
-            "version": "1.0.0",
-            "title": "Swagger-Stats Petstore ",
-            "contact": {
-                "email": "sv2@slanatech.com"
-            },
-            "license": {
-                "name": "MIT"
-            }
-        },
-        "host": "localhost",
-        "basePath": "/api/v1",
-        "tags": [
-            {
-                "name": "pet",
-                "description": "Everything about your Pets",
-                "externalDocs": {
-                    "description": "Find out more",
-                    "url": "http://swaggerstats.io"
-                }
-            },
-            {
-                "name": "store",
-                "description": "Access to Petstore orders"
-            },
-            {
-                "name": "user",
-                "description": "Operations about user",
-                "externalDocs": {
-                    "description": "Find out more about our store",
-                    "url": "http://swaggerstats.io"
-                }
-            }
-        ],
-        "schemes": ["http"]
-    },
-    apis: [apifile]  // Path to the API files with swagger docs in comments
-};
-
-// Initialize swagger-jsdoc -> returns validated swagger spec in json format
-var swaggerSpec = swaggerJSDoc(swOptions);
+const swaggerSpec = require('./petstore.json');
 
 // Testing validation of 3rd-party API spec
-var parser = new swaggerParser();
+const parser = new swaggerParser();
 
 parser.validate(swaggerSpec,function(err, api) {
     if (!err) {
         debug('Success validating swagger file!');
-        swaggerSpec = api;
+        //swaggerSpec = api;
 
         // Enable swagger-stats middleware
         app.use(swStats.getMiddleware({
             name: 'swagger-stats-testapp',
-            version: '0.95.17',
+            version: '0.99.7',
             timelineBucketDuration: tlBucket,
             uriPath: '/swagger-stats',
             swaggerSpec:swaggerSpec,

package/lib/sws-api-swagger.yaml

@@ -1,6 +1,6 @@
 swagger: '2.0'
 info:
-  version: 0.95.17
+  version: 0.99.7
   title: swagger-stats API
   description: |
     ### Telemetry for your APIs

package/lib/swsAPIStats.js

@@ -185,6 +185,7 @@ swsAPIStats.prototype.addPathSpecs = function (pathSpecs) {
         if( fullExpressPath.indexOf('{') !== -1 ) {
             fullExpressPath = fullExpressPath.replace(/\{/g, ':');
             fullExpressPath = fullExpressPath.replace(/\}/g, '');
+            fullExpressPath = fullExpressPath.replace(/\?(\w+=)/g, '\\?$1');
             re = pathToRegexp(fullExpressPath, keys);
         }
 
@@ -332,6 +333,10 @@ swsAPIStats.prototype.extractPathParams = function (matchResult,keys) {
 swsAPIStats.prototype.matchRequest = function (req) {
 
     var url = req.sws.originalUrl;
+    // Handle "/pets" and "/pets/" the same way - #105
+    if( url.endsWith('/') ) {
+        url = url.slice(0,-1);
+    }
 
     req.sws.match = false;  // No match by default
 
@@ -351,7 +356,7 @@ swsAPIStats.prototype.matchRequest = function (req) {
         matchEntry = this.apiMatchIndex[url];
         apiPath = url;
         debug('SWS:MATCH: %s exact match', url);
-    }else{
+    } else {
         // if not, search by regex matching
         for(var swPath in this.apiMatchIndex) {
             if( this.apiMatchIndex[swPath].re !== null) {

package/lib/swsAuth.js

@@ -0,0 +1,140 @@
+const swsSettings = require('./swssettings');
+const debug = require('debug')('sws:auth');
+const basicAuth = require("basic-auth");
+const Cookies = require('cookies');
+const { v1: uuidv1 } = require('uuid');
+
+/* Authentication */
+class SwsAuth {
+
+    constructor() {
+        this.sessionIDs = {};
+        this.expireIntervalId = null;
+    }
+
+    storeSessionID(sid){
+        // PJS - Create sid value if not passed
+        sid = sid || uuidv1();
+
+        let tsSec = Date.now() + swsSettings.sessionMaxAge*1000;
+        this.sessionIDs[sid] = tsSec;
+        //debug('Session ID updated: %s=%d', sid,tssec);
+        if( !this.expireIntervalId ){
+            this.expireIntervalId = setInterval(() => {this.expireSessionIDs();},500);
+        }
+
+        // PJS - Send back sid value
+        return sid;
+    }
+
+    removeSessionID(sid){
+        delete this.sessionIDs[sid];
+    }
+
+    // If authentication is enabled, executed periodically and expires old session IDs
+    expireSessionIDs(){
+        let tssec = Date.now();
+        let expired = [];
+        for(let sid in this.sessionIDs){
+            if(this.sessionIDs[sid] < (tssec + 500)){
+                expired.push(sid);
+            }
+        }
+        for(let i=0;i<expired.length;i++){
+            delete this.sessionIDs[expired[i]];
+            debug('Session ID expired: %s', expired[i]);
+        }
+    }
+
+    async processAuth(req,res) {
+
+        if( !swsSettings.authentication ){
+            return true;
+        }
+
+        if( swsSettings.customAuth ){
+            // return swsSettings.customAuth(req);
+            // PJS Custom
+            return swsSettings.customAuth(req, res);
+        }
+
+        let cookies = new Cookies( req, res );
+
+        // Check session cookie
+        let sessionIdCookie = cookies.get('sws-session-id');
+        if( (sessionIdCookie !== undefined) && (sessionIdCookie !== null) ){
+
+            if( sessionIdCookie in this.sessionIDs ){
+                // renew it
+                //sessionIDs[sessionIdCookie] = Date.now();
+                this.storeSessionID(sessionIdCookie);
+                cookies.set('sws-session-id',sessionIdCookie,{path:swsSettings.basePath+swsSettings.uriPath,maxAge:swsSettings.sessionMaxAge*1000});
+                // Ok
+                req['sws-auth'] = true;
+                return true;
+            }
+        }
+
+        let authInfo = basicAuth(req);
+
+        let authenticated = false;
+        let msg = 'Authentication required';
+
+        if( (authInfo !== undefined) && (authInfo!==null) && ('name' in authInfo) && ('pass' in authInfo)){
+            if(typeof swsSettings.onAuthenticate === 'function'){
+
+                let onAuthResult = null;
+                try{
+                    onAuthResult = await swsSettings.onAuthenticate(req, authInfo.name, authInfo.pass);
+                }catch (e){
+                    msg = `Authentication error: ${e.message}`;
+                    res.statusCode = 403;
+                    res.end(msg);
+                    return false;
+                }
+                if( onAuthResult ){
+                    authenticated = true;
+                    // Session is only for stats requests
+                    if(req.url.startsWith(swsSettings.pathStats)){
+                        // Generate session id
+                        let sessid = uuidv1();
+                        this.storeSessionID(sessid);
+                        // Set session cookie with expiration in 15 min
+                        cookies.set('sws-session-id',sessid,{path:swsSettings.basePath+swsSettings.uriPath,maxAge:swsSettings.sessionMaxAge*1000});
+                    }
+                    req['sws-auth'] = true;
+                    return true;
+                } else {
+                    res.statusCode = 403;
+                    res.end(msg);
+                    return false;
+                }
+            }else{
+                res.statusCode = 403;
+                res.end(msg);
+                return false;
+            }
+        }else{
+            res.statusCode = 403;
+            res.end(msg);
+            return false;
+        }
+    }
+
+    processLogout(req,res){
+        let cookies = new Cookies( req, res );
+        let sessionIdCookie = cookies.get('sws-session-id');
+        if( (sessionIdCookie !== undefined) && (sessionIdCookie !== null) ){
+            if( sessionIdCookie in this.sessionIDs ){
+                this.removeSessionID(sessionIdCookie);
+                cookies.set('sws-session-id'); // deletes cookie
+            }
+        }
+        res.statusCode = 200;
+        res.end('Logged out');
+    }
+
+}
+
+let swsAuth = new SwsAuth();
+module.exports = swsAuth;

package/lib/swsElasticEmitter.js

@@ -4,17 +4,29 @@
 
 'use strict';
 
-var os = require('os');
-var util = require('util');
-var http = require('http');
-var url = require('url');
-var request = require('request');
-
-var debug = require('debug')('sws:elastic');
-var swsUtil = require('./swsUtil');
-var moment = require('moment');
-
-var indexTemplate = require('../schema/elasticsearch/api_index_template.json');
+const os = require('os');
+const util = require('util');
+const http = require('http');
+const url = require('url');
+const https = require('https');
+//const axios = require('axios');
+let axios = null;
+let axiosPromise = new Promise(async (resolve) => {
+    axios = (await import('axios')).default;
+    resolve(axios);
+});
+/*
+(async () => {
+    axios = (await import('axios')).default;
+})();
+*/
+
+const debug = require('debug')('sws:elastic');
+const swsUtil = require('./swsUtil');
+const moment = require('moment');
+
+const indexTemplate = require('../schema/elasticsearch/api_index_template.json');
+const indexTemplate7X = require('../schema/elasticsearch/api_index_template_7x.json');
 
 
 const ES_MAX_BUFF = 50;
@@ -25,6 +37,9 @@ function swsElasticEmitter() {
     // Options
     this.options = null;
 
+    this.es7 = false;
+    this.es8 = true;
+
     this.indexBuffer = '';
     this.bufferCount = 0;
     this.lastFlush = 0;
@@ -38,6 +53,9 @@ function swsElasticEmitter() {
     this.elasticUsername = null;
     this.elasticPassword = null;
 
+    this.elasticsearchCert = null;
+    this.elasticsearchKey = null
+
     this.indexPrefix = "api-";
 
     this.enabled = false;
@@ -78,11 +96,26 @@ swsElasticEmitter.prototype.initialize = function (swsOptions) {
         this.elasticPassword = swsOptions[swsUtil.supportedOptions.elasticsearchPassword];
     }
 
-    // Check / Initialize schema
-    this.initTemplate();
+    if(swsUtil.supportedOptions.elasticsearchCert in swsOptions) {
+        this.elasticsearchCert = swsOptions[swsUtil.supportedOptions.elasticsearchCert]
+    }
 
-    this.enabled = true;
+    if( swsUtil.supportedOptions.elasticsearchKey in swsOptions) {
+        this.elasticsearchKey = swsOptions[swsUtil.supportedOptions.elasticsearchKey]
+    }
+
+    if(this.elasticsearchKey && this.elasticsearchCert){
+        axios.defaults.httpsAgent = new https.Agent({
+            rejectUnauthorized: false, // (NOTE: this will disable client verification)
+            cert: this.elasticsearchCert,
+            key: this.elasticsearchKey,
+        });
+    }
 
+    // Check / Initialize schema
+    Promise.resolve(axiosPromise).then( () => {
+        this.initTemplate();
+    });
 };
 
 // initialize index template
@@ -90,50 +123,70 @@ swsElasticEmitter.prototype.initTemplate = function(rrr) {
 
     var that = this;
 
-    var requiredTemplateVersion = indexTemplate.version;
+    var requiredTemplateVersion = indexTemplate7X.version;
 
     // Check if there is a template
     var templateURL = this.elasticURL+'/_template/template_api';
-    var getOptions = {url:templateURL, json:true};
-    var putOptions = {url:templateURL, json:indexTemplate};
+    var getOptionsVersion = {method:'get',url:this.elasticURL};
+    var getOptions = {method:'get',url:templateURL};
+    let putOptions = {method:'put',url:templateURL, data:indexTemplate7X};
 
     if (this.elasticUsername && this.elasticPassword) {
         var auth = {
             username: this.elasticUsername,
             password: this.elasticPassword,
         }
+        getOptionsVersion.auth = auth;
         getOptions.auth = auth;
         putOptions.auth = auth;
     }
 
-    request.get(getOptions, function (error, response, body) {
-        if(error) {
-            debug("Error querying template:", JSON.stringify(error) );
-        }else {
+    axios(getOptionsVersion).then( (response) => {
+        const body = response.data || {};
+        if(body && ('version' in body) && ('number' in body.version)){
+            that.es7 = body.version.number.startsWith('7');
+            that.es8 = body.version.number.startsWith('8');
+        }
+
+        if( !that.es7  && !that.es8){
+            putOptions.json = indexTemplate;
+        }
 
-            var initializeNeeded = false;
+        let initializeNeeded = false;
 
-            if(response.statusCode === 404){
-                initializeNeeded = true;
-            }else if(response.statusCode === 200){
-                if( 'template_api' in body ) {
-                    if (!('version' in body.template_api) || (body.template_api.version < requiredTemplateVersion)) {
-                        initializeNeeded = true;
-                    }
+        axios(getOptions).then( (response) => {
+            const body = response.data || {};
+            if( 'template_api' in body ) {
+                if (!('version' in body.template_api) || (body.template_api.version < requiredTemplateVersion)) {
+                    initializeNeeded = true;
                 }
             }
-
+        }).catch((error)=>{
+            if(error.response && error.response.status === 404) {
+                initializeNeeded = true;
+            } else {
+                debug(`Error querying template: ${error.message}`);
+                that.enabled = false;
+            }
+        }).finally(()=> {
             if(initializeNeeded){
-                request.put(putOptions, function (error, response, body) {
-                    if(error) {
-                        debug("Failed to update template:", JSON.stringify(error));
-                    }
+                axios(putOptions).then((response) =>{
+                    debug("Elasticsearch template updated");
+                    that.enabled = true;
+                }).catch((error)=>{
+                    debug(`Failed to update template: ${error.message}`);
+                    that.enabled = false;
                 });
+            }else{
+                that.enabled = true;
             }
+        });
 
-        }
-    });
 
+    }).catch((error)=>{
+        debug(`Error getting version: ${error.message}`);
+        that.enabled = false;
+    });
 };
 
 // Update timeline and stats per tick
@@ -180,7 +233,11 @@ swsElasticEmitter.prototype.processRecord = function(rrr){
 
     // Create metadata
     var indexName = this.indexPrefix+moment(rrr['@timestamp']).utc().format('YYYY.MM.DD');
-    var meta = {index:{_index:indexName,_type:'api',_id:rrr.id}};
+
+    let meta = {index:{_index:indexName,_id:rrr.id}};
+    if(!this.es7 && !this.es8){
+        meta = {index:{_index:indexName,_type:'api',_id:rrr.id}};
+    }
 
     // Add to buffer
     this.indexBuffer += JSON.stringify(meta) + '\n';
@@ -201,12 +258,13 @@ swsElasticEmitter.prototype.flush = function(){
 
     this.lastFlush = Date.now();
 
-    var options = {
+    let options = {
+        method: 'post',
         url: this.elasticURLBulk,
         headers: {
             'Content-Type': 'application/x-ndjson'
         },
-        body: this.indexBuffer
+        data: this.indexBuffer,
     };
 
     if (this.elasticUsername && this.elasticPassword) {
@@ -216,13 +274,13 @@ swsElasticEmitter.prototype.flush = function(){
         }
     }
 
-    request.post(options, function (error, response, body) {
-        if (error) {
-            debug("Indexing Error:", JSON.stringify(error) );
-        }
-        if (response && ('statusCode' in response) && (response.statusCode !== 200)) {
-            debug('Indexing Error: %d %s',response.statusCode, response.message);
+    axios(options).then((response)=> {
+        if (response && ('status' in response) && (response.status !== 200)) {
+            debug('Indexing Error: %d %s',response.status, response.message);
         }
+    }).catch((error)=>{
+        debug(`Indexing Error: ${error.message}`);
+        that.enabled = false;
     });
 
     this.indexBuffer = '';

package/lib/swsHapi.js

@@ -3,6 +3,7 @@ const path = require('path');
 const promClient = require("prom-client");
 const swsSettings = require('./swssettings');
 const swsProcessor = require('./swsProcessor');
+const swsAuth = require('./swsAuth');
 const swsUtil = require('./swsUtil');
 const debug = require('debug')('sws:hapi');
 const url = require('url');
@@ -33,7 +34,7 @@ class SwsHapi {
                 debug("processResponse:ERROR: " + e);
             }
         });
-        await server.ext('onRequest', function (request, h) {
+        await server.ext('onRequest', async function (request, h) {
             let nodeReq = request.raw.req;
             let nodeRes = request.raw.res;
             nodeRes._swsReq = nodeReq;
@@ -43,6 +44,14 @@ class SwsHapi {
             if(reqUrl.startsWith(swsSettings.uriPath)){
                 // Don't track sws requests
                 nodeReq.sws.track = false;
+                /*
+                if(reqUrl.startsWith(swsSettings.pathStats)){
+                    let authResult  = await swsAuth.processAuth(request.raw.req,request.raw.res);
+                    if( authResult ){
+                        //return h.continue;
+                        return processor.getStats(request.raw.req.sws.query);
+                    }
+                }*/
                 return h.continue;
             }
             try {
@@ -56,7 +65,14 @@ class SwsHapi {
         server.route({
             method: 'GET',
             path: swsSettings.pathStats,
-            handler: function (request, h) {
+            handler: async function (request, h) {
+                let authResult  = await swsAuth.processAuth(request.raw.req,request.raw.res);
+                if( !authResult ){
+                    return h.abandon;
+                }
+                if(('sws-auth' in request.raw.req) && request.raw.req['sws-auth']){
+                    request.raw.res.setHeader('x-sws-authenticated','true');
+                }
                 return processor.getStats(request.raw.req.sws.query);
             },
             options: options.routeOptions
@@ -65,45 +81,24 @@ class SwsHapi {
         server.route({
             method: 'GET',
             path: swsSettings.pathMetrics,
-            handler: function (request, h) {
-                const response = h.response(promClient.register.metrics());
+            handler: async function (request, h) {
+                let authResult  = await swsAuth.processAuth(request.raw.req,request.raw.res);
+                if( !authResult ){
+                    return h.abandon;
+                }
+                const response = h.response(await promClient.register.metrics());
                 response.code(200);
                 response.header('Content-Type', 'text/plain');
                 return response;
             },
             options: options.routeOptions
         });
-        // Redirect to ui
-        server.route({
-            method: 'GET',
-            path: swsSettings.uriPath,
-            handler: function (request, h) {
-                return h.redirect(swsSettings.pathUI);
-            },
-            options: options.routeOptions
-        });
-        // Return UI
+        // Logout
         server.route({
             method: 'GET',
-            path: swsSettings.pathUI,
+            path: swsSettings.pathLogout,
             handler: function (request, h) {
-                return swsUtil.swsEmbeddedUIMarkup;
-            },
-            options: options.routeOptions
-        });
-        // Return Dist
-        server.route({
-            method: 'GET',
-            path: swsSettings.pathDist+'/{file*}',
-            handler: function (request, h) {
-                let fileName = request.params.file;
-                var options = {
-                    root: path.join(__dirname,'..','dist'),
-                    dotfiles: 'deny'
-                    // TODO Caching
-                };
-                request.raw.res.setHeader('Content-Type', send.mime.lookup(path.basename(fileName)));
-                send(request.raw.req, fileName, options).pipe(request.raw.res);
+                swsAuth.processLogout(request.raw.req,request.raw.res);
                 return h.abandon;
             },
             options: options.routeOptions
@@ -111,13 +106,13 @@ class SwsHapi {
         // Return UX
         server.route({
             method: 'GET',
-            path: swsSettings.pathUX+'/{file*}',
+            path: swsSettings.pathUX+'{file*}',
             handler: function (request, h) {
                 let fileName = request.params.file;
                 if(!fileName){
                     fileName = 'index.html';
                 }
-                var options = {
+                let options = {
                     root: path.join(__dirname,'..','ux'),
                     dotfiles: 'deny'
                     // TODO Caching