pixl-server-web 1.3.6 → 1.3.9

package/README.md

@@ -20,6 +20,7 @@ This module is a component for use in [pixl-server](https://www.github.com/jhuck
 	* [http_regex_json](#http_regex_json)
 	* [http_response_headers](#http_response_headers)
 	* [http_timeout](#http_timeout)
+	* [http_request_timeout](#http_request_timeout)
 	* [http_keep_alives](#http_keep_alives)
 		+ [default](#default)
 		+ [request](#request)
@@ -42,6 +43,9 @@ This module is a component for use in [pixl-server](https://www.github.com/jhuck
 	* [http_clean_headers](#http_clean_headers)
 	* [http_log_socket_errors](#http_log_socket_errors)
 	* [http_full_uri_match](#http_full_uri_match)
+	* [http_req_max_dump_enabled](#http_req_max_dump_enabled)
+	* [http_req_max_dump_dir](#http_req_max_dump_dir)
+	* [http_req_max_dump_debounce](#http_req_max_dump_debounce)
 	* [https](#https)
 	* [https_port](#https_port)
 	* [https_cert_file](#https_cert_file)
@@ -53,6 +57,7 @@ This module is a component for use in [pixl-server](https://www.github.com/jhuck
 - [Custom URI Handlers](#custom-uri-handlers)
 	* [Access Control Lists](#access-control-lists)
 	* [Internal File Redirects](#internal-file-redirects)
+	* [Static Directory Handlers](#static-directory-handlers)
 	* [Sending Responses](#sending-responses)
 		+ [Standard Response](#standard-response)
 		+ [Custom Response](#custom-response)
@@ -72,6 +77,7 @@ This module is a component for use in [pixl-server](https://www.github.com/jhuck
 		+ [args.cookies](#argscookies)
 		+ [args.perf](#argsperf)
 		+ [args.server](#argsserver)
+		+ [args.id](#argsid)
 	* [Request Filters](#request-filters)
 - [Logging](#logging)
 - [Stats](#stats)
@@ -88,6 +94,7 @@ This module is a component for use in [pixl-server](https://www.github.com/jhuck
 	* [Self-Referencing URLs](#self-referencing-urls)
 	* [Custom Method Handlers](#custom-method-handlers)
 	* [Let's Encrypt SSL Certificates](#lets-encrypt-ssl-certificates)
+	* [Request Max Dump](#request-max-dump)
 - [License](#license)
 
 # Usage
@@ -220,6 +227,34 @@ This param allows you to send back any additional custom HTTP headers with each
 }
 ```
 
+## http_code_response_headers
+
+This property allows you to include *conditional* response headers, based on the HTTP response code.  For example, you can instruct the web server to send back a custom header with `404` (File Not Found) responses, like this:
+
+```js
+{
+	"http_code_response_headers": {
+		"404": {
+			"X-Message": "And don't come back!"
+		}
+	}
+}
+```
+
+An actual useful case would be to include a [Retry-After](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After) header with all `429` (Too Many Requests) responses, like this:
+
+```js
+{
+	"http_code_response_headers": {
+		"429": {
+			"Retry-After": "10"
+		}
+	}
+}
+```
+
+This would give a hint to clients when they receive a `429` (Too Many Requests) response from the web server, that they should wait `10` seconds before trying again.
+
 ## http_timeout
 
 This sets the idle socket timeout for all incoming HTTP requests, in seconds.  If omitted, the Node.js default is 120 seconds.  Example:
@@ -464,6 +499,18 @@ When this boolean is set to `true`, [Custom URI Handlers](#custom-uri-handlers)
 }
 ```
 
+## http_req_max_dump_enabled
+
+When this boolean is set to `true`, the [Request Max Dump](#request-max-dump) system is enabled.  This will produce a JSON dump file when the web server is maxed out on requests.
+
+## http_req_max_dump_dir
+
+When the [Request Max Dump](#request-max-dump) system is enabled, the `http_req_max_dump_dir` property sets the directory path where JSON dump files are dropped.  The directory will be created if needed.
+
+## http_req_max_dump_debounce
+
+When the [Request Max Dump](#request-max-dump) system is enabled, the `http_req_max_dump_debounce` property sets how many seconds should elapse between dumps, as to not overwhelm the filesystem.
+
 ## https
 
 This boolean allows you to enable HTTPS (SSL) support in the web server.  It defaults to `false`.  Note that you must also set `https_port`, and possibly `https_cert_file` and `https_key_file` for this to work.
@@ -984,31 +1031,51 @@ The result is an object in this format:
 	"stats": {
 		"total": {
 			"st": "mma",
-			"min": 0.479,
-			"max": 2.57,
-			"total": 11.3748,
-			"count": 11
+			"min": 0.108,
+			"max": 19.964,
+			"total": 18719.696,
+			"count": 2997,
+			"avg": 6.246
+		},
+		"queue": {
+			"st": "mma",
+			"min": 3.707,
+			"max": 10.917,
+			"total": 8510.662,
+			"count": 1373,
+			"avg": 6.198
 		},
 		"read": {
 			"st": "mma",
-			"min": 0.005,
-			"max": 0.071,
-			"total": 0.170,
-			"count": 11
+			"min": 0,
+			"max": 0.134,
+			"total": 2.533,
+			"count": 1373,
+			"avg": 0.001
 		},
+		"filter": {
+			"st": "mma",
+			"min": 0,
+			"max": 0,
+			"total": 0,
+			"count": 0,
+			"avg": 0
+		}
 		"process": {
 			"st": "mma",
-			"min": 0.123,
-			"max": 0.625,
-			"total": 2.691,
-			"count": 11
+			"min": 0.834,
+			"max": 6.1,
+			"total": 3513.736,
+			"count": 1373,
+			"avg": 2.559
 		},
 		"write": {
 			"st": "mma",
-			"min": 0.313,
-			"max": 1.747,
-			"total": 7.679,
-			"count": 11
+			"min": 0.08,
+			"max": 8.85,
+			"total": 6523.865,
+			"count": 2997,
+			"avg": 2.176
 		},
 		"bytes_in": 0,
 		"bytes_out": 1175,
@@ -1107,7 +1174,9 @@ The `stats` object contains real-time performance metrics, representing one whol
 | Property | Type | Description |
 |----------|------|-------------|
 | `total` | Min/Max/Avg | Total request elapsed time. |
+| `queue` | Min/Max/Avg | Total request time in queue. |
 | `read` | Min/Max/Avg | Total request read time. |
+| `filter` | Min/Max/Avg | Total request filter time. |
 | `process` | Min/Max/Avg | Total request process time (i.e. custom URI handler). |
 | `write` | Min/Max/Avg | Total request write time. |
 | `bytes_in` | Simple Counter | Total bytes received in the last full second. |
@@ -1344,6 +1413,65 @@ Toss that command into a shell script in `/etc/cron.daily/` and it'll run daily
 
 Certbot produces its own log file here: `/var/log/letsencrypt/letsencrypt.log`
 
+## Request Max Dump
+
+For debugging and troubleshooting purposes, pixl-server-web can optionally generate a "dump" file when it reaches certain traffic limits.  Specifically, when one of these events occur:
+
+- When the [http_max_connections](#http_max_connections) limit is reached.
+- When the [http_max_queue_length](#http_max_queue_length) limit is reached.
+- When the [http_max_queue_active](#http_max_queue_active) limit is reached.
+
+To enable this feature, set the [http_req_max_dump_enabled](#http_req_max_dump_enabled) configuration property to `true`, the [http_req_max_dump_dir](#http_req_max_dump_dir) property to a path on your filesystem to hold your dump files (this will be created if needed), and [http_req_max_dump_debounce](#http_req_max_dump_debounce) to the maximum frequency you want files dumped (in seconds).  Example:
+
+```json
+"http_req_max_dump_enabled": true,
+"http_req_max_dump_dir": "/var/log/web-server-dumps",
+"http_req_max_dump_debounce": 10
+```
+
+This would generate dump files in the `/var/log/web-server-dumps` directory every 10 seconds, while one or more maximum limits are maxed out.
+
+The dump files themselves are in JSON format, and contain everything from the [Stats API](#stats), as well as a list of all active and pending requests.  For each request, the following information is dumped:
+
+```json
+"r2945": {
+	"uri": "/api/test/sleep?ms=1",
+	"ip": "127.0.0.1",
+	"ips": [
+		"127.0.0.1"
+	],
+	"headers": {
+		"accept-encoding": "gzip, deflate, br",
+		"user-agent": "Mozilla/5.0; wperf/1.0.4",
+		"host": "localhost:3012",
+		"connection": "keep-alive"
+	},
+	"state": "writing",
+	"date": 1644617758.688,
+	"elapsed": 0.009999990463256836
+}
+```
+
+Here is a description of each property:
+
+| Property Name | Type | Description |
+|---------------|------|-------------|
+| `uri` | String | The request URI path (sans protocol and hostname). |
+| `ip` | String | The client's public IP address (may be a load balancer or proxy). |
+| `ips` | All the client IPs as an array (includes those from proxy headers). |
+| `headers` | String | All the incoming HTTP request headers from the client (lower-cased keys). |
+| `state` | String | The state of the request, will be one of `queued`, `reading`, `filtering`, `processing` or `writing`. |
+| `date` | String | The timestamp of the start of the request, in Epoch seconds. |
+| `elapsed` | String | The elapsed time of the request in seconds. |
+
+Each dump file is given a unique filename using the current server hostname, the pixl-server-web process PID, and a high-resolution timestamp in [Base36](https://en.wikipedia.org/wiki/Base36) format.  Example:
+
+```
+req-dump-joemax.local-67463-kziyy7eq.json
+req-dump-joemax.local-67463-kziyy86i.json
+req-dump-joemax.local-67463-kziyy8yc.json
+```
+
 # License
 
 **The MIT License (MIT)**

package/lib/http.js

@@ -63,6 +63,7 @@ module.exports = class HTTP {
 				socket.end();
 				socket.unref();
 				socket.destroy(); // hard close
+				self.dumpAllRequests();
 				return;
 			}
 			if (self.server.shut) {

package/lib/https.js

@@ -61,6 +61,7 @@ module.exports = class HTTP2 {
 				socket.end();
 				socket.unref();
 				socket.destroy(); // hard close
+				self.dumpAllRequests();
 				return;
 			}
 			if (self.server.shut) {

package/lib/request.js

@@ -16,20 +16,21 @@ module.exports = class Request {
 			id: this.getNextId('r'),
 			date: Date.now() / 1000,
 			request: request,
-			response: response
+			response: response,
+			state: 'queued',
+			perf: new Perf()
 		};
 		
+		args.perf.begin();
+		
+		var ips = args.ips = this.getAllClientIPs(request);
+		var ip = args.ip = this.getPublicIP(ips);
+		
 		if (this.server.shut) {
 			// server is shutting down, deny new requests
-			var ips = args.ips = this.getAllClientIPs(request);
-			var ip = args.ip = this.getPublicIP(ips);
-			
 			this.logError(503, "Server is shutting down, denying request from: " + ip, 
 				{ id: args.id, ips: ips, uri: request.url, headers: request.headers }
 			);
-			
-			args.perf = new Perf();
-			args.perf.begin();
 			this.sendHTTPResponse( args, "503 Service Unavailable", {}, "503 Service Unavailable (server shutting down)" );
 			return;
 		}
@@ -38,15 +39,13 @@ module.exports = class Request {
 		if (this.queueSkipMatch && request.url.match(this.queueSkipMatch)) {
 			this.logDebug(8, "Bumping request to front of queue: " + request.url);
 			this.requests[ args.id ] = args;
+			args.perf.begin('queue');
 			this.queue.unshift(args);
 			return;
 		}
 		
 		if (this.maxQueueActive && (this.queue.running() >= this.maxQueueActive)) {
 			// queue is maxed out on active reqs, reject request immediately
-			var ips = args.ips = this.getAllClientIPs(request);
-			var ip = args.ip = this.getPublicIP(ips);
-			
 			this.logError(429, "Queue is maxed out (" + this.queue.running() + " active reqs), denying new request from: " + ip, { 
 				id: args.id,
 				ips: ips, 
@@ -56,18 +55,13 @@ module.exports = class Request {
 				active: this.queue.running(),
 				sockets: this.numConns
 			});
-			
-			args.perf = new Perf();
-			args.perf.begin();
-			this.sendHTTPResponse( args, "429 Too Many Requests", {}, "429 Too Many Requests (queue maxed out)" );
+			this.sendHTTPResponse( args, "429 Too Many Requests", {}, "429 Too Many Requests (queue active maxed out)" );
+			this.dumpAllRequests();
 			return;
 		}
 		
 		if (this.maxQueueLength && (this.queue.length() >= this.maxQueueLength)) {
 			// queue is maxed out on pending reqs, reject request immediately
-			var ips = args.ips = this.getAllClientIPs(request);
-			var ip = args.ip = this.getPublicIP(ips);
-			
 			this.logError(429, "Queue is maxed out (" + this.queue.length() + " pending reqs), denying new request from: " + ip, { 
 				id: args.id,
 				ips: ips, 
@@ -77,14 +71,14 @@ module.exports = class Request {
 				active: this.queue.running(),
 				sockets: this.numConns
 			});
-			
-			args.perf = new Perf();
-			args.perf.begin();
-			this.sendHTTPResponse( args, "429 Too Many Requests", {}, "429 Too Many Requests (queue maxed out)" );
+			this.sendHTTPResponse( args, "429 Too Many Requests", {}, "429 Too Many Requests (queue pending maxed out)" );
+			this.dumpAllRequests();
 			return;
 		}
 		
 		this.requests[ args.id ] = args;
+		
+		args.perf.begin('queue');
 		this.queue.push(args);
 	}
 	
@@ -93,7 +87,10 @@ module.exports = class Request {
 		// (async dequeue handler function)
 		var self = this;
 		var request = args.request;
-		var response = args.response;
+		var ips = args.ips;
+		var ip = args.ip;
+		
+		args.perf.end('queue');
 		
 		// all requests will end up in this callback here
 		args.callback = function() {
@@ -135,9 +132,6 @@ module.exports = class Request {
 			return;
 		}
 		
-		var ips = this.getAllClientIPs(request);
-		var ip = this.getPublicIP(ips);
-		
 		this.logDebug(8, "New HTTP request: " + request.method + " " + request.url + " (" + ips.join(', ') + ")", {
 			id: args.id,
 			socket: request.socket._pixl_data.id,
@@ -187,10 +181,6 @@ module.exports = class Request {
 			} // foreach cookie
 		} // headers.cookie
 		
-		// track performance of request
-		args.perf = new Perf();
-		args.perf.begin();
-		
 		if (this.server.shut) {
 			// server is shutting down, deny new requests
 			this.logError(503, "Server is shutting down, denying request from: " + ip, 

package/lib/response.js

@@ -79,6 +79,14 @@ module.exports = class Response {
 		args.http_code = http_code;
 		args.http_status = http_status;
 		
+		// merge in conditional headers based on response code
+		var code_headers = this.config.get('http_code_response_headers');
+		if (code_headers && (http_code in code_headers)) {
+			for (var key in code_headers[http_code]) {
+				headers[key] = code_headers[http_code][key];
+			}
+		}
+		
 		// use duck typing to see if we have a stream, buffer or string
 		var is_stream = (body && body.pipe);
 		var is_buffer = (body && body.fill);

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-web",
-	"version": "1.3.6",
+	"version": "1.3.9",
 	"description": "A web server component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-web",
@@ -27,7 +27,7 @@
 		"formidable": "2.0.1",
 		"errno": "0.1.7",
 		"stream-meter": "1.0.4",
-		"async": "3.2.0",
+		"async": "3.2.2",
 		"mime": "2.5.2"
 	},
 	"devDependencies": {

package/test/test.js

@@ -1373,6 +1373,42 @@ module.exports = {
 			);
 		},
 		
+		function testConditionalResponseHeaders(test) {
+			// test response headers per http code
+			var self = this;
+			var web = this.web_server;
+			
+			web.config.set('http_code_response_headers', {
+				"403": { 'X-Test-Cond': "Tree Frogs" }
+			});
+			
+			request.get( 'http://127.0.0.1:3020/server-status', // ACL'ed endpoint
+				{
+					headers: {
+						"X-Forwarded-For": "1.2.3.4" // external IP
+					}
+				},
+				function(err, resp, data, perf) {
+					test.ok( !err, "No error from PixlRequest: " + err );
+					test.ok( !!resp, "Got resp from PixlRequest" );
+					test.ok( resp.statusCode == 403, "Got 403 response: " + resp.statusCode );
+					test.ok( resp.headers['x-test-cond'] === "Tree Frogs", "Unexpected header: " + resp.headers['X-Test-Cond'] );
+					
+					// make sure basic 200 doesn't have header
+					request.json( 'http://127.0.0.1:3020/json', false, {}, 
+						function(err, resp, json, perf) {
+							test.ok( !err, "No error from PixlRequest: " + err );
+							test.ok( !!resp, "Got resp from PixlRequest" );
+							test.ok( resp.statusCode == 200, "Got 200 response: " + resp.statusCode );
+							test.ok( !resp.headers['x-test-cond'], "Unexpected X-Test-Cond header for HTTP 200!" );
+							web.config.set('http_code_response_headers', null); // reset config
+							test.done();
+						}
+					);
+				} 
+			);
+		},
+		
 		// get stats
 		function testStats(test) {
 			// test stats API (this also tests ACL pass)

package/web_server.js

@@ -1,6 +1,6 @@
 // Simple HTTP / HTTPS Web Server
 // A component for the pixl-server daemon framework.
-// Copyright (c) 2015 - 2021 Joseph Huckaby
+// Copyright (c) 2015 - 2022 Joseph Huckaby
 // Released under the MIT License
 
 const fs = require('fs');
@@ -25,39 +25,45 @@ module.exports = Class({
 	version: require( __dirname + '/package.json' ).version,
 	
 	defaultConfig: {
-		http_private_ip_ranges: ['127.0.0.1', '10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16', '::1/128', 'fd00::/8', '169.254.0.0/16', 'fe80::/10'],
-		http_regex_text: "(text|javascript|json|css|html)",
-		http_regex_json: "(javascript|js|json)",
-		http_keep_alives: "default",
-		http_timeout: 120,
-		http_static_index: "index.html",
-		http_static_ttl: 0,
-		http_max_upload_size: 32 * 1024 * 1024,
-		http_temp_dir: os.tmpdir(),
-		http_gzip_opts: {
-			level: zlib.constants.Z_DEFAULT_COMPRESSION, 
-			memLevel: 8 
+		"http_private_ip_ranges": ['127.0.0.1', '10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16', '::1/128', 'fd00::/8', '169.254.0.0/16', 'fe80::/10'],
+		"http_regex_text": "(text|javascript|json|css|html)",
+		"http_regex_json": "(javascript|js|json)",
+		"http_keep_alives": "default",
+		"http_timeout": 120,
+		"http_static_index": "index.html",
+		"http_static_ttl": 0,
+		"http_max_upload_size": 32 * 1024 * 1024,
+		"http_temp_dir": os.tmpdir(),
+		"http_gzip_opts": {
+			"level": zlib.constants.Z_DEFAULT_COMPRESSION, 
+			"memLevel": 8 
 		},
-		http_brotli_opts: {
-			chunkSize: 16 * 1024,
-			mode: "text",
-			level: 4
+		"http_brotli_opts": {
+			"chunkSize": 16 * 1024,
+			"mode": "text",
+			"level": 4
 		},
-		http_compress_text: false,
-		http_enable_brotli: false,
-		http_default_acl: ['127.0.0.1', '10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16', '::1/128', 'fd00::/8', '169.254.0.0/16', 'fe80::/10'],
-		http_log_requests: false,
-		http_recent_requests: 10,
-		http_max_connections: 0,
-		http_max_requests_per_connection: 0,
-		http_max_concurrent_requests: 0,
-		http_max_queue_length: 0,
-		http_max_queue_active: 0,
-		http_queue_skip_uri_match: false,
-		http_clean_headers: false,
-		http_log_socket_errors: true,
-		http_full_uri_match: false,
-		http_request_timeout: 0
+		"http_compress_text": false,
+		"http_enable_brotli": false,
+		"http_default_acl": ['127.0.0.1', '10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16', '::1/128', 'fd00::/8', '169.254.0.0/16', 'fe80::/10'],
+		"http_log_requests": false,
+		"http_recent_requests": 10,
+		"http_max_connections": 0,
+		"http_max_requests_per_connection": 0,
+		"http_max_concurrent_requests": 0,
+		"http_max_queue_length": 0,
+		"http_max_queue_active": 0,
+		"http_queue_skip_uri_match": false,
+		"http_clean_headers": false,
+		"http_log_socket_errors": true,
+		"http_full_uri_match": false,
+		"http_request_timeout": 0,
+		
+		"http_req_max_dump_enabled": false,
+		"http_req_max_dump_dir": "",
+		"http_req_max_dump_debounce": 10,
+		
+		"http_code_response_headers": null
 	},
 	
 	conns: null,
@@ -169,6 +175,16 @@ class WebServer extends Component {
 			}
 		}
 		
+		// initialize request max dump system, if enabled
+		this.reqMaxDumpEnabled = this.config.get('http_req_max_dump_enabled');
+		this.reqMaxDumpDir = this.config.get('http_req_max_dump_dir');
+		this.reqMaxDumpDebounce = this.config.get('http_req_max_dump_debounce');
+		this.reqMaxDumpLast = 0;
+		
+		if (this.reqMaxDumpEnabled && this.reqMaxDumpDir && !fs.existsSync(this.reqMaxDumpDir)) {
+			fs.mkdirSync( this.reqMaxDumpDir, { mode: 0o777, recursive: true } );
+		}
+		
 		// listen for tick events to swap stat buffers
 		this.server.on( 'tick', this.tick.bind(this) );
 		
@@ -184,6 +200,40 @@ class WebServer extends Component {
 		} );
 	}
 	
+	dumpAllRequests(callback) {
+		// create dump file containing info on all active/pending requests
+		// this is called when requests or sockets are maxed out
+		// only write file every N seconds
+		var self = this;
+		var now = Date.now() / 1000;
+		if (now - this.reqMaxDumpLast < this.reqMaxDumpDebounce) return;
+		this.reqMaxDumpLast = now;
+		
+		var dump_file = this.reqMaxDumpDir + '/req-dump-' + os.hostname() + '-' + process.pid + '-' + Date.now().toString(36) + '.json';
+		var json = this.getStats();
+		json.requests = {};
+		
+		for (var id in this.requests) {
+			var args = this.requests[id];
+			var info = {
+				uri: args.request.url,
+				ip: args.ip,
+				ips: args.ips,
+				headers: args.request.headers,
+				state: args.state,
+				date: args.date,
+				elapsed: now - args.date
+			};
+			json.requests[id] = info;
+		}
+		
+		this.logDebug(5, "Writing dump file: " + dump_file );
+		fs.writeFile( dump_file, JSON.stringify(json, null, "\t") + "\n", function(err) {
+			if (err) self.logError('dump', "Failed to write dump file: " + dump_file + ": " + err, err);
+			if (callback) callback(err);
+		} );
+	}
+	
 	deleteUploadTempFiles(args) {
 		// delete leftover temp files created by Formidable
 		for (var key in args.files) {
@@ -244,7 +294,7 @@ class WebServer extends Component {
 		if (!stats.bytes_in) stats.bytes_in = 0;
 		if (!stats.bytes_out) stats.bytes_out = 0;
 		
-		['total', 'read', 'process', 'write'].forEach( function(key) {
+		['total', 'queue', 'read', 'filter', 'process', 'write'].forEach( function(key) {
 			if (!stats[key]) stats[key] = { "st": "mma", "min": 0, "max": 0, "total": 0, "count": 0 };
 		} );