whistle 2.9.37 → 2.9.38-beta

package/lib/plugins/load-plugin.js

@@ -5,13 +5,12 @@ var express = require('express');
 var os = require('os');
 var format = require('util').format;
 var LRU = require('lru-cache');
-var iconv = require('iconv-lite');
 var wsParser = require('ws-parser');
 var http = require('http');
 var https = require('https');
 var extend = require('extend');
+var gzip = require('zlib').gzip;
 var request = require('../util/http-mgr').request;
-var isUtf8 = require('../util/is-utf8');
 var Storage = require('../rules/storage');
 var getServer = require('hagent').create(null, 40500);
 var Buffer = require('safe-buffer').Buffer;
@@ -341,7 +340,7 @@ var addParserApi = function (req, conn, state, reqId) {
     }
     if (Array.isArray(data.toClient)) {
       data.toClient.forEach(function (frame) {
-        var buf = toBuffer(frame.base64);
+        var buf = base64ToBuffer(frame.base64);
         try {
           buf && req.emit('sendToClient', buf, frame.binary);
         } catch (e) {}
@@ -349,7 +348,7 @@ var addParserApi = function (req, conn, state, reqId) {
     }
     if (Array.isArray(data.toServer)) {
       data.toServer.forEach(function (frame) {
-        var buf = toBuffer(frame.base64);
+        var buf = base64ToBuffer(frame.base64);
         try {
           buf && req.emit('sendToServer', buf, frame.binary);
         } catch (e) {}
@@ -411,7 +410,7 @@ var ADDITIONAL_FIELDS = [
 
 function getPluginVars(req, key) {
   var value = req.headers[key];
-  value = toBuffer(value);
+  value = base64ToBuffer(value);
   if (value) {
     delete req.headers[key];
     try {
@@ -465,7 +464,7 @@ var initReq = function (req, res, isServer) {
   oReq.isUIRequest = req.isUIRequest =
     req.headers[pluginOpts.UI_REQUEST_HEADER] == '1';
   oReq.enableCapture = req.headers[pluginOpts.ENABLE_CAPTURE_HEADER] == '1';
-  oReq.isFromPlugin = req.headers[pluginOpts.PLUGIN_REQUEST_HEADER] == '1';
+  oReq.isFromPlugin = req.fromPlugin = oReq.fromPlugin = req.headers[pluginOpts.PLUGIN_REQUEST_HEADER] == '1';
   oReq.ruleValue = getValue(req, pluginOpts.RULE_VALUE_HEADER);
   oReq.ruleUrl = getValue(req, pluginOpts.RULE_URL_HEADER) || oReq.ruleValue;
   oReq.pipeValue = getValue(req, pluginOpts.PIPE_VALUE_HEADER);
@@ -542,7 +541,7 @@ var getOptions = function (opts, binary, toServer) {
   }
   return binary ? BINARY_OPTIONS : '';
 };
-var toBuffer = function (base64) {
+var base64ToBuffer = function (base64) {
   if (base64) {
     try {
       return new Buffer(base64, 'base64');
@@ -550,16 +549,11 @@ var toBuffer = function (base64) {
   }
 };
 var getBuffer = function (item) {
-  return toBuffer(item.base64);
+  return base64ToBuffer(item.base64);
 };
 var getText = function (item) {
-  var body = toBuffer(item.base64) || '';
-  if (body && !isUtf8(body)) {
-    try {
-      body = iconv.encode(body, 'GB18030');
-    } catch (e) {}
-  }
-  return body + '';
+  var body = base64ToBuffer(item.base64) || '';
+  return common.bufferToString(body);
 };
 
 var defineProps = function (obj) {
@@ -885,31 +879,33 @@ function handleError(socket, sender, receiver) {
   receiver && (receiver.onerror = emitError);
 }
 
-function toBinary(data) {
-  if (!data || Buffer.isBuffer(data)) {
-    return data && data.length && data;
-  }
-  if (typeof data !== 'string') {
-    try {
-      data = JSON.stringify(data);
-    } catch (e) {
-      return;
-    }
-  }
-  return Buffer.from(data);
-}
+var GZIP_RE = /^\s*gzip\s*$/i;
 
-function getCustomBody(body, headers) {
+function getCustomBody(body, req, cb) {
+  var headers = req.headers;
   if (body && typeof body.pipe === 'function') {
     delete headers['content-length'];
-    return body;
+    return cb(body);
   }
-  body = toBinary(body) || '';
-  if (headers['content-length'] != null) {
-    headers['content-length'] = body ? body.length : 0;
-    delete headers['transfer-encoding'];
+  var handleCb = function() {
+    if (headers['content-length'] != null) {
+      headers['content-length'] = body.length;
+      delete headers['transfer-encoding'];
+    }
+    cb(body);
+  };
+  if (Buffer.isBuffer(body)) {
+    return handleCb();
+  }
+  var encoding = headers['content-encoding'];
+  body = common.toBuffer(body, req) || '';
+  if (!GZIP_RE.test(encoding)) {
+    return handleCb();
   }
-  return body;
+  gzip(body, function(_, result) {
+    body = result || body;
+    handleCb();
+  });
 }
 
 function wrapTunnelWriter(socket, toServer) {
@@ -918,7 +914,7 @@ function wrapTunnelWriter(socket, toServer) {
   var sender = wsParser.getSender(socket, toServer);
   handleError(socket, sender);
   socket.write = function (chunk, encoding, cb) {
-    if ((chunk = toBinary(chunk))) {
+    if ((chunk = common.toBuffer(chunk))) {
       if (encoding === 'binary') {
         return write.call(this, chunk, encoding, cb);
       }
@@ -931,7 +927,7 @@ function wrapTunnelWriter(socket, toServer) {
   };
   if (toServer) {
     socket.write = function (chunk, opts, cb) {
-      if ((chunk = toBinary(chunk))) {
+      if ((chunk = common.toBuffer(chunk))) {
         if (opts === 'binary') {
           return write.call(this, chunk, opts, cb);
         }
@@ -939,12 +935,12 @@ function wrapTunnelWriter(socket, toServer) {
       }
     };
     socket.writeText = function (chunk) {
-      if ((chunk = toBinary(chunk))) {
+      if ((chunk = common.toBuffer(chunk))) {
         sender.send(chunk, getOptions(null, false, true));
       }
     };
     socket.writeBin = function (chunk) {
-      if ((chunk = toBinary(chunk))) {
+      if ((chunk = common.toBuffer(chunk))) {
         sender.send(chunk, getOptions(null, true, true));
       }
     };
@@ -956,7 +952,7 @@ function wrapTunnelWriter(socket, toServer) {
     };
   } else {
     socket.write = function (chunk, encoding, cb) {
-      if ((chunk = toBinary(chunk))) {
+      if ((chunk = common.toBuffer(chunk))) {
         if (encoding === 'binary') {
           return write.call(this, chunk, encoding, cb);
         }
@@ -995,9 +991,9 @@ function setReqRules(uri, reqRules) {
   if (!reqRules) {
     return;
   }
-  var rules = reqRules.rules || reqRules;
+  var rules = formatRules(reqRules);
   var values = reqRules.values;
-  if (typeof rules !== 'string') {
+  if (!rules) {
     return;
   }
   uri.headers = uri.headers || {};
@@ -1039,7 +1035,7 @@ function addFrameHandler(req, socket, maxWsPayload, fromClient, toServer) {
     emit.call(socket, 'data', chunk, opts);
   };
   socket.write = function (chunk, opts, cb) {
-    if ((chunk = toBinary(chunk))) {
+    if ((chunk = common.toBuffer(chunk))) {
       if (opts === 'binary') {
         return write.call(this, chunk, opts, cb);
       }
@@ -1047,12 +1043,12 @@ function addFrameHandler(req, socket, maxWsPayload, fromClient, toServer) {
     }
   };
   socket.writeText = function (chunk) {
-    if ((chunk = toBinary(chunk))) {
+    if ((chunk = common.toBuffer(chunk))) {
       sender.send(chunk, getOptions(null, false, toServer));
     }
   };
   socket.writeBin = function (chunk) {
-    if ((chunk = toBinary(chunk))) {
+    if ((chunk = common.toBuffer(chunk))) {
       sender.send(chunk, getOptions(null, true, toServer));
     }
   };
@@ -1114,6 +1110,14 @@ function destroySocket() {
   this.destroy();
 }
 
+function formatRules(rules) {
+  if (Array.isArray(rules)) {
+    return rules.join('\n');
+  }
+  rules = rules.rules || rules;
+  return typeof rules === 'string' ? rules : undefined;
+}
+
 module.exports = async function (options, callback) {
   var root = options.value;
   if (options.isDev) {
@@ -1180,13 +1184,17 @@ module.exports = async function (options, callback) {
   var boundPort = config.port;
   var PLUGIN_HOOKS = config.PLUGIN_HOOKS;
   var RES_RULES_HEAD = config.RES_RULES_HEAD;
-  var getRulesStr = function (obj) {
+  var setResRules = function(headers, obj) {
+    var rules = formatRules(obj);
+    if (!rules) {
+      return;
+    }
     obj = JSON.stringify({
-      rules: typeof obj === 'string' ? obj : obj.rules,
+      rules: rules,
       values: obj.values,
       root: root
     });
-    return encodeURIComponent(obj);
+    headers[RES_RULES_HEAD] =  encodeURIComponent(obj);
   };
   var SHOW_LOGIN_BOX = options.SHOW_LOGIN_BOX;
   FROM_TUNNEL_HEADER = options.FROM_TUNNEL_HEADER;
@@ -2036,7 +2044,7 @@ module.exports = async function (options, callback) {
               headers = headers || svrRes.headers;
               if (resRules) {
                 headers = headers || {};
-                headers[RES_RULES_HEAD] = getRulesStr(resRules);
+                setResRules(headers, resRules);
                 req.setResRules = res.setResRules = noop;
                 resRules = null;
               }
@@ -2108,15 +2116,17 @@ module.exports = async function (options, callback) {
                     }
                   };
                   if (handleRes) {
-                    common.readStream(_res, function(buffer) {
-                      handleRes(buffer, function(result) {
+                    common.readStream(_res, function(rawBuffer) {
+                      handleRes(rawBuffer, function(result) {
                         if (result) {
                           req.setResRules(result.rules);
-                          _response(getCustomBody(result.body, _res.headers), result.trailers);
+                          getCustomBody(result.body, _res, function(body) {
+                            _response(body, result.trailers);
+                          });
                         } else {
                           _response('');
                         }
-                      });
+                      }, _res);
                     });
                   } else {
                     _response();
@@ -2131,15 +2141,17 @@ module.exports = async function (options, callback) {
                 }
               };
               if (handleReq) {
-                common.readStream(req, function(buffer) {
-                  handleReq(buffer, function(result) {
+                common.readStream(req, function(rawBuffer) {
+                  handleReq(rawBuffer, function(result) {
                     if (result) {
                       req.setReqRules(result.rules);
-                      _request(getCustomBody(result.body, req.headers) || '', result.uri || result.url);
+                      getCustomBody(result.body, req, function(body) {
+                        _request(body, result.uri || result.url);
+                      });
                     } else {
                       _request('');
                     }
-                  });
+                  }, req);
                 });
               } else {
                 _request();
@@ -2220,7 +2232,7 @@ module.exports = async function (options, callback) {
               headers = headers || svrRes.headers;
               if (resRules) {
                 headers = headers || {};
-                headers[RES_RULES_HEAD] = getRulesStr(resRules);
+                setResRules(headers, resRules);
                 req.setResRules = socket.setResRules = noop;
                 resRules = null;
               }

package/lib/util/common.js

@@ -2,7 +2,12 @@ var os = require('os');
 var path = require('path');
 var fse = require('fs-extra2');
 var fs = require('fs');
+var iconv = require('iconv-lite');
+var qs = require('qs');
+var Buffer = require('safe-buffer').Buffer;
+var zlib = require('./zlib');
 var pkgConf = require('../../package.json');
+var isUtf8 = require('./is-utf8');
 
 var HEAD_RE = /^head$/i;
 var HOME_DIR_RE = /^[~～]\//;
@@ -325,12 +330,113 @@ exports.isGroup = function(name) {
   return name && name[0] === '\r';
 };
 
+var UTF8_RE = /^utf-?8$/i;
+function bufferToString(buffer, encoding) {
+  if (typeof encoding === 'string') {
+    encoding = encoding.trim();
+  } else {
+    encoding = null;
+  }
+  if (Buffer.isBuffer(buffer) && !UTF8_RE.test(encoding) && !isUtf8(buffer)) {
+    try {
+      buffer = iconv.encode(buffer, encoding || 'GB18030');
+    } catch (e) {}
+  }
+  return String(buffer || '');
+}
+
+exports.bufferToString = bufferToString;
+
+var URLENCODED_RE = /application\/x-www-form-urlencoded/i;
+var POST_RE = /^post$/i;
+
+function isUrlEncoded(req) {
+  var type = req.method && req.headers && req.headers['content-type'];
+  return type && POST_RE.test(req.method) && URLENCODED_RE.test(type);
+}
+
+exports.isUrlEncoded = isUrlEncoded;
+
+function queryToString(query, req) {
+  if (!req || typeof query !== 'object' || !isUrlEncoded(req)) {
+    return query;
+  }
+  return qs.stringify(query);
+}
+
+exports.toBuffer = function(data, req) {
+  data = queryToString(data, req);
+  if (!data || Buffer.isBuffer(data)) {
+    return data && data.length && data;
+  }
+  if (typeof data !== 'string') {
+    try {
+      data = JSON.stringify(data);
+    } catch (e) {
+      return;
+    }
+  }
+  return Buffer.from(data);
+};
+
+
 exports.readStream = function(stream, callback) {
   var buffer = null;
   stream.on('data', function(chunk) {
     buffer = buffer ? Buffer.concat([buffer, chunk]) : chunk;
   });
   stream.once('end', function() {
+    var buf;
+    var text;
+    var json;
+    var err;
+    var jsonErr;
+    stream.zlib = zlib;
+    var encoding = stream.headers && stream.headers['content-encoding'];
+    var getBuffer = function(cb) {
+      if (err || buf !== undefined) {
+        return cb(err, buf);
+      }
+      zlib.unzip(encoding, buffer, function(e, ctn) {
+        err = e;
+        buf = ctn || null;
+        cb(err, buf);
+      });
+    };
+    var getText = function(cb, ecd) {
+      if (err || text != null) {
+        return cb(err, text);
+      }
+      getBuffer(function(e, ctn) {
+        err = e;
+        text = bufferToString(ctn, ecd);
+        cb(err, text);
+      });
+    };
+    stream.getBuffer = getBuffer;
+    stream.getText = getText;
+    stream.getJson = function(cb, ecd) {
+      getText(function() {
+        if (err || jsonErr || !text || json !== undefined) {
+          return cb(err || jsonErr, json);
+        }
+        try {
+          if (isUrlEncoded(stream)) {
+            json = qs.parse(text);
+          } else {
+            text = text.trim();
+            if (text[0] === '[' || text[0] === '{') {
+              json = JSON.parse(text) || null;
+            } else {
+              json = null;
+            }
+          }
+        } catch (e) {
+          jsonErr = e;
+        }
+        cb(jsonErr, json);
+      }, ecd);
+    };
     callback(buffer);
   });
 };

package/lib/util/data-server.js

@@ -347,16 +347,16 @@ module.exports = function init(_proxy) {
     return result;
   }
   function checkClientIp(item, filter) {
-    var clientId = getClientId(item);
-    var ipList = filter.ipList;
     var clientIp = item.req.ip;
     if (filter.clientIp) {
-      return clientId === filter.clientIp;
+      return clientIp === filter.clientIp;
     }
     if (filter.ip && clientIp === filter.ip) {
       return true;
     }
     // 有 clientId 过滤条件时，必须匹配 clientId
+    var clientId = getClientId(item);
+    var ipList = filter.ipList;
     if (filter.clientId) {
       if (clientId === filter.clientId) {
         return true;

package/lib/util/index.js

@@ -1823,14 +1823,8 @@ exports.getClientPort = function (req) {
 
 exports.removeIPV6Prefix = removeIPV6Prefix;
 
-function isUrlEncoded(req) {
-  return (
-    /^post$/i.test(req.method) &&
-    /urlencoded/i.test(req.headers && req.headers['content-type'])
-  );
-}
+exports.isUrlEncoded = common.isUrlEncoded;
 
-exports.isUrlEncoded = isUrlEncoded;
 function isJSONContent(req) {
   if (!hasRequestBody(req)) {
     return false;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "whistle",
   "description": "HTTP, HTTP2, HTTPS, Websocket debugging proxy",
-  "version": "2.9.37",
+  "version": "2.9.38-beta",
   "dataDirname": ".whistle",
   "localUIHost": "local.whistlejs.com",
   "port": 8899,