whatap 0.4.97 → 0.5.0

package/lib/observers/grpc-observer.js

@@ -0,0 +1,339 @@
+/**
+ * Copyright 2016 the WHATAP project authors. All rights reserved.
+ * Use of this source code is governed by a license that
+ * can be found in the LICENSE file.
+ */
+
+var TraceContextManager = require('../trace/trace-context-manager'),
+    conf = require('../conf/configure'),
+    Logger = require('../logger');
+const { Detector: URLPatternDetector } = require("../trace/serviceurl-pattern-detector");
+const HashUtil = require("../util/hashutil");
+const DataTextAgent = require("../data/datatext-agent");
+const ResourceProfile = require("../util/resourceprofile");
+const ProfilePack = require('../pack/profile-pack');
+const TxRecord = require('../service/tx-record');
+const DateUtil = require('../util/dateutil');
+const SecurityMaster = require('../net/security-master');
+const DataProfileAgent = require("../data/dataprofile-agent");
+const MessageStep = require("../step/message-step");
+const MeterService = require('../counter/meter/meter-service').MeterService;
+
+const shimmer = require('../core/shimmer');
+
+var grpc_profile_enabled = conf.getProperty('grpc_profile_enabled', true);
+var grpc_profile_stream_client_enabled = conf.getProperty('grpc_profile_stream_client_enabled', true);
+var grpc_profile_stream_server_enabled = conf.getProperty('grpc_profile_stream_server_enabled', true);
+var grpc_profile_ignore_method = conf.getProperty('grpc_profile_ignore_method', '');
+var ignore_method_set = null;
+conf.on('grpc_profile_enabled', function(newProperty) {
+    grpc_profile_enabled = newProperty;
+});
+conf.on('grpc_profile_stream_client_enabled', function(newProperty) {
+    grpc_profile_stream_client_enabled = newProperty;
+});
+conf.on('grpc_profile_stream_server_enabled', function(newProperty) {
+    grpc_profile_stream_server_enabled = newProperty;
+});
+conf.on('grpc_profile_ignore_method', function(newProperty) {
+    grpc_profile_ignore_method = newProperty;
+});
+
+var GRpcObserver = function(agent) {
+    this.agent = agent;
+    this.packages = ['@grpc/grpc-js'];
+};
+
+const types = {
+    unary: 'unary',
+    client_stream: 'clientStream',
+    server_stream: 'serverStream',
+    bidi: 'bidi'
+};
+
+var _module;
+
+GRpcObserver.prototype.inject = function(mod, moduleName) {
+    if (mod.__whatap_observe__) {
+        return;
+    }
+    _module = mod;
+    mod.__whatap_observe__ = true;
+    Logger.initPrint("GRpcObserver");
+
+    if (grpc_profile_enabled) {
+        shimmer.wrap(mod.Server.prototype, 'register', wrapRegister);
+        shimmer.wrap(mod.Client.prototype, 'register', wrapRegister);
+    }
+};
+
+function checkIgnoreMethod(ignore_method, method_name){
+    try{
+        if (ignore_method) {
+            ignore_method_set = new Set(ignore_method.split(','));
+        } else {
+            ignore_method_set = null;
+        }
+        if (ignore_method_set && ignore_method_set.has(method_name)) {
+            return true;
+        }
+    }catch (e) {
+        Logger.printError('WHATAP-703', 'gRPC checkIgnoreMethod error: ' + e, false);
+    }
+    return false;
+}
+
+function wrapHandler(handler, methodName, type) {
+    return function(call, callback) {
+        var method_name = methodName.includes('/') ? methodName.substring(methodName.lastIndexOf('/')+1, methodName.length) : methodName
+        if (!grpc_profile_enabled || checkIgnoreMethod(conf.getProperty('grpc_profile_ignore_method', ''), method_name)) {
+            return handler.call(this, call, callback);
+        }
+
+        TraceContextManager._asyncLocalStorage.run(initCtx(), () => {
+            try {
+                var ctx = TraceContextManager._asyncLocalStorage.getStore();
+                if (!ctx) {
+                    return handler.call(this, call, callback);
+                }
+
+                ctx.service_name = methodName;
+                ctx.service_hash = HashUtil.hashFromString(methodName);
+
+                var step_type = new MessageStep();
+                step_type.hash = HashUtil.hashFromString('Type');
+                step_type.start_time = ctx.getElapsedTime();
+                step_type.desc = type;
+                DataTextAgent.MESSAGE.add(step_type.hash, "Type");
+                ctx.profile.push(step_type);
+
+                var step_method = new MessageStep();
+                step_method.hash = HashUtil.hashFromString('Method');
+                step_method.start_time = ctx.getElapsedTime();
+                step_method.desc = method_name;
+                DataTextAgent.MESSAGE.add(step_method.hash, "Method");
+                ctx.profile.push(step_method);
+
+                if(call.request && Object.keys(call.request).length > 0){
+                    var step_param = new MessageStep();
+                    step_param.hash = HashUtil.hashFromString('Parameter');
+                    step_param.start_time = ctx.getElapsedTime();
+                    step_param.desc = JSON.stringify(Object.keys(call.request));
+                    DataTextAgent.MESSAGE.add(step_param.hash, "Parameter");
+                    ctx.profile.push(step_param);
+                }
+
+                console.log(call.pendingStatus)
+
+                ctx.grpc_method = method_name;
+
+                function wrappedCallback(err, response, ctx) {
+
+                    if (err) {
+                        ctx.error = err.stack;
+                        ctx.statusCode = 500;
+
+                        var step_error = new MessageStep();
+                        step_error.hash = HashUtil.hashFromString("EXCEPTION");
+                        step_error.start_time = ctx.getElapsedTime();
+                        step_error.desc = err.stack;
+                        DataTextAgent.MESSAGE.add(step_error.hash, "EXCEPTION");
+                        ctx.profile.push(step_error);
+                    }
+
+                    endTransaction(ctx);
+
+                    if (typeof callback === 'function') {
+                        return callback(err, response);
+                    }
+                }
+
+                return handler.call(this, call, (err, response) => wrappedCallback(err, response, ctx));
+                // return handler.call(this, call, (err, response) => {
+                //     if (!err) {
+                //         err = new Error('Exception error occured');
+                //     }
+                //     wrappedCallback(err, response, ctx);
+            } catch (err) {
+                Logger.printError('WHATAP-701', 'gRPC wrapHandler error: ' + err, false);
+
+                ctx.error = err.stack;
+
+                var step_error = new MessageStep();
+                step_error.hash = HashUtil.hashFromString("EXCEPTION");
+                step_error.start_time = ctx.getElapsedTime();
+                step_error.desc = err.stack;
+                DataTextAgent.MESSAGE.add(step_error.hash, "EXCEPTION");
+                ctx.profile.push(step_error);
+
+                endTransaction(ctx);
+
+                return handler.call(this, call, callback);
+            }
+        });
+    };
+}
+
+function wrapStreamHandler(handler, methodName, type) {
+    return function(call, callback) {
+        var method_name = methodName.includes('/') ? methodName.substring(methodName.lastIndexOf('/')+1, methodName.length) : methodName
+        if (!grpc_profile_enabled || checkIgnoreMethod(conf.getProperty('grpc_profile_ignore_method', ''), method_name)) {
+            return handler.call(this, call, callback);
+        }
+
+        switch (type) {
+            case 'server_stream':
+                if (!grpc_profile_stream_server_enabled) {
+                    return handler.call(this, call, callback);
+                }
+                break;
+            case 'client_stream':
+                if (!grpc_profile_stream_client_enabled) {
+                    return handler.call(this, call, callback);
+                }
+                break;
+            case 'bidi':
+                if (!grpc_profile_stream_server_enabled || !grpc_profile_stream_client_enabled) {
+                    return handler.call(this, call, callback);
+                }
+                break;
+        }
+
+        TraceContextManager._asyncLocalStorage.run(initCtx(), () => {
+            try {
+                var ctx = TraceContextManager._asyncLocalStorage.getStore();
+                if (!ctx) {
+                    return handler.call(this, call, callback);
+                }
+
+                ctx.service_name = methodName;
+                ctx.service_hash = HashUtil.hashFromString(methodName);
+
+                var step_type = new MessageStep();
+                step_type.hash = HashUtil.hashFromString('Type');
+                step_type.start_time = ctx.getElapsedTime();
+                step_type.desc = type;
+                DataTextAgent.MESSAGE.add(step_type.hash, "Type");
+                ctx.profile.push(step_type);
+
+                var step_method = new MessageStep();
+                step_method.hash = HashUtil.hashFromString('Method');
+                step_method.start_time = ctx.getElapsedTime();
+                step_method.desc = method_name;
+                DataTextAgent.MESSAGE.add(step_method.hash, "Method");
+                ctx.profile.push(step_method);
+
+                ctx.grpc_method = method_name;
+
+                const startTime = Date.now();
+
+                call.on('end', () => {
+                    const duration = Date.now() - startTime;
+                    endTransaction(ctx);
+                });
+
+                call.once('cancelled', () => {
+                    ctx.cancelled = true;
+                    var step_cancel = new MessageStep();
+                    step_cancel.hash = HashUtil.hashFromString("CANCELLED");
+                    step_cancel.start_time = ctx.getElapsedTime();
+                    step_cancel.desc = "Request cancelled";
+                    DataTextAgent.MESSAGE.add(step_cancel.hash, "CANCELLED");
+                    ctx.profile.push(step_cancel);
+                    endTransaction(ctx);
+                });
+
+                call.on('error', (err) => {
+                    ctx.error = err.stack;
+                    ctx.statusCode = err.code || grpc.status.INTERNAL;
+
+                    var step_error = new MessageStep();
+                    step_error.hash = HashUtil.hashFromString("EXCEPTION");
+                    step_error.start_time = ctx.getElapsedTime();
+                    step_error.desc = err.stack;
+                    DataTextAgent.MESSAGE.add(step_error.hash, "EXCEPTION");
+                    ctx.profile.push(step_error);
+
+                    endTransaction(ctx);
+                });
+
+                return handler.call(this, call);
+            } catch (e) {
+                Logger.printError('WHATAP-702', 'gRPC wrapStreamHandler error: ' + e, false);
+                return handler.call(this, call);
+            }
+        });
+    };
+}
+
+function wrapRegister(register) {
+    return function(name, handler, serialize, deserialize, type) {
+        if (typeof handler === 'function') {
+            switch (type) {
+                case types.client_stream:
+                    handler = wrapStreamHandler(handler, name, 'clientStream');
+                    break;
+                case types.server_stream:
+                    handler = wrapStreamHandler(handler, name, 'serverStream');
+                    break;
+                case types.bidi:
+                    handler = wrapStreamHandler(handler, name, 'bidi');
+                    break;
+                case types.unary:
+                default:
+                    handler = wrapHandler(handler, name, 'unary');
+            }
+        }
+        return register.call(this, name, handler, serialize, deserialize, type);
+    };
+}
+
+function endTransaction(ctx) {
+    try {
+        var profile = new ProfilePack();
+        var wtx = new TxRecord();
+        wtx.endTime = DateUtil.currentTime();
+        profile.time = wtx.endTime;
+        wtx.elapsed = ctx.getElapsedTime();
+
+        DataTextAgent.SERVICE.add(ctx.service_hash, ctx.service_name);
+
+        wtx.seq = ctx.txid;
+        wtx.service = ctx.service_hash;
+        wtx.cpuTime = ResourceProfile.getCPUTime() - ctx.start_cpu;
+        wtx.malloc = ResourceProfile.getUsedHeapSize() - ctx.start_malloc;
+        if (wtx.malloc < 0) { wtx.malloc = 0; }
+        wtx.status = 2;
+        wtx.http_method = ctx.grpc_method;
+
+        wtx.ipaddr = ctx.remoteIp;
+
+        MeterService.add(wtx.service, wtx.elapsed,
+            wtx.errorLevel, ctx.mcaller_pcode, ctx.mcaller_okind, ctx.mcaller_oid);
+
+        profile.oid = SecurityMaster.OID;
+        profile.service = wtx;
+
+        TraceContextManager.end(ctx._id);
+
+        setTimeout(function() {
+            DataProfileAgent.sendProfile(ctx, profile, false);
+            ctx = null;
+        }, 100);
+    } catch (e) {
+        Logger.printError('WHATAP-615', 'Websocket end transaction error..', e, false);
+        TraceContextManager.end(ctx._id);
+        ctx = null;
+    }
+};
+
+function initCtx() {
+    const ctx = TraceContextManager.start();
+    if (!ctx) { return; }
+
+    ctx.start_malloc = ResourceProfile.getUsedHeapSize();
+    ctx.start_cpu = ResourceProfile.getCPUTime();
+    return ctx;
+}
+
+exports.GRpcObserver = GRpcObserver;

package/lib/observers/http-observer.js

@@ -59,8 +59,8 @@ var httpc_not_found_ignore_time = conf.getProperty('httpc_not_found_ignore_time'
 var profile_http_header_ignore_keys = conf.getProperty('profile_http_header_ignore_keys', 'Cookie,cookie,accept,user-agent,referer');
 var profile_http_parameter_enabled = conf.getProperty('profile_http_parameter_enabled', true);
 var profile_http_parameter_keys = conf.getProperty('profile_http_parameter_keys', '');
-var ignore_nextjs_build_file_enabled = conf.getProperty('ignore_nextjs_build_file_enabled', true);
-var ignore_nextjs_build_file_path = conf.getProperty('ignore_nextjs_build_file_path', '/_next/');
+var ignore_build_file_enabled = conf.getProperty('ignore_build_file_enabled', true);
+var ignore_build_file_path = conf.getProperty('ignore_build_file_path', '/_next/');
 conf.on('trace_http_client_ip_header_key', function(newProperty) {
     configIpHeaderKey = newProperty;
 });
@@ -118,11 +118,11 @@ conf.on('profile_http_parameter_enabled', function (newProps) {
 conf.on('profile_http_parameter_keys', function (newProps) {
     profile_http_parameter_keys = newProps;
 })
-conf.on('ignore_nextjs_build_file_enabled', function (newProps) {
-    ignore_nextjs_build_file_enabled = newProps;
+conf.on('ignore_build_file_enabled', function (newProps) {
+    ignore_build_file_enabled = newProps;
 })
-conf.on('ignore_nextjs_build_file_path', function (newProps) {
-    ignore_nextjs_build_file_path = newProps;
+conf.on('ignore_build_file_path', function (newProps) {
+    ignore_build_file_path = newProps;
 })
 var staticConents = function (newProps) {
     var x=new Set();
@@ -241,7 +241,7 @@ function initCtx(req, res) {
     /*url이 없으면 추적하지 않는다*/
     if(!req.url) { return null; }
     if(ignore_http_method && ignore_http_method.toUpperCase().split(',').includes(req.method)) { return null; }
-    if (ignore_nextjs_build_file_enabled && ignore_nextjs_build_file_path && ignore_nextjs_build_file_path.split(',').some(path => req.url.startsWith(path))) {
+    if (ignore_build_file_enabled && ignore_build_file_path && ignore_build_file_path.split(',').some(path => req.url.startsWith(path))) {
         return null;
     }
 

package/lib/observers/process-observer.js

@@ -4,23 +4,72 @@
  * can be found in the LICENSE file.
  */
 
-var TraceContextManager = require('../trace/trace-context-manager');
+const TraceContextManager = require('../trace/trace-context-manager');
+const conf = require('../conf/configure');
+const LogTracer = require('../logsink/log-tracer');
 
-var ProcessObserver = function (agent) {
+let logsink_enabled = conf.getProperty('logsink_enabled', false);
+let logTracer = logsink_enabled ? new LogTracer() : null;
+
+conf.on('logsink_enabled', function(newProperty) {
+    logsink_enabled = newProperty;
+    logTracer = logsink_enabled ? new LogTracer() : null;
+});
+
+const ProcessObserver = function (agent) {
     this.agent = agent;
     this.packages = ['process'];
 };
+
 ProcessObserver.prototype.inject = function (mod, moduleName) {
-    var self = this;
-    self.agent.aop.before(mod, 'nextTick', function (obj, args) {
-        var cached_id = TraceContextManager.getCurrentId();
-        self.agent.aop.functionHook(args, -1, function (obj, args) {
-            if(cached_id != null) {
+    this._hookNextTick(mod);
+    this._hookStdOutWrite();
+    this._hookStdErrWrite();
+};
+
+ProcessObserver.prototype._hookNextTick = function (mod) {
+    this.agent.aop.before(mod, 'nextTick', (obj, args) => {
+        const cached_id = TraceContextManager.getCurrentId();
+        this.agent.aop.functionHook(args, -1, (obj, args) => {
+            if (cached_id != null) {
                 TraceContextManager.resume(cached_id);
-                cached_id = null;
             }
         });
     });
 };
 
+ProcessObserver.prototype._hookStdOutWrite = function () {
+    this.agent.aop.after(process.stdout, 'write', (obj, args) => {
+        if (conf.getProperty('logsink_enabled', false) && args[0]) {
+            return;
+            let content = typeof args[0] === 'string' ? args[0] : (args[0].message || '');
+            try {
+                const parsedContent = JSON.parse(content);
+                content = parsedContent.message ? parsedContent.message : JSON.stringify(parsedContent);
+            } catch (e) {
+            }
+            if (logTracer && content) {
+                logTracer.addStdWrite(content, conf.logsink_category_stdout);
+            }
+        }
+    });
+};
+
+ProcessObserver.prototype._hookStdErrWrite = function () {
+    this.agent.aop.after(process.stderr, 'write', (obj, args) => {
+        if (conf.getProperty('logsink_enabled', false) && args[0]) {
+            return;
+            let content = typeof args[0] === 'string' ? args[0] : (args[0].message || '');
+            try {
+                const parsedContent = JSON.parse(content);
+                content = parsedContent.message ? parsedContent.message : JSON.stringify(parsedContent);
+            } catch (e) {
+            }
+            if (logTracer && content) {
+                logTracer.addStdWrite(content, conf.logsink_category_stderr);
+            }
+        }
+    });
+};
+
 module.exports.ProcessObserver = ProcessObserver;

package/lib/observers/redis-observer.js

@@ -6,12 +6,12 @@ var TraceContextManager = require('../trace/trace-context-manager'),
     HashUtil = require('../util/hashutil'),
     Long = require('long'),
     Logger = require('../logger');
-
+const shimmer = require('../core/shimmer');
 
 var RedisObserver = function (agent) {
     this.agent = agent;
     this.aop = agent.aop;
-    this.packages = ['redis'];
+    this.packages = ['redis', 'ioredis'];
 };
 
 RedisObserver.prototype.inject = function (mod, modName) {
@@ -23,13 +23,13 @@ RedisObserver.prototype.inject = function (mod, modName) {
     Logger.initPrint("RedisObserver");
     var self = this;
 
-    var dbc_hash= 0;
+    var dbc_hash = 0;
     var dbc;
     var dbc_time;
     self.aop.both(mod, 'createClient',
         function (obj, args, lctx) {
-            if(dbc_hash === 0){
-                if(args.length > 0) {
+            if (dbc_hash === 0) {
+                if (args.length > 0) {
                     var info = (args[0] || {});
                     dbc = 'redis://';
                     dbc += info.host || '';
@@ -43,13 +43,15 @@ RedisObserver.prototype.inject = function (mod, modName) {
             var selectCommand = new Set(['get', 'hGet', 'hmGet']);
             var insertCommand = new Set(['set', 'hSet', 'hset', 'hmSet', 'hmset', 'zAdd', 'zadd', 'lSet', 'lset']);
             var updateCommand = new Set(['set', 'lSet', 'lset', 'hSet', 'hset', 'zAdd', 'zadd']);
-            var deleteCommand = new Set(['del', 'lRem', 'sRem', 'srem', 'hDel',  'hdel', 'zRem', 'zrem']);
+            var deleteCommand = new Set(['del', 'lRem', 'sRem', 'srem', 'hDel', 'hdel', 'zRem', 'zrem']);
             var commands = new Set([...selectCommand, ...insertCommand, ...updateCommand, ...deleteCommand]);
             commands.forEach(function (command) {
                 self.aop.after(ret, command,
                     function (obj, args, ret, lctx) {
                         var ctx = TraceContextManager.getCurrentContext();
-                        if(ctx == null) {return;}
+                        if (ctx == null) {
+                            return;
+                        }
 
                         var dbc_step = new DBCStep();
                         dbc_step.hash = dbc_hash;
@@ -62,18 +64,18 @@ RedisObserver.prototype.inject = function (mod, modName) {
                             sql_step = new SqlStepX();
                             const key = args[0];
                             const values = [];
-                            for(let i = 1; i < args.length; i++)
+                            for (let i = 1; i < args.length; i++)
                                 values.push(args[i]);
 
                             var sql = 'Redis ' + command + ': ' + JSON.stringify([key]);
-                            if(values.length > 0)
-                                sql += ' ' + JSON.stringify(values);
+                            // if (values.length > 0)
+                            //     sql += ' ' + JSON.stringify(values);
                             sql_step.hash = HashUtil.hashFromString(sql);
                             sql_step.start_time = ctx.getElapsedTime();
                             //sql_step.crud = 'I'.charCodeAt(0);
                             // DataTextAgent.SQL.add(sql_step.hash, sql);
                             ctx.profile.push(sql_step);
-                        } catch(e) {
+                        } catch (e) {
                             Logger.printError("WHATAP-605", "Redis CRUD error", e, false);
                             sql_step = null;
                         }
@@ -152,6 +154,62 @@ RedisObserver.prototype.inject = function (mod, modName) {
                 // }
             });
         });
+
+    var ioredis_dbc_hash = 0;
+    var ioredis_dbc;
+    self.aop.both(mod.prototype, 'sendCommand', function (obj, args, lctx) {
+        if (ioredis_dbc_hash === 0) {
+            if (obj && Object.keys(obj.options).length > 0) {
+                var info = obj.options
+                ioredis_dbc = 'redis://';
+                ioredis_dbc += info.host || '';
+                ioredis_dbc += ':';
+                ioredis_dbc += info.port || '';
+                ioredis_dbc_hash = HashUtil.hashFromString(ioredis_dbc);
+            }
+        }
+    }, function (obj, args, ret, lctx) {
+        var selectCommand = new Set(['get', 'hget', 'hmget', 'zrange', 'smembers', 'lrange']);
+        var insertCommand = new Set(['set', 'hset', 'hmset', 'zadd', 'lset', 'sadd', 'lpush', 'rpush']);
+        var updateCommand = new Set(['set', 'lset', 'hset', 'zadd']);
+        var deleteCommand = new Set(['del', 'lrem', 'srem', 'hdel', 'zrem']);
+        var commands = new Set([...selectCommand, ...insertCommand, ...updateCommand, ...deleteCommand]);
+
+        if(!args[0] || (args[0] && !args[0].name)){
+            return;
+        }
+        var command = args[0].name;
+        if(commands.has(command.toLowerCase())){
+            var ctx = TraceContextManager.getCurrentContext();
+            if (ctx == null) {
+                return;
+            }
+
+            var dbc_step = new DBCStep();
+            dbc_step.hash = ioredis_dbc_hash;
+            dbc_step.start_time = ctx.getElapsedTime();
+            DataTextAgent.DBC.add(ioredis_dbc_hash, ioredis_dbc);
+            ctx.profile.push(dbc_step);
+
+            var sql_step;
+            var args = args[0].args;
+            try {
+                sql_step = new SqlStepX();
+                let key = args[0];
+                if(typeof key === 'string')
+                    key = [key];
+
+                var sql = 'Redis ' + command + ': ' + JSON.stringify(key);
+                sql_step.hash = HashUtil.hashFromString(sql);
+                sql_step.start_time = ctx.getElapsedTime();
+                DataTextAgent.SQL.add(sql_step.hash, sql);
+                ctx.profile.push(sql_step);
+            } catch (e) {
+                Logger.printError("WHATAP-605", "Redis CRUD error", e, false);
+                sql_step = null;
+            }
+        }
+    })
 };
 
 exports.RedisObserver = RedisObserver;

package/lib/observers/websocket-observer.js

@@ -24,7 +24,7 @@ const MeterUsers = require("whatap/lib/counter/meter/meter-users");
 const DataPackSender = require("whatap/lib/data/datapack-sender");
 const MeterService        = require('../counter/meter/meter-service').MeterService;
 
-var trace_background_socket_enabled = conf.getProperty('trace_background_socket_enabled', false);
+var trace_background_socket_enabled = conf.getProperty('trace_background_socket_enabled', true);
 conf.on('trace_background_socket_enabled', function (newProps) {
     trace_background_socket_enabled = newProps;
 })

package/lib/pack/event-pack.js

@@ -27,7 +27,7 @@ EventPack.prototype.write = function(dout) {
     dout.writeText(this.message);
     dout.writeByte(Object.keys(this.attr).length);
 
-    for(key in this.attr) {
+    for(var key in this.attr) {
         dout.writeText(key);
         dout.writeText(this.attr[key]);
     }