un-cli 0.0.69 → 0.0.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adminlog.mjs +12 -3
- package/cmd.txt +1 -1
- package/index.html +677 -0
- package/index.mjs +141 -87
- package/logger.mjs +8 -1
- package/makerequest.mjs +14 -2
- package/package.json +2 -4
- package/portal.mjs +5 -4
- package/utilitynetwork.node.mjs +7 -5
package/adminlog.mjs
CHANGED
|
@@ -2,8 +2,6 @@
|
|
|
2
2
|
//Author : Hussein Nasser
|
|
3
3
|
//Date : Oct-13-2021
|
|
4
4
|
//Twitter: @hnasr
|
|
5
|
-
import fetch from "node-fetch";
|
|
6
|
-
import { logger } from "./logger.mjs"
|
|
7
5
|
|
|
8
6
|
export class AdminLog {
|
|
9
7
|
|
|
@@ -13,12 +11,22 @@ export class AdminLog {
|
|
|
13
11
|
this.token = token;
|
|
14
12
|
}
|
|
15
13
|
|
|
16
|
-
|
|
14
|
+
async getFetch() {
|
|
15
|
+
try {
|
|
16
|
+
const nodeFetch = await import ("node-fetch");
|
|
17
|
+
return nodeFetch.default;
|
|
18
|
+
}
|
|
19
|
+
catch(ex) {
|
|
20
|
+
return fetch;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
async query (codes, serviceName ="*", pageSize = 100000, startTime = null, endTime = null, logLevel = "DEBUG")
|
|
17
24
|
{
|
|
18
25
|
const url = this.adminServerUrl + "/logs/query?f=pjson"
|
|
19
26
|
const level = logLevel
|
|
20
27
|
const filterType="json"
|
|
21
28
|
const token = this.token
|
|
29
|
+
|
|
22
30
|
const filter = {
|
|
23
31
|
"codes": codes,
|
|
24
32
|
"services": serviceName
|
|
@@ -32,6 +40,7 @@ export class AdminLog {
|
|
|
32
40
|
queryLogUrl += `&endTime=${endTime}`
|
|
33
41
|
|
|
34
42
|
//logger.info(queryLogUrl);
|
|
43
|
+
const fetch = await this.getFetch()
|
|
35
44
|
return fetch(queryLogUrl);
|
|
36
45
|
}
|
|
37
46
|
|
package/cmd.txt
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
|
|
1
|
+
update subnetworks --all --desc
|
|
2
2
|
exit
|
package/index.html
ADDED
|
@@ -0,0 +1,677 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8">
|
|
5
|
+
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
|
6
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
7
|
+
<title>UN Logs</title>
|
|
8
|
+
</head>
|
|
9
|
+
<body>
|
|
10
|
+
|
|
11
|
+
<table>
|
|
12
|
+
<tr><td>Service</td>
|
|
13
|
+
<td colspan =3 ><select id ='cmbService'>
|
|
14
|
+
<option>(All Services)</option>
|
|
15
|
+
</select>
|
|
16
|
+
</td>
|
|
17
|
+
</tr>
|
|
18
|
+
|
|
19
|
+
<tr>
|
|
20
|
+
<td >Age</td>
|
|
21
|
+
<td colspan =3><input type ='text' id = 'txtAge' value = '30' style="width:30px"> minutes</td>
|
|
22
|
+
</tr>
|
|
23
|
+
<tr colspan =4>
|
|
24
|
+
<td><button id = 'btnTraceLogs'>Trace Logs</button></td>
|
|
25
|
+
<td><button id = 'btnValidateLogs'>Validate Logs</button></td>
|
|
26
|
+
<td><button id = 'btnUpdateSubnetworkLogs'>Update Subnetwork Logs</button></td>
|
|
27
|
+
<td><button id = 'btnAR'>Attribute Rules Logs</button></td>
|
|
28
|
+
<td><button id = 'btnSQLLogs'>SQL Logs</button></td>
|
|
29
|
+
|
|
30
|
+
</tr>
|
|
31
|
+
|
|
32
|
+
</table>
|
|
33
|
+
|
|
34
|
+
<table id = 'tblResult' border =1 cellpadding = 4 cellspacing = 4 style="text-align:center">
|
|
35
|
+
|
|
36
|
+
</table>
|
|
37
|
+
<script type = 'module'>
|
|
38
|
+
import { Portal } from "./portal.mjs"
|
|
39
|
+
import { UtilityNetwork } from "./utilitynetwork.node.mjs"
|
|
40
|
+
import { AdminLog } from "./adminlog.mjs"
|
|
41
|
+
import logger from "./logger.mjs"
|
|
42
|
+
|
|
43
|
+
const parameters = {
|
|
44
|
+
"user": "unadmin",
|
|
45
|
+
"password": "unadmin.109",
|
|
46
|
+
"portal": "https://utilitynetwork.esri.com/portal",
|
|
47
|
+
"service": "RedTrolley_SQLServer",
|
|
48
|
+
"referer": "https://utilitynetwork.esri.com/log"
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
let portal;
|
|
52
|
+
let adminLog = null;
|
|
53
|
+
let cmbService
|
|
54
|
+
let token;
|
|
55
|
+
async function init() {
|
|
56
|
+
|
|
57
|
+
cmbService = document.getElementById("cmbService")
|
|
58
|
+
|
|
59
|
+
token = await getToken(parameters);
|
|
60
|
+
const services = await portal.services()
|
|
61
|
+
const featureServices = services.services.filter(s => s.type == "FeatureServer")
|
|
62
|
+
featureServices.forEach(s=> {
|
|
63
|
+
const o = document.createElement("option");
|
|
64
|
+
o.text = s.name;
|
|
65
|
+
cmbService.appendChild(o);
|
|
66
|
+
})
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
async function getToken(parameters) {
|
|
71
|
+
portal = new Portal(parameters.portal, parameters.user, parameters.password,300, parameters.server, parameters.referer)
|
|
72
|
+
console.log("About to connect..")
|
|
73
|
+
const token = await portal.connect()
|
|
74
|
+
console.log(`Token generanted successfully.`)
|
|
75
|
+
return token;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
async function loadTraceLogs () {
|
|
80
|
+
|
|
81
|
+
//build table
|
|
82
|
+
const tblResult = document.getElementById("tblResult");
|
|
83
|
+
//clear
|
|
84
|
+
while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
|
|
85
|
+
tblResult.appendChild(document.createTextNode("Loading...."));
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
parameters.service = cmbService.options[cmbService.selectedIndex].text
|
|
89
|
+
|
|
90
|
+
//create a new admin object (user might not be admin we won't use it until the user call log )
|
|
91
|
+
adminLog = new AdminLog(token, portal.serverUrl)
|
|
92
|
+
|
|
93
|
+
logger.info("Connected.")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
const topLogCount = 1000;
|
|
97
|
+
const pageSize = 10000
|
|
98
|
+
|
|
99
|
+
let mins = document.getElementById("txtAge").value //query logs for the last 30 minutes
|
|
100
|
+
|
|
101
|
+
console.log(`Querying trace logs for ${parameters.service} for the last ${mins} minutes ...`)
|
|
102
|
+
const startTime = Date.now() - mins*60*1000
|
|
103
|
+
const endTime = Date.now();
|
|
104
|
+
let services = [parameters.service+ ".MapServer"]
|
|
105
|
+
if (parameters.service == "(All Services)")
|
|
106
|
+
services = "*"
|
|
107
|
+
let result= await adminLog.query([102002], services, topLogCount, startTime ,endTime , "VERBOSE")
|
|
108
|
+
let jsonRes = await result.json()
|
|
109
|
+
let allMessages = [].concat(jsonRes.logMessages)
|
|
110
|
+
allMessages = allMessages.filter(m => m.message.indexOf("------ Trace Parameters ----") > -1)
|
|
111
|
+
while (jsonRes.hasMore)
|
|
112
|
+
{
|
|
113
|
+
//start paging
|
|
114
|
+
logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
|
|
115
|
+
result= await adminLog.query([102002], services, pageSize, jsonRes.endTime, null, "VERBOSE")
|
|
116
|
+
jsonRes = await result.json()
|
|
117
|
+
|
|
118
|
+
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("------ Trace Parameters ----") > -1))
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
await populateMessages(allMessages)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
async function loadValidateLogs () {
|
|
129
|
+
//build table
|
|
130
|
+
const tblResult = document.getElementById("tblResult");
|
|
131
|
+
//clear
|
|
132
|
+
while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
|
|
133
|
+
tblResult.appendChild(document.createTextNode("Loading...."));
|
|
134
|
+
|
|
135
|
+
parameters.service = cmbService.options[cmbService.selectedIndex].text
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
//create a new admin object (user might not be admin we won't use it until the user call log )
|
|
139
|
+
adminLog = new AdminLog(token, portal.serverUrl)
|
|
140
|
+
|
|
141
|
+
logger.info("Connected.")
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
const topLogCount = 1000;
|
|
145
|
+
const pageSize = 10000
|
|
146
|
+
|
|
147
|
+
let mins = document.getElementById("txtAge").value //query logs for the last 30 minutes
|
|
148
|
+
|
|
149
|
+
console.log(`Querying validate logs for ${parameters.service} for the last ${mins} minutes ...`)
|
|
150
|
+
const startTime = Date.now() - mins*60*1000
|
|
151
|
+
const endTime = Date.now();
|
|
152
|
+
let services = [parameters.service+ ".MapServer"]
|
|
153
|
+
if (parameters.service == "(All Services)")
|
|
154
|
+
services = "*"
|
|
155
|
+
let result= await adminLog.query([102003], services, topLogCount, startTime ,endTime , "VERBOSE")
|
|
156
|
+
let jsonRes = await result.json()
|
|
157
|
+
let allMessages = [].concat(jsonRes.logMessages)
|
|
158
|
+
allMessages = allMessages.filter(m => m.message.indexOf("-------- Environment ---") > -1 && m.message.indexOf("The network is built.") > -1 && m.methodName == 'BuildEngineLog')
|
|
159
|
+
while (jsonRes.hasMore)
|
|
160
|
+
{
|
|
161
|
+
//start paging
|
|
162
|
+
logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
|
|
163
|
+
result= await adminLog.query([102003], services, pageSize, jsonRes.endTime, null, "VERBOSE")
|
|
164
|
+
jsonRes = await result.json()
|
|
165
|
+
|
|
166
|
+
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("-------- Environment ---") > -1 && m.message.indexOf("------ Trace Parameters ----") == -1))
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
//validate logs missing elapsed populate it
|
|
173
|
+
allMessages = allMessages.map( m => {
|
|
174
|
+
try{
|
|
175
|
+
|
|
176
|
+
//The network is built. 0.093 seconds (4.982 total) - 29 MB memory
|
|
177
|
+
|
|
178
|
+
let re = /The network is built. [-+]?([0-9]*\.[0-9]+|[0-9]+) seconds \([-+]?([0-9]*\.[0-9]+|[0-9]+) total\)/;
|
|
179
|
+
let res = re.exec(m.message)
|
|
180
|
+
if (res && res.length > 1)
|
|
181
|
+
m.elapsed = res[2]
|
|
182
|
+
|
|
183
|
+
return m;
|
|
184
|
+
}
|
|
185
|
+
catch(ex){
|
|
186
|
+
return m;
|
|
187
|
+
}
|
|
188
|
+
})
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
await populateMessages(allMessages)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
async function loadUpdateSubnetworkLogs () {
|
|
198
|
+
//build table
|
|
199
|
+
const tblResult = document.getElementById("tblResult");
|
|
200
|
+
//clear
|
|
201
|
+
while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
|
|
202
|
+
tblResult.appendChild(document.createTextNode("Loading...."));
|
|
203
|
+
|
|
204
|
+
parameters.service = cmbService.options[cmbService.selectedIndex].text
|
|
205
|
+
//create a new admin object (user might not be admin we won't use it until the user call log )
|
|
206
|
+
adminLog = new AdminLog(token, portal.serverUrl)
|
|
207
|
+
|
|
208
|
+
logger.info("Connected.")
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
const topLogCount = 1000;
|
|
212
|
+
const pageSize = 10000
|
|
213
|
+
|
|
214
|
+
let mins = document.getElementById("txtAge").value //query logs for the last 30 minutes
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
console.log(`Querying subnetwork logs for ${parameters.service} for the last ${mins} minutes ...`)
|
|
218
|
+
const startTime = Date.now() - mins*60*1000
|
|
219
|
+
const endTime = Date.now();
|
|
220
|
+
let services = [parameters.service+ ".MapServer"]
|
|
221
|
+
if (parameters.service == "(All Services)")
|
|
222
|
+
services = "*"
|
|
223
|
+
let result= await adminLog.query([102003], services, topLogCount, startTime ,endTime , "VERBOSE")
|
|
224
|
+
let jsonRes = await result.json()
|
|
225
|
+
let allMessages = [].concat(jsonRes.logMessages)
|
|
226
|
+
allMessages = allMessages.filter(m => m.message.indexOf("---- Subnetwork Parameters ----") > -1)
|
|
227
|
+
while (jsonRes.hasMore)
|
|
228
|
+
{
|
|
229
|
+
//start paging
|
|
230
|
+
logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
|
|
231
|
+
result= await adminLog.query([102003], services, pageSize, jsonRes.endTime, null, "VERBOSE")
|
|
232
|
+
jsonRes = await result.json()
|
|
233
|
+
|
|
234
|
+
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("---- Subnetwork Parameters ----") > -1))
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
//update subnetwork missing elapsed populate it
|
|
238
|
+
allMessages = allMessages.map( m => {
|
|
239
|
+
try{
|
|
240
|
+
|
|
241
|
+
let re = /Total \([-+]?([0-9]*\.[0-9]+|[0-9]+) seconds\)/;
|
|
242
|
+
let res = re.exec(m.message)
|
|
243
|
+
if (res && res.length > 1)
|
|
244
|
+
m.elapsed = res[1]
|
|
245
|
+
|
|
246
|
+
re = /Total update subnetwork time \([-+]?([0-9]*\.[0-9]+|[0-9]+) seconds\)/;
|
|
247
|
+
res = re.exec(m.message)
|
|
248
|
+
if (res && res.length > 1)
|
|
249
|
+
m.elapsed = res[1]
|
|
250
|
+
|
|
251
|
+
return m;
|
|
252
|
+
}
|
|
253
|
+
catch(ex){
|
|
254
|
+
return m;
|
|
255
|
+
}
|
|
256
|
+
})
|
|
257
|
+
await populateMessages(allMessages)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
async function loadAttributeRules () {
|
|
265
|
+
|
|
266
|
+
//build table
|
|
267
|
+
const tblResult = document.getElementById("tblResult");
|
|
268
|
+
tblResult.appendChild(document.createTextNode("Loading...."));
|
|
269
|
+
|
|
270
|
+
let mins = document.getElementById("txtAge").value //query logs for the last 30 minutes
|
|
271
|
+
|
|
272
|
+
parameters.service = cmbService.options[cmbService.selectedIndex].text
|
|
273
|
+
//create a new admin object (user might not be admin we won't use it until the user call log )
|
|
274
|
+
adminLog = new AdminLog(token, portal.serverUrl)
|
|
275
|
+
|
|
276
|
+
let showMaxGuid = true
|
|
277
|
+
let showMinGuid = false
|
|
278
|
+
const startTime = Date.now() - mins*60*1000
|
|
279
|
+
const endTime = Date.now();
|
|
280
|
+
|
|
281
|
+
let services = [parameters.service+ ".MapServer"]
|
|
282
|
+
if (parameters.service == "(All Services)")
|
|
283
|
+
services = "*"
|
|
284
|
+
|
|
285
|
+
const pageSize = 10000 //maximum messages per page
|
|
286
|
+
logger.info(`Querying attribute rules logs for ${parameters.service} in the past ${mins} minutes...`)
|
|
287
|
+
let result= await adminLog.query([102003], services, pageSize, startTime ,endTime , "DEBUG")
|
|
288
|
+
let jsonRes = await result.json()
|
|
289
|
+
let allMessages = [].concat(jsonRes.logMessages)
|
|
290
|
+
|
|
291
|
+
while (jsonRes.hasMore && jsonRes.endTime > startTime)
|
|
292
|
+
{
|
|
293
|
+
//start paging
|
|
294
|
+
logger.info(`Aggregating messages... total so far ${allMessages.length} debug entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
result= await adminLog.query([102003], services, pageSize, jsonRes.endTime )
|
|
298
|
+
jsonRes = await result.json()
|
|
299
|
+
allMessages = allMessages.concat(jsonRes.logMessages)
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
const arMessages = allMessages
|
|
303
|
+
.filter(m => m.message.indexOf("Attribute rule execution complete:") > -1)
|
|
304
|
+
.map (m => JSON.parse(m.message.replace("Attribute rule execution complete:", "")))
|
|
305
|
+
.map( m => {
|
|
306
|
+
m["Elapsed Time (ms)"] = Math.round(m["Elapsed Time"]*1000000)/1000
|
|
307
|
+
// m["Arcade Evaluation Time:"] = Math.round(m["Arcade Evaluation Time:"]*1000,6)
|
|
308
|
+
//m.ArcadeTime = m["Arcade Evaluation Time:"]
|
|
309
|
+
|
|
310
|
+
delete m["Arcade Evaluation Time:"];
|
|
311
|
+
delete m["Elapsed Time"];
|
|
312
|
+
//delete m ['GlobalID'];
|
|
313
|
+
return m
|
|
314
|
+
})
|
|
315
|
+
.sort( (m1, m2) => m2["Elapsed Time (ms)"]- m1["Elapsed Time (ms)"])
|
|
316
|
+
.reduce( ( prev, cur ) => {
|
|
317
|
+
if (prev [cur["Rule name"]] === undefined)
|
|
318
|
+
{
|
|
319
|
+
prev [cur["Rule name"]] = {
|
|
320
|
+
"totalTime": 0,
|
|
321
|
+
"occurrence": 0,
|
|
322
|
+
"minTime": Number.MAX_SAFE_INTEGER,
|
|
323
|
+
"maxTime": -1,
|
|
324
|
+
"avgTime": 0,
|
|
325
|
+
"maxGuid": null,
|
|
326
|
+
"minGuid": null
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
prev [cur["Rule name"]].totalTime = prev [cur["Rule name"]].totalTime + cur["Elapsed Time (ms)"]
|
|
331
|
+
|
|
332
|
+
if (cur["Elapsed Time (ms)"] < prev [cur["Rule name"]].minTime ) {
|
|
333
|
+
prev [cur["Rule name"]].minTime = cur["Elapsed Time (ms)"];
|
|
334
|
+
prev [cur["Rule name"]].minGuid = cur["GlobalID"];
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
if (cur["Elapsed Time (ms)"] > prev [cur["Rule name"]].maxTime ){
|
|
338
|
+
prev [cur["Rule name"]].maxTime = cur["Elapsed Time (ms)"];
|
|
339
|
+
prev [cur["Rule name"]].maxGuid = cur["GlobalID"];
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
prev [cur["Rule name"]].occurrence++
|
|
343
|
+
|
|
344
|
+
prev [cur["Rule name"]].avgTime = prev [cur["Rule name"]].totalTime / prev [cur["Rule name"]].occurrence
|
|
345
|
+
|
|
346
|
+
return prev
|
|
347
|
+
}, {})
|
|
348
|
+
|
|
349
|
+
const rules = Object.keys(arMessages)
|
|
350
|
+
.map(a => {
|
|
351
|
+
const rule = {}
|
|
352
|
+
rule["Attribute Rule"] = a;
|
|
353
|
+
rule["Total Cost (ms)"] = parseFloat(arMessages[a].totalTime.toFixed(2))
|
|
354
|
+
if (!showMinGuid && !showMaxGuid) rule["Average Cost (ms)"] = parseFloat(arMessages[a].avgTime.toFixed(2))
|
|
355
|
+
rule["Max execution time (ms)"] = parseFloat(arMessages[a].maxTime.toFixed(2))
|
|
356
|
+
if (showMaxGuid) rule["Max GUID"] = arMessages[a].maxGuid
|
|
357
|
+
rule["Min execution time (ms)"] = parseFloat(arMessages[a].minTime.toFixed(2))
|
|
358
|
+
if (showMinGuid) rule["Min GUID"] = arMessages[a].minGuid
|
|
359
|
+
if (!showMinGuid && !showMaxGuid) rule["Occurrence"] = arMessages[a].occurrence;
|
|
360
|
+
return rule;
|
|
361
|
+
})
|
|
362
|
+
.sort( (m1, m2) => m2["Total Cost (ms)"] -m1["Total Cost (ms)"])
|
|
363
|
+
console.table(rules)
|
|
364
|
+
|
|
365
|
+
/*
|
|
366
|
+
Attribute Rule
|
|
367
|
+
│ Total Cost (ms)
|
|
368
|
+
│ Max execution time (ms)
|
|
369
|
+
│ Max GUID
|
|
370
|
+
│ Min execution time (m
|
|
371
|
+
|
|
372
|
+
*/
|
|
373
|
+
|
|
374
|
+
const header = document.createElement("tr");
|
|
375
|
+
const cAttributeRule = document.createElement("th");
|
|
376
|
+
cAttributeRule.textContent = "Attribute Rule"
|
|
377
|
+
|
|
378
|
+
const cTotalCost = document.createElement("th");
|
|
379
|
+
cTotalCost.textContent = "Total Cost (ms)"
|
|
380
|
+
|
|
381
|
+
const cMaxExecutionTime = document.createElement("th");
|
|
382
|
+
cMaxExecutionTime.textContent = "Max execution time (ms)"
|
|
383
|
+
|
|
384
|
+
const cMaxGuid = document.createElement("th");
|
|
385
|
+
cMaxGuid.textContent = "Max GUID"
|
|
386
|
+
|
|
387
|
+
const cMinExecutionTime = document.createElement("th");
|
|
388
|
+
cMinExecutionTime.textContent = "Min execution time (ms)"
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
header.appendChild (cAttributeRule)
|
|
392
|
+
header.appendChild (cTotalCost)
|
|
393
|
+
header.appendChild (cMaxExecutionTime)
|
|
394
|
+
header.appendChild (cMaxGuid)
|
|
395
|
+
header.appendChild (cMinExecutionTime)
|
|
396
|
+
|
|
397
|
+
tblResult.appendChild(header)
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
rules.forEach(rule =>
|
|
401
|
+
{
|
|
402
|
+
|
|
403
|
+
const logRow = document.createElement("tr");
|
|
404
|
+
const cAttributeRule = document.createElement("td");
|
|
405
|
+
cAttributeRule.textContent = rule["Attribute Rule"]
|
|
406
|
+
|
|
407
|
+
const cTotalCost = document.createElement("td");
|
|
408
|
+
cTotalCost.textContent = rule["Total Cost (ms)"]
|
|
409
|
+
|
|
410
|
+
const cMaxExecutionTime = document.createElement("td");
|
|
411
|
+
cMaxExecutionTime.textContent = rule["Max execution time (ms)"]
|
|
412
|
+
|
|
413
|
+
const cMaxGuid = document.createElement("td");
|
|
414
|
+
cMaxGuid.textContent = rule["Max GUID"]
|
|
415
|
+
|
|
416
|
+
const cMinExecutionTime = document.createElement("td");
|
|
417
|
+
cMinExecutionTime.textContent = rule["Min execution time (ms)"]
|
|
418
|
+
|
|
419
|
+
logRow.appendChild (cAttributeRule)
|
|
420
|
+
logRow.appendChild (cTotalCost)
|
|
421
|
+
logRow.appendChild (cMaxExecutionTime)
|
|
422
|
+
logRow.appendChild (cMaxGuid)
|
|
423
|
+
logRow.appendChild (cMinExecutionTime)
|
|
424
|
+
tblResult.appendChild(logRow)
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
})
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
const totalARExecution = rules.reduce( (prev, cur) => prev + cur["Total Cost (ms)"], 0)
|
|
431
|
+
console.log(`Total time spend executing attribute rules (${Math.round(totalARExecution)} ms) (${Math.round(totalARExecution/1000)} s) (${Math.round(totalARExecution/(1000*60))} m)`)
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
}
|
|
436
|
+
async function loadSQLLogs () {
|
|
437
|
+
|
|
438
|
+
//build table
|
|
439
|
+
const tblResult = document.getElementById("tblResult");
|
|
440
|
+
//clear
|
|
441
|
+
while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
|
|
442
|
+
tblResult.appendChild(document.createTextNode("Loading...."));
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
parameters.service = cmbService.options[cmbService.selectedIndex].text
|
|
446
|
+
|
|
447
|
+
//create a new admin object (user might not be admin we won't use it until the user call log )
|
|
448
|
+
adminLog = new AdminLog(token, portal.serverUrl)
|
|
449
|
+
|
|
450
|
+
logger.info("Connected.")
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
const topLogCount = 1000;
|
|
454
|
+
const pageSize = 10000
|
|
455
|
+
|
|
456
|
+
let mins = document.getElementById("txtAge").value //query logs for the last 30 minutes
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
console.log(`Querying cursor sql logs for ${parameters.service} for the last ${mins} minutes ...`)
|
|
460
|
+
const startTime = Date.now() - mins*60*1000
|
|
461
|
+
const endTime = Date.now();
|
|
462
|
+
let services = [parameters.service+ ".MapServer"]
|
|
463
|
+
if (parameters.service == "(All Services)")
|
|
464
|
+
services = "*"
|
|
465
|
+
let result= await adminLog.query([102023], services, topLogCount, startTime ,endTime , "DEBUG")
|
|
466
|
+
let jsonRes = await result.json()
|
|
467
|
+
let allMessages = [].concat(jsonRes.logMessages)
|
|
468
|
+
allMessages = allMessages.filter(m => m.message.indexOf("EndCursor;") > -1)
|
|
469
|
+
while (jsonRes.hasMore)
|
|
470
|
+
{
|
|
471
|
+
//start paging
|
|
472
|
+
logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
|
|
473
|
+
result= await adminLog.query([102023], services, pageSize, jsonRes.endTime, null, "DEBUG")
|
|
474
|
+
jsonRes = await result.json()
|
|
475
|
+
|
|
476
|
+
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("EndCursor;") > -1))
|
|
477
|
+
}
|
|
478
|
+
console.log ("Filtering messages...")
|
|
479
|
+
|
|
480
|
+
allMessages = allMessages
|
|
481
|
+
.map( m=> {
|
|
482
|
+
m.dataAccessElapsed = parseFloat(m.message.split(";")[1].split(" ")[1])
|
|
483
|
+
m.executeQueryElapsed = parseFloat(m.message.split(";")[2].split(" ")[1])
|
|
484
|
+
m.totalExecutionElapsed = m.dataAccessElapsed + m.executeQueryElapsed
|
|
485
|
+
m.elapsed = parseFloat(m.elapsed); return m;
|
|
486
|
+
|
|
487
|
+
})
|
|
488
|
+
.sort( (m1,m2) => m2.totalExecutionElapsed - m1.totalExecutionElapsed)
|
|
489
|
+
.slice(0, 100) ;//first 100
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
console.log("-----Top 10 SQL----")
|
|
493
|
+
while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
|
|
494
|
+
|
|
495
|
+
let i =0;
|
|
496
|
+
|
|
497
|
+
const header = document.createElement("tr");
|
|
498
|
+
const cService = document.createElement("th");
|
|
499
|
+
cService.textContent = "Source"
|
|
500
|
+
const cTime = document.createElement("th");
|
|
501
|
+
cTime.textContent = "Time"
|
|
502
|
+
const cUser = document.createElement("th");
|
|
503
|
+
cUser.textContent = "User"
|
|
504
|
+
const cMethod = document.createElement("th");
|
|
505
|
+
cMethod.textContent = "Method"
|
|
506
|
+
const cQueryTime = document.createElement("th");
|
|
507
|
+
cQueryTime.textContent = "Query Time (ms)"
|
|
508
|
+
cQueryTime.title = "includes search + data access nextRow"
|
|
509
|
+
const cTotalTime = document.createElement("th");
|
|
510
|
+
cTotalTime.title = "Total time the cursor was opened"
|
|
511
|
+
cTotalTime.textContent = "Total Time (ms)"
|
|
512
|
+
const cLog = document.createElement("th");
|
|
513
|
+
cLog.textContent = "Full SQL"
|
|
514
|
+
header.appendChild (cService)
|
|
515
|
+
header.appendChild (cTime)
|
|
516
|
+
header.appendChild (cUser)
|
|
517
|
+
header.appendChild (cMethod)
|
|
518
|
+
header.appendChild (cQueryTime)
|
|
519
|
+
header.appendChild (cTotalTime)
|
|
520
|
+
header.appendChild (cLog)
|
|
521
|
+
tblResult.appendChild(header)
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
allMessages= allMessages.forEach(m =>
|
|
525
|
+
{
|
|
526
|
+
|
|
527
|
+
const logRow = document.createElement("tr");
|
|
528
|
+
const x = m.message.split(";")
|
|
529
|
+
x.shift()
|
|
530
|
+
console.log(`id: ${i++}`)
|
|
531
|
+
console.log(`\tAt: ${new Date(m.time)} (${m.time})`)
|
|
532
|
+
console.log(`\tUser: ${m.user}`)
|
|
533
|
+
console.log(`\tTotal Time: ${numberWithCommas(Math.round(m.elapsed*1000))} ms (Total time the cursor was opened)`)
|
|
534
|
+
console.log(`\tQuery Time: ${numberWithCommas(m.totalExecutionElapsed)} ms (includes search + data access nextRow)`)
|
|
535
|
+
console.log(`\tQuery:`)
|
|
536
|
+
x.forEach(a => console.log(`\t${a}`))
|
|
537
|
+
console.log(`\n`)
|
|
538
|
+
|
|
539
|
+
const cService = document.createElement("th");
|
|
540
|
+
cService.textContent = m.source.replace(".MapServer", "")
|
|
541
|
+
|
|
542
|
+
const cTime = document.createElement("td");
|
|
543
|
+
cTime.textContent = new Date(m.time).toLocaleString()
|
|
544
|
+
|
|
545
|
+
const cUser = document.createElement("td");
|
|
546
|
+
cUser.textContent = m.user
|
|
547
|
+
const cMethod = document.createElement("td");
|
|
548
|
+
cMethod.textContent = "Cursor"
|
|
549
|
+
const cQueryTime = document.createElement("td");
|
|
550
|
+
cQueryTime.textContent = numberWithCommas(m.totalExecutionElapsed)
|
|
551
|
+
const cTotalTime = document.createElement("td");
|
|
552
|
+
cTotalTime.textContent = numberWithCommas(Math.round(m.elapsed*1000))
|
|
553
|
+
const cLog = document.createElement("td");
|
|
554
|
+
cLog.textContent = m.message.substr( m.message.indexOf("SQL: ") + 5, 50) + "..."
|
|
555
|
+
cLog.fullLog = m.message;
|
|
556
|
+
cLog.title = m.message + "\nClick to copy"
|
|
557
|
+
cLog.addEventListener("click", e=> {navigator.clipboard.writeText(e.target.fullLog); alert("Copied to clipboard")})
|
|
558
|
+
logRow.appendChild (cService)
|
|
559
|
+
logRow.appendChild (cTime)
|
|
560
|
+
logRow.appendChild (cUser)
|
|
561
|
+
logRow.appendChild (cMethod)
|
|
562
|
+
logRow.appendChild (cQueryTime)
|
|
563
|
+
logRow.appendChild (cTotalTime)
|
|
564
|
+
logRow.appendChild (cLog)
|
|
565
|
+
tblResult.appendChild(logRow)
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
})
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
//await populateMessages(allMessages)
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
function numberWithCommas(x) {
|
|
583
|
+
// return x.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
|
|
584
|
+
return x.toLocaleString()
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
init();
|
|
588
|
+
|
|
589
|
+
const btnTraceLogs = document.getElementById("btnTraceLogs");
|
|
590
|
+
btnTraceLogs.addEventListener("click", e => loadTraceLogs())
|
|
591
|
+
|
|
592
|
+
const btnValidateLogs = document.getElementById("btnValidateLogs");
|
|
593
|
+
btnValidateLogs.addEventListener("click", e => loadValidateLogs())
|
|
594
|
+
|
|
595
|
+
const btnUpdateSubnetworkLogs = document.getElementById("btnUpdateSubnetworkLogs");
|
|
596
|
+
btnUpdateSubnetworkLogs.addEventListener("click", e => loadUpdateSubnetworkLogs())
|
|
597
|
+
|
|
598
|
+
const btnSQLLogs = document.getElementById("btnSQLLogs");
|
|
599
|
+
btnSQLLogs.addEventListener("click", e => loadSQLLogs())
|
|
600
|
+
|
|
601
|
+
const btnAR = document.getElementById("btnAR");
|
|
602
|
+
btnAR.addEventListener("click", e => loadAttributeRules())
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
async function populateMessages(allTheMessages) {
|
|
607
|
+
|
|
608
|
+
//sort messages
|
|
609
|
+
|
|
610
|
+
const allMessages = allTheMessages.map(m => {
|
|
611
|
+
const newMessage = Object.assign({}, m);
|
|
612
|
+
delete newMessage.machine;
|
|
613
|
+
delete newMessage.type;
|
|
614
|
+
delete newMessage.code;
|
|
615
|
+
delete newMessage.requestID;
|
|
616
|
+
delete newMessage.process;
|
|
617
|
+
delete newMessage.thread;
|
|
618
|
+
newMessage.elapsedms = parseInt (parseFloat(newMessage.elapsed) * 1000)
|
|
619
|
+
newMessage.time = new Date(newMessage.time).toLocaleString()
|
|
620
|
+
delete newMessage.elapsed
|
|
621
|
+
return newMessage;
|
|
622
|
+
})
|
|
623
|
+
.sort( (m1,m2) => m2.elapsedms - m1.elapsedms)
|
|
624
|
+
|
|
625
|
+
//build table
|
|
626
|
+
const tblResult = document.getElementById("tblResult");
|
|
627
|
+
//clear
|
|
628
|
+
while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
|
|
629
|
+
|
|
630
|
+
const header = document.createElement("tr");
|
|
631
|
+
const cService = document.createElement("th");
|
|
632
|
+
cService.textContent = "Source"
|
|
633
|
+
const cTime = document.createElement("th");
|
|
634
|
+
cTime.textContent = "Time"
|
|
635
|
+
const cUser = document.createElement("th");
|
|
636
|
+
cUser.textContent = "User"
|
|
637
|
+
const cMethod = document.createElement("th");
|
|
638
|
+
cMethod.textContent = "Method"
|
|
639
|
+
const cElapsedMS = document.createElement("th");
|
|
640
|
+
cElapsedMS.textContent = "Elapsed (ms)"
|
|
641
|
+
const cLog = document.createElement("th");
|
|
642
|
+
cLog.textContent = "Full Log"
|
|
643
|
+
header.appendChild (cService)
|
|
644
|
+
header.appendChild (cTime)
|
|
645
|
+
header.appendChild (cUser)
|
|
646
|
+
header.appendChild (cMethod)
|
|
647
|
+
header.appendChild (cElapsedMS)
|
|
648
|
+
header.appendChild (cLog)
|
|
649
|
+
tblResult.appendChild(header)
|
|
650
|
+
allMessages.forEach(m => {
|
|
651
|
+
const logRow = document.createElement("tr");
|
|
652
|
+
const cService = document.createElement("td");
|
|
653
|
+
cService.textContent = m.source.replace(".MapServer", "")
|
|
654
|
+
const cTime = document.createElement("td");
|
|
655
|
+
cTime.textContent = m.time
|
|
656
|
+
const cUser = document.createElement("td");
|
|
657
|
+
cUser.textContent = m.user
|
|
658
|
+
const cMethod = document.createElement("td");
|
|
659
|
+
cMethod.textContent = m.methodName
|
|
660
|
+
const cElapsedMS = document.createElement("td");
|
|
661
|
+
cElapsedMS.textContent = m.elapsedms
|
|
662
|
+
const cLog = document.createElement("td");
|
|
663
|
+
cLog.textContent = "..."
|
|
664
|
+
cLog.fullLog = m.message;
|
|
665
|
+
cLog.addEventListener("click", e=> {navigator.clipboard.writeText(e.target.fullLog); alert("Copied to clipboard")})
|
|
666
|
+
logRow.appendChild (cService)
|
|
667
|
+
logRow.appendChild (cTime)
|
|
668
|
+
logRow.appendChild (cUser)
|
|
669
|
+
logRow.appendChild (cMethod)
|
|
670
|
+
logRow.appendChild (cElapsedMS)
|
|
671
|
+
logRow.appendChild (cLog)
|
|
672
|
+
tblResult.appendChild(logRow)
|
|
673
|
+
})
|
|
674
|
+
}
|
|
675
|
+
</script>
|
|
676
|
+
</body>
|
|
677
|
+
</html>
|
package/index.mjs
CHANGED
|
@@ -4,10 +4,10 @@ import { Portal } from "./portal.mjs"
|
|
|
4
4
|
import { UtilityNetwork } from "./utilitynetwork.node.mjs"
|
|
5
5
|
import { AdminLog } from "./adminlog.mjs"
|
|
6
6
|
|
|
7
|
-
import
|
|
7
|
+
import logger from "./logger.mjs"
|
|
8
8
|
import fetch from "node-fetch"
|
|
9
9
|
//update version
|
|
10
|
-
let version = "0.0.
|
|
10
|
+
let version = "0.0.70";
|
|
11
11
|
const GENERATE_TOKEN_TIME_MIN = 30;
|
|
12
12
|
|
|
13
13
|
let rl = null;
|
|
@@ -408,11 +408,15 @@ const inputs = {
|
|
|
408
408
|
}
|
|
409
409
|
},
|
|
410
410
|
|
|
411
|
-
"^update subnetworks --all
|
|
411
|
+
"^update subnetworks --all" : async input => {
|
|
412
412
|
|
|
413
413
|
do {
|
|
414
|
+
|
|
415
|
+
let sort = "asc";
|
|
416
|
+
if (input.indexOf("--desc") > 0) sort = "desc"
|
|
417
|
+
|
|
414
418
|
console.log("Querying all subnetworks that are dirty.");
|
|
415
|
-
let subnetworks = await un.queryDistinct(500002, "domainnetworkname,tiername,subnetworkname", "isdirty=1",
|
|
419
|
+
let subnetworks = await un.queryDistinct(500002, "domainnetworkname,tiername,subnetworkname", "isdirty=1", `domainnetworkname ${sort},tiername ${sort},subnetworkname ${sort}`);
|
|
416
420
|
console.log(`Discovered ${subnetworks.features.length} dirty subnetworks.`);
|
|
417
421
|
|
|
418
422
|
for (let i = 0; i < subnetworks.features.length; i++) {
|
|
@@ -774,16 +778,30 @@ const inputs = {
|
|
|
774
778
|
|
|
775
779
|
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("------ Trace Parameters ----") > -1))
|
|
776
780
|
}
|
|
777
|
-
|
|
781
|
+
|
|
782
|
+
allMessages = allMessages.map(m => {
|
|
778
783
|
const newMessage = Object.assign({}, m);
|
|
779
|
-
delete newMessage.message
|
|
780
784
|
delete newMessage.source;
|
|
781
785
|
delete newMessage.machine;
|
|
782
786
|
delete newMessage.type;
|
|
787
|
+
delete newMessage.code;
|
|
783
788
|
delete newMessage.requestID;
|
|
789
|
+
delete newMessage.process;
|
|
784
790
|
delete newMessage.thread;
|
|
785
|
-
|
|
791
|
+
newMessage.elapsedms = parseInt (parseFloat(newMessage.elapsed) * 1000)
|
|
792
|
+
newMessage.time = new Date(newMessage.time).toLocaleString()
|
|
793
|
+
delete newMessage.elapsed
|
|
794
|
+
return newMessage;
|
|
795
|
+
})
|
|
796
|
+
.sort( (m1,m2) => m2.elapsedms - m1.elapsedms)
|
|
786
797
|
|
|
798
|
+
const summaryMessages = allMessages.map(m => {const newM = Object.assign({}, m); delete newM.message; return newM})
|
|
799
|
+
console.table(summaryMessages)
|
|
800
|
+
|
|
801
|
+
allMessages.forEach(m => {
|
|
802
|
+
const newMessage = Object.assign({}, m);
|
|
803
|
+
delete newMessage.message
|
|
804
|
+
|
|
787
805
|
console.table([newMessage])
|
|
788
806
|
console.log(m.message)
|
|
789
807
|
})
|
|
@@ -802,37 +820,76 @@ const inputs = {
|
|
|
802
820
|
if (inputParam != null && inputParam.length > 0)
|
|
803
821
|
mins = inputParam[0].replace("--age ", "")
|
|
804
822
|
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
823
|
+
|
|
824
|
+
console.log(`Querying validate logs for ${parameters.service} for the last ${mins} minutes ...`)
|
|
825
|
+
const startTime = Date.now() - mins*60*1000
|
|
826
|
+
const endTime = Date.now();
|
|
827
|
+
let result= await adminLog.query([102003], [parameters.service+ ".MapServer"], topLogCount, startTime ,endTime , "VERBOSE")
|
|
828
|
+
let jsonRes = await result.json()
|
|
829
|
+
let allMessages = [].concat(jsonRes.logMessages)
|
|
830
|
+
allMessages = allMessages.filter(m => m.message.indexOf("-------- Environment ---") > -1 && m.message.indexOf("The network is built.") > -1 && m.methodName == 'BuildEngineLog')
|
|
831
|
+
while (jsonRes.hasMore)
|
|
832
|
+
{
|
|
833
|
+
//start paging
|
|
834
|
+
logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
|
|
835
|
+
let services = [parameters.service + ".MapServer"]
|
|
836
|
+
result= await adminLog.query([102003], services, pageSize, jsonRes.endTime, null, "VERBOSE")
|
|
837
|
+
jsonRes = await result.json()
|
|
838
|
+
|
|
839
|
+
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("-------- Environment ---") > -1 && m.message.indexOf("------ Trace Parameters ----") == -1))
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
|
|
843
|
+
|
|
844
|
+
|
|
845
|
+
//validate logs missing elapsed populate it
|
|
846
|
+
allMessages = allMessages.map( m => {
|
|
847
|
+
try{
|
|
848
|
+
|
|
849
|
+
//The network is built. 0.093 seconds (4.982 total) - 29 MB memory
|
|
850
|
+
|
|
851
|
+
let re = /The network is built. [-+]?([0-9]*\.[0-9]+|[0-9]+) seconds \([-+]?([0-9]*\.[0-9]+|[0-9]+) total\)/;
|
|
852
|
+
let res = re.exec(m.message)
|
|
853
|
+
if (res && res.length > 1)
|
|
854
|
+
m.elapsed = res[2]
|
|
855
|
+
|
|
856
|
+
return m;
|
|
857
|
+
}
|
|
858
|
+
catch(ex){
|
|
859
|
+
return m;
|
|
860
|
+
}
|
|
861
|
+
})
|
|
821
862
|
|
|
822
863
|
|
|
823
|
-
|
|
864
|
+
|
|
865
|
+
allMessages = allMessages.map(m => {
|
|
824
866
|
const newMessage = Object.assign({}, m);
|
|
825
|
-
delete newMessage.message
|
|
826
867
|
delete newMessage.source;
|
|
827
868
|
delete newMessage.machine;
|
|
828
869
|
delete newMessage.type;
|
|
870
|
+
delete newMessage.code;
|
|
829
871
|
delete newMessage.requestID;
|
|
872
|
+
delete newMessage.process;
|
|
830
873
|
delete newMessage.thread;
|
|
831
|
-
|
|
874
|
+
newMessage.elapsedms = parseInt (parseFloat(newMessage.elapsed) * 1000)
|
|
875
|
+
newMessage.time = new Date(newMessage.time).toLocaleString()
|
|
876
|
+
delete newMessage.elapsed
|
|
877
|
+
return newMessage;
|
|
878
|
+
})
|
|
879
|
+
.sort( (m1,m2) => m2.elapsedms - m1.elapsedms)
|
|
832
880
|
|
|
881
|
+
const summaryMessages = allMessages.map(m => {const newM = Object.assign({}, m); delete newM.message; return newM})
|
|
882
|
+
console.table(summaryMessages)
|
|
883
|
+
|
|
884
|
+
allMessages.forEach(m => {
|
|
885
|
+
const newMessage = Object.assign({}, m);
|
|
886
|
+
delete newMessage.message
|
|
887
|
+
|
|
833
888
|
console.table([newMessage])
|
|
834
889
|
console.log(m.message)
|
|
835
890
|
})
|
|
891
|
+
|
|
892
|
+
|
|
836
893
|
|
|
837
894
|
},
|
|
838
895
|
|
|
@@ -864,20 +921,59 @@ const inputs = {
|
|
|
864
921
|
|
|
865
922
|
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("---- Subnetwork Parameters ----") > -1))
|
|
866
923
|
}
|
|
867
|
-
|
|
924
|
+
|
|
925
|
+
|
|
926
|
+
|
|
927
|
+
//update subnetwork missing elapsed populate it
|
|
928
|
+
allMessages = allMessages.map( m => {
|
|
929
|
+
try{
|
|
930
|
+
|
|
931
|
+
let re = /Total \([-+]?([0-9]*\.[0-9]+|[0-9]+) seconds\)/;
|
|
932
|
+
let res = re.exec(m.message)
|
|
933
|
+
if (res && res.length > 1)
|
|
934
|
+
m.elapsed = res[1]
|
|
935
|
+
|
|
936
|
+
re = /Total update subnetwork time \([-+]?([0-9]*\.[0-9]+|[0-9]+) seconds\)/;
|
|
937
|
+
res = re.exec(m.message)
|
|
938
|
+
if (res && res.length > 1)
|
|
939
|
+
m.elapsed = res[1]
|
|
940
|
+
|
|
941
|
+
return m;
|
|
942
|
+
}
|
|
943
|
+
catch(ex){
|
|
944
|
+
return m;
|
|
945
|
+
}
|
|
946
|
+
})
|
|
947
|
+
|
|
948
|
+
|
|
949
|
+
allMessages = allMessages.map(m => {
|
|
868
950
|
const newMessage = Object.assign({}, m);
|
|
869
|
-
delete newMessage.message
|
|
870
951
|
delete newMessage.source;
|
|
871
952
|
delete newMessage.machine;
|
|
872
953
|
delete newMessage.type;
|
|
954
|
+
delete newMessage.code;
|
|
873
955
|
delete newMessage.requestID;
|
|
956
|
+
delete newMessage.process;
|
|
874
957
|
delete newMessage.thread;
|
|
875
|
-
|
|
958
|
+
newMessage.elapsedms = parseInt (parseFloat(newMessage.elapsed) * 1000)
|
|
959
|
+
newMessage.time = new Date(newMessage.time).toLocaleString()
|
|
960
|
+
delete newMessage.elapsed
|
|
961
|
+
return newMessage;
|
|
962
|
+
})
|
|
963
|
+
.sort( (m1,m2) => m2.elapsedms - m1.elapsedms)
|
|
964
|
+
|
|
965
|
+
const summaryMessages = allMessages.map(m => {const newM = Object.assign({}, m); delete newM.message; return newM})
|
|
966
|
+
console.table(summaryMessages)
|
|
876
967
|
|
|
968
|
+
allMessages.forEach(m => {
|
|
969
|
+
const newMessage = Object.assign({}, m);
|
|
970
|
+
delete newMessage.message
|
|
971
|
+
|
|
877
972
|
console.table([newMessage])
|
|
878
973
|
console.log(m.message)
|
|
879
974
|
})
|
|
880
975
|
|
|
976
|
+
|
|
881
977
|
},
|
|
882
978
|
|
|
883
979
|
|
|
@@ -942,60 +1038,6 @@ const inputs = {
|
|
|
942
1038
|
},
|
|
943
1039
|
|
|
944
1040
|
|
|
945
|
-
|
|
946
|
-
"^cursorlogs --age": async input => {
|
|
947
|
-
const topLogCount = 1000;
|
|
948
|
-
const pageSize = 10000
|
|
949
|
-
|
|
950
|
-
const inputParam = input.match(/--age .*/gm)
|
|
951
|
-
let mins = 30; //query logs for the last 30 minutes
|
|
952
|
-
if (inputParam != null && inputParam.length > 0)
|
|
953
|
-
mins = inputParam[0].replace("--age ", "")
|
|
954
|
-
|
|
955
|
-
console.log(`Querying cursor sql logs for ${parameters.service} for the last ${mins} minutes ...`)
|
|
956
|
-
const startTime = Date.now() - mins*60*1000
|
|
957
|
-
const endTime = Date.now();
|
|
958
|
-
let result= await adminLog.query([102023], [parameters.service+ ".MapServer"], topLogCount, startTime ,endTime , "DEBUG")
|
|
959
|
-
let jsonRes = await result.json()
|
|
960
|
-
let allMessages = [].concat(jsonRes.logMessages)
|
|
961
|
-
allMessages = allMessages.filter(m => m.message.indexOf("EndCursor;") > -1)
|
|
962
|
-
while (jsonRes.hasMore)
|
|
963
|
-
{
|
|
964
|
-
//start paging
|
|
965
|
-
logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
|
|
966
|
-
result= await adminLog.query([102023], [parameters.service + ".MapServer"], pageSize, jsonRes.endTime, null, "DEBUG")
|
|
967
|
-
jsonRes = await result.json()
|
|
968
|
-
|
|
969
|
-
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("EndCursor;") > -1))
|
|
970
|
-
}
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
//
|
|
974
|
-
allMessages= allMessages.map(m => {
|
|
975
|
-
|
|
976
|
-
const r= m.message.split (";")
|
|
977
|
-
delete r[0];
|
|
978
|
-
delete r[1];
|
|
979
|
-
delete r[3];
|
|
980
|
-
return r;
|
|
981
|
-
|
|
982
|
-
const newMessage = Object.assign({}, m);
|
|
983
|
-
|
|
984
|
-
delete newMessage.source;
|
|
985
|
-
delete newMessage.machine;
|
|
986
|
-
delete newMessage.type;
|
|
987
|
-
delete newMessage.requestID;
|
|
988
|
-
delete newMessage.methodName;
|
|
989
|
-
delete newMessage.process;
|
|
990
|
-
delete newMessage.thread;
|
|
991
|
-
delete newMessage.time;
|
|
992
|
-
delete newMessage.code;
|
|
993
|
-
|
|
994
|
-
return newMessage
|
|
995
|
-
|
|
996
|
-
})
|
|
997
|
-
console.table(allMessages)
|
|
998
|
-
},
|
|
999
1041
|
|
|
1000
1042
|
|
|
1001
1043
|
"^arlogs --byrule": async input => {
|
|
@@ -1010,17 +1052,29 @@ const inputs = {
|
|
|
1010
1052
|
if (inputParam != null && inputParam.length > 0 && inputParam[0].indexOf("--minguid") > -1)
|
|
1011
1053
|
showMinGuid = true
|
|
1012
1054
|
|
|
1055
|
+
|
|
1056
|
+
const ageInputParam = input.match(/--age [0-9]*/)
|
|
1057
|
+
let mins = 30; //query logs for the last 30 minutes
|
|
1058
|
+
if (ageInputParam != null && ageInputParam.length > 0)
|
|
1059
|
+
mins = ageInputParam[0].replace("--age ", "")
|
|
1060
|
+
|
|
1061
|
+
|
|
1062
|
+
const startTime = Date.now() - mins*60*1000
|
|
1063
|
+
const endTime = Date.now();
|
|
1064
|
+
|
|
1013
1065
|
const pageSize = 10000 //maximum messages per page
|
|
1014
|
-
logger.info(`Querying attribute rules logs for ${parameters.service} ...`)
|
|
1015
|
-
let result= await adminLog.query([102003], [parameters.service + ".MapServer"], pageSize)
|
|
1066
|
+
logger.info(`Querying attribute rules logs for ${parameters.service} in the past ${mins} minutes...`)
|
|
1067
|
+
let result= await adminLog.query([102003], [parameters.service + ".MapServer"], pageSize, startTime ,endTime , "DEBUG")
|
|
1016
1068
|
let jsonRes = await result.json()
|
|
1017
1069
|
let allMessages = [].concat(jsonRes.logMessages)
|
|
1018
1070
|
|
|
1019
|
-
while (jsonRes.hasMore)
|
|
1071
|
+
while (jsonRes.hasMore && jsonRes.endTime > startTime)
|
|
1020
1072
|
{
|
|
1021
1073
|
//start paging
|
|
1022
1074
|
logger.info(`Aggregating messages... total so far ${allMessages.length} debug entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
|
|
1023
|
-
|
|
1075
|
+
|
|
1076
|
+
|
|
1077
|
+
result= await adminLog.query([102003], [parameters.service + ".MapServer"], pageSize, jsonRes.endTime )
|
|
1024
1078
|
jsonRes = await result.json()
|
|
1025
1079
|
allMessages = allMessages.concat(jsonRes.logMessages)
|
|
1026
1080
|
}
|
package/logger.mjs
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
const logger = {}
|
|
2
|
+
logger.info = console.log
|
|
3
|
+
logger.error = console.error
|
|
4
|
+
|
|
5
|
+
export default logger;
|
|
6
|
+
|
|
7
|
+
/*
|
|
1
8
|
import winston from "winston"
|
|
2
9
|
|
|
3
10
|
export const logger = winston.createLogger();
|
|
@@ -5,4 +12,4 @@ export const logger = winston.createLogger();
|
|
|
5
12
|
logger.add(
|
|
6
13
|
new winston.transports.Console({"format": winston.format.json()})
|
|
7
14
|
);
|
|
8
|
-
|
|
15
|
+
*/
|
package/makerequest.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
|
|
1
|
+
|
|
2
|
+
|
|
2
3
|
|
|
3
4
|
export function makeRequest (opts) {
|
|
4
5
|
|
|
@@ -24,7 +25,18 @@ export function makeRequest (opts) {
|
|
|
24
25
|
Object.keys(opts.headers).forEach( key => headers[key] = opts.headers[key] )
|
|
25
26
|
|
|
26
27
|
//console.log(opts)
|
|
27
|
-
|
|
28
|
+
|
|
29
|
+
let f;
|
|
30
|
+
try {
|
|
31
|
+
const nodeFetch = await import ("node-fetch");
|
|
32
|
+
f = nodeFetch.default;
|
|
33
|
+
}
|
|
34
|
+
catch(ex) {
|
|
35
|
+
f = fetch;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
const result = await f(opts.url, {
|
|
28
40
|
"method" : opts.method,
|
|
29
41
|
"headers": headers,
|
|
30
42
|
"body": params
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "un-cli",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.70",
|
|
4
4
|
"description": "Command line interface for working with ArcGIS Utility Network Extension",
|
|
5
5
|
"main": "index.mjs",
|
|
6
6
|
"bin": {
|
|
@@ -17,8 +17,6 @@
|
|
|
17
17
|
"author": "Hussein Nasser",
|
|
18
18
|
"license": "ISC",
|
|
19
19
|
"dependencies": {
|
|
20
|
-
"node-fetch": "^2.6.0"
|
|
21
|
-
"progress": "^2.0.3",
|
|
22
|
-
"winston": "^3.2.1"
|
|
20
|
+
"node-fetch": "^2.6.0"
|
|
23
21
|
}
|
|
24
22
|
}
|
package/portal.mjs
CHANGED
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
import { makeRequest } from "./makerequest.mjs"
|
|
2
|
-
import
|
|
2
|
+
import logger from "./logger.mjs"
|
|
3
3
|
|
|
4
4
|
export class Portal{
|
|
5
5
|
|
|
6
|
-
constructor(url, username, password, expiration = 300, serverUrl = undefined)
|
|
6
|
+
constructor(url, username, password, expiration = 300, serverUrl = undefined, referer)
|
|
7
7
|
{
|
|
8
8
|
this.url = url;
|
|
9
9
|
this.username = username;
|
|
10
10
|
this.password = password;
|
|
11
11
|
this.expiration = expiration;
|
|
12
12
|
this.serverUrl = serverUrl;
|
|
13
|
+
this.referer = referer
|
|
13
14
|
}
|
|
14
15
|
|
|
15
16
|
|
|
@@ -25,7 +26,7 @@ export class Portal{
|
|
|
25
26
|
const postJson = {
|
|
26
27
|
username: self.username,
|
|
27
28
|
password: self.password,
|
|
28
|
-
referer:
|
|
29
|
+
referer: self.referer,
|
|
29
30
|
expiration: self.expiration,
|
|
30
31
|
f: "json"
|
|
31
32
|
}
|
|
@@ -71,7 +72,7 @@ export class Portal{
|
|
|
71
72
|
token: self.token
|
|
72
73
|
}
|
|
73
74
|
|
|
74
|
-
const serversUrl = self.url +
|
|
75
|
+
const serversUrl = self.url + `/sharing/rest/portals/self/servers?token=${self.token}`
|
|
75
76
|
logger.info( "About to query federated servers");
|
|
76
77
|
|
|
77
78
|
//query for federated servers.
|
package/utilitynetwork.node.mjs
CHANGED
|
@@ -8,9 +8,9 @@
|
|
|
8
8
|
/*
|
|
9
9
|
import fetch from "node-fetch"
|
|
10
10
|
*/
|
|
11
|
-
import
|
|
12
|
-
import
|
|
13
|
-
|
|
11
|
+
import logger from "./logger.mjs"
|
|
12
|
+
import { makeRequest } from "./makerequest.mjs"
|
|
13
|
+
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
export class UtilityNetwork {
|
|
@@ -122,6 +122,7 @@ export class UtilityNetwork {
|
|
|
122
122
|
|
|
123
123
|
}
|
|
124
124
|
|
|
125
|
+
/*
|
|
125
126
|
//a function that detects associations with from/to globalId that don't exist in the source tables.
|
|
126
127
|
//if one of the endpoints do not exists fails..
|
|
127
128
|
returnInvalidAssociations() {
|
|
@@ -178,7 +179,7 @@ export class UtilityNetwork {
|
|
|
178
179
|
|
|
179
180
|
|
|
180
181
|
}
|
|
181
|
-
|
|
182
|
+
*/
|
|
182
183
|
queryMoment(moment = ["definitionModification"]) {
|
|
183
184
|
|
|
184
185
|
let thisObj = this;
|
|
@@ -1086,6 +1087,7 @@ export class UtilityNetwork {
|
|
|
1086
1087
|
|
|
1087
1088
|
|
|
1088
1089
|
|
|
1090
|
+
/*
|
|
1089
1091
|
//Makes a request
|
|
1090
1092
|
function makeRequest (opts) {
|
|
1091
1093
|
|
|
@@ -1134,7 +1136,7 @@ export class UtilityNetwork {
|
|
|
1134
1136
|
|
|
1135
1137
|
});
|
|
1136
1138
|
}
|
|
1137
|
-
|
|
1139
|
+
*/
|
|
1138
1140
|
|
|
1139
1141
|
|
|
1140
1142
|
const v = (o, f, vIfNotFound=null) => {
|