un-cli 0.0.69 → 0.0.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adminlog.mjs +12 -3
- package/cmd.txt +1 -1
- package/index.html +744 -0
- package/index.mjs +156 -90
- package/logger.mjs +8 -1
- package/makerequest.mjs +14 -2
- package/package.json +2 -4
- package/portal.mjs +5 -4
- package/utilitynetwork.node.mjs +7 -5
package/adminlog.mjs
CHANGED
|
@@ -2,8 +2,6 @@
|
|
|
2
2
|
//Author : Hussein Nasser
|
|
3
3
|
//Date : Oct-13-2021
|
|
4
4
|
//Twitter: @hnasr
|
|
5
|
-
import fetch from "node-fetch";
|
|
6
|
-
import { logger } from "./logger.mjs"
|
|
7
5
|
|
|
8
6
|
export class AdminLog {
|
|
9
7
|
|
|
@@ -13,12 +11,22 @@ export class AdminLog {
|
|
|
13
11
|
this.token = token;
|
|
14
12
|
}
|
|
15
13
|
|
|
16
|
-
|
|
14
|
+
async getFetch() {
|
|
15
|
+
try {
|
|
16
|
+
const nodeFetch = await import ("node-fetch");
|
|
17
|
+
return nodeFetch.default;
|
|
18
|
+
}
|
|
19
|
+
catch(ex) {
|
|
20
|
+
return fetch;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
async query (codes, serviceName ="*", pageSize = 100000, startTime = null, endTime = null, logLevel = "DEBUG")
|
|
17
24
|
{
|
|
18
25
|
const url = this.adminServerUrl + "/logs/query?f=pjson"
|
|
19
26
|
const level = logLevel
|
|
20
27
|
const filterType="json"
|
|
21
28
|
const token = this.token
|
|
29
|
+
|
|
22
30
|
const filter = {
|
|
23
31
|
"codes": codes,
|
|
24
32
|
"services": serviceName
|
|
@@ -32,6 +40,7 @@ export class AdminLog {
|
|
|
32
40
|
queryLogUrl += `&endTime=${endTime}`
|
|
33
41
|
|
|
34
42
|
//logger.info(queryLogUrl);
|
|
43
|
+
const fetch = await this.getFetch()
|
|
35
44
|
return fetch(queryLogUrl);
|
|
36
45
|
}
|
|
37
46
|
|
package/cmd.txt
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
|
|
1
|
+
update subnetworks --all --desc
|
|
2
2
|
exit
|
package/index.html
ADDED
|
@@ -0,0 +1,744 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8">
|
|
5
|
+
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
|
6
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
7
|
+
<title>UN Logs</title>
|
|
8
|
+
</head>
|
|
9
|
+
<body>
|
|
10
|
+
|
|
11
|
+
<table>
|
|
12
|
+
<tr>
|
|
13
|
+
<td>
|
|
14
|
+
Admin User
|
|
15
|
+
</td>
|
|
16
|
+
<td colspan =3><input type = 'text' id = 'txtAdmin' value = 'admin'></td>
|
|
17
|
+
</tr>
|
|
18
|
+
<tr>
|
|
19
|
+
<td>
|
|
20
|
+
Admin Password
|
|
21
|
+
</td>
|
|
22
|
+
<td colspan =3><input type = 'password' id = 'txtPassword' value = ''></td>
|
|
23
|
+
</tr>
|
|
24
|
+
<tr>
|
|
25
|
+
|
|
26
|
+
<td colspan =5><button id = 'btnLogin'>Login</button></td>
|
|
27
|
+
</tr>
|
|
28
|
+
<tr><td>Service</td>
|
|
29
|
+
<td colspan =3 ><select id ='cmbService'>
|
|
30
|
+
|
|
31
|
+
</select>
|
|
32
|
+
</td>
|
|
33
|
+
</tr>
|
|
34
|
+
|
|
35
|
+
<tr>
|
|
36
|
+
<td >Age</td>
|
|
37
|
+
<td colspan =3><input type ='text' id = 'txtAge' value = '10' style="width:30px"> minutes</td>
|
|
38
|
+
</tr>
|
|
39
|
+
<tr colspan =4>
|
|
40
|
+
<td><button id = 'btnTraceLogs'>Trace Logs</button></td>
|
|
41
|
+
<td><button id = 'btnValidateLogs'>Validate Logs</button></td>
|
|
42
|
+
<td><button id = 'btnUpdateSubnetworkLogs'>Update Subnetwork Logs</button></td>
|
|
43
|
+
<td><button id = 'btnAR'>Attribute Rules Logs</button></td>
|
|
44
|
+
<td><button id = 'btnSQLLogs'>SQL Logs</button></td>
|
|
45
|
+
<td>Message<input id = 'txtFilter' type = 'text'></td>
|
|
46
|
+
<td>User<input id = 'txtUser' type = 'text'></td>
|
|
47
|
+
</tr>
|
|
48
|
+
|
|
49
|
+
</table>
|
|
50
|
+
|
|
51
|
+
<table id = 'tblResult' border =1 cellpadding = 4 cellspacing = 4 style="text-align:center">
|
|
52
|
+
|
|
53
|
+
</table>
|
|
54
|
+
<script type = 'module'>
|
|
55
|
+
import { Portal } from "./portal.mjs"
|
|
56
|
+
import { UtilityNetwork } from "./utilitynetwork.node.mjs"
|
|
57
|
+
import { AdminLog } from "./adminlog.mjs"
|
|
58
|
+
import logger from "./logger.mjs"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
let parameters = {
|
|
63
|
+
"user": "unadmin",
|
|
64
|
+
"password": "",
|
|
65
|
+
"portal": "https://utilitynetwork.esri.com/portal",
|
|
66
|
+
"service": "",
|
|
67
|
+
"referer": "https://utilitynetwork.esri.com/log"
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
let portal;
|
|
72
|
+
let adminLog = null;
|
|
73
|
+
let cmbService
|
|
74
|
+
let token;
|
|
75
|
+
const btnLogin = document.getElementById("btnLogin")
|
|
76
|
+
const txtAdmin = document.getElementById("txtAdmin");
|
|
77
|
+
const txtPassword = document.getElementById("txtPassword")
|
|
78
|
+
|
|
79
|
+
btnLogin.addEventListener("click", () => init())
|
|
80
|
+
async function init() {
|
|
81
|
+
try {
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
parameters.user = txtAdmin.value;
|
|
85
|
+
parameters.password = txtPassword.value;
|
|
86
|
+
parameters.referer = window.location.href;
|
|
87
|
+
|
|
88
|
+
cmbService = document.getElementById("cmbService")
|
|
89
|
+
while (cmbService.firstChild) cmbService.removeChild(cmbService.firstChild)
|
|
90
|
+
|
|
91
|
+
const opt = document.createElement("option")
|
|
92
|
+
opt.textContent = "(All Services)"
|
|
93
|
+
cmbService.appendChild (opt)
|
|
94
|
+
|
|
95
|
+
token = await getToken(parameters);
|
|
96
|
+
const services = await portal.services()
|
|
97
|
+
const featureServices = services.services.filter(s => s.type == "FeatureServer")
|
|
98
|
+
featureServices.forEach(s=> {
|
|
99
|
+
const o = document.createElement("option");
|
|
100
|
+
o.text = s.name;
|
|
101
|
+
cmbService.appendChild(o);
|
|
102
|
+
})
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
document.getElementById("btnTraceLogs").disabled = false
|
|
107
|
+
document.getElementById("btnValidateLogs").disabled = false
|
|
108
|
+
document.getElementById("btnUpdateSubnetworkLogs").disabled = false
|
|
109
|
+
document.getElementById("btnAR").disabled = false
|
|
110
|
+
document.getElementById("btnSQLLogs").disabled = false
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
}
|
|
114
|
+
catch(ex){
|
|
115
|
+
alert(`Failed to login to ${parameters.portal} check the user/password and make sure its an admin`)
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
async function getToken(parameters) {
|
|
121
|
+
portal = new Portal(parameters.portal, parameters.user, parameters.password,300, parameters.server, parameters.referer)
|
|
122
|
+
console.log("About to connect..")
|
|
123
|
+
const token = await portal.connect()
|
|
124
|
+
console.log(`Token generanted successfully.`)
|
|
125
|
+
return token;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
async function loadTraceLogs () {
|
|
130
|
+
|
|
131
|
+
//build table
|
|
132
|
+
const tblResult = document.getElementById("tblResult");
|
|
133
|
+
//clear
|
|
134
|
+
while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
parameters.service = cmbService.options[cmbService.selectedIndex].text
|
|
138
|
+
|
|
139
|
+
let mins = document.getElementById("txtAge").value //query logs for the last 30 minutes
|
|
140
|
+
|
|
141
|
+
console.log(`Querying trace logs for ${parameters.service} for the last ${mins} minutes ...`)
|
|
142
|
+
|
|
143
|
+
//page query the admin log
|
|
144
|
+
let allMessages = await adminLogQueryWithPaging(mins, parameters.service, [102002], "------ Trace Parameters ----", "VERBOSE")
|
|
145
|
+
|
|
146
|
+
await populateMessages(allMessages)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
async function loadValidateLogs () {
|
|
154
|
+
//build table
|
|
155
|
+
const tblResult = document.getElementById("tblResult");
|
|
156
|
+
//clear
|
|
157
|
+
while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
|
|
158
|
+
|
|
159
|
+
parameters.service = cmbService.options[cmbService.selectedIndex].text
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
let mins = document.getElementById("txtAge").value //query logs for the last 30 minutes
|
|
163
|
+
|
|
164
|
+
console.log(`Querying validate logs for ${parameters.service} for the last ${mins} minutes ...`)
|
|
165
|
+
|
|
166
|
+
//-1 && m.message.indexOf("The network is built.") > -1 && m.methodName == 'BuildEngineLog')
|
|
167
|
+
//page query the admin log
|
|
168
|
+
let allMessages = await adminLogQueryWithPaging(mins, parameters.service, [102003], "The network is built.", "VERBOSE")
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
//validate logs missing elapsed populate it
|
|
172
|
+
allMessages = allMessages.map( m => {
|
|
173
|
+
try{
|
|
174
|
+
|
|
175
|
+
//The network is built. 0.093 seconds (4.982 total) - 29 MB memory
|
|
176
|
+
|
|
177
|
+
let re = /The network is built. [-+]?([0-9]*\.[0-9]+|[0-9]+) seconds \([-+]?([0-9]*\.[0-9]+|[0-9]+) total\)/;
|
|
178
|
+
let res = re.exec(m.message)
|
|
179
|
+
if (res && res.length > 1)
|
|
180
|
+
m.elapsed = res[2]
|
|
181
|
+
|
|
182
|
+
return m;
|
|
183
|
+
}
|
|
184
|
+
catch(ex){
|
|
185
|
+
return m;
|
|
186
|
+
}
|
|
187
|
+
})
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
await populateMessages(allMessages)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
async function loadUpdateSubnetworkLogs () {
|
|
197
|
+
//build table
|
|
198
|
+
const tblResult = document.getElementById("tblResult");
|
|
199
|
+
//clear
|
|
200
|
+
while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
|
|
201
|
+
|
|
202
|
+
parameters.service = cmbService.options[cmbService.selectedIndex].text
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
let mins = document.getElementById("txtAge").value //query logs for the last 30 minutes
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
console.log(`Querying subnetwork logs for ${parameters.service} for the last ${mins} minutes ...`)
|
|
209
|
+
let allMessages = await adminLogQueryWithPaging(mins, parameters.service, [102003], "---- Subnetwork Parameters ----", "VERBOSE")
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
//update subnetwork missing elapsed populate it
|
|
213
|
+
allMessages = allMessages.map( m => {
|
|
214
|
+
try{
|
|
215
|
+
|
|
216
|
+
let re = /Total \([-+]?([0-9]*\.[0-9]+|[0-9]+) seconds\)/;
|
|
217
|
+
let res = re.exec(m.message)
|
|
218
|
+
if (res && res.length > 1)
|
|
219
|
+
m.elapsed = res[1]
|
|
220
|
+
|
|
221
|
+
re = /Total update subnetwork time \([-+]?([0-9]*\.[0-9]+|[0-9]+) seconds\)/;
|
|
222
|
+
res = re.exec(m.message)
|
|
223
|
+
if (res && res.length > 1)
|
|
224
|
+
m.elapsed = res[1]
|
|
225
|
+
|
|
226
|
+
return m;
|
|
227
|
+
}
|
|
228
|
+
catch(ex){
|
|
229
|
+
return m;
|
|
230
|
+
}
|
|
231
|
+
})
|
|
232
|
+
await populateMessages(allMessages)
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
async function loadAttributeRules () {
|
|
240
|
+
|
|
241
|
+
//build table
|
|
242
|
+
const tblResult = document.getElementById("tblResult");
|
|
243
|
+
while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
|
|
244
|
+
|
|
245
|
+
let mins = document.getElementById("txtAge").value //query logs for the last 30 minutes
|
|
246
|
+
|
|
247
|
+
parameters.service = cmbService.options[cmbService.selectedIndex].text
|
|
248
|
+
|
|
249
|
+
let showMaxGuid = true
|
|
250
|
+
let showMinGuid = false
|
|
251
|
+
|
|
252
|
+
logger.info(`Querying attribute rules logs for ${parameters.service} in the past ${mins} minutes...`)
|
|
253
|
+
let allMessages = await adminLogQueryWithPaging(mins, parameters.service, [102003], "Attribute rule execution complete:", "DEBUG")
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
const arMessages = filterMessages(allMessages)
|
|
257
|
+
.filter(m => m.message.indexOf("Attribute rule execution complete:") > -1)
|
|
258
|
+
.map (m => JSON.parse(m.message.replace("Attribute rule execution complete:", "")))
|
|
259
|
+
.map( m => {
|
|
260
|
+
m["Elapsed Time (ms)"] = Math.round(m["Elapsed Time"]*1000000)/1000
|
|
261
|
+
// m["Arcade Evaluation Time:"] = Math.round(m["Arcade Evaluation Time:"]*1000,6)
|
|
262
|
+
//m.ArcadeTime = m["Arcade Evaluation Time:"]
|
|
263
|
+
|
|
264
|
+
delete m["Arcade Evaluation Time:"];
|
|
265
|
+
delete m["Elapsed Time"];
|
|
266
|
+
//delete m ['GlobalID'];
|
|
267
|
+
return m
|
|
268
|
+
})
|
|
269
|
+
.sort( (m1, m2) => m2["Elapsed Time (ms)"]- m1["Elapsed Time (ms)"])
|
|
270
|
+
.reduce( ( prev, cur ) => {
|
|
271
|
+
if (prev [cur["Rule name"]] === undefined)
|
|
272
|
+
{
|
|
273
|
+
prev [cur["Rule name"]] = {
|
|
274
|
+
"totalTime": 0,
|
|
275
|
+
"occurrence": 0,
|
|
276
|
+
"minTime": Number.MAX_SAFE_INTEGER,
|
|
277
|
+
"maxTime": -1,
|
|
278
|
+
"avgTime": 0,
|
|
279
|
+
"maxGuid": null,
|
|
280
|
+
"minGuid": null
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
prev [cur["Rule name"]].totalTime = prev [cur["Rule name"]].totalTime + cur["Elapsed Time (ms)"]
|
|
285
|
+
|
|
286
|
+
if (cur["Elapsed Time (ms)"] < prev [cur["Rule name"]].minTime ) {
|
|
287
|
+
prev [cur["Rule name"]].minTime = cur["Elapsed Time (ms)"];
|
|
288
|
+
prev [cur["Rule name"]].minGuid = cur["GlobalID"];
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
if (cur["Elapsed Time (ms)"] > prev [cur["Rule name"]].maxTime ){
|
|
292
|
+
prev [cur["Rule name"]].maxTime = cur["Elapsed Time (ms)"];
|
|
293
|
+
prev [cur["Rule name"]].maxGuid = cur["GlobalID"];
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
prev [cur["Rule name"]].occurrence++
|
|
297
|
+
|
|
298
|
+
prev [cur["Rule name"]].avgTime = prev [cur["Rule name"]].totalTime / prev [cur["Rule name"]].occurrence
|
|
299
|
+
|
|
300
|
+
return prev
|
|
301
|
+
}, {})
|
|
302
|
+
|
|
303
|
+
const rules = Object.keys(arMessages)
|
|
304
|
+
.map(a => {
|
|
305
|
+
const rule = {}
|
|
306
|
+
rule["Attribute Rule"] = a;
|
|
307
|
+
rule["Total Cost (ms)"] = parseFloat(arMessages[a].totalTime.toFixed(2))
|
|
308
|
+
if (!showMinGuid && !showMaxGuid) rule["Average Cost (ms)"] = parseFloat(arMessages[a].avgTime.toFixed(2))
|
|
309
|
+
rule["Max execution time (ms)"] = parseFloat(arMessages[a].maxTime.toFixed(2))
|
|
310
|
+
if (showMaxGuid) rule["Max GUID"] = arMessages[a].maxGuid
|
|
311
|
+
rule["Min execution time (ms)"] = parseFloat(arMessages[a].minTime.toFixed(2))
|
|
312
|
+
if (showMinGuid) rule["Min GUID"] = arMessages[a].minGuid
|
|
313
|
+
if (!showMinGuid && !showMaxGuid) rule["Occurrence"] = arMessages[a].occurrence;
|
|
314
|
+
return rule;
|
|
315
|
+
})
|
|
316
|
+
.sort( (m1, m2) => m2["Total Cost (ms)"] -m1["Total Cost (ms)"])
|
|
317
|
+
console.table(rules)
|
|
318
|
+
|
|
319
|
+
/*
|
|
320
|
+
Attribute Rule
|
|
321
|
+
│ Total Cost (ms)
|
|
322
|
+
│ Max execution time (ms)
|
|
323
|
+
│ Max GUID
|
|
324
|
+
│ Min execution time (m
|
|
325
|
+
|
|
326
|
+
*/
|
|
327
|
+
while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
|
|
328
|
+
|
|
329
|
+
const header = document.createElement("tr");
|
|
330
|
+
const cAttributeRule = document.createElement("th");
|
|
331
|
+
cAttributeRule.textContent = "Attribute Rule"
|
|
332
|
+
|
|
333
|
+
const cTotalCost = document.createElement("th");
|
|
334
|
+
cTotalCost.textContent = "Total Cost (ms)"
|
|
335
|
+
|
|
336
|
+
const cMaxExecutionTime = document.createElement("th");
|
|
337
|
+
cMaxExecutionTime.textContent = "Max execution time (ms)"
|
|
338
|
+
|
|
339
|
+
const cMaxGuid = document.createElement("th");
|
|
340
|
+
cMaxGuid.textContent = "Max GUID"
|
|
341
|
+
|
|
342
|
+
const cMinExecutionTime = document.createElement("th");
|
|
343
|
+
cMinExecutionTime.textContent = "Min execution time (ms)"
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
header.appendChild (cAttributeRule)
|
|
347
|
+
header.appendChild (cTotalCost)
|
|
348
|
+
header.appendChild (cMaxExecutionTime)
|
|
349
|
+
header.appendChild (cMaxGuid)
|
|
350
|
+
header.appendChild (cMinExecutionTime)
|
|
351
|
+
|
|
352
|
+
tblResult.appendChild(header)
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
rules.forEach(rule =>
|
|
356
|
+
{
|
|
357
|
+
|
|
358
|
+
const logRow = document.createElement("tr");
|
|
359
|
+
const cAttributeRule = document.createElement("td");
|
|
360
|
+
cAttributeRule.textContent = rule["Attribute Rule"]
|
|
361
|
+
|
|
362
|
+
const cTotalCost = document.createElement("td");
|
|
363
|
+
cTotalCost.textContent = rule["Total Cost (ms)"]
|
|
364
|
+
|
|
365
|
+
const cMaxExecutionTime = document.createElement("td");
|
|
366
|
+
cMaxExecutionTime.textContent = rule["Max execution time (ms)"]
|
|
367
|
+
|
|
368
|
+
const cMaxGuid = document.createElement("td");
|
|
369
|
+
cMaxGuid.textContent = rule["Max GUID"]
|
|
370
|
+
|
|
371
|
+
const cMinExecutionTime = document.createElement("td");
|
|
372
|
+
cMinExecutionTime.textContent = rule["Min execution time (ms)"]
|
|
373
|
+
|
|
374
|
+
logRow.appendChild (cAttributeRule)
|
|
375
|
+
logRow.appendChild (cTotalCost)
|
|
376
|
+
logRow.appendChild (cMaxExecutionTime)
|
|
377
|
+
logRow.appendChild (cMaxGuid)
|
|
378
|
+
logRow.appendChild (cMinExecutionTime)
|
|
379
|
+
tblResult.appendChild(logRow)
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
})
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
const totalARExecution = rules.reduce( (prev, cur) => prev + cur["Total Cost (ms)"], 0)
|
|
386
|
+
console.log(`Total time spend executing attribute rules (${Math.round(totalARExecution)} ms) (${Math.round(totalARExecution/1000)} s) (${Math.round(totalARExecution/(1000*60))} m)`)
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
}
|
|
391
|
+
async function loadSQLLogs () {
|
|
392
|
+
|
|
393
|
+
//build table
|
|
394
|
+
const tblResult = document.getElementById("tblResult");
|
|
395
|
+
//clear
|
|
396
|
+
while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
parameters.service = cmbService.options[cmbService.selectedIndex].text
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
let mins = document.getElementById("txtAge").value //query logs for the last 30 minutes
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
console.log(`Querying cursor sql logs for ${parameters.service} for the last ${mins} minutes ...`)
|
|
406
|
+
|
|
407
|
+
//startTime is the most recent
|
|
408
|
+
//endTime is the oldest
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
/*
|
|
413
|
+
const topLogCount = 100;
|
|
414
|
+
const startTime = Date.now()
|
|
415
|
+
const endTime = Date.now() - mins*60*1000
|
|
416
|
+
let services = [theService + ".MapServer"]
|
|
417
|
+
if (theService == "(All Services)")
|
|
418
|
+
services = "*"
|
|
419
|
+
let result= await adminLog.query([code], services, topLogCount, startTime ,endTime , "DEBUG")
|
|
420
|
+
let jsonRes = await result.json()
|
|
421
|
+
let allMessages = [].concat(jsonRes.logMessages)
|
|
422
|
+
allMessages = allMessages.filter(m => m.message.indexOf("EndCursor;") > -1)
|
|
423
|
+
while (jsonRes.hasMore)
|
|
424
|
+
{
|
|
425
|
+
//start paging
|
|
426
|
+
logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs between ${new Date(jsonRes.endTime)} and ${new Date(endTime)}`)
|
|
427
|
+
result= await adminLog.query([102023], services, pageSize, jsonRes.endTime, endTime, "DEBUG")
|
|
428
|
+
jsonRes = await result.json()
|
|
429
|
+
|
|
430
|
+
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("EndCursor;") > -1))
|
|
431
|
+
}
|
|
432
|
+
*/
|
|
433
|
+
|
|
434
|
+
//page query the admin log
|
|
435
|
+
let allMessages = await adminLogQueryWithPaging(mins, parameters.service, [102023], "EndCursor;", "DEBUG")
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
console.log ("Filtering messages...")
|
|
439
|
+
|
|
440
|
+
allMessages = filterMessages(allMessages)
|
|
441
|
+
.map( m=> {
|
|
442
|
+
m.dataAccessElapsed =0;
|
|
443
|
+
m.executeQueryElapsed = 0;
|
|
444
|
+
const sDataAccess = "DataAccess "
|
|
445
|
+
const idxDataAccess = m.message.indexOf(sDataAccess)
|
|
446
|
+
if (idxDataAccess >=0) {
|
|
447
|
+
const idxNext = m.message.indexOf(" ",idxDataAccess + sDataAccess.length )
|
|
448
|
+
m.dataAccessElapsed = parseInt(m.message.substr( idxDataAccess + sDataAccess.length, idxNext - idxDataAccess - sDataAccess.length))
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
const sExecuteQuery = "ExecuteQuery "
|
|
452
|
+
const idxExecuteQuery = m.message.indexOf(sExecuteQuery)
|
|
453
|
+
if (idxExecuteQuery >=0) {
|
|
454
|
+
const idxNext = m.message.indexOf(" ",idxExecuteQuery + sExecuteQuery.length)
|
|
455
|
+
m.executeQueryElapsed = parseInt(m.message.substr( idxExecuteQuery + sExecuteQuery.length, idxNext - idxExecuteQuery - sExecuteQuery.length))
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
//m.dataAccessElapsed = parseFloat(m.message.split(";")[1].split(" ")[1])
|
|
460
|
+
//m.executeQueryElapsed = parseFloat(m.message.split(";")[2].split(" ")[1])
|
|
461
|
+
m.totalExecutionElapsed = m.dataAccessElapsed + m.executeQueryElapsed
|
|
462
|
+
m.elapsed = parseFloat(m.elapsed); return m;
|
|
463
|
+
|
|
464
|
+
})
|
|
465
|
+
.sort( (m1,m2) => m2.totalExecutionElapsed - m1.totalExecutionElapsed)
|
|
466
|
+
.slice(0, 100) ;//first 100
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
console.log("-----Top SQL----")
|
|
470
|
+
while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
|
|
471
|
+
|
|
472
|
+
let i =0;
|
|
473
|
+
|
|
474
|
+
const header = document.createElement("tr");
|
|
475
|
+
const cService = document.createElement("th");
|
|
476
|
+
cService.textContent = "Source"
|
|
477
|
+
const cTime = document.createElement("th");
|
|
478
|
+
cTime.textContent = "Time"
|
|
479
|
+
const cUser = document.createElement("th");
|
|
480
|
+
cUser.textContent = "User"
|
|
481
|
+
const cVersion = document.createElement("th");
|
|
482
|
+
cVersion.textContent = "Version"
|
|
483
|
+
const cQueryTime = document.createElement("th");
|
|
484
|
+
cQueryTime.textContent = "Query Time (ms)"
|
|
485
|
+
cQueryTime.title = "includes search + data access nextRow"
|
|
486
|
+
const cTotalTime = document.createElement("th");
|
|
487
|
+
cTotalTime.title = "Total time the cursor was opened"
|
|
488
|
+
cTotalTime.textContent = "Total Time (ms)"
|
|
489
|
+
const cLog = document.createElement("th");
|
|
490
|
+
cLog.textContent = "Full SQL"
|
|
491
|
+
header.appendChild (cService)
|
|
492
|
+
header.appendChild (cTime)
|
|
493
|
+
header.appendChild (cUser)
|
|
494
|
+
header.appendChild (cVersion)
|
|
495
|
+
header.appendChild (cQueryTime)
|
|
496
|
+
header.appendChild (cTotalTime)
|
|
497
|
+
header.appendChild (cLog)
|
|
498
|
+
tblResult.appendChild(header)
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
allMessages= allMessages.forEach(m =>
|
|
502
|
+
{
|
|
503
|
+
|
|
504
|
+
const logRow = document.createElement("tr");
|
|
505
|
+
const x = m.message.split(";")
|
|
506
|
+
x.shift()
|
|
507
|
+
console.log(`id: ${i++}`)
|
|
508
|
+
console.log(`\tAt: ${new Date(m.time)} (${m.time})`)
|
|
509
|
+
console.log(`\tUser: ${m.user}`)
|
|
510
|
+
console.log(`\tTotal Time: ${numberWithCommas(Math.round(m.elapsed*1000))} ms (Total time the cursor was opened)`)
|
|
511
|
+
console.log(`\tQuery Time: ${numberWithCommas(m.totalExecutionElapsed)} ms (includes search + data access nextRow)`)
|
|
512
|
+
console.log(`\tQuery:`)
|
|
513
|
+
x.forEach(a => console.log(`\t${a}`))
|
|
514
|
+
console.log(`\n`)
|
|
515
|
+
|
|
516
|
+
const cService = document.createElement("th");
|
|
517
|
+
cService.textContent = m.source.replace(".MapServer", "")
|
|
518
|
+
|
|
519
|
+
const cTime = document.createElement("td");
|
|
520
|
+
cTime.textContent = new Date(m.time).toLocaleString()
|
|
521
|
+
|
|
522
|
+
const cUser = document.createElement("td");
|
|
523
|
+
cUser.textContent = m.user
|
|
524
|
+
const cVersion = document.createElement("td");
|
|
525
|
+
const idxVersion = m.message.indexOf("Version: ")
|
|
526
|
+
if (idxVersion >=0) {
|
|
527
|
+
const idxNext = m.message.indexOf(";",idxVersion)
|
|
528
|
+
cVersion.textContent = m.message.substr( idxVersion + 9, idxNext - idxVersion - 9 )
|
|
529
|
+
}
|
|
530
|
+
const cQueryTime = document.createElement("td");
|
|
531
|
+
cQueryTime.textContent = numberWithCommas(m.totalExecutionElapsed)
|
|
532
|
+
const cTotalTime = document.createElement("td");
|
|
533
|
+
cTotalTime.textContent = numberWithCommas(Math.round(m.elapsed*1000))
|
|
534
|
+
const cLog = document.createElement("td");
|
|
535
|
+
const cLogText = document.createElement("input")
|
|
536
|
+
cLogText.type = "text"
|
|
537
|
+
cLogText.style = "width:500px"
|
|
538
|
+
cLogText.value = m.message.substr( m.message.indexOf("SQL: ") + 5, m.message.length)
|
|
539
|
+
cLogText.readOnly= true;
|
|
540
|
+
cLog.appendChild(cLogText)
|
|
541
|
+
//cLog.textContent = m.message.substr( m.message.indexOf("SQL: ") + 5, 50) + "..."
|
|
542
|
+
cLog.fullLog = m.message;
|
|
543
|
+
cLog.title = m.message + "\nClick to copy"
|
|
544
|
+
cLog.addEventListener("click", e=> {navigator.clipboard.writeText(e.target.fullLog); e.target.selectionStart = 0; e.target.selectionEnd = e.target.value.length;
|
|
545
|
+
/* alert("Copied to clipboard") */})
|
|
546
|
+
logRow.appendChild (cService)
|
|
547
|
+
logRow.appendChild (cTime)
|
|
548
|
+
logRow.appendChild (cUser)
|
|
549
|
+
logRow.appendChild (cVersion)
|
|
550
|
+
logRow.appendChild (cQueryTime)
|
|
551
|
+
logRow.appendChild (cTotalTime)
|
|
552
|
+
logRow.appendChild (cLog)
|
|
553
|
+
tblResult.appendChild(logRow)
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
})
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
//await populateMessages(allMessages)
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
function numberWithCommas(x) {
|
|
571
|
+
// return x.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
|
|
572
|
+
return x.toLocaleString()
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
//init();
|
|
576
|
+
|
|
577
|
+
const btnTraceLogs = document.getElementById("btnTraceLogs");
|
|
578
|
+
btnTraceLogs.addEventListener("click", e => loadTraceLogs())
|
|
579
|
+
btnTraceLogs.disabled = true
|
|
580
|
+
|
|
581
|
+
const btnValidateLogs = document.getElementById("btnValidateLogs");
|
|
582
|
+
btnValidateLogs.addEventListener("click", e => loadValidateLogs())
|
|
583
|
+
btnValidateLogs.disabled = true
|
|
584
|
+
|
|
585
|
+
const btnUpdateSubnetworkLogs = document.getElementById("btnUpdateSubnetworkLogs");
|
|
586
|
+
btnUpdateSubnetworkLogs.addEventListener("click", e => loadUpdateSubnetworkLogs())
|
|
587
|
+
btnUpdateSubnetworkLogs.disabled = true
|
|
588
|
+
|
|
589
|
+
const btnSQLLogs = document.getElementById("btnSQLLogs");
|
|
590
|
+
btnSQLLogs.addEventListener("click", e => loadSQLLogs())
|
|
591
|
+
btnSQLLogs.disabled = true
|
|
592
|
+
|
|
593
|
+
const btnAR = document.getElementById("btnAR");
|
|
594
|
+
btnAR.addEventListener("click", e => loadAttributeRules())
|
|
595
|
+
btnAR.disabled = true
|
|
596
|
+
|
|
597
|
+
function filterMessages (allMessages) {
|
|
598
|
+
const txtFilter = document.getElementById("txtFilter")
|
|
599
|
+
const txtUser = document.getElementById("txtUser")
|
|
600
|
+
if (txtFilter.value == "" && txtUser.value == "") return allMessages;
|
|
601
|
+
//const filtered= allMessages.filter(m => m.message.toUpperCase().indexOf(txtFilter.value.toUpperCase()) > 0 && m.user.toUpperCase().indexOf(txtUser.value.toUpperCase()) > 0)
|
|
602
|
+
const filtered= allMessages.filter(m => {
|
|
603
|
+
|
|
604
|
+
if (m.message.toUpperCase().indexOf(txtFilter.value.toUpperCase()) >= 0
|
|
605
|
+
&&
|
|
606
|
+
m.user.toUpperCase().indexOf(txtUser.value.toUpperCase()) >= 0)
|
|
607
|
+
return true
|
|
608
|
+
|
|
609
|
+
return false
|
|
610
|
+
})
|
|
611
|
+
return filtered
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
async function populateMessages(allTheMessages) {
|
|
615
|
+
|
|
616
|
+
//sort messages
|
|
617
|
+
|
|
618
|
+
const allMessages = filterMessages (allTheMessages).map(m => {
|
|
619
|
+
const newMessage = Object.assign({}, m);
|
|
620
|
+
delete newMessage.machine;
|
|
621
|
+
delete newMessage.type;
|
|
622
|
+
delete newMessage.code;
|
|
623
|
+
delete newMessage.requestID;
|
|
624
|
+
delete newMessage.process;
|
|
625
|
+
delete newMessage.thread;
|
|
626
|
+
newMessage.elapsedms = parseInt (parseFloat(newMessage.elapsed) * 1000)
|
|
627
|
+
newMessage.time = new Date(newMessage.time).toLocaleString()
|
|
628
|
+
delete newMessage.elapsed
|
|
629
|
+
return newMessage;
|
|
630
|
+
})
|
|
631
|
+
.sort( (m1,m2) => m2.elapsedms - m1.elapsedms)
|
|
632
|
+
|
|
633
|
+
//build table
|
|
634
|
+
const tblResult = document.getElementById("tblResult");
|
|
635
|
+
//clear
|
|
636
|
+
while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
|
|
637
|
+
|
|
638
|
+
const header = document.createElement("tr");
|
|
639
|
+
const cService = document.createElement("th");
|
|
640
|
+
cService.textContent = "Source"
|
|
641
|
+
const cTime = document.createElement("th");
|
|
642
|
+
cTime.textContent = "Time"
|
|
643
|
+
const cUser = document.createElement("th");
|
|
644
|
+
cUser.textContent = "User"
|
|
645
|
+
const cMethod = document.createElement("th");
|
|
646
|
+
cMethod.textContent = "Method"
|
|
647
|
+
const cElapsedMS = document.createElement("th");
|
|
648
|
+
cElapsedMS.textContent = "Elapsed (ms)"
|
|
649
|
+
const cLog = document.createElement("th");
|
|
650
|
+
cLog.textContent = "Full Log"
|
|
651
|
+
header.appendChild (cService)
|
|
652
|
+
header.appendChild (cTime)
|
|
653
|
+
header.appendChild (cUser)
|
|
654
|
+
header.appendChild (cMethod)
|
|
655
|
+
header.appendChild (cElapsedMS)
|
|
656
|
+
header.appendChild (cLog)
|
|
657
|
+
tblResult.appendChild(header)
|
|
658
|
+
allMessages.forEach(m => {
|
|
659
|
+
const logRow = document.createElement("tr");
|
|
660
|
+
const cService = document.createElement("td");
|
|
661
|
+
cService.textContent = m.source.replace(".MapServer", "")
|
|
662
|
+
const cTime = document.createElement("td");
|
|
663
|
+
cTime.textContent = m.time
|
|
664
|
+
const cUser = document.createElement("td");
|
|
665
|
+
cUser.textContent = m.user
|
|
666
|
+
const cMethod = document.createElement("td");
|
|
667
|
+
cMethod.textContent = m.methodName
|
|
668
|
+
const cElapsedMS = document.createElement("td");
|
|
669
|
+
cElapsedMS.textContent = m.elapsedms
|
|
670
|
+
const cLog = document.createElement("td");
|
|
671
|
+
cLog.textContent = "..."
|
|
672
|
+
cLog.fullLog = m.message;
|
|
673
|
+
cLog.addEventListener("click", e=> {navigator.clipboard.writeText(e.target.fullLog); alert("Copied to clipboard")})
|
|
674
|
+
logRow.appendChild (cService)
|
|
675
|
+
logRow.appendChild (cTime)
|
|
676
|
+
logRow.appendChild (cUser)
|
|
677
|
+
logRow.appendChild (cMethod)
|
|
678
|
+
logRow.appendChild (cElapsedMS)
|
|
679
|
+
logRow.appendChild (cLog)
|
|
680
|
+
tblResult.appendChild(logRow)
|
|
681
|
+
})
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
async function adminLogQueryWithPaging(mins, theService, code, filter, level){
|
|
686
|
+
const topLogCount = 1000;
|
|
687
|
+
let stop = false;
|
|
688
|
+
const prg = document.createTextNode("Loading.... 0%");
|
|
689
|
+
const btn = document.createElement("button")
|
|
690
|
+
btn.textContent = '🛑'
|
|
691
|
+
btn.addEventListener("click", ()=> stop = true )
|
|
692
|
+
//create a new admin object (user might not be admin we won't use it until the user call log )
|
|
693
|
+
let adminLog = new AdminLog(token, portal.serverUrl)
|
|
694
|
+
tblResult.appendChild(prg);
|
|
695
|
+
tblResult.appendChild(btn)
|
|
696
|
+
const startTime = Date.now()
|
|
697
|
+
const endTime = Date.now() - mins*60*1000
|
|
698
|
+
let services = [theService + ".MapServer"]
|
|
699
|
+
if (theService == "(All Services)")
|
|
700
|
+
services = "*"
|
|
701
|
+
let result= await adminLog.query(code, services, topLogCount, startTime ,endTime , level)
|
|
702
|
+
let jsonRes = await result.json()
|
|
703
|
+
let allMessages = [].concat(jsonRes.logMessages)
|
|
704
|
+
allMessages = allMessages.filter(m => m.message.indexOf(filter) > -1)
|
|
705
|
+
while (jsonRes.hasMore && !stop)
|
|
706
|
+
{
|
|
707
|
+
//start paging
|
|
708
|
+
prg.textContent = "Loading ... " + parseInt((( (startTime - endTime) - (jsonRes.endTime - endTime) ) /((startTime - endTime)))*100) + "%"
|
|
709
|
+
logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs between ${new Date(jsonRes.endTime)} and ${new Date(endTime)}`)
|
|
710
|
+
result= await adminLog.query(code, services, topLogCount, jsonRes.endTime, endTime, level)
|
|
711
|
+
jsonRes = await result.json()
|
|
712
|
+
|
|
713
|
+
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf(filter) > -1))
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
return allMessages
|
|
717
|
+
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
|
|
721
|
+
/*
|
|
722
|
+
const topLogCount = 100;f
|
|
723
|
+
const startTime = Date.now()
|
|
724
|
+
const endTime = Date.now() - mins*60*1000
|
|
725
|
+
let services = [theService + ".MapServer"]
|
|
726
|
+
if (theService == "(All Services)")
|
|
727
|
+
services = "*"
|
|
728
|
+
let result= await adminLog.query([code], services, topLogCount, startTime ,endTime , "DEBUG")
|
|
729
|
+
let jsonRes = await result.json()
|
|
730
|
+
let allMessages = [].concat(jsonRes.logMessages)
|
|
731
|
+
allMessages = allMessages.filter(m => m.message.indexOf("EndCursor;") > -1)
|
|
732
|
+
while (jsonRes.hasMore)
|
|
733
|
+
{
|
|
734
|
+
//start paging
|
|
735
|
+
logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs between ${new Date(jsonRes.endTime)} and ${new Date(endTime)}`)
|
|
736
|
+
result= await adminLog.query([102023], services, pageSize, jsonRes.endTime, endTime, "DEBUG")
|
|
737
|
+
jsonRes = await result.json()
|
|
738
|
+
|
|
739
|
+
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("EndCursor;") > -1))
|
|
740
|
+
}
|
|
741
|
+
*/
|
|
742
|
+
</script>
|
|
743
|
+
</body>
|
|
744
|
+
</html>
|
package/index.mjs
CHANGED
|
@@ -4,10 +4,10 @@ import { Portal } from "./portal.mjs"
|
|
|
4
4
|
import { UtilityNetwork } from "./utilitynetwork.node.mjs"
|
|
5
5
|
import { AdminLog } from "./adminlog.mjs"
|
|
6
6
|
|
|
7
|
-
import
|
|
7
|
+
import logger from "./logger.mjs"
|
|
8
8
|
import fetch from "node-fetch"
|
|
9
9
|
//update version
|
|
10
|
-
let version = "0.0.
|
|
10
|
+
let version = "0.0.72";
|
|
11
11
|
const GENERATE_TOKEN_TIME_MIN = 30;
|
|
12
12
|
|
|
13
13
|
let rl = null;
|
|
@@ -408,21 +408,36 @@ const inputs = {
|
|
|
408
408
|
}
|
|
409
409
|
},
|
|
410
410
|
|
|
411
|
-
"^update subnetworks --all
|
|
411
|
+
"^update subnetworks --all" : async input => {
|
|
412
412
|
|
|
413
|
+
let subnetworks;
|
|
414
|
+
let more = false;
|
|
415
|
+
let failedSubnetworks = []
|
|
413
416
|
do {
|
|
417
|
+
|
|
418
|
+
let sort = "asc";
|
|
419
|
+
if (input.indexOf("--desc") > 0) sort = "desc"
|
|
420
|
+
let failedSubWhereClause = ""
|
|
421
|
+
|
|
422
|
+
if (failedSubnetworks.length > 0 )
|
|
423
|
+
failedSubWhereClause = " AND SUBNETWORKNAME NOT IN (" + failedSubnetworks.join(",") + ")"
|
|
424
|
+
|
|
414
425
|
console.log("Querying all subnetworks that are dirty.");
|
|
415
|
-
|
|
426
|
+
subnetworks = await un.queryDistinct(500002, "domainnetworkname,tiername,subnetworkname", "isdirty=1 " + failedSubWhereClause, `domainnetworkname ${sort},tiername ${sort},subnetworkname ${sort}`);
|
|
416
427
|
console.log(`Discovered ${subnetworks.features.length} dirty subnetworks.`);
|
|
417
428
|
|
|
418
429
|
for (let i = 0; i < subnetworks.features.length; i++) {
|
|
419
430
|
const f = subnetworks.features[i]
|
|
420
|
-
|
|
431
|
+
const subnetworkName = v(f.attributes,"subnetworkName")
|
|
432
|
+
console.log("Updating Subnetwork " + subnetworkName);
|
|
421
433
|
|
|
422
434
|
const fromDate = new Date();
|
|
423
435
|
|
|
424
436
|
const subnetworkResult = await un.updateSubnetworks(v(f.attributes,"domainNetworkName"), v(f.attributes,"tierName"), v(f.attributes,"subnetworkName"),false);
|
|
425
|
-
|
|
437
|
+
//check if we have processed this subnetwork (maybe be an error)
|
|
438
|
+
if (subnetworkResult.success == false)
|
|
439
|
+
failedSubnetworks.push("'" + subnetworkName + "'")
|
|
440
|
+
|
|
426
441
|
const toDate = new Date();
|
|
427
442
|
const timeEnable = toDate.getTime() - fromDate.getTime();
|
|
428
443
|
subnetworkResult.duration = numberWithCommas(Math.round(timeEnable)) + " ms"
|
|
@@ -447,6 +462,7 @@ const inputs = {
|
|
|
447
462
|
},
|
|
448
463
|
"^export subnetworks --all --folder .*$|^export subnetworks --all$" : async input => {
|
|
449
464
|
|
|
465
|
+
let subnetworks
|
|
450
466
|
//create folder
|
|
451
467
|
const file = input.match(/--folder .*/gm)
|
|
452
468
|
let inputDir = "Exported"
|
|
@@ -458,7 +474,7 @@ const inputs = {
|
|
|
458
474
|
do {
|
|
459
475
|
|
|
460
476
|
console.log("Querying all subnetworks that are clean.");
|
|
461
|
-
|
|
477
|
+
subnetworks = await un.queryDistinct(500002, "domainnetworkname,tiername,subnetworkname", "isdirty=0","domainnetworkname,tiername,subnetworkname");
|
|
462
478
|
console.log(`Discovered ${subnetworks.features.length} subnetworks that can be exported.`);
|
|
463
479
|
for (let i = 0; i < subnetworks.features.length; i++) {
|
|
464
480
|
const f = subnetworks.features[i]
|
|
@@ -774,16 +790,30 @@ const inputs = {
|
|
|
774
790
|
|
|
775
791
|
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("------ Trace Parameters ----") > -1))
|
|
776
792
|
}
|
|
777
|
-
|
|
793
|
+
|
|
794
|
+
allMessages = allMessages.map(m => {
|
|
778
795
|
const newMessage = Object.assign({}, m);
|
|
779
|
-
delete newMessage.message
|
|
780
796
|
delete newMessage.source;
|
|
781
797
|
delete newMessage.machine;
|
|
782
798
|
delete newMessage.type;
|
|
799
|
+
delete newMessage.code;
|
|
783
800
|
delete newMessage.requestID;
|
|
801
|
+
delete newMessage.process;
|
|
784
802
|
delete newMessage.thread;
|
|
785
|
-
|
|
803
|
+
newMessage.elapsedms = parseInt (parseFloat(newMessage.elapsed) * 1000)
|
|
804
|
+
newMessage.time = new Date(newMessage.time).toLocaleString()
|
|
805
|
+
delete newMessage.elapsed
|
|
806
|
+
return newMessage;
|
|
807
|
+
})
|
|
808
|
+
.sort( (m1,m2) => m2.elapsedms - m1.elapsedms)
|
|
809
|
+
|
|
810
|
+
const summaryMessages = allMessages.map(m => {const newM = Object.assign({}, m); delete newM.message; return newM})
|
|
811
|
+
console.table(summaryMessages)
|
|
786
812
|
|
|
813
|
+
allMessages.forEach(m => {
|
|
814
|
+
const newMessage = Object.assign({}, m);
|
|
815
|
+
delete newMessage.message
|
|
816
|
+
|
|
787
817
|
console.table([newMessage])
|
|
788
818
|
console.log(m.message)
|
|
789
819
|
})
|
|
@@ -802,37 +832,76 @@ const inputs = {
|
|
|
802
832
|
if (inputParam != null && inputParam.length > 0)
|
|
803
833
|
mins = inputParam[0].replace("--age ", "")
|
|
804
834
|
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
835
|
+
|
|
836
|
+
console.log(`Querying validate logs for ${parameters.service} for the last ${mins} minutes ...`)
|
|
837
|
+
const startTime = Date.now() - mins*60*1000
|
|
838
|
+
const endTime = Date.now();
|
|
839
|
+
let result= await adminLog.query([102003], [parameters.service+ ".MapServer"], topLogCount, startTime ,endTime , "VERBOSE")
|
|
840
|
+
let jsonRes = await result.json()
|
|
841
|
+
let allMessages = [].concat(jsonRes.logMessages)
|
|
842
|
+
allMessages = allMessages.filter(m => m.message.indexOf("-------- Environment ---") > -1 && m.message.indexOf("The network is built.") > -1 && m.methodName == 'BuildEngineLog')
|
|
843
|
+
while (jsonRes.hasMore)
|
|
844
|
+
{
|
|
845
|
+
//start paging
|
|
846
|
+
logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
|
|
847
|
+
let services = [parameters.service + ".MapServer"]
|
|
848
|
+
result= await adminLog.query([102003], services, pageSize, jsonRes.endTime, null, "VERBOSE")
|
|
849
|
+
jsonRes = await result.json()
|
|
850
|
+
|
|
851
|
+
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("-------- Environment ---") > -1 && m.message.indexOf("------ Trace Parameters ----") == -1))
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
|
|
855
|
+
|
|
856
|
+
|
|
857
|
+
//validate logs missing elapsed populate it
|
|
858
|
+
allMessages = allMessages.map( m => {
|
|
859
|
+
try{
|
|
860
|
+
|
|
861
|
+
//The network is built. 0.093 seconds (4.982 total) - 29 MB memory
|
|
862
|
+
|
|
863
|
+
let re = /The network is built. [-+]?([0-9]*\.[0-9]+|[0-9]+) seconds \([-+]?([0-9]*\.[0-9]+|[0-9]+) total\)/;
|
|
864
|
+
let res = re.exec(m.message)
|
|
865
|
+
if (res && res.length > 1)
|
|
866
|
+
m.elapsed = res[2]
|
|
867
|
+
|
|
868
|
+
return m;
|
|
869
|
+
}
|
|
870
|
+
catch(ex){
|
|
871
|
+
return m;
|
|
872
|
+
}
|
|
873
|
+
})
|
|
821
874
|
|
|
822
875
|
|
|
823
|
-
|
|
876
|
+
|
|
877
|
+
allMessages = allMessages.map(m => {
|
|
824
878
|
const newMessage = Object.assign({}, m);
|
|
825
|
-
delete newMessage.message
|
|
826
879
|
delete newMessage.source;
|
|
827
880
|
delete newMessage.machine;
|
|
828
881
|
delete newMessage.type;
|
|
882
|
+
delete newMessage.code;
|
|
829
883
|
delete newMessage.requestID;
|
|
884
|
+
delete newMessage.process;
|
|
830
885
|
delete newMessage.thread;
|
|
831
|
-
|
|
886
|
+
newMessage.elapsedms = parseInt (parseFloat(newMessage.elapsed) * 1000)
|
|
887
|
+
newMessage.time = new Date(newMessage.time).toLocaleString()
|
|
888
|
+
delete newMessage.elapsed
|
|
889
|
+
return newMessage;
|
|
890
|
+
})
|
|
891
|
+
.sort( (m1,m2) => m2.elapsedms - m1.elapsedms)
|
|
892
|
+
|
|
893
|
+
const summaryMessages = allMessages.map(m => {const newM = Object.assign({}, m); delete newM.message; return newM})
|
|
894
|
+
console.table(summaryMessages)
|
|
832
895
|
|
|
896
|
+
allMessages.forEach(m => {
|
|
897
|
+
const newMessage = Object.assign({}, m);
|
|
898
|
+
delete newMessage.message
|
|
899
|
+
|
|
833
900
|
console.table([newMessage])
|
|
834
901
|
console.log(m.message)
|
|
835
902
|
})
|
|
903
|
+
|
|
904
|
+
|
|
836
905
|
|
|
837
906
|
},
|
|
838
907
|
|
|
@@ -864,20 +933,59 @@ const inputs = {
|
|
|
864
933
|
|
|
865
934
|
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("---- Subnetwork Parameters ----") > -1))
|
|
866
935
|
}
|
|
867
|
-
|
|
936
|
+
|
|
937
|
+
|
|
938
|
+
|
|
939
|
+
//update subnetwork missing elapsed populate it
|
|
940
|
+
allMessages = allMessages.map( m => {
|
|
941
|
+
try{
|
|
942
|
+
|
|
943
|
+
let re = /Total \([-+]?([0-9]*\.[0-9]+|[0-9]+) seconds\)/;
|
|
944
|
+
let res = re.exec(m.message)
|
|
945
|
+
if (res && res.length > 1)
|
|
946
|
+
m.elapsed = res[1]
|
|
947
|
+
|
|
948
|
+
re = /Total update subnetwork time \([-+]?([0-9]*\.[0-9]+|[0-9]+) seconds\)/;
|
|
949
|
+
res = re.exec(m.message)
|
|
950
|
+
if (res && res.length > 1)
|
|
951
|
+
m.elapsed = res[1]
|
|
952
|
+
|
|
953
|
+
return m;
|
|
954
|
+
}
|
|
955
|
+
catch(ex){
|
|
956
|
+
return m;
|
|
957
|
+
}
|
|
958
|
+
})
|
|
959
|
+
|
|
960
|
+
|
|
961
|
+
allMessages = allMessages.map(m => {
|
|
868
962
|
const newMessage = Object.assign({}, m);
|
|
869
|
-
delete newMessage.message
|
|
870
963
|
delete newMessage.source;
|
|
871
964
|
delete newMessage.machine;
|
|
872
965
|
delete newMessage.type;
|
|
966
|
+
delete newMessage.code;
|
|
873
967
|
delete newMessage.requestID;
|
|
968
|
+
delete newMessage.process;
|
|
874
969
|
delete newMessage.thread;
|
|
875
|
-
|
|
970
|
+
newMessage.elapsedms = parseInt (parseFloat(newMessage.elapsed) * 1000)
|
|
971
|
+
newMessage.time = new Date(newMessage.time).toLocaleString()
|
|
972
|
+
delete newMessage.elapsed
|
|
973
|
+
return newMessage;
|
|
974
|
+
})
|
|
975
|
+
.sort( (m1,m2) => m2.elapsedms - m1.elapsedms)
|
|
976
|
+
|
|
977
|
+
const summaryMessages = allMessages.map(m => {const newM = Object.assign({}, m); delete newM.message; return newM})
|
|
978
|
+
console.table(summaryMessages)
|
|
876
979
|
|
|
980
|
+
allMessages.forEach(m => {
|
|
981
|
+
const newMessage = Object.assign({}, m);
|
|
982
|
+
delete newMessage.message
|
|
983
|
+
|
|
877
984
|
console.table([newMessage])
|
|
878
985
|
console.log(m.message)
|
|
879
986
|
})
|
|
880
987
|
|
|
988
|
+
|
|
881
989
|
},
|
|
882
990
|
|
|
883
991
|
|
|
@@ -942,60 +1050,6 @@ const inputs = {
|
|
|
942
1050
|
},
|
|
943
1051
|
|
|
944
1052
|
|
|
945
|
-
|
|
946
|
-
"^cursorlogs --age": async input => {
|
|
947
|
-
const topLogCount = 1000;
|
|
948
|
-
const pageSize = 10000
|
|
949
|
-
|
|
950
|
-
const inputParam = input.match(/--age .*/gm)
|
|
951
|
-
let mins = 30; //query logs for the last 30 minutes
|
|
952
|
-
if (inputParam != null && inputParam.length > 0)
|
|
953
|
-
mins = inputParam[0].replace("--age ", "")
|
|
954
|
-
|
|
955
|
-
console.log(`Querying cursor sql logs for ${parameters.service} for the last ${mins} minutes ...`)
|
|
956
|
-
const startTime = Date.now() - mins*60*1000
|
|
957
|
-
const endTime = Date.now();
|
|
958
|
-
let result= await adminLog.query([102023], [parameters.service+ ".MapServer"], topLogCount, startTime ,endTime , "DEBUG")
|
|
959
|
-
let jsonRes = await result.json()
|
|
960
|
-
let allMessages = [].concat(jsonRes.logMessages)
|
|
961
|
-
allMessages = allMessages.filter(m => m.message.indexOf("EndCursor;") > -1)
|
|
962
|
-
while (jsonRes.hasMore)
|
|
963
|
-
{
|
|
964
|
-
//start paging
|
|
965
|
-
logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
|
|
966
|
-
result= await adminLog.query([102023], [parameters.service + ".MapServer"], pageSize, jsonRes.endTime, null, "DEBUG")
|
|
967
|
-
jsonRes = await result.json()
|
|
968
|
-
|
|
969
|
-
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("EndCursor;") > -1))
|
|
970
|
-
}
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
//
|
|
974
|
-
allMessages= allMessages.map(m => {
|
|
975
|
-
|
|
976
|
-
const r= m.message.split (";")
|
|
977
|
-
delete r[0];
|
|
978
|
-
delete r[1];
|
|
979
|
-
delete r[3];
|
|
980
|
-
return r;
|
|
981
|
-
|
|
982
|
-
const newMessage = Object.assign({}, m);
|
|
983
|
-
|
|
984
|
-
delete newMessage.source;
|
|
985
|
-
delete newMessage.machine;
|
|
986
|
-
delete newMessage.type;
|
|
987
|
-
delete newMessage.requestID;
|
|
988
|
-
delete newMessage.methodName;
|
|
989
|
-
delete newMessage.process;
|
|
990
|
-
delete newMessage.thread;
|
|
991
|
-
delete newMessage.time;
|
|
992
|
-
delete newMessage.code;
|
|
993
|
-
|
|
994
|
-
return newMessage
|
|
995
|
-
|
|
996
|
-
})
|
|
997
|
-
console.table(allMessages)
|
|
998
|
-
},
|
|
999
1053
|
|
|
1000
1054
|
|
|
1001
1055
|
"^arlogs --byrule": async input => {
|
|
@@ -1010,17 +1064,29 @@ const inputs = {
|
|
|
1010
1064
|
if (inputParam != null && inputParam.length > 0 && inputParam[0].indexOf("--minguid") > -1)
|
|
1011
1065
|
showMinGuid = true
|
|
1012
1066
|
|
|
1067
|
+
|
|
1068
|
+
const ageInputParam = input.match(/--age [0-9]*/)
|
|
1069
|
+
let mins = 30; //query logs for the last 30 minutes
|
|
1070
|
+
if (ageInputParam != null && ageInputParam.length > 0)
|
|
1071
|
+
mins = ageInputParam[0].replace("--age ", "")
|
|
1072
|
+
|
|
1073
|
+
|
|
1074
|
+
const startTime = Date.now() - mins*60*1000
|
|
1075
|
+
const endTime = Date.now();
|
|
1076
|
+
|
|
1013
1077
|
const pageSize = 10000 //maximum messages per page
|
|
1014
|
-
logger.info(`Querying attribute rules logs for ${parameters.service} ...`)
|
|
1015
|
-
let result= await adminLog.query([102003], [parameters.service + ".MapServer"], pageSize)
|
|
1078
|
+
logger.info(`Querying attribute rules logs for ${parameters.service} in the past ${mins} minutes...`)
|
|
1079
|
+
let result= await adminLog.query([102003], [parameters.service + ".MapServer"], pageSize, startTime ,endTime , "DEBUG")
|
|
1016
1080
|
let jsonRes = await result.json()
|
|
1017
1081
|
let allMessages = [].concat(jsonRes.logMessages)
|
|
1018
1082
|
|
|
1019
|
-
while (jsonRes.hasMore)
|
|
1083
|
+
while (jsonRes.hasMore && jsonRes.endTime > startTime)
|
|
1020
1084
|
{
|
|
1021
1085
|
//start paging
|
|
1022
1086
|
logger.info(`Aggregating messages... total so far ${allMessages.length} debug entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
|
|
1023
|
-
|
|
1087
|
+
|
|
1088
|
+
|
|
1089
|
+
result= await adminLog.query([102003], [parameters.service + ".MapServer"], pageSize, jsonRes.endTime )
|
|
1024
1090
|
jsonRes = await result.json()
|
|
1025
1091
|
allMessages = allMessages.concat(jsonRes.logMessages)
|
|
1026
1092
|
}
|
package/logger.mjs
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
const logger = {}
|
|
2
|
+
logger.info = console.log
|
|
3
|
+
logger.error = console.error
|
|
4
|
+
|
|
5
|
+
export default logger;
|
|
6
|
+
|
|
7
|
+
/*
|
|
1
8
|
import winston from "winston"
|
|
2
9
|
|
|
3
10
|
export const logger = winston.createLogger();
|
|
@@ -5,4 +12,4 @@ export const logger = winston.createLogger();
|
|
|
5
12
|
logger.add(
|
|
6
13
|
new winston.transports.Console({"format": winston.format.json()})
|
|
7
14
|
);
|
|
8
|
-
|
|
15
|
+
*/
|
package/makerequest.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
|
|
1
|
+
|
|
2
|
+
|
|
2
3
|
|
|
3
4
|
export function makeRequest (opts) {
|
|
4
5
|
|
|
@@ -24,7 +25,18 @@ export function makeRequest (opts) {
|
|
|
24
25
|
Object.keys(opts.headers).forEach( key => headers[key] = opts.headers[key] )
|
|
25
26
|
|
|
26
27
|
//console.log(opts)
|
|
27
|
-
|
|
28
|
+
|
|
29
|
+
let f;
|
|
30
|
+
try {
|
|
31
|
+
const nodeFetch = await import ("node-fetch");
|
|
32
|
+
f = nodeFetch.default;
|
|
33
|
+
}
|
|
34
|
+
catch(ex) {
|
|
35
|
+
f = fetch;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
const result = await f(opts.url, {
|
|
28
40
|
"method" : opts.method,
|
|
29
41
|
"headers": headers,
|
|
30
42
|
"body": params
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "un-cli",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.72",
|
|
4
4
|
"description": "Command line interface for working with ArcGIS Utility Network Extension",
|
|
5
5
|
"main": "index.mjs",
|
|
6
6
|
"bin": {
|
|
@@ -17,8 +17,6 @@
|
|
|
17
17
|
"author": "Hussein Nasser",
|
|
18
18
|
"license": "ISC",
|
|
19
19
|
"dependencies": {
|
|
20
|
-
"node-fetch": "^2.6.0"
|
|
21
|
-
"progress": "^2.0.3",
|
|
22
|
-
"winston": "^3.2.1"
|
|
20
|
+
"node-fetch": "^2.6.0"
|
|
23
21
|
}
|
|
24
22
|
}
|
package/portal.mjs
CHANGED
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
import { makeRequest } from "./makerequest.mjs"
|
|
2
|
-
import
|
|
2
|
+
import logger from "./logger.mjs"
|
|
3
3
|
|
|
4
4
|
export class Portal{
|
|
5
5
|
|
|
6
|
-
constructor(url, username, password, expiration = 300, serverUrl = undefined)
|
|
6
|
+
constructor(url, username, password, expiration = 300, serverUrl = undefined, referer)
|
|
7
7
|
{
|
|
8
8
|
this.url = url;
|
|
9
9
|
this.username = username;
|
|
10
10
|
this.password = password;
|
|
11
11
|
this.expiration = expiration;
|
|
12
12
|
this.serverUrl = serverUrl;
|
|
13
|
+
this.referer = referer
|
|
13
14
|
}
|
|
14
15
|
|
|
15
16
|
|
|
@@ -25,7 +26,7 @@ export class Portal{
|
|
|
25
26
|
const postJson = {
|
|
26
27
|
username: self.username,
|
|
27
28
|
password: self.password,
|
|
28
|
-
referer:
|
|
29
|
+
referer: self.referer,
|
|
29
30
|
expiration: self.expiration,
|
|
30
31
|
f: "json"
|
|
31
32
|
}
|
|
@@ -71,7 +72,7 @@ export class Portal{
|
|
|
71
72
|
token: self.token
|
|
72
73
|
}
|
|
73
74
|
|
|
74
|
-
const serversUrl = self.url +
|
|
75
|
+
const serversUrl = self.url + `/sharing/rest/portals/self/servers?token=${self.token}`
|
|
75
76
|
logger.info( "About to query federated servers");
|
|
76
77
|
|
|
77
78
|
//query for federated servers.
|
package/utilitynetwork.node.mjs
CHANGED
|
@@ -8,9 +8,9 @@
|
|
|
8
8
|
/*
|
|
9
9
|
import fetch from "node-fetch"
|
|
10
10
|
*/
|
|
11
|
-
import
|
|
12
|
-
import
|
|
13
|
-
|
|
11
|
+
import logger from "./logger.mjs"
|
|
12
|
+
import { makeRequest } from "./makerequest.mjs"
|
|
13
|
+
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
export class UtilityNetwork {
|
|
@@ -122,6 +122,7 @@ export class UtilityNetwork {
|
|
|
122
122
|
|
|
123
123
|
}
|
|
124
124
|
|
|
125
|
+
/*
|
|
125
126
|
//a function that detects associations with from/to globalId that don't exist in the source tables.
|
|
126
127
|
//if one of the endpoints do not exists fails..
|
|
127
128
|
returnInvalidAssociations() {
|
|
@@ -178,7 +179,7 @@ export class UtilityNetwork {
|
|
|
178
179
|
|
|
179
180
|
|
|
180
181
|
}
|
|
181
|
-
|
|
182
|
+
*/
|
|
182
183
|
queryMoment(moment = ["definitionModification"]) {
|
|
183
184
|
|
|
184
185
|
let thisObj = this;
|
|
@@ -1086,6 +1087,7 @@ export class UtilityNetwork {
|
|
|
1086
1087
|
|
|
1087
1088
|
|
|
1088
1089
|
|
|
1090
|
+
/*
|
|
1089
1091
|
//Makes a request
|
|
1090
1092
|
function makeRequest (opts) {
|
|
1091
1093
|
|
|
@@ -1134,7 +1136,7 @@ export class UtilityNetwork {
|
|
|
1134
1136
|
|
|
1135
1137
|
});
|
|
1136
1138
|
}
|
|
1137
|
-
|
|
1139
|
+
*/
|
|
1138
1140
|
|
|
1139
1141
|
|
|
1140
1142
|
const v = (o, f, vIfNotFound=null) => {
|