un-cli 0.0.68 → 0.0.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adminlog.mjs +12 -3
- package/cmd.txt +1 -1
- package/index.html +677 -0
- package/index.mjs +205 -143
- package/logger.mjs +8 -1
- package/makerequest.mjs +14 -2
- package/package.json +2 -4
- package/portal.mjs +5 -4
- package/utilitynetwork.node.mjs +7 -5
package/adminlog.mjs
CHANGED
|
@@ -2,8 +2,6 @@
|
|
|
2
2
|
//Author : Hussein Nasser
|
|
3
3
|
//Date : Oct-13-2021
|
|
4
4
|
//Twitter: @hnasr
|
|
5
|
-
import fetch from "node-fetch";
|
|
6
|
-
import { logger } from "./logger.mjs"
|
|
7
5
|
|
|
8
6
|
export class AdminLog {
|
|
9
7
|
|
|
@@ -13,12 +11,22 @@ export class AdminLog {
|
|
|
13
11
|
this.token = token;
|
|
14
12
|
}
|
|
15
13
|
|
|
16
|
-
|
|
14
|
+
async getFetch() {
|
|
15
|
+
try {
|
|
16
|
+
const nodeFetch = await import ("node-fetch");
|
|
17
|
+
return nodeFetch.default;
|
|
18
|
+
}
|
|
19
|
+
catch(ex) {
|
|
20
|
+
return fetch;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
async query (codes, serviceName ="*", pageSize = 100000, startTime = null, endTime = null, logLevel = "DEBUG")
|
|
17
24
|
{
|
|
18
25
|
const url = this.adminServerUrl + "/logs/query?f=pjson"
|
|
19
26
|
const level = logLevel
|
|
20
27
|
const filterType="json"
|
|
21
28
|
const token = this.token
|
|
29
|
+
|
|
22
30
|
const filter = {
|
|
23
31
|
"codes": codes,
|
|
24
32
|
"services": serviceName
|
|
@@ -32,6 +40,7 @@ export class AdminLog {
|
|
|
32
40
|
queryLogUrl += `&endTime=${endTime}`
|
|
33
41
|
|
|
34
42
|
//logger.info(queryLogUrl);
|
|
43
|
+
const fetch = await this.getFetch()
|
|
35
44
|
return fetch(queryLogUrl);
|
|
36
45
|
}
|
|
37
46
|
|
package/cmd.txt
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
update subnetworks --all
|
|
1
|
+
update subnetworks --all --desc
|
|
2
2
|
exit
|
package/index.html
ADDED
|
@@ -0,0 +1,677 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8">
|
|
5
|
+
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
|
6
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
7
|
+
<title>UN Logs</title>
|
|
8
|
+
</head>
|
|
9
|
+
<body>
|
|
10
|
+
|
|
11
|
+
<table>
|
|
12
|
+
<tr><td>Service</td>
|
|
13
|
+
<td colspan =3 ><select id ='cmbService'>
|
|
14
|
+
<option>(All Services)</option>
|
|
15
|
+
</select>
|
|
16
|
+
</td>
|
|
17
|
+
</tr>
|
|
18
|
+
|
|
19
|
+
<tr>
|
|
20
|
+
<td >Age</td>
|
|
21
|
+
<td colspan =3><input type ='text' id = 'txtAge' value = '30' style="width:30px"> minutes</td>
|
|
22
|
+
</tr>
|
|
23
|
+
<tr colspan =4>
|
|
24
|
+
<td><button id = 'btnTraceLogs'>Trace Logs</button></td>
|
|
25
|
+
<td><button id = 'btnValidateLogs'>Validate Logs</button></td>
|
|
26
|
+
<td><button id = 'btnUpdateSubnetworkLogs'>Update Subnetwork Logs</button></td>
|
|
27
|
+
<td><button id = 'btnAR'>Attribute Rules Logs</button></td>
|
|
28
|
+
<td><button id = 'btnSQLLogs'>SQL Logs</button></td>
|
|
29
|
+
|
|
30
|
+
</tr>
|
|
31
|
+
|
|
32
|
+
</table>
|
|
33
|
+
|
|
34
|
+
<table id = 'tblResult' border =1 cellpadding = 4 cellspacing = 4 style="text-align:center">
|
|
35
|
+
|
|
36
|
+
</table>
|
|
37
|
+
<script type = 'module'>
|
|
38
|
+
import { Portal } from "./portal.mjs"
|
|
39
|
+
import { UtilityNetwork } from "./utilitynetwork.node.mjs"
|
|
40
|
+
import { AdminLog } from "./adminlog.mjs"
|
|
41
|
+
import logger from "./logger.mjs"
|
|
42
|
+
|
|
43
|
+
const parameters = {
|
|
44
|
+
"user": "unadmin",
|
|
45
|
+
"password": "unadmin.109",
|
|
46
|
+
"portal": "https://utilitynetwork.esri.com/portal",
|
|
47
|
+
"service": "RedTrolley_SQLServer",
|
|
48
|
+
"referer": "https://utilitynetwork.esri.com/log"
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
let portal;
|
|
52
|
+
let adminLog = null;
|
|
53
|
+
let cmbService
|
|
54
|
+
let token;
|
|
55
|
+
async function init() {
|
|
56
|
+
|
|
57
|
+
cmbService = document.getElementById("cmbService")
|
|
58
|
+
|
|
59
|
+
token = await getToken(parameters);
|
|
60
|
+
const services = await portal.services()
|
|
61
|
+
const featureServices = services.services.filter(s => s.type == "FeatureServer")
|
|
62
|
+
featureServices.forEach(s=> {
|
|
63
|
+
const o = document.createElement("option");
|
|
64
|
+
o.text = s.name;
|
|
65
|
+
cmbService.appendChild(o);
|
|
66
|
+
})
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
async function getToken(parameters) {
|
|
71
|
+
portal = new Portal(parameters.portal, parameters.user, parameters.password,300, parameters.server, parameters.referer)
|
|
72
|
+
console.log("About to connect..")
|
|
73
|
+
const token = await portal.connect()
|
|
74
|
+
console.log(`Token generanted successfully.`)
|
|
75
|
+
return token;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
async function loadTraceLogs () {
|
|
80
|
+
|
|
81
|
+
//build table
|
|
82
|
+
const tblResult = document.getElementById("tblResult");
|
|
83
|
+
//clear
|
|
84
|
+
while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
|
|
85
|
+
tblResult.appendChild(document.createTextNode("Loading...."));
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
parameters.service = cmbService.options[cmbService.selectedIndex].text
|
|
89
|
+
|
|
90
|
+
//create a new admin object (user might not be admin we won't use it until the user call log )
|
|
91
|
+
adminLog = new AdminLog(token, portal.serverUrl)
|
|
92
|
+
|
|
93
|
+
logger.info("Connected.")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
const topLogCount = 1000;
|
|
97
|
+
const pageSize = 10000
|
|
98
|
+
|
|
99
|
+
let mins = document.getElementById("txtAge").value //query logs for the last 30 minutes
|
|
100
|
+
|
|
101
|
+
console.log(`Querying trace logs for ${parameters.service} for the last ${mins} minutes ...`)
|
|
102
|
+
const startTime = Date.now() - mins*60*1000
|
|
103
|
+
const endTime = Date.now();
|
|
104
|
+
let services = [parameters.service+ ".MapServer"]
|
|
105
|
+
if (parameters.service == "(All Services)")
|
|
106
|
+
services = "*"
|
|
107
|
+
let result= await adminLog.query([102002], services, topLogCount, startTime ,endTime , "VERBOSE")
|
|
108
|
+
let jsonRes = await result.json()
|
|
109
|
+
let allMessages = [].concat(jsonRes.logMessages)
|
|
110
|
+
allMessages = allMessages.filter(m => m.message.indexOf("------ Trace Parameters ----") > -1)
|
|
111
|
+
while (jsonRes.hasMore)
|
|
112
|
+
{
|
|
113
|
+
//start paging
|
|
114
|
+
logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
|
|
115
|
+
result= await adminLog.query([102002], services, pageSize, jsonRes.endTime, null, "VERBOSE")
|
|
116
|
+
jsonRes = await result.json()
|
|
117
|
+
|
|
118
|
+
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("------ Trace Parameters ----") > -1))
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
await populateMessages(allMessages)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
async function loadValidateLogs () {
|
|
129
|
+
//build table
|
|
130
|
+
const tblResult = document.getElementById("tblResult");
|
|
131
|
+
//clear
|
|
132
|
+
while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
|
|
133
|
+
tblResult.appendChild(document.createTextNode("Loading...."));
|
|
134
|
+
|
|
135
|
+
parameters.service = cmbService.options[cmbService.selectedIndex].text
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
//create a new admin object (user might not be admin we won't use it until the user call log )
|
|
139
|
+
adminLog = new AdminLog(token, portal.serverUrl)
|
|
140
|
+
|
|
141
|
+
logger.info("Connected.")
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
const topLogCount = 1000;
|
|
145
|
+
const pageSize = 10000
|
|
146
|
+
|
|
147
|
+
let mins = document.getElementById("txtAge").value //query logs for the last 30 minutes
|
|
148
|
+
|
|
149
|
+
console.log(`Querying validate logs for ${parameters.service} for the last ${mins} minutes ...`)
|
|
150
|
+
const startTime = Date.now() - mins*60*1000
|
|
151
|
+
const endTime = Date.now();
|
|
152
|
+
let services = [parameters.service+ ".MapServer"]
|
|
153
|
+
if (parameters.service == "(All Services)")
|
|
154
|
+
services = "*"
|
|
155
|
+
let result= await adminLog.query([102003], services, topLogCount, startTime ,endTime , "VERBOSE")
|
|
156
|
+
let jsonRes = await result.json()
|
|
157
|
+
let allMessages = [].concat(jsonRes.logMessages)
|
|
158
|
+
allMessages = allMessages.filter(m => m.message.indexOf("-------- Environment ---") > -1 && m.message.indexOf("The network is built.") > -1 && m.methodName == 'BuildEngineLog')
|
|
159
|
+
while (jsonRes.hasMore)
|
|
160
|
+
{
|
|
161
|
+
//start paging
|
|
162
|
+
logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
|
|
163
|
+
result= await adminLog.query([102003], services, pageSize, jsonRes.endTime, null, "VERBOSE")
|
|
164
|
+
jsonRes = await result.json()
|
|
165
|
+
|
|
166
|
+
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("-------- Environment ---") > -1 && m.message.indexOf("------ Trace Parameters ----") == -1))
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
//validate logs missing elapsed populate it
|
|
173
|
+
allMessages = allMessages.map( m => {
|
|
174
|
+
try{
|
|
175
|
+
|
|
176
|
+
//The network is built. 0.093 seconds (4.982 total) - 29 MB memory
|
|
177
|
+
|
|
178
|
+
let re = /The network is built. [-+]?([0-9]*\.[0-9]+|[0-9]+) seconds \([-+]?([0-9]*\.[0-9]+|[0-9]+) total\)/;
|
|
179
|
+
let res = re.exec(m.message)
|
|
180
|
+
if (res && res.length > 1)
|
|
181
|
+
m.elapsed = res[2]
|
|
182
|
+
|
|
183
|
+
return m;
|
|
184
|
+
}
|
|
185
|
+
catch(ex){
|
|
186
|
+
return m;
|
|
187
|
+
}
|
|
188
|
+
})
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
await populateMessages(allMessages)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
async function loadUpdateSubnetworkLogs () {
|
|
198
|
+
//build table
|
|
199
|
+
const tblResult = document.getElementById("tblResult");
|
|
200
|
+
//clear
|
|
201
|
+
while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
|
|
202
|
+
tblResult.appendChild(document.createTextNode("Loading...."));
|
|
203
|
+
|
|
204
|
+
parameters.service = cmbService.options[cmbService.selectedIndex].text
|
|
205
|
+
//create a new admin object (user might not be admin we won't use it until the user call log )
|
|
206
|
+
adminLog = new AdminLog(token, portal.serverUrl)
|
|
207
|
+
|
|
208
|
+
logger.info("Connected.")
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
const topLogCount = 1000;
|
|
212
|
+
const pageSize = 10000
|
|
213
|
+
|
|
214
|
+
let mins = document.getElementById("txtAge").value //query logs for the last 30 minutes
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
console.log(`Querying subnetwork logs for ${parameters.service} for the last ${mins} minutes ...`)
|
|
218
|
+
const startTime = Date.now() - mins*60*1000
|
|
219
|
+
const endTime = Date.now();
|
|
220
|
+
let services = [parameters.service+ ".MapServer"]
|
|
221
|
+
if (parameters.service == "(All Services)")
|
|
222
|
+
services = "*"
|
|
223
|
+
let result= await adminLog.query([102003], services, topLogCount, startTime ,endTime , "VERBOSE")
|
|
224
|
+
let jsonRes = await result.json()
|
|
225
|
+
let allMessages = [].concat(jsonRes.logMessages)
|
|
226
|
+
allMessages = allMessages.filter(m => m.message.indexOf("---- Subnetwork Parameters ----") > -1)
|
|
227
|
+
while (jsonRes.hasMore)
|
|
228
|
+
{
|
|
229
|
+
//start paging
|
|
230
|
+
logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
|
|
231
|
+
result= await adminLog.query([102003], services, pageSize, jsonRes.endTime, null, "VERBOSE")
|
|
232
|
+
jsonRes = await result.json()
|
|
233
|
+
|
|
234
|
+
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("---- Subnetwork Parameters ----") > -1))
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
//update subnetwork missing elapsed populate it
|
|
238
|
+
allMessages = allMessages.map( m => {
|
|
239
|
+
try{
|
|
240
|
+
|
|
241
|
+
let re = /Total \([-+]?([0-9]*\.[0-9]+|[0-9]+) seconds\)/;
|
|
242
|
+
let res = re.exec(m.message)
|
|
243
|
+
if (res && res.length > 1)
|
|
244
|
+
m.elapsed = res[1]
|
|
245
|
+
|
|
246
|
+
re = /Total update subnetwork time \([-+]?([0-9]*\.[0-9]+|[0-9]+) seconds\)/;
|
|
247
|
+
res = re.exec(m.message)
|
|
248
|
+
if (res && res.length > 1)
|
|
249
|
+
m.elapsed = res[1]
|
|
250
|
+
|
|
251
|
+
return m;
|
|
252
|
+
}
|
|
253
|
+
catch(ex){
|
|
254
|
+
return m;
|
|
255
|
+
}
|
|
256
|
+
})
|
|
257
|
+
await populateMessages(allMessages)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
async function loadAttributeRules () {
|
|
265
|
+
|
|
266
|
+
//build table
|
|
267
|
+
const tblResult = document.getElementById("tblResult");
|
|
268
|
+
tblResult.appendChild(document.createTextNode("Loading...."));
|
|
269
|
+
|
|
270
|
+
let mins = document.getElementById("txtAge").value //query logs for the last 30 minutes
|
|
271
|
+
|
|
272
|
+
parameters.service = cmbService.options[cmbService.selectedIndex].text
|
|
273
|
+
//create a new admin object (user might not be admin we won't use it until the user call log )
|
|
274
|
+
adminLog = new AdminLog(token, portal.serverUrl)
|
|
275
|
+
|
|
276
|
+
let showMaxGuid = true
|
|
277
|
+
let showMinGuid = false
|
|
278
|
+
const startTime = Date.now() - mins*60*1000
|
|
279
|
+
const endTime = Date.now();
|
|
280
|
+
|
|
281
|
+
let services = [parameters.service+ ".MapServer"]
|
|
282
|
+
if (parameters.service == "(All Services)")
|
|
283
|
+
services = "*"
|
|
284
|
+
|
|
285
|
+
const pageSize = 10000 //maximum messages per page
|
|
286
|
+
logger.info(`Querying attribute rules logs for ${parameters.service} in the past ${mins} minutes...`)
|
|
287
|
+
let result= await adminLog.query([102003], services, pageSize, startTime ,endTime , "DEBUG")
|
|
288
|
+
let jsonRes = await result.json()
|
|
289
|
+
let allMessages = [].concat(jsonRes.logMessages)
|
|
290
|
+
|
|
291
|
+
while (jsonRes.hasMore && jsonRes.endTime > startTime)
|
|
292
|
+
{
|
|
293
|
+
//start paging
|
|
294
|
+
logger.info(`Aggregating messages... total so far ${allMessages.length} debug entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
result= await adminLog.query([102003], services, pageSize, jsonRes.endTime )
|
|
298
|
+
jsonRes = await result.json()
|
|
299
|
+
allMessages = allMessages.concat(jsonRes.logMessages)
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
const arMessages = allMessages
|
|
303
|
+
.filter(m => m.message.indexOf("Attribute rule execution complete:") > -1)
|
|
304
|
+
.map (m => JSON.parse(m.message.replace("Attribute rule execution complete:", "")))
|
|
305
|
+
.map( m => {
|
|
306
|
+
m["Elapsed Time (ms)"] = Math.round(m["Elapsed Time"]*1000000)/1000
|
|
307
|
+
// m["Arcade Evaluation Time:"] = Math.round(m["Arcade Evaluation Time:"]*1000,6)
|
|
308
|
+
//m.ArcadeTime = m["Arcade Evaluation Time:"]
|
|
309
|
+
|
|
310
|
+
delete m["Arcade Evaluation Time:"];
|
|
311
|
+
delete m["Elapsed Time"];
|
|
312
|
+
//delete m ['GlobalID'];
|
|
313
|
+
return m
|
|
314
|
+
})
|
|
315
|
+
.sort( (m1, m2) => m2["Elapsed Time (ms)"]- m1["Elapsed Time (ms)"])
|
|
316
|
+
.reduce( ( prev, cur ) => {
|
|
317
|
+
if (prev [cur["Rule name"]] === undefined)
|
|
318
|
+
{
|
|
319
|
+
prev [cur["Rule name"]] = {
|
|
320
|
+
"totalTime": 0,
|
|
321
|
+
"occurrence": 0,
|
|
322
|
+
"minTime": Number.MAX_SAFE_INTEGER,
|
|
323
|
+
"maxTime": -1,
|
|
324
|
+
"avgTime": 0,
|
|
325
|
+
"maxGuid": null,
|
|
326
|
+
"minGuid": null
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
prev [cur["Rule name"]].totalTime = prev [cur["Rule name"]].totalTime + cur["Elapsed Time (ms)"]
|
|
331
|
+
|
|
332
|
+
if (cur["Elapsed Time (ms)"] < prev [cur["Rule name"]].minTime ) {
|
|
333
|
+
prev [cur["Rule name"]].minTime = cur["Elapsed Time (ms)"];
|
|
334
|
+
prev [cur["Rule name"]].minGuid = cur["GlobalID"];
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
if (cur["Elapsed Time (ms)"] > prev [cur["Rule name"]].maxTime ){
|
|
338
|
+
prev [cur["Rule name"]].maxTime = cur["Elapsed Time (ms)"];
|
|
339
|
+
prev [cur["Rule name"]].maxGuid = cur["GlobalID"];
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
prev [cur["Rule name"]].occurrence++
|
|
343
|
+
|
|
344
|
+
prev [cur["Rule name"]].avgTime = prev [cur["Rule name"]].totalTime / prev [cur["Rule name"]].occurrence
|
|
345
|
+
|
|
346
|
+
return prev
|
|
347
|
+
}, {})
|
|
348
|
+
|
|
349
|
+
const rules = Object.keys(arMessages)
|
|
350
|
+
.map(a => {
|
|
351
|
+
const rule = {}
|
|
352
|
+
rule["Attribute Rule"] = a;
|
|
353
|
+
rule["Total Cost (ms)"] = parseFloat(arMessages[a].totalTime.toFixed(2))
|
|
354
|
+
if (!showMinGuid && !showMaxGuid) rule["Average Cost (ms)"] = parseFloat(arMessages[a].avgTime.toFixed(2))
|
|
355
|
+
rule["Max execution time (ms)"] = parseFloat(arMessages[a].maxTime.toFixed(2))
|
|
356
|
+
if (showMaxGuid) rule["Max GUID"] = arMessages[a].maxGuid
|
|
357
|
+
rule["Min execution time (ms)"] = parseFloat(arMessages[a].minTime.toFixed(2))
|
|
358
|
+
if (showMinGuid) rule["Min GUID"] = arMessages[a].minGuid
|
|
359
|
+
if (!showMinGuid && !showMaxGuid) rule["Occurrence"] = arMessages[a].occurrence;
|
|
360
|
+
return rule;
|
|
361
|
+
})
|
|
362
|
+
.sort( (m1, m2) => m2["Total Cost (ms)"] -m1["Total Cost (ms)"])
|
|
363
|
+
console.table(rules)
|
|
364
|
+
|
|
365
|
+
/*
|
|
366
|
+
Attribute Rule
|
|
367
|
+
│ Total Cost (ms)
|
|
368
|
+
│ Max execution time (ms)
|
|
369
|
+
│ Max GUID
|
|
370
|
+
│ Min execution time (m
|
|
371
|
+
|
|
372
|
+
*/
|
|
373
|
+
|
|
374
|
+
const header = document.createElement("tr");
|
|
375
|
+
const cAttributeRule = document.createElement("th");
|
|
376
|
+
cAttributeRule.textContent = "Attribute Rule"
|
|
377
|
+
|
|
378
|
+
const cTotalCost = document.createElement("th");
|
|
379
|
+
cTotalCost.textContent = "Total Cost (ms)"
|
|
380
|
+
|
|
381
|
+
const cMaxExecutionTime = document.createElement("th");
|
|
382
|
+
cMaxExecutionTime.textContent = "Max execution time (ms)"
|
|
383
|
+
|
|
384
|
+
const cMaxGuid = document.createElement("th");
|
|
385
|
+
cMaxGuid.textContent = "Max GUID"
|
|
386
|
+
|
|
387
|
+
const cMinExecutionTime = document.createElement("th");
|
|
388
|
+
cMinExecutionTime.textContent = "Min execution time (ms)"
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
header.appendChild (cAttributeRule)
|
|
392
|
+
header.appendChild (cTotalCost)
|
|
393
|
+
header.appendChild (cMaxExecutionTime)
|
|
394
|
+
header.appendChild (cMaxGuid)
|
|
395
|
+
header.appendChild (cMinExecutionTime)
|
|
396
|
+
|
|
397
|
+
tblResult.appendChild(header)
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
rules.forEach(rule =>
|
|
401
|
+
{
|
|
402
|
+
|
|
403
|
+
const logRow = document.createElement("tr");
|
|
404
|
+
const cAttributeRule = document.createElement("td");
|
|
405
|
+
cAttributeRule.textContent = rule["Attribute Rule"]
|
|
406
|
+
|
|
407
|
+
const cTotalCost = document.createElement("td");
|
|
408
|
+
cTotalCost.textContent = rule["Total Cost (ms)"]
|
|
409
|
+
|
|
410
|
+
const cMaxExecutionTime = document.createElement("td");
|
|
411
|
+
cMaxExecutionTime.textContent = rule["Max execution time (ms)"]
|
|
412
|
+
|
|
413
|
+
const cMaxGuid = document.createElement("td");
|
|
414
|
+
cMaxGuid.textContent = rule["Max GUID"]
|
|
415
|
+
|
|
416
|
+
const cMinExecutionTime = document.createElement("td");
|
|
417
|
+
cMinExecutionTime.textContent = rule["Min execution time (ms)"]
|
|
418
|
+
|
|
419
|
+
logRow.appendChild (cAttributeRule)
|
|
420
|
+
logRow.appendChild (cTotalCost)
|
|
421
|
+
logRow.appendChild (cMaxExecutionTime)
|
|
422
|
+
logRow.appendChild (cMaxGuid)
|
|
423
|
+
logRow.appendChild (cMinExecutionTime)
|
|
424
|
+
tblResult.appendChild(logRow)
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
})
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
const totalARExecution = rules.reduce( (prev, cur) => prev + cur["Total Cost (ms)"], 0)
|
|
431
|
+
console.log(`Total time spend executing attribute rules (${Math.round(totalARExecution)} ms) (${Math.round(totalARExecution/1000)} s) (${Math.round(totalARExecution/(1000*60))} m)`)
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
}
|
|
436
|
+
async function loadSQLLogs () {
|
|
437
|
+
|
|
438
|
+
//build table
|
|
439
|
+
const tblResult = document.getElementById("tblResult");
|
|
440
|
+
//clear
|
|
441
|
+
while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
|
|
442
|
+
tblResult.appendChild(document.createTextNode("Loading...."));
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
parameters.service = cmbService.options[cmbService.selectedIndex].text
|
|
446
|
+
|
|
447
|
+
//create a new admin object (user might not be admin we won't use it until the user call log )
|
|
448
|
+
adminLog = new AdminLog(token, portal.serverUrl)
|
|
449
|
+
|
|
450
|
+
logger.info("Connected.")
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
const topLogCount = 1000;
|
|
454
|
+
const pageSize = 10000
|
|
455
|
+
|
|
456
|
+
let mins = document.getElementById("txtAge").value //query logs for the last 30 minutes
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
console.log(`Querying cursor sql logs for ${parameters.service} for the last ${mins} minutes ...`)
|
|
460
|
+
const startTime = Date.now() - mins*60*1000
|
|
461
|
+
const endTime = Date.now();
|
|
462
|
+
let services = [parameters.service+ ".MapServer"]
|
|
463
|
+
if (parameters.service == "(All Services)")
|
|
464
|
+
services = "*"
|
|
465
|
+
let result= await adminLog.query([102023], services, topLogCount, startTime ,endTime , "DEBUG")
|
|
466
|
+
let jsonRes = await result.json()
|
|
467
|
+
let allMessages = [].concat(jsonRes.logMessages)
|
|
468
|
+
allMessages = allMessages.filter(m => m.message.indexOf("EndCursor;") > -1)
|
|
469
|
+
while (jsonRes.hasMore)
|
|
470
|
+
{
|
|
471
|
+
//start paging
|
|
472
|
+
logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
|
|
473
|
+
result= await adminLog.query([102023], services, pageSize, jsonRes.endTime, null, "DEBUG")
|
|
474
|
+
jsonRes = await result.json()
|
|
475
|
+
|
|
476
|
+
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("EndCursor;") > -1))
|
|
477
|
+
}
|
|
478
|
+
console.log ("Filtering messages...")
|
|
479
|
+
|
|
480
|
+
allMessages = allMessages
|
|
481
|
+
.map( m=> {
|
|
482
|
+
m.dataAccessElapsed = parseFloat(m.message.split(";")[1].split(" ")[1])
|
|
483
|
+
m.executeQueryElapsed = parseFloat(m.message.split(";")[2].split(" ")[1])
|
|
484
|
+
m.totalExecutionElapsed = m.dataAccessElapsed + m.executeQueryElapsed
|
|
485
|
+
m.elapsed = parseFloat(m.elapsed); return m;
|
|
486
|
+
|
|
487
|
+
})
|
|
488
|
+
.sort( (m1,m2) => m2.totalExecutionElapsed - m1.totalExecutionElapsed)
|
|
489
|
+
.slice(0, 100) ;//first 100
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
console.log("-----Top 10 SQL----")
|
|
493
|
+
while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
|
|
494
|
+
|
|
495
|
+
let i =0;
|
|
496
|
+
|
|
497
|
+
const header = document.createElement("tr");
|
|
498
|
+
const cService = document.createElement("th");
|
|
499
|
+
cService.textContent = "Source"
|
|
500
|
+
const cTime = document.createElement("th");
|
|
501
|
+
cTime.textContent = "Time"
|
|
502
|
+
const cUser = document.createElement("th");
|
|
503
|
+
cUser.textContent = "User"
|
|
504
|
+
const cMethod = document.createElement("th");
|
|
505
|
+
cMethod.textContent = "Method"
|
|
506
|
+
const cQueryTime = document.createElement("th");
|
|
507
|
+
cQueryTime.textContent = "Query Time (ms)"
|
|
508
|
+
cQueryTime.title = "includes search + data access nextRow"
|
|
509
|
+
const cTotalTime = document.createElement("th");
|
|
510
|
+
cTotalTime.title = "Total time the cursor was opened"
|
|
511
|
+
cTotalTime.textContent = "Total Time (ms)"
|
|
512
|
+
const cLog = document.createElement("th");
|
|
513
|
+
cLog.textContent = "Full SQL"
|
|
514
|
+
header.appendChild (cService)
|
|
515
|
+
header.appendChild (cTime)
|
|
516
|
+
header.appendChild (cUser)
|
|
517
|
+
header.appendChild (cMethod)
|
|
518
|
+
header.appendChild (cQueryTime)
|
|
519
|
+
header.appendChild (cTotalTime)
|
|
520
|
+
header.appendChild (cLog)
|
|
521
|
+
tblResult.appendChild(header)
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
allMessages= allMessages.forEach(m =>
|
|
525
|
+
{
|
|
526
|
+
|
|
527
|
+
const logRow = document.createElement("tr");
|
|
528
|
+
const x = m.message.split(";")
|
|
529
|
+
x.shift()
|
|
530
|
+
console.log(`id: ${i++}`)
|
|
531
|
+
console.log(`\tAt: ${new Date(m.time)} (${m.time})`)
|
|
532
|
+
console.log(`\tUser: ${m.user}`)
|
|
533
|
+
console.log(`\tTotal Time: ${numberWithCommas(Math.round(m.elapsed*1000))} ms (Total time the cursor was opened)`)
|
|
534
|
+
console.log(`\tQuery Time: ${numberWithCommas(m.totalExecutionElapsed)} ms (includes search + data access nextRow)`)
|
|
535
|
+
console.log(`\tQuery:`)
|
|
536
|
+
x.forEach(a => console.log(`\t${a}`))
|
|
537
|
+
console.log(`\n`)
|
|
538
|
+
|
|
539
|
+
const cService = document.createElement("th");
|
|
540
|
+
cService.textContent = m.source.replace(".MapServer", "")
|
|
541
|
+
|
|
542
|
+
const cTime = document.createElement("td");
|
|
543
|
+
cTime.textContent = new Date(m.time).toLocaleString()
|
|
544
|
+
|
|
545
|
+
const cUser = document.createElement("td");
|
|
546
|
+
cUser.textContent = m.user
|
|
547
|
+
const cMethod = document.createElement("td");
|
|
548
|
+
cMethod.textContent = "Cursor"
|
|
549
|
+
const cQueryTime = document.createElement("td");
|
|
550
|
+
cQueryTime.textContent = numberWithCommas(m.totalExecutionElapsed)
|
|
551
|
+
const cTotalTime = document.createElement("td");
|
|
552
|
+
cTotalTime.textContent = numberWithCommas(Math.round(m.elapsed*1000))
|
|
553
|
+
const cLog = document.createElement("td");
|
|
554
|
+
cLog.textContent = m.message.substr( m.message.indexOf("SQL: ") + 5, 50) + "..."
|
|
555
|
+
cLog.fullLog = m.message;
|
|
556
|
+
cLog.title = m.message + "\nClick to copy"
|
|
557
|
+
cLog.addEventListener("click", e=> {navigator.clipboard.writeText(e.target.fullLog); alert("Copied to clipboard")})
|
|
558
|
+
logRow.appendChild (cService)
|
|
559
|
+
logRow.appendChild (cTime)
|
|
560
|
+
logRow.appendChild (cUser)
|
|
561
|
+
logRow.appendChild (cMethod)
|
|
562
|
+
logRow.appendChild (cQueryTime)
|
|
563
|
+
logRow.appendChild (cTotalTime)
|
|
564
|
+
logRow.appendChild (cLog)
|
|
565
|
+
tblResult.appendChild(logRow)
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
})
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
//await populateMessages(allMessages)
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
function numberWithCommas(x) {
|
|
583
|
+
// return x.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
|
|
584
|
+
return x.toLocaleString()
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
init();
|
|
588
|
+
|
|
589
|
+
const btnTraceLogs = document.getElementById("btnTraceLogs");
|
|
590
|
+
btnTraceLogs.addEventListener("click", e => loadTraceLogs())
|
|
591
|
+
|
|
592
|
+
const btnValidateLogs = document.getElementById("btnValidateLogs");
|
|
593
|
+
btnValidateLogs.addEventListener("click", e => loadValidateLogs())
|
|
594
|
+
|
|
595
|
+
const btnUpdateSubnetworkLogs = document.getElementById("btnUpdateSubnetworkLogs");
|
|
596
|
+
btnUpdateSubnetworkLogs.addEventListener("click", e => loadUpdateSubnetworkLogs())
|
|
597
|
+
|
|
598
|
+
const btnSQLLogs = document.getElementById("btnSQLLogs");
|
|
599
|
+
btnSQLLogs.addEventListener("click", e => loadSQLLogs())
|
|
600
|
+
|
|
601
|
+
const btnAR = document.getElementById("btnAR");
|
|
602
|
+
btnAR.addEventListener("click", e => loadAttributeRules())
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
async function populateMessages(allTheMessages) {
|
|
607
|
+
|
|
608
|
+
//sort messages
|
|
609
|
+
|
|
610
|
+
const allMessages = allTheMessages.map(m => {
|
|
611
|
+
const newMessage = Object.assign({}, m);
|
|
612
|
+
delete newMessage.machine;
|
|
613
|
+
delete newMessage.type;
|
|
614
|
+
delete newMessage.code;
|
|
615
|
+
delete newMessage.requestID;
|
|
616
|
+
delete newMessage.process;
|
|
617
|
+
delete newMessage.thread;
|
|
618
|
+
newMessage.elapsedms = parseInt (parseFloat(newMessage.elapsed) * 1000)
|
|
619
|
+
newMessage.time = new Date(newMessage.time).toLocaleString()
|
|
620
|
+
delete newMessage.elapsed
|
|
621
|
+
return newMessage;
|
|
622
|
+
})
|
|
623
|
+
.sort( (m1,m2) => m2.elapsedms - m1.elapsedms)
|
|
624
|
+
|
|
625
|
+
//build table
|
|
626
|
+
const tblResult = document.getElementById("tblResult");
|
|
627
|
+
//clear
|
|
628
|
+
while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
|
|
629
|
+
|
|
630
|
+
const header = document.createElement("tr");
|
|
631
|
+
const cService = document.createElement("th");
|
|
632
|
+
cService.textContent = "Source"
|
|
633
|
+
const cTime = document.createElement("th");
|
|
634
|
+
cTime.textContent = "Time"
|
|
635
|
+
const cUser = document.createElement("th");
|
|
636
|
+
cUser.textContent = "User"
|
|
637
|
+
const cMethod = document.createElement("th");
|
|
638
|
+
cMethod.textContent = "Method"
|
|
639
|
+
const cElapsedMS = document.createElement("th");
|
|
640
|
+
cElapsedMS.textContent = "Elapsed (ms)"
|
|
641
|
+
const cLog = document.createElement("th");
|
|
642
|
+
cLog.textContent = "Full Log"
|
|
643
|
+
header.appendChild (cService)
|
|
644
|
+
header.appendChild (cTime)
|
|
645
|
+
header.appendChild (cUser)
|
|
646
|
+
header.appendChild (cMethod)
|
|
647
|
+
header.appendChild (cElapsedMS)
|
|
648
|
+
header.appendChild (cLog)
|
|
649
|
+
tblResult.appendChild(header)
|
|
650
|
+
allMessages.forEach(m => {
|
|
651
|
+
const logRow = document.createElement("tr");
|
|
652
|
+
const cService = document.createElement("td");
|
|
653
|
+
cService.textContent = m.source.replace(".MapServer", "")
|
|
654
|
+
const cTime = document.createElement("td");
|
|
655
|
+
cTime.textContent = m.time
|
|
656
|
+
const cUser = document.createElement("td");
|
|
657
|
+
cUser.textContent = m.user
|
|
658
|
+
const cMethod = document.createElement("td");
|
|
659
|
+
cMethod.textContent = m.methodName
|
|
660
|
+
const cElapsedMS = document.createElement("td");
|
|
661
|
+
cElapsedMS.textContent = m.elapsedms
|
|
662
|
+
const cLog = document.createElement("td");
|
|
663
|
+
cLog.textContent = "..."
|
|
664
|
+
cLog.fullLog = m.message;
|
|
665
|
+
cLog.addEventListener("click", e=> {navigator.clipboard.writeText(e.target.fullLog); alert("Copied to clipboard")})
|
|
666
|
+
logRow.appendChild (cService)
|
|
667
|
+
logRow.appendChild (cTime)
|
|
668
|
+
logRow.appendChild (cUser)
|
|
669
|
+
logRow.appendChild (cMethod)
|
|
670
|
+
logRow.appendChild (cElapsedMS)
|
|
671
|
+
logRow.appendChild (cLog)
|
|
672
|
+
tblResult.appendChild(logRow)
|
|
673
|
+
})
|
|
674
|
+
}
|
|
675
|
+
</script>
|
|
676
|
+
</body>
|
|
677
|
+
</html>
|
package/index.mjs
CHANGED
|
@@ -4,10 +4,10 @@ import { Portal } from "./portal.mjs"
|
|
|
4
4
|
import { UtilityNetwork } from "./utilitynetwork.node.mjs"
|
|
5
5
|
import { AdminLog } from "./adminlog.mjs"
|
|
6
6
|
|
|
7
|
-
import
|
|
7
|
+
import logger from "./logger.mjs"
|
|
8
8
|
import fetch from "node-fetch"
|
|
9
9
|
//update version
|
|
10
|
-
let version = "0.0.
|
|
10
|
+
let version = "0.0.71";
|
|
11
11
|
const GENERATE_TOKEN_TIME_MIN = 30;
|
|
12
12
|
|
|
13
13
|
let rl = null;
|
|
@@ -394,8 +394,7 @@ const inputs = {
|
|
|
394
394
|
console.log("Updating Subnetwork " + v(f.attributes,"subnetworkName"));
|
|
395
395
|
|
|
396
396
|
const fromDate = new Date();
|
|
397
|
-
|
|
398
|
-
|
|
397
|
+
|
|
399
398
|
const subnetworkResult = await un.updateSubnetworks(v(f.attributes,"domainNetworkName"), v(f.attributes,"tierName"), v(f.attributes,"subnetworkName"),false);
|
|
400
399
|
|
|
401
400
|
//code
|
|
@@ -409,27 +408,36 @@ const inputs = {
|
|
|
409
408
|
}
|
|
410
409
|
},
|
|
411
410
|
|
|
412
|
-
"^update subnetworks --all
|
|
413
|
-
|
|
414
|
-
let subnetworks
|
|
415
|
-
|
|
411
|
+
"^update subnetworks --all" : async input => {
|
|
412
|
+
|
|
413
|
+
let subnetworks;
|
|
414
|
+
do {
|
|
416
415
|
|
|
416
|
+
let sort = "asc";
|
|
417
|
+
if (input.indexOf("--desc") > 0) sort = "desc"
|
|
417
418
|
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
console.log(
|
|
421
|
-
|
|
422
|
-
const fromDate = new Date();
|
|
423
|
-
|
|
419
|
+
console.log("Querying all subnetworks that are dirty.");
|
|
420
|
+
subnetworks = await un.queryDistinct(500002, "domainnetworkname,tiername,subnetworkname", "isdirty=1", `domainnetworkname ${sort},tiername ${sort},subnetworkname ${sort}`);
|
|
421
|
+
console.log(`Discovered ${subnetworks.features.length} dirty subnetworks.`);
|
|
424
422
|
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
423
|
+
for (let i = 0; i < subnetworks.features.length; i++) {
|
|
424
|
+
const f = subnetworks.features[i]
|
|
425
|
+
console.log("Updating Subnetwork " + v(f.attributes,"subnetworkName"));
|
|
426
|
+
|
|
427
|
+
const fromDate = new Date();
|
|
428
|
+
|
|
429
|
+
const subnetworkResult = await un.updateSubnetworks(v(f.attributes,"domainNetworkName"), v(f.attributes,"tierName"), v(f.attributes,"subnetworkName"),false);
|
|
430
|
+
|
|
431
|
+
const toDate = new Date();
|
|
432
|
+
const timeEnable = toDate.getTime() - fromDate.getTime();
|
|
433
|
+
subnetworkResult.duration = numberWithCommas(Math.round(timeEnable)) + " ms"
|
|
434
|
+
|
|
435
|
+
console.log(`Result ${JSON.stringify(subnetworkResult)}`)
|
|
436
|
+
}
|
|
430
437
|
|
|
431
|
-
|
|
432
|
-
|
|
438
|
+
}
|
|
439
|
+
while (subnetworks.features.length > 0)
|
|
440
|
+
|
|
433
441
|
},
|
|
434
442
|
"^update subnetworks --all --async$" : async () => {
|
|
435
443
|
console.log("Querying all subnetworks that are dirty.");
|
|
@@ -443,8 +451,8 @@ const inputs = {
|
|
|
443
451
|
}
|
|
444
452
|
},
|
|
445
453
|
"^export subnetworks --all --folder .*$|^export subnetworks --all$" : async input => {
|
|
446
|
-
|
|
447
|
-
|
|
454
|
+
|
|
455
|
+
let subnetworks
|
|
448
456
|
//create folder
|
|
449
457
|
const file = input.match(/--folder .*/gm)
|
|
450
458
|
let inputDir = "Exported"
|
|
@@ -452,45 +460,49 @@ const inputs = {
|
|
|
452
460
|
inputDir = file[0].replace("--folder ", "")
|
|
453
461
|
//create directory if doesn't exists
|
|
454
462
|
if (!fs.existsSync(inputDir)) fs.mkdirSync(inputDir)
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
463
|
+
|
|
464
|
+
do {
|
|
465
|
+
|
|
466
|
+
console.log("Querying all subnetworks that are clean.");
|
|
467
|
+
subnetworks = await un.queryDistinct(500002, "domainnetworkname,tiername,subnetworkname", "isdirty=0","domainnetworkname,tiername,subnetworkname");
|
|
468
|
+
console.log(`Discovered ${subnetworks.features.length} subnetworks that can be exported.`);
|
|
469
|
+
for (let i = 0; i < subnetworks.features.length; i++) {
|
|
470
|
+
const f = subnetworks.features[i]
|
|
471
|
+
const subnetworkName = v(f.attributes,"subnetworkName")
|
|
472
|
+
console.log("Exporting subnetworks " + v(f.attributes,"subnetworkName"));
|
|
473
|
+
|
|
474
|
+
const fromDate = new Date();
|
|
475
|
+
|
|
476
|
+
const subnetworkResult = await un.exportSubnetworks(v(f.attributes,"domainNetworkName"), v(f.attributes,"tierName"), v(f.attributes,"subnetworkName"),false);
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
//code
|
|
480
|
+
|
|
481
|
+
const toDate = new Date();
|
|
482
|
+
const timeEnable = toDate.getTime() - fromDate.getTime();
|
|
483
|
+
subnetworkResult.duration = numberWithCommas(timeEnable) + " ms"
|
|
484
|
+
//if undefined exit
|
|
485
|
+
if (!subnetworkResult.url)
|
|
486
|
+
{
|
|
487
|
+
console.log("Export subnetwork failed " + JSON.stringify(subnetworkResult))
|
|
488
|
+
continue;
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
//fetch the json and write it to disk
|
|
492
|
+
const subContent = await fetch(subnetworkResult.url);
|
|
493
|
+
//check if the response is 200 only then attempt to parse to json
|
|
494
|
+
//although the response is json, its easier to treat it as text (handle error cases) since we will only write it to disk.
|
|
495
|
+
// if we want to do something with the response then make it json
|
|
496
|
+
const jsonExport = await subContent.text();
|
|
497
|
+
fs.writeFileSync(`${inputDir}/${subnetworkName}.json`, jsonExport)
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
console.log(`Result ${JSON.stringify(subnetworkResult)} written to file ${process.cwd()}/${inputDir}/${subnetworkName}.json`)
|
|
501
|
+
|
|
480
502
|
}
|
|
481
|
-
|
|
482
|
-
//fetch the json and write it to disk
|
|
483
|
-
const subContent = await fetch(subnetworkResult.url);
|
|
484
|
-
//check if the response is 200 only then attempt to parse to json
|
|
485
|
-
//although the response is json, its easier to treat it as text (handle error cases) since we will only write it to disk.
|
|
486
|
-
// if we want to do something with the response then make it json
|
|
487
|
-
const jsonExport = await subContent.text();
|
|
488
|
-
fs.writeFileSync(`${inputDir}/${subnetworkName}.json`, jsonExport)
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
console.log(`Result ${JSON.stringify(subnetworkResult)} written to file ${process.cwd()}/${inputDir}/${subnetworkName}.json`)
|
|
492
|
-
|
|
493
503
|
}
|
|
504
|
+
while (subnetworks.features.length > 0)
|
|
505
|
+
|
|
494
506
|
},
|
|
495
507
|
|
|
496
508
|
|
|
@@ -768,16 +780,30 @@ const inputs = {
|
|
|
768
780
|
|
|
769
781
|
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("------ Trace Parameters ----") > -1))
|
|
770
782
|
}
|
|
771
|
-
|
|
783
|
+
|
|
784
|
+
allMessages = allMessages.map(m => {
|
|
772
785
|
const newMessage = Object.assign({}, m);
|
|
773
|
-
delete newMessage.message
|
|
774
786
|
delete newMessage.source;
|
|
775
787
|
delete newMessage.machine;
|
|
776
788
|
delete newMessage.type;
|
|
789
|
+
delete newMessage.code;
|
|
777
790
|
delete newMessage.requestID;
|
|
791
|
+
delete newMessage.process;
|
|
778
792
|
delete newMessage.thread;
|
|
779
|
-
|
|
793
|
+
newMessage.elapsedms = parseInt (parseFloat(newMessage.elapsed) * 1000)
|
|
794
|
+
newMessage.time = new Date(newMessage.time).toLocaleString()
|
|
795
|
+
delete newMessage.elapsed
|
|
796
|
+
return newMessage;
|
|
797
|
+
})
|
|
798
|
+
.sort( (m1,m2) => m2.elapsedms - m1.elapsedms)
|
|
799
|
+
|
|
800
|
+
const summaryMessages = allMessages.map(m => {const newM = Object.assign({}, m); delete newM.message; return newM})
|
|
801
|
+
console.table(summaryMessages)
|
|
780
802
|
|
|
803
|
+
allMessages.forEach(m => {
|
|
804
|
+
const newMessage = Object.assign({}, m);
|
|
805
|
+
delete newMessage.message
|
|
806
|
+
|
|
781
807
|
console.table([newMessage])
|
|
782
808
|
console.log(m.message)
|
|
783
809
|
})
|
|
@@ -796,37 +822,76 @@ const inputs = {
|
|
|
796
822
|
if (inputParam != null && inputParam.length > 0)
|
|
797
823
|
mins = inputParam[0].replace("--age ", "")
|
|
798
824
|
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
825
|
+
|
|
826
|
+
console.log(`Querying validate logs for ${parameters.service} for the last ${mins} minutes ...`)
|
|
827
|
+
const startTime = Date.now() - mins*60*1000
|
|
828
|
+
const endTime = Date.now();
|
|
829
|
+
let result= await adminLog.query([102003], [parameters.service+ ".MapServer"], topLogCount, startTime ,endTime , "VERBOSE")
|
|
830
|
+
let jsonRes = await result.json()
|
|
831
|
+
let allMessages = [].concat(jsonRes.logMessages)
|
|
832
|
+
allMessages = allMessages.filter(m => m.message.indexOf("-------- Environment ---") > -1 && m.message.indexOf("The network is built.") > -1 && m.methodName == 'BuildEngineLog')
|
|
833
|
+
while (jsonRes.hasMore)
|
|
834
|
+
{
|
|
835
|
+
//start paging
|
|
836
|
+
logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
|
|
837
|
+
let services = [parameters.service + ".MapServer"]
|
|
838
|
+
result= await adminLog.query([102003], services, pageSize, jsonRes.endTime, null, "VERBOSE")
|
|
839
|
+
jsonRes = await result.json()
|
|
840
|
+
|
|
841
|
+
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("-------- Environment ---") > -1 && m.message.indexOf("------ Trace Parameters ----") == -1))
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
|
|
845
|
+
|
|
846
|
+
|
|
847
|
+
//validate logs missing elapsed populate it
|
|
848
|
+
allMessages = allMessages.map( m => {
|
|
849
|
+
try{
|
|
850
|
+
|
|
851
|
+
//The network is built. 0.093 seconds (4.982 total) - 29 MB memory
|
|
852
|
+
|
|
853
|
+
let re = /The network is built. [-+]?([0-9]*\.[0-9]+|[0-9]+) seconds \([-+]?([0-9]*\.[0-9]+|[0-9]+) total\)/;
|
|
854
|
+
let res = re.exec(m.message)
|
|
855
|
+
if (res && res.length > 1)
|
|
856
|
+
m.elapsed = res[2]
|
|
857
|
+
|
|
858
|
+
return m;
|
|
859
|
+
}
|
|
860
|
+
catch(ex){
|
|
861
|
+
return m;
|
|
862
|
+
}
|
|
863
|
+
})
|
|
815
864
|
|
|
816
865
|
|
|
817
|
-
|
|
866
|
+
|
|
867
|
+
allMessages = allMessages.map(m => {
|
|
818
868
|
const newMessage = Object.assign({}, m);
|
|
819
|
-
delete newMessage.message
|
|
820
869
|
delete newMessage.source;
|
|
821
870
|
delete newMessage.machine;
|
|
822
871
|
delete newMessage.type;
|
|
872
|
+
delete newMessage.code;
|
|
823
873
|
delete newMessage.requestID;
|
|
874
|
+
delete newMessage.process;
|
|
824
875
|
delete newMessage.thread;
|
|
825
|
-
|
|
876
|
+
newMessage.elapsedms = parseInt (parseFloat(newMessage.elapsed) * 1000)
|
|
877
|
+
newMessage.time = new Date(newMessage.time).toLocaleString()
|
|
878
|
+
delete newMessage.elapsed
|
|
879
|
+
return newMessage;
|
|
880
|
+
})
|
|
881
|
+
.sort( (m1,m2) => m2.elapsedms - m1.elapsedms)
|
|
826
882
|
|
|
883
|
+
const summaryMessages = allMessages.map(m => {const newM = Object.assign({}, m); delete newM.message; return newM})
|
|
884
|
+
console.table(summaryMessages)
|
|
885
|
+
|
|
886
|
+
allMessages.forEach(m => {
|
|
887
|
+
const newMessage = Object.assign({}, m);
|
|
888
|
+
delete newMessage.message
|
|
889
|
+
|
|
827
890
|
console.table([newMessage])
|
|
828
891
|
console.log(m.message)
|
|
829
892
|
})
|
|
893
|
+
|
|
894
|
+
|
|
830
895
|
|
|
831
896
|
},
|
|
832
897
|
|
|
@@ -858,20 +923,59 @@ const inputs = {
|
|
|
858
923
|
|
|
859
924
|
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("---- Subnetwork Parameters ----") > -1))
|
|
860
925
|
}
|
|
861
|
-
|
|
926
|
+
|
|
927
|
+
|
|
928
|
+
|
|
929
|
+
//update subnetwork missing elapsed populate it
|
|
930
|
+
allMessages = allMessages.map( m => {
|
|
931
|
+
try{
|
|
932
|
+
|
|
933
|
+
let re = /Total \([-+]?([0-9]*\.[0-9]+|[0-9]+) seconds\)/;
|
|
934
|
+
let res = re.exec(m.message)
|
|
935
|
+
if (res && res.length > 1)
|
|
936
|
+
m.elapsed = res[1]
|
|
937
|
+
|
|
938
|
+
re = /Total update subnetwork time \([-+]?([0-9]*\.[0-9]+|[0-9]+) seconds\)/;
|
|
939
|
+
res = re.exec(m.message)
|
|
940
|
+
if (res && res.length > 1)
|
|
941
|
+
m.elapsed = res[1]
|
|
942
|
+
|
|
943
|
+
return m;
|
|
944
|
+
}
|
|
945
|
+
catch(ex){
|
|
946
|
+
return m;
|
|
947
|
+
}
|
|
948
|
+
})
|
|
949
|
+
|
|
950
|
+
|
|
951
|
+
allMessages = allMessages.map(m => {
|
|
862
952
|
const newMessage = Object.assign({}, m);
|
|
863
|
-
delete newMessage.message
|
|
864
953
|
delete newMessage.source;
|
|
865
954
|
delete newMessage.machine;
|
|
866
955
|
delete newMessage.type;
|
|
956
|
+
delete newMessage.code;
|
|
867
957
|
delete newMessage.requestID;
|
|
958
|
+
delete newMessage.process;
|
|
868
959
|
delete newMessage.thread;
|
|
869
|
-
|
|
960
|
+
newMessage.elapsedms = parseInt (parseFloat(newMessage.elapsed) * 1000)
|
|
961
|
+
newMessage.time = new Date(newMessage.time).toLocaleString()
|
|
962
|
+
delete newMessage.elapsed
|
|
963
|
+
return newMessage;
|
|
964
|
+
})
|
|
965
|
+
.sort( (m1,m2) => m2.elapsedms - m1.elapsedms)
|
|
870
966
|
|
|
967
|
+
const summaryMessages = allMessages.map(m => {const newM = Object.assign({}, m); delete newM.message; return newM})
|
|
968
|
+
console.table(summaryMessages)
|
|
969
|
+
|
|
970
|
+
allMessages.forEach(m => {
|
|
971
|
+
const newMessage = Object.assign({}, m);
|
|
972
|
+
delete newMessage.message
|
|
973
|
+
|
|
871
974
|
console.table([newMessage])
|
|
872
975
|
console.log(m.message)
|
|
873
976
|
})
|
|
874
977
|
|
|
978
|
+
|
|
875
979
|
},
|
|
876
980
|
|
|
877
981
|
|
|
@@ -936,60 +1040,6 @@ const inputs = {
|
|
|
936
1040
|
},
|
|
937
1041
|
|
|
938
1042
|
|
|
939
|
-
|
|
940
|
-
"^cursorlogs --age": async input => {
|
|
941
|
-
const topLogCount = 1000;
|
|
942
|
-
const pageSize = 10000
|
|
943
|
-
|
|
944
|
-
const inputParam = input.match(/--age .*/gm)
|
|
945
|
-
let mins = 30; //query logs for the last 30 minutes
|
|
946
|
-
if (inputParam != null && inputParam.length > 0)
|
|
947
|
-
mins = inputParam[0].replace("--age ", "")
|
|
948
|
-
|
|
949
|
-
console.log(`Querying cursor sql logs for ${parameters.service} for the last ${mins} minutes ...`)
|
|
950
|
-
const startTime = Date.now() - mins*60*1000
|
|
951
|
-
const endTime = Date.now();
|
|
952
|
-
let result= await adminLog.query([102023], [parameters.service+ ".MapServer"], topLogCount, startTime ,endTime , "DEBUG")
|
|
953
|
-
let jsonRes = await result.json()
|
|
954
|
-
let allMessages = [].concat(jsonRes.logMessages)
|
|
955
|
-
allMessages = allMessages.filter(m => m.message.indexOf("EndCursor;") > -1)
|
|
956
|
-
while (jsonRes.hasMore)
|
|
957
|
-
{
|
|
958
|
-
//start paging
|
|
959
|
-
logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
|
|
960
|
-
result= await adminLog.query([102023], [parameters.service + ".MapServer"], pageSize, jsonRes.endTime, null, "DEBUG")
|
|
961
|
-
jsonRes = await result.json()
|
|
962
|
-
|
|
963
|
-
allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("EndCursor;") > -1))
|
|
964
|
-
}
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
//
|
|
968
|
-
allMessages= allMessages.map(m => {
|
|
969
|
-
|
|
970
|
-
const r= m.message.split (";")
|
|
971
|
-
delete r[0];
|
|
972
|
-
delete r[1];
|
|
973
|
-
delete r[3];
|
|
974
|
-
return r;
|
|
975
|
-
|
|
976
|
-
const newMessage = Object.assign({}, m);
|
|
977
|
-
|
|
978
|
-
delete newMessage.source;
|
|
979
|
-
delete newMessage.machine;
|
|
980
|
-
delete newMessage.type;
|
|
981
|
-
delete newMessage.requestID;
|
|
982
|
-
delete newMessage.methodName;
|
|
983
|
-
delete newMessage.process;
|
|
984
|
-
delete newMessage.thread;
|
|
985
|
-
delete newMessage.time;
|
|
986
|
-
delete newMessage.code;
|
|
987
|
-
|
|
988
|
-
return newMessage
|
|
989
|
-
|
|
990
|
-
})
|
|
991
|
-
console.table(allMessages)
|
|
992
|
-
},
|
|
993
1043
|
|
|
994
1044
|
|
|
995
1045
|
"^arlogs --byrule": async input => {
|
|
@@ -1004,17 +1054,29 @@ const inputs = {
|
|
|
1004
1054
|
if (inputParam != null && inputParam.length > 0 && inputParam[0].indexOf("--minguid") > -1)
|
|
1005
1055
|
showMinGuid = true
|
|
1006
1056
|
|
|
1057
|
+
|
|
1058
|
+
const ageInputParam = input.match(/--age [0-9]*/)
|
|
1059
|
+
let mins = 30; //query logs for the last 30 minutes
|
|
1060
|
+
if (ageInputParam != null && ageInputParam.length > 0)
|
|
1061
|
+
mins = ageInputParam[0].replace("--age ", "")
|
|
1062
|
+
|
|
1063
|
+
|
|
1064
|
+
const startTime = Date.now() - mins*60*1000
|
|
1065
|
+
const endTime = Date.now();
|
|
1066
|
+
|
|
1007
1067
|
const pageSize = 10000 //maximum messages per page
|
|
1008
|
-
logger.info(`Querying attribute rules logs for ${parameters.service} ...`)
|
|
1009
|
-
let result= await adminLog.query([102003], [parameters.service + ".MapServer"], pageSize)
|
|
1068
|
+
logger.info(`Querying attribute rules logs for ${parameters.service} in the past ${mins} minutes...`)
|
|
1069
|
+
let result= await adminLog.query([102003], [parameters.service + ".MapServer"], pageSize, startTime ,endTime , "DEBUG")
|
|
1010
1070
|
let jsonRes = await result.json()
|
|
1011
1071
|
let allMessages = [].concat(jsonRes.logMessages)
|
|
1012
1072
|
|
|
1013
|
-
while (jsonRes.hasMore)
|
|
1073
|
+
while (jsonRes.hasMore && jsonRes.endTime > startTime)
|
|
1014
1074
|
{
|
|
1015
1075
|
//start paging
|
|
1016
1076
|
logger.info(`Aggregating messages... total so far ${allMessages.length} debug entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
|
|
1017
|
-
|
|
1077
|
+
|
|
1078
|
+
|
|
1079
|
+
result= await adminLog.query([102003], [parameters.service + ".MapServer"], pageSize, jsonRes.endTime )
|
|
1018
1080
|
jsonRes = await result.json()
|
|
1019
1081
|
allMessages = allMessages.concat(jsonRes.logMessages)
|
|
1020
1082
|
}
|
package/logger.mjs
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
const logger = {}
|
|
2
|
+
logger.info = console.log
|
|
3
|
+
logger.error = console.error
|
|
4
|
+
|
|
5
|
+
export default logger;
|
|
6
|
+
|
|
7
|
+
/*
|
|
1
8
|
import winston from "winston"
|
|
2
9
|
|
|
3
10
|
export const logger = winston.createLogger();
|
|
@@ -5,4 +12,4 @@ export const logger = winston.createLogger();
|
|
|
5
12
|
logger.add(
|
|
6
13
|
new winston.transports.Console({"format": winston.format.json()})
|
|
7
14
|
);
|
|
8
|
-
|
|
15
|
+
*/
|
package/makerequest.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
|
|
1
|
+
|
|
2
|
+
|
|
2
3
|
|
|
3
4
|
export function makeRequest (opts) {
|
|
4
5
|
|
|
@@ -24,7 +25,18 @@ export function makeRequest (opts) {
|
|
|
24
25
|
Object.keys(opts.headers).forEach( key => headers[key] = opts.headers[key] )
|
|
25
26
|
|
|
26
27
|
//console.log(opts)
|
|
27
|
-
|
|
28
|
+
|
|
29
|
+
let f;
|
|
30
|
+
try {
|
|
31
|
+
const nodeFetch = await import ("node-fetch");
|
|
32
|
+
f = nodeFetch.default;
|
|
33
|
+
}
|
|
34
|
+
catch(ex) {
|
|
35
|
+
f = fetch;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
const result = await f(opts.url, {
|
|
28
40
|
"method" : opts.method,
|
|
29
41
|
"headers": headers,
|
|
30
42
|
"body": params
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "un-cli",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.71",
|
|
4
4
|
"description": "Command line interface for working with ArcGIS Utility Network Extension",
|
|
5
5
|
"main": "index.mjs",
|
|
6
6
|
"bin": {
|
|
@@ -17,8 +17,6 @@
|
|
|
17
17
|
"author": "Hussein Nasser",
|
|
18
18
|
"license": "ISC",
|
|
19
19
|
"dependencies": {
|
|
20
|
-
"node-fetch": "^2.6.0"
|
|
21
|
-
"progress": "^2.0.3",
|
|
22
|
-
"winston": "^3.2.1"
|
|
20
|
+
"node-fetch": "^2.6.0"
|
|
23
21
|
}
|
|
24
22
|
}
|
package/portal.mjs
CHANGED
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
import { makeRequest } from "./makerequest.mjs"
|
|
2
|
-
import
|
|
2
|
+
import logger from "./logger.mjs"
|
|
3
3
|
|
|
4
4
|
export class Portal{
|
|
5
5
|
|
|
6
|
-
constructor(url, username, password, expiration = 300, serverUrl = undefined)
|
|
6
|
+
constructor(url, username, password, expiration = 300, serverUrl = undefined, referer)
|
|
7
7
|
{
|
|
8
8
|
this.url = url;
|
|
9
9
|
this.username = username;
|
|
10
10
|
this.password = password;
|
|
11
11
|
this.expiration = expiration;
|
|
12
12
|
this.serverUrl = serverUrl;
|
|
13
|
+
this.referer = referer
|
|
13
14
|
}
|
|
14
15
|
|
|
15
16
|
|
|
@@ -25,7 +26,7 @@ export class Portal{
|
|
|
25
26
|
const postJson = {
|
|
26
27
|
username: self.username,
|
|
27
28
|
password: self.password,
|
|
28
|
-
referer:
|
|
29
|
+
referer: self.referer,
|
|
29
30
|
expiration: self.expiration,
|
|
30
31
|
f: "json"
|
|
31
32
|
}
|
|
@@ -71,7 +72,7 @@ export class Portal{
|
|
|
71
72
|
token: self.token
|
|
72
73
|
}
|
|
73
74
|
|
|
74
|
-
const serversUrl = self.url +
|
|
75
|
+
const serversUrl = self.url + `/sharing/rest/portals/self/servers?token=${self.token}`
|
|
75
76
|
logger.info( "About to query federated servers");
|
|
76
77
|
|
|
77
78
|
//query for federated servers.
|
package/utilitynetwork.node.mjs
CHANGED
|
@@ -8,9 +8,9 @@
|
|
|
8
8
|
/*
|
|
9
9
|
import fetch from "node-fetch"
|
|
10
10
|
*/
|
|
11
|
-
import
|
|
12
|
-
import
|
|
13
|
-
|
|
11
|
+
import logger from "./logger.mjs"
|
|
12
|
+
import { makeRequest } from "./makerequest.mjs"
|
|
13
|
+
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
export class UtilityNetwork {
|
|
@@ -122,6 +122,7 @@ export class UtilityNetwork {
|
|
|
122
122
|
|
|
123
123
|
}
|
|
124
124
|
|
|
125
|
+
/*
|
|
125
126
|
//a function that detects associations with from/to globalId that don't exist in the source tables.
|
|
126
127
|
//if one of the endpoints do not exists fails..
|
|
127
128
|
returnInvalidAssociations() {
|
|
@@ -178,7 +179,7 @@ export class UtilityNetwork {
|
|
|
178
179
|
|
|
179
180
|
|
|
180
181
|
}
|
|
181
|
-
|
|
182
|
+
*/
|
|
182
183
|
queryMoment(moment = ["definitionModification"]) {
|
|
183
184
|
|
|
184
185
|
let thisObj = this;
|
|
@@ -1086,6 +1087,7 @@ export class UtilityNetwork {
|
|
|
1086
1087
|
|
|
1087
1088
|
|
|
1088
1089
|
|
|
1090
|
+
/*
|
|
1089
1091
|
//Makes a request
|
|
1090
1092
|
function makeRequest (opts) {
|
|
1091
1093
|
|
|
@@ -1134,7 +1136,7 @@ export class UtilityNetwork {
|
|
|
1134
1136
|
|
|
1135
1137
|
});
|
|
1136
1138
|
}
|
|
1137
|
-
|
|
1139
|
+
*/
|
|
1138
1140
|
|
|
1139
1141
|
|
|
1140
1142
|
const v = (o, f, vIfNotFound=null) => {
|