un-cli 0.0.84 → 0.0.86

package/README.md

@@ -1,17 +1,68 @@
 # uncli
-utility network javascript command line works on Windows and Linux
+Server Log Parser and utility network javascript command line works on Windows and Linux
 
-## Download Node JS 13.5 or later
+
+
+## Server Log web parser
+This tool also provides a way to parse server logs and view them for each utility network and geodatabase functions. (Note you do not need to install nodejs for the web parser)
+
+### IIS
+To install on IIS follow these steps: This server assume your machine name where the Webadaptor lives is `utilitynetwork.esri.com` 
+
+- Login to machine where you have the webadaptor
+- Create a folder called `log` in `c:\inetpub\wwwroot`
+- Copy the entire content of the repro https://github.com/hussein-nasser/uncli to c:\inetpub\wwwroot\log
+- Open IIS (Internet Information Services)
+- Open MIME Types
+- Under action click Add 
+- under file extension type `.mjs` and under MIME Type write `application/javascript`
+- Restart IIS
+- (OPTIONAL only change when your webadaptor is not /portal) In c:\inetpub\wwwroot\log edit the `index.html` and find this line of code
+- Update the JSON object with the correct URL, for example if your webadaptor is https://utilitynetwork.esri.com/portal set the "portal" to be that. You don't have to update the referer it will be automatically calculated it.  
+- (Optional) if you have multiple federated server , you have to set the correct one you want to query admin api for in the `"server"` parameter below.
+
+```js
+let parameters = {
+        "user": "unadmin",
+        "password": "",
+        "portal": "",
+        "service": "",
+        "referer": "",
+        "server": undefined
+    }
+```
+- Save the HTML and then visit `https://utilitynetwork.esri.com/log` put in your username and password and login to use the parser
+
+
+### Tomcat or any other similar Webservers
+-Create a folder under tomcat/webapps/ call it log 
+-Then copy the content there. 
+-Then go to /home/tomcat/conf/web.xml 
+-And add this entry if doesn’t exist 
+ ```xml
+    <mime-mapping>
+        <extension>mjs</extension>
+        <mime-type>text/javascript</mime-type>
+    </mime-mapping>
+ ```
+
+-Restart tomcat and visit the page/log should work
+
+
+
+
+## Utility Network Command Line tool (Utility Network Only)
+### Download Node JS 13.5 or later
 
 https://nodejs.org/en/
 
-## Open command prompt and run this command
+### Open command prompt and run this command
 
 ```bash
 npm install -g un-cli
 ```
 
-## Once installed here is how you connect 
+### Once installed here is how you connect 
 
 ```bash
 > uncli --portal https://utilitynetwork.esri.com/portal --service NapervilleElectric_SQLServer --user tester --password tester.108 --verify true
@@ -67,7 +118,7 @@ uncli> help
 
 
  
-## Execute bulk of commands
+### Execute bulk of commands
 Create a commands.txt file and type in the commands in that file
 command.txt
 ```text
@@ -77,32 +128,3 @@ export subnetworks --new
 
 
 > uncli --portal https://utilitynetwork.esri.com/portal --service NapervilleElectric_SQLServer --user tester --password tester.108 --file commands.txt --verify true
-
-## Server Log web parser
-This tool also provides a way to parse server logs and view them for each utility network and geodatabase function
-
-To install on IIS follow these steps: This server assume your machine name where the Webadaptor lives is `utilitynetwork.esri.com` 
-
-- Login to machine where you have the webadaptor
-- Create a folder called `log` in `c:\inetpub\wwwroot`
-- Copy the entire content of the repro https://github.com/hussein-nasser/uncli to c:\inetpub\wwwroot\log
-- Open IIS (Internet Information Services)
-- Open MIME Types
-- Under action click Add 
-- under file extension type `.mjs` and under MIME Type write `application/javascript`
-- Restart IIS
-- (OPTIONAL only change when your webadaptor is not /portal) In c:\inetpub\wwwroot\log edit the `index.html` and find this line of code
-- Update the JSON object with the correct URL, for example if your webadaptor is https://utilitynetwork.esri.com/portal set the "portal" to be that. You don't have to update the referer it will be automatically calculated it.  
-- (Optional) if you have multiple federated server , you have to set the correct one you want to query admin api for in the `"server"` parameter below.
-
-```js
-let parameters = {
-        "user": "unadmin",
-        "password": "",
-        "portal": "",
-        "service": "",
-        "referer": "",
-        "server": undefined
-    }
-    ```
-- Save the HTML and then visit `https://utilitynetwork.esri.com/log` put in your username and password and login to use the parser

package/examples.txt

@@ -0,0 +1,4 @@
+export subnetworks --all --resulttype [{"type":"features","includeGeometry":true,"includePropagatedValues":false,"includeDomainDescriptions":true,"networkAttributeNames":["Source ID","E:Riser"],"diagramTemplateName":"","resultTypeFields":[{"networkSourceId":14,"fieldName":"GLOBALID"},{"networkSourceId":15,"fieldName":"CLUSTERKEY"},{"networkSourceId":9,"fieldName":"LIFECYCLESTATUS"},{"networkSourceId":4,"fieldName":"ASSETGROUP"},{"networkSourceId":12,"fieldName":"ASSETGROUP"}]},{"type":"connectivity","includeGeometry":true,"includePropagatedValues":false,"includeDomainDescriptions":true,"networkAttributeNames":[],"diagramTemplateName":"","resultTypeFields":[]},{"type":"associations","includeGeometry":false,"includePropagatedValues":false,"includeDomainDescriptions":true,"networkAttributeNames":[],"diagramTemplateName":"","resultTypeFields":[]}]
+update subnewtorks --all
+topology --validate
+trace --subnetwork test

package/index.html

@@ -54,7 +54,7 @@
         <td>Method Name</td>
         <td><input id = 'txtMethodName' type = 'text'></td>
         <td>RequestID</td>
-        <td colspan =3><input id = 'txtRequestId' type = 'text'></td>
+        <td colspan =3><input id = 'txtRequestId' type = 'text'  style="width:250px"></td>
     </tr>
 </table>
 
@@ -69,13 +69,15 @@
             <button id = 'btnApplyEditsLogs'>ApplyEdits Logs</button>
             <button id = 'btnQueryLogs'>Query Logs</button>
             <button id = 'btnReconcileLogs'>Reconcile Logs</button>
+            <button id = 'btnstartEditingLogs'>StartEditing Logs</button>
             <button id = 'btnstopEditingLogs'>StopEditing Logs</button>
-            <button id = 'btnWaitLogs'>Wait Logs</button>
+            <button id = 'btnWaitLogs'>Startup and Wait Logs</button>
             <button id = 'btnDownloadLogs'>Raw Logs</button>
         </td>
         
     </tr>
     <tr><td colspan=9><small>Elapsed time is in minutes:seconds:milliseconds</small></td></tr>
+    <tr><td colspan=9><small>To capture the logs effectively, enable debug logging from server manager, login as admin, run the operation (e.g. edit in Pro) and click the corresponding logs to load.</small></td></tr>
     </table>
 
     <table id = 'tblResult' border =1 cellpadding = 4 cellspacing = 4 style="text-align:left">
@@ -170,6 +172,7 @@
         document.getElementById("btnDownloadLogs").disabled = false 
         document.getElementById("btnReconcileLogs").disabled = false 
         document.getElementById("btnstopEditingLogs").disabled = false 
+        document.getElementById("btnstartEditingLogs").disabled = false 
         document.getElementById("btnWaitLogs").disabled = false 
         
 
@@ -239,7 +242,9 @@
 
                 x = /Moment:.*\r\n/.exec(m.message);
                 if (x && x.length > 0) newMessage.moment = x[0].replace("Moment:", "").replace("\r\n", "") * 1000000
-   
+                
+                x = /Moment:\s*([\d.]+)/.exec(m.message);
+                if (x) newMessage.moment = parseFloat(x[1]) * 1000000;
  
             }
             catch(ex){
@@ -261,8 +266,6 @@
         cTime.textContent = "Time"
         const cUser = document.createElement("th");
         cUser.textContent = "User"
-        const cMethod = document.createElement("th");
-        cMethod.textContent = "Method"
         const cVersion = document.createElement("th");
         cVersion.textContent = "Version"
         const cMoment = document.createElement("th");
@@ -273,6 +276,8 @@
         cTraceType.textContent = "Trace Type"
         const cTraceTime = document.createElement("th");
         cTraceTime.textContent = "Trace Time"
+        const cReadAmp = document.createElement("th");
+        cReadAmp.textContent = "Elements / Pages"
         const cRequestId = document.createElement("th");
         cRequestId.textContent = "RequestId"
         const cLog = document.createElement("th");
@@ -280,11 +285,11 @@
         header.appendChild (cService)
         header.appendChild (cTime)
         header.appendChild (cUser)
-        header.appendChild (cMethod)
         header.appendChild (cVersion)
         header.appendChild (cMoment)
         header.appendChild (cTraceType)
         header.appendChild (cTraceTime)
+        header.appendChild (cReadAmp)
         header.appendChild (cRequestId)
         header.appendChild (cElapsedMS)
 
@@ -306,6 +311,8 @@
             cTraceType.textContent = m.traceType
             const cTraceTime = document.createElement("td");
             cTraceTime.textContent = msToMinSecMs(m.traceTime)
+            const cReadAmp = document.createElement("td");
+            cReadAmp.textContent = ParseTraceReadAmplication(m.message)
             const cRequestId = document.createElement("td");
             cRequestId.textContent = m.requestID
             const cVersion = document.createElement("td");
@@ -319,11 +326,11 @@
             logRow.appendChild (cService)
             logRow.appendChild (cTime)
             logRow.appendChild (cUser)
-            logRow.appendChild (cMethod)
             logRow.appendChild (cVersion)
             logRow.appendChild (cMoment)
             logRow.appendChild (cTraceType)
             logRow.appendChild (cTraceTime)
+            logRow.appendChild (cReadAmp)
             logRow.appendChild (cRequestId)
             logRow.appendChild (cElapsedMS)
 
@@ -379,9 +386,16 @@
                 let x ;
                 x = /Extent:.*\r\n/.exec(m.message);
                 if (x && x.length > 0) m.extent = x[0].replace("Extent:", "").replace("\r\n", "")
+
+                x = /"extent":\{[^}]+\}/.exec(m.message);
+                if (x) m.extent = x[0].replace(`"extent":`,"")
+
                 x = /Version name:.*\r\n/.exec(m.message);
                 if (x && x.length > 0) m.versionName = x[0].replace("Version name:", "").replace("\r\n", "")
 
+                x = /Version name:(.*)/.exec(m.message);
+                if (x) m.versionName = x[1].trim();
+
                 let re = new RegExp(`Network built. [-+]?([0-9]*\\.[0-9]+|[0-9]+) seconds \\([-+]?([0-9]*\\.[0-9]+|[0-9]+) total\\)`)
                 let res = re.exec(m.message)
                 
@@ -510,6 +524,7 @@
 
         const cRequestId = document.createElement("th");
         cRequestId.textContent = "RequestId"
+ 
         const cLog = document.createElement("th");
         cLog.textContent = "Full Log"
 
@@ -547,6 +562,12 @@
 
             const cRequestId = document.createElement("td");
             cRequestId.textContent = m.requestID;
+            cRequestId.style.color = "blue";
+            cRequestId.style.textDecoration = "underline";
+            cRequestId.style.cursor = "pointer";
+            cRequestId.style.fontOpticalSizing
+            cRequestId.addEventListener("click", e=> viewRawLogs( m.requestID ) );
+
             const cLog = document.createElement("td");
             cLog.textContent = "..."
             cLog.fullLog = m.message;
@@ -678,6 +699,12 @@ async function loadUpdateSubnetworkLogs () {
             cSubnetworkName.textContent = m.subnetworkName
             const cRequestId = document.createElement("td");
             cRequestId.textContent = m.requestID
+            cRequestId.style.color = "blue";
+            cRequestId.style.textDecoration = "underline";
+            cRequestId.style.cursor = "pointer";
+            cRequestId.style.fontOpticalSizing
+            cRequestId.addEventListener("click", e=> viewRawLogs( m.requestID ) );
+
             const cLog = document.createElement("td");
             cLog.textContent = "..."
             cLog.fullLog = m.message;
@@ -719,7 +746,7 @@ async function loadAttributeRules () {
  
         const arMessages = filterMessages(allMessages)
             .filter(m => m.message.indexOf("Attribute rule executed: ") > -1)
-            .map (m =>  JSON.parse(m.message.replace("Attribute rule executed: ", "")))
+            .map (m =>  decodeAndParseJSON(m.message.replace("Attribute rule executed: ", "")))
             .map( m => {
                 m["Elapsed Time (ms)"] =  Math.round(m["Elapsed Time"]*1000000)/1000
                // m["Arcade Evaluation Time:"] = Math.round(m["Arcade Evaluation Time:"]*1000,6)
@@ -1016,7 +1043,8 @@ console.log(`Querying applyEdits logs for ${parameters.service} for the last ${m
         if (m.message.indexOf("GraphicFeatureServer::HandleRESTRequest ## inputJSON ") > -1)
         {
             applyEditslogs[m.requestID].message = m.message.replace("GraphicFeatureServer::HandleRESTRequest ## inputJSON = ","")
-            const applyEditsPayload = JSON.parse(applyEditslogs[m.requestID].message)
+            applyEditslogs[m.requestID].message = decodeHTMLEntities(applyEditslogs[m.requestID].message)
+            const applyEditsPayload = decodeAndParseJSON(applyEditslogs[m.requestID].message)
             applyEditslogs[m.requestID].gdbVersion = applyEditsPayload.gdbVersion
             applyEditslogs[m.requestID].ids =  [...new Set(applyEditsPayload.edits.reduce ( (prev, cur) => {prev.push(cur.id); return prev}, []) )]; 
         }
@@ -1066,6 +1094,13 @@ console.log(`Querying applyEdits logs for ${parameters.service} for the last ${m
             const cRequestId = document.createElement("td");
             cRequestId.textContent = m.requestID
 
+            cRequestId.style.color = "blue";
+            cRequestId.style.textDecoration = "underline";
+            cRequestId.style.cursor = "pointer";
+            cRequestId.style.fontOpticalSizing
+            cRequestId.addEventListener("click", e=> viewRawLogs( m.requestID ) );
+
+
             const cLog = document.createElement("td");
             const cLogText = document.createElement("input")
             cLogText.type = "text"
@@ -1097,6 +1132,7 @@ console.log(`Querying applyEdits logs for ${parameters.service} for the last ${m
 
 async function stopEditingLogs () {
 
+    
 //build table
 const tblResult = document.getElementById("tblResult");
 //clear
@@ -1109,7 +1145,7 @@ parameters.service = cmbService.options[cmbService.selectedIndex].text
 let mins = document.getElementById("txtAge").value  //query logs for the last 30 minutes
 
 
-console.log(`Querying stop editing logs for ${parameters.service} for the last ${mins} minutes ...`)
+console.log(`Querying start editing logs for ${parameters.service} for the last ${mins} minutes ...`)
 
     //startTime is the most recent
     //endTime is the oldest
@@ -1158,20 +1194,22 @@ console.log(`Querying stop editing logs for ${parameters.service} for the last $
     const queryLogs = {}
     //sort by time
     allMessages = allMessages.sort ( (m1, m2) => m2.time - m1.time )
+   // allMessages = allMessages.filter( m => m.methodName.indexOf("Geodatabase.FeatureDataset.StartEditing") > -1   )
     allMessages.forEach (m => {
 
         if (!queryLogs[m.requestID])
             queryLogs[m.requestID] = {"message": "Time,Method,Elapsed_ms,Message"}
 
-        queryLogs[m.requestID].message += "\r\n" + m.time + "," + m.methodName + "," + Math.round(m.elapsed*1000) + "," + m.message 
+        if (m.methodName.indexOf("Geodatabase.FeatureDataset.StopEditing") > -1)
+            queryLogs[m.requestID].message += "\r\n" + m.time + "," + m.methodName + "," + Math.round(m.elapsed*1000) + "," + m.message 
 
         //get elapsed
         //check for async (method GPReconcileVersionAsync::Execute)
         //sync
         //VersionManagementServer::HandleREST_ReconcileOperation
         //message Returned moment: 
-        if  ( m.methodName.indexOf("VersionManagementServer::HandleREST_StopEditingOperation") > -1  
-           && m.message.indexOf("End StopEditing") > -1
+        if  ( m.methodName.indexOf("Geodatabase.FeatureDataset.StopEditing") > -1  
+           && m.message.indexOf("Starting editing") > -1
         )
         {
             queryLogs[m.requestID].elapsed = m.elapsed
@@ -1180,13 +1218,13 @@ console.log(`Querying stop editing logs for ${parameters.service} for the last $
             queryLogs[m.requestID].time = m.time
             queryLogs[m.requestID].requestID = m.requestID
             queryLogs[m.requestID].methodName = m.methodName.replace("VersionManagementServer::HandleREST_","")
- 
+            queryLogs[m.requestID].textMessage = m.message 
         }
         
         if (m.message.indexOf("In WorkspaceInfo::GetVersionedWorkspaceInfo : Cache hit for versionOrBranchName=") > -1)
         {   
-            let v =  m.message.replace(": In WorkspaceInfo::GetVersionedWorkspaceInfo : Cache hit for versionOrBranchName=","")
-            v = v.substr(v.indexOf("{"))
+            let v =  m.message.replace(" In WorkspaceInfo::GetVersionedWorkspaceInfo : Cache hit for versionOrBranchName=","")
+            v = v.substr(v.indexOf(":")+1)
             queryLogs[m.requestID].gdbVersion =v
     
         }
@@ -1195,6 +1233,7 @@ console.log(`Querying stop editing logs for ${parameters.service} for the last $
         
     })
 
+    /*
     allMessages = []
 
      Object.keys(queryLogs).forEach(k =>
@@ -1205,14 +1244,17 @@ console.log(`Querying stop editing logs for ${parameters.service} for the last $
               
         })
 
-
+*/
     
   console.log ("Filtering messages...") 
- 
+  allMessages = allMessages.filter( m => m.methodName.indexOf("Geodatabase.FeatureDataset.StopEditing") > -1   )
+
   allMessages = filterMessages(allMessages)
   .sort( (m1,m2) => Math.round(m2.elapsed*1000) -Math.round(m1.elapsed*1000))
 
   allMessages.forEach (m => {  
+
+        
             const logRow = document.createElement("tr");
   
             const cService = document.createElement("th");
@@ -1225,21 +1267,27 @@ console.log(`Querying stop editing logs for ${parameters.service} for the last $
             cUser.textContent = m.user
 
             const cVersion = document.createElement("td");
-            cVersion.textContent = m.gdbVersion 
+            cVersion.textContent =  queryLogs[m.requestID].gdbVersion
            
             const cMethodName = document.createElement("td");
             cMethodName.textContent = m.methodName         
 
             const cRequestId = document.createElement("td");
-            cRequestId.textContent = m.requestID 
-  
+            cRequestId.textContent = m.requestID
+
+            cRequestId.style.color = "blue";
+            cRequestId.style.textDecoration = "underline";
+            cRequestId.style.cursor = "pointer";
+            cRequestId.style.fontOpticalSizing
+            cRequestId.addEventListener("click", e=> viewRawLogs( m.requestID ) );
+
             const cTotalTime = document.createElement("td");
             cTotalTime.textContent =  msToMinSecMs(m.elapsed*1000 )
             const cLog = document.createElement("td");
-            cLog.textContent = "..."
-            cLog.title = "Click to copy full stopediting log"
-            cLog.fullLog = m.message;
-            cLog.addEventListener("click", e=> {navigator.clipboard.writeText(e.target.fullLog); alert("Copied to clipboard")})
+            cLog.textContent = m.message
+           // cLog.title = "Click to copy full startediting log"
+           // cLog.fullLog = m.message;
+           // cLog.addEventListener("click", e=> {navigator.clipboard.writeText(e.target.fullLog); alert("Copied to clipboard")})
         
             //cLog.fullLog = m.payLoad;
             //cLog.title = m.payLoad + "\nClick to copy"
@@ -1267,6 +1315,197 @@ console.log(`Querying stop editing logs for ${parameters.service} for the last $
 
 
 
+async function viewRawLogs(requestId){
+
+    const txtRequestId =  document.getElementById("txtRequestId")
+    txtRequestId.value = requestId;
+    document.getElementById("btnDownloadLogs").click();
+
+    
+}
+
+
+async function startEditingLogs () {
+
+//build table
+const tblResult = document.getElementById("tblResult");
+//clear
+while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
+    
+
+parameters.service = cmbService.options[cmbService.selectedIndex].text
+
+
+let mins = document.getElementById("txtAge").value  //query logs for the last 30 minutes
+
+
+console.log(`Querying start editing logs for ${parameters.service} for the last ${mins} minutes ...`)
+
+    //startTime is the most recent
+    //endTime is the oldest
+
+     
+ //page query the admin log , search for /stopediting logs by methodname
+  let allMessages = await adminLogQueryWithPaging(mins, parameters.service, [102003,17003], "", "DEBUG")
+
+
+  while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
+
+    let i =0;
+    
+    const header = document.createElement("tr");
+    const cService = document.createElement("th");
+    cService.textContent = "Source"
+    const cTime = document.createElement("th");
+    cTime.textContent = "Time"
+    const cUser = document.createElement("th");
+    cUser.textContent = "User"
+    const cVersion = document.createElement("th");
+    cVersion.textContent = "Version"
+    const cMethodName = document.createElement("th");
+    cMethodName.textContent = "MethodName"
+    const cRequestId = document.createElement("th");
+    cRequestId.textContent = "RequestId"
+    const cTotalTime = document.createElement("th");
+    cTotalTime.title  = "Total time startEditing took"
+    cTotalTime.textContent = "Total Time"
+ 
+
+    const cLog = document.createElement("th");
+    cLog.textContent = "StartEditing details"
+    header.appendChild (cService)
+    header.appendChild (cTime)
+    header.appendChild (cUser)
+    header.appendChild (cVersion)
+    header.appendChild (cMethodName)
+    header.appendChild (cRequestId)
+    header.appendChild (cTotalTime)
+ 
+    header.appendChild (cLog) 
+    tblResult.appendChild(header)
+
+    //build out the dictionary, key is request id, value is another dictionary
+    const queryLogs = {}
+    //sort by time
+    allMessages = allMessages.sort ( (m1, m2) => m2.time - m1.time )
+   // allMessages = allMessages.filter( m => m.methodName.indexOf("Geodatabase.FeatureDataset.StartEditing") > -1   )
+    allMessages.forEach (m => {
+
+        if (!queryLogs[m.requestID])
+            queryLogs[m.requestID] = {"message": "Time,Method,Elapsed_ms,Message"}
+
+        if (m.methodName.indexOf("Geodatabase.FeatureDataset.StartEditing") > -1)
+            queryLogs[m.requestID].message += "\r\n" + m.time + "," + m.methodName + "," + Math.round(m.elapsed*1000) + "," + m.message 
+
+        //get elapsed
+        //check for async (method GPReconcileVersionAsync::Execute)
+        //sync
+        //VersionManagementServer::HandleREST_ReconcileOperation
+        //message Returned moment: 
+        if  ( m.methodName.indexOf("Geodatabase.FeatureDataset.StartEditing") > -1  
+           && m.message.indexOf("Starting editing") > -1
+        )
+        {
+            queryLogs[m.requestID].elapsed = m.elapsed
+            queryLogs[m.requestID].source = m.source.replace(".MapServer", "")
+            queryLogs[m.requestID].user = m.user
+            queryLogs[m.requestID].time = m.time
+            queryLogs[m.requestID].requestID = m.requestID
+            queryLogs[m.requestID].methodName = m.methodName.replace("VersionManagementServer::HandleREST_","")
+            queryLogs[m.requestID].textMessage = m.message 
+        }
+        
+        if (m.message.indexOf("In WorkspaceInfo::GetVersionedWorkspaceInfo : Cache hit for versionOrBranchName=") > -1)
+        {   
+            let v =  m.message.replace(" In WorkspaceInfo::GetVersionedWorkspaceInfo : Cache hit for versionOrBranchName=","")
+            v = v.substr(v.indexOf(":")+1)
+            queryLogs[m.requestID].gdbVersion =v
+    
+        }
+
+        
+        
+    })
+
+    /*
+    allMessages = []
+
+     Object.keys(queryLogs).forEach(k =>
+        {
+            const m = queryLogs[k]
+            if (m.methodName)
+                allMessages.push(m)
+              
+        })
+
+*/
+    
+  console.log ("Filtering messages...") 
+  allMessages = allMessages.filter( m => m.methodName.indexOf("Geodatabase.FeatureDataset.StartEditing") > -1   )
+
+  allMessages = filterMessages(allMessages)
+  .sort( (m1,m2) => Math.round(m2.elapsed*1000) -Math.round(m1.elapsed*1000))
+
+  allMessages.forEach (m => {  
+
+        
+            const logRow = document.createElement("tr");
+  
+            const cService = document.createElement("th");
+            cService.textContent = m.source
+
+            const cTime = document.createElement("td");
+            cTime.textContent =  new Date(m.time).toLocaleString()
+
+            const cUser = document.createElement("td");
+            cUser.textContent = m.user
+
+            const cVersion = document.createElement("td");
+            cVersion.textContent =  queryLogs[m.requestID].gdbVersion
+           
+            const cMethodName = document.createElement("td");
+            cMethodName.textContent = m.methodName         
+
+            const cRequestId = document.createElement("td");
+            cRequestId.textContent = m.requestID 
+            cRequestId.style.color = "blue";
+            cRequestId.style.textDecoration = "underline";
+            cRequestId.style.cursor = "pointer";
+            cRequestId.style.fontOpticalSizing
+            cRequestId.addEventListener("click", e=> viewRawLogs( m.requestID ) );
+
+            const cTotalTime = document.createElement("td");
+            cTotalTime.textContent =  msToMinSecMs(m.elapsed*1000 )
+            const cLog = document.createElement("td");
+            cLog.textContent = m.message
+           // cLog.title = "Click to copy full startediting log"
+           // cLog.fullLog = m.message;
+           // cLog.addEventListener("click", e=> {navigator.clipboard.writeText(e.target.fullLog); alert("Copied to clipboard")})
+        
+            //cLog.fullLog = m.payLoad;
+            //cLog.title = m.payLoad + "\nClick to copy"
+           // cLog.addEventListener("click", e=> {navigator.clipboard.writeText(e.target.fullLog); e.target.selectionStart = 0; e.target.selectionEnd = e.target.value.length;
+            //    alert("Copied to clipboard")  })
+            logRow.appendChild (cService)
+            logRow.appendChild (cTime)
+            logRow.appendChild (cUser)
+            logRow.appendChild (cVersion)
+            logRow.appendChild (cMethodName)
+            logRow.appendChild (cRequestId)
+            logRow.appendChild (cTotalTime)
+ 
+            logRow.appendChild (cLog) 
+            tblResult.appendChild(logRow)
+  });
+  
+    addCSV(allMessages)
+
+
+
+}
+
+
+
 
 
 async function waitLogs () {
@@ -1642,7 +1881,8 @@ console.log(`Querying query logs for ${parameters.service} for the last ${mins}
         if (m.message.indexOf("GraphicFeatureServer::HandleRESTRequest ## inputJSON = ") > -1)
         {
             queryLogs[m.requestID].message = m.message.replace("GraphicFeatureServer::HandleRESTRequest ## inputJSON = ","")
-            const queryPayload = JSON.parse(queryLogs[m.requestID].message)
+            queryLogs[m.requestID].message = decodeHTMLEntities(queryLogs[m.requestID].message)
+            const queryPayload = decodeAndParseJSON(queryLogs[m.requestID].message)
             queryLogs[m.requestID].gdbVersion = queryPayload.gdbVersion
             queryLogs[m.requestID].moment = queryPayload.historicMoment
             queryLogs[m.requestID].methodName = m.methodName;
@@ -1942,6 +2182,9 @@ function numberWithCommas(x) {
     btnstopEditingLogs.disabled = true
 
 
+    const btnstartEditingLogs = document.getElementById("btnstartEditingLogs");
+    btnstartEditingLogs.addEventListener("click", e => startEditingLogs())
+    btnstartEditingLogs.disabled = true
     
     const btnWait = document.getElementById("btnWaitLogs");
     btnWait.addEventListener("click", e => waitLogs())
@@ -2144,6 +2387,12 @@ function numberWithCommas(x) {
         })
     }
 
+
+    function decodeAndParseJSON (x) {
+          const y = decodeHTMLEntities(x);
+          return JSON.parse(y);
+    }
+
     function decodeHTMLEntities (x) {
         const decodedString = document.createElement('div');
         decodedString.innerHTML = x;
@@ -2218,6 +2467,14 @@ function numberWithCommas(x) {
             
             allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf(messageFilter) > -1 && m.methodName.indexOf(methodFilter) > -1))
         }
+        //DECODE html for all messages.
+        logger.info(`Decoding HTML entities ${allMessages.length}`);
+
+        allMessages = allMessages.map(m=> {  
+            m.message = decodeHTMLEntities(m.message)
+            return m
+        })
+
         logger.info(`Completed aggregation total messages ${allMessages.length}`);
 
         return allMessages
@@ -2225,6 +2482,58 @@ function numberWithCommas(x) {
     }
 
 
+
+    function ParseTraceReadAmplication (traceLog) {
+        
+        try {
+
+    
+        const text = traceLog;
+        const traversedMatch = text.match(/Total\s+#\s*Traversed\s+Elements:\s*([\d,]+)/i);
+
+        const traversedElements = traversedMatch
+        ? parseInt(traversedMatch[1].replace(/,/g, ''), 10)
+        : 0;
+        
+        let minPagesRead = traversedElements / 1024;
+        if (minPagesRead <= 1024) minPagesRead =1;
+        
+
+        // -------------------------------------------------------------------
+        // Sum all Cache misses values
+        // -------------------------------------------------------------------
+        let actualPagesRead = 0;
+
+        // Match rows that begin with a table name and then a cache miss number
+        const cacheMissRegex =
+        /^\s*[A-Za-z0-9:#()\-\s]+?\s+(\d+)\s+\d+\s+\d+\s+\d+\s+\d+/gm;
+
+        let match;
+
+        while ((match = cacheMissRegex.exec(text)) !== null) {
+        actualPagesRead += parseInt(match[1], 10);
+        }
+
+
+        let readAmp;
+        if (actualPagesRead < minPagesRead)
+            readAmp= "0%"
+        else
+            readAmp=  Math.floor((actualPagesRead-minPagesRead)/actualPagesRead  *100) + "%"
+
+        readAmp= numberWithCommas(traversedElements) + " / " + numberWithCommas(actualPagesRead)
+
+        console.log(`traversedElements = ${traversedElements}, minPagesRead = ${minPagesRead}, actualPagesRead =  ${actualPagesRead} ,, readAmp=${readAmp}` );
+
+        return readAmp
+      }
+            catch(ex){
+                console.error(ex)
+                return "n/a";
+            }
+    }
+
+
      
 
     </script>

package/index.mjs

@@ -7,7 +7,7 @@ import { AdminLog } from "./adminlog.mjs"
 import  logger  from "./logger.mjs"
 import  fetch  from "node-fetch"
 //update version
-let version = "0.0.84";
+let version = "0.0.86";
 const GENERATE_TOKEN_TIME_MIN = 30;
 
 let rl = null;
@@ -166,17 +166,21 @@ const inputs = {
             "subnetworks --deleted" : "Lists dirty and deleted subnetworks",
             "evaluate" : "Evaluate in parallel",
             "trace --subnetwork <subnetwork>": "Traces input subnetwork and returns the time and number of elements returned .",
+            "trace --all": "Traces all subnetworks.",
             "topology" : "Displays the topology status",
             "topology --disable" : "Disable topology",
             "topology --enable" : "Enable topology",
             "topology --validate" : "Validate topology (full extent)",
             "update subnetworks --subnetwork": "Update the input subnetwork synchronously",
             "update subnetworks --all": "Update all dirty subnetworks synchronously",
+            "update subnetworks --all --async": "Update all dirty subnetworks asynchronously",
             "update subnetworks --deleted": "Update all deleted dirty subnetworks synchronously",
             "update subnetworks --all --async": "Update all dirty subnetworks asynchronously",          
             "export subnetworks --all [--folder]": "Export all subnetworks with ACK --folder where exported files are saved",
             "export subnetworks --new [--folder]": "Export all subnetworks with ACK that haven't been exported --folder where exported files are saved",
             "export subnetworks --deleted": "Export all subnetworks with ACK that are deleted ",
+            "export subnetworks --all [--resulttype]": "Export all subnetworks with ACK --resulttype is a json configuration",
+
             "updateisconnected": "Run update is connected ",
             "versions": "List all versions available to the current logged in user.",
             "versions --summary": "Summary of versions.",
@@ -792,8 +796,8 @@ const inputs = {
             logger.info(`Result from submitting job ${JSON.stringify(subnetworkResult)}`)
         }
     },
-   "^export subnetworks --all --folder .*$|^export subnetworks --all$" : async input => {
- 
+   "^export subnetworks --all --folder .*$|^export subnetworks --all$|^export subnetworks --all --resulttype .*$" : async input => {
+  
         let subnetworks 
         let sort = "asc";
         if (input.indexOf("--desc") > 0) sort = "desc"
@@ -802,6 +806,27 @@ const inputs = {
         let inputDir = "Exported"
         if (file != null && file.length > 0)
              inputDir = file[0].replace("--folder ", "")
+
+        //default resulttype
+        let resultType = null;
+
+        const rt = input.match(/--resulttype .*/gm)
+        if (rt != null && v.length > 0)
+            resultType = rt[0].replace("--resulttype ", "")
+        
+        logger.info(resultType)
+
+        try
+        {
+            resultType = JSON.parse(resultType)
+        }
+        catch(ex)
+        {
+            logger.error("Can't parse resulttype" + ex)
+            return;
+        }
+
+        
         //create directory if doesn't exists
         if (!fs.existsSync(inputDir))  fs.mkdirSync(inputDir)
         let exportedSubnetworks = [];
@@ -823,7 +848,7 @@ const inputs = {
                 
                 const fromDate = new Date();
                 
-                const subnetworkResult = await un.exportSubnetworks(v(f.attributes,"domainNetworkName"), v(f.attributes,"tierName"), v(f.attributes,"subnetworkName"),false);
+                const subnetworkResult = await un.exportSubnetworks(v(f.attributes,"domainNetworkName"), v(f.attributes,"tierName"), v(f.attributes,"subnetworkName"),false, resultType);
                   
                 //code
                 exportedSubnetworks.push("'" + v(f.attributes,"subnetworkName") + "'")
@@ -845,10 +870,10 @@ const inputs = {
                 //although the response is json, its easier to treat it as text (handle error cases) since we will only write it to disk.
                 // if we want to do something with the response then make it json
                 const jsonExport = await subContent.text();
-                fs.writeFileSync(`${inputDir}/${subnetworkName}.json`, jsonExport)            
+                fs.writeFileSync(`${inputDir}/${v(f.attributes,"domainNetworkName")}_${v(f.attributes,"tierName")}_${subnetworkName}.json`, jsonExport)            
                
     
-                logger.info(`Result ${JSON.stringify(subnetworkResult)} written to file ${process.cwd()}/${inputDir}/${subnetworkName}.json`)
+                logger.info(`Result ${JSON.stringify(subnetworkResult)} written to file ${process.cwd()}/${inputDir}/${v(f.attributes,"domainNetworkName")}_${v(f.attributes,"tierName")}_${subnetworkName}.json`)
     
             }
         }
@@ -856,7 +881,55 @@ const inputs = {
        
     },
    
+ "^export subnetworks --all --async --folder .*$|^export subnetworks --all --async$|^export subnetworks --all --async --resulttype .*$" : async input => {
+ 
+        let subnetworks 
+        let sort = "asc";
+        if (input.indexOf("--desc") > 0) sort = "desc"
+        //create folder
+        const file = input.match(/--folder .*/gm)
+        let inputDir = "Exported"
+        if (file != null && file.length > 0)
+             inputDir = file[0].replace("--folder ", "")
+
+ 
+        //default resulttype
+        let resultType = null;
+        
+        const rt = input.match(/--resulttype .*/gm)
+        if (rt != null && v.length > 0)
+            resultType = rt[0].replace("--resulttype ", "")
+
+        logger.info(resultType)
+
+        //create directory if doesn't exists
+        if (!fs.existsSync(inputDir))  fs.mkdirSync(inputDir)
+        let exportedSubnetworks = [];
+
+        do {
+
+            let exportedSubnetworksWhereClause = ""
+            
+            if (exportedSubnetworks.length > 0 )
+                exportedSubnetworksWhereClause = " AND SUBNETWORKNAME NOT IN (" + exportedSubnetworks.join(",") + ")"
+ 
+            logger.info("Querying all subnetworks that are clean.");
+            subnetworks = await un.queryDistinct(500002, "domainnetworkname,tiername,subnetworkname", "isdirty=0 " + exportedSubnetworksWhereClause,`domainnetworkname  ${sort},tiername ${sort},subnetworkname ${sort}`);
+            logger.info(`Discovered ${subnetworks.features.length} subnetworks that can be exported.`);
+            for (let i = 0;  i < subnetworks.features.length; i++) {
+                const f = subnetworks.features[i]
+                const subnetworkName = v(f.attributes,"subnetworkName")
+                logger.info("Exporting subnetworks " + subnetworkName);
+                 
+                
+                const subnetworkResult = await un.exportSubnetworks(v(f.attributes,"domainNetworkName"), v(f.attributes,"tierName"), v(f.attributes,"subnetworkName"),true);
+                logger.info(`Result from submitting job ${JSON.stringify(subnetworkResult)}`)
 
+            }
+        }
+        while (subnetworks?.features?.length > 0)
+       
+    },
     "^export subnetworks --new --folder .*$|^export subnetworks --new$" : async input => {
 
         //create folder
@@ -879,9 +952,11 @@ const inputs = {
             //fetch the json and write it to disk 
             const subContent = await fetch(subnetworkResult.url);
             const jsonExport = await subContent.text();
-            fs.writeFileSync(`${inputDir}/${subnetworkName}.json`, JSON.stringify(jsonExport))            
-
-            logger.info(`Result ${JSON.stringify(subnetworkResult)} written to file ${process.cwd()}/${inputDir}/${subnetworkName}.json`)
+            fs.writeFileSync(`${inputDir}/${v(f.attributes,"domainNetworkName")}_${v(f.attributes,"tierName")}_${subnetworkName}.json`, jsonExport)            
+               
+    
+            logger.info(`Result ${JSON.stringify(subnetworkResult)} written to file ${process.cwd()}/${inputDir}/${v(f.attributes,"domainNetworkName")}_${v(f.attributes,"tierName")}_${subnetworkName}.json`)
+    
         }
 
  
@@ -968,19 +1043,68 @@ const inputs = {
             logger.error(JSON.stringify(ex))
         }
     },
-    "^export subnetworks --deleted$" : async input => {
+
+
+
+     "^trace --all": async input => {
+        //get subnetwork name
+        try {
+
+             
+        const inputDir= "Exported"
+        let full = true; 
+
+       
+        const subnetworks = await un.getSubnetworks();
+        if (subnetworks.features.length === 0) {
+            logger.info("No dirty subnetworks found.")
+            return;
+        }
+
+        const subs = subnetworks.features.map(a => a.attributes)
+        const rowCount = subs.length;
+        logger.info (`${numberWithCommas(rowCount)} subnetworks returned.`)
+        for (let i = 0; i < subs.length ; i++)
+        {
+            const fromDate = new Date();
+             const subnetworkName = v(subs[i],"subnetworkname")
+            logger.info(`Tracing subnetwork ${subnetworkName}`);
+            const result = await un.subnetworkTraceSimple(subnetworkName)
+            if (result == null) {
+                logger.info(`Subnetwork ${subnetworkName} doesn't exist`);
+                return null;
+            }
+            const toDate = new Date();
+            const timeRun = toDate.getTime() - fromDate.getTime();
+            const newResult = {}
+            newResult.duration =  numberWithCommas(Math.round(timeRun)) + " ms"
+            newResult.elementsCount = result.traceResults.elements.length;
+            console.table(newResult)
+
+        } 
+
+         
+  
+    }
+        catch(ex){
+            logger.error(JSON.stringify(ex))
+        }
+    },
+
+
+    "^export subnetworks --deleted --folder" : async input => {
 
         //create folder
-        const file = input.match(/-f .*/gm)
+        const file = input.match(/--folder .*/gm)
         let inputDir = "Exported"
         if (file != null && file.length > 0)
-             inputDir = file[0].replace("-f ", "")
+             inputDir = file[0].replace("--folder ", "")
         //create directory if doesn't exists
         if (!fs.existsSync(inputDir))  fs.mkdirSync(inputDir)
 
 
         logger.info("Querying all subnetworks that are clean and deleted.");
-        let subnetworks = await un.queryDistinct(500002, "domainnetworkname,tiername,subnetworkname", "isdirty = 0 and isdeleted=1");
+        let subnetworks = await un.queryDistinct(500002, "domainnetworkname,tiername,subnetworkname", "isdeleted=1", "subnetworkname");
         logger.info(`Discovered ${subnetworks.features.length} subnetworks that can be exported.`);
         for (let i = 0;  i < subnetworks.features.length; i++) {
             const f = subnetworks.features[i]
@@ -998,9 +1122,11 @@ const inputs = {
             //fetch the json and write it to disk 
             const subContent = await fetch(subnetworkResult.url);
             const jsonExport = await subContent.text();
-            fs.writeFileSync(`${inputDir}/${subnetworkName}.json`, JSON.stringify(jsonExport))            
-
-            logger.info(`Result ${JSON.stringify(subnetworkResult)} written to file ${process.cwd()}/${inputDir}/${subnetworkName}.json`)
+            fs.writeFileSync(`${inputDir}/${v(f.attributes,"domainNetworkName")}_${v(f.attributes,"tierName")}_${subnetworkName}.json`, jsonExport)            
+               
+    
+            logger.info(`Result ${JSON.stringify(subnetworkResult)} written to file ${process.cwd()}/${inputDir}/${v(f.attributes,"domainNetworkName")}_${v(f.attributes,"tierName")}_${subnetworkName}.json`)
+    
         }
 
  

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "un-cli",
-  "version": "0.0.84",
+  "version": "0.0.86",
   "description": "Command line interface for working with ArcGIS Utility Network Extension",
   "main": "index.mjs",
   "bin": {

package/utilitynetwork.node.mjs

@@ -280,6 +280,7 @@ export class UtilityNetwork {
                 orderByFields: "DOMAINNETWORKNAME, TIERNAME, SUBNETWORKNAME",
                 gdbVersion:this.gdbVersion,
                 returnDistinctValues: true,
+                resultRecordCount: 20000,
                 f: "json"
             }
             
@@ -1180,7 +1181,7 @@ export class UtilityNetwork {
         }
 
 
-        exportSubnetworks(domainNetworkName, tierName, subnetworkName, async=false) {
+        exportSubnetworks(domainNetworkName, tierName, subnetworkName, async=false,resultTypes=null) {
 
             let thisObj = this; 
             let tier = this.getTier(domainNetworkName, tierName);
@@ -1190,37 +1191,9 @@ export class UtilityNetwork {
              let ar = thisObj.featureServiceUrl.split("/");
              ar[ar.length-1]="UtilityNetworkServer";
              let exportsubnetworkUrl = ar.join("/") + "/exportSubnetwork"
-             const resultTypes = [
-                {
-                  "type": "features",
-                  "includeGeometry": true,
-                  "includePropagatedValues": false,
-                  "includeDomainDescriptions": true,
-                  "networkAttributeNames": [
-                    "Is subnetwork controller"
-                  ],
-                  "diagramTemplateName": "",
-                  "resultTypeFields": []
-                },
-                {
-                  "type": "connectivity",
-                  "includeGeometry": true,
-                  "includePropagatedValues": false,
-                  "includeDomainDescriptions": true,
-                  "networkAttributeNames": [],
-                  "diagramTemplateName": "",
-                  "resultTypeFields": []
-                },
-                {
-                  "type": "associations",
-                  "includeGeometry": false,
-                  "includePropagatedValues": false,
-                  "includeDomainDescriptions": true,
-                  "networkAttributeNames": [],
-                  "diagramTemplateName": "",
-                  "resultTypeFields": []
-                }
-              ]
+             if (!resultTypes)
+                resultTypes = [{"type":"features","includeGeometry":true,"includePropagatedValues":false,"includeDomainDescriptions":true,"networkAttributeNames":["Is subnetwork controller"],"diagramTemplateName":"","resultTypeFields":[]},{"type":"connectivity","includeGeometry":true,"includePropagatedValues":false,"includeDomainDescriptions":true,"networkAttributeNames":[],"diagramTemplateName":"","resultTypeFields":[]},{"type":"associations","includeGeometry":false,"includePropagatedValues":false,"includeDomainDescriptions":true,"networkAttributeNames":[],"diagramTemplateName":"","resultTypeFields":[]}]
+
              //traceConfiguration: JSON.stringify(subnetworkDef),   
                let exportsubnetworkJson = {
                   f: "json",
@@ -1231,7 +1204,7 @@ export class UtilityNetwork {
                   exportAcknowledgement: true,
                   allSubnetworksInTier: false,
                   continueOnFailure: false,
-                  traceConfiguration: subnetworkDef,   
+                  traceConfiguration: JSON.stringify(subnetworkDef),   
                   async: async,
                   gdbVersion:this.gdbVersion,
                   resultTypes: JSON.stringify(resultTypes)