un-cli 0.0.62 → 0.0.66

package/README.md

@@ -18,26 +18,44 @@ npm install -g un-cli
 ```
 
 ```bash
+
 uncli> help
-┌─────────────────────────┬───────────────────────────────────────────────────────────────┐
-│         (index)         │                            Values                             │
-├─────────────────────────┼───────────────────────────────────────────────────────────────┤
-│          help           │                     'Displays this help'                      │
-│         version         │                'Displays the version of uncli'                │
-│           ls            │                      'List all services'                      │
-│           def           │             'Show the feature service definition'             │
-│       def -layers       │               'List all layers in this service'               │
-│       subnetworks       │                    'Lists all subnetworks'                    │
-│     subnetworks -d      │                'Lists only dirty subnetworks'                 │
-│ update subnetworks -all │                'Update all dirty subnetworks '                │
-│ export subnetworks -all │              'Export all subnetworks with ACK '               │
-│ export subnetworks -new │ "Export all subnetworks with ACK that haven't been exported " │
-│          count          │       'Lists the number of rows in all feature layers.'       │
-│      count -system      │         'Lists the number of rows in system layers.'          │
-│         whoami          │                 'List the current login info'                 │
-│          clear          │                     'Clears this screen'                      │
-│          quit           │                      'Exit this program'                      │
-└─────────────────────────┴───────────────────────────────────────────────────────────────┘
+┌───────────────────────────────────────┬──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
+│                (index)                │                                                          Values                                                          │
+├───────────────────────────────────────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
+│                 help                  │                                                   'Displays this help'                                                   │
+│                version                │                                             'Displays the version of uncli'                                              │
+│                  ls                   │                                                   'List all services'                                                    │
+│                  def                  │                                          'Show the feature service definition'                                           │
+│             def --layers              │                                            'List all layers in this service'                                             │
+│              subnetworks              │                                                 'Lists all subnetworks'                                                  │
+│          subnetworks --dirty          │                                              'Lists only dirty subnetworks'                                              │
+│         subnetworks --deleted         │                                          'Lists dirty and deleted subnetworks'                                           │
+│               evaluate                │                                                  'Evaluate in parallel'                                                  │
+│    trace --subnetwork <subnetwork>    │                     'Traces input subnetwork and returns the time and number of elements returned .'                     │
+│               topology                │                                              'Displays the topology status'                                              │
+│          topology --disable           │                                                    'Disable topology'                                                    │
+│           topology --enable           │                                                    'Enable topology'                                                     │
+│          topology --validate          │                                            'Validate topology (full extent)'                                             │
+│       update subnetworks --all        │                                       'Update all dirty subnetworks synchronously'                                       │
+│     update subnetworks --deleted      │                                   'Update all deleted dirty subnetworks synchronously'                                   │
+│   update subnetworks --all --async    │                                      'Update all dirty subnetworks asynchronously'                                       │
+│       export subnetworks --all        │                                            'Export all subnetworks with ACK '                                            │
+│       export subnetworks --new        │                              "Export all subnetworks with ACK that haven't been exported "                               │
+│     export subnetworks --deleted      │                                   'Export all subnetworks with ACK that are deleted '                                    │
+│                 count                 │                                    'Lists the number of rows in all feature layers.'                                     │
+│            count --system             │                                       'Lists the number of rows in system layers.'                                       │
+│           connect --service           │                                            'Connects to the another service'                                             │
+│       tracelogs --age <minutes>       │                    'Lists utility network trace summary logs for the last x minutes (requires admin)'                    │
+│     validatelogs --age <minutes>      │                  'Lists utility network validate summary logs for the last x minutes (requires admin)'                   │
+│ updatesubnetworkslog --age <minutes>  │             'Lists utility network update subnetworks summary logs for the last x minutes (requires admin)'              │
+│        arlogs --age <minutes>         │                     'Lists attribute rules execution logs for the last x minutes  (requires admin)'                      │
+│ arlogs --byrule [--minguid --maxguid] │ 'Lists attribute rules execution summary by rule (requires admin), --maxguid and --minguid show the GUID of the feature' │
+│        topsql --age <minutes>         │                           'Lists all queries executed in the last x minutes  (requires admin)'                           │
+│                whoami                 │                                              'Lists the current login info'                                              │
+│                 clear                 │                                                   'Clears this screen'                                                   │
+│                 quit                  │                                                   'Exit this program'                                                    │
+└───────────────────────────────────────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
 ```
 
 

package/cmd.txt

@@ -1 +1,2 @@
-tracelogs --m 300
+export subnetworks --all --folder c:\temp
+exit

package/index.mjs

@@ -7,7 +7,7 @@ import { AdminLog } from "./adminlog.mjs"
 import { logger } from "./logger.mjs"
 import  fetch  from "node-fetch"
 //update version
-let version = "0.0.62";
+let version = "0.0.66";
 const GENERATE_TOKEN_TIME_MIN = 30;
 
 let rl = null;
@@ -178,9 +178,13 @@ const inputs = {
             "count": "Lists the number of rows in all feature layers.",
             "count --system": "Lists the number of rows in system layers.",
             "connect --service": "Connects to the another service",
-            "tracelogs --m <minutes>": "Lists utility network trace summary logs for the last x minutes (requires admin)",
-            "arlogs": "Lists attribute rules execution logs (requires admin)",
+            "tracelogs --age <minutes>": "Lists utility network trace summary logs for the last x minutes (requires admin)",
+            "validatelogs --age <minutes>": "Lists utility network validate summary logs for the last x minutes (requires admin)",
+            "updatesubnetworkslog --age <minutes>": "Lists utility network update subnetworks summary logs for the last x minutes (requires admin)",
+            "arlogs --age <minutes>": "Lists attribute rules execution logs for the last x minutes  (requires admin)",
             "arlogs --byrule [--minguid --maxguid]": "Lists attribute rules execution summary by rule (requires admin), --maxguid and --minguid show the GUID of the feature",
+            "topsql --age <minutes>": "Lists all queries executed in the last x minutes  (requires admin)",
+
             "whoami": "Lists the current login info",
             "clear": "Clears this screen",
             "quit": "Exit this program"
@@ -438,14 +442,14 @@ const inputs = {
             console.log(`Result from submitting job ${JSON.stringify(subnetworkResult)}`)
         }
     },
-    "^export subnetworks --all$" : async input => {
+   "^export subnetworks --all --folder .*$|^export subnetworks --all$" : async input => {
 
         
         //create folder
-        const file = input.match(/-f .*/gm)
+        const file = input.match(/--folder .*/gm)
         let inputDir = "Exported"
         if (file != null && file.length > 0)
-             inputDir = file[0].replace("-f ", "")
+             inputDir = file[0].replace("--folder ", "")
         //create directory if doesn't exists
         if (!fs.existsSync(inputDir))  fs.mkdirSync(inputDir)
 
@@ -467,7 +471,7 @@ const inputs = {
 
             const toDate = new Date();
             const timeEnable = toDate.getTime() - fromDate.getTime();
-            subnetworkResult.duration =  numberWithCommas(Math.round(timeEnable/1000)) + " s"
+            subnetworkResult.duration =  numberWithCommas(timeEnable) + " ms"
             
 
             //fetch the json and write it to disk 
@@ -674,17 +678,27 @@ const inputs = {
 
     },
 
-    "^arlogs$": async () => {
+    "^arlogs --age": async input => {
         const topLogCount = 200;
         const pageSize = 10000
-        console.log(`Querying attribute rules logs for ${parameters.service} ...`)
-        console.log(`Displaying top ${topLogCount} entries only..`)
 
-        let result= await adminLog.query([102003], [parameters.service+ ".MapServer"], topLogCount)
+        
+        const inputParam = input.match(/--age .*/gm)
+        let mins = 30;  //query logs for the last 30 minutes
+        if (inputParam != null && inputParam.length > 0)
+            mins = inputParam[0].replace("--age ", "")
+
+
+       console.log(`Querying attribute rules logs for ${parameters.service} for the last ${mins} minutes ...`)
+
+       const startTime = Date.now() - mins*60*1000
+       const endTime = Date.now();
+
+        let result= await adminLog.query([102003], [parameters.service+ ".MapServer"], topLogCount, startTime, endTime , "DEBUG")
         let jsonRes = await result.json()
         let allMessages = [].concat(jsonRes.logMessages)
         
-        while (jsonRes.hasMore && allMessages.filter(m => m.message.indexOf("Attribute rule execution complete:") > -1).length < topLogCount )
+        while (jsonRes.hasMore  )
         {  
             //start paging
             logger.info(`Aggregating messages... total so far ${allMessages.length} debug entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
@@ -714,37 +728,255 @@ const inputs = {
          
     },
 
-
-
+ 
     
-    "^tracelogs --m": async input => {
+    "^tracelogs --age": async input => {
         const topLogCount = 1000;
         const pageSize = 10000
 
-        const inputParam = input.match(/--m .*/gm)
+        const inputParam = input.match(/--age .*/gm)
         let mins = 30;  //query logs for the last 30 minutes
         if (inputParam != null && inputParam.length > 0)
-            mins = inputParam[0].replace("--m ", "")
+            mins = inputParam[0].replace("--age ", "")
  
         console.log(`Querying trace logs for ${parameters.service} for the last ${mins} minutes ...`)
         const startTime = Date.now() - mins*60*1000
         const endTime = Date.now();
         let result= await adminLog.query([102002], [parameters.service+ ".MapServer"], topLogCount, startTime ,endTime , "VERBOSE")
         let jsonRes = await result.json()
-        let allMessages = [].concat(jsonRes.logMessages)
-        
-        while (jsonRes.hasMore && allMessages.filter(m => m.message.indexOf(" Environment -") > -1).length < topLogCount )
+        let allMessages = [].concat(jsonRes.logMessages) 
+        allMessages = allMessages.filter(m => m.message.indexOf("------ Trace Parameters ----") > -1)
+        while (jsonRes.hasMore)
         {  
             //start paging
-            logger.info(`Aggregating messages... total so far ${allMessages.length} debug entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
-            result= await adminLog.query([102002], [parameters.service + ".MapServer"], pageSize, jsonRes.endTime)
+            logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
+            result= await adminLog.query([102002], [parameters.service + ".MapServer"], pageSize, jsonRes.endTime, null, "VERBOSE")
             jsonRes = await result.json()
-            allMessages = allMessages.concat(jsonRes.logMessages)
+  
+            allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("------ Trace Parameters ----") > -1))
+        }  
+        allMessages.forEach(m => {
+            const newMessage = Object.assign({}, m);
+            delete newMessage.message
+            delete newMessage.source;
+            delete newMessage.machine;
+            delete newMessage.type;
+            delete newMessage.requestID;
+            delete newMessage.thread;
+            delete newMessage.time;
+
+            console.table([newMessage])
+            console.log(m.message)  
+        })
+         
+    },
+
+
+    
+    "^validatelogs --age": async input => {
+      
+        const topLogCount = 1000;
+        const pageSize = 10000
+
+        const inputParam = input.match(/--age .*/gm)
+        let mins = 30;  //query logs for the last 30 minutes
+        if (inputParam != null && inputParam.length > 0)
+            mins = inputParam[0].replace("--age ", "")
+ 
+        console.log(`Querying validate logs for ${parameters.service} for the last ${mins} minutes ...`)
+        const startTime = Date.now() - mins*60*1000
+        const endTime = Date.now();
+        let result= await adminLog.query([102003], [parameters.service+ ".MapServer"], topLogCount, startTime ,endTime , "VERBOSE")
+        let jsonRes = await result.json()
+        let allMessages = [].concat(jsonRes.logMessages) 
+        allMessages = allMessages.filter(m => m.message.indexOf("-------- Environment ---") > -1 && m.message.indexOf("------ Trace Parameters ----") < 0)
+        while (jsonRes.hasMore)
+        {  
+            //start paging
+            logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
+            result= await adminLog.query([102003], [parameters.service + ".MapServer"], pageSize, jsonRes.endTime, null, "VERBOSE")
+            jsonRes = await result.json()
+  
+            allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("-------- Environment ---") > -1 && m.message.indexOf("------ Trace Parameters ----") == -1))
         }  
 
-        allMessages.forEach(m => console.log(m.message))
+    
+        allMessages.forEach(m => {
+            const newMessage = Object.assign({}, m);
+            delete newMessage.message
+            delete newMessage.source;
+            delete newMessage.machine;
+            delete newMessage.type;
+            delete newMessage.requestID;
+            delete newMessage.thread;
+            delete newMessage.time;
+
+            console.table([newMessage])
+            console.log(m.message)  
+        })
+
+    },
+
+
+
+    
+    "^updatesubnetworkslogs --age": async input => {
+        const topLogCount = 1000;
+        const pageSize = 10000
+
+        const inputParam = input.match(/--age .*/gm)
+        let mins = 30;  //query logs for the last 30 minutes
+        if (inputParam != null && inputParam.length > 0)
+            mins = inputParam[0].replace("--age ", "")
+ 
+        console.log(`Querying subnetwork logs for ${parameters.service} for the last ${mins} minutes ...`)
+        const startTime = Date.now() - mins*60*1000
+        const endTime = Date.now();
+        let result= await adminLog.query([102003], [parameters.service+ ".MapServer"], topLogCount, startTime ,endTime , "VERBOSE")
+        let jsonRes = await result.json()
+        let allMessages = [].concat(jsonRes.logMessages) 
+        allMessages = allMessages.filter(m => m.message.indexOf("---- Subnetwork Parameters ----") > -1)
+        while (jsonRes.hasMore)
+        {  
+            //start paging
+            logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
+            result= await adminLog.query([102003], [parameters.service + ".MapServer"], pageSize, jsonRes.endTime, null, "VERBOSE")
+            jsonRes = await result.json()
+  
+            allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("---- Subnetwork Parameters ----") > -1))
+        }  
+        allMessages.forEach(m => {
+            const newMessage = Object.assign({}, m);
+            delete newMessage.message
+            delete newMessage.source;
+            delete newMessage.machine;
+            delete newMessage.type;
+            delete newMessage.requestID;
+            delete newMessage.thread;
+            delete newMessage.time;
+
+            console.table([newMessage])
+            console.log(m.message)  
+        })
          
     },
+
+
+    
+    "^topsql --age": async input => {
+        const topLogCount = 1000;
+        const pageSize = 10000
+
+        const inputParam = input.match(/--age .*/gm)
+        let mins = 30;  //query logs for the last 30 minutes
+        if (inputParam != null && inputParam.length > 0)
+            mins = inputParam[0].replace("--age ", "")
+ 
+        console.log(`Querying cursor sql logs for ${parameters.service} for the last ${mins} minutes ...`)
+        const startTime = Date.now() - mins*60*1000
+        const endTime = Date.now();
+        let result= await adminLog.query([102023], [parameters.service+ ".MapServer"], topLogCount, startTime ,endTime , "DEBUG")
+        let jsonRes = await result.json()
+        let allMessages = [].concat(jsonRes.logMessages) 
+        allMessages = allMessages.filter(m => m.message.indexOf("EndCursor;") > -1)
+        while (jsonRes.hasMore)
+        {  
+            //start paging
+            logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
+            result= await adminLog.query([102023], [parameters.service + ".MapServer"], pageSize, jsonRes.endTime, null, "DEBUG")
+            jsonRes = await result.json()
+  
+            allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("EndCursor;") > -1))
+        }
+      console.log ("Filtering messages...")
+      allMessages = allMessages
+            .map( m=> {
+                m.dataAccessElapsed = parseFloat(m.message.split(";")[1].split(" ")[1])
+                m.ExecuteQueryElapsed = parseFloat(m.message.split(";")[2].split(" ")[1])
+                m.totalTimeElapsed = m.dataAccessElapsed +  m.ExecuteQueryElapsed 
+                m.elapsed = parseFloat(m.elapsed); return m;
+
+            })
+            .sort( (m1,m2) => m2.totalTimeElapsed - m1.totalTimeElapsed)
+            .slice(0, 10) ;//first 10
+    
+     
+        console.log("-----Top 10 SQL----")
+        let i =0;
+        allMessages= allMessages.forEach(m => 
+            {
+             
+                const x = m.message.split(";")
+                console.log(`id: ${i++}`)
+                console.log(`\tAt: ${new Date(m.time)} (${m.time})`)
+                console.log(`\tUser: ${m.user}`)
+                console.log(`\tElapsed: ${m.elapsed*1000} ms`)
+                console.log(`\ttotalTimeElapsed: ${m.totalTimeElapsed} ms`)
+                console.log(`\tQuery:`)
+                x.forEach(a => console.log(`\t${a}`))
+                console.log(`\n`)
+ 
+        })
+ 
+    },
+        
+
+    
+    "^cursorlogs --age": async input => {
+        const topLogCount = 1000;
+        const pageSize = 10000
+
+        const inputParam = input.match(/--age .*/gm)
+        let mins = 30;  //query logs for the last 30 minutes
+        if (inputParam != null && inputParam.length > 0)
+            mins = inputParam[0].replace("--age ", "")
+ 
+        console.log(`Querying cursor sql logs for ${parameters.service} for the last ${mins} minutes ...`)
+        const startTime = Date.now() - mins*60*1000
+        const endTime = Date.now();
+        let result= await adminLog.query([102023], [parameters.service+ ".MapServer"], topLogCount, startTime ,endTime , "DEBUG")
+        let jsonRes = await result.json()
+        let allMessages = [].concat(jsonRes.logMessages) 
+        allMessages = allMessages.filter(m => m.message.indexOf("EndCursor;") > -1)
+        while (jsonRes.hasMore)
+        {  
+            //start paging
+            logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
+            result= await adminLog.query([102023], [parameters.service + ".MapServer"], pageSize, jsonRes.endTime, null, "DEBUG")
+            jsonRes = await result.json()
+  
+            allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("EndCursor;") > -1))
+        }  
+
+
+     //   
+     allMessages= allMessages.map(m => {
+
+            const r= m.message.split (";")
+            delete r[0];
+            delete r[1];
+            delete r[3];
+            return r;
+            
+            const newMessage = Object.assign({}, m);
+        
+            delete newMessage.source;
+            delete newMessage.machine;
+            delete newMessage.type;
+            delete newMessage.requestID;
+            delete newMessage.methodName;
+            delete newMessage.process;
+            delete newMessage.thread;
+            delete newMessage.time;
+            delete newMessage.code;
+
+            return newMessage
+ 
+        })
+        console.table(allMessages)
+    },
+
+
     "^arlogs --byrule": async input => {
         //--minguid to show min guid
         //--maxguid to show max guid

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "un-cli",
-  "version": "0.0.62",
+  "version": "0.0.66",
   "description": "Command line interface for working with ArcGIS Utility Network Extension",
   "main": "index.mjs",
   "bin": {

package/utilitynetwork.node.mjs

@@ -997,6 +997,37 @@ export class UtilityNetwork {
              let ar = thisObj.featureServiceUrl.split("/");
              ar[ar.length-1]="UtilityNetworkServer";
              let exportsubnetworkUrl = ar.join("/") + "/exportSubnetwork"
+             const resultTypes = [
+                {
+                  "type": "features",
+                  "includeGeometry": true,
+                  "includePropagatedValues": false,
+                  "includeDomainDescriptions": true,
+                  "networkAttributeNames": [
+                    "Is subnetwork controller"
+                  ],
+                  "diagramTemplateName": "",
+                  "resultTypeFields": []
+                },
+                {
+                  "type": "connectivity",
+                  "includeGeometry": true,
+                  "includePropagatedValues": false,
+                  "includeDomainDescriptions": true,
+                  "networkAttributeNames": [],
+                  "diagramTemplateName": "",
+                  "resultTypeFields": []
+                },
+                {
+                  "type": "associations",
+                  "includeGeometry": false,
+                  "includePropagatedValues": false,
+                  "includeDomainDescriptions": true,
+                  "networkAttributeNames": [],
+                  "diagramTemplateName": "",
+                  "resultTypeFields": []
+                }
+              ]
              //traceConfiguration: JSON.stringify(subnetworkDef),   
                let exportsubnetworkJson = {
                   f: "json",
@@ -1007,9 +1038,10 @@ export class UtilityNetwork {
                   exportAcknowledgement: true,
                   allSubnetworksInTier: false,
                   continueOnFailure: false,
-                  traceConfiguration: subnetworkDef,               
+                  traceConfiguration: subnetworkDef,   
                   async: async,
                   gdbVersion:this.gdbVersion,
+                  resultTypes: JSON.stringify(resultTypes)
                 }
               let un = this;