un-cli 0.0.58 → 0.0.61

package/adminlog.mjs

@@ -13,8 +13,8 @@ export class AdminLog {
         this.token = token;         
     }
 
-    query (codes, serviceName ="*", pageSize = 10000) 
-    {
+    query (codes, serviceName ="*", pageSize = 100000, startTime = null, endTime = null) 
+    {   
         const url =  this.adminServerUrl  + "/logs/query?f=pjson"
         const level = "DEBUG"
         const filterType="json"
@@ -24,7 +24,13 @@ export class AdminLog {
             "services": serviceName
         }
       
-        const queryLogUrl = url + `&token=${token}&level=${level}&filterType=${filterType}&filter=${encodeURIComponent(JSON.stringify(filter))}&pageSize=${pageSize}`
+        let queryLogUrl = url + `&token=${token}&level=${level}&filterType=${filterType}&filter=${encodeURIComponent(JSON.stringify(filter))}&pageSize=${pageSize}`
+        if (startTime != null)
+            queryLogUrl += `&startTime=${startTime}`
+
+        if (endTime != null)
+            queryLogUrl += `&endTime=${endTime}`
+
         //logger.info(queryLogUrl);
         return fetch(queryLogUrl);
     }

package/cmd.txt

@@ -1 +1,4 @@
+subnetworks
+arlogs --byrule --maxguid
+arlogs --byrule --minguid
 arlogs --byrule

package/index.mjs

@@ -7,13 +7,10 @@ import { AdminLog } from "./adminlog.mjs"
 import { logger } from "./logger.mjs"
 import  fetch  from "node-fetch"
 //update version
-let version = "0.0.58";
+let version = "0.0.61";
 const GENERATE_TOKEN_TIME_MIN = 30;
 
 let rl = null;
-
-
-//uncli --portal https://utilitynetwork.esri.com/portal --service AllStar_oracle --user unadmin --password unadmin.108
 let portal = null;
 let un  = null;
 let adminLog = null;
@@ -29,9 +26,11 @@ function parseInput(){
           "--command",
           "--gdbversion",
           "--file",
-          "--verify"
+          "--verify",
+          "--server"
            ]     
-
+        
+      //null marked parmaters are required
       const params = {
           "portal": null,
           "service": null,
@@ -40,7 +39,8 @@ function parseInput(){
           "command": "",
           "gdbversion": "SDE.DEFAULT",
           "file": "",
-          "verify": "true"
+          "verify": "true",
+          "server": undefined
       }
 
       for (let i = 0; i < process.argv.length ; i++){
@@ -53,9 +53,10 @@ function parseInput(){
 
       if (Object.values(params).includes(null))
       {
-        console.log ("HELP: uncli --portal https://unportal.domain.com/portal --service servicename --user username --password password [--gdbversion* user.version --file commandfile* --verify true|false]")    
+        console.log ("HELP: uncli --portal https://unportal.domain.com/portal --service servicename --user username --password password [--gdbversion* user.version --server  https://federatedserver.domain.com/server --file commandfile* --verify true|false]")    
         console.log("--file commandfile is optional and you can pass a path to a file with a list of command to execute. ")
         console.log("--gdbversion is optional and allows the UN to be opened in that version. When not specified sde.DEFAULT is used.")
+        console.log("--server is optional except when there are more than one federated server sites to the portal. If the portal only has one server it will be selected.")
         process.exit();
       }
      
@@ -64,7 +65,7 @@ function parseInput(){
 
 //
 async function getToken(parameters) {
-    portal = new Portal(parameters.portal, parameters.user, parameters.password)
+    portal = new Portal(parameters.portal, parameters.user, parameters.password,300, parameters.server)
     logger.info("About to connect..")
     const token = await portal.connect()
     logger.info(`Token generanted successfully.`)
@@ -102,9 +103,9 @@ async function connect(parameters) {
     un = new UtilityNetwork(token, serviceUrl, parameters.gdbversion)
     //create a new admin object (user might not be admin we won't use it until the user call log )
     adminLog = new AdminLog(token, portal.serverUrl)
-    console.log("Loading utility network...")
+    logger.info("Loading utility network...")
     await un.load();
-    console.log("Connected.")
+    logger.info("Connected.")
     /*test here*/
  
     //if user specified file path open it read all commands and execute them
@@ -178,7 +179,7 @@ const inputs = {
             "count --system": "Lists the number of rows in system layers.",
             "connect --service": "Connects to the another service",
             "arlogs": "Lists attribute rules execution logs (requires admin)",
-            "arlogs --byrule": "Lists attribute rules execution summary by rule (requires admin)",
+            "arlogs --byrule [--minguid --maxguid]": "Lists attribute rules execution summary by rule (requires admin), --maxguid and --minguid show the GUID of the feature",
             "whoami": "Lists the current login info",
             "clear": "Clears this screen",
             "quit": "Exit this program"
@@ -673,10 +674,26 @@ const inputs = {
     },
 
     "^arlogs$": async () => {
+        const topLogCount = 200;
+        const pageSize = 10000
         console.log(`Querying attribute rules logs for ${parameters.service} ...`)
-        const result= await adminLog.query([102003], parameters.service)
-        const jsonRes = await result.json()
-        const arMessages = jsonRes.logMessages
+        console.log(`Displaying top ${topLogCount} entries only..`)
+
+        let result= await adminLog.query([102003], [parameters.service+ ".MapServer"], topLogCount)
+        let jsonRes = await result.json()
+        let allMessages = [].concat(jsonRes.logMessages)
+        
+        while (jsonRes.hasMore && allMessages.filter(m => m.message.indexOf("Attribute rule execution complete:") > -1).length < topLogCount )
+        {  
+            //start paging
+            logger.info(`Aggregating messages... total so far ${allMessages.length} debug entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
+            result= await adminLog.query([102003], [parameters.service + ".MapServer"], pageSize, jsonRes.endTime)
+            jsonRes = await result.json()
+            allMessages = allMessages.concat(jsonRes.logMessages)
+        }  
+
+
+        const arMessages = allMessages
             .filter(m => m.message.indexOf("Attribute rule execution complete:") > -1)
             .map (m =>  JSON.parse(m.message.replace("Attribute rule execution complete:", "")))
             .map( m => {
@@ -690,30 +707,41 @@ const inputs = {
                 return m
             })
             .sort( (m1, m2) => m2["Elapsed Time (ms)"]- m1["Elapsed Time (ms)"])
+            .slice(0, topLogCount);
         console.table(arMessages)
-        /*
-        code:102003
-        elapsed:''
-        machine:'DEV0015932.ESRI.COM'
-        message:'requestProperties = {"token":"HfauSFGoSwTMA5KLvsL-I8EORea_KEvz1GcAMNCsvTuzeJ1QuYbQm0EGI7eC2zr1lOm8857U18oZOjG0BeuEwKj7fvCk-_DuWKFvClU5p06SRLE8RjEzPB0gjMTFQnjVnTRmQzZFWXCj1VRssMECQg..","referer":null,"privilege":"ADMINISTER","privileges":["features:user:edit","features:user:fullEdit","features:user:manageVersions","portal:user:viewTracks","premium:user:geocode:stored","premium:user:geocode:temporary","premium:user:networkanalysis:closestfacility","premium:user:networkanalysis:locationallocation","premium:user:networkanalysis:optimizedrouting","premium:user:networkanalysis:origindestinationcostmatrix","premium:user:networkanalysis:routing","premium:user:networkanalysis:servicearea","premium:user:networkanalysis:vehiclerouting","traceNetwork","utilityNetwork","parcelFabric"],"securityProvider":"portal"}'
-        methodName:'GetServerEnvironmentRequestProperties'
-        process:'30940'
-        requestID:'fccb7fba-cba9-4ebb-84b2-cb3645979d8e'
-        source:'RedTrolley_Postgres.MapServer'
-        thread:'29408'
-        time:1634165691074
-        type:'DEBUG'
-        user:'unadmin'
-        */
+       
          
     },
 
 
-    "^arlogs --byrule$": async () => {
-        console.log(`Querying attribute rules logs for ${parameters.service} ...`)
-        const result= await adminLog.query([102003], parameters.service)
-        const jsonRes = await result.json()
-        const arMessages = jsonRes.logMessages
+    "^arlogs --byrule": async input => {
+        //--minguid to show min guid
+        //--maxguid to show max guid
+        const inputParam = input.match(/--byrule .*/gm)
+        let showMaxGuid = false
+        let showMinGuid = false
+        if (inputParam != null && inputParam.length > 0 && inputParam[0].indexOf("--maxguid") > -1)
+            showMaxGuid = true
+
+        if (inputParam != null && inputParam.length > 0 && inputParam[0].indexOf("--minguid") > -1)
+            showMinGuid = true
+
+        const pageSize = 10000 //maximum messages per page
+        logger.info(`Querying attribute rules logs for ${parameters.service} ...`)
+        let result= await adminLog.query([102003], [parameters.service + ".MapServer"], pageSize)
+        let jsonRes = await result.json()
+        let allMessages = [].concat(jsonRes.logMessages)
+        
+        while (jsonRes.hasMore)
+        {  
+            //start paging
+            logger.info(`Aggregating messages... total so far ${allMessages.length} debug entries but more left, pulling logs before ${new Date(jsonRes.endTime)}`)
+            result= await adminLog.query([102003], [parameters.service + ".MapServer"], pageSize, jsonRes.endTime)
+            jsonRes = await result.json()
+            allMessages = allMessages.concat(jsonRes.logMessages)
+        }  
+
+        const arMessages = allMessages
             .filter(m => m.message.indexOf("Attribute rule execution complete:") > -1)
             .map (m =>  JSON.parse(m.message.replace("Attribute rule execution complete:", "")))
             .map( m => {
@@ -729,30 +757,62 @@ const inputs = {
             .sort( (m1, m2) => m2["Elapsed Time (ms)"]- m1["Elapsed Time (ms)"])
             .reduce( ( prev, cur ) => {
                 if (prev [cur["Rule name"]] === undefined)
-                    prev [cur["Rule name"]] = 0;
+                   { 
+                       prev [cur["Rule name"]] =  {
+                           "totalTime": 0,
+                           "occurrence": 0,
+                           "minTime":  Number.MAX_SAFE_INTEGER,
+                           "maxTime": -1,
+                           "avgTime": 0,
+                           "maxGuid": null,
+                           "minGuid": null
+                       };
+                     }
+                
+                     prev [cur["Rule name"]].totalTime = prev [cur["Rule name"]].totalTime + cur["Elapsed Time (ms)"]
+                      
+                     if (cur["Elapsed Time (ms)"] < prev [cur["Rule name"]].minTime ) {
+                        prev [cur["Rule name"]].minTime = cur["Elapsed Time (ms)"];
+                        prev [cur["Rule name"]].minGuid = cur["GlobalID"];
+                     }
+
+                    if (cur["Elapsed Time (ms)"] > prev [cur["Rule name"]].maxTime ){
+                        prev [cur["Rule name"]].maxTime = cur["Elapsed Time (ms)"];
+                        prev [cur["Rule name"]].maxGuid = cur["GlobalID"];
+                    }
+                     
+                    prev [cur["Rule name"]].occurrence++
+                     
+                    prev [cur["Rule name"]].avgTime = prev [cur["Rule name"]].totalTime / prev [cur["Rule name"]].occurrence
 
-                prev [cur["Rule name"]] = cur["Elapsed Time (ms)"]
                 return prev
             }, {})
 
         const rules = Object.keys(arMessages)
         .map(a => {
-
                 const rule = {}
                 rule["Attribute Rule"] = a;
-                rule["Total Cost (ms)"] = arMessages[a];
+                rule["Total Cost (ms)"] = parseFloat(arMessages[a].totalTime.toFixed(2))
+                if (!showMinGuid && !showMaxGuid) rule["Average Cost (ms)"] = parseFloat(arMessages[a].avgTime.toFixed(2))
+                rule["Max execution time (ms)"] = parseFloat(arMessages[a].maxTime.toFixed(2))
+                if (showMaxGuid) rule["Max GUID"] = arMessages[a].maxGuid
+                rule["Min execution time (ms)"] = parseFloat(arMessages[a].minTime.toFixed(2))
+                if (showMinGuid) rule["Min GUID"] = arMessages[a].minGuid
+                if (!showMinGuid && !showMaxGuid) rule["Occurrence"] = arMessages[a].occurrence;
                 return rule;
         })
         .sort( (m1, m2) => m2["Total Cost (ms)"] -m1["Total Cost (ms)"])
         console.table(rules)
-       
+
+        const totalARExecution = rules.reduce( (prev, cur) =>  prev + cur["Total Cost (ms)"], 0)
+        console.log(`Total time spend executing attribute rules (${Math.round(totalARExecution)} ms) (${Math.round(totalARExecution/1000)} s) (${Math.round(totalARExecution/(1000*60))} m)`)
          
     },
 
     "^version$": () => console.log(version),
     "^clear$|^cls$": () => console.clear(),
     "^quit$": () => {
-        rl.close();
+        if (rl) rl.close();
         process.exit();
     },
     "^exit$|^quit$|^bye$": () => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "un-cli",
-  "version": "0.0.58",
+  "version": "0.0.61",
   "description": "Command line interface for working with ArcGIS Utility Network Extension",
   "main": "index.mjs",
   "bin": {

package/portal.mjs

@@ -3,13 +3,13 @@ import { logger } from "./logger.mjs"
 
 export class Portal{
         
-    constructor(url, username, password, expiration = 300)
+    constructor(url, username, password, expiration = 300, serverUrl = undefined)
     {
         this.url = url;
         this.username = username;
         this.password = password;
         this.expiration = expiration;
-        this.serverUrl = "bad";
+        this.serverUrl = serverUrl;
     }
 
 
@@ -20,7 +20,6 @@ export class Portal{
         
                 try { 
 
-                    //https://utilitynetwork.esri.com/portal/sharing/rest/portals/self/servers?f=json
                     const tokenUrl = self.url + "/sharing/rest/generateToken";
                    
                     const postJson = {
@@ -44,9 +43,9 @@ export class Portal{
                   
             }
             catch(ex){
-                logger.error(ex.status.message)
-                console.error(ex.status.errno)
-                reject(`Failed to connect to portal, check your username or password or add --verify false if you are using a self-signed certificate. Normally a production system should have a valid certificate signed by a CA and you should NOT disable verification in that case.)`)
+                logger.error(ex?.status?.message)
+                console.error(ex?.status?.errno)
+                reject(`Failed to connect to portal, check your username or password or add --verify false if you are using a self-signed certificate. Normally a production system should have a valid certificate signed by a CA and you should NOT disable verification in that case.) \n\n${ex}`)
             }
 
             }
@@ -55,32 +54,54 @@ export class Portal{
     }
  
     async updateServices () {
+        const self = this;
+        return new Promise( async (resolve, reject) => {
+            //if the user specified a serverUrl no need to do anything
+            if (self.serverUrl !== undefined) {
+                logger.info(`Using server ${self.serverUrl} supplied in the --server parameter`)
+                resolve(self.serverUrl);
+                return;
+            }
 
-        try {
+            //else we need to calculate it
+            try {
 
-            const postJsonServers= {
-                f: "json",
-                token: this.token
+                const postJsonServers= {
+                    f: "json",
+                    token: self.token
+                }
+        
+                const serversUrl =  self.url + "/sharing/rest/portals/self/servers"
+                logger.info( "About to query federated servers");
+                    
+                //query for federated servers.
+                const servers = await makeRequest({method: 'POST', url: serversUrl, params: postJsonServers });
+                
+                //if we don't have any federated servers quit.
+                if (servers.servers.length === 0) 
+                    {
+                        reject( "No federeated servers");
+                    return
+                }
+                
+                //if we have more than one then we let the user pick.
+                if (servers.servers.length > 1){
+                    let serverUrls = "";
+                    servers.servers.forEach(s => serverUrls += '\n  * ' + s.url + '\n' ) 
+                    reject("more than one federated server found, run the command with --server and specify one of the servers below\n" + serverUrls)
+                }
+                this.serverUrl = servers.servers[0].url;
+                resolve(self.serverUrl)
+                logger.info(`Found one federate server, using server url ${self.serverUrl} by default`)
             }
-    
-            const serversUrl =  this.url + "/sharing/rest/portals/self/servers"
-            logger.info( "About to query federated servers");
-                  
-            //query for federated servers.
-            const servers = await makeRequest({method: 'POST', url: serversUrl, params: postJsonServers });
-            
-            //get the first one
-            if (servers.servers.length === 0) 
-                reject( "No federeated servers");
-    
-            this.serverUrl = servers.servers[0].url;
-            logger.info(`Found server url ${this.serverUrl}`)
-        }
-       catch(ex){
-           logger.error(ex)
-       }
+        catch(ex){
+            logger.error(ex)
+            reject(ex)
+            }       
+         });
+
+    } 
 
-    }
     //get the feature service definition 
     async serviceDef(serviceName) {
         this.token = this.token;
@@ -109,9 +130,20 @@ export class Portal{
                         token: this.token,
                         f: "json"
                     }
-             
+                    
+                    let allServices = []
                     const services = await makeRequest({method: 'POST', url: servicesUrl, params: postJson })
-                    resolve(services);
+                    allServices = allServices.concat(services.services);
+                    const folders = services.folders;
+                    for (let f = 0; f < folders.length; f++)
+                     {
+                        const folderUrl = servicesUrl + "/" + folders[f];
+                        const folderServices = await makeRequest({method: 'POST', url: folderUrl, params: postJson })
+                        allServices = allServices.concat(folderServices.services)
+                    }
+                        
+                         
+                    resolve({"services": allServices});
                 }
                 catch(ex) {
                     reject(ex)