un-cli 0.0.72 → 0.0.75

package/cmd.txt

@@ -1,2 +1,2 @@
-update subnetworks --all --desc
+updateisconnected
 exit

package/index.html

@@ -87,12 +87,14 @@
            
         cmbService = document.getElementById("cmbService")
         while (cmbService.firstChild) cmbService.removeChild(cmbService.firstChild)
-        
+         
+        token = await getToken(parameters);
+
+        //if we logged in successfully 
         const opt = document.createElement("option")
         opt.textContent = "(All Services)"
         cmbService.appendChild (opt)
 
-        token = await getToken(parameters);
         const services = await portal.services()
         const featureServices = services.services.filter(s => s.type == "FeatureServer")
         featureServices.forEach(s=> {
@@ -390,16 +392,16 @@ async function loadAttributeRules () {
 }
 async function loadSQLLogs () {
 
-//build table
-const tblResult = document.getElementById("tblResult");
-//clear
-while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
- 
+    //build table
+    const tblResult = document.getElementById("tblResult");
+    //clear
+    while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
+        
 
-  parameters.service = cmbService.options[cmbService.selectedIndex].text
-   
-  
-  let mins = document.getElementById("txtAge").value  //query logs for the last 30 minutes
+    parameters.service = cmbService.options[cmbService.selectedIndex].text
+    
+    
+    let mins = document.getElementById("txtAge").value  //query logs for the last 30 minutes
 
   
     console.log(`Querying cursor sql logs for ${parameters.service} for the last ${mins} minutes ...`)
@@ -407,30 +409,7 @@ while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
         //startTime is the most recent
         //endTime is the oldest
 
-        
-
-    /*
-     const topLogCount = 100;
-        const startTime = Date.now()
-        const endTime = Date.now() - mins*60*1000
-        let services = [theService + ".MapServer"] 
-        if (theService == "(All Services)")
-            services = "*"
-        let result= await adminLog.query([code], services, topLogCount, startTime ,endTime , "DEBUG")
-        let jsonRes = await result.json()
-        let allMessages = [].concat(jsonRes.logMessages) 
-        allMessages = allMessages.filter(m => m.message.indexOf("EndCursor;") > -1)
-        while (jsonRes.hasMore)
-        {  
-            //start paging
-            logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs between ${new Date(jsonRes.endTime)} and ${new Date(endTime)}`)
-            result= await adminLog.query([102023], services, pageSize, jsonRes.endTime, endTime, "DEBUG")
-            jsonRes = await result.json()
-  
-            allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("EndCursor;") > -1))
-        }
-        */
-
+         
      //page query the admin log
       let allMessages = await adminLogQueryWithPaging(mins, parameters.service, [102023], "EndCursor;", "DEBUG")
 
@@ -534,7 +513,7 @@ while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
                 const cLog = document.createElement("td");
                 const cLogText = document.createElement("input")
                 cLogText.type = "text"
-                cLogText.style = "width:500px"
+                cLogText.style = "width:800px"
                 cLogText.value = m.message.substr( m.message.indexOf("SQL: ") + 5, m.message.length)
                 cLogText.readOnly= true;
                 cLog.appendChild(cLogText)
@@ -555,12 +534,6 @@ while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
  
         })
  
-        
-       
-    
-
-
-
    //await populateMessages(allMessages)
 
    
@@ -718,27 +691,8 @@ function numberWithCommas(x) {
     }
 
 
-    /*
-     const topLogCount = 100;f
-        const startTime = Date.now()
-        const endTime = Date.now() - mins*60*1000
-        let services = [theService + ".MapServer"] 
-        if (theService == "(All Services)")
-            services = "*"
-        let result= await adminLog.query([code], services, topLogCount, startTime ,endTime , "DEBUG")
-        let jsonRes = await result.json()
-        let allMessages = [].concat(jsonRes.logMessages) 
-        allMessages = allMessages.filter(m => m.message.indexOf("EndCursor;") > -1)
-        while (jsonRes.hasMore)
-        {  
-            //start paging
-            logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs between ${new Date(jsonRes.endTime)} and ${new Date(endTime)}`)
-            result= await adminLog.query([102023], services, pageSize, jsonRes.endTime, endTime, "DEBUG")
-            jsonRes = await result.json()
-  
-            allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("EndCursor;") > -1))
-        }
-        */
+     
+
     </script>
 </body>
 </html>

package/index.mjs

@@ -7,7 +7,7 @@ import { AdminLog } from "./adminlog.mjs"
 import  logger  from "./logger.mjs"
 import  fetch  from "node-fetch"
 //update version
-let version = "0.0.72";
+let version = "0.0.75";
 const GENERATE_TOKEN_TIME_MIN = 30;
 
 let rl = null;
@@ -53,10 +53,10 @@ function parseInput(){
 
       if (Object.values(params).includes(null))
       {
-        console.log ("HELP: uncli --portal https://unportal.domain.com/portal --service servicename --user username --password password [--gdbversion* user.version --server  https://federatedserver.domain.com/server --file commandfile* --verify true|false]")    
-        console.log("--file commandfile is optional and you can pass a path to a file with a list of command to execute. ")
-        console.log("--gdbversion is optional and allows the UN to be opened in that version. When not specified sde.DEFAULT is used.")
-        console.log("--server is optional except when there are more than one federated server sites to the portal. If the portal only has one server it will be selected.")
+        logger.info ("HELP: uncli --portal https://unportal.domain.com/portal --service servicename --user username --password password [--gdbversion* user.version --server  https://federatedserver.domain.com/server --file commandfile* --verify true|false]")    
+        logger.info("--file commandfile is optional and you can pass a path to a file with a list of command to execute. ")
+        logger.info("--gdbversion is optional and allows the UN to be opened in that version. When not specified sde.DEFAULT is used.")
+        logger.info("--server is optional except when there are more than one federated server sites to the portal. If the portal only has one server it will be selected.")
         process.exit();
       }
      
@@ -73,7 +73,7 @@ async function getToken(parameters) {
 }
 
 async function regenerateToken(parameters) {
-    console.log("Regenerating token.")
+    logger.info("Regenerating token.")
     const token = await getToken(parameters);
     un.token = token;
     executeInput("clear");
@@ -175,6 +175,8 @@ const inputs = {
             "export subnetworks --all [--folder]": "Export all subnetworks with ACK --folder where exported files are saved",
             "export subnetworks --new [--folder]": "Export all subnetworks with ACK that haven't been exported --folder where exported files are saved",
             "export subnetworks --deleted": "Export all subnetworks with ACK that are deleted ",
+            "updateisconnected": "Run update is connected ",
+            
             "count": "Lists the number of rows in all feature layers.",
             "count --system": "Lists the number of rows in system layers.",
             "connect --service": "Connects to the another service",
@@ -193,7 +195,7 @@ const inputs = {
     },
     "^whoami$": async () => {
  
-        console.log(`${parameters.user}@${parameters.service}@${parameters.gdbversion}`)
+        logger.info(`${parameters.user}@${parameters.service}@${parameters.gdbversion}`)
 
     },
     "^def --layers$|^layers$": async () => {
@@ -231,20 +233,20 @@ const inputs = {
         
         const subnetworks = await un.getSubnetworks();
         if (subnetworks.features.length === 0) {
-            console.log("No dirty subnetworks found.")
+            logger.info("No dirty subnetworks found.")
             return;
         }
         const subs = subnetworks.features.map(a => a.attributes)
         console.table(subs)
         const rowCount = subs.length;
-        console.log (`${numberWithCommas(rowCount)} rows returned.`)
+        logger.info (`${numberWithCommas(rowCount)} rows returned.`)
     },
     "^topology$": async () => {  
         const moments = ["initialEnableTopology","fullValidateTopology","partialValidateTopology","enableTopology","disableTopology","definitionModification","updateIsConnected"]      
         const networkMoments = await un.queryMoment(moments)
        // networkMoments.forEach (m => momentsText += `\n${m.moment} : ${m.time === 0 ? "N/A": new Date(m.time*1000)} ${m.duration === 0 ? "" : ` Duration: ${Math.round(m.duration/1000)}s `} `)
        // networkMoments.forEach (m => momentsText += `\n${m.moment} : ${m.time === 0 ? "N/A": new Date(m.time*1000)} ${m.duration === 0 ? "" : ` Duration: ${Math.round(m.duration/1000)}s `} `)
-      //  console.log('\x1b[36m%s\x1b[0m', 'I am cyan');  //cyan
+      //  logger.info('\x1b[36m%s\x1b[0m', 'I am cyan');  //cyan
         const topoMoments = networkMoments.networkMoments.map(m => {
             const t = m.time === 0 ? "N/A": new Date(m.time*1000)
             const d = m.duration === 0 ? "N/A" : numberWithCommas(Math.round(m.duration)) + " ms"
@@ -260,7 +262,7 @@ const inputs = {
         console.table(topoMoments) 
     },
     "^topology --enable$": async () => {
-        console.log("Enabling topology ...");
+        logger.info("Enabling topology ...");
         const fromDate = new Date();
         const result = await un.enableTopology()
         const toDate = new Date();
@@ -270,34 +272,44 @@ const inputs = {
     },
     "^topology --disable$": async () => {
         const fromDate = new Date();
-        console.log("Disabling topology ...");
+        logger.info("Disabling topology ...");
         const result = await un.disableTopology()
         const toDate = new Date();
         const timeEnable = toDate.getTime() - fromDate.getTime();
         result.duration =  numberWithCommas(Math.round(timeEnable)) + " ms"
         console.table(result) 
     },
+    
+    "^updateisconnected$": async () => {
+        const fromDate = new Date();
+        logger.info("Updating is connected ...");
+        const result = await un.updateIsConnected()
+        const toDate = new Date();
+        const timeEnable = toDate.getTime() - fromDate.getTime();
+        result.duration =  numberWithCommas(Math.round(timeEnable)) + " ms"
+        console.table(result) 
+    },
     "^evaluate$": async () => {
         const fromDate = new Date();
-        console.log("Building Evaluation Blocks ...");
+        logger.info("Building Evaluation Blocks ...");
         //return evaluation blocks for layer 5
         const blocks = await buildEvaluationBlocks(5);
 
-        console.log("Evaluating Attribute Rules ...");
+        logger.info("Evaluating Attribute Rules ...");
         //blocks.forEach(b => un.evaluate (null, b, ["validationRules", "calculationRules"], async = false, gdbVersion = "sde.DEFAULT"))
-        //Object.keys(blocks).forEach(k => console.log(blocks[k]))
+        //Object.keys(blocks).forEach(k => logger.info(blocks[k]))
         const promises = []
         Object.keys(blocks).forEach(k => promises.push(un.evaluate (null, blocks[k], ["validationRules", "calculationRules"] )))
         
-        console.log("done sending all requests.. now waiting for response ")
+        logger.info("done sending all requests.. now waiting for response ")
         
-        Promise.all(promises).then(a=>console.log("done")).catch(a=>console.log("failed" + JSON.stringify(a)))
+        Promise.all(promises).then(a=>logger.info("done")).catch(a=>logger.info("failed" + JSON.stringify(a)))
 
         const result = {}
         const toDate = new Date();
         const timeEnable = toDate.getTime() - fromDate.getTime();
         result.duration =  numberWithCommas(Math.round(timeEnable)) + " ms"
-        console.log(result)
+        logger.info(result)
     },
 
     //partition so that we can run and commit incrementally.. 
@@ -305,7 +317,7 @@ const inputs = {
     //timeouts
     //in case failure you don't lose everything
     "^topology --validate -fn$": async () => {
-        console.log("Validating Network topology ...");
+        logger.info("Validating Network topology ...");
 
         const fullExtent = un.featureServiceJson.fullExtent;
         /*
@@ -343,7 +355,7 @@ const inputs = {
             const timeEnable = toDate.getTime() - fromDate.getTime();
             const duration =  numberWithCommas(Math.round(timeEnable)) + " ms"
             console.clear()
-            console.log("Validating extent " + e.xmin)
+            logger.info("Validating extent " + e.xmin)
             console.table({duration}) 
 
         })
@@ -352,7 +364,7 @@ const inputs = {
     },
 
     "^topology --validate$": async () => {
-        console.log("Validating Network topology ...");
+        logger.info("Validating Network topology ...");
         const fromDate = new Date();
         const result = await un.validateNetworkTopology()
         const toDate = new Date();
@@ -363,35 +375,35 @@ const inputs = {
     "^subnetworks --dirty$": async () => {        
         const subnetworks = await un.getSubnetworks("isdirty=1");
         if (subnetworks.features.length === 0) {
-            console.log("No dirty subnetworks found.")
+            logger.info("No dirty subnetworks found.")
             return;
         }
             
         const subs = subnetworks.features.map(a => a.attributes)
         console.table(subs)
         const rowCount = subs.length;
-        console.log (`${numberWithCommas(rowCount)} rows returned.`)
+        logger.info (`${numberWithCommas(rowCount)} rows returned.`)
     },
     "^subnetworks --deleted$": async () => {        
         const subnetworks = await un.getSubnetworks("isdirty=1 and isdeleted=1");
         if (subnetworks.features.length === 0) {
-            console.log("No dirty and deleted subnetworks found.")
+            logger.info("No dirty and deleted subnetworks found.")
             return;
         }
             
         const subs = subnetworks.features.map(a => a.attributes)
         console.table(subs)
         const rowCount = subs.length;
-        console.log (`${numberWithCommas(rowCount)} rows returned.`)
+        logger.info (`${numberWithCommas(rowCount)} rows returned.`)
     },
 
     "^update subnetworks --deleted$" : async () => {
-        console.log("Querying all subnetworks that are dirty and deleted.");
+        logger.info("Querying all subnetworks that are dirty and deleted.");
         let subnetworks = await un.queryDistinct(500002, "domainnetworkname,tiername,subnetworkname", "isdirty=1 and isdeleted=1","domainnetworkname,tiername,subnetworkname");
-        console.log(`Discovered ${subnetworks.features.length} dirty deleted subnetworks.`);
+        logger.info(`Discovered ${subnetworks.features.length} dirty deleted subnetworks.`);
         for (let i = 0;  i < subnetworks.features.length; i++) {
             const f = subnetworks.features[i]
-            console.log("Updating Subnetwork " + v(f.attributes,"subnetworkName"));
+            logger.info("Updating Subnetwork " + v(f.attributes,"subnetworkName"));
             
             const fromDate = new Date();
               
@@ -404,7 +416,7 @@ const inputs = {
             subnetworkResult.duration =  numberWithCommas(Math.round(timeEnable)) + " ms"
             
 
-            console.log(`Result ${JSON.stringify(subnetworkResult)}`)
+            logger.info(`Result ${JSON.stringify(subnetworkResult)}`)
         }
     },
 
@@ -422,27 +434,28 @@ const inputs = {
             if (failedSubnetworks.length > 0 )
                 failedSubWhereClause = " AND SUBNETWORKNAME NOT IN (" + failedSubnetworks.join(",") + ")"
 
-            console.log("Querying all subnetworks that are dirty.");
+            logger.info("Querying all subnetworks that are dirty.");
             subnetworks = await un.queryDistinct(500002, "domainnetworkname,tiername,subnetworkname", "isdirty=1 " + failedSubWhereClause, `domainnetworkname  ${sort},tiername ${sort},subnetworkname ${sort}`);
-            console.log(`Discovered ${subnetworks.features.length} dirty subnetworks.`);
+            logger.info(`Discovered ${subnetworks.features.length} dirty subnetworks.`);
 
             for (let i = 0;  i < subnetworks.features.length; i++) {
                 const f = subnetworks.features[i]
                 const subnetworkName = v(f.attributes,"subnetworkName")
-                console.log("Updating Subnetwork " + subnetworkName);
+                logger.info("Updating Subnetwork " + subnetworkName);
                 
                 const fromDate = new Date();
                  
                 const subnetworkResult = await un.updateSubnetworks(v(f.attributes,"domainNetworkName"), v(f.attributes,"tierName"), v(f.attributes,"subnetworkName"),false);
                 //check if we have processed this subnetwork (maybe be an error)
-                if (subnetworkResult.success == false)
+                const tier = un.getTier(v(f.attributes,"domainNetworkName"), v(f.attributes,"tierName"))
+                if (subnetworkResult.success == false || tier.manageSubnetwork?.propertySetItems?.includes("IsDirty"))
                     failedSubnetworks.push("'" + subnetworkName + "'")
 
                 const toDate = new Date();
                 const timeEnable = toDate.getTime() - fromDate.getTime();
                 subnetworkResult.duration =  numberWithCommas(Math.round(timeEnable)) + " ms"
     
-                console.log(`Result ${JSON.stringify(subnetworkResult)}`)
+                logger.info(`Result ${JSON.stringify(subnetworkResult)}`)
             }
 
         } 
@@ -450,19 +463,21 @@ const inputs = {
         
     },
     "^update subnetworks --all --async$" : async () => {
-        console.log("Querying all subnetworks that are dirty.");
+        logger.info("Querying all subnetworks that are dirty.");
         let subnetworks = await un.queryDistinct(500002, "domainnetworkname,tiername,subnetworkname", "isdirty=1", "domainnetworkname,tiername,subnetworkname");
-        console.log(`Discovered ${subnetworks.features.length} dirty subnetworks.`);
+        logger.info(`Discovered ${subnetworks.features.length} dirty subnetworks.`);
         for (let i = 0;  i < subnetworks.features.length; i++) {
             const f = subnetworks.features[i]
-            console.log("Sending job for " + v(f.attributes,"subnetworkName"));
+            logger.info("Sending job for " + v(f.attributes,"subnetworkName"));
             const subnetworkResult = await un.updateSubnetworks(v(f.attributes,"domainNetworkName"), v(f.attributes,"tierName"), v(f.attributes,"subnetworkName"),true);
-            console.log(`Result from submitting job ${JSON.stringify(subnetworkResult)}`)
+            logger.info(`Result from submitting job ${JSON.stringify(subnetworkResult)}`)
         }
     },
    "^export subnetworks --all --folder .*$|^export subnetworks --all$" : async input => {
  
         let subnetworks 
+        let sort = "asc";
+        if (input.indexOf("--desc") > 0) sort = "desc"
         //create folder
         const file = input.match(/--folder .*/gm)
         let inputDir = "Exported"
@@ -470,33 +485,40 @@ const inputs = {
              inputDir = file[0].replace("--folder ", "")
         //create directory if doesn't exists
         if (!fs.existsSync(inputDir))  fs.mkdirSync(inputDir)
- 
+        let exportedSubnetworks = [];
+
         do {
 
-            console.log("Querying all subnetworks that are clean.");
-            subnetworks = await un.queryDistinct(500002, "domainnetworkname,tiername,subnetworkname", "isdirty=0","domainnetworkname,tiername,subnetworkname");
-            console.log(`Discovered ${subnetworks.features.length} subnetworks that can be exported.`);
+            let exportedSubnetworksWhereClause = ""
+            
+            if (exportedSubnetworks.length > 0 )
+                exportedSubnetworksWhereClause = " AND SUBNETWORKNAME NOT IN (" + exportedSubnetworks.join(",") + ")"
+ 
+            logger.info("Querying all subnetworks that are clean.");
+            subnetworks = await un.queryDistinct(500002, "domainnetworkname,tiername,subnetworkname", "isdirty=0 " + exportedSubnetworksWhereClause,`domainnetworkname  ${sort},tiername ${sort},subnetworkname ${sort}`);
+            logger.info(`Discovered ${subnetworks.features.length} subnetworks that can be exported.`);
             for (let i = 0;  i < subnetworks.features.length; i++) {
                 const f = subnetworks.features[i]
                 const subnetworkName = v(f.attributes,"subnetworkName")
-                console.log("Exporting subnetworks " + v(f.attributes,"subnetworkName"));
+                logger.info("Exporting subnetworks " + v(f.attributes,"subnetworkName"));
                 
                 const fromDate = new Date();
                 
                 const subnetworkResult = await un.exportSubnetworks(v(f.attributes,"domainNetworkName"), v(f.attributes,"tierName"), v(f.attributes,"subnetworkName"),false);
-                 
-                
+                  
                 //code
-    
+                exportedSubnetworks.push("'" + v(f.attributes,"subnetworkName") + "'")
+
                 const toDate = new Date();
                 const timeEnable = toDate.getTime() - fromDate.getTime();
                 subnetworkResult.duration =  numberWithCommas(timeEnable) + " ms"
                 //if undefined exit
                 if (!subnetworkResult.url)
                 {
-                    console.log("Export subnetwork failed " + JSON.stringify(subnetworkResult))
+                    logger.info("Export subnetwork failed " + JSON.stringify(subnetworkResult))
                     continue;
                 }
+
     
                 //fetch the json and write it to disk 
                 const subContent = await fetch(subnetworkResult.url);
@@ -507,11 +529,11 @@ const inputs = {
                 fs.writeFileSync(`${inputDir}/${subnetworkName}.json`, jsonExport)            
                
     
-                console.log(`Result ${JSON.stringify(subnetworkResult)} written to file ${process.cwd()}/${inputDir}/${subnetworkName}.json`)
+                logger.info(`Result ${JSON.stringify(subnetworkResult)} written to file ${process.cwd()}/${inputDir}/${subnetworkName}.json`)
     
             }
         }
-        while (subnetworks.features.length > 0)
+        while (subnetworks?.features?.length > 0)
        
     },
    
@@ -527,20 +549,20 @@ const inputs = {
         if (!fs.existsSync(inputDir))  fs.mkdirSync(inputDir)
 
 
-        console.log("Querying all subnetworks that are clean and not exported.");
+        logger.info("Querying all subnetworks that are clean and not exported.");
         let subnetworks = await un.queryDistinct(500002, "domainnetworkname,tiername,subnetworkname", "isdirty = 0 and (LASTACKEXPORTSUBNETWORK is null or LASTACKEXPORTSUBNETWORK < LASTUPDATESUBNETWORK)","domainnetworkname,tiername,subnetworkname");
-        console.log(`Discovered ${subnetworks.features.length} subnetworks that can be exported.`);
+        logger.info(`Discovered ${subnetworks.features.length} subnetworks that can be exported.`);
         for (let i = 0;  i < subnetworks.features.length; i++) {
             const f = subnetworks.features[i]
             const subnetworkName = v(f.attributes,"subnetworkName")
-            console.log(`Exporting subnetwork ${subnetworkName}` );
+            logger.info(`Exporting subnetwork ${subnetworkName}` );
             const subnetworkResult = await un.exportSubnetworks(v(f.attributes,"domainNetworkName"), v(f.attributes,"tierName"), v(f.attributes,"subnetworkName"),false);
             //fetch the json and write it to disk 
             const subContent = await fetch(subnetworkResult.url);
             const jsonExport = await subContent.text();
             fs.writeFileSync(`${inputDir}/${subnetworkName}.json`, JSON.stringify(jsonExport))            
 
-            console.log(`Result ${JSON.stringify(subnetworkResult)} written to file ${process.cwd()}/${inputDir}/${subnetworkName}.json`)
+            logger.info(`Result ${JSON.stringify(subnetworkResult)} written to file ${process.cwd()}/${inputDir}/${subnetworkName}.json`)
         }
 
  
@@ -550,7 +572,7 @@ const inputs = {
     "^ia$": async input => {
 
         const result = await un.returnInvalidAssociations();
-        console.log("Invalid Associations " + JSON.stringify(result))
+        logger.info("Invalid Associations " + JSON.stringify(result))
     },
     "^connect --service": async input =>{
 
@@ -564,19 +586,23 @@ const inputs = {
     },
     "^trace --subnetwork": async input => {
         //get subnetwork name
+        try {
 
+             
         const fromDate = new Date();
-       
+        const inputDir= "Exported"
+        let full = true;
+      //  if (input.indexOf("--full") > 0) full = true
 
         const inputParam = input.match(/--subnetwork .*/gm)
         let subnetworkName = null;
         if (inputParam != null && inputParam.length > 0)
             subnetworkName = inputParam[0].replace("--subnetwork ", "")
 
-        console.log(`Tracing subnetwork ${subnetworkName}`);
+        logger.info(`Tracing subnetwork ${subnetworkName}`);
         const result = await un.subnetworkTraceSimple(subnetworkName)
         if (result == null) {
-            console.log(`Subnetwork ${subnetworkName} doesn't exist`);
+            logger.info(`Subnetwork ${subnetworkName} doesn't exist`);
             return null;
         }
         const toDate = new Date();
@@ -584,8 +610,44 @@ const inputs = {
         const newResult = {}
         newResult.duration =  numberWithCommas(Math.round(timeRun)) + " ms"
         newResult.elementsCount = result.traceResults.elements.length;
-        console.table(newResult) 
+        const traceRes = {}
+        logger.info("Grouping trace results...")
+         result.traceResults.elements.forEach(e => {
+            const layerid = un.getLayerIdfromSourceId(e.networkSourceId).layerId
+            if (!traceRes["l" + layerid])
+                traceRes["l" + layerid] = []
+            traceRes["l" + layerid].push(e.objectId)
+          
+        })
+        console.table(newResult)
 
+        //if the trace is full turn around the pull all features
+        if (full){
+
+            //loop through all layers and query
+            logger.info("Removing duplicates")
+                
+            console.log(traceRes)
+             
+            //send all queries async 
+            const allQueries = []
+            Object.keys(traceRes).forEach (k => {
+                traceRes[k] = [...new Set(traceRes[k])]
+                allQueries.push(un.query(k.replace("l",""), "", k, traceRes[k]))
+            })
+            logger.info(`${Object.keys(traceRes)} queries sent, waiting for response..`)
+            const result = await Promise.all(allQueries)
+            logger.info("All queries returned.")
+            fs.writeFileSync(`${inputDir}/traceout${subnetworkName}.json`, JSON.stringify(result))
+            logger.info(`Result written to file ${inputDir}/traceout${subnetworkName}.json`)
+
+            //logger.info(JSON.stringify(result))
+            
+        }
+    }
+        catch(ex){
+            logger.error(JSON.stringify(ex))
+        }
     },
     "^export subnetworks --deleted$" : async input => {
 
@@ -598,19 +660,19 @@ const inputs = {
         if (!fs.existsSync(inputDir))  fs.mkdirSync(inputDir)
 
 
-        console.log("Querying all subnetworks that are clean and deleted.");
+        logger.info("Querying all subnetworks that are clean and deleted.");
         let subnetworks = await un.queryDistinct(500002, "domainnetworkname,tiername,subnetworkname", "isdirty = 0 and isdeleted=1");
-        console.log(`Discovered ${subnetworks.features.length} subnetworks that can be exported.`);
+        logger.info(`Discovered ${subnetworks.features.length} subnetworks that can be exported.`);
         for (let i = 0;  i < subnetworks.features.length; i++) {
             const f = subnetworks.features[i]
             const subnetworkName = v(f.attributes,"subnetworkName")
-            console.log(`Exporting subnetwork ${subnetworkName}` );
+            logger.info(`Exporting subnetwork ${subnetworkName}` );
             const subnetworkResult = await un.exportSubnetworks(v(f.attributes,"domainNetworkName"), v(f.attributes,"tierName"), v(f.attributes,"subnetworkName"),false);
             
             //if undefined exit
             if (!subnetworkResult.url)
             {
-                console.log("Export subnetwork failed " + JSON.stringify(subnetworkResult))
+                logger.info("Export subnetwork failed " + JSON.stringify(subnetworkResult))
                 continue;
             }
 
@@ -619,7 +681,7 @@ const inputs = {
             const jsonExport = await subContent.text();
             fs.writeFileSync(`${inputDir}/${subnetworkName}.json`, JSON.stringify(jsonExport))            
 
-            console.log(`Result ${JSON.stringify(subnetworkResult)} written to file ${process.cwd()}/${inputDir}/${subnetworkName}.json`)
+            logger.info(`Result ${JSON.stringify(subnetworkResult)} written to file ${process.cwd()}/${inputDir}/${subnetworkName}.json`)
         }
 
  
@@ -628,7 +690,7 @@ const inputs = {
     "^cwd$" : async input => {
           
 
-        console.log(process.cwd())
+        logger.info(process.cwd())
  
      },
  
@@ -640,14 +702,14 @@ const inputs = {
          //create directory if doesn't exists
          if (!fs.existsSync(inputDir))  fs.mkdirSync(inputDir)
          fs.writeFileSync(`${inputDir}/${Math.random()}`, Math.random())
-         console.log(inputDir)
+         logger.info(inputDir)
    
   
       },
 
 
     "^count$": async () => {
-        console.log("Querying all layers....")
+        logger.info("Querying all layers....")
         const layerProperties = [
             "id",
             "name",
@@ -681,12 +743,12 @@ const inputs = {
  
        
         console.table(layerCount)
-        console.log(`Total number of rows in all layers : ${numberWithCommas(totalRows)} .`)
+        logger.info(`Total number of rows in all layers : ${numberWithCommas(totalRows)} .`)
     },
 
     
     "^count --system$": async () => {
-        console.log("Querying all system layers....")
+        logger.info("Querying all system layers....")
         
         const systemLayers = un.getSystemLayers();
         let totalRows = 0;
@@ -709,7 +771,7 @@ const inputs = {
         } 
         
         console.table(layerCount)
-        console.log(`Total number of rows in all system layers : ${numberWithCommas(totalRows)} .`)
+        logger.info(`Total number of rows in all system layers : ${numberWithCommas(totalRows)} .`)
 
     },
 
@@ -724,7 +786,7 @@ const inputs = {
             mins = inputParam[0].replace("--age ", "")
 
 
-       console.log(`Querying attribute rules logs for ${parameters.service} for the last ${mins} minutes ...`)
+       logger.info(`Querying attribute rules logs for ${parameters.service} for the last ${mins} minutes ...`)
 
        const startTime = Date.now() - mins*60*1000
        const endTime = Date.now();
@@ -774,7 +836,7 @@ const inputs = {
         if (inputParam != null && inputParam.length > 0)
             mins = inputParam[0].replace("--age ", "")
  
-        console.log(`Querying trace logs for ${parameters.service} for the last ${mins} minutes ...`)
+        logger.info(`Querying trace logs for ${parameters.service} for the last ${mins} minutes ...`)
         const startTime = Date.now() - mins*60*1000
         const endTime = Date.now();
         let result= await adminLog.query([102002], [parameters.service+ ".MapServer"], topLogCount, startTime ,endTime , "VERBOSE")
@@ -815,7 +877,7 @@ const inputs = {
             delete newMessage.message
          
             console.table([newMessage])
-            console.log(m.message)  
+            logger.info(m.message)  
         })
          
     },
@@ -833,7 +895,7 @@ const inputs = {
             mins = inputParam[0].replace("--age ", "")
  
             
-  console.log(`Querying validate logs for ${parameters.service} for the last ${mins} minutes ...`)
+  logger.info(`Querying validate logs for ${parameters.service} for the last ${mins} minutes ...`)
   const startTime = Date.now() - mins*60*1000
   const endTime = Date.now();
   let result= await adminLog.query([102003], [parameters.service+ ".MapServer"], topLogCount, startTime ,endTime , "VERBOSE")
@@ -898,7 +960,7 @@ const inputs = {
             delete newMessage.message
          
             console.table([newMessage])
-            console.log(m.message)  
+            logger.info(m.message)  
         })
          
 
@@ -917,7 +979,7 @@ const inputs = {
         if (inputParam != null && inputParam.length > 0)
             mins = inputParam[0].replace("--age ", "")
  
-        console.log(`Querying subnetwork logs for ${parameters.service} for the last ${mins} minutes ...`)
+        logger.info(`Querying subnetwork logs for ${parameters.service} for the last ${mins} minutes ...`)
         const startTime = Date.now() - mins*60*1000
         const endTime = Date.now();
         let result= await adminLog.query([102003], [parameters.service+ ".MapServer"], topLogCount, startTime ,endTime , "VERBOSE")
@@ -982,7 +1044,7 @@ const inputs = {
             delete newMessage.message
          
             console.table([newMessage])
-            console.log(m.message)  
+            logger.info(m.message)  
         })
          
          
@@ -999,7 +1061,7 @@ const inputs = {
         if (inputParam != null && inputParam.length > 0)
             mins = inputParam[0].replace("--age ", "")
  
-        console.log(`Querying cursor sql logs for ${parameters.service} for the last ${mins} minutes ...`)
+        logger.info(`Querying cursor sql logs for ${parameters.service} for the last ${mins} minutes ...`)
         const startTime = Date.now() - mins*60*1000
         const endTime = Date.now();
         let result= await adminLog.query([102023], [parameters.service+ ".MapServer"], topLogCount, startTime ,endTime , "DEBUG")
@@ -1015,7 +1077,7 @@ const inputs = {
   
             allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("EndCursor;") > -1))
         }
-      console.log ("Filtering messages...")
+      logger.info ("Filtering messages...")
       
       allMessages = allMessages
             .map( m=> {
@@ -1029,21 +1091,21 @@ const inputs = {
             .slice(0, 10) ;//first 10
     
      
-        console.log("-----Top 10 SQL----")
+        logger.info("-----Top 10 SQL----")
         let i =0;
         allMessages= allMessages.forEach(m => 
             {
              
                 const x = m.message.split(";")
                 x.shift()
-                console.log(`id: ${i++}`)
-                console.log(`\tAt: ${new Date(m.time)} (${m.time})`)
-                console.log(`\tUser: ${m.user}`)
-                console.log(`\tTotal Time: ${numberWithCommas(Math.round(m.elapsed*1000))} ms (Total time the cursor was opened)`)
-                console.log(`\tQuery Time: ${numberWithCommas(m.totalExecutionElapsed)} ms (includes search + data access nextRow)`)
-                console.log(`\tQuery:`)
-                x.forEach(a => console.log(`\t${a}`))
-                console.log(`\n`)
+                logger.info(`id: ${i++}`)
+                logger.info(`\tAt: ${new Date(m.time)} (${m.time})`)
+                logger.info(`\tUser: ${m.user}`)
+                logger.info(`\tTotal Time: ${numberWithCommas(Math.round(m.elapsed*1000))} ms (Total time the cursor was opened)`)
+                logger.info(`\tQuery Time: ${numberWithCommas(m.totalExecutionElapsed)} ms (includes search + data access nextRow)`)
+                logger.info(`\tQuery:`)
+                x.forEach(a => logger.info(`\t${a}`))
+                logger.info(`\n`)
  
         })
  
@@ -1155,11 +1217,11 @@ const inputs = {
         console.table(rules)
 
         const totalARExecution = rules.reduce( (prev, cur) =>  prev + cur["Total Cost (ms)"], 0)
-        console.log(`Total time spend executing attribute rules (${Math.round(totalARExecution)} ms) (${Math.round(totalARExecution/1000)} s) (${Math.round(totalARExecution/(1000*60))} m)`)
+        logger.info(`Total time spend executing attribute rules (${Math.round(totalARExecution)} ms) (${Math.round(totalARExecution/1000)} s) (${Math.round(totalARExecution/(1000*60))} m)`)
          
     },
 
-    "^version$": () => console.log(version),
+    "^version$": () => logger.info(version),
     "^clear$|^cls$": () => console.clear(),
     "^quit$": () => {
         if (rl) rl.close();
@@ -1184,7 +1246,7 @@ const inputs = {
    
     while(true) {
         const result = await un.query(layerId, `1=1`, undefined, undefined, ["globalId"], "sde.DEFAULT", offset, recordCount)
-        console.log(`Processing ${recordCount} rows`)
+        logger.info(`Processing ${recordCount} rows`)
         //for each assocaition check if its valid
         for (let i = 0 ; i < result.features.length; i++){
             const row = result.features[i]
@@ -1299,7 +1361,7 @@ function numberWithCommas(x) {
 /*
 rl.question("What is your name ? ", function(name) {
     rl.question("Where do you live ? ", function(country) {
-        console.log(`${name}, is a citizen of ${country}`);
+        logger.info(`${name}, is a citizen of ${country}`);
         rl.close();
     });
 });
@@ -1312,7 +1374,7 @@ function setupReadLine() {
     });
 
     rl.on("close", function() {
-        console.log("\nbye");
+        logger.info("\nbye");
         process.exit(0);
     });
 
@@ -1334,7 +1396,7 @@ export async function run (){
         console.error(`Minimum required node js is ${minVer} your version is ${process.version}`)
         process.exit(0);
     }
-    console.log(`uncli ${version} is experimental command line utility for basic utility network services. Use as is.`)
+    logger.info(`uncli ${version} is experimental command line utility for basic utility network services. Use as is.`)
     parameters = await parseInput( )
     //set certificate verification 
     const verifyCert = parameters["verify"] === 'true' ? 1 : 0;

package/logger.mjs

@@ -1,9 +1,23 @@
 const logger = {}
-logger.info = console.log
-logger.error = console.error
+logger.info = m => console.log (`${shortDate()}: ${m}`)
+logger.error =m => console.error (`${shortDate()}: ${m}`)
 
 export default logger;
 
+function shortDate()
+{
+
+  const t = new Date();
+  const date = ('0' + t.getDate()).slice(-2);
+  const month = ('0' + (t.getMonth() + 1)).slice(-2);
+  const year = t.getFullYear();
+  const hours = ('0' + t.getHours()).slice(-2);
+  const minutes = ('0' + t.getMinutes()).slice(-2);
+  const seconds = ('0' + t.getSeconds()).slice(-2);
+  const time = `${month}/${date}/${year}:${hours}:${minutes}:${seconds}`;
+  return time;
+}
+
 /*
 import  winston  from "winston"
 

package/makerequest.mjs

@@ -28,25 +28,28 @@ export function makeRequest (opts) {
 
         let f;
         try {
+ 
             const nodeFetch = await import ("node-fetch");
             f = nodeFetch.default;
         }
         catch(ex) {
             f = fetch;
         }
- 
-
+         
+        let jsonRes
+        
        const result = await f(opts.url, {
            "method" : opts.method,
             "headers":  headers,
             "body": params
        });
 
-       const jsonRes  = await result.json();
-      
+        jsonRes  = await result.json();
+        
         if (typeof jsonRes !== "object") 
             jsonRes = JSON.parse(jsonRes);
 
+ 
         resolve(jsonRes);
     
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "un-cli",
-  "version": "0.0.72",
+  "version": "0.0.75",
   "description": "Command line interface for working with ArcGIS Utility Network Extension",
   "main": "index.mjs",
   "bin": {

package/utilitynetwork.node.mjs

@@ -21,12 +21,14 @@ export class UtilityNetwork {
             this.featureServiceUrl = featureServiceUrl;
             this.token = token;         
             this.gdbVersion = gdbVersion;
+            this.sourceMapping = {}
         }
 
       ///first function one should call after creating an instance of a utility network
        load ()
         {
             let thisObj = this;
+            
             return new Promise (function (resolve, reject)
             {
                 //run async mode
@@ -68,7 +70,30 @@ export class UtilityNetwork {
             
                     thisObj.dataElement = undataElement.layerDataElements[0].dataElement;
                     thisObj.layerDefinition = unLayerDef
-                    thisObj.subnetLineLayerId = thisObj.getSubnetLineLayerId();                    
+                    thisObj.subnetLineLayerId = thisObj.getSubnetLineLayerId();     
+                    
+                    //build out source mapping hash
+                    let domainNetworks = thisObj.dataElement.domainNetworks;
+                    let layerObj = undefined;
+
+                    for (let i = 0; i < domainNetworks.length; i ++)
+                    {
+                        let domainNetwork = domainNetworks[i];
+                        for (let j = 0; j < domainNetwork.junctionSources.length; j ++)
+                            {  
+                                layerObj =  {type: domainNetwork.junctionSources[j].shapeType, layerId: domainNetwork.junctionSources[j].layerId}
+                                layerObj.type = layerObj.type.replace("esriGeometry", "").toLowerCase();
+                                thisObj.sourceMapping["s" + domainNetwork.junctionSources[j].sourceId] = layerObj
+                            }
+
+                        for (let j = 0; j < domainNetwork.edgeSources.length; j ++)
+                             { 
+                                layerObj = {type: domainNetwork.edgeSources[j].shapeType, layerId: domainNetwork.edgeSources[j].layerId} 
+                                layerObj.type = layerObj.type.replace("esriGeometry", "").toLowerCase();
+                                thisObj.sourceMapping["s" + domainNetwork.edgeSources[j].sourceId] = layerObj
+                            }
+                    }
+  
                     resolve(thisObj);
                 }
                 else
@@ -333,16 +358,49 @@ export class UtilityNetwork {
             if (objectids != undefined) 
             queryJson.objectIds = objectids;
             queryJson.layerId = layerId
-            return new Promise((resolve, reject) => {
+            return new Promise(async(resolve, reject) => {
+
+
+                
+                let rowsJson;
+                let allRowsJson;
+                let recordOffset = 0;
+
+                try {
+
+                    do {
+                        queryJson.resultOffset = recordOffset;
+                        rowsJson  = await makeRequest({method: 'POST', params: queryJson, url: this.featureServiceUrl + "/" + layerId + "/query"}) ;                
+                        recordOffset = recordOffset + resultRecordCount + 1;
+                        rowsJson.obj = obj;
+
+                        if (!allRowsJson)
+                          allRowsJson = rowsJson
+                        else
+                          allRowsJson?.features.push(...rowsJson?.features)
+                        //page the result until done
+                    }
+                    while(rowsJson?.exceededTransferLimit == true)
+
+                    resolve(allRowsJson);
+                }
+                catch(ex){
+                    reject("failed to query" + JSON.stringify(ex))
+                }
 
-                makeRequest({method: 'POST', params: queryJson, url: this.featureServiceUrl + "/" + layerId + "/query"}).then(rowsJson=> {                  
-                    rowsJson.obj = obj;
-                    resolve(rowsJson);
-                }).catch(rej => reject("failed to query"));
                 
+           
+
             });
                 
-              
+
+            
+       // }
+       // while(jsonRes?.exceededTransferLimit == true)
+      // if (!allJsonRes)
+     //  allJsonRes = jsonRes
+ // else
+      // allJsonRes?.features.push(...jsonRes?.features)
         }
         //get the terminal configuration using the id
         getTerminalConfiguration(terminalConfigurationId)
@@ -538,8 +596,10 @@ export class UtilityNetwork {
         //get layer id from Source Id used to map sourceid to layer id
         getLayerIdfromSourceId(sourceId)
         { 
+ 
+            return this.sourceMapping["s" + sourceId]
+            /*
             let domainNetworks = this.dataElement.domainNetworks;
-            let layerObj = undefined;
 
             for (let i = 0; i < domainNetworks.length; i ++)
             {
@@ -561,6 +621,7 @@ export class UtilityNetwork {
 
             if (layerObj != undefined)
                 layerObj.type = layerObj.type.replace("esriGeometry", "").toLowerCase();
+            */
 
             return layerObj;
         }
@@ -958,6 +1019,27 @@ export class UtilityNetwork {
 
  
 
+        updateIsConnected(async=false) {
+
+            let thisObj = this;  
+            let ar = thisObj.featureServiceUrl.split("/");
+             ar[ar.length-1]="UtilityNetworkServer";
+             let updateisconnectedURL = ar.join("/") + "/updateIsConnected"
+     
+               let payload = {
+                  f: "json",  
+                  token: this.token,  
+                  async: async
+                }
+              let un = this;
+            
+            return makeRequest({method:'POST', params: payload, url: updateisconnectedURL })
+        
+ 
+
+        }
+
+
         updateSubnetworks(domainNetworkName, tierName, subnetworkName, async=false) {
 
             let thisObj = this;