un-cli 0.0.73 → 0.0.76

package/cmd.txt

@@ -1,2 +1,2 @@
-update subnetworks --all --desc
+topology --validate --fishnet
 exit

package/index.html

@@ -44,6 +44,8 @@
         <td><button id = 'btnSQLLogs'>SQL Logs</button></td>
         <td>Message<input id = 'txtFilter' type = 'text'></td>
         <td>User<input id = 'txtUser' type = 'text'></td>
+        <td>Start Time<input id = 'txtStartTime' type = 'text'></td>
+        <td>End Time<input id = 'txtEndTime' type = 'text'></td>
     </tr>
 
     </table>
@@ -87,12 +89,14 @@
            
         cmbService = document.getElementById("cmbService")
         while (cmbService.firstChild) cmbService.removeChild(cmbService.firstChild)
-        
+         
+        token = await getToken(parameters);
+
+        //if we logged in successfully 
         const opt = document.createElement("option")
         opt.textContent = "(All Services)"
         cmbService.appendChild (opt)
 
-        token = await getToken(parameters);
         const services = await portal.services()
         const featureServices = services.services.filter(s => s.type == "FeatureServer")
         featureServices.forEach(s=> {
@@ -127,7 +131,7 @@
 
 
     async function loadTraceLogs () {
-  
+   
         //build table
         const tblResult = document.getElementById("tblResult");
         //clear
@@ -141,9 +145,110 @@
         console.log(`Querying trace logs for ${parameters.service} for the last ${mins} minutes ...`)
             
          //page query the admin log
-         let allMessages = await adminLogQueryWithPaging(mins, parameters.service, [102002], "------ Trace Parameters ----", "VERBOSE")
+       let allTheMessages = await adminLogQueryWithPaging(mins, parameters.service, [102002], "------ Trace Parameters ----", "VERBOSE")
+            
+       let allMessages = filterMessages (allTheMessages).map(m => {
+            const newMessage = Object.assign({}, m);
+            delete newMessage.machine;
+            delete newMessage.type;
+            delete newMessage.code;
+            delete newMessage.requestID;
+            delete newMessage.process;
+            delete newMessage.thread;
+            newMessage.elapsedms = parseInt (parseFloat(newMessage.elapsed) * 1000)
+            newMessage.time = new Date(newMessage.time).toLocaleString()
+            delete newMessage.elapsed
+
+            try{
+                
+                //Total Trace Time (4.745 seconds)
+    
+                let re = /Total Trace Time \([-+]?([0-9]*\.[0-9]+|[0-9]+) seconds\)/;
+                let res = re.exec(m.message)
+                if (res && res.length  > 1)
+                    newMessage.traceTime = parseFloat(res[1])*1000
+        
+                let re1 = /Trace Type: .\w.*/;
+                let res1 = re1.exec(m.message)
+                if (res1 && res1.length  > 0)
+                    newMessage.traceType = res1[0].replace("Trace Type: ","")
  
-         await populateMessages(allMessages)
+            }
+            catch(ex){
+               console.error(ex)
+            }
+ 
+
+            return newMessage;
+        })
+        .sort( (m1,m2) => m2.elapsedms - m1.elapsedms)
+        
+        //clear
+        while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
+
+        const header = document.createElement("tr");
+        const cService = document.createElement("th");
+        cService.textContent = "Source"
+        const cTime = document.createElement("th");
+        cTime.textContent = "Time"
+        const cUser = document.createElement("th");
+        cUser.textContent = "User"
+        const cMethod = document.createElement("th");
+        cMethod.textContent = "Method"
+        const cElapsedMS = document.createElement("th");
+        cElapsedMS.textContent = "Elapsed (ms)"
+        const cTraceType = document.createElement("th");
+        cTraceType.textContent = "Trace Type"
+        const cTraceTime = document.createElement("th");
+        cTraceTime.textContent = "Trace Time"
+        const cLog = document.createElement("th");
+        cLog.textContent = "Full Log"
+        header.appendChild (cService)
+        header.appendChild (cTime)
+        header.appendChild (cUser)
+        header.appendChild (cMethod)
+        header.appendChild (cTraceType)
+        header.appendChild (cTraceTime)
+        header.appendChild (cElapsedMS)
+
+        header.appendChild (cLog) 
+        tblResult.appendChild(header)
+        allMessages.forEach(m => {
+            const logRow = document.createElement("tr");
+            const cService = document.createElement("td");
+            cService.textContent = m.source.replace(".MapServer", "")
+            const cTime = document.createElement("td");
+            cTime.textContent = m.time
+            const cUser = document.createElement("td");
+            cUser.textContent = m.user
+            const cMethod = document.createElement("td");
+            cMethod.textContent = m.methodName
+            const cElapsedMS = document.createElement("td");
+            cElapsedMS.textContent = m.elapsedms
+            const cTraceType = document.createElement("td");
+            cTraceType.textContent = m.traceType
+            const cTraceTime = document.createElement("td");
+            cTraceTime.textContent = m.traceTime
+
+            const cLog = document.createElement("td");
+            cLog.textContent = "..."
+            cLog.fullLog = m.message;
+            cLog.addEventListener("click", e=> {navigator.clipboard.writeText(e.target.fullLog); alert("Copied to clipboard")})
+            logRow.appendChild (cService)
+            logRow.appendChild (cTime)
+            logRow.appendChild (cUser)
+            logRow.appendChild (cMethod)
+            logRow.appendChild (cTraceType)
+            logRow.appendChild (cTraceTime)
+            logRow.appendChild (cElapsedMS)
+
+            logRow.appendChild (cLog) 
+            tblResult.appendChild(logRow)
+        })
+
+
+
+         //await populateMessages(allMessages)
  
          
     }
@@ -206,31 +311,114 @@ async function loadUpdateSubnetworkLogs () {
 
         
         console.log(`Querying subnetwork logs for ${parameters.service} for the last ${mins} minutes ...`)
-        let allMessages = await adminLogQueryWithPaging(mins, parameters.service, [102003], "---- Subnetwork Parameters ----", "VERBOSE")
+        let allTheMessages = await adminLogQueryWithPaging(mins, parameters.service, [102003], "---- Subnetwork Parameters ----", "VERBOSE")
 
+          
         
-        //update subnetwork missing elapsed populate it
-        allMessages = allMessages.map( m => {
-            try{
+       let allMessages = filterMessages (allTheMessages).map(m => {
+           
+        const newMessage = Object.assign({}, m);
+        try{
+          
+            delete newMessage.machine;
+            delete newMessage.type;
+            delete newMessage.code;
+            delete newMessage.requestID;
+            delete newMessage.process;
+            delete newMessage.thread;
+         
+                
+                let re = /Total \([-+]?([0-9]*\.[0-9]+|[0-9]+) seconds\)/;
+                let res = re.exec(m.message)
+                if (res && res.length  > 0)
+                    newMessage.elapsed = res[1]
+
+                re = /Total update subnetwork time \([-+]?([0-9]*\.[0-9]+|[0-9]+) seconds\)/;
+                res = re.exec(m.message)
+                if (res && res.length  > 0)
+                    newMessage.elapsed = res[1]
  
-            let re = /Total \([-+]?([0-9]*\.[0-9]+|[0-9]+) seconds\)/;
-            let res = re.exec(m.message)
-            if (res && res.length  > 1)
-                m.elapsed = res[1]
-
-            re = /Total update subnetwork time \([-+]?([0-9]*\.[0-9]+|[0-9]+) seconds\)/;
-            res = re.exec(m.message)
-            if (res && res.length  > 1)
-                m.elapsed = res[1]
-            
-            return m;
+                    
+                let re1 = /Subnetwork name: .*/;
+                let res1 = re1.exec(m.message)
+                if (res1 && res1.length  > 0)
+                    newMessage.subnetworkName = res1[0].replace("Subnetwork name: ","")
+                    
+         
+                newMessage.elapsedms = parseInt (parseFloat(newMessage.elapsed) * 1000)
+                newMessage.time = new Date(newMessage.time).toLocaleString()
+                delete newMessage.elapsed
+                
+  
             }
             catch(ex){
-                return m;
+                console.error(ex)
             }
+            finally {
+                return newMessage;
+            }
+
+        })
+        .sort( (m1,m2) => m2.elapsedms - m1.elapsedms)
+        
+        //clear
+        while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
+
+        const header = document.createElement("tr");
+        const cService = document.createElement("th");
+        cService.textContent = "Source"
+        const cTime = document.createElement("th");
+        cTime.textContent = "Time"
+        const cUser = document.createElement("th");
+        cUser.textContent = "User"
+        const cMethod = document.createElement("th");
+        cMethod.textContent = "Method"
+        const cElapsedMS = document.createElement("th");
+        cElapsedMS.textContent = "Elapsed (ms)"
+        const cSubnetworkName = document.createElement("th");
+        cSubnetworkName.textContent = "Subnetwork Name"
+        const cLog = document.createElement("th");
+        cLog.textContent = "Full Log"
+        header.appendChild (cService)
+        header.appendChild (cTime)
+        header.appendChild (cUser)
+        header.appendChild (cMethod)
+        header.appendChild (cSubnetworkName)
+        header.appendChild (cElapsedMS)
+
+        header.appendChild (cLog) 
+        tblResult.appendChild(header)
+        allMessages.forEach(m => {
+            const logRow = document.createElement("tr");
+            const cService = document.createElement("td");
+            cService.textContent = m.source.replace(".MapServer", "")
+            const cTime = document.createElement("td");
+            cTime.textContent = m.time
+            const cUser = document.createElement("td");
+            cUser.textContent = m.user
+            const cMethod = document.createElement("td");
+            cMethod.textContent = m.methodName
+            const cElapsedMS = document.createElement("td");
+            cElapsedMS.textContent = m.elapsedms
+            const cSubnetworkName = document.createElement("td");
+            cSubnetworkName.textContent = m.subnetworkName
+
+            const cLog = document.createElement("td");
+            cLog.textContent = "..."
+            cLog.fullLog = m.message;
+            cLog.addEventListener("click", e=> {navigator.clipboard.writeText(e.target.fullLog); alert("Copied to clipboard")})
+            logRow.appendChild (cService)
+            logRow.appendChild (cTime)
+            logRow.appendChild (cUser)
+            logRow.appendChild (cMethod)
+            logRow.appendChild (cSubnetworkName)
+            logRow.appendChild (cElapsedMS)
+
+            logRow.appendChild (cLog) 
+            tblResult.appendChild(logRow)
         })
-        await populateMessages(allMessages)
 
+  
    
 }
 
@@ -390,16 +578,16 @@ async function loadAttributeRules () {
 }
 async function loadSQLLogs () {
 
-//build table
-const tblResult = document.getElementById("tblResult");
-//clear
-while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
- 
+    //build table
+    const tblResult = document.getElementById("tblResult");
+    //clear
+    while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
+        
 
-  parameters.service = cmbService.options[cmbService.selectedIndex].text
-   
-  
-  let mins = document.getElementById("txtAge").value  //query logs for the last 30 minutes
+    parameters.service = cmbService.options[cmbService.selectedIndex].text
+    
+    
+    let mins = document.getElementById("txtAge").value  //query logs for the last 30 minutes
 
   
     console.log(`Querying cursor sql logs for ${parameters.service} for the last ${mins} minutes ...`)
@@ -407,30 +595,7 @@ while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
         //startTime is the most recent
         //endTime is the oldest
 
-        
-
-    /*
-     const topLogCount = 100;
-        const startTime = Date.now()
-        const endTime = Date.now() - mins*60*1000
-        let services = [theService + ".MapServer"] 
-        if (theService == "(All Services)")
-            services = "*"
-        let result= await adminLog.query([code], services, topLogCount, startTime ,endTime , "DEBUG")
-        let jsonRes = await result.json()
-        let allMessages = [].concat(jsonRes.logMessages) 
-        allMessages = allMessages.filter(m => m.message.indexOf("EndCursor;") > -1)
-        while (jsonRes.hasMore)
-        {  
-            //start paging
-            logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs between ${new Date(jsonRes.endTime)} and ${new Date(endTime)}`)
-            result= await adminLog.query([102023], services, pageSize, jsonRes.endTime, endTime, "DEBUG")
-            jsonRes = await result.json()
-  
-            allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("EndCursor;") > -1))
-        }
-        */
-
+         
      //page query the admin log
       let allMessages = await adminLogQueryWithPaging(mins, parameters.service, [102023], "EndCursor;", "DEBUG")
 
@@ -534,7 +699,7 @@ while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
                 const cLog = document.createElement("td");
                 const cLogText = document.createElement("input")
                 cLogText.type = "text"
-                cLogText.style = "width:500px"
+                cLogText.style = "width:800px"
                 cLogText.value = m.message.substr( m.message.indexOf("SQL: ") + 5, m.message.length)
                 cLogText.readOnly= true;
                 cLog.appendChild(cLogText)
@@ -555,12 +720,6 @@ while(tblResult.firstChild) tblResult.removeChild(tblResult.firstChild)
  
         })
  
-        
-       
-    
-
-
-
    //await populateMessages(allMessages)
 
    
@@ -693,8 +852,16 @@ function numberWithCommas(x) {
         let adminLog = new AdminLog(token, portal.serverUrl)
         tblResult.appendChild(prg);
         tblResult.appendChild(btn)
-        const startTime = Date.now()
-        const endTime = Date.now() - mins*60*1000
+        let startTime = Date.now()
+        const txtStartTime = document.getElementById("txtStartTime")
+        if (txtStartTime.value != "")
+            startTime = new Date(txtStartTime.value).getTime()
+
+        let endTime = startTime - mins*60*1000
+        const txtEndTime = document.getElementById("txtEndTime")
+        if (txtEndTime.value != "")
+            endTime = new Date(txtEndTime.value).getTime()
+
         let services = [theService + ".MapServer"] 
         if (theService == "(All Services)")
             services = "*"
@@ -718,27 +885,8 @@ function numberWithCommas(x) {
     }
 
 
-    /*
-     const topLogCount = 100;f
-        const startTime = Date.now()
-        const endTime = Date.now() - mins*60*1000
-        let services = [theService + ".MapServer"] 
-        if (theService == "(All Services)")
-            services = "*"
-        let result= await adminLog.query([code], services, topLogCount, startTime ,endTime , "DEBUG")
-        let jsonRes = await result.json()
-        let allMessages = [].concat(jsonRes.logMessages) 
-        allMessages = allMessages.filter(m => m.message.indexOf("EndCursor;") > -1)
-        while (jsonRes.hasMore)
-        {  
-            //start paging
-            logger.info(`Aggregating messages... total so far ${allMessages.length} entries but more left, pulling logs between ${new Date(jsonRes.endTime)} and ${new Date(endTime)}`)
-            result= await adminLog.query([102023], services, pageSize, jsonRes.endTime, endTime, "DEBUG")
-            jsonRes = await result.json()
-  
-            allMessages = allMessages.concat(jsonRes.logMessages.filter(m => m.message.indexOf("EndCursor;") > -1))
-        }
-        */
+     
+
     </script>
 </body>
 </html>

package/index.mjs

@@ -7,7 +7,7 @@ import { AdminLog } from "./adminlog.mjs"
 import  logger  from "./logger.mjs"
 import  fetch  from "node-fetch"
 //update version
-let version = "0.0.73";
+let version = "0.0.75";
 const GENERATE_TOKEN_TIME_MIN = 30;
 
 let rl = null;
@@ -86,7 +86,7 @@ async function regenerateToken(parameters) {
 async function connect(parameters) {
     try{
      //print the parameters
-    logger.info(parameters);
+    logger.info(JSON.stringify(parameters));
     //connect to portal
     
     const token = await getToken(parameters);
@@ -175,6 +175,8 @@ const inputs = {
             "export subnetworks --all [--folder]": "Export all subnetworks with ACK --folder where exported files are saved",
             "export subnetworks --new [--folder]": "Export all subnetworks with ACK that haven't been exported --folder where exported files are saved",
             "export subnetworks --deleted": "Export all subnetworks with ACK that are deleted ",
+            "updateisconnected": "Run update is connected ",
+            
             "count": "Lists the number of rows in all feature layers.",
             "count --system": "Lists the number of rows in system layers.",
             "connect --service": "Connects to the another service",
@@ -277,6 +279,16 @@ const inputs = {
         result.duration =  numberWithCommas(Math.round(timeEnable)) + " ms"
         console.table(result) 
     },
+    
+    "^updateisconnected$": async () => {
+        const fromDate = new Date();
+        logger.info("Updating is connected ...");
+        const result = await un.updateIsConnected()
+        const toDate = new Date();
+        const timeEnable = toDate.getTime() - fromDate.getTime();
+        result.duration =  numberWithCommas(Math.round(timeEnable)) + " ms"
+        console.table(result) 
+    },
     "^evaluate$": async () => {
         const fromDate = new Date();
         logger.info("Building Evaluation Blocks ...");
@@ -304,10 +316,12 @@ const inputs = {
     //progress
     //timeouts
     //in case failure you don't lose everything
-    "^topology --validate -fn$": async () => {
+    "^topology --validate --fishnet$": async () => {
         logger.info("Validating Network topology ...");
 
-        const fullExtent = un.featureServiceJson.fullExtent;
+        const fullExtent = un.layerDefinition.extent;
+        //console.log(`add_env(${ JSON.stringify(fullExtent)})`)
+
         /*
         fullExtent.xmin = 0;
         fullExtent.xmax = 100;
@@ -316,13 +330,14 @@ const inputs = {
         https://desktop.arcgis.com/en/arcmap/10.3/tools/cartography-toolbox/create-cartographic-partitions.htm
         */
         //fish net
-        const grids = 4;
+        const grids = 5; //divide the grid 5 x 5
         const dx = (fullExtent.xmax - fullExtent.xmin) / grids;
         const dy = (fullExtent.ymax - fullExtent.ymin) / grids;
-        
-        const extents = [];
+         
+        const fishnet = [];
         for (let i =0; i < grids; i ++) {
-            
+            const row = [];
+
             for (let j = 0; j < grids; j++) {
                 const extent = {
                     "xmin": fullExtent.xmin + j*dx,
@@ -331,24 +346,47 @@ const inputs = {
                     "ymax": fullExtent.ymin + i*dy + dy,
                     "spatialReference": fullExtent.spatialReference
                 }
-                extents.push(extent);
+                row.push({"content": " ", "extent": extent});
+ 
             }
+            fishnet.push(row);
         }
          
-        extents.forEach(async e => {
-            const fromDate = new Date(); 
 
-            const result = await un.validateNetworkTopology("sde.DEFAULT", e)
-            const toDate = new Date();
-            const timeEnable = toDate.getTime() - fromDate.getTime();
-            const duration =  numberWithCommas(Math.round(timeEnable)) + " ms"
-            console.clear()
-            logger.info("Validating extent " + e.xmin)
-            console.table({duration}) 
 
-        })
-        
-        
+        for (let i = 0; i < fishnet.length; i ++) {
+
+            for (let j = 0 ; j < fishnet[i].length; j++) {
+                
+                fishnet[i][j].content = '⚙️'
+                const e =  fishnet[i][j].extent
+            
+                    try {
+
+                
+                    const fromDate = new Date(); 
+                    console.log("Validating Extent " + JSON.stringify(e));
+                    printFishnet(fishnet)
+
+                    const result = await un.validateNetworkTopology(  e)
+                    const toDate = new Date();
+                    const timeEnable = toDate.getTime() - fromDate.getTime();
+                    const duration =  numberWithCommas(Math.round(timeEnable)) + " ms"
+                    result.duration = duration
+                    console.table(result) 
+                    //console.log(`add_env(${ JSON.stringify(e)})`)
+                    fishnet[i][j].content = '✔️'
+
+                    }
+                    catch(ex){
+                        console.log(JSON.stringify(ex))
+                    }
+
+        }
+                
+    }
+    console.log("Done")
+
     },
 
     "^topology --validate$": async () => {
@@ -464,6 +502,8 @@ const inputs = {
    "^export subnetworks --all --folder .*$|^export subnetworks --all$" : async input => {
  
         let subnetworks 
+        let sort = "asc";
+        if (input.indexOf("--desc") > 0) sort = "desc"
         //create folder
         const file = input.match(/--folder .*/gm)
         let inputDir = "Exported"
@@ -471,11 +511,17 @@ const inputs = {
              inputDir = file[0].replace("--folder ", "")
         //create directory if doesn't exists
         if (!fs.existsSync(inputDir))  fs.mkdirSync(inputDir)
- 
+        let exportedSubnetworks = [];
+
         do {
 
+            let exportedSubnetworksWhereClause = ""
+            
+            if (exportedSubnetworks.length > 0 )
+                exportedSubnetworksWhereClause = " AND SUBNETWORKNAME NOT IN (" + exportedSubnetworks.join(",") + ")"
+ 
             logger.info("Querying all subnetworks that are clean.");
-            subnetworks = await un.queryDistinct(500002, "domainnetworkname,tiername,subnetworkname", "isdirty=0","domainnetworkname,tiername,subnetworkname");
+            subnetworks = await un.queryDistinct(500002, "domainnetworkname,tiername,subnetworkname", "isdirty=0 " + exportedSubnetworksWhereClause,`domainnetworkname  ${sort},tiername ${sort},subnetworkname ${sort}`);
             logger.info(`Discovered ${subnetworks.features.length} subnetworks that can be exported.`);
             for (let i = 0;  i < subnetworks.features.length; i++) {
                 const f = subnetworks.features[i]
@@ -485,10 +531,10 @@ const inputs = {
                 const fromDate = new Date();
                 
                 const subnetworkResult = await un.exportSubnetworks(v(f.attributes,"domainNetworkName"), v(f.attributes,"tierName"), v(f.attributes,"subnetworkName"),false);
-                 
-                
+                  
                 //code
-    
+                exportedSubnetworks.push("'" + v(f.attributes,"subnetworkName") + "'")
+
                 const toDate = new Date();
                 const timeEnable = toDate.getTime() - fromDate.getTime();
                 subnetworkResult.duration =  numberWithCommas(timeEnable) + " ms"
@@ -498,6 +544,7 @@ const inputs = {
                     logger.info("Export subnetwork failed " + JSON.stringify(subnetworkResult))
                     continue;
                 }
+
     
                 //fetch the json and write it to disk 
                 const subContent = await fetch(subnetworkResult.url);
@@ -512,7 +559,7 @@ const inputs = {
     
             }
         }
-        while (subnetworks.features.length > 0)
+        while (subnetworks?.features?.length > 0)
        
     },
    
@@ -565,9 +612,13 @@ const inputs = {
     },
     "^trace --subnetwork": async input => {
         //get subnetwork name
+        try {
 
+             
         const fromDate = new Date();
-       
+        const inputDir= "Exported"
+        let full = true;
+      //  if (input.indexOf("--full") > 0) full = true
 
         const inputParam = input.match(/--subnetwork .*/gm)
         let subnetworkName = null;
@@ -585,8 +636,44 @@ const inputs = {
         const newResult = {}
         newResult.duration =  numberWithCommas(Math.round(timeRun)) + " ms"
         newResult.elementsCount = result.traceResults.elements.length;
-        console.table(newResult) 
+        const traceRes = {}
+        logger.info("Grouping trace results...")
+         result.traceResults.elements.forEach(e => {
+            const layerid = un.getLayerIdfromSourceId(e.networkSourceId).layerId
+            if (!traceRes["l" + layerid])
+                traceRes["l" + layerid] = []
+            traceRes["l" + layerid].push(e.objectId)
+          
+        })
+        console.table(newResult)
 
+        //if the trace is full turn around the pull all features
+        if (full){
+
+            //loop through all layers and query
+            logger.info("Removing duplicates")
+                
+            console.log(traceRes)
+             
+            //send all queries async 
+            const allQueries = []
+            Object.keys(traceRes).forEach (k => {
+                traceRes[k] = [...new Set(traceRes[k])]
+                allQueries.push(un.query(k.replace("l",""), "", k, traceRes[k]))
+            })
+            logger.info(`${Object.keys(traceRes)} queries sent, waiting for response..`)
+            const result = await Promise.all(allQueries)
+            logger.info("All queries returned.")
+            fs.writeFileSync(`${inputDir}/traceout${subnetworkName}.json`, JSON.stringify(result))
+            logger.info(`Result written to file ${inputDir}/traceout${subnetworkName}.json`)
+
+            //logger.info(JSON.stringify(result))
+            
+        }
+    }
+        catch(ex){
+            logger.error(JSON.stringify(ex))
+        }
     },
     "^export subnetworks --deleted$" : async input => {
 
@@ -1346,3 +1433,27 @@ export async function run (){
 
 
 
+function printFishnet(fishnet) {
+
+    //y is flipped x is ok
+    //i = 0 that is i = fishnet.length -1
+    for (let i =0; i < fishnet.length ; i++){
+        
+        for (let j = 0; j < fishnet[i].length;j++) 
+            process.stdout.write("+------");
+
+        process.stdout.write("+\n")
+
+        for (let j = 0; j < fishnet[i].length;j++)
+            process.stdout.write(`|   ${fishnet[fishnet.length - 1 - i][j].content}  `);
+ 
+        process.stdout.write("|\n")
+    }
+
+
+    for (let j = 0; j < fishnet.length;j++) 
+        process.stdout.write("+------");
+    
+    process.stdout.write("+\n\n")
+
+}

package/makerequest.mjs

@@ -28,25 +28,28 @@ export function makeRequest (opts) {
 
         let f;
         try {
-            const nodeFetch = await import ("node-fetch");
+ 
+            let nodeFetch = await import ("node-fetch");
             f = nodeFetch.default;
         }
         catch(ex) {
             f = fetch;
         }
- 
-
+         
+        let jsonRes
+        
        const result = await f(opts.url, {
            "method" : opts.method,
             "headers":  headers,
             "body": params
        });
 
-       const jsonRes  = await result.json();
-      
+        jsonRes  = await result.json();
+        
         if (typeof jsonRes !== "object") 
             jsonRes = JSON.parse(jsonRes);
 
+ 
         resolve(jsonRes);
     
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "un-cli",
-  "version": "0.0.73",
+  "version": "0.0.76",
   "description": "Command line interface for working with ArcGIS Utility Network Extension",
   "main": "index.mjs",
   "bin": {

package/utilitynetwork.node.mjs

@@ -21,12 +21,14 @@ export class UtilityNetwork {
             this.featureServiceUrl = featureServiceUrl;
             this.token = token;         
             this.gdbVersion = gdbVersion;
+            this.sourceMapping = {}
         }
 
       ///first function one should call after creating an instance of a utility network
        load ()
         {
             let thisObj = this;
+            
             return new Promise (function (resolve, reject)
             {
                 //run async mode
@@ -68,7 +70,30 @@ export class UtilityNetwork {
             
                     thisObj.dataElement = undataElement.layerDataElements[0].dataElement;
                     thisObj.layerDefinition = unLayerDef
-                    thisObj.subnetLineLayerId = thisObj.getSubnetLineLayerId();                    
+                    thisObj.subnetLineLayerId = thisObj.getSubnetLineLayerId();     
+                    
+                    //build out source mapping hash
+                    let domainNetworks = thisObj.dataElement.domainNetworks;
+                    let layerObj = undefined;
+
+                    for (let i = 0; i < domainNetworks.length; i ++)
+                    {
+                        let domainNetwork = domainNetworks[i];
+                        for (let j = 0; j < domainNetwork.junctionSources.length; j ++)
+                            {  
+                                layerObj =  {type: domainNetwork.junctionSources[j].shapeType, layerId: domainNetwork.junctionSources[j].layerId}
+                                layerObj.type = layerObj.type.replace("esriGeometry", "").toLowerCase();
+                                thisObj.sourceMapping["s" + domainNetwork.junctionSources[j].sourceId] = layerObj
+                            }
+
+                        for (let j = 0; j < domainNetwork.edgeSources.length; j ++)
+                             { 
+                                layerObj = {type: domainNetwork.edgeSources[j].shapeType, layerId: domainNetwork.edgeSources[j].layerId} 
+                                layerObj.type = layerObj.type.replace("esriGeometry", "").toLowerCase();
+                                thisObj.sourceMapping["s" + domainNetwork.edgeSources[j].sourceId] = layerObj
+                            }
+                    }
+  
                     resolve(thisObj);
                 }
                 else
@@ -333,16 +358,49 @@ export class UtilityNetwork {
             if (objectids != undefined) 
             queryJson.objectIds = objectids;
             queryJson.layerId = layerId
-            return new Promise((resolve, reject) => {
+            return new Promise(async(resolve, reject) => {
+
+
+                
+                let rowsJson;
+                let allRowsJson;
+                let recordOffset = 0;
+
+                try {
+
+                    do {
+                        queryJson.resultOffset = recordOffset;
+                        rowsJson  = await makeRequest({method: 'POST', params: queryJson, url: this.featureServiceUrl + "/" + layerId + "/query"}) ;                
+                        recordOffset = recordOffset + resultRecordCount + 1;
+                        rowsJson.obj = obj;
+
+                        if (!allRowsJson)
+                          allRowsJson = rowsJson
+                        else
+                          allRowsJson?.features.push(...rowsJson?.features)
+                        //page the result until done
+                    }
+                    while(rowsJson?.exceededTransferLimit == true)
+
+                    resolve(allRowsJson);
+                }
+                catch(ex){
+                    reject("failed to query" + JSON.stringify(ex))
+                }
 
-                makeRequest({method: 'POST', params: queryJson, url: this.featureServiceUrl + "/" + layerId + "/query"}).then(rowsJson=> {                  
-                    rowsJson.obj = obj;
-                    resolve(rowsJson);
-                }).catch(rej => reject("failed to query"));
                 
+           
+
             });
                 
-              
+
+            
+       // }
+       // while(jsonRes?.exceededTransferLimit == true)
+      // if (!allJsonRes)
+     //  allJsonRes = jsonRes
+ // else
+      // allJsonRes?.features.push(...jsonRes?.features)
         }
         //get the terminal configuration using the id
         getTerminalConfiguration(terminalConfigurationId)
@@ -538,8 +596,10 @@ export class UtilityNetwork {
         //get layer id from Source Id used to map sourceid to layer id
         getLayerIdfromSourceId(sourceId)
         { 
+ 
+            return this.sourceMapping["s" + sourceId]
+            /*
             let domainNetworks = this.dataElement.domainNetworks;
-            let layerObj = undefined;
 
             for (let i = 0; i < domainNetworks.length; i ++)
             {
@@ -561,6 +621,7 @@ export class UtilityNetwork {
 
             if (layerObj != undefined)
                 layerObj.type = layerObj.type.replace("esriGeometry", "").toLowerCase();
+            */
 
             return layerObj;
         }
@@ -958,6 +1019,27 @@ export class UtilityNetwork {
 
  
 
+        updateIsConnected(async=false) {
+
+            let thisObj = this;  
+            let ar = thisObj.featureServiceUrl.split("/");
+             ar[ar.length-1]="UtilityNetworkServer";
+             let updateisconnectedURL = ar.join("/") + "/updateIsConnected"
+     
+               let payload = {
+                  f: "json",  
+                  token: this.token,  
+                  async: async
+                }
+              let un = this;
+            
+            return makeRequest({method:'POST', params: payload, url: updateisconnectedURL })
+        
+ 
+
+        }
+
+
         updateSubnetworks(domainNetworkName, tierName, subnetworkName, async=false) {
 
             let thisObj = this;