pinokiod 7.1.68 → 7.1.70

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "7.1.68",
+  "version": "7.1.70",
   "description": "",
   "main": "index.js",
   "scripts": {

package/server/index.js

@@ -79,6 +79,8 @@ const { createInjectRouter, resolveInjectList } = require("./lib/inject_router")
 const { createTaskPackageService } = require("./lib/task_packages")
 const { createTaskWorkspaceLinkService } = require("./lib/task_workspace_links")
 const { createContentValidationService } = require("./lib/content_validation")
+const { buildSecureRouterDebugSnapshot, createSecureRouterDebugStore } = require("./lib/secure_router_debug")
+const { ensureSecureRouterReady } = require("./lib/secure_router_ready")
 const AppRegistryService = require("./lib/app_registry")
 const AppLogService = require("./lib/app_logs")
 const AppSearchService = require("./lib/app_search")
@@ -253,6 +255,7 @@ class Server {
       updated_at: new Date().toISOString()
     }
     this.startup_status_run_id = 0
+    this.secure_router_debug = createSecureRouterDebugStore()
     this.installFatalHandlers()
   }
   setStartupStatus(patch = {}) {
@@ -5480,53 +5483,45 @@ class Server {
     return list
   }
   async check_router_up() {
-    let response
-    let config
+    // check if caddy is runnign properly
+    //    try https://pinokio.localhost
+    //    if it works, proceed
+    //    if not, redirect
+    let https_running = false
     try {
       let res = await axios.get(`http://127.0.0.1:2019/config/`, {
         timeout: 2000
       })
-      config = res.data
-    } catch (e) {
-      response = {
-        stage: "caddy_admin_unavailable",
-        error: "caddy admin unavailable",
-        detail: e && e.message ? e.message : String(e)
+      let test = /pinokio\.localhost/.test(JSON.stringify(res.data))
+      if (test) {
+        https_running = true
       }
+    } catch (e) {
+      return { error: "caddy admin unavailable" }
     }
-    if (!response) {
-      if (config === null || typeof config === "undefined") {
-        response = {
-          stage: "caddy_config_empty",
-          error: "caddy config not yet loaded"
-        }
-      } else if (!/pinokio\.localhost/.test(JSON.stringify(config))) {
-        response = {
-          stage: "caddy_route_missing",
-          error: "pinokio.localhost missing from caddy config"
-        }
-      } else {
-        let router_running = false
-        let router = this.kernel.router.published()
-        for (let ip in router) {
-          let domains = router[ip]
-          if (domains.includes("pinokio.localhost")) {
-            router_running = true
-            break
-          }
-        }
-        if (!router_running) {
-          response = {
-            stage: "pinokio_router_missing",
-            error: "pinokio.localhost not yet published by Pinokio router"
-          }
-        }
+//    console.log({ https_running })
+    if (!https_running) {
+      return { error: "pinokio.host not yet available" }
+    }
+
+    // check if pinokio.localhost router is running
+    let router_running = false
+    let router = this.kernel.router.published()
+    for(let ip in router) {
+      let domains = router[ip]
+      if (domains.includes("pinokio.localhost")) {
+        router_running = true
+        break
       }
     }
-    if (!response) {
-      response = { success: true, stage: "ready" }
+    if (!router_running) {
+      return { error: "pinokio.localhost not yet available" }
     }
-    return response
+
+    return { success: true }
+  }
+  async ensureSecureRouterReady(timeout = 120000, interval = 500) {
+    return ensureSecureRouterReady(this, timeout, interval)
   }
   async start(options) {
     this.debug = false
@@ -5797,6 +5792,12 @@ class Server {
           phase: "initializing_environment"
         })
         await Environment.init({}, this.kernel)
+        if (this.kernel && this.kernel.peer && this.kernel.peer.https_active) {
+          setStartupStatus({
+            phase: "waiting_for_secure_router"
+          })
+          await this.ensureSecureRouterReady()
+        }
         setStartupStatus({
           sys_ready: this.getStartupStatus().sys_ready,
           managed_ready: true,
@@ -11438,8 +11439,36 @@ class Server {
         return
       }
 
-      const routerStatus = await this.check_router_up()
-      if (!routerStatus.success) {
+      let https_running = false
+      try {
+        let res = await axios.get(`http://127.0.0.1:2019/config/`, {
+          timeout: 2000
+        })
+        let test = /pinokio\.localhost/.test(JSON.stringify(res.data))
+        if (test) {
+          https_running = true
+        }
+      } catch (e) {
+        console.log(e)
+      }
+      if (!https_running) {
+//        res.json({ error: "pinokio.host not yet available" })
+        res.redirect("/setup/connect?callback=/connect/" + req.params.provider)
+        return
+      }
+
+      // check if pinokio.localhost router is running
+      let router_running = false
+      let router = this.kernel.router.published()
+      for(let ip in router) {
+        let domains = router[ip]
+        if (domains.includes("pinokio.localhost")) {
+          router_running = true
+          break
+        }
+      }
+      if (!router_running) {
+//        res.json({ error: "pinokio.localhost not yet available" })
         res.redirect("/setup/connect?callback=/connect/" + req.params.provider)
         return
       }
@@ -15422,6 +15451,9 @@ class Server {
     this.app.get("/pinokio/startup_status", ex((req, res) => {
       res.json(this.getStartupStatus())
     }))
+    this.app.get("/pinokio/secure_router_debug", ex(async (req, res) => {
+      res.json(await buildSecureRouterDebugSnapshot(this, this.secure_router_debug))
+    }))
     this.app.get("/pinokio/requirements_ready", ex((req, res) => {
       res.json(this.getStartupStatus())
     }))
@@ -15557,13 +15589,6 @@ class Server {
           server_ready: true,
           phase: this.getStartupStatus().phase === "idle" ? "ready" : this.getStartupStatus().phase
         })
-        if (this.kernel && this.kernel.peer && this.kernel.peer.https_active) {
-          this.kernel.refresh(true).then(() => {
-            return this.check_router_up()
-          }).catch((error) => {
-            console.warn("[Pinokiod] initial secure router refresh failed", error && error.message ? error.message : error)
-          })
-        }
         resolve()
       });
       this.httpTerminator = createHttpTerminator({

package/server/lib/secure_router_debug.js

@@ -0,0 +1,116 @@
+function createSecureRouterDebugStore() {
+  return {
+    events: [],
+    state: {
+      active: false,
+      started_at: null,
+      updated_at: null,
+      last_status: null,
+      caddy_start_attempted: false,
+      caddy_start_finished: false,
+      caddy_start_error: null,
+      refresh_attempted: false,
+      refresh_finished: false,
+      refresh_error: null,
+    }
+  }
+}
+
+function clone(value) {
+  return JSON.parse(JSON.stringify(value))
+}
+
+function recordSecureRouterDebug(store, message, patch = {}) {
+  if (!store) {
+    return
+  }
+  const timestamp = new Date().toISOString()
+  store.state = {
+    ...(store.state || {}),
+    ...patch,
+    updated_at: timestamp
+  }
+  store.events.push({
+    at: timestamp,
+    message
+  })
+  if (store.events.length > 20) {
+    store.events.shift()
+  }
+}
+
+async function buildSecureRouterDebugSnapshot(server, store) {
+  const kernel = server && server.kernel ? server.kernel : null
+  const peer = kernel && kernel.peer ? kernel.peer : null
+  const router = kernel && kernel.router && typeof kernel.router.published === "function"
+    ? (kernel.router.published() || {})
+    : {}
+  const routerDialKeys = Object.keys(router)
+  const hasPinokioLocalhost = routerDialKeys.some((dial) => {
+    const domains = Array.isArray(router[dial]) ? router[dial] : []
+    return domains.includes("pinokio.localhost")
+  })
+  const caddy = kernel && kernel.bin && kernel.bin.mod
+    ? kernel.bin.mod.caddy
+    : null
+  let caddyInstalled = null
+  let caddyRunning = null
+  try {
+    if (caddy && typeof caddy.installed === "function") {
+      caddyInstalled = await caddy.installed()
+    }
+  } catch (error) {
+    caddyInstalled = { error: error && error.message ? error.message : String(error) }
+  }
+  try {
+    if (caddy && typeof caddy.running === "function") {
+      caddyRunning = await caddy.running()
+    }
+  } catch (error) {
+    caddyRunning = { error: error && error.message ? error.message : String(error) }
+  }
+  const startupStatus = server && typeof server.getStartupStatus === "function"
+    ? server.getStartupStatus()
+    : null
+  const state = store && store.state ? clone(store.state) : {}
+  const events = store && Array.isArray(store.events) ? clone(store.events) : []
+  return {
+    wait: state,
+    flags: {
+      peer_active: peer ? !!peer.peer_active : null,
+      https_active: peer ? !!peer.https_active : null,
+      active: peer ? !!peer.active : null,
+    },
+    caddy: {
+      installed: caddyInstalled,
+      running: caddyRunning,
+      start_attempted: !!state.caddy_start_attempted,
+      start_finished: !!state.caddy_start_finished,
+      start_error: state.caddy_start_error || null,
+    },
+    peer: {
+      host: peer && peer.host ? peer.host : null,
+      name: peer && peer.name ? peer.name : null,
+      has_info: !!(peer && peer.info),
+      has_host_info: !!(peer && peer.info && peer.host && peer.info[peer.host]),
+      refreshing: !!(peer && peer.refreshing),
+    },
+    router: {
+      published_count: routerDialKeys.length,
+      published_dials: routerDialKeys.slice(0, 10),
+      has_pinokio_localhost: hasPinokioLocalhost,
+      refresh_attempted: !!state.refresh_attempted,
+      refresh_finished: !!state.refresh_finished,
+      refresh_error: state.refresh_error || null,
+      last_status: state.last_status || null,
+    },
+    startup: startupStatus,
+    events
+  }
+}
+
+module.exports = {
+  buildSecureRouterDebugSnapshot,
+  createSecureRouterDebugStore,
+  recordSecureRouterDebug,
+}

package/server/lib/secure_router_ready.js

@@ -0,0 +1,96 @@
+const { createSecureRouterDebugStore, recordSecureRouterDebug } = require("./secure_router_debug")
+
+async function ensureSecureRouterReady(server, timeout = 120000, interval = 500) {
+  if (!(server.kernel && server.kernel.peer && server.kernel.peer.https_active)) {
+    return { success: true, stage: "disabled" }
+  }
+  server.secure_router_debug = createSecureRouterDebugStore()
+  recordSecureRouterDebug(server.secure_router_debug, "wait started", {
+    active: true,
+    started_at: new Date().toISOString(),
+    last_status: null,
+    caddy_start_attempted: false,
+    caddy_start_finished: false,
+    caddy_start_error: null,
+    refresh_attempted: false,
+    refresh_finished: false,
+    refresh_error: null,
+  })
+  const caddy = server.kernel && server.kernel.bin && server.kernel.bin.mod
+    ? server.kernel.bin.mod.caddy
+    : null
+  const deadline = Date.now() + timeout
+  let startedCaddyManually = false
+  let refreshedRouter = false
+  let lastStatus = null
+  let lastStatusSummary = null
+  while (Date.now() < deadline) {
+    lastStatus = await server.check_router_up()
+    const statusSummary = lastStatus && lastStatus.success
+      ? "success"
+      : (lastStatus && lastStatus.error ? lastStatus.error : JSON.stringify(lastStatus))
+    if (statusSummary !== lastStatusSummary) {
+      lastStatusSummary = statusSummary
+      recordSecureRouterDebug(server.secure_router_debug, `check_router_up -> ${statusSummary}`, {
+        last_status: lastStatus
+      })
+    }
+    if (lastStatus && lastStatus.success) {
+      recordSecureRouterDebug(server.secure_router_debug, "wait success", {
+        active: false
+      })
+      return lastStatus
+    }
+    if (
+      lastStatus &&
+      lastStatus.error === "caddy admin unavailable" &&
+      !startedCaddyManually &&
+      caddy &&
+      typeof caddy.start === "function"
+    ) {
+      startedCaddyManually = true
+      recordSecureRouterDebug(server.secure_router_debug, "manual caddy.start() attempted", {
+        caddy_start_attempted: true
+      })
+      try {
+        await caddy.start()
+        recordSecureRouterDebug(server.secure_router_debug, "manual caddy.start() finished", {
+          caddy_start_finished: true
+        })
+      } catch (e) {
+        recordSecureRouterDebug(server.secure_router_debug, "manual caddy.start() failed", {
+          caddy_start_error: e && e.message ? e.message : String(e)
+        })
+        throw e
+      }
+      continue
+    }
+    if (!refreshedRouter) {
+      refreshedRouter = true
+      recordSecureRouterDebug(server.secure_router_debug, "kernel.refresh(true) attempted", {
+        refresh_attempted: true
+      })
+      await server.kernel.refresh(true).then(() => {
+        recordSecureRouterDebug(server.secure_router_debug, "kernel.refresh(true) finished", {
+          refresh_finished: true
+        })
+      }).catch((e) => {
+        recordSecureRouterDebug(server.secure_router_debug, "kernel.refresh(true) failed", {
+          refresh_error: e && e.message ? e.message : String(e)
+        })
+      })
+      continue
+    }
+    await new Promise((resolve) => {
+      setTimeout(resolve, interval)
+    })
+  }
+  recordSecureRouterDebug(server.secure_router_debug, "wait timeout", {
+    active: false
+  })
+  throw new Error(lastStatus && lastStatus.error ? lastStatus.error : "secure router did not come up")
+}
+
+module.exports = {
+  ensureSecureRouterReady,
+}

package/server/public/setup-wait-debug.js

@@ -0,0 +1,108 @@
+(function () {
+  const HISTORY_LIMIT = 12
+
+  function create(waitRoot) {
+    const state = {
+      events: [],
+      phase: null,
+      router: null,
+      startupError: null,
+    }
+
+    function ensurePanel() {
+      if (!waitRoot) {
+        return null
+      }
+      let panel = waitRoot.querySelector(".wait-debug")
+      if (!panel) {
+        panel = document.createElement("div")
+        panel.className = "wait-debug"
+        panel.style.marginTop = "16px"
+        panel.style.padding = "12px"
+        panel.style.border = "1px solid rgba(255,255,255,0.12)"
+        panel.style.borderRadius = "8px"
+        panel.style.background = "rgba(255,255,255,0.04)"
+        panel.style.textAlign = "left"
+        panel.innerHTML = [
+          '<div style="font-size:12px;font-weight:600;letter-spacing:0.04em;text-transform:uppercase;opacity:0.75;margin-bottom:8px;">Debug</div>',
+          '<pre class="wait-debug-body" style="margin:0;text-align:left;white-space:pre-wrap;word-break:break-word;font-size:12px;line-height:1.5;opacity:0.9;"></pre>'
+        ].join("")
+        waitRoot.appendChild(panel)
+      }
+      return panel.querySelector(".wait-debug-body")
+    }
+
+    function event(message) {
+      const timestamp = new Date().toLocaleTimeString()
+      const entry = `[${timestamp}] ${message}`
+      if (state.events[state.events.length - 1] !== entry) {
+        state.events.push(entry)
+        if (state.events.length > HISTORY_LIMIT) {
+          state.events.shift()
+        }
+      }
+    }
+
+    function track({ routerStatus, startupStatus } = {}) {
+      if (startupStatus && startupStatus.phase && startupStatus.phase !== state.phase) {
+        state.phase = startupStatus.phase
+        event(`phase -> ${startupStatus.phase}`)
+      }
+      const routerSummary = routerStatus
+        ? (routerStatus.success ? "success" : (routerStatus.error || JSON.stringify(routerStatus)))
+        : null
+      if (routerSummary && routerSummary !== state.router) {
+        state.router = routerSummary
+        event(`router -> ${routerSummary}`)
+      }
+      if (startupStatus && startupStatus.error && startupStatus.error !== state.startupError) {
+        state.startupError = startupStatus.error
+        event(`startup error -> ${startupStatus.error}`)
+      }
+    }
+
+    function render({ startedAt, routerStatus, startupStatus, serverDebug } = {}) {
+      const body = ensurePanel()
+      if (!body) {
+        return
+      }
+      const elapsedSeconds = startedAt ? Math.max(0, Math.floor((Date.now() - startedAt) / 1000)) : 0
+      const lines = [
+        `Elapsed: ${elapsedSeconds}s`,
+        `Startup phase: ${startupStatus && startupStatus.phase ? startupStatus.phase : "-"}`,
+        `Startup error: ${startupStatus && startupStatus.error ? startupStatus.error : "-"}`,
+        `Router check: ${routerStatus ? JSON.stringify(routerStatus) : "-"}`,
+      ]
+      if (serverDebug) {
+        lines.push(`Flags: ${JSON.stringify(serverDebug.flags || null)}`)
+        lines.push(`Caddy: ${JSON.stringify(serverDebug.caddy || null)}`)
+        lines.push(`Peer: ${JSON.stringify(serverDebug.peer || null)}`)
+        lines.push(`Router: ${JSON.stringify(serverDebug.router || null)}`)
+        lines.push(`Wait: ${JSON.stringify(serverDebug.wait || null)}`)
+      }
+      if (state.events.length > 0) {
+        lines.push("")
+        lines.push("Recent page events:")
+        lines.push(...state.events)
+      }
+      if (serverDebug && Array.isArray(serverDebug.events) && serverDebug.events.length > 0) {
+        lines.push("")
+        lines.push("Recent server events:")
+        for (const entry of serverDebug.events) {
+          lines.push(`[${entry.at}] ${entry.message}`)
+        }
+      }
+      body.textContent = lines.join("\n")
+    }
+
+    return {
+      event,
+      render,
+      track,
+    }
+  }
+
+  window.PinokioSetupWaitDebug = {
+    create
+  }
+})();

package/server/views/setup.ejs

@@ -13,6 +13,7 @@
 <% if (agent === "electron") { %>
   <link href="/electron.css" rel="stylesheet"/>
 <% } %>
+<script src="/setup-wait-debug.js"></script>
 <style>
 html {
   scroll-behavior: smooth;
@@ -243,17 +244,45 @@ document.addEventListener("DOMContentLoaded", async () => {
     const waitRoot = document.querySelector(".requirements .content")
     const waitLabel = document.querySelector(".requirements .content .loading span")
     const WAIT_TIMEOUT_MS = 120000
-    const ROUTER_STAGE_MESSAGES = {
-      caddy_admin_unavailable: "Restarting Pinokio... (waiting for Caddy)",
-      caddy_config_empty: "Restarting Pinokio... (publishing secure router)",
-      caddy_route_missing: "Restarting Pinokio... (loading pinokio.localhost)",
-      pinokio_router_missing: "Restarting Pinokio... (waiting for Pinokio router)"
-    }
+    const waitDebug = window.PinokioSetupWaitDebug ? window.PinokioSetupWaitDebug.create(waitRoot) : null
+    let lastRouterStatus = null
+    let lastStartupStatus = null
+    let lastServerDebug = null
     const setWaitLabel = (message) => {
       if (waitLabel) {
         waitLabel.textContent = message
       }
     }
+    const pushWaitDebugEvent = (message) => {
+      if (waitDebug) {
+        waitDebug.event(message)
+      }
+    }
+    const renderWaitDebug = ({ startedAt, routerStatus, startupStatus, serverDebug } = {}) => {
+      if (typeof routerStatus !== "undefined") {
+        lastRouterStatus = routerStatus
+      }
+      if (typeof startupStatus !== "undefined") {
+        lastStartupStatus = startupStatus
+      }
+      if (typeof serverDebug !== "undefined") {
+        lastServerDebug = serverDebug
+      }
+      if (!waitDebug) {
+        return
+      }
+      waitDebug.render({
+        startedAt,
+        routerStatus: lastRouterStatus,
+        startupStatus: lastStartupStatus,
+        serverDebug: lastServerDebug,
+      })
+    }
+    const trackWaitDebug = ({ routerStatus, startupStatus } = {}) => {
+      if (waitDebug) {
+        waitDebug.track({ routerStatus, startupStatus })
+      }
+    }
     const showWaitError = (message) => {
       if (waitRoot) {
         waitRoot.innerHTML = `
@@ -265,6 +294,7 @@ document.addEventListener("DOMContentLoaded", async () => {
             <a class='btn' href='${location.href}'>Retry</a>
           </div>
         `
+        renderWaitDebug()
       } else {
         alert(message)
       }
@@ -272,6 +302,8 @@ document.addEventListener("DOMContentLoaded", async () => {
 
     try {
       setWaitLabel("Enabling secure localhost access...")
+      pushWaitDebugEvent("saving secure localhost flags")
+      renderWaitDebug()
       const networkResponse = await fetch("/network", {
         method: "post",
         headers: { "Content-Type": "application/json" },
@@ -285,8 +317,12 @@ document.addEventListener("DOMContentLoaded", async () => {
       if (networkResponse && networkResponse.error) {
         throw new Error(networkResponse.error)
       }
+      pushWaitDebugEvent("secure localhost flags saved")
+      renderWaitDebug()
 
       setWaitLabel("Restarting Pinokio...")
+      pushWaitDebugEvent("restart requested")
+      renderWaitDebug()
       try {
         await fetch("/restart", {
           method: "post"
@@ -298,13 +334,15 @@ document.addEventListener("DOMContentLoaded", async () => {
       const startedAt = Date.now()
       let lastKnownError = null
       let lastKnownPhase = null
+      renderWaitDebug({ startedAt })
 
       await new Promise((resolve, reject) => {
         const interval = setInterval(async () => {
           try {
-            const [routerStatus, startupStatus] = await Promise.all([
+            const [routerStatus, startupStatus, serverDebug] = await Promise.all([
               fetch("/check_router_up").then((res) => res.json()),
-              fetch("/pinokio/startup_status").then((res) => res.json()).catch(() => null)
+              fetch("/pinokio/startup_status").then((res) => res.json()).catch(() => null),
+              fetch("/pinokio/secure_router_debug").then((res) => res.json()).catch(() => null)
             ])
 
             if (routerStatus && routerStatus.success) {
@@ -319,16 +357,15 @@ document.addEventListener("DOMContentLoaded", async () => {
             if (startupStatus && startupStatus.error) {
               lastKnownError = startupStatus.error
             }
+            trackWaitDebug({ routerStatus, startupStatus })
             if (startupStatus && startupStatus.phase) {
-              lastKnownPhase = (routerStatus && routerStatus.stage) || startupStatus.phase
-              const routerStageMessage = routerStatus && routerStatus.stage
-                ? ROUTER_STAGE_MESSAGES[routerStatus.stage]
-                : null
-              const phaseMessage = routerStageMessage || (startupStatus.phase === "ready"
+              lastKnownPhase = startupStatus.phase
+              const phaseMessage = startupStatus.phase === "ready"
                 ? "Secure router is almost ready..."
-                : `Restarting Pinokio... (${startupStatus.phase.replace(/_/g, " ")})`)
+                : `Restarting Pinokio... (${startupStatus.phase.replace(/_/g, " ")})`
               setWaitLabel(phaseMessage)
             }
+            renderWaitDebug({ startedAt, routerStatus, startupStatus, serverDebug })
 
             if (startupStatus && startupStatus.phase === "error") {
               clearInterval(interval)
@@ -339,10 +376,14 @@ document.addEventListener("DOMContentLoaded", async () => {
             if ((Date.now() - startedAt) >= WAIT_TIMEOUT_MS) {
               clearInterval(interval)
               const detail = lastKnownError || lastKnownPhase || "secure localhost router did not come up"
+              pushWaitDebugEvent(`timeout -> ${detail}`)
+              renderWaitDebug({ startedAt, routerStatus, startupStatus })
               reject(new Error(`Timed out waiting for restart: ${detail}`))
             }
           } catch (error) {
             lastKnownError = error && error.message ? error.message : String(error)
+            pushWaitDebugEvent(`poll error -> ${lastKnownError}`)
+            renderWaitDebug({ startedAt })
             if ((Date.now() - startedAt) >= WAIT_TIMEOUT_MS) {
               clearInterval(interval)
               reject(new Error(`Timed out waiting for restart: ${lastKnownError}`))