pinokiod 8.0.71 → 8.0.73

package/kernel/vault/automatic_scans.js

@@ -7,7 +7,7 @@ const {
   ENTRY_BATCH_SIZE
 } = require("./constants")
 const { statMany } = require("./walker")
-const { sameSnapshot } = require("./snapshot")
+const { fileSnapshot, sameSnapshot } = require("./snapshot")
 const {
   cancelledError,
   isPathError
@@ -52,10 +52,12 @@ class AutomaticScans {
     this.manualDepth = 0
     this.observedApps = new Set()
     this.changedPaths = new Map()
+    this.changedPathIndex = new Map()
     this.pendingStops = new Map()
     this.lifecycleWork = new Set()
     this.stopSettleMs = STOP_SETTLE_MS
     this.drainQueued = false
+    this.drainRequested = false
     this.waitingFor = null
     this.listeners = new Set()
     this.appTransitions = new Map()
@@ -136,6 +138,7 @@ class AutomaticScans {
     }
     this.observedApps.clear()
     this.changedPaths.clear()
+    this.changedPathIndex.clear()
     if (watcherError) throw watcherError
   }
 
@@ -152,7 +155,8 @@ class AutomaticScans {
       if (!event || !["create", "update", "delete"].includes(event.type)) {
         continue
       }
-      this.recordChangedPath(event.path)
+      if (event.type === "delete") this.removeChangedPath(event.path)
+      else this.recordChangedPath(event.path)
     }
   }
 
@@ -171,25 +175,105 @@ class AutomaticScans {
       paths = new Set()
       this.changedPaths.set(app, paths)
     }
+    let index = this.changedPathIndex.get(app)
+    if (!index) {
+      index = { exact: new Map(), descendants: new Map() }
+      this.changedPathIndex.set(app, index)
+    }
+    const key = pathKey(resolved)
+    if (index.exact.has(key)) return true
     paths.add(resolved)
+    index.exact.set(key, resolved)
+    this.addChangedPathToIndex(index.descendants, appRoot, resolved)
+    return true
+  }
+
+  changedPathParentKeys(appRoot, filePath) {
+    const rootKey = pathKey(appRoot)
+    const keys = []
+    let current = path.dirname(path.resolve(filePath))
+    while (true) {
+      const key = pathKey(current)
+      keys.push(key)
+      if (key === rootKey) return keys
+      current = path.dirname(current)
+    }
+  }
+
+  addChangedPathToIndex(index, appRoot, filePath) {
+    for (const key of this.changedPathParentKeys(appRoot, filePath)) {
+      let descendants = index.get(key)
+      if (!descendants) {
+        descendants = new Set()
+        index.set(key, descendants)
+      }
+      descendants.add(filePath)
+    }
+  }
+
+  removeChangedPathFromIndex(index, appRoot, filePath) {
+    index.exact.delete(pathKey(filePath))
+    for (const key of this.changedPathParentKeys(appRoot, filePath)) {
+      const descendants = index.descendants.get(key)
+      if (!descendants) continue
+      descendants.delete(filePath)
+      if (!descendants.size) index.descendants.delete(key)
+    }
+  }
+
+  removeChangedPath(filePath) {
+    if (this.disposed || !this.supported ||
+        typeof filePath !== "string" || !filePath) {
+      return false
+    }
+    const app = this.appForLaunchPath(filePath)
+    if (!app || !this.observedApps.has(app)) return false
+    const paths = this.changedPaths.get(app)
+    if (!paths) return false
+    const resolved = path.resolve(filePath)
+    const appRoot = path.resolve(this.vault.kernel.homedir, "api", app)
+    const index = this.changedPathIndex.get(app)
+    if (!index) return false
+    const key = pathKey(resolved)
+    const exact = index.exact.get(key)
+    const descendants = index.descendants.get(key)
+    if (!exact && !descendants) return false
+    const matches = new Set(descendants || [])
+    if (exact) matches.add(exact)
+    for (const candidate of matches) {
+      paths.delete(candidate)
+      this.removeChangedPathFromIndex(index, appRoot, candidate)
+    }
+    if (!paths.size) this.discardChangedPaths(app)
     return true
   }
 
   discardChangedPaths(app) {
     this.changedPaths.delete(app)
+    this.changedPathIndex.delete(app)
   }
 
   consumeChangedPaths(app, paths) {
     const collected = this.changedPaths.get(app)
     if (!collected) return
-    for (const filePath of paths || []) collected.delete(filePath)
-    if (!collected.size) this.changedPaths.delete(app)
+    const appRoot = path.resolve(this.vault.kernel.homedir, "api", app)
+    const index = this.changedPathIndex.get(app)
+    for (const filePath of paths || []) {
+      const resolved = path.resolve(filePath)
+      if (!collected.delete(resolved) || !index) continue
+      this.removeChangedPathFromIndex(index, appRoot, resolved)
+    }
+    if (!collected.size) this.discardChangedPaths(app)
   }
 
   async candidateThreshold() {
-    const scan = this.vault.registry
-      ? await this.vault.registry.scanFor()
+    const cached = this.vault.lastScanCache &&
+      typeof this.vault.lastScanCache.get === "function"
+      ? this.vault.lastScanCache.get("")
       : null
+    const scan = cached || (this.vault.registry
+      ? await this.vault.registry.scanFor()
+      : null)
     const threshold = scan ? Number(scan.candidate_min_bytes) : NaN
     return Number.isFinite(threshold) && threshold >= 0
       ? threshold
@@ -726,24 +810,38 @@ class AutomaticScans {
   }
 
   schedule() {
-    if (this.disposed || this.drainQueued) return
+    if (this.disposed) return
+    if (this.drainQueued) {
+      this.drainRequested = true
+      return
+    }
     this.drainQueued = true
     queueMicrotask(() => {
-      this.drainQueued = false
-      const pending = this.drain()
+      const pending = (async () => {
+        do {
+          this.drainRequested = false
+          await this.drain()
+        } while (!this.disposed && this.drainRequested)
+      })()
       this.lifecycleWork.add(pending)
       pending.then(
-        () => this.lifecycleWork.delete(pending),
-        () => this.lifecycleWork.delete(pending)
+        () => {
+          this.lifecycleWork.delete(pending)
+          this.drainQueued = false
+          if (this.drainRequested) this.schedule()
+        },
+        (error) => {
+          this.lifecycleWork.delete(pending)
+          this.drainQueued = false
+          this.log("error", {
+            stage: "queue",
+            message: error && error.message ? error.message : String(error)
+          })
+          console.warn("Automatic Disk Saver check failed:",
+            error && error.message ? error.message : error)
+          if (this.drainRequested) this.schedule()
+        }
       )
-      pending.catch((error) => {
-        this.log("error", {
-          stage: "queue",
-          message: error && error.message ? error.message : String(error)
-        })
-        console.warn("Automatic Disk Saver check failed:",
-          error && error.message ? error.message : error)
-      })
     })
   }
 
@@ -799,37 +897,58 @@ class AutomaticScans {
       current = await fs.promises.lstat(entry.path)
     } catch (error) {
       if (!isPathError(error)) throw error
+      if (entry.kind === "changed" && isMissing(error)) {
+        counts.path_errors += 1
+        return null
+      }
       this.logVerificationSkip(active.app, entry, "unreadable", counts, error)
       return null
     }
     this.checkpoint(active)
     if (!current.isFile() || current.isSymbolicLink() ||
-        !sameSnapshot(entry, current)) {
+        current.dev !== entry.dev || current.size !== entry.size) {
       this.logVerificationSkip(
         active.app, entry, "snapshot-changed", counts)
       return null
     }
-    const key = this.automaticHashKey(entry)
-    const reusable = SHA256_RE.test(entry.hash || "")
-      ? entry.hash
-      : memory.get(key)
+    const currentEntry = Object.assign({}, entry, fileSnapshot(current))
+    const key = this.automaticHashKey(currentEntry)
+    const cachedSnapshot = {
+      size: entry.cached_size,
+      mtime: entry.cached_mtime,
+      ctime: entry.cached_ctime,
+      dev: entry.cached_dev,
+      ino: entry.cached_ino
+    }
+    const reusable = memory.get(key) ||
+      (sameSnapshot(entry, current) && SHA256_RE.test(entry.hash || "")
+        ? entry.hash
+        : null) ||
+      (sameSnapshot(cachedSnapshot, current) &&
+        SHA256_RE.test(entry.cached_hash || "")
+        ? entry.cached_hash
+        : null)
     if (reusable) {
       counts.hash_reuses += 1
       memory.set(key, reusable)
-      verifiedHashes.set(pathKey(entry.path), Object.assign({}, entry, {
+      verifiedHashes.set(pathKey(entry.path), Object.assign({}, currentEntry, {
         hash: reusable
       }))
       return reusable
     }
     let verified
     try {
-      verified = await this.vault.scanner.hashStable(entry, {
+      verified = await this.vault.scanner.hashStable(currentEntry, {
         signal: active.controller && active.controller.signal
       })
     } catch (error) {
       if (error && error.code === "EVAULTCANCELLED") throw error
       if (!isPathError(error)) throw error
       counts.hash_failures += 1
+      if (entry.kind === "changed" && isMissing(error)) {
+        counts.path_errors += 1
+        return null
+      }
       this.logVerificationSkip(active.app, entry, "hash-failed", counts, error)
       return null
     }
@@ -842,7 +961,7 @@ class AutomaticScans {
     counts.hashed += 1
     counts.hash_bytes += Math.max(0, Number(verified.result.size) || 0)
     memory.set(key, verified.result.hash)
-    verifiedHashes.set(pathKey(entry.path), Object.assign({}, entry, {
+    verifiedHashes.set(pathKey(entry.path), Object.assign({}, currentEntry, {
       hash: verified.result.hash
     }))
     return verified.result.hash
@@ -872,7 +991,6 @@ class AutomaticScans {
     const paths = [...new Set(active.paths || [])].filter((filePath) =>
       typeof filePath === "string" && inside(root, filePath))
     const verifiedHashes = new Map()
-    await this.vault.registry.beginAutomaticPrecheck(app)
     this.log("check-started", {
       app,
       root,
@@ -880,6 +998,7 @@ class AutomaticScans {
       changed_paths: paths.length,
       policy: "metadata-prefilter-sha256"
     })
+    await this.vault.registry.beginAutomaticPrecheck(app)
     try {
       this.checkpoint(active)
       const metadataOptions = {
@@ -888,6 +1007,7 @@ class AutomaticScans {
         onError: (error, filePath) => {
           if (!isPathError(error)) return false
           counts.path_errors += 1
+          if (isMissing(error)) return true
           this.log("path-skipped", {
             app,
             path: filePath,
@@ -1063,7 +1183,19 @@ class AutomaticScans {
       item.state === "checking")
     if (!entry) return
     await this.refreshGlobalScanReady()
-    if (this.disposed || !this.globalScanReady) return
+    if (this.disposed) return
+    if (!this.globalScanReady) {
+      if (this.entries.get(entry.app) === entry) {
+        this.restorePrevious(entry.app)
+        this.discardChangedPaths(entry.app)
+        this.log("check-skipped", {
+          app: entry.app,
+          reason: "global-scan-required"
+        })
+      }
+      this.schedule()
+      return
+    }
     if (this.manualDepth > 0 || this.currentBusyPromise() ||
         this.vault.fileActionProgress) {
       this.waitForBusyWork()

package/kernel/vault/registry_core.js

@@ -2867,23 +2867,30 @@ class RegistryCore {
       rows = this.database.prepare(`
         SELECT
           'anchor' AS kind,
-          path,
-          size,
-          mtime,
-          ctime,
-          dev,
-          ino,
-          nlink,
-          mode,
-          uid,
-          gid,
-          hash
-        FROM anchors
-        WHERE dev = @dev
-          AND size = @size
-          AND path > @path
-          AND verified_at IS NOT NULL
-        ORDER BY path
+          anchor.path,
+          anchor.size,
+          anchor.mtime,
+          anchor.ctime,
+          anchor.dev,
+          anchor.ino,
+          anchor.nlink,
+          anchor.mode,
+          anchor.uid,
+          anchor.gid,
+          anchor.hash,
+          cached.hash AS cached_hash,
+          cached.size AS cached_size,
+          cached.mtime AS cached_mtime,
+          cached.ctime AS cached_ctime,
+          cached.dev AS cached_dev,
+          cached.ino AS cached_ino
+        FROM anchors anchor
+        LEFT JOIN hash_cache cached ON cached.path = anchor.path
+        WHERE anchor.dev = @dev
+          AND anchor.size = @size
+          AND anchor.path > @path
+          AND anchor.verified_at IS NOT NULL
+        ORDER BY anchor.path
         LIMIT @limit
       `).all({ dev, size, path: afterPath, limit: pageSize })
     } else {
@@ -2900,15 +2907,15 @@ class RegistryCore {
           peer.mode,
           peer.uid,
           peer.gid,
-          COALESCE(peer.hash, cached.hash) AS hash
+          peer.hash,
+          cached.hash AS cached_hash,
+          cached.size AS cached_size,
+          cached.mtime AS cached_mtime,
+          cached.ctime AS cached_ctime,
+          cached.dev AS cached_dev,
+          cached.ino AS cached_ino
         FROM files peer
-        LEFT JOIN hash_cache cached
-          ON cached.path = peer.path
-          AND cached.size = peer.size
-          AND cached.mtime = peer.mtime
-          AND cached.ctime = peer.ctime
-          AND cached.dev = peer.dev
-          AND cached.ino = peer.ino
+        LEFT JOIN hash_cache cached ON cached.path = peer.path
         WHERE peer.dev = @dev
           AND peer.size = @size
           AND peer.path > @path

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "8.0.71",
+  "version": "8.0.73",
   "description": "",
   "main": "index.js",
   "scripts": {

package/test/vault-automatic-scans.test.js

@@ -226,6 +226,45 @@ describe("automatic app checks", () => {
     await close(vault)
   })
 
+  test("Windows watcher deletes remove retained paths and descendants", async () => {
+    const home = await makeHome()
+    const appRoot = path.join(home, "api", "windows-app")
+    const scriptPath = path.join(appRoot, "start.js")
+    const first = path.join(appRoot, "first.bin")
+    const nestedRoot = path.join(appRoot, "temporary")
+    const nested = path.join(nestedRoot, "nested.bin")
+    await fs.promises.mkdir(appRoot)
+    let callback = null
+    const vault = await makeVault(home, {
+      deferStorage: true,
+      platform: "win32",
+      automaticWatcher: {
+        subscribe: async (_root, listener) => {
+          callback = listener
+          return { unsubscribe: async () => {} }
+        }
+      }
+    })
+
+    assert.equal(await vault.automaticScans.startWatcher(), true)
+    vault.automaticScans.handleStarted(scriptPath)
+    callback(null, [
+      { type: "create", path: first },
+      { type: "update", path: nestedRoot },
+      { type: "update", path: nested }
+    ])
+    assert.deepEqual([...vault.automaticScans.changedPaths.get(
+      "windows-app")].sort(), [first, nestedRoot, nested].sort())
+
+    callback(null, [{ type: "delete", path: nestedRoot }])
+    assert.deepEqual([...vault.automaticScans.changedPaths.get(
+      "windows-app")], [first])
+    callback(null, [{ type: "delete", path: first }])
+    assert.equal(vault.automaticScans.changedPaths.has("windows-app"), false)
+    assert.equal(vault.registry, null)
+    await close(vault)
+  })
+
   test("disposing automatic checks releases the watcher and pending work", async () => {
     const home = await makeHome()
     const appRoot = path.join(home, "api", "disposed-app")
@@ -366,6 +405,71 @@ describe("automatic app checks", () => {
     assert.equal(checksStarted, 0)
   })
 
+  test("scheduler coalesces wakeups while dispatch is in progress", async () => {
+    let releaseReadiness
+    let readinessCalls = 0
+    let dispatches = 0
+    const automatic = makeAutomatic({
+      enabled: true,
+      registry: {},
+      runExclusive: () => {
+        dispatches += 1
+        return new Promise(() => {})
+      }
+    })
+    automatic.entries.set("demo", {
+      app: "demo",
+      state: "checking",
+      paths: []
+    })
+    automatic.globalScanReady = true
+    automatic.refreshGlobalScanReady = async () => {
+      readinessCalls += 1
+      if (readinessCalls === 1) {
+        await new Promise((resolve) => { releaseReadiness = resolve })
+      }
+      automatic.globalScanReady = true
+    }
+    automatic.appRootIsAvailable = async () => true
+    automatic.appIsRunning = () => false
+
+    automatic.schedule()
+    await waitFor(() => typeof releaseReadiness === "function",
+      "scheduler readiness pause")
+    automatic.schedule()
+    releaseReadiness()
+    await waitFor(() => dispatches > 0, "automatic dispatch")
+    await new Promise((resolve) => setImmediate(resolve))
+
+    assert.equal(dispatches, 1)
+    assert.equal(readinessCalls, 1)
+  })
+
+  test("queued checks are cleared if global readiness disappears", async () => {
+    const automatic = makeAutomatic({
+      enabled: true,
+      registry: {}
+    })
+    automatic.entries.set("demo", {
+      app: "demo",
+      state: "checking",
+      paths: ["/pinokio/api/demo/model.bin"]
+    })
+    automatic.changedPaths.set("demo", new Set([
+      "/pinokio/api/demo/model.bin"
+    ]))
+    automatic.refreshGlobalScanReady = async () => {
+      automatic.globalScanReady = false
+    }
+    automatic.log = () => {}
+
+    automatic.schedule()
+    await waitFor(() => !automatic.drainQueued, "automatic queue cleanup")
+
+    assert.equal(automatic.entries.has("demo"), false)
+    assert.equal(automatic.changedPaths.has("demo"), false)
+  })
+
   test("Linux starts no watcher and ignores automatic lifecycle work", async () => {
     const home = await makeHome()
     const appRoot = path.join(home, "api", "linux-app")
@@ -1150,6 +1254,31 @@ describe("automatic app checks", () => {
     await close(vault)
   })
 
+  test("transient missing watcher paths do not emit per-path logs", async () => {
+    const home = await makeHome()
+    const app = "temporary-path-app"
+    const appRoot = path.join(home, "api", app)
+    const missing = [
+      path.join(appRoot, "temporary-1.bin"),
+      path.join(appRoot, "temporary-2.bin")
+    ]
+    await fs.promises.mkdir(appRoot)
+    const vault = await makeVault(home)
+    const logs = []
+    vault.automaticScans.log = (event, details) => {
+      logs.push({ event, details })
+    }
+
+    vault.automaticScans.queueApp(app, missing)
+    await waitFor(() => !vault.automaticScans.active &&
+      !vault.automaticScans.entries.has(app))
+
+    assert.equal(logs.some((record) =>
+      ["path-skipped", "verification-skipped"].includes(record.event) &&
+      missing.includes(record.details.path)), false)
+    await close(vault)
+  })
+
   test("stable published hashes are reused without rereading contents", async () => {
     const home = await makeHome()
     const app = "published-hash-app"
@@ -1203,6 +1332,102 @@ describe("automatic app checks", () => {
     await close(vault)
   })
 
+  test("a metadata-changed Windows peer is freshly hashed and cached", async () => {
+    const home = await makeHome()
+    const app = "windows-peer-app"
+    const appRoot = path.join(home, "api", app)
+    const candidate = path.join(appRoot, "candidate.bin")
+    const peer = path.join(home, "published-peer.bin")
+    await fs.promises.mkdir(appRoot)
+    await fs.promises.writeFile(candidate, "same-content")
+    await fs.promises.writeFile(peer, "same-content")
+    const vault = await makeVault(home, {
+      deferStorage: true,
+      platform: "win32"
+    })
+    await vault.ensureRegistryInitialized()
+    vault.automaticScans.candidateThreshold = async () => 1
+    const peerStat = await fs.promises.lstat(peer)
+    await vault.registry.upsertFile({
+      path: peer,
+      hash: crypto.createHash("sha256").update("old-content!").digest("hex"),
+      size: peerStat.size,
+      mtime: peerStat.mtimeMs - 1,
+      ctime: peerStat.ctimeMs - 1,
+      dev: peerStat.dev,
+      ino: peerStat.ino,
+      mode: peerStat.mode,
+      uid: peerStat.uid,
+      gid: peerStat.gid,
+      source_id: "app:peer",
+      app: "peer",
+      status: "reference",
+      unavailable_reason: null,
+      updated_at: Date.now()
+    })
+    const hashedPaths = []
+    const originalHashFile = vault.hashFile.bind(vault)
+    vault.hashFile = async (filePath, options) => {
+      hashedPaths.push(filePath)
+      return originalHashFile(filePath, options)
+    }
+
+    vault.automaticScans.queueApp(app, [candidate])
+    await waitFor(() => !vault.automaticScans.active &&
+      vault.automaticScans.entries.get(app)?.state === "result")
+    assert.deepEqual(hashedPaths.sort(), [candidate, peer].sort())
+
+    await vault.registry.setAutomaticAppScanState(app, null)
+    vault.automaticScans.entries.delete(app)
+    vault.hashFile = async () => {
+      throw new Error("Fresh automatic hashes must be reused.")
+    }
+    vault.automaticScans.queueApp(app, [candidate])
+    await waitFor(() => !vault.automaticScans.active &&
+      vault.automaticScans.entries.get(app)?.state === "result")
+
+    await close(vault)
+  })
+
+  test("a stale peer hash cannot match different equal-size content", async () => {
+    const home = await makeHome()
+    const app = "stale-peer-hash-app"
+    const appRoot = path.join(home, "api", app)
+    const candidate = path.join(appRoot, "candidate.bin")
+    const peer = path.join(home, "stale-peer.bin")
+    await fs.promises.mkdir(appRoot)
+    await fs.promises.writeFile(candidate, "same-content")
+    await fs.promises.writeFile(peer, "other-bytes!")
+    const vault = await makeVault(home, { deferStorage: true })
+    await vault.ensureRegistryInitialized()
+    vault.automaticScans.candidateThreshold = async () => 1
+    const peerStat = await fs.promises.lstat(peer)
+    await vault.registry.upsertFile({
+      path: peer,
+      hash: crypto.createHash("sha256").update("same-content").digest("hex"),
+      size: peerStat.size,
+      mtime: peerStat.mtimeMs - 1,
+      ctime: peerStat.ctimeMs - 1,
+      dev: peerStat.dev,
+      ino: peerStat.ino,
+      mode: peerStat.mode,
+      uid: peerStat.uid,
+      gid: peerStat.gid,
+      source_id: "app:peer",
+      app: "peer",
+      status: "reference",
+      unavailable_reason: null,
+      updated_at: Date.now()
+    })
+
+    vault.automaticScans.queueApp(app, [candidate])
+    await waitFor(() => !vault.automaticScans.active &&
+      !vault.automaticScans.entries.has(app))
+
+    assert.deepEqual(vault.automaticScans.snapshot().rows, [])
+    await close(vault)
+  })
+
   test("stale file rows and known identical inodes are not peers", async () => {
     const home = await makeHome()
     const appRoot = path.join(home, "api", "no-peer-app")