pinokiod 8.0.73 → 8.0.74

package/kernel/vault/automatic_scans.js

@@ -52,7 +52,6 @@ class AutomaticScans {
     this.manualDepth = 0
     this.observedApps = new Set()
     this.changedPaths = new Map()
-    this.changedPathIndex = new Map()
     this.pendingStops = new Map()
     this.lifecycleWork = new Set()
     this.stopSettleMs = STOP_SETTLE_MS
@@ -138,7 +137,6 @@ class AutomaticScans {
     }
     this.observedApps.clear()
     this.changedPaths.clear()
-    this.changedPathIndex.clear()
     if (watcherError) throw watcherError
   }
 
@@ -151,117 +149,123 @@ class AutomaticScans {
       })
       return
     }
-    for (const event of events || []) {
+    const watcherEvents = Array.isArray(events) ? events : []
+    if (!watcherEvents.some((event) => event && event.type === "delete")) {
+      for (const event of watcherEvents) {
+        if (event && ["create", "update"].includes(event.type)) {
+          this.recordChangedPath(event.path)
+        }
+      }
+      return
+    }
+    const batches = new Map()
+    for (let index = 0; index < watcherEvents.length; index++) {
+      const event = watcherEvents[index]
       if (!event || !["create", "update", "delete"].includes(event.type)) {
         continue
       }
-      if (event.type === "delete") this.removeChangedPath(event.path)
-      else this.recordChangedPath(event.path)
+      const target = this.changedPathTarget(
+        event.path, event.type === "delete")
+      if (!target) continue
+      let batch = batches.get(target.app)
+      if (!batch) {
+        batch = {
+          root: target.root,
+          updates: new Map(),
+          deletes: new Map()
+        }
+        batches.set(target.app, batch)
+      }
+      const key = pathKey(target.path)
+      if (event.type === "delete") batch.deletes.set(key, index)
+      else batch.updates.set(key, { index, path: target.path })
+    }
+    for (const [app, batch] of batches) {
+      this.applyChangedPathBatch(app, batch)
     }
   }
 
-  recordChangedPath(filePath) {
+  changedPathTarget(filePath, allowRoot = false) {
     if (this.disposed || !this.supported ||
         typeof filePath !== "string" || !filePath) {
-      return false
+      return null
     }
     const app = this.appForLaunchPath(filePath)
-    if (!app || !this.observedApps.has(app)) return false
-    const appRoot = path.resolve(this.vault.kernel.homedir, "api", app)
+    if (!app || !this.observedApps.has(app)) return null
+    const root = path.resolve(this.vault.kernel.homedir, "api", app)
     const resolved = path.resolve(filePath)
-    if (resolved === appRoot || !inside(appRoot, resolved)) return false
+    if ((!allowRoot && resolved === root) || !inside(root, resolved)) return null
+    return { app, root, path: resolved }
+  }
+
+  recordChangedPath(filePath) {
+    const target = this.changedPathTarget(filePath)
+    if (!target) return false
+    const { app, path: resolved } = target
     let paths = this.changedPaths.get(app)
     if (!paths) {
       paths = new Set()
       this.changedPaths.set(app, paths)
     }
-    let index = this.changedPathIndex.get(app)
-    if (!index) {
-      index = { exact: new Map(), descendants: new Map() }
-      this.changedPathIndex.set(app, index)
-    }
-    const key = pathKey(resolved)
-    if (index.exact.has(key)) return true
     paths.add(resolved)
-    index.exact.set(key, resolved)
-    this.addChangedPathToIndex(index.descendants, appRoot, resolved)
     return true
   }
 
-  changedPathParentKeys(appRoot, filePath) {
-    const rootKey = pathKey(appRoot)
-    const keys = []
-    let current = path.dirname(path.resolve(filePath))
+  lastDeleteIndex(root, filePath, deletes) {
+    const rootKey = pathKey(root)
+    let current = path.resolve(filePath)
+    let latest = -1
     while (true) {
       const key = pathKey(current)
-      keys.push(key)
-      if (key === rootKey) return keys
+      const index = deletes.get(key)
+      if (index !== undefined && index > latest) latest = index
+      if (key === rootKey) return latest
       current = path.dirname(current)
     }
   }
 
-  addChangedPathToIndex(index, appRoot, filePath) {
-    for (const key of this.changedPathParentKeys(appRoot, filePath)) {
-      let descendants = index.get(key)
-      if (!descendants) {
-        descendants = new Set()
-        index.set(key, descendants)
-      }
-      descendants.add(filePath)
+  applyChangedPathBatch(app, { root, updates, deletes }) {
+    const paths = this.changedPaths.get(app)
+    if (!deletes.size) {
+      const collected = paths || new Set()
+      for (const update of updates.values()) collected.add(update.path)
+      if (collected.size) this.changedPaths.set(app, collected)
+      return
     }
-  }
-
-  removeChangedPathFromIndex(index, appRoot, filePath) {
-    index.exact.delete(pathKey(filePath))
-    for (const key of this.changedPathParentKeys(appRoot, filePath)) {
-      const descendants = index.descendants.get(key)
-      if (!descendants) continue
-      descendants.delete(filePath)
-      if (!descendants.size) index.descendants.delete(key)
+    const retained = new Set()
+    if (paths) {
+      for (const candidate of paths) {
+        const key = pathKey(candidate)
+        const update = updates.get(key)
+        if (this.lastDeleteIndex(root, candidate, deletes) >
+            (update ? update.index : -1) || retained.has(key)) {
+          paths.delete(candidate)
+        } else {
+          retained.add(key)
+        }
+      }
     }
-  }
-
-  removeChangedPath(filePath) {
-    if (this.disposed || !this.supported ||
-        typeof filePath !== "string" || !filePath) {
-      return false
+    let collected = paths
+    for (const [key, update] of updates) {
+      if (update.index <= this.lastDeleteIndex(root, update.path, deletes) ||
+          retained.has(key)) continue
+      if (!collected) collected = new Set()
+      collected.add(update.path)
+      retained.add(key)
     }
-    const app = this.appForLaunchPath(filePath)
-    if (!app || !this.observedApps.has(app)) return false
-    const paths = this.changedPaths.get(app)
-    if (!paths) return false
-    const resolved = path.resolve(filePath)
-    const appRoot = path.resolve(this.vault.kernel.homedir, "api", app)
-    const index = this.changedPathIndex.get(app)
-    if (!index) return false
-    const key = pathKey(resolved)
-    const exact = index.exact.get(key)
-    const descendants = index.descendants.get(key)
-    if (!exact && !descendants) return false
-    const matches = new Set(descendants || [])
-    if (exact) matches.add(exact)
-    for (const candidate of matches) {
-      paths.delete(candidate)
-      this.removeChangedPathFromIndex(index, appRoot, candidate)
-    }
-    if (!paths.size) this.discardChangedPaths(app)
-    return true
+    if (collected && collected.size) this.changedPaths.set(app, collected)
+    else this.changedPaths.delete(app)
   }
 
   discardChangedPaths(app) {
     this.changedPaths.delete(app)
-    this.changedPathIndex.delete(app)
   }
 
   consumeChangedPaths(app, paths) {
     const collected = this.changedPaths.get(app)
     if (!collected) return
-    const appRoot = path.resolve(this.vault.kernel.homedir, "api", app)
-    const index = this.changedPathIndex.get(app)
     for (const filePath of paths || []) {
-      const resolved = path.resolve(filePath)
-      if (!collected.delete(resolved) || !index) continue
-      this.removeChangedPathFromIndex(index, appRoot, resolved)
+      collected.delete(path.resolve(filePath))
     }
     if (!collected.size) this.discardChangedPaths(app)
   }
@@ -931,10 +935,11 @@ class AutomaticScans {
     if (reusable) {
       counts.hash_reuses += 1
       memory.set(key, reusable)
-      verifiedHashes.set(pathKey(entry.path), Object.assign({}, currentEntry, {
+      const verifiedEntry = Object.assign({}, currentEntry, {
         hash: reusable
-      }))
-      return reusable
+      })
+      verifiedHashes.set(pathKey(entry.path), verifiedEntry)
+      return verifiedEntry
     }
     let verified
     try {
@@ -961,10 +966,11 @@ class AutomaticScans {
     counts.hashed += 1
     counts.hash_bytes += Math.max(0, Number(verified.result.size) || 0)
     memory.set(key, verified.result.hash)
-    verifiedHashes.set(pathKey(entry.path), Object.assign({}, currentEntry, {
+    const verifiedEntry = Object.assign({}, currentEntry, {
       hash: verified.result.hash
-    }))
-    return verified.result.hash
+    })
+    verifiedHashes.set(pathKey(entry.path), verifiedEntry)
+    return verifiedEntry
   }
 
   async runPrecheck(active) {
@@ -1061,7 +1067,6 @@ class AutomaticScans {
       const preparePeers = () => {
         if (!group || group.unmatched) return
         group.unmatched = new Map()
-        group.unmatchedIdentities = new Map()
         for (const [hash, value] of group.candidatesByHash) {
           if (value.identities.size > 1) {
             for (const candidate of value.candidates) {
@@ -1073,9 +1078,6 @@ class AutomaticScans {
               identity,
               candidates: value.candidates
             })
-            group.unmatchedIdentities.set(
-              identity,
-              (group.unmatchedIdentities.get(identity) || 0) + 1)
           }
         }
       }
@@ -1104,47 +1106,39 @@ class AutomaticScans {
               candidatesByHash: new Map(),
               matched: new Set(),
               unmatched: null,
-              unmatchedIdentities: null,
               memory: new Map()
             }
           }
           if (entry.kind === "changed") {
             const candidate = entry
             this.checkpoint(active)
-            const hash = await this.verifiedAutomaticHash(
+            const verified = await this.verifiedAutomaticHash(
               active, candidate, group.memory, verifiedHashes, counts)
-            if (!hash) continue
+            if (!verified) continue
+            const hash = verified.hash
             let value = group.candidatesByHash.get(hash)
             if (!value) {
               value = { candidates: [], identities: new Set() }
               group.candidatesByHash.set(hash, value)
             }
-            value.candidates.push(candidate)
-            value.identities.add(this.automaticIdentityKey(candidate))
+            value.candidates.push(verified)
+            value.identities.add(this.automaticIdentityKey(verified))
             continue
           }
           preparePeers()
           if (!group.unmatched.size) continue
           this.checkpoint(active)
-          const identity = this.automaticIdentityKey(entry)
-          if (group.unmatchedIdentities.size === 1 &&
-              group.unmatchedIdentities.has(identity)) continue
-          const hash = await this.verifiedAutomaticHash(
+          const verified = await this.verifiedAutomaticHash(
             active, entry, group.memory, verifiedHashes, counts)
-          const wanted = hash && group.unmatched.get(hash)
+          const wanted = verified && group.unmatched.get(verified.hash)
+          const identity = verified && this.automaticIdentityKey(verified)
           if (!wanted || wanted.identity === identity) {
             continue
           }
           for (const candidate of wanted.candidates) {
             group.matched.add(pathKey(candidate.path))
           }
-          group.unmatched.delete(hash)
-          const remaining = group.unmatchedIdentities.get(wanted.identity) - 1
-          if (remaining) {
-            group.unmatchedIdentities.set(wanted.identity, remaining)
-          } else {
-            group.unmatchedIdentities.delete(wanted.identity)
-          }
+          group.unmatched.delete(verified.hash)
         }
         if (verifiedHashes.size >= ENTRY_BATCH_SIZE) {
           await this.vault.registry.rememberHashCache(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "8.0.73",
+  "version": "8.0.74",
   "description": "",
   "main": "index.js",
   "scripts": {

package/server/public/app-vault-mode.js

@@ -167,7 +167,8 @@
         tab.classList.remove("app-vault-result-attention")
       }
     })
-    tab.addEventListener("click", () => {
+    tab.addEventListener("click", (event) => {
+      if (!event.isTrusted) return
       const signature = status.dataset.resultSignature || ""
       if (!signature) return
       requestAutomaticScanFocus(signature)

package/test/vault-automatic-scan-ui.test.js

@@ -221,7 +221,7 @@ test("the app workspace presents the automatic-result coachmark without starting
   assert.doesNotMatch(present, /innerHTML|post\(|btn-scan\.click/)
 })
 
-test("selecting a badged row acknowledges it without delaying navigation", async () => {
+test("user-selecting a badged row acknowledges it without delaying navigation", async () => {
   const script = await fs.promises.readFile(
     path.join(root, "server", "public", "app-vault-mode.js"), "utf8")
   const parent = new JSDOM("", { url: "http://localhost/" })
@@ -256,14 +256,19 @@ test("selecting a badged row acknowledges it without delaying navigation", async
     }
   }
 
-  dom.window.eval(script)
   const tab = dom.window.document.getElementById("save-space-tab")
-  let preventedByHandler = null
-  tab.addEventListener("click", (event) => {
-    preventedByHandler = event.defaultPrevented
-    event.preventDefault()
+  let clickHandler
+  const addEventListener = tab.addEventListener.bind(tab)
+  tab.addEventListener = (type, listener, options) => {
+    if (type === "click") clickHandler = listener
+    addEventListener(type, listener, options)
+  }
+  dom.window.eval(script)
+  let preventedByHandler = false
+  clickHandler({
+    isTrusted: true,
+    preventDefault: () => { preventedByHandler = true }
   })
-  tab.click()
 
   await waitFor(() => requests.length === 1)
   assert.equal(preventedByHandler, false)
@@ -289,6 +294,49 @@ test("selecting a badged row acknowledges it without delaying navigation", async
   parent.window.close()
 })
 
+test("programmatic tab clicks do not acknowledge a result", async () => {
+  const script = await fs.promises.readFile(
+    path.join(root, "server", "public", "app-vault-mode.js"), "utf8")
+  const signature = "e".repeat(64)
+  const parent = new JSDOM("", { url: "http://localhost/" })
+  const dom = new JSDOM(`<a id="save-space-tab" href="/vault/app/ComfyUI">
+    <span data-app-vault-result-badge></span>
+    <span data-app-vault-mode data-app="ComfyUI" data-mode="automatic"
+      data-ready="true" data-result-signature="${signature}">
+      <span data-app-vault-mode-label>Auto</span>
+    </span>
+  </a>`, {
+    runScripts: "outside-only",
+    url: "http://localhost/v/ComfyUI"
+  })
+  Object.defineProperty(dom.window, "parent", {
+    configurable: true,
+    value: parent.window
+  })
+  parent.window.postMessage = () => {}
+  const requests = []
+  dom.window.fetch = async (url, options = {}) => {
+    requests.push({ url, options })
+    return { ok: true, json: async () => ({}) }
+  }
+
+  dom.window.eval(script)
+  const tab = dom.window.document.getElementById("save-space-tab")
+  tab.addEventListener("click", (event) => event.preventDefault())
+  tab.click()
+
+  assert.equal(requests.length, 0)
+  assert.equal(tab.querySelector("[data-app-vault-result-badge]").hidden,
+    false)
+  assert.equal(dom.window.document.querySelector("[data-app-vault-mode]")
+    .dataset.resultSignature, signature)
+  assert.equal(dom.window.sessionStorage.getItem(
+    "pinokio:vault:auto-scan-focus:ComfyUI"), null)
+
+  dom.window.close()
+  parent.window.close()
+})
+
 test("a retained Disk Saver handoff targets the visible existing frame", async () => {
   const template = await fs.promises.readFile(
     path.join(root, "server", "views", "app.ejs"), "utf8")
@@ -336,8 +384,15 @@ test("a retained Disk Saver handoff targets the visible existing frame", async (
     json: async () => ({ acknowledged: true, app: "ComfyUI" })
   })
 
+  const tab = dom.window.document.getElementById("save-space-tab")
+  let clickHandler
+  const addEventListener = tab.addEventListener.bind(tab)
+  tab.addEventListener = (type, listener, options) => {
+    if (type === "click") clickHandler = listener
+    addEventListener(type, listener, options)
+  }
   dom.window.eval(script)
-  dom.window.document.getElementById("save-space-tab").click()
+  clickHandler({ isTrusted: true })
   await waitFor(() => visibleMessages.length === 1)
 
   assert.deepEqual(hiddenMessages, [])

package/test/vault-automatic-scans.test.js

@@ -265,6 +265,46 @@ describe("automatic app checks", () => {
     await close(vault)
   })
 
+  test("a watcher batch retains a path recreated after deletion", async () => {
+    const home = await makeHome()
+    const app = "delete-create-app"
+    const root = path.join(home, "api", app)
+    const temporary = path.join(root, "temporary")
+    const recreated = path.join(temporary, "model.bin")
+    const automatic = makeAutomatic({
+      enabled: true,
+      kernel: { homedir: home, platform: "darwin" }
+    })
+    automatic.observedApps.add(app)
+
+    automatic.handleWatcherEvents(null, [
+      { type: "delete", path: temporary },
+      { type: "create", path: recreated }
+    ])
+
+    assert.deepEqual([...automatic.changedPaths.get(app)], [recreated])
+  })
+
+  test("a watcher batch removes a path deleted after creation", async () => {
+    const home = await makeHome()
+    const app = "create-delete-app"
+    const root = path.join(home, "api", app)
+    const temporary = path.join(root, "temporary")
+    const removed = path.join(temporary, "model.bin")
+    const automatic = makeAutomatic({
+      enabled: true,
+      kernel: { homedir: home, platform: "darwin" }
+    })
+    automatic.observedApps.add(app)
+
+    automatic.handleWatcherEvents(null, [
+      { type: "create", path: removed },
+      { type: "delete", path: temporary }
+    ])
+
+    assert.equal(automatic.changedPaths.has(app), false)
+  })
+
   test("disposing automatic checks releases the watcher and pending work", async () => {
     const home = await makeHome()
     const appRoot = path.join(home, "api", "disposed-app")
@@ -1428,6 +1468,80 @@ describe("automatic app checks", () => {
     await close(vault)
   })
 
+  test("files that become hardlinks after staging are not duplicates", async () => {
+    const home = await makeHome()
+    const app = "became-hardlinks-app"
+    const appRoot = path.join(home, "api", app)
+    const first = path.join(appRoot, "first.bin")
+    const second = path.join(appRoot, "second.bin")
+    await fs.promises.mkdir(appRoot)
+    await fs.promises.writeFile(first, "same-content")
+    await fs.promises.writeFile(second, "same-content")
+    const before = await Promise.all([
+      fs.promises.lstat(first),
+      fs.promises.lstat(second)
+    ])
+    assert.notEqual(before[0].ino, before[1].ino)
+    const vault = await makeVault(home)
+    vault.automaticScans.candidateThreshold = async () => 1
+    const stage = vault.registry.stageAutomaticPrecheckFiles.bind(
+      vault.registry)
+    vault.registry.stageAutomaticPrecheckFiles = async (...args) => {
+      await stage(...args)
+      await fs.promises.unlink(second)
+      await fs.promises.link(first, second)
+    }
+
+    vault.automaticScans.queueApp(app, [first, second])
+    await waitFor(() => !vault.automaticScans.active &&
+      !vault.automaticScans.entries.has(app))
+
+    const after = await Promise.all([
+      fs.promises.lstat(first),
+      fs.promises.lstat(second)
+    ])
+    assert.equal(after[0].ino, after[1].ino)
+    assert.deepEqual(vault.automaticScans.snapshot().rows, [])
+    await close(vault)
+  })
+
+  test("hardlinks that become independent after staging are duplicates", async () => {
+    const home = await makeHome()
+    const app = "split-hardlinks-app"
+    const appRoot = path.join(home, "api", app)
+    const first = path.join(appRoot, "first.bin")
+    const second = path.join(appRoot, "second.bin")
+    await fs.promises.mkdir(appRoot)
+    await fs.promises.writeFile(first, "same-content")
+    await fs.promises.link(first, second)
+    const before = await Promise.all([
+      fs.promises.lstat(first),
+      fs.promises.lstat(second)
+    ])
+    assert.equal(before[0].ino, before[1].ino)
+    const vault = await makeVault(home)
+    vault.automaticScans.candidateThreshold = async () => 1
+    const stage = vault.registry.stageAutomaticPrecheckFiles.bind(
+      vault.registry)
+    vault.registry.stageAutomaticPrecheckFiles = async (...args) => {
+      await stage(...args)
+      await fs.promises.unlink(second)
+      await fs.promises.writeFile(second, "same-content")
+    }
+
+    vault.automaticScans.queueApp(app, [first, second])
+    await waitFor(() => !vault.automaticScans.active &&
+      vault.automaticScans.entries.get(app)?.state === "result")
+
+    const after = await Promise.all([
+      fs.promises.lstat(first),
+      fs.promises.lstat(second)
+    ])
+    assert.notEqual(after[0].ino, after[1].ino)
+    assert.equal(vault.automaticScans.snapshot().rows[0].state, "result")
+    await close(vault)
+  })
+
   test("stale file rows and known identical inodes are not peers", async () => {
     const home = await makeHome()
     const appRoot = path.join(home, "api", "no-peer-app")