pinokiod 8.0.74 → 8.0.76

package/kernel/vault/automatic_scans.js

@@ -894,7 +894,7 @@ class AutomaticScans {
     })
   }
 
-  async verifiedAutomaticHash(active, entry, memory, verifiedHashes, counts) {
+  async currentAutomaticEntry(active, entry, counts) {
     this.checkpoint(active)
     let current
     try {
@@ -915,7 +915,20 @@ class AutomaticScans {
         active.app, entry, "snapshot-changed", counts)
       return null
     }
-    const currentEntry = Object.assign({}, entry, fileSnapshot(current))
+    return {
+      stat: current,
+      entry: Object.assign({}, entry, fileSnapshot(current))
+    }
+  }
+
+  async verifiedAutomaticHash(
+    active, entry, memory, verifiedHashes, counts, prepared = null
+  ) {
+    const checked = prepared || await this.currentAutomaticEntry(
+      active, entry, counts)
+    if (!checked) return null
+    const current = checked.stat
+    const currentEntry = checked.entry
     const key = this.automaticHashKey(currentEntry)
     const cachedSnapshot = {
       size: entry.cached_size,
@@ -976,16 +989,26 @@ class AutomaticScans {
   async runPrecheck(active) {
     const app = active.app
     const root = path.resolve(this.vault.kernel.homedir, "api", app)
+    const startedAt = Date.now()
+    active.startedAt = startedAt
+    const markStage = (stage, entry = null) => {
+      active.stage = stage
+      active.stagePath = entry && entry.path ? entry.path : null
+      active.stageKind = entry && entry.kind ? entry.kind : null
+    }
+    markStage("app-root-check")
     if (!await this.appRootIsAvailable(app)) {
       const error = new Error("That app is no longer available.")
       error.code = "ENOENT"
       throw error
     }
-    const startedAt = Date.now()
     const counts = {
       files: 0,
       bytes: 0,
       candidates: 0,
+      registry_pages: 0,
+      candidate_checks: 0,
+      peer_checks: 0,
       path_errors: 0,
       hashed: 0,
       hash_bytes: 0,
@@ -993,6 +1016,7 @@ class AutomaticScans {
       hash_failures: 0,
       unstable_hashes: 0
     }
+    markStage("threshold-read")
     const threshold = await this.candidateThreshold()
     const paths = [...new Set(active.paths || [])].filter((filePath) =>
       typeof filePath === "string" && inside(root, filePath))
@@ -1004,6 +1028,7 @@ class AutomaticScans {
       changed_paths: paths.length,
       policy: "metadata-prefilter-sha256"
     })
+    markStage("staging-reset")
     await this.vault.registry.beginAutomaticPrecheck(app)
     try {
       this.checkpoint(active)
@@ -1023,9 +1048,11 @@ class AutomaticScans {
           return true
         }
       }
+      markStage("changed-parent-metadata")
       const safePaths = await this.changedPathsWithSafeParents(
         paths, root, metadataOptions, () => this.checkpoint(active))
       this.checkpoint(active)
+      markStage("changed-path-metadata")
       const safeStats = await this.statChangedPaths(
         safePaths, metadataOptions, () => this.checkpoint(active))
       const statsByPath = new Map(safePaths.map((filePath, index) =>
@@ -1052,95 +1079,164 @@ class AutomaticScans {
           gid: stat.gid
         })
       }
-      counts.candidates = candidates.length
       if (candidates.length) {
+        markStage("staging-write")
         await this.vault.registry.stageAutomaticPrecheckFiles(app, candidates)
       }
       this.checkpoint(active)
+      markStage("app-root-recheck")
       if (!await this.appRootIsAvailable(app)) {
         const error = new Error("That app is no longer available.")
         error.code = "ENOENT"
         throw error
       }
-      const verifiedMatches = []
+      this.log("verification-started", {
+        app,
+        files: counts.files,
+        bytes: counts.bytes,
+        threshold_candidates: candidates.length,
+        duration_ms: Date.now() - startedAt
+      })
+      let proof = null
       let group = null
+      const acceptProof = (candidate, peer, hash) => {
+        proof = { path: candidate.path, hash }
+        this.log("first-proof-exit", {
+          app,
+          candidate_path: candidate.path,
+          peer_path: peer.path,
+          peer_kind: peer.kind,
+          size: candidate.size,
+          duration_ms: Date.now() - startedAt,
+          registry_pages: counts.registry_pages,
+          candidate_checks: counts.candidate_checks,
+          peer_checks: counts.peer_checks,
+          hashed: counts.hashed,
+          hash_bytes: counts.hash_bytes,
+          hash_reuses: counts.hash_reuses
+        })
+      }
       const preparePeers = () => {
         if (!group || group.unmatched) return
         group.unmatched = new Map()
         for (const [hash, value] of group.candidatesByHash) {
-          if (value.identities.size > 1) {
-            for (const candidate of value.candidates) {
-              group.matched.add(pathKey(candidate.path))
-            }
-          } else {
-            const identity = value.identities.values().next().value
-            group.unmatched.set(hash, {
-              identity,
-              candidates: value.candidates
-            })
-          }
+          group.unmatched.set(hash, {
+            identity: value.identities.values().next().value,
+            candidate: value.candidate
+          })
         }
       }
-      const finishGroup = () => {
-        if (!group) return
-        preparePeers()
-        for (const [hash, value] of group.candidatesByHash) {
-          for (const candidate of value.candidates) {
-            if (!group.matched.has(pathKey(candidate.path))) continue
-            verifiedMatches.push({ path: candidate.path, hash })
+      const hashCandidates = async (peerIdentity = null) => {
+        if (!group || group.candidatesHashed) return !!group
+        const identities = new Set(group.candidates.map((candidate) =>
+          this.automaticIdentityKey(candidate.entry)))
+        if (identities.size < 2 &&
+            (!peerIdentity || identities.has(peerIdentity))) {
+          return false
+        }
+        group.candidatesHashed = true
+        counts.candidates += group.candidates.length
+        for (const candidate of group.candidates) {
+          this.checkpoint(active)
+          markStage("candidate-hash", candidate.entry)
+          const verified = await this.verifiedAutomaticHash(
+            active,
+            candidate.entry,
+            group.memory,
+            verifiedHashes,
+            counts,
+            candidate
+          )
+          if (!verified) continue
+          const hash = verified.hash
+          let value = group.candidatesByHash.get(hash)
+          if (!value) {
+            value = { candidate: verified, identities: new Set() }
+            group.candidatesByHash.set(hash, value)
+          }
+          value.identities.add(this.automaticIdentityKey(verified))
+          if (value.identities.size > 1) {
+            acceptProof(value.candidate, verified, hash)
+            return true
           }
         }
+        preparePeers()
+        return true
+      }
+      const finishGroup = async () => {
+        if (!group) return false
+        await hashCandidates()
+        if (proof) return true
         group = null
+        return false
+      }
+      const complete = async () => {
+        this.checkpoint(active)
+        markStage("result-signature")
+        const result = await this.vault.registry.automaticPrecheckResult(
+          app, proof ? [proof] : [])
+        this.checkpoint(active)
+        return Object.assign({}, counts, {
+          signature: result.signature,
+          verified_files: Math.max(0, Number(result.files) || 0),
+          duration_ms: Date.now() - startedAt
+        })
       }
       let cursor = null
       do {
+        markStage("candidate-query")
+        counts.registry_pages += 1
         const page = await this.vault.registry.automaticPrecheckEntries(
           app, cursor, ENTRY_BATCH_SIZE)
         this.checkpoint(active)
         for (const entry of page.entries) {
           const key = `${entry.dev}\0${entry.size}`
           if (!group || group.key !== key) {
-            finishGroup()
+            if (await finishGroup()) return await complete()
             group = {
               key,
+              candidates: [],
+              candidatesHashed: false,
               candidatesByHash: new Map(),
-              matched: new Set(),
               unmatched: null,
               memory: new Map()
             }
           }
           if (entry.kind === "changed") {
-            const candidate = entry
             this.checkpoint(active)
-            const verified = await this.verifiedAutomaticHash(
-              active, candidate, group.memory, verifiedHashes, counts)
-            if (!verified) continue
-            const hash = verified.hash
-            let value = group.candidatesByHash.get(hash)
-            if (!value) {
-              value = { candidates: [], identities: new Set() }
-              group.candidatesByHash.set(hash, value)
-            }
-            value.candidates.push(verified)
-            value.identities.add(this.automaticIdentityKey(verified))
+            markStage("candidate-metadata", entry)
+            counts.candidate_checks += 1
+            const candidate = await this.currentAutomaticEntry(
+              active, entry, counts)
+            if (candidate) group.candidates.push(candidate)
             continue
           }
-          preparePeers()
+          if (!group.candidatesHashed) await hashCandidates()
+          if (proof) return await complete()
+          if (group.candidatesHashed && !group.unmatched.size) continue
+          markStage("peer-metadata", entry)
+          counts.peer_checks += 1
+          const peer = await this.currentAutomaticEntry(active, entry, counts)
+          if (!peer) continue
+          const identity = this.automaticIdentityKey(peer.entry)
+          if (!await hashCandidates(identity)) continue
+          if (proof) return await complete()
           if (!group.unmatched.size) continue
+          if (![...group.unmatched.values()].some((wanted) =>
+            wanted.identity !== identity)) continue
           this.checkpoint(active)
+          markStage("peer-hash", entry)
           const verified = await this.verifiedAutomaticHash(
-            active, entry, group.memory, verifiedHashes, counts)
+            active, entry, group.memory, verifiedHashes, counts, peer)
           const wanted = verified && group.unmatched.get(verified.hash)
-          const identity = verified && this.automaticIdentityKey(verified)
           if (!wanted || wanted.identity === identity) {
             continue
           }
-          for (const candidate of wanted.candidates) {
-            group.matched.add(pathKey(candidate.path))
-          }
-          group.unmatched.delete(verified.hash)
+          acceptProof(wanted.candidate, verified, verified.hash)
+          return await complete()
         }
         if (verifiedHashes.size >= ENTRY_BATCH_SIZE) {
+          markStage("hash-cache-write")
           await this.vault.registry.rememberHashCache(
             [...verifiedHashes.values()])
           verifiedHashes.clear()
@@ -1148,24 +1244,26 @@ class AutomaticScans {
         }
         cursor = page.next_cursor
       } while (cursor)
-      finishGroup()
-      this.checkpoint(active)
-      const result = await this.vault.registry.automaticPrecheckResult(
-        app, verifiedMatches)
-      this.checkpoint(active)
-      return Object.assign({}, counts, {
-        signature: result.signature,
-        verified_files: Math.max(0, Number(result.files) || 0),
-        duration_ms: Date.now() - startedAt
-      })
+      await finishGroup()
+      return await complete()
     } finally {
       try {
         if (verifiedHashes.size) {
-          await this.vault.registry.rememberHashCache(
-            [...verifiedHashes.values()])
+          try {
+            await this.vault.registry.rememberHashCache(
+              [...verifiedHashes.values()])
+          } catch (error) {
+            markStage("hash-cache-write")
+            throw error
+          }
         }
       } finally {
-        await this.vault.registry.abortAutomaticPrecheck(app)
+        try {
+          await this.vault.registry.abortAutomaticPrecheck(app)
+        } catch (error) {
+          markStage("staging-cleanup")
+          throw error
+        }
       }
     }
   }
@@ -1269,10 +1367,22 @@ class AutomaticScans {
             ? (error.code === "EVAULTCANCELLED" ? "cancelled" : "failed")
             : "complete",
           cancel_reason: reason,
-          duration_ms: result && result.duration_ms,
+          duration_ms: result
+            ? result.duration_ms
+            : error && active.startedAt
+              ? Date.now() - active.startedAt
+              : undefined,
+          failure_stage: error && active.stage,
+          failure_path: error && (error.path || active.stagePath),
+          failure_kind: error && active.stageKind,
+          error_code: error && error.code,
+          error_syscall: error && error.syscall,
           files: result && result.files,
           bytes: result && result.bytes,
           candidates: result && result.candidates,
+          registry_pages: result && result.registry_pages,
+          candidate_checks: result && result.candidate_checks,
+          peer_checks: result && result.peer_checks,
           verified_files: result && result.verified_files,
           hashed: result && result.hashed,
           hash_bytes: result && result.hash_bytes,

package/kernel/vault/registry_core.js

@@ -2801,15 +2801,44 @@ class RegistryCore {
     let dev = Number(after.dev) || 0
     let size = Number(after.size) || 0
     let afterPath = typeof after.path === "string" ? after.path : ""
+    const possiblePeer = `(
+      EXISTS (
+        SELECT 1 FROM automatic_precheck_files peer
+        WHERE peer.app = candidate.app
+          AND peer.dev = candidate.dev
+          AND peer.size = candidate.size
+          AND pinokio_path_key(peer.path) !=
+            pinokio_path_key(candidate.path)
+      )
+      OR EXISTS (
+        SELECT 1 FROM files peer
+        WHERE peer.dev = candidate.dev
+          AND peer.size = candidate.size
+          AND peer.unavailable_reason IS NOT 'stale'
+          AND pinokio_path_key(peer.path) !=
+            pinokio_path_key(candidate.path)
+      )
+      OR EXISTS (
+        SELECT 1 FROM anchors peer
+        WHERE peer.verified_at IS NOT NULL
+          AND peer.dev = candidate.dev
+          AND peer.size = candidate.size
+          AND pinokio_path_key(peer.path) !=
+            pinokio_path_key(candidate.path)
+      )
+    )`
     if (!phase) {
       const group = this.database.prepare(`
-        SELECT dev, size
-        FROM automatic_precheck_files
-        WHERE app = @app AND (
-          @has_cursor = 0 OR dev > @dev OR (dev = @dev AND size > @size)
-        )
-        GROUP BY dev, size
-        ORDER BY dev, size
+        SELECT candidate.dev, candidate.size
+        FROM automatic_precheck_files candidate
+        WHERE candidate.app = @app
+          AND ${possiblePeer}
+          AND (
+            @has_cursor = 0 OR candidate.dev > @dev OR
+            (candidate.dev = @dev AND candidate.size > @size)
+          )
+        GROUP BY candidate.dev, candidate.size
+        ORDER BY candidate.dev, candidate.size
         LIMIT 1
       `).get({
         app,
@@ -3001,29 +3030,42 @@ class RegistryCore {
       hashes.set(canonicalPathKey(entry.path), entry.hash)
     }
     const rows = this.database.prepare(`
-      SELECT path, dev, size
+      SELECT path, dev, ino, size, mtime, ctime
       FROM automatic_precheck_files
       WHERE app = ?
-      ORDER BY pinokio_path_key(path), dev, size
+      ORDER BY pinokio_path_key(path), path, dev, ino, size, mtime, ctime
     `).iterate(app)
     const digest = crypto.createHash("sha256")
-    let files = 0
+    let proof = null
+    digest.update("automatic-precheck-snapshots\0")
     for (const row of rows) {
-      const hash = hashes.get(canonicalPathKey(row.path))
-      if (!hash) continue
-      digest.update(canonicalPathKey(row.path))
-      digest.update("\0")
-      digest.update(String(row.dev))
-      digest.update("\0")
-      digest.update(String(row.size))
-      digest.update("\0")
-      digest.update(hash)
-      digest.update("\n")
-      files += 1
+      const canonicalPath = canonicalPathKey(row.path)
+      const hash = hashes.get(canonicalPath)
+      if (!proof && hash) proof = { canonicalPath, row, hash }
+      for (const value of [
+        canonicalPath,
+        row.dev,
+        row.ino,
+        row.size,
+        row.mtime,
+        row.ctime
+      ]) {
+        digest.update(String(value))
+        digest.update("\0")
+      }
     }
+    if (!proof) return { signature: null, files: 0 }
+    digest.update("automatic-precheck-proof\0")
+    digest.update(proof.canonicalPath)
+    digest.update("\0")
+    digest.update(String(proof.row.dev))
+    digest.update("\0")
+    digest.update(String(proof.row.size))
+    digest.update("\0")
+    digest.update(proof.hash)
     return {
-      signature: files ? digest.digest("hex") : null,
-      files
+      signature: digest.digest("hex"),
+      files: 1
     }
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "8.0.74",
+  "version": "8.0.76",
   "description": "",
   "main": "index.js",
   "scripts": {

package/server/public/vault.css

@@ -1678,6 +1678,40 @@ body.vault-page .vault-view-tabs {
 .vault-view-tabs .vault-nav-count {
   font-size: 10.5px;
 }
+.vault-view-tabs .vault-nav-row.attention {
+  background: color-mix(in srgb, var(--vault-warning) 9%, var(--task-panel));
+  color: var(--vault-warning);
+}
+.vault-view-tabs .vault-nav-row.attention:hover {
+  background: color-mix(in srgb, var(--vault-warning) 14%, var(--task-panel));
+}
+.vault-view-tabs .vault-nav-row.attention.selected {
+  border-bottom-color: var(--task-accent);
+}
+.vault-view-tabs .vault-nav-row.attention .vault-nav-name {
+  color: var(--vault-warning);
+  font-weight: 680;
+}
+.vault-view-tabs .vault-nav-row.attention .vault-nav-name::before {
+  width: 6px;
+  height: 6px;
+  display: inline-block;
+  margin-right: 6px;
+  border-radius: 50%;
+  background: var(--vault-warning);
+  content: "";
+  vertical-align: 1px;
+}
+.vault-view-tabs .vault-nav-row.attention .vault-nav-count {
+  min-width: 20px;
+  padding: 1px 5px;
+  border-radius: 5px;
+  background: color-mix(in srgb, var(--vault-warning) 72%, var(--task-panel));
+  color: #241300;
+  font-weight: 720;
+  line-height: 1.35;
+  text-align: center;
+}
 .vault-pane-action-note { color: var(--task-muted); font-size: 11.5px; white-space: nowrap; }
 .vault-toolbar {
   display: flex;

package/server/public/vault.js

@@ -707,16 +707,20 @@ const eventLabels = {
 }
 const renderViews = () => {
   const counts = state.data.inventory.counts
+  const attentionViews = new Set(["duplicates", "reclaimable"])
   el("views-label").textContent = COPY.views
   const views = IS_APP_MODE
     ? ["all", "duplicates", "unavailable", "shared", "tracked", "activity"]
     : ["all", "duplicates", "unavailable", "shared", "tracked", "reclaimable", "activity"]
-  el("vault-views").innerHTML = views.map((view) => `
-    <button class="vault-nav-row ${state.view === view ? "selected" : ""}" type="button" data-view="${view}" ${state.view === view ? 'aria-current="page"' : ""}>
+  el("vault-views").innerHTML = views.map((view) => {
+    const needsAttention = attentionViews.has(view) && Number(counts[view]) > 0
+    return `
+    <button class="vault-nav-row ${state.view === view ? "selected" : ""} ${needsAttention ? "attention" : ""}" type="button" data-view="${view}" ${state.view === view ? 'aria-current="page"' : ""}>
       <i class="${viewIcon[view]}"></i>
       <span class="vault-nav-copy"><span class="vault-nav-name">${esc(viewLabel[view])}</span></span>
-      <span class="vault-nav-count ${view === "duplicates" && counts[view] ? "attention" : ""}">${counts[view]}</span>
-    </button>`).join("")
+      <span class="vault-nav-count ${needsAttention ? "attention" : ""}">${counts[view]}</span>
+    </button>`
+  }).join("")
 }
 
 const renderSourceNode = (source, depth, counts) => {