pinokiod 8.0.76 → 8.0.78

package/kernel/vault/automatic_scans.js

@@ -32,6 +32,12 @@ const pathKey = (value) => {
   return process.platform === "win32" ? resolved.toLowerCase() : resolved
 }
 
+const metadataKey = (entry) => [
+  Number(entry.mode) & 0o7777,
+  Number(entry.uid),
+  Number(entry.gid)
+].join(":")
+
 class AutomaticScans {
   constructor(vault) {
     this.vault = vault
@@ -270,7 +276,7 @@ class AutomaticScans {
     if (!collected.size) this.discardChangedPaths(app)
   }
 
-  async candidateThreshold() {
+  async candidatePolicy() {
     const cached = this.vault.lastScanCache &&
       typeof this.vault.lastScanCache.get === "function"
       ? this.vault.lastScanCache.get("")
@@ -279,9 +285,24 @@ class AutomaticScans {
       ? await this.vault.registry.scanFor()
       : null)
     const threshold = scan ? Number(scan.candidate_min_bytes) : NaN
-    return Number.isFinite(threshold) && threshold >= 0
-      ? threshold
-      : SIZE_THRESHOLD
+    const hasLinkableDevices = !!(
+      scan && Array.isArray(scan.linkable_devices))
+    return {
+      threshold: Number.isFinite(threshold) && threshold >= 0
+        ? threshold
+        : SIZE_THRESHOLD,
+      linkability_known: hasLinkableDevices,
+      linkable_devices: new Set(hasLinkableDevices
+        ? scan.linkable_devices
+          .map((dev) => Number(dev))
+          .filter((dev) => Number.isFinite(dev))
+        : [])
+    }
+  }
+
+  async candidateThreshold(policy = null) {
+    const current = policy || await this.candidatePolicy()
+    return current.threshold
   }
 
   statChangedPaths(paths, options, onSettled = null) {
@@ -910,7 +931,8 @@ class AutomaticScans {
     }
     this.checkpoint(active)
     if (!current.isFile() || current.isSymbolicLink() ||
-        current.dev !== entry.dev || current.size !== entry.size) {
+        current.dev !== entry.dev || current.size !== entry.size ||
+        metadataKey(current) !== metadataKey(entry)) {
       this.logVerificationSkip(
         active.app, entry, "snapshot-changed", counts)
       return null
@@ -1016,8 +1038,10 @@ class AutomaticScans {
       hash_failures: 0,
       unstable_hashes: 0
     }
-    markStage("threshold-read")
-    const threshold = await this.candidateThreshold()
+    markStage("policy-read")
+    const policy = await this.candidatePolicy()
+    const threshold = await this.candidateThreshold(policy)
+    const linkableDevices = policy.linkable_devices
     const paths = [...new Set(active.paths || [])].filter((filePath) =>
       typeof filePath === "string" && inside(root, filePath))
     const verifiedHashes = new Map()
@@ -1025,8 +1049,10 @@ class AutomaticScans {
       app,
       root,
       threshold_bytes: threshold,
+      linkability_known: policy.linkability_known,
+      linkable_devices: [...linkableDevices],
       changed_paths: paths.length,
-      policy: "metadata-prefilter-sha256"
+      policy: "eligibility-metadata-prefilter-sha256"
     })
     markStage("staging-reset")
     await this.vault.registry.beginAutomaticPrecheck(app)
@@ -1065,7 +1091,9 @@ class AutomaticScans {
         if (!stat || !stat.isFile() || stat.isSymbolicLink()) continue
         counts.files += 1
         counts.bytes += Math.max(0, Number(stat.size) || 0)
-        if (!isCandidateFileSize(stat.size, threshold)) continue
+        if (!isCandidateFileSize(stat.size, threshold) ||
+            (policy.linkability_known &&
+              !linkableDevices.has(stat.dev))) continue
         candidates.push({
           path: paths[index],
           size: stat.size,
@@ -1119,8 +1147,8 @@ class AutomaticScans {
       const preparePeers = () => {
         if (!group || group.unmatched) return
         group.unmatched = new Map()
-        for (const [hash, value] of group.candidatesByHash) {
-          group.unmatched.set(hash, {
+        for (const [match, value] of group.candidatesByMatch) {
+          group.unmatched.set(match, {
             identity: value.identities.values().next().value,
             candidate: value.candidate
           })
@@ -1149,10 +1177,11 @@ class AutomaticScans {
           )
           if (!verified) continue
           const hash = verified.hash
-          let value = group.candidatesByHash.get(hash)
+          const match = `${hash}\0${metadataKey(verified)}`
+          let value = group.candidatesByMatch.get(match)
           if (!value) {
             value = { candidate: verified, identities: new Set() }
-            group.candidatesByHash.set(hash, value)
+            group.candidatesByMatch.set(match, value)
           }
           value.identities.add(this.automaticIdentityKey(verified))
           if (value.identities.size > 1) {
@@ -1197,7 +1226,7 @@ class AutomaticScans {
               key,
               candidates: [],
               candidatesHashed: false,
-              candidatesByHash: new Map(),
+              candidatesByMatch: new Map(),
               unmatched: null,
               memory: new Map()
             }
@@ -1222,13 +1251,17 @@ class AutomaticScans {
           if (!await hashCandidates(identity)) continue
           if (proof) return await complete()
           if (!group.unmatched.size) continue
+          const peerMetadata = metadataKey(peer.entry)
           if (![...group.unmatched.values()].some((wanted) =>
-            wanted.identity !== identity)) continue
+            wanted.identity !== identity &&
+            metadataKey(wanted.candidate) === peerMetadata)) continue
           this.checkpoint(active)
           markStage("peer-hash", entry)
           const verified = await this.verifiedAutomaticHash(
             active, entry, group.memory, verifiedHashes, counts, peer)
-          const wanted = verified && group.unmatched.get(verified.hash)
+          const match = verified &&
+            `${verified.hash}\0${metadataKey(verified)}`
+          const wanted = match && group.unmatched.get(match)
           if (!wanted || wanted.identity === identity) {
             continue
           }

package/kernel/vault/registry_core.js

@@ -2807,6 +2807,9 @@ class RegistryCore {
         WHERE peer.app = candidate.app
           AND peer.dev = candidate.dev
           AND peer.size = candidate.size
+          AND (peer.mode & 4095) = (candidate.mode & 4095)
+          AND peer.uid = candidate.uid
+          AND peer.gid = candidate.gid
           AND pinokio_path_key(peer.path) !=
             pinokio_path_key(candidate.path)
       )
@@ -2814,7 +2817,10 @@ class RegistryCore {
         SELECT 1 FROM files peer
         WHERE peer.dev = candidate.dev
           AND peer.size = candidate.size
-          AND peer.unavailable_reason IS NOT 'stale'
+          AND peer.unavailable_reason IS NULL
+          AND (peer.mode & 4095) = (candidate.mode & 4095)
+          AND peer.uid = candidate.uid
+          AND peer.gid = candidate.gid
           AND pinokio_path_key(peer.path) !=
             pinokio_path_key(candidate.path)
       )
@@ -2823,6 +2829,9 @@ class RegistryCore {
         WHERE peer.verified_at IS NOT NULL
           AND peer.dev = candidate.dev
           AND peer.size = candidate.size
+          AND (peer.mode & 4095) = (candidate.mode & 4095)
+          AND peer.uid = candidate.uid
+          AND peer.gid = candidate.gid
           AND pinokio_path_key(peer.path) !=
             pinokio_path_key(candidate.path)
       )
@@ -2888,6 +2897,7 @@ class RegistryCore {
         WHERE candidate.app = @app
           AND candidate.dev = @dev
           AND candidate.size = @size
+          AND ${possiblePeer}
           AND candidate.path > @path
         ORDER BY candidate.path
         LIMIT @limit
@@ -2948,7 +2958,7 @@ class RegistryCore {
         WHERE peer.dev = @dev
           AND peer.size = @size
           AND peer.path > @path
-          AND peer.unavailable_reason IS NOT 'stale'
+          AND peer.unavailable_reason IS NULL
         ORDER BY peer.path
         LIMIT @limit
       `).all({ dev, size, path: afterPath, limit: pageSize })
@@ -3030,7 +3040,7 @@ class RegistryCore {
       hashes.set(canonicalPathKey(entry.path), entry.hash)
     }
     const rows = this.database.prepare(`
-      SELECT path, dev, ino, size, mtime, ctime
+      SELECT path, dev, ino, size, mtime, ctime, mode, uid, gid
       FROM automatic_precheck_files
       WHERE app = ?
       ORDER BY pinokio_path_key(path), path, dev, ino, size, mtime, ctime
@@ -3048,7 +3058,10 @@ class RegistryCore {
         row.ino,
         row.size,
         row.mtime,
-        row.ctime
+        row.ctime,
+        Number(row.mode) & 0o7777,
+        row.uid,
+        row.gid
       ]) {
         digest.update(String(value))
         digest.update("\0")
@@ -3062,6 +3075,12 @@ class RegistryCore {
     digest.update("\0")
     digest.update(String(proof.row.size))
     digest.update("\0")
+    digest.update(String(Number(proof.row.mode) & 0o7777))
+    digest.update("\0")
+    digest.update(String(proof.row.uid))
+    digest.update("\0")
+    digest.update(String(proof.row.gid))
+    digest.update("\0")
     digest.update(proof.hash)
     return {
       signature: digest.digest("hex"),

package/kernel/vault/sweeper.js

@@ -155,7 +155,6 @@ class Sweeper {
         : "complete"
       this.state.phase = "publishing"
       this.state.duration_ms = Date.now() - this.state.started
-      const metadata = this.scanMetadata(scopeId, outcome)
       const stores = anchorStores
         .filter((store) =>
           store.available && Number.isFinite(store.dev))
@@ -165,6 +164,12 @@ class Sweeper {
           can_link: store.mode !== "copy",
           root: store.root
         }))
+      const metadata = this.scanMetadata(scopeId, outcome)
+      if (!scopeId) {
+        metadata.linkable_devices = [...new Set(stores
+          .filter((store) => store.can_link)
+          .map((store) => store.dev))].sort((left, right) => left - right)
+      }
       const publication = await registry.publishScan(
         runId,
         this.publicationSourceIds(scopeId),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "8.0.76",
+  "version": "8.0.78",
   "description": "",
   "main": "index.js",
   "scripts": {

package/server/public/vault.css

@@ -2133,7 +2133,11 @@ body.dark .vault-row-action .vault-text-button:hover:not(:disabled) {
 .vault-event-time { color: var(--task-muted); font-size: 10.5px; }
 .vault-unavailable { color: var(--task-muted); font-size: 11px; }
 @media (max-width: 1140px) {
-  .vault-overview { align-items: flex-start; flex-direction: column; }
+  .vault-overview {
+    min-height: auto;
+    align-items: flex-start;
+    flex-direction: column;
+  }
   .vault-overview-actions { width: 100%; justify-content: flex-start; }
   .vault-scan-coachmark { right: auto; left: 0; }
   .vault-metrics { width: 100%; }

package/test/vault-automatic-scans.test.js

@@ -48,6 +48,13 @@ const makeVault = async (home, options = {}) => {
     : true
   vault.globalScanReady = async () => globalScanReady
   vault.automaticScans.globalScanReady = globalScanReady
+  if (globalScanReady) {
+    const homeStat = await fs.promises.stat(home)
+    vault.lastScanCache.set("", {
+      candidate_min_bytes: SIZE_THRESHOLD,
+      linkable_devices: [homeStat.dev]
+    })
+  }
   return vault
 }
 
@@ -1037,7 +1044,10 @@ describe("automatic app checks", () => {
       ["ino", 22],
       ["size", 201],
       ["mtime", 203],
-      ["ctime", 204]
+      ["ctime", 204],
+      ["mode", 0o640],
+      ["uid", 1],
+      ["gid", 1]
     ]) {
       assert.notEqual(
         signature([proof, Object.assign({}, unmatched, { [field]: value })]),
@@ -1161,6 +1171,103 @@ describe("automatic app checks", () => {
     await close(vault)
   })
 
+  test("metadata-incompatible copies do not publish a result", async (t) => {
+    if (process.platform === "win32") {
+      t.skip("POSIX permission metadata is required")
+      return
+    }
+    const home = await makeHome()
+    const app = "metadata-incompatible-app"
+    const appRoot = path.join(home, "api", app)
+    const first = path.join(appRoot, "first.bin")
+    const second = path.join(appRoot, "second.bin")
+    await fs.promises.mkdir(appRoot)
+    await fs.promises.writeFile(first, "same-content")
+    await fs.promises.writeFile(second, "same-content")
+    await fs.promises.chmod(first, 0o600)
+    await fs.promises.chmod(second, 0o644)
+    const vault = await makeVault(home)
+    vault.automaticScans.candidateThreshold = async () => 1
+    const hashedPaths = []
+    const originalHashFile = vault.hashFile.bind(vault)
+    vault.hashFile = async (filePath, options) => {
+      hashedPaths.push(filePath)
+      return originalHashFile(filePath, options)
+    }
+
+    vault.automaticScans.queueApp(app, [first, second])
+    await waitFor(() => !vault.automaticScans.active &&
+      !vault.automaticScans.entries.has(app))
+
+    assert.deepEqual(hashedPaths, [])
+    assert.deepEqual(vault.automaticScans.snapshot().rows, [])
+    await close(vault)
+  })
+
+  test("devices without hardlink support do not publish a result", async () => {
+    const home = await makeHome()
+    const app = "copy-only-device-app"
+    const appRoot = path.join(home, "api", app)
+    const first = path.join(appRoot, "first.bin")
+    const second = path.join(appRoot, "second.bin")
+    await fs.promises.mkdir(appRoot)
+    await fs.promises.writeFile(first, "same-content")
+    await fs.promises.writeFile(second, "same-content")
+    const vault = await makeVault(home)
+    vault.automaticScans.candidateThreshold = async () => 1
+    vault.automaticScans.candidatePolicy = async () => ({
+      threshold: 1,
+      linkability_known: true,
+      linkable_devices: new Set()
+    })
+    const hashedPaths = []
+    const originalHashFile = vault.hashFile.bind(vault)
+    vault.hashFile = async (filePath, options) => {
+      hashedPaths.push(filePath)
+      return originalHashFile(filePath, options)
+    }
+
+    vault.automaticScans.queueApp(app, [first, second])
+    await waitFor(() => !vault.automaticScans.active &&
+      !vault.automaticScans.entries.has(app))
+
+    assert.deepEqual(hashedPaths, [])
+    assert.deepEqual(vault.automaticScans.snapshot().rows, [])
+    await close(vault)
+  })
+
+  test("legacy global scans without linkability metadata preserve automatic checks", async () => {
+    const home = await makeHome()
+    const app = "legacy-global-scan-app"
+    const appRoot = path.join(home, "api", app)
+    const first = path.join(appRoot, "first.bin")
+    const second = path.join(appRoot, "second.bin")
+    await fs.promises.mkdir(appRoot)
+    await fs.promises.writeFile(first, "same-content")
+    await fs.promises.writeFile(second, "same-content")
+    const vault = await makeVault(home)
+    vault.lastScanCache.set("", { candidate_min_bytes: 1 })
+    const hashedPaths = []
+    const originalHashFile = vault.hashFile.bind(vault)
+    vault.hashFile = async (filePath, options) => {
+      hashedPaths.push(filePath)
+      return originalHashFile(filePath, options)
+    }
+
+    vault.automaticScans.queueApp(app, [first, second])
+    await waitFor(() => {
+      const row = vault.automaticScans.entries.get(app)
+      return !vault.automaticScans.active && row && row.state === "result"
+    })
+
+    assert.deepEqual(hashedPaths.sort(), [first, second].sort())
+    assert.deepEqual(vault.automaticScans.snapshot().rows.map((row) => ({
+      app: row.app,
+      state: row.state
+    })), [{ app, state: "result" }])
+    await close(vault)
+  })
+
   test("hash worker cancellation interrupts a read without poisoning the worker", async () => {
     const home = await makeHome()
     const large = path.join(home, "large.bin")
@@ -2113,7 +2220,7 @@ describe("automatic app checks", () => {
     await close(vault)
   })
 
-  test("peer verification rejects changed type, device, and size", async (t) => {
+  test("peer verification rejects changed eligibility metadata", async (t) => {
     const home = await makeHome()
     const automatic = makeAutomatic({
       enabled: true,
@@ -2141,6 +2248,18 @@ describe("automatic app checks", () => {
       {
         name: "size",
         stat: { file: true, link: false, dev: 1, size: 5 }
+      },
+      {
+        name: "permissions",
+        stat: { file: true, link: false, dev: 1, size: 4, mode: 1 }
+      },
+      {
+        name: "owner",
+        stat: { file: true, link: false, dev: 1, size: 4, uid: 1 }
+      },
+      {
+        name: "group",
+        stat: { file: true, link: false, dev: 1, size: 4, gid: 1 }
       }
     ]
     const currentStats = new Map(cases.map(({ name, stat }) => [
@@ -2154,9 +2273,9 @@ describe("automatic app checks", () => {
         mtimeMs: 1,
         ctimeMs: 1,
         nlink: 1,
-        mode: 0,
-        uid: 0,
-        gid: 0
+        mode: stat.mode || 0,
+        uid: stat.uid || 0,
+        gid: stat.gid || 0
       }
     ]))
     const lstat = fs.promises.lstat
@@ -2172,7 +2291,10 @@ describe("automatic app checks", () => {
         size: 4,
         ino: 1,
         mtime: 1,
-        ctime: 1
+        ctime: 1,
+        mode: 0,
+        uid: 0,
+        gid: 0
       }, counts), null)
     }
 
@@ -3426,7 +3548,11 @@ describe("automatic app checks", () => {
       candidate("hardlink-a.bin", 108, 7),
       candidate("hardlink-b.bin", 108, 7),
       candidate("file-peer.bin", 109, 8),
-      candidate("anchor-peer.bin", 110, 9)
+      candidate("anchor-peer.bin", 110, 9),
+      candidate("unavailable-peer.bin", 111, 10),
+      Object.assign(candidate("metadata-mismatch.bin", 108, 11), {
+        mode: 1
+      })
     ]
     const file = (filePath, size, dev, ino, unavailableReason = null) => ({
       path: filePath,
@@ -3454,6 +3580,13 @@ describe("automatic app checks", () => {
       path.join(home, "stale-peer.bin"), 106, 1, 23, "stale"))
     await registry.upsertFile(file(
       path.join(home, "valid-file-peer.bin"), 109, 1, 24))
+    const unavailableWithinEligibleGroup = path.join(
+      home, "unavailable-within-eligible-group.bin")
+    await registry.upsertFile(file(
+      unavailableWithinEligibleGroup, 109, 1, 25, "metadata"))
+    await registry.upsertFile(file(
+      path.join(home, "unavailable-file-peer.bin"), 111, 1, 26,
+      "metadata"))
 
     for (const [name, size, verifiedAt] of [
       ["unverified", 107, null],
@@ -3497,6 +3630,9 @@ describe("automatic app checks", () => {
             WHERE peer.app = candidate.app
               AND peer.dev = candidate.dev
               AND peer.size = candidate.size
+              AND (peer.mode & 4095) = (candidate.mode & 4095)
+              AND peer.uid = candidate.uid
+              AND peer.gid = candidate.gid
               AND pinokio_path_key(peer.path) !=
                 pinokio_path_key(candidate.path)
           )
@@ -3504,7 +3640,10 @@ describe("automatic app checks", () => {
             SELECT 1 FROM files peer
             WHERE peer.dev = candidate.dev
               AND peer.size = candidate.size
-              AND peer.unavailable_reason IS NOT 'stale'
+              AND peer.unavailable_reason IS NULL
+              AND (peer.mode & 4095) = (candidate.mode & 4095)
+              AND peer.uid = candidate.uid
+              AND peer.gid = candidate.gid
               AND pinokio_path_key(peer.path) !=
                 pinokio_path_key(candidate.path)
           )
@@ -3513,6 +3652,9 @@ describe("automatic app checks", () => {
             WHERE peer.verified_at IS NOT NULL
               AND peer.dev = candidate.dev
               AND peer.size = candidate.size
+              AND (peer.mode & 4095) = (candidate.mode & 4095)
+              AND peer.uid = candidate.uid
+              AND peer.gid = candidate.gid
               AND pinokio_path_key(peer.path) !=
                 pinokio_path_key(candidate.path)
           )
@@ -3525,9 +3667,11 @@ describe("automatic app checks", () => {
     assert.match(plan, /files_size_device_path_idx/)
     assert.match(plan, /anchors_size_device_path_idx/)
     const changed = []
+    const returnedPaths = []
     let cursor = null
     do {
       const page = registry.automaticPrecheckEntries(app, cursor, 2)
+      returnedPaths.push(...page.entries.map((entry) => entry.path))
       changed.push(...page.entries
         .filter((entry) => entry.kind === "changed")
         .map((entry) => entry.path))
@@ -3540,6 +3684,7 @@ describe("automatic app checks", () => {
       entries[8].path,
       entries[9].path
     ].sort())
+    assert.equal(returnedPaths.includes(unavailableWithinEligibleGroup), false)
     registry.close()
   })
 

package/test/vault-engine.test.js

@@ -189,6 +189,13 @@ describe("Save Space engine", () => {
 
     assert.equal((await vault.status()).global_candidate_min_bytes, selected)
     assert.equal(await vault.automaticScans.candidateThreshold(), selected)
+    const scan = await vault.registry.scanFor()
+    const expectedDevices = vault.anchorStores()
+      .filter((store) => store.available && Number.isFinite(store.dev) &&
+        store.mode !== "copy")
+      .map((store) => store.dev)
+      .sort((left, right) => left - right)
+    assert.deepEqual(scan.linkable_devices, expectedDevices)
 
     await close(vault)
   })

package/test/vault-ui.test.js

@@ -546,6 +546,8 @@ describe("Save Space interface", () => {
     assert.match(combined, /unique to this app/)
     assert.match(vaultCss,
       /body\.vault-page \.vault-view-tabs\s*\{[^}]*padding:\s*0;/s)
+    assert.match(vaultCss,
+      /@media \(max-width: 1140px\)[\s\S]*?\.vault-overview\s*\{[^}]*min-height:\s*auto;/s)
     assert.match(vaultCss,
       /body\.dark\.vault-page\s*\{[^}]*--task-panel:\s*var\(--pinokio-sidebar-tabbar-bg\);/s)
     assert.match(vaultCss,