pinokiod 8.0.41 → 8.0.44

package/kernel/vault/registry.js

@@ -1,7 +1,7 @@
 const fs = require('fs')
 const path = require('path')
 const crypto = require('crypto')
-const { SHA256_RE, CANDIDATE_SIZE_OPTIONS } = require('./constants')
+const { SHA256_RE } = require('./constants')
 
 const isMissingError = (error) => !!(error && (error.code === "ENOENT" || error.code === "ENOTDIR"))
 const isRecord = (value) => !!(value && typeof value === "object" && !Array.isArray(value))
@@ -42,15 +42,13 @@ class Registry {
   }
   reset() {
     this.blobs = new Map()      // hash -> { size, first_seen, source_urls, verified_at, orphan }
-    this.links = new Map()      // path -> { hash, app, source_id, dev, ino, created, batch_id? }
+    this.links = new Map()      // path -> { hash, app, source_id, dev, ino, created, mode, batch_id? }
     this.scanIndex = new Map()  // path -> { size, mtime, dev, ino, hash, source_id }
     this.duplicates = new Map() // path -> { hash, size, app, source_id, discovered } pending user conversion
     this.excluded = new Map()   // path -> { ts, source_id, size } user chose "keep independent"
-    this.lastScan = null         // last complete global scan
-    this.lastScanAttempt = null  // last global scan attempt, complete or partial
+    this.lastScan = null        // scan totals, duration, and hash work
     this.sourceScans = new Map() // source id -> last complete scoped scan
-    this.sourceScanAttempts = new Map() // source id -> last scoped scan attempt
-    this.settings = { external_sources: [] } // durable user preferences, not derived scan state
+    this.totals = { lifetime_bytes_saved: 0 }
     this.byIno = new Map()      // "dev:ino" -> hash
     this.pathsByIno = new Map() // "dev:ino" -> Set(paths), for bounded group updates
     this.eventsCache = null
@@ -170,24 +168,9 @@ class Registry {
   scanFor(sourceId = null) {
     return sourceId ? (this.sourceScans.get(sourceId) || null) : this.lastScan
   }
-  scanAttemptFor(sourceId = null) {
-    return sourceId
-      ? (this.sourceScanAttempts.get(sourceId) || null)
-      : this.lastScanAttempt
-  }
   setLastScan(meta, sourceId = null) {
-    if (sourceId) {
-      this.sourceScans.set(sourceId, meta)
-      this.sourceScanAttempts.set(sourceId, meta)
-    } else {
-      this.lastScan = meta
-      this.lastScanAttempt = meta
-    }
-    this.schedulePersist()
-  }
-  setScanAttempt(meta, sourceId = null) {
-    if (sourceId) this.sourceScanAttempts.set(sourceId, meta)
-    else this.lastScanAttempt = meta
+    if (sourceId) this.sourceScans.set(sourceId, meta)
+    else this.lastScan = meta
     this.schedulePersist()
   }
   inoKey(dev, ino) {
@@ -217,10 +200,9 @@ class Registry {
     // A path has exactly one derived classification. Registering a physical
     // name always clears an older pending-duplicate classification first.
     this.duplicates.delete(filePath)
-    const next = Object.assign({ created: Date.now() }, entry)
-    delete next.mode
+    const next = Object.assign({ created: Date.now(), mode: "link" }, entry)
     const sameTrackedFile = previous && previous.hash === next.hash &&
-      previous.dev === next.dev && previous.ino === next.ino
+      previous.mode === next.mode && previous.dev === next.dev && previous.ino === next.ino
     if (next.batch_id === undefined && sameTrackedFile && previous.batch_id) {
       next.batch_id = previous.batch_id
     }
@@ -303,46 +285,22 @@ class Registry {
     }
     this.schedulePersist()
   }
-  snapshotState() {
-    const cloneMap = (source) => new Map([...source].map(([key, value]) => [
-      key,
-      value && typeof value === "object"
-        ? Object.assign({}, value, Array.isArray(value.source_urls)
-          ? { source_urls: [...value.source_urls] }
-          : {})
-        : value
-    ]))
-    return {
-      blobs: cloneMap(this.blobs),
-      links: cloneMap(this.links),
-      scanIndex: cloneMap(this.scanIndex),
-      duplicates: cloneMap(this.duplicates),
-      excluded: cloneMap(this.excluded),
-      lastScan: this.lastScan ? Object.assign({}, this.lastScan) : null,
-      lastScanAttempt: this.lastScanAttempt ? Object.assign({}, this.lastScanAttempt) : null,
-      sourceScans: cloneMap(this.sourceScans),
-      sourceScanAttempts: cloneMap(this.sourceScanAttempts),
-      settings: Object.assign({}, this.settings, {
-        external_sources: [...(this.settings.external_sources || [])]
-      }),
-      byIno: new Map(this.byIno),
-      pathsByIno: new Map([...this.pathsByIno].map(([key, paths]) => [key, new Set(paths)]))
-    }
-  }
-  replaceState(state, options = {}) {
+  addSaved(bytes) {
+    this.totals.lifetime_bytes_saved += bytes
+    this.schedulePersist()
+  }
+  replaceState(state) {
     this.blobs = state.blobs
     this.links = state.links
     this.scanIndex = state.scanIndex
     this.duplicates = state.duplicates
     this.excluded = state.excluded
     this.lastScan = state.lastScan
-    this.lastScanAttempt = state.lastScanAttempt || null
     this.sourceScans = state.sourceScans
-    this.sourceScanAttempts = state.sourceScanAttempts || new Map()
-    this.settings = Object.assign({ external_sources: [] }, state.settings)
+    this.totals = state.totals
     this.byIno = state.byIno
     this.pathsByIno = state.pathsByIno
-    if (options.persist !== false) this.schedulePersist()
+    this.schedulePersist()
   }
   // Returns { corrupt: true } when the snapshot exists but cannot be parsed,
   // so the caller can rebuild from disk instead of blocking startup.
@@ -375,16 +333,10 @@ class Registry {
         }))
       }
     }
-    const legacyCopies = []
     for (const [k, v] of Object.entries(json.links || {})) {
       if (!isRecord(v) || !SHA256_RE.test(v.hash) ||
           !Number.isFinite(v.dev) || !Number.isFinite(v.ino)) continue
-      if (v.mode === "copy") {
-        legacyCopies.push([k, v])
-        continue
-      }
-      const link = Object.assign({}, v)
-      delete link.mode
+      const link = Object.assign({}, v, { mode: v.mode === "copy" ? "copy" : "link" })
       this.links.set(k, link)
       if (link.dev !== undefined && link.ino !== undefined) {
         const key = this.inoKey(link.dev, link.ino)
@@ -406,26 +358,6 @@ class Registry {
         }))
       }
     }
-    // Older builds recorded independent files as mode:"copy" links. They were
-    // never shared. Migrate only their review metadata and never recreate that
-    // unsupported state.
-    for (const [filePath, link] of legacyCopies) {
-      if (this.duplicates.has(filePath) || !this.blobs.has(link.hash)) continue
-      const indexed = this.scanIndex.get(filePath)
-      if (!indexed || indexed.hash !== link.hash) continue
-      this.duplicates.set(filePath, {
-        hash: link.hash,
-        size: indexed.size,
-        app: link.app || null,
-        source_id: link.source_id || indexed.source_id || null,
-        discovered: link.created || Date.now(),
-        dev: indexed.dev,
-        ino: indexed.ino,
-        mtime: indexed.mtime,
-        ctime: indexed.ctime,
-        unavailable_reason: "unsupported_disk"
-      })
-    }
     for (const [k, v] of Object.entries(json.excluded || {})) {
       if (!isRecord(v)) continue
       const linked = this.links.get(k)
@@ -439,45 +371,14 @@ class Registry {
         size: Number.isFinite(v.size) && v.size >= 0 ? v.size : 0
       }))
     }
-    if (isRecord(json.scans)) {
-      for (const [scopeId, value] of Object.entries(json.scans)) {
-        if (!isRecord(value)) continue
-        const complete = isRecord(value.last_complete) ? value.last_complete : null
-        const attempt = isRecord(value.last_attempt) ? value.last_attempt : null
-        if (scopeId === "global") {
-          this.lastScan = complete
-          this.lastScanAttempt = attempt
-        } else {
-          if (complete) this.sourceScans.set(scopeId, complete)
-          if (attempt) this.sourceScanAttempts.set(scopeId, attempt)
-        }
-      }
-    } else {
-      if (isRecord(json.last_scan)) {
-        this.lastScan = json.last_scan
-        this.lastScanAttempt = json.last_scan
-      }
-      for (const [sourceId, scan] of Object.entries(json.source_scans || {})) {
-        if (!isRecord(scan)) continue
-        this.sourceScans.set(sourceId, scan)
-        this.sourceScanAttempts.set(sourceId, scan)
-      }
+    if (isRecord(json.last_scan)) {
+      this.lastScan = json.last_scan
     }
-    if (isRecord(json.settings)) {
-      if (CANDIDATE_SIZE_OPTIONS.includes(json.settings.candidate_min_bytes)) {
-        this.settings.candidate_min_bytes = json.settings.candidate_min_bytes
-      }
-      if (Array.isArray(json.settings.external_sources)) {
-        const seen = new Set()
-        for (const value of json.settings.external_sources) {
-          if (typeof value !== "string" || !path.isAbsolute(value)) continue
-          const resolved = path.resolve(value)
-          const key = process.platform === "win32" ? resolved.toLowerCase() : resolved
-          if (seen.has(key)) continue
-          seen.add(key)
-          this.settings.external_sources.push(resolved)
-        }
-      }
+    for (const [sourceId, scan] of Object.entries(json.source_scans || {})) {
+      if (isRecord(scan)) this.sourceScans.set(sourceId, scan)
+    }
+    if (json.totals && Number.isFinite(json.totals.lifetime_bytes_saved)) {
+      this.totals = { lifetime_bytes_saved: Math.max(0, json.totals.lifetime_bytes_saved) }
     }
     return { corrupt: false, existed: true }
   }
@@ -491,12 +392,7 @@ class Registry {
   }
   async endBatch(options = {}) {
     if (this.persistDepth > 0) this.persistDepth -= 1
-    if (this.persistDepth) return
-    if (options.discard) {
-      this.persistDirty = false
-      return
-    }
-    if (!this.persistDirty) return
+    if (this.persistDepth || !this.persistDirty) return
     if (options.flush === false) this.schedulePersist()
     else await this.flush()
   }
@@ -518,21 +414,6 @@ class Registry {
       this.persistTimer = null
     }
     this.persistDirty = false
-    const scans = {
-      global: {
-        last_complete: this.lastScan,
-        last_attempt: this.lastScanAttempt
-      }
-    }
-    for (const sourceId of new Set([
-      ...this.sourceScans.keys(),
-      ...this.sourceScanAttempts.keys()
-    ])) {
-      scans[sourceId] = {
-        last_complete: this.sourceScans.get(sourceId) || null,
-        last_attempt: this.sourceScanAttempts.get(sourceId) || null
-      }
-    }
     const json = {
       version: 1,
       blobs: Object.fromEntries(this.blobs),
@@ -540,8 +421,9 @@ class Registry {
       scan_index: Object.fromEntries(this.scanIndex),
       duplicates: Object.fromEntries(this.duplicates),
       excluded: Object.fromEntries(this.excluded),
-      scans,
-      settings: this.settings
+      last_scan: this.lastScan,
+      source_scans: Object.fromEntries(this.sourceScans),
+      totals: this.totals
     }
     const write = async () => {
       await this.atomicWrite(this.snapshotPath, JSON.stringify(json))
@@ -561,25 +443,21 @@ class Registry {
     }
   }
   async appendEvent(event) {
-    return this.appendEvents([event])
-  }
-  async appendEvents(events) {
-    if (!events.length) return
-    const entries = events.map((event) => Object.assign({ ts: Date.now() }, event))
-    const contents = entries.map((event) => JSON.stringify(event)).join("\n") + "\n"
+    const line = JSON.stringify(Object.assign({ ts: Date.now() }, event)) + "\n"
     const append = async () => {
-      await this.appendFile(this.eventsPath, contents)
-      if (this.eventsCache) this.eventsCache.push(...entries)
-      this.eventsSinceCompact += entries.length
+      await this.appendFile(this.eventsPath, line)
+      if (this.eventsCache) this.eventsCache.push(JSON.parse(line))
+      this.eventsSinceCompact += 1
       if ((this.eventsCache && this.eventsCache.length > this.maxEvents + this.compactEvery) ||
           this.eventsSinceCompact >= this.compactEvery) {
         await this.compactEventsNow()
       }
     }
     const pending = this.eventPromise.then(append, append)
-    // One ordered writer keeps action and scan batches ordered. Keep it usable
-    // after a failure while returning the real append result so the dashboard
-    // can expose missing activity without misreporting the action.
+    // One ordered writer prevents a scan with many matches from opening an
+    // unbounded number of concurrent append operations. Keep the writer
+    // usable after a failure while returning the real append result so the
+    // dashboard can expose missing activity without misreporting the action.
     this.eventPromise = pending.then(
       () => { this.eventError = null },
       (error) => {