pinokiod 8.0.50 → 8.0.51

package/kernel/vault/index.js

@@ -103,8 +103,8 @@ class Vault {
   get blobRoot() {
     return path.resolve(this.root, "sha256")
   }
-  get sourceRoot() {
-    return path.resolve(this.root, "sources")
+  get externalSourcesPath() {
+    return path.resolve(this.root, "external-sources.json")
   }
   storePathFor(hash) {
     if (typeof hash !== "string" || !SHA256_RE.test(hash)) {
@@ -129,6 +129,43 @@ class Vault {
     if (!st) throw unsafeStoragePath(directory)
     return st
   }
+  async readExternalSourcePaths() {
+    let raw
+    try {
+      raw = await this.registry.readFile(this.externalSourcesPath)
+    } catch (error) {
+      if (isMissingError(error)) return []
+      throw error
+    }
+    let parsed
+    try {
+      parsed = JSON.parse(raw)
+    } catch (error) {
+      throw new Error("External folder settings are invalid.")
+    }
+    if (!parsed || !Array.isArray(parsed.paths)) {
+      throw new Error("External folder settings are invalid.")
+    }
+    const paths = []
+    for (const value of parsed.paths) {
+      if (typeof value !== "string" || !path.isAbsolute(value.trim())) continue
+      const resolved = path.resolve(value.trim())
+      if (!paths.some((existing) => samePath(existing, resolved))) paths.push(resolved)
+    }
+    return paths
+  }
+  async writeExternalSourcePaths(folderPaths) {
+    const paths = []
+    for (const value of folderPaths) {
+      if (typeof value !== "string" || !path.isAbsolute(value.trim())) continue
+      const resolved = path.resolve(value.trim())
+      if (!paths.some((existing) => samePath(existing, resolved))) paths.push(resolved)
+    }
+    await this.registry.atomicWrite(
+      this.externalSourcesPath,
+      JSON.stringify({ paths }, null, 2)
+    )
+  }
   async storeStatIfPresent(storePath, options = {}) {
     if (!isPathWithin(this.blobRoot, storePath)) throw unsafeStoragePath(storePath)
     if (!await this.directoryIfSafe(this.root) || !await this.directoryIfSafe(this.blobRoot)) {
@@ -205,6 +242,11 @@ class Vault {
           }
         }
       }
+      case "remove_source":
+        if (typeof payload.source_id !== "string" || !payload.source_id) {
+          return { error: "Choose an external folder to remove." }
+        }
+        return this.runMutation(() => this.removeExternalSource(payload.source_id))
       case "scan": {
         if (payload.scope_id && !this.scanSource(payload.scope_id)) {
           return { error: "That scan location is no longer available." }
@@ -313,34 +355,6 @@ class Vault {
       return source
     }
 
-    const seenExternalRoots = new Set()
-    const addExternalEntries = async (importRoot, idPrefix = "") => {
-      let entries = []
-      try {
-        entries = await fs.promises.readdir(importRoot, { withFileTypes: true })
-      } catch (error) {
-        if (!isMissingError(error)) throw error
-      }
-      for (const entry of entries) {
-        if (!entry.isSymbolicLink()) continue
-        const mountPath = path.resolve(importRoot, entry.name)
-        try {
-          const root = await fs.promises.realpath(mountPath)
-          const st = await fs.promises.stat(root)
-          if (!st.isDirectory()) continue
-          const canonical = path.resolve(root)
-          if (seenExternalRoots.has(canonical)) continue
-          seenExternalRoots.add(canonical)
-          sources.push(await decorate({
-            id: sourceId("external", `${idPrefix}${entry.name}`), kind: "external", label: entry.name,
-            root: canonical, mount_path: mountPath, parent_id: "external"
-          }))
-        } catch (error) {
-          if (!isMissingError(error)) throw error
-        }
-      }
-    }
-
     let apiEntries = []
     try {
       apiEntries = await fs.promises.readdir(apiRoot, { withFileTypes: true })
@@ -356,9 +370,15 @@ class Vault {
         }))
       }
     }
-    await addExternalEntries(apiRoot)
-    if (await this.directoryIfSafe(this.sourceRoot)) {
-      await addExternalEntries(this.sourceRoot, "vault:")
+    for (const configuredPath of await this.readExternalSourcePaths()) {
+      sources.push(await decorate({
+        id: sourceId("external", configuredPath),
+        kind: "external",
+        label: path.basename(configuredPath) || configuredPath,
+        root: configuredPath,
+        parent_id: "external",
+        configured: true
+      }))
     }
 
     let homeEntries = []
@@ -421,49 +441,38 @@ class Vault {
       return { created: false, source: existing }
     }
 
-    const importRoot = this.sourceRoot
-    await this.ensureDirectory(importRoot)
-    const baseLabel = path.basename(canonical) || "external-folder"
-    let mountPath = null
-    for (let index = 1; index < 10000; index++) {
-      const label = index === 1 ? baseLabel : `${baseLabel}-${index}`
-      const candidate = path.resolve(importRoot, label)
-      try {
-        await fs.promises.symlink(canonical, candidate, this.kernel.platform === "win32" ? "junction" : "dir")
-        mountPath = candidate
-        break
-      } catch (error) {
-        if (error && error.code === "EEXIST") continue
-        if (error && (error.code === "EACCES" || error.code === "EPERM")) {
-          throw new Error("Pinokio does not have permission to add that folder.")
-        }
-        throw new Error("Pinokio could not add that folder.")
-      }
+    const configuredPaths = await this.readExternalSourcePaths()
+    configuredPaths.push(canonical)
+    await this.writeExternalSourcePaths(configuredPaths)
+    await this.refreshSources()
+    const source = this._sources.find((item) =>
+      item.kind === "external" && item.configured && samePath(item.root, canonical))
+    if (!source) {
+      throw new Error("The folder could not be added. Restart Pinokio and try again.")
     }
-    if (!mountPath) throw new Error("Pinokio could not create a unique name for that folder.")
+    return { created: true, source }
+  }
+  async removeExternalSource(sourceIdToRemove) {
+    await this.refreshSources()
+    const source = this._sources.find((item) =>
+      item.id === sourceIdToRemove && item.kind === "external" && item.configured)
+    if (!source) return { error: "That external folder is no longer configured." }
 
-    try {
-      await this.refreshSources()
-      const source = this._sources.find((item) => item.kind === "external" && item.mount_path && samePath(item.mount_path, mountPath))
-      if (!source) throw new Error("The folder could not be added. Restart Pinokio and try again.")
-      return { created: true, source }
-    } catch (error) {
-      // Adding a source is transactional. Roll back only the exact directory
-      // link created above; preserve any path an external writer replaced.
-      try {
-        const [mounted, target] = await Promise.all([
-          fs.promises.lstat(mountPath),
-          fs.promises.realpath(mountPath)
-        ])
-        if (mounted.isSymbolicLink() && samePath(target, canonical)) {
-          await fs.promises.unlink(mountPath)
-        }
-      } catch {
-        // The original error remains authoritative; an unverified path is
-        // deliberately preserved rather than deleted.
-      }
-      throw error
+    const configuredPaths = await this.readExternalSourcePaths()
+    const remaining = configuredPaths.filter((folderPath) => !samePath(folderPath, source.root))
+    if (remaining.length === configuredPaths.length) {
+      return { error: "That external folder is no longer configured." }
     }
+    await this.writeExternalSourcePaths(remaining)
+    await this.refreshSources()
+    this.reconcileConfiguredSources()
+    for (const [filePath, entry] of [...this.registry.excluded]) {
+      if (entry.source_id === source.id) this.registry.excluded.delete(filePath)
+    }
+    this.registry.sourceScans.delete(source.id)
+    this.registry.lastScan = null
+    this.registry.schedulePersist()
+    return { removed: true, source_id: source.id, label: source.label }
   }
   sourceForPath(filePath, preferredId) {
     let cursor = path.resolve(filePath)
@@ -1811,7 +1820,8 @@ class Vault {
       display_path: source.kind === "pinokio" ? source.root : (source.mount_path || source.root),
       target_path: source.kind === "external" ? source.root : null,
       parent_id: scopeId ? null : source.parent_id, app: source.app || null,
-      available: source.available !== false, shareable: source.kind === "virtual" ? null : !!source.shareable
+      available: source.available !== false, shareable: source.kind === "virtual" ? null : !!source.shareable,
+      removable: source.kind === "external" && source.configured === true
     }))
     const result = {
       enabled: true,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "8.0.50",
+  "version": "8.0.51",
   "description": "",
   "main": "index.js",
   "scripts": {

package/server/public/vault.js

@@ -20,6 +20,9 @@ const COPY = {
   external_added: "Added to Locations. Run a scan when you’re ready.",
   external_exists: "That folder is already in Locations.",
   external_other_disk: "Added to Locations. It can be scanned, but files on this disk cannot be deduplicated with files in your Pinokio folder.",
+  remove_external_folder: "Remove from Locations",
+  remove_external_confirm: "Remove “{name}” from Locations? This does not delete or modify any files.",
+  external_removed: "Removed from Locations. No files were changed.",
   pinokio_folder: "Pinokio folder",
   save_space: "Save space",
   disk_space_freed: "of disk space freed",
@@ -530,6 +533,9 @@ const batchAction = (source) => {
   if (!source.shareable || !shareable.length) return `<div class="vault-pane-action-note">${esc(COPY.sharing_unavailable)}</div>`
   return `<button class="vault-button" type="button" data-deduplicate-scope="${attr(source.id)}">${esc(COPY.deduplicate)} ${countLabel(shareable.length)}</button>`
 }
+const removeSourceAction = (source) => source && source.removable
+  ? `<button class="vault-button" type="button" data-remove-source="${attr(source.id)}">${esc(COPY.remove_external_folder)}</button>`
+  : ""
 
 const toolbarSummary = (visibleItems) => {
   if (state.view === "duplicates") {
@@ -588,7 +594,8 @@ const renderToolbar = (visibleItems, allItems) => {
     ${displayModeControl()}
     ${descriptionMarkup}
     <span class="vault-toolbar-count" id="vault-toolbar-summary">${esc(toolbarSummary(visibleItems))}</span>
-    ${state.view === "all" ? batchAction(source) : bulkDeduplicationAction(allItems)}`
+    ${state.view === "all" ? batchAction(source) : bulkDeduplicationAction(allItems)}
+    ${removeSourceAction(source)}`
 }
 
 const statusMarkup = (item) => {
@@ -1582,6 +1589,18 @@ document.addEventListener("click", async (event) => {
     if (state.expandedFiles.has(file)) state.expandedFiles.delete(file)
     else state.expandedFiles.add(file)
     render()
+  } else if (target.dataset.removeSource) {
+    const source = sourceById(target.dataset.removeSource)
+    if (!source) return
+    const message = COPY.remove_external_confirm.replace("{name}", source.label)
+    if (!window.confirm(message)) return
+    await runAction({
+      action: "remove_source",
+      source_id: source.id
+    }, () => {
+      state.sourceId = null
+      return COPY.external_removed
+    })
   } else if (target.id === "btn-add-source") {
     target.disabled = true
     try {

package/test/vault-sweep.test.js

@@ -265,7 +265,7 @@ describe('vault manual scan (phase 3)', () => {
     assert.strictEqual((await fs.promises.stat(outside)).nlink, 1)
   })
 
-  test('top-level linked imports are scanned as external sources while nested links stay skipped', async (t) => {
+  test('top-level api links are not adopted as external sources', async (t) => {
     const { home, vault } = await makeEnv()
     const external = await fs.promises.mkdtemp(path.resolve(os.tmpdir(), 'pinokio-external-'))
     const nested = await fs.promises.mkdtemp(path.resolve(os.tmpdir(), 'pinokio-external-nested-'))
@@ -286,22 +286,20 @@ describe('vault manual scan (phase 3)', () => {
     const source = vault.sources().find((item) => item.kind === 'external' && item.label === 'linked-models')
     const canonicalImported = path.resolve(await fs.promises.realpath(external), 'models', 'm.bin')
     const canonicalNestedFile = path.resolve(await fs.promises.realpath(nested), 'should-not-scan.bin')
-    assert.ok(source, 'linked import becomes an external source')
-    assert.strictEqual(source.parent_id, 'external')
-    assert.ok(vault.registry.scanIndex.has(canonicalImported), 'external root was scanned')
+    assert.strictEqual(source, undefined, 'api links are not Vault configuration')
+    assert.strictEqual(vault.registry.scanIndex.has(canonicalImported), false)
     assert.strictEqual(vault.registry.scanIndex.has(canonicalNestedFile), false, 'nested directory link was not followed')
-    assert.strictEqual(vault.registry.duplicates.get(canonicalImported).source_id, source.id)
     assert.strictEqual((await fs.promises.stat(local)).nlink, 2)
 
     const status = await vault.status()
-    assert.ok(status.sources.some((item) => item.id === source.id && item.kind === 'external'))
-    assert.strictEqual(status.duplicates.find((item) => item.path === canonicalImported).source_id, source.id)
+    assert.strictEqual(status.sources.some((item) => item.kind === 'external'), false)
+    assert.strictEqual(status.duplicates.some((item) => item.path === canonicalImported), false)
     assert.strictEqual(status.last_scan.home_bytes_total, content.length, 'Pinokio metric excludes external files')
-    assert.strictEqual(status.last_scan.source_bytes[source.id], content.length, 'external totals remain attributable')
+    assert.strictEqual(status.last_scan.bytes_total, content.length)
   })
 
-  test('adding an external source creates one persistent import without starting a scan', async () => {
-    const { home, vault } = await makeEnv()
+  test('adding and removing an external source persists only its path and never creates a link', async () => {
+    const { home, vault, kernel } = await makeEnv()
     const external = await fs.promises.mkdtemp(path.resolve(os.tmpdir(), 'pinokio-added-external-'))
     homes.push(external)
     const content = crypto.randomBytes(4096)
@@ -312,27 +310,42 @@ describe('vault manual scan (phase 3)', () => {
     assert.strictEqual(added.source.kind, 'external')
     assert.strictEqual(added.source.parent_id, 'external')
     assert.strictEqual(vault.registry.scanIndex.size, 0, 'adding a source never scans it')
-    assert.strictEqual((await fs.promises.lstat(added.source.mount_path)).isSymbolicLink(), true)
-    assert.strictEqual(path.dirname(added.source.mount_path), path.resolve(home, 'vault', 'sources'))
+    assert.strictEqual(added.source.root, path.resolve(await fs.promises.realpath(external)))
+    assert.strictEqual(added.source.mount_path, undefined)
+    assert.deepStrictEqual(
+      JSON.parse(await fs.promises.readFile(vault.externalSourcesPath, 'utf8')).paths,
+      [added.source.root]
+    )
+    assert.strictEqual(fs.existsSync(path.resolve(home, 'vault', 'sources')), false)
     assert.deepStrictEqual(await fs.promises.readdir(path.resolve(home, 'api')), [],
-      'a scan-only external source never enters global app discovery')
+      'adding a Vault source never creates an api link')
 
     const repeated = await vault.addExternalSource(external)
     assert.strictEqual(repeated.created, false, 'the same physical folder is not imported twice')
     assert.strictEqual(vault.sources().filter((source) => source.kind === 'external').length, 1)
 
+    const fresh = new Vault(kernel)
+    await fresh.init()
+    assert.ok(fresh.sources().some((source) =>
+      source.kind === 'external' && source.root === added.source.root))
+
     await vault.sweeper.scan()
     const canonicalImported = path.resolve(await fs.promises.realpath(imported))
     assert.ok(vault.registry.scanIndex.has(canonicalImported))
 
-    await fs.promises.unlink(added.source.mount_path)
-    await vault.sweeper.scan()
+    const removed = await vault.perform('remove_source', { source_id: added.source.id })
+    assert.strictEqual(removed.removed, true)
+    assert.deepStrictEqual(
+      JSON.parse(await fs.promises.readFile(vault.externalSourcesPath, 'utf8')).paths,
+      []
+    )
     const status = await vault.status()
     assert.strictEqual(vault.registry.links.has(canonicalImported), false)
     assert.strictEqual(vault.registry.duplicates.has(canonicalImported), false)
     assert.strictEqual(vault.registry.scanIndex.has(canonicalImported), false)
     assert.strictEqual(status.sources.some((source) => source.kind === 'external'), false)
     assert.strictEqual(status.duplicates.some((item) => item.path === canonicalImported), false)
+    assert.deepStrictEqual(await fs.promises.readFile(imported), content)
   })
 
   test('nested external sources are walked once and attributed to the deepest source', async () => {
@@ -393,7 +406,7 @@ describe('vault manual scan (phase 3)', () => {
     const realStat = fs.promises.stat
     let externalStats = 0
     fs.promises.stat = async (target, options) => {
-      if (path.resolve(String(target)) === path.resolve(canonicalExternal) && ++externalStats === 2) {
+      if (path.resolve(String(target)) === path.resolve(canonicalExternal) && ++externalStats === 1) {
         const error = new Error('temporary external metadata failure')
         error.code = 'EIO'
         throw error
@@ -452,28 +465,26 @@ describe('vault manual scan (phase 3)', () => {
     assert.strictEqual(vault.registry.lastScan.files, 2)
   })
 
-  test('a failed source refresh rolls back only the import it just created', async () => {
+  test('a failed external-source settings write creates no configuration', async () => {
     const { vault } = await makeEnv()
     const external = await fs.promises.mkdtemp(path.resolve(os.tmpdir(), 'pinokio-add-rollback-'))
     homes.push(external)
-    const importRoot = vault.sourceRoot
-    const realReaddir = fs.promises.readdir
-    let importReads = 0
-    fs.promises.readdir = async (target, options) => {
-      if (path.resolve(target) === importRoot && ++importReads === 1) {
+    const realAtomicWrite = vault.registry.atomicWrite
+    vault.registry.atomicWrite = async (target, contents) => {
+      if (path.resolve(target) === vault.externalSourcesPath) {
         const error = new Error('transient read failure')
         error.code = 'EIO'
         throw error
       }
-      return realReaddir(target, options)
+      return realAtomicWrite.call(vault.registry, target, contents)
     }
     try {
       await assert.rejects(vault.addExternalSource(external), (error) => error.code === 'EIO')
     } finally {
-      fs.promises.readdir = realReaddir
+      vault.registry.atomicWrite = realAtomicWrite
     }
 
-    assert.deepStrictEqual(await fs.promises.readdir(importRoot), [])
+    assert.strictEqual(fs.existsSync(vault.externalSourcesPath), false)
     assert.strictEqual(vault.sources().some((source) => source.kind === 'external'), false)
   })
 

package/test/vault-ui.test.js

@@ -1334,6 +1334,8 @@ describe('vault dashboard backend (phase 4)', () => {
     assert.match(vaultPage, /id='btn-add-source'/)
     assert.match(vaultPage, /<script src="\/Socket\.js"><\/script>/)
     assert.match(vaultPage, /action:\s*"add_source"/)
+    assert.match(vaultPage, /action:\s*"remove_source"/)
+    assert.match(vaultPage, /remove_external_folder:\s*"Remove from Locations"/)
     assert.match(vaultPage, /identical_contents_at:\s*"Identical contents at"/)
     assert.doesNotMatch(vaultPage, /Matching locations/)
     assert.match(vaultPage, /scan_waiting:\s*"Waiting for scan results"/)
@@ -2592,7 +2594,7 @@ describe('vault dashboard backend (phase 4)', () => {
     assert.doesNotMatch(actionRoute, /convertPending|runExclusive/)
   })
 
-  test('a stale external source id cannot authorize files outside its current target', async (t) => {
+  test('a stale external source id cannot authorize files outside its current target', async () => {
     const { home, vault } = await makeEnv()
     const firstTarget = await fs.promises.mkdtemp(path.resolve(os.tmpdir(), 'pinokio-scope-old-'))
     const secondTarget = await fs.promises.mkdtemp(path.resolve(os.tmpdir(), 'pinokio-scope-new-'))
@@ -2601,16 +2603,8 @@ describe('vault dashboard backend (phase 4)', () => {
     const hash = sha256(content)
     const original = await writeFile(path.resolve(home, 'api', 'appA', 'model.bin'), content)
     const outside = await writeFile(path.resolve(firstTarget, 'model.bin'), content)
-    const mount = path.resolve(home, 'api', 'external-models')
-    try {
-      await fs.promises.symlink(firstTarget, mount, process.platform === 'win32' ? 'junction' : 'dir')
-    } catch (error) {
-      t.skip(`directory links unavailable: ${error.message}`)
-      return
-    }
-    await vault.refreshSources()
+    const externalSource = (await vault.addExternalSource(firstTarget)).source
     const originalSource = vault.sources().find((source) => source.kind === 'app' && source.app === 'appA')
-    const externalSource = vault.sources().find((source) => source.kind === 'external' && source.label === 'external-models')
     await vault.adopt(original, hash, { app: 'appA', source_id: originalSource.id })
     const outsideStat = await fs.promises.stat(outside)
     vault.registry.duplicates.set(outside, {
@@ -2618,12 +2612,12 @@ describe('vault dashboard backend (phase 4)', () => {
       dev: outsideStat.dev, ino: outsideStat.ino,
       mtime: outsideStat.mtimeMs, ctime: outsideStat.ctimeMs
     })
-    await fs.promises.unlink(mount)
-    await fs.promises.symlink(secondTarget, mount, process.platform === 'win32' ? 'junction' : 'dir')
+    await vault.writeExternalSourcePaths([secondTarget])
+    await vault.refreshSources()
 
     const result = await vault.perform('deduplicate', { scope_id: externalSource.id })
 
-    assert.strictEqual(result.converted, 0)
+    assert.match(result.error, /no longer available/i)
     assert.strictEqual((await fs.promises.stat(outside)).nlink, 1)
     assert.ok(vault.registry.duplicates.has(outside))
   })
@@ -2772,16 +2766,13 @@ describe('vault dashboard backend (phase 4)', () => {
     assert.strictEqual(vault.registry.links.has(pending), false)
   })
 
-  test('status separates app, Pinokio folder, and external source metadata', async (t) => {
+  test('status separates app, Pinokio folder, and external source metadata', async () => {
     const { home, vault } = await makeEnv()
-    const external = await fs.promises.mkdtemp(path.resolve(os.tmpdir(), 'pinokio-ui-external-'))
-    homes.push(external)
-    try {
-      await fs.promises.symlink(external, path.resolve(home, 'api', 'linked-models'), process.platform === 'win32' ? 'junction' : 'dir')
-    } catch (error) {
-      t.skip(`directory links unavailable: ${error.message}`)
-      return
-    }
+    const externalParent = await fs.promises.mkdtemp(path.resolve(os.tmpdir(), 'pinokio-ui-external-'))
+    const external = path.resolve(externalParent, 'linked-models')
+    homes.push(externalParent)
+    await fs.promises.mkdir(external)
+    await vault.addExternalSource(external)
     await fs.promises.mkdir(path.resolve(home, 'api', 'local-app'), { recursive: true })
     await fs.promises.mkdir(path.resolve(home, 'cache'), { recursive: true })
     await vault.refreshSources()
@@ -2789,6 +2780,8 @@ describe('vault dashboard backend (phase 4)', () => {
     const byKind = new Map(status.sources.map((source) => [source.id, source]))
     assert.ok([...byKind.values()].some((source) => source.kind === 'app' && source.label === 'local-app' && source.parent_id === 'apps'))
     assert.ok([...byKind.values()].some((source) => source.kind === 'folder' && source.label === 'cache' && source.parent_id === 'pinokio'))
-    assert.ok([...byKind.values()].some((source) => source.kind === 'external' && source.label === 'linked-models' && source.parent_id === 'external'))
+    assert.ok([...byKind.values()].some((source) =>
+      source.kind === 'external' && source.label === 'linked-models' &&
+      source.parent_id === 'external' && source.removable === true))
   })
 })