pinokiod 7.5.42 → 7.5.44

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "7.5.42",
+  "version": "7.5.44",
   "description": "",
   "main": "index.js",
   "scripts": {

package/server/index.js

@@ -85,6 +85,7 @@ const { createTaskWorkspaceLinkService } = require("./lib/task_workspace_links")
 const { createContentValidationService } = require("./lib/content_validation")
 const { buildSecureRouterDebugSnapshot, createSecureRouterDebugStore } = require("./lib/secure_router_debug")
 const logRedaction = require("./lib/log_redaction")
+const privacyFilterCache = require("./lib/privacy_filter_cache")
 const AppRegistryService = require("./lib/app_registry")
 const AppLogService = require("./lib/app_logs")
 const AppLogReportService = require("./lib/app_log_report")
@@ -6401,6 +6402,12 @@ class Server {
         }
       });
       this.app.use('/files', serve2, serveIndex(this.kernel.homedir, {icons: true, hidden: true, theme: this.theme }))
+      this.app.use('/pinokio/privacy-filter/models', express.static(privacyFilterCache.cacheRoot(this.kernel), {
+        fallthrough: true,
+        index: false,
+        immutable: true,
+        maxAge: '1y'
+      }))
     } else {
       this.app.set("views", [
         path.resolve(__dirname, "views")
@@ -8415,6 +8422,24 @@ class Server {
         entries
       })
     }))
+    this.app.get("/pinokio/privacy-filter/status", ex(async (req, res) => {
+      const payload = await privacyFilterCache.status(this.kernel, {
+        dtype: req.query.dtype
+      })
+      res.set("Cache-Control", "no-store")
+      res.json(payload)
+    }))
+    this.app.post("/pinokio/privacy-filter/ensure", ex(async (req, res) => {
+      if (!privacyFilterCache.isInstallRequestAllowed(req)) {
+        res.status(403).json({ error: "Privacy filter cache install requires a same-origin request" })
+        return
+      }
+      const payload = await privacyFilterCache.ensure(this.kernel, {
+        dtype: req.query.dtype
+      })
+      res.set("Cache-Control", "no-store")
+      res.json(payload)
+    }))
     this.app.get("/pinokio/logs/file", ex(logRedaction.createTopLevelLogFileHandler(this)))
     this.app.get("/api/logs/stream", ex(async (req, res) => {
       const workspace = typeof req.query.workspace === 'string' ? req.query.workspace.trim() : ''

package/server/lib/privacy_filter_cache.js

@@ -0,0 +1,286 @@
+const fs = require('fs')
+const path = require('path')
+const { pipeline } = require('stream/promises')
+const axios = require('axios')
+
+const PRIVACY_FILTER_MODEL_ID = 'openai/privacy-filter'
+const PRIVACY_FILTER_REVISION = '7ffa9a043d54d1be65afb281eddf0ffbe629385b'
+const PRIVACY_FILTER_LOCAL_MODEL_PATH = `/pinokio/privacy-filter/models/${PRIVACY_FILTER_REVISION}/`
+const PRIVACY_FILTER_SHARED_FILES = [
+  'config.json',
+  'tokenizer_config.json',
+  'tokenizer.json'
+]
+const PRIVACY_FILTER_DTYPE_FILES = {
+  q4f16: [
+    'onnx/model_q4f16.onnx',
+    'onnx/model_q4f16.onnx_data'
+  ],
+  q4: [
+    'onnx/model_q4.onnx',
+    'onnx/model_q4.onnx_data'
+  ],
+  q8: [
+    'onnx/model_quantized.onnx',
+    'onnx/model_quantized.onnx_data'
+  ]
+}
+const PRIVACY_FILTER_ASSET_SET = new Set([
+  ...PRIVACY_FILTER_SHARED_FILES,
+  ...Object.values(PRIVACY_FILTER_DTYPE_FILES).flat()
+])
+
+const ensurePromises = new Map()
+const installStates = new Map()
+
+function normalizeDtype(value) {
+  const dtype = String(value || '').trim().toLowerCase()
+  return Object.prototype.hasOwnProperty.call(PRIVACY_FILTER_DTYPE_FILES, dtype) ? dtype : 'q8'
+}
+
+function cacheRoot(kernel) {
+  const home = kernel && typeof kernel.homedir === 'string' ? kernel.homedir : ''
+  if (!home.trim()) {
+    throw new Error('Pinokio home directory is required for privacy filter cache.')
+  }
+  return path.resolve(home, 'cache', 'privacy-filter', 'models')
+}
+
+function modelRoot(kernel) {
+  return path.resolve(cacheRoot(kernel), PRIVACY_FILTER_REVISION, 'openai', 'privacy-filter')
+}
+
+function assetFilesForDtype(dtype) {
+  const normalized = normalizeDtype(dtype)
+  return [
+    ...PRIVACY_FILTER_SHARED_FILES,
+    ...PRIVACY_FILTER_DTYPE_FILES[normalized]
+  ]
+}
+
+function assertKnownAsset(relativePath) {
+  const normalized = String(relativePath || '').replace(/\\/g, '/')
+  if (!PRIVACY_FILTER_ASSET_SET.has(normalized)) {
+    throw new Error(`Unknown privacy filter asset: ${relativePath}`)
+  }
+  return normalized
+}
+
+function isInstallRequestAllowed(req) {
+  const fetchSite = String(req && typeof req.get === 'function' ? req.get('Sec-Fetch-Site') || '' : '').trim().toLowerCase()
+  if (fetchSite && fetchSite !== 'same-origin' && fetchSite !== 'none') {
+    return false
+  }
+
+  const origin = String(req && typeof req.get === 'function' ? req.get('Origin') || '' : '').trim()
+  if (!origin) {
+    return true
+  }
+
+  const host = String(req && typeof req.get === 'function' ? req.get('Host') || '' : '').trim().toLowerCase()
+  if (!host) {
+    return false
+  }
+
+  try {
+    const parsed = new URL(origin)
+    return (parsed.protocol === 'http:' || parsed.protocol === 'https:') && parsed.host.toLowerCase() === host
+  } catch (_) {
+    return false
+  }
+}
+
+function assetPath(kernel, relativePath) {
+  const normalized = assertKnownAsset(relativePath)
+  return path.resolve(modelRoot(kernel), ...normalized.split('/'))
+}
+
+function ensureKey(kernel, dtype) {
+  return `${modelRoot(kernel)}:${normalizeDtype(dtype)}`
+}
+
+function remoteAssetUrl(relativePath) {
+  const normalized = assertKnownAsset(relativePath)
+  return `https://huggingface.co/${PRIVACY_FILTER_MODEL_ID}/resolve/${PRIVACY_FILTER_REVISION}/${normalized}`
+}
+
+function copyInstallState(install) {
+  return install ? { ...install } : null
+}
+
+async function cleanupStaleAssetTemps(target) {
+  const dir = path.dirname(target)
+  const base = path.basename(target)
+  const currentProcessPrefix = `${base}.${process.pid}.`
+  let entries
+  try {
+    entries = await fs.promises.readdir(dir, { withFileTypes: true })
+  } catch (error) {
+    if (error && error.code === 'ENOENT') {
+      return
+    }
+    throw error
+  }
+  await Promise.all(entries
+    .filter((entry) => {
+      return entry.isFile() &&
+        entry.name.startsWith(`${base}.`) &&
+        entry.name.endsWith('.tmp') &&
+        !entry.name.startsWith(currentProcessPrefix)
+    })
+    .map((entry) => fs.promises.rm(path.resolve(dir, entry.name), { force: true })))
+}
+
+async function fileStatus(kernel, relativePath) {
+  const target = assetPath(kernel, relativePath)
+  try {
+    const stats = await fs.promises.stat(target)
+    return {
+      path: relativePath,
+      exists: stats.isFile() && stats.size > 0,
+      size: stats.isFile() ? stats.size : 0
+    }
+  } catch (_) {
+    return {
+      path: relativePath,
+      exists: false,
+      size: 0
+    }
+  }
+}
+
+async function status(kernel, options = {}) {
+  const dtype = normalizeDtype(options.dtype)
+  const install = copyInstallState(installStates.get(ensureKey(kernel, dtype)))
+  const files = await Promise.all(assetFilesForDtype(dtype).map((file) => fileStatus(kernel, file)))
+  const missing = files.filter((file) => !file.exists).map((file) => file.path)
+  return {
+    model_id: PRIVACY_FILTER_MODEL_ID,
+    revision: PRIVACY_FILTER_REVISION,
+    dtype,
+    local_model_path: PRIVACY_FILTER_LOCAL_MODEL_PATH,
+    ready: missing.length === 0,
+    files,
+    missing,
+    missing_count: missing.length,
+    installing: Boolean(install),
+    install
+  }
+}
+
+async function downloadAsset(kernel, relativePath, onProgress) {
+  const target = assetPath(kernel, relativePath)
+  const existing = await fileStatus(kernel, relativePath)
+  if (existing.exists) {
+    return { path: relativePath, status: 'cached', size: existing.size }
+  }
+
+  await fs.promises.mkdir(path.dirname(target), { recursive: true })
+  await cleanupStaleAssetTemps(target)
+  const tmp = `${target}.${process.pid}.${Date.now()}.tmp`
+  try {
+    const response = await axios.get(remoteAssetUrl(relativePath), {
+      responseType: 'stream',
+      maxRedirects: 5,
+      timeout: 0,
+      maxContentLength: Infinity,
+      maxBodyLength: Infinity,
+      validateStatus: (statusCode) => statusCode >= 200 && statusCode < 300
+    })
+    let loaded = 0
+    const total = Number(response.headers && response.headers['content-length']) || 0
+    const encoded = Boolean(response.headers && response.headers['content-encoding'])
+    if (typeof onProgress === 'function') {
+      onProgress({ loaded, total })
+    }
+    response.data.on('data', (chunk) => {
+      loaded += chunk && chunk.length ? chunk.length : 0
+      if (typeof onProgress === 'function') {
+        onProgress({ loaded, total })
+      }
+    })
+    await pipeline(response.data, fs.createWriteStream(tmp))
+    const stats = await fs.promises.stat(tmp)
+    if (!stats.isFile() || stats.size <= 0) {
+      throw new Error(`Downloaded empty privacy filter asset: ${relativePath}`)
+    }
+    if (total > 0 && !encoded && stats.size !== total) {
+      throw new Error(`Incomplete privacy filter asset: ${relativePath} (${stats.size}/${total} bytes)`)
+    }
+    await fs.promises.rename(tmp, target)
+    return { path: relativePath, status: 'downloaded', size: stats.size }
+  } catch (error) {
+    await fs.promises.rm(tmp, { force: true }).catch(() => {})
+    throw error
+  }
+}
+
+async function ensure(kernel, options = {}) {
+  const dtype = normalizeDtype(options.dtype)
+  const key = ensureKey(kernel, dtype)
+  if (ensurePromises.has(key)) {
+    return ensurePromises.get(key)
+  }
+  const promise = (async () => {
+    const files = assetFilesForDtype(dtype)
+    const install = {
+      dtype,
+      total_files: files.length,
+      completed_files: 0,
+      cached_files: 0,
+      downloaded_files: 0,
+      current_file: '',
+      current_loaded: 0,
+      current_total: 0,
+      started_at: new Date().toISOString(),
+      updated_at: new Date().toISOString()
+    }
+    installStates.set(key, install)
+    const results = []
+    for (let index = 0; index < files.length; index += 1) {
+      const file = files[index]
+      install.current_file = file
+      install.current_index = index + 1
+      install.current_loaded = 0
+      install.current_total = 0
+      install.updated_at = new Date().toISOString()
+      const result = await downloadAsset(kernel, file, ({ loaded, total }) => {
+        install.current_loaded = loaded
+        install.current_total = total
+        install.updated_at = new Date().toISOString()
+      })
+      results.push(result)
+      install.completed_files += 1
+      if (result.status === 'cached') {
+        install.cached_files += 1
+      } else if (result.status === 'downloaded') {
+        install.downloaded_files += 1
+      }
+      install.updated_at = new Date().toISOString()
+    }
+    const current = await status(kernel, { dtype })
+    return {
+      ...current,
+      results,
+      downloaded: results.filter((result) => result.status === 'downloaded').length,
+      cached: results.filter((result) => result.status === 'cached').length
+    }
+  })().finally(() => {
+    ensurePromises.delete(key)
+    installStates.delete(key)
+  })
+  ensurePromises.set(key, promise)
+  return promise
+}
+
+module.exports = {
+  PRIVACY_FILTER_MODEL_ID,
+  PRIVACY_FILTER_REVISION,
+  PRIVACY_FILTER_LOCAL_MODEL_PATH,
+  normalizeDtype,
+  cacheRoot,
+  modelRoot,
+  isInstallRequestAllowed,
+  status,
+  ensure
+}

package/server/public/logs-top-redaction.js

@@ -387,10 +387,30 @@
       const name = file && file.name ? file.name : 'file'
       if (progress.type === 'runtime') {
         this.setStatus(`Loading privacy filter locally (${progress.device}/${progress.dtype}) for ${name}.`)
-      } else if (progress.type === 'download') {
+      } else if (progress.type === 'cache-check') {
+        this.setStatus(`Checking local privacy filter cache for ${name}.`)
+      } else if (progress.type === 'cache-install') {
+        this.setStatus('Installing privacy filter locally. This can take a few minutes on first run.')
+      } else if (progress.type === 'cache-install-progress') {
+        const totalFiles = Number(progress.totalFiles) || 0
+        const fileIndex = Number(progress.fileIndex) || 0
+        const fileCount = totalFiles > 0 && fileIndex > 0 ? ` (${Math.min(fileIndex, totalFiles)} / ${totalFiles} files)` : ''
+        if (progress.total) {
+          this.setStatus(`Installing privacy filter locally${fileCount}: ${humanBytes(progress.loaded)} / ${humanBytes(progress.total)}.`)
+        } else {
+          this.setStatus(`Installing privacy filter locally${fileCount}.`)
+        }
+      } else if (progress.type === 'cache-ready') {
+        const downloaded = Number(progress.downloaded) || 0
+        this.setStatus(downloaded > 0 ? 'Privacy filter installed locally.' : `Privacy filter loaded from local cache for ${name}.`)
+      } else if (progress.type === 'cache-fallback') {
+        this.setStatus(progress.message || `Local privacy filter cache unavailable. Loading remote fallback for ${name}.`)
+      } else if (progress.type === 'asset-loading') {
+        this.setStatus('Loading privacy filter asset.')
+      } else if (progress.type === 'asset-progress') {
         this.setStatus(progress.total
-          ? `Downloading privacy filter: ${humanBytes(progress.loaded)} / ${humanBytes(progress.total)}.`
-          : 'Downloading privacy filter.')
+          ? `Loading privacy filter: ${humanBytes(progress.loaded)} / ${humanBytes(progress.total)}.`
+          : 'Loading privacy filter.')
       } else if (progress.type === 'chunk') {
         const total = Number(progress.total) || 0
         const done = Number(progress.done) || 0
@@ -472,6 +492,9 @@
         this.setStatus(error && error.message ? error.message : 'Privacy filtering failed.', true)
       } finally {
         this.setBusy(false)
+        if (this.files.length && this.filesEl && this.filesEl.children.length) {
+          this.renderFileList()
+        }
         this.updateCount()
       }
     }

package/server/public/logs.js

@@ -207,7 +207,15 @@
       return this.worker
     }
     handleMessage(message) {
-      if (message.type === 'download') {
+      if (!message.id && (
+        message.type === 'asset-loading' ||
+        message.type === 'asset-progress' ||
+        message.type === 'cache-check' ||
+        message.type === 'cache-install' ||
+        message.type === 'cache-install-progress' ||
+        message.type === 'cache-ready' ||
+        message.type === 'cache-fallback'
+      )) {
         for (const pending of this.pending.values()) {
           pending.onProgress(message)
         }
@@ -221,6 +229,16 @@
         pending.onProgress(message)
       } else if (message.type === 'fallback') {
         pending.onProgress(message)
+      } else if (
+        message.type === 'asset-loading' ||
+        message.type === 'asset-progress' ||
+        message.type === 'cache-check' ||
+        message.type === 'cache-install' ||
+        message.type === 'cache-install-progress' ||
+        message.type === 'cache-ready' ||
+        message.type === 'cache-fallback'
+      ) {
+        pending.onProgress(message)
       } else if (message.type === 'result') {
         this.pending.delete(message.id)
         pending.resolve(message)
@@ -2109,15 +2127,37 @@
         return
       }
       if (progress.type === 'runtime') {
-        this.setStatus(`Loading OpenAI Privacy Filter locally (${progress.device}/${progress.dtype}). First run downloads and caches the model.`)
+        this.setStatus(`Preparing OpenAI Privacy Filter locally (${progress.device}/${progress.dtype}).`)
+      } else if (progress.type === 'cache-check') {
+        this.setStatus('Checking local privacy filter cache.')
+      } else if (progress.type === 'cache-install') {
+        this.setStatus('Installing privacy filter locally. This can take a few minutes on first run.')
+      } else if (progress.type === 'cache-install-progress') {
+        const fileLabel = progress.file ? ` ${progress.file}` : ''
+        const totalFiles = Number(progress.totalFiles) || 0
+        const fileIndex = Number(progress.fileIndex) || 0
+        const fileCount = totalFiles > 0 && fileIndex > 0 ? ` (${Math.min(fileIndex, totalFiles)} / ${totalFiles} files)` : ''
+        if (progress.total) {
+          this.setStatus(`Installing privacy filter locally${fileCount}${fileLabel}: ${humanBytes(progress.loaded)} / ${humanBytes(progress.total)}.`)
+        } else {
+          this.setStatus(`Installing privacy filter locally${fileCount}${fileLabel}.`)
+        }
+      } else if (progress.type === 'cache-ready') {
+        const downloaded = Number(progress.downloaded) || 0
+        this.setStatus(downloaded > 0 ? 'Privacy filter installed locally.' : 'Privacy filter loaded from local cache.')
+      } else if (progress.type === 'cache-fallback') {
+        this.setStatus(progress.message || 'Local privacy filter cache unavailable. Loading remote fallback.')
       } else if (progress.type === 'fallback') {
         this.setStatus(progress.message || `Retrying privacy filtering locally with ${progress.device}/${progress.dtype}.`)
-      } else if (progress.type === 'download') {
+      } else if (progress.type === 'asset-loading') {
+        const fileLabel = progress.file ? ` ${progress.file}` : ''
+        this.setStatus(`Loading privacy filter asset${fileLabel}.`)
+      } else if (progress.type === 'asset-progress') {
         const fileLabel = progress.file ? ` ${progress.file}` : ''
         if (progress.total) {
-          this.setStatus(`Downloading privacy filter${fileLabel}: ${humanBytes(progress.loaded)} / ${humanBytes(progress.total)}. Cached for future reports.`)
+          this.setStatus(`Loading privacy filter${fileLabel}: ${humanBytes(progress.loaded)} / ${humanBytes(progress.total)}.`)
         } else {
-          this.setStatus(`Downloading privacy filter${fileLabel}. Cached for future reports.`)
+          this.setStatus(`Loading privacy filter${fileLabel}.`)
         }
       } else if (progress.type === 'chunk') {
         const total = Number(progress.total) || 0

package/server/public/privacy_filter_worker.js

@@ -1,5 +1,7 @@
 const TRANSFORMERS_URL = 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0/+esm'
 const MODEL_ID = 'openai/privacy-filter'
+const MODEL_REVISION = '7ffa9a043d54d1be65afb281eddf0ffbe629385b'
+const LOCAL_MODEL_PATH = `/pinokio/privacy-filter/models/${MODEL_REVISION}/`
 const MAX_CHUNK_CHARS = 1800
 const CHUNK_OVERLAP_CHARS = 120
 const URL_COMPONENT_LABELS = new Set([
@@ -15,11 +17,52 @@ let pipelinePromise = null
 let pipelineKey = ''
 let activeDevice = 'webgpu'
 let activeDtype = 'q4f16'
+const localCachePromises = new Map()
 
 const post = (message) => {
   self.postMessage(message)
 }
 
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+
+const cacheUrl = (pathname, params) => {
+  const url = new URL(pathname, self.location.origin)
+  Object.entries(params || {}).forEach(([key, value]) => {
+    if (value != null && value !== '') {
+      url.searchParams.set(key, value)
+    }
+  })
+  return url.toString()
+}
+
+const fetchCacheStatus = async (dtype) => {
+  const response = await fetch(cacheUrl('/pinokio/privacy-filter/status', { dtype }), {
+    headers: { 'Accept': 'application/json' }
+  })
+  return response.ok ? response.json() : null
+}
+
+const postCacheInstallProgress = ({ id, device, dtype, status }) => {
+  const install = status && status.install
+  if (!install) {
+    return
+  }
+  post({
+    type: 'cache-install-progress',
+    id,
+    device,
+    dtype,
+    file: install.current_file || '',
+    fileIndex: Number(install.current_index) || 0,
+    doneFiles: Number(install.completed_files) || 0,
+    totalFiles: Number(install.total_files) || 0,
+    loaded: Number(install.current_loaded) || 0,
+    total: Number(install.current_total) || 0,
+    cachedFiles: Number(install.cached_files) || 0,
+    downloadedFiles: Number(install.downloaded_files) || 0
+  })
+}
+
 const normalizeLabel = (value) => {
   const label = String(value || 'private')
     .replace(/^[bieos]-/i, '')
@@ -272,8 +315,9 @@ const loadTransformers = async () => {
   if (!transformersPromise) {
     transformersPromise = import(TRANSFORMERS_URL).then((mod) => {
       if (mod.env) {
-        mod.env.allowLocalModels = false
+        mod.env.allowLocalModels = true
         mod.env.allowRemoteModels = true
+        mod.env.localModelPath = LOCAL_MODEL_PATH
         mod.env.useBrowserCache = true
         mod.env.useWasmCache = true
         mod.env.cacheKey = 'pinokio-privacy-filter-cache'
@@ -284,7 +328,93 @@ const loadTransformers = async () => {
   return transformersPromise
 }
 
-const getPipeline = async (device, dtype) => {
+const ensureLocalCache = async ({ id, device, dtype }) => {
+  const key = dtype || activeDtype
+  if (localCachePromises.has(key)) {
+    return localCachePromises.get(key)
+  }
+  const promise = (async () => {
+    post({ type: 'cache-check', id, device, dtype })
+    let status = null
+    try {
+      status = await fetchCacheStatus(dtype)
+    } catch (_) {}
+    if (status && status.ready) {
+      post({ type: 'cache-ready', id, device, dtype, downloaded: 0, cached: Array.isArray(status.files) ? status.files.length : 0 })
+      return true
+    }
+
+    post({
+      type: 'cache-install',
+      id,
+      device,
+      dtype,
+      missing: status && Array.isArray(status.missing) ? status.missing : [],
+      missingCount: status && Number.isFinite(status.missing_count) ? status.missing_count : 0
+    })
+    const ensurePromise = fetch(cacheUrl('/pinokio/privacy-filter/ensure', { dtype }), {
+      method: 'POST',
+      headers: { 'Accept': 'application/json' }
+    }).then(async (ensureResponse) => {
+      if (!ensureResponse.ok) {
+        const message = await ensureResponse.text().catch(() => '')
+        throw new Error(message || `HTTP ${ensureResponse.status}`)
+      }
+      return ensureResponse.json()
+    })
+    const settledEnsure = ensurePromise.then(
+      (value) => ({ done: true, value }),
+      (error) => ({ done: true, error })
+    )
+
+    let payload = null
+    while (!payload) {
+      const result = await Promise.race([
+        settledEnsure,
+        delay(1000).then(() => ({ done: false }))
+      ])
+      if (result.done) {
+        if (result.error) {
+          throw result.error
+        }
+        payload = result.value
+        break
+      }
+      try {
+        status = await fetchCacheStatus(dtype)
+        if (status && status.installing) {
+          postCacheInstallProgress({ id, device, dtype, status })
+        }
+      } catch (_) {}
+    }
+    if (!payload || payload.ready !== true) {
+      throw new Error('Privacy filter cache is incomplete.')
+    }
+    post({
+      type: 'cache-ready',
+      id,
+      device,
+      dtype,
+      downloaded: Number(payload.downloaded) || 0,
+      cached: Number(payload.cached) || 0
+    })
+    return true
+  })().catch((error) => {
+    localCachePromises.delete(key)
+    post({
+      type: 'cache-fallback',
+      id,
+      device,
+      dtype,
+      message: `Local privacy filter cache unavailable. Loading from remote cache fallback. ${error && error.message ? error.message : ''}`.trim()
+    })
+    return false
+  })
+  localCachePromises.set(key, promise)
+  return promise
+}
+
+const getPipeline = async (id, device, dtype) => {
   const nextDevice = device || activeDevice
   const nextDtype = dtype || activeDtype
   const nextKey = `${nextDevice}:${nextDtype}`
@@ -298,14 +428,24 @@ const getPipeline = async (device, dtype) => {
     return pipeline('token-classification', MODEL_ID, {
       device: activeDevice,
       dtype: activeDtype,
+      revision: MODEL_REVISION,
       progress_callback: (progress) => {
-        if (progress && progress.status === 'progress') {
+        if (!progress || !progress.status) {
+          return
+        }
+        if (progress.status === 'download') {
+          post({
+            type: 'asset-loading',
+            file: progress.file || ''
+          })
+        } else if (progress.status === 'progress' || progress.status === 'progress_total') {
           post({
-            type: 'download',
+            type: 'asset-progress',
             file: progress.file || '',
             loaded: progress.loaded || 0,
             total: progress.total || 0,
-            progress: progress.progress || 0
+            progress: progress.progress || 0,
+            aggregate: progress.status === 'progress_total'
           })
         }
       }
@@ -340,7 +480,8 @@ const filterText = async ({ id, text, device, dtype }) => {
   let classifier = null
   let entities = []
   try {
-    classifier = await getPipeline(requestedDevice, requestedDtype)
+    await ensureLocalCache({ id, device: requestedDevice, dtype: requestedDtype })
+    classifier = await getPipeline(id, requestedDevice, requestedDtype)
     entities = await classifyChunks({ classifier, chunks, id })
   } catch (error) {
     if (requestedDevice === 'wasm') {
@@ -357,7 +498,8 @@ const filterText = async ({ id, text, device, dtype }) => {
       dtype: requestedDtype,
       message: 'WebGPU privacy filtering failed. Retrying locally with WASM.'
     })
-    classifier = await getPipeline(requestedDevice, requestedDtype)
+    await ensureLocalCache({ id, device: requestedDevice, dtype: requestedDtype })
+    classifier = await getPipeline(id, requestedDevice, requestedDtype)
     entities = await classifyChunks({ classifier, chunks, id })
   }
   const merged = mergeEntities(prepareMaskEntities(reportText, entities))

package/test/logs-ask-ai.test.js

@@ -255,6 +255,19 @@ async function createRawLogsDom(options = {}) {
           if (tokenMatch) {
             addItem("private_key", tokenMatch[0])
           }
+          setTimeout(() => {
+            const listener = this.listeners.get("message")
+            if (listener) {
+              for (const progress of (options.workerProgress || [])) {
+                listener({
+                  data: {
+                    id: message.id,
+                    ...progress
+                  }
+                })
+              }
+            }
+          }, 0)
           setTimeout(() => {
             const listener = this.listeners.get("message")
             if (listener) {
@@ -268,7 +281,7 @@ async function createRawLogsDom(options = {}) {
                 }
               })
             }
-          }, 0)
+          }, options.workerResultDelayMs || 0)
         }
       }
       window.fetch = async (url, options = {}) => {
@@ -476,6 +489,10 @@ test("raw logs redaction sends reviewed top-level overrides when generating zip"
       document.getElementById("logs-top-redaction-count").textContent.includes("masked")
   }, "top-level redaction review")
 
+  await waitFor(() => document.getElementById("logs-top-redaction-status").textContent.includes("Report redacted"), "top-level redaction complete")
+  const fileModes = Array.from(document.querySelectorAll(".logs-top-redaction-mode"))
+  assert.equal(fileModes.length > 0, true)
+  assert.equal(fileModes.every((select) => select.disabled === false), true)
   assert.match(document.getElementById("logs-top-redaction-list").textContent, /private_key/)
 
   document.getElementById("logs-redaction-collapse").click()
@@ -497,6 +514,48 @@ test("raw logs redaction sends reviewed top-level overrides when generating zip"
   assert.equal(systemOverride.text.includes("sk-proj-123456789012345678901234"), false)
 })
 
+test("raw logs redaction describes model asset progress as loading", async () => {
+  const { dom } = await createRawLogsDom({
+    workerResultDelayMs: 50,
+    workerProgress: [{
+      id: null,
+      type: "asset-progress",
+      loaded: 25,
+      total: 100
+    }]
+  })
+  const { document } = dom.window
+
+  document.getElementById("logs-redact-top-level").click()
+
+  await waitFor(() => /Loading privacy filter/.test(document.getElementById("logs-top-redaction-status").textContent), "privacy filter loading status")
+  assert.equal(/Downloading privacy filter/.test(document.getElementById("logs-top-redaction-status").textContent), false)
+})
+
+test("raw logs redaction shows privacy filter install progress", async () => {
+  const { dom } = await createRawLogsDom({
+    workerResultDelayMs: 50,
+    workerProgress: [{
+      id: null,
+      type: "cache-install-progress",
+      fileIndex: 5,
+      totalFiles: 5,
+      loaded: 512,
+      total: 1024
+    }]
+  })
+  const { document } = dom.window
+
+  document.getElementById("logs-redact-top-level").click()
+
+  await waitFor(() => {
+    const status = document.getElementById("logs-top-redaction-status").textContent
+    return /Installing privacy filter locally/.test(status) &&
+      /5 \/ 5 files/.test(status) &&
+      /512 B \/ 1\.0 KB/.test(status)
+  }, "privacy filter install progress status")
+})
+
 test("raw log report drawer opens with file choices before redaction", async () => {
   const { dom, archiveRequests } = await createRawLogsDom()
   const { document } = dom.window

package/test/privacy-filter-cache.test.js

@@ -0,0 +1,242 @@
+const assert = require('node:assert/strict')
+const fs = require('node:fs/promises')
+const os = require('node:os')
+const path = require('node:path')
+const { PassThrough, Readable } = require('node:stream')
+const test = require('node:test')
+
+const cacheModulePath = path.resolve(__dirname, '..', 'server', 'lib', 'privacy_filter_cache.js')
+const axiosModulePath = require.resolve('axios')
+
+function loadCacheWithAxiosMock(mockAxios) {
+  const originalAxios = require.cache[axiosModulePath]
+  delete require.cache[cacheModulePath]
+  require.cache[axiosModulePath] = {
+    id: axiosModulePath,
+    filename: axiosModulePath,
+    loaded: true,
+    exports: mockAxios
+  }
+  const mod = require(cacheModulePath)
+  return {
+    mod,
+    restore() {
+      delete require.cache[cacheModulePath]
+      if (originalAxios) {
+        require.cache[axiosModulePath] = originalAxios
+      } else {
+        delete require.cache[axiosModulePath]
+      }
+    }
+  }
+}
+
+function fakeRequest(headers) {
+  const normalized = {}
+  for (const [key, value] of Object.entries(headers || {})) {
+    normalized[key.toLowerCase()] = value
+  }
+  return {
+    get(name) {
+      return normalized[String(name).toLowerCase()]
+    }
+  }
+}
+
+async function waitForValue(fn, label) {
+  const start = Date.now()
+  while (Date.now() - start < 1000) {
+    const value = await fn()
+    if (value) {
+      return value
+    }
+    await new Promise((resolve) => setTimeout(resolve, 10))
+  }
+  assert.fail(`Timed out waiting for ${label}`)
+}
+
+test('privacy filter cache installs requested dtype files under local model path', async () => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), 'pinokio-privacy-filter-cache-'))
+  const requests = []
+  const { mod, restore } = loadCacheWithAxiosMock({
+    get: async (url) => {
+      requests.push(url)
+      return {
+        data: Readable.from([Buffer.from(`asset:${url}`)])
+      }
+    }
+  })
+
+  try {
+    const kernel = { homedir: root }
+    const staleRoot = path.resolve(root, 'cache', 'privacy-filter', 'models', 'openai', 'privacy-filter')
+    await fs.mkdir(staleRoot, { recursive: true })
+    await fs.writeFile(path.resolve(staleRoot, 'config.json'), 'stale')
+
+    const before = await mod.status(kernel, { dtype: 'q4f16' })
+    assert.equal(before.ready, false)
+    assert.deepEqual(before.missing, [
+      'config.json',
+      'tokenizer_config.json',
+      'tokenizer.json',
+      'onnx/model_q4f16.onnx',
+      'onnx/model_q4f16.onnx_data'
+    ])
+
+    const ensured = await mod.ensure(kernel, { dtype: 'q4f16' })
+    assert.equal(ensured.ready, true)
+    assert.equal(ensured.downloaded, 5)
+    assert.equal(ensured.local_model_path, `/pinokio/privacy-filter/models/${mod.PRIVACY_FILTER_REVISION}/`)
+    assert.equal(requests.length, 5)
+    assert.ok(requests.every((url) => url.includes(mod.PRIVACY_FILTER_REVISION)))
+
+    const modelRoot = mod.modelRoot(kernel)
+    assert.equal(modelRoot.includes(mod.PRIVACY_FILTER_REVISION), true)
+    const config = await fs.readFile(path.resolve(modelRoot, 'config.json'), 'utf8')
+    assert.match(config, /asset:https:\/\/huggingface\.co\/openai\/privacy-filter\/resolve\//)
+
+    const after = await mod.ensure(kernel, { dtype: 'q4f16' })
+    assert.equal(after.ready, true)
+    assert.equal(after.downloaded, 0)
+    assert.equal(after.cached, 5)
+    assert.equal(requests.length, 5)
+  } finally {
+    restore()
+    await fs.rm(root, { recursive: true, force: true })
+  }
+})
+
+test('privacy filter cache install endpoint accepts only same-origin browser writes', async () => {
+  const { mod, restore } = loadCacheWithAxiosMock({ get: async () => ({ data: Readable.from(['']) }) })
+  try {
+    assert.equal(mod.isInstallRequestAllowed(fakeRequest({})), true)
+    assert.equal(mod.isInstallRequestAllowed(fakeRequest({
+      Host: 'localhost:42000',
+      Origin: 'http://localhost:42000',
+      'Sec-Fetch-Site': 'same-origin'
+    })), true)
+    assert.equal(mod.isInstallRequestAllowed(fakeRequest({
+      Host: 'localhost:42000',
+      Origin: 'http://evil.example',
+      'Sec-Fetch-Site': 'cross-site'
+    })), false)
+    assert.equal(mod.isInstallRequestAllowed(fakeRequest({
+      Host: 'localhost:42000',
+      Origin: 'http://localhost:5173',
+      'Sec-Fetch-Site': 'same-site'
+    })), false)
+    assert.equal(mod.isInstallRequestAllowed(fakeRequest({
+      Host: 'localhost:42000',
+      Origin: 'not a url',
+      'Sec-Fetch-Site': 'same-origin'
+    })), false)
+  } finally {
+    restore()
+  }
+})
+
+test('privacy filter cache requires an explicit Pinokio home', async () => {
+  const { mod, restore } = loadCacheWithAxiosMock({
+    get: async () => {
+      throw new Error('download should not start')
+    }
+  })
+  try {
+    assert.throws(() => mod.cacheRoot({}), /Pinokio home directory is required/)
+    await assert.rejects(() => mod.status({}, { dtype: 'q8' }), /Pinokio home directory is required/)
+    await assert.rejects(() => mod.ensure({}, { dtype: 'q8' }), /Pinokio home directory is required/)
+  } finally {
+    restore()
+  }
+})
+
+test('privacy filter cache removes stale temp files before retrying an asset', async () => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), 'pinokio-privacy-filter-cache-temp-'))
+  const { mod, restore } = loadCacheWithAxiosMock({
+    get: async (url) => ({
+      data: Readable.from([Buffer.from(`asset:${url}`)])
+    })
+  })
+
+  try {
+    const kernel = { homedir: root }
+    const target = path.resolve(mod.modelRoot(kernel), 'config.json')
+    const staleTemp = `${target}.999999.1.tmp`
+    await fs.mkdir(path.dirname(target), { recursive: true })
+    await fs.writeFile(staleTemp, 'partial')
+
+    await mod.ensure(kernel, { dtype: 'q8' })
+
+    await assert.rejects(() => fs.stat(staleTemp), /ENOENT/)
+    const installed = await fs.readFile(target, 'utf8')
+    assert.match(installed, /asset:https:\/\/huggingface\.co\/openai\/privacy-filter\/resolve\//)
+  } finally {
+    restore()
+    await fs.rm(root, { recursive: true, force: true })
+  }
+})
+
+test('privacy filter cache rejects truncated downloads when content length is available', async () => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), 'pinokio-privacy-filter-cache-truncated-'))
+  const { mod, restore } = loadCacheWithAxiosMock({
+    get: async () => ({
+      headers: {
+        'content-length': '8'
+      },
+      data: Readable.from([Buffer.from('short')])
+    })
+  })
+
+  try {
+    const kernel = { homedir: root }
+    const target = path.resolve(mod.modelRoot(kernel), 'config.json')
+    await assert.rejects(
+      () => mod.ensure(kernel, { dtype: 'q8' }),
+      /Incomplete privacy filter asset: config\.json/
+    )
+    await assert.rejects(() => fs.stat(target), /ENOENT/)
+    const dir = path.dirname(target)
+    const entries = await fs.readdir(dir)
+    assert.equal(entries.some((entry) => entry.endsWith('.tmp')), false)
+  } finally {
+    restore()
+    await fs.rm(root, { recursive: true, force: true })
+  }
+})
+
+test('privacy filter cache exposes in-progress install status', async () => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), 'pinokio-privacy-filter-cache-progress-'))
+  const { mod, restore } = loadCacheWithAxiosMock({
+    get: async () => {
+      const stream = new PassThrough()
+      setTimeout(() => stream.write(Buffer.alloc(512)), 20)
+      setTimeout(() => stream.end(Buffer.alloc(512)), 80)
+      return {
+        headers: {
+          'content-length': '1024'
+        },
+        data: stream
+      }
+    }
+  })
+
+  try {
+    const kernel = { homedir: root }
+    const ensurePromise = mod.ensure(kernel, { dtype: 'q8' })
+    const installing = await waitForValue(async () => {
+      const current = await mod.status(kernel, { dtype: 'q8' })
+      return current.installing && current.install && current.install.current_file && current.install.current_total === 1024 ? current : null
+    }, 'privacy filter install progress')
+
+    assert.equal(installing.install.total_files, 5)
+    assert.equal(installing.install.current_file, 'config.json')
+    assert.equal(installing.install.current_total, 1024)
+
+    const ensured = await ensurePromise
+    assert.equal(ensured.ready, true)
+    assert.equal(ensured.downloaded, 5)
+  } finally {
+    restore()
+    await fs.rm(root, { recursive: true, force: true })
+  }
+})