termux-vision 1.0.0

package/lib/cache.js

@@ -0,0 +1,300 @@
+/**
+ * Model Cache Management and Artifact Discovery for Node.js.
+ * Open-Source under Apache License 2.0.
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const https = require('https');
+const http = require('http');
+const crypto = require('crypto');
+
+const {
+  ModelNotFoundError,
+  NoInstalledModelsError,
+  ModelDownloadError
+} = require('./errors');
+
+const CATALOG = {
+  'smolvlm-500m-q4': {
+    modelId: 'smolvlm-500m-q4',
+    adapter: 'smolvlm',
+    tier: 'M',
+    estimatedMemoryMb: 750,
+    contextLimit: 1024,
+    preferredResolution: 384,
+    artifacts: [
+      {
+        role: 'language_model',
+        filename: 'smolvlm-500m-instruct-q4_k_m.gguf',
+        sizeBytes: 350000000,
+        downloadUrl: 'https://huggingface.co/HuggingFaceTB/SmolVLM-500M-Instruct-GGUF/resolve/main/smolvlm-500m-instruct-q4_k_m.gguf'
+      },
+      {
+        role: 'vision_projector',
+        filename: 'mmproj-smolvlm-500m-instruct-f16.gguf',
+        sizeBytes: 200000000,
+        downloadUrl: 'https://huggingface.co/HuggingFaceTB/SmolVLM-500M-Instruct-GGUF/resolve/main/mmproj-smolvlm-500m-instruct-f16.gguf'
+      }
+    ]
+  },
+  'qwen2-vl-2b-q4': {
+    modelId: 'qwen2-vl-2b-q4',
+    adapter: 'qwen2vl',
+    tier: 'L',
+    estimatedMemoryMb: 2100,
+    contextLimit: 1024,
+    preferredResolution: 384,
+    artifacts: [
+      {
+        role: 'language_model',
+        filename: 'Qwen2-VL-2B-Instruct-Q4_K_M.gguf',
+        sizeBytes: 986046944,
+        downloadUrl: 'https://huggingface.co/second-state/Qwen2-VL-2B-Instruct-GGUF/resolve/main/Qwen2-VL-2B-Instruct-Q4_K_M.gguf'
+      },
+      {
+        role: 'vision_projector',
+        filename: 'Qwen2-VL-2B-Instruct-vision-encoder.gguf',
+        sizeBytes: 2600000000,
+        downloadUrl: 'https://huggingface.co/second-state/Qwen2-VL-2B-Instruct-GGUF/resolve/main/Qwen2-VL-2B-Instruct-vision-encoder.gguf'
+      }
+    ]
+  }
+};
+
+class ModelCacheManager {
+  constructor(customCacheRoot = null) {
+    this.cacheRoot = customCacheRoot || path.join(os.homedir(), '.cache', 'termux-vision');
+    this.legacyCache = path.join(os.homedir(), '.cache', 'vlm_models');
+    this.modelsDir = path.join(this.cacheRoot, 'models');
+    this.downloadsDir = path.join(this.cacheRoot, 'downloads');
+
+    fs.mkdirSync(this.modelsDir, { recursive: true });
+    fs.mkdirSync(this.downloadsDir, { recursive: true });
+  }
+
+  getAvailableCatalogModels() {
+    return Object.keys(CATALOG).sort();
+  }
+
+  isModelInstalled(modelId) {
+    const mdir = path.join(this.modelsDir, modelId);
+    if (!fs.existsSync(mdir)) return false;
+
+    // Check ready marker or custom gguf + mmproj
+    const readyMarker = path.join(mdir, 'READY');
+    if (fs.existsSync(readyMarker)) return true;
+
+    if (fs.statSync(mdir).isDirectory()) {
+      const files = fs.readdirSync(mdir);
+      const ggufs = files.filter(f => f.endsWith('.gguf'));
+      const visionGgufs = ggufs.filter(f => /mmproj|encoder|projector/i.test(f));
+      const textGgufs = ggufs.filter(f => !/mmproj|encoder|projector/i.test(f));
+      if ((textGgufs.length > 0 && visionGgufs.length > 0) || ggufs.length >= 2) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  listInstalled() {
+    const results = [];
+    if (fs.existsSync(this.modelsDir)) {
+      const entries = fs.readdirSync(this.modelsDir);
+      for (const name of entries) {
+        const mdir = path.join(this.modelsDir, name);
+        if (fs.statSync(mdir).isDirectory() && this.isModelInstalled(name)) {
+          let totalBytes = 0;
+          const files = fs.readdirSync(mdir);
+          for (const f of files) {
+            try {
+              totalBytes += fs.statSync(path.join(mdir, f)).size;
+            } catch (e) {}
+          }
+          const tier = CATALOG[name] ? CATALOG[name].tier : 'CUSTOM';
+          results.push({
+            modelId: name,
+            tier: tier,
+            state: 'READY',
+            sizeMb: Math.round(totalBytes / (1024 * 1024)),
+            path: mdir
+          });
+        }
+      }
+    }
+
+    if (fs.existsSync(this.legacyCache)) {
+      for (const [key, val] of Object.entries(CATALOG)) {
+        if (!results.find(r => r.modelId === key)) {
+          const allFound = val.artifacts.every(a => fs.existsSync(path.join(this.legacyCache, a.filename)));
+          if (allFound) {
+            results.push({
+              modelId: key,
+              tier: val.tier,
+              state: 'READY',
+              sizeMb: Math.round(val.artifacts.reduce((acc, a) => acc + (a.sizeBytes || 0), 0) / (1024 * 1024)),
+              path: this.legacyCache
+            });
+          }
+        }
+      }
+    }
+    return results;
+  }
+
+  requireInstalledModel(modelId, mmprojPath = null) {
+    // 1. Direct file path check (Custom 싸제 GGUF)
+    const expanded = path.resolve(modelId.replace(/^~(?=$|\/|\\)/, os.homedir()));
+    if (fs.existsSync(expanded) && fs.statSync(expanded).isFile()) {
+      const dir = path.dirname(expanded);
+      const filename = path.basename(expanded);
+      let visionPath = mmprojPath ? path.resolve(mmprojPath.replace(/^~(?=$|\/|\\)/, os.homedir())) : null;
+      if (!visionPath) {
+        const files = fs.readdirSync(dir);
+        const candidate = files.find(f => f.endsWith('.gguf') && /mmproj|encoder|projector/i.test(f));
+        if (candidate) visionPath = path.join(dir, candidate);
+      }
+      return {
+        manifest: {
+          modelId: path.parse(filename).name,
+          adapter: /smol/i.test(filename) ? 'smolvlm' : 'qwen2vl',
+          tier: 'CUSTOM',
+          contextLimit: 1024
+        },
+        textModelPath: expanded,
+        visionModelPath: visionPath
+      };
+    }
+
+    // 2. Installed model search
+    const installed = this.listInstalled();
+    const installedIds = installed.map(m => m.modelId);
+
+    if (installedIds.length === 0) {
+      throw new NoInstalledModelsError(this.getAvailableCatalogModels());
+    }
+
+    if (!installedIds.includes(modelId)) {
+      throw new ModelNotFoundError(modelId, installedIds, this.getAvailableCatalogModels());
+    }
+
+    const mdir = path.join(this.modelsDir, modelId);
+    let textPath = null;
+    let visionPath = mmprojPath ? path.resolve(mmprojPath) : null;
+
+    if (CATALOG[modelId]) {
+      const cat = CATALOG[modelId];
+      for (const a of cat.artifacts) {
+        const p = path.join(mdir, a.filename);
+        if (fs.existsSync(p)) {
+          if (a.role === 'language_model') textPath = p;
+          if (a.role === 'vision_projector') visionPath = p;
+        }
+      }
+      return {
+        manifest: cat,
+        textModelPath: textPath,
+        visionModelPath: visionPath
+      };
+    }
+
+    // Custom directory
+    const files = fs.readdirSync(mdir);
+    const ggufs = files.filter(f => f.endsWith('.gguf'));
+    const visionFiles = ggufs.filter(f => /mmproj|encoder|projector/i.test(f));
+    const textFiles = ggufs.filter(f => !/mmproj|encoder|projector/i.test(f));
+
+    textPath = textFiles.length > 0 ? path.join(mdir, textFiles[0]) : (ggufs.length > 0 ? path.join(mdir, ggufs[0]) : null);
+    if (!visionPath) {
+      visionPath = visionFiles.length > 0 ? path.join(mdir, visionFiles[0]) : (ggufs.length > 1 ? path.join(mdir, ggufs[1]) : null);
+    }
+
+    return {
+      manifest: {
+        modelId: modelId,
+        adapter: /smol/i.test(modelId) ? 'smolvlm' : 'qwen2vl',
+        tier: 'CUSTOM',
+        contextLimit: 1024
+      },
+      textModelPath: textPath,
+      visionModelPath: visionPath
+    };
+  }
+
+  async downloadFile(url, destPath, onProgress = null) {
+    const partialPath = destPath + '.partial';
+    if (fs.existsSync(partialPath)) fs.unlinkSync(partialPath);
+
+    return new Promise((resolve, reject) => {
+      const getFollow = (currentUrl) => {
+        const client = currentUrl.startsWith('https') ? https : http;
+        const req = client.get(currentUrl, { headers: { 'User-Agent': 'Mozilla/5.0 (termux-vision node)' } }, (res) => {
+          if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+            return getFollow(res.headers.location);
+          }
+          if (res.statusCode !== 200) {
+            return reject(new Error(`HTTP ${res.statusCode}: ${res.statusMessage}`));
+          }
+          const total = parseInt(res.headers['content-length'] || '0', 10);
+          let downloaded = 0;
+          const out = fs.createWriteStream(partialPath);
+
+          res.on('data', (chunk) => {
+            downloaded += chunk.length;
+            if (onProgress) onProgress(path.basename(destPath), downloaded, total);
+          });
+
+          res.pipe(out);
+          out.on('finish', () => {
+            out.close(() => {
+              if (fs.existsSync(destPath)) fs.unlinkSync(destPath);
+              fs.renameSync(partialPath, destPath);
+              resolve();
+            });
+          });
+        });
+        req.on('error', (err) => {
+          if (fs.existsSync(partialPath)) try { fs.unlinkSync(partialPath); } catch (e) {}
+          reject(err);
+        });
+      };
+      getFollow(url);
+    });
+  }
+
+  async install(modelId, onProgress = null) {
+    if (!CATALOG[modelId]) {
+      const installed = this.listInstalled().map(m => m.modelId);
+      throw new ModelDownloadError(modelId, `Model '${modelId}' is not in catalog.`, installed);
+    }
+    const manifest = CATALOG[modelId];
+    const targetDir = path.join(this.modelsDir, modelId);
+    fs.mkdirSync(targetDir, { recursive: true });
+
+    for (const art of manifest.artifacts) {
+      const dest = path.join(targetDir, art.filename);
+      if (fs.existsSync(dest) && fs.statSync(dest).size > 0) continue;
+      await this.downloadFile(art.downloadUrl, dest, onProgress);
+    }
+
+    fs.writeFileSync(path.join(targetDir, 'READY'), 'OK\n', 'utf-8');
+    fs.writeFileSync(path.join(targetDir, 'manifest.json'), JSON.stringify(manifest, null, 2), 'utf-8');
+  }
+
+  remove(modelId) {
+    const mdir = path.join(this.modelsDir, modelId);
+    if (fs.existsSync(mdir)) {
+      fs.rmSync(mdir, { recursive: true, force: true });
+      return true;
+    }
+    return false;
+  }
+}
+
+module.exports = {
+  CATALOG,
+  ModelCacheManager
+};

package/lib/cv.js

@@ -0,0 +1,54 @@
+/**
+ * Fast Pure-JS Spatial Filters & Classical Computer Vision for Node.js.
+ * Open-Source under Apache License 2.0.
+ */
+
+'use strict';
+
+function sobel(grayPixels, width, height) {
+  const gradX = new Float32Array(width * height);
+  const gradY = new Float32Array(width * height);
+  const magnitude = new Float32Array(width * height);
+
+  for (let y = 1; y < height - 1; y++) {
+    for (let x = 1; x < width - 1; x++) {
+      const idx = y * width + x;
+      const gx =
+        -grayPixels[(y - 1) * width + (x - 1)] + grayPixels[(y - 1) * width + (x + 1)] +
+        -2 * grayPixels[y * width + (x - 1)] + 2 * grayPixels[y * width + (x + 1)] +
+        -grayPixels[(y + 1) * width + (x - 1)] + grayPixels[(y + 1) * width + (x + 1)];
+
+      const gy =
+        -grayPixels[(y - 1) * width + (x - 1)] - 2 * grayPixels[(y - 1) * width + x] - grayPixels[(y - 1) * width + (x + 1)] +
+        grayPixels[(y + 1) * width + (x - 1)] + 2 * grayPixels[(y + 1) * width + x] + grayPixels[(y + 1) * width + (x + 1)];
+
+      gradX[idx] = gx;
+      gradY[idx] = gy;
+      magnitude[idx] = Math.sqrt(gx * gx + gy * gy);
+    }
+  }
+
+  return { gradX, gradY, magnitude };
+}
+
+function canny(grayPixels, width, height, lowThreshold = 40.0, highThreshold = 120.0) {
+  const { magnitude } = sobel(grayPixels, width, height);
+  const edges = new Uint8Array(width * height);
+
+  for (let i = 0; i < magnitude.length; i++) {
+    if (magnitude[i] >= highThreshold) {
+      edges[i] = 255;
+    } else if (magnitude[i] >= lowThreshold) {
+      edges[i] = 128;
+    } else {
+      edges[i] = 0;
+    }
+  }
+
+  return edges;
+}
+
+module.exports = {
+  sobel,
+  canny
+};

package/lib/detect.js

@@ -0,0 +1,47 @@
+/**
+ * BoundingBox Geometry & Non-Maximum Suppression (NMS) for Node.js.
+ * Open-Source under Apache License 2.0.
+ */
+
+'use strict';
+
+function computeIoU(boxA, boxB) {
+  const x1 = Math.max(boxA.x, boxB.x);
+  const y1 = Math.max(boxA.y, boxB.y);
+  const x2 = Math.min(boxA.x + boxA.width, boxB.x + boxB.width);
+  const y2 = Math.min(boxA.y + boxA.height, boxB.y + boxB.height);
+
+  const interArea = Math.max(0, x2 - x1) * Math.max(0, y2 - y1);
+  const areaA = boxA.width * boxA.height;
+  const areaB = boxB.width * boxB.height;
+  const unionArea = areaA + areaB - interArea;
+
+  return unionArea <= 0 ? 0 : interArea / unionArea;
+}
+
+function nms(boxes, iouThreshold = 0.45) {
+  if (!boxes || boxes.length === 0) return [];
+  const sorted = [...boxes].sort((a, b) => (b.score || 0) - (a.score || 0));
+  const keep = [];
+
+  for (let i = 0; i < sorted.length; i++) {
+    const current = sorted[i];
+    let suppressed = false;
+    for (let j = 0; j < keep.length; j++) {
+      if (computeIoU(current, keep[j]) > iouThreshold) {
+        suppressed = true;
+        break;
+      }
+    }
+    if (!suppressed) {
+      keep.push(current);
+    }
+  }
+
+  return keep;
+}
+
+module.exports = {
+  computeIoU,
+  nms
+};

package/lib/doctor.js

@@ -0,0 +1,77 @@
+/**
+ * Hardware, Memory, Vulkan, and Environment Diagnostic Doctor for Node.js.
+ * Open-Source under Apache License 2.0.
+ */
+
+'use strict';
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { ModelCacheManager } = require('./cache');
+
+function runDoctor(probeVulkan = false) {
+  const cache = new ModelCacheManager();
+  const isAndroid = fs.existsSync('/system/build.prop') || Boolean(process.env.ANDROID_ROOT);
+
+  let totalRamMb = null;
+  let availableRamMb = null;
+
+  try {
+    if (fs.existsSync('/proc/meminfo')) {
+      const content = fs.readFileSync('/proc/meminfo', 'utf-8');
+      for (const line of content.split('\n')) {
+        if (line.startsWith('MemTotal:')) {
+          totalRamMb = Math.round(parseInt(line.split(/\s+/)[1], 10) / 1024);
+        } else if (line.startsWith('MemAvailable:')) {
+          availableRamMb = Math.round(parseInt(line.split(/\s+/)[1], 10) / 1024);
+        }
+      }
+    }
+  } catch (e) {}
+
+  if (!totalRamMb) {
+    totalRamMb = Math.round(os.totalmem() / (1024 * 1024));
+    availableRamMb = Math.round(os.freemem() / (1024 * 1024));
+  }
+
+  const loaderDetected = fs.existsSync('/system/lib64/libvulkan.so') || fs.existsSync('/system/lib/libvulkan.so');
+  const driverDetected = fs.existsSync('/vendor/lib64/hw/vulkan.adreno.so') || fs.existsSync('/vendor/lib64/hw/vulkan.mali.so');
+
+  let vulkanStatus = 'unverified';
+  if (probeVulkan) {
+    vulkanStatus = (loaderDetected && driverDetected) ? 'driver_detected_experimental' : 'disabled';
+  }
+
+  const installed = cache.listInstalled();
+
+  return {
+    schemaVersion: 1,
+    clientVersion: '0.2.0-alpha.1',
+    platform: {
+      system: os.type(),
+      machine: os.arch(),
+      nodeVersion: process.version,
+      isAndroid: isAndroid
+    },
+    hardware: {
+      cpuCores: os.cpus().length,
+      totalRamMb: totalRamMb,
+      availableRamMb: availableRamMb
+    },
+    vulkan: {
+      loaderDetected: loaderDetected,
+      driverDetected: driverDetected,
+      status: vulkanStatus
+    },
+    vlmRuntime: {
+      installedModelsCount: installed.length,
+      cacheDir: cache.cacheRoot
+    },
+    recommendedPreset: 'Tier M (smolvlm-500m / 4-Threads CPU Reference)'
+  };
+}
+
+module.exports = {
+  runDoctor
+};

package/lib/errors.js

@@ -0,0 +1,119 @@
+/**
+ * Structured Exception Classes and Dynamic User Error Contracts for Node.js.
+ * Open-Source under Apache License 2.0.
+ */
+
+'use strict';
+
+class TermuxVisionError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = this.constructor.name;
+  }
+}
+
+class ModelNotFoundError extends TermuxVisionError {
+  constructor(modelId, availableLocal = [], catalogModels = []) {
+    let msg = `Model '${modelId}' was not found locally or on remote registry.`;
+    if (availableLocal && availableLocal.length > 0) {
+      msg += `\n\nCurrently installed local models (${availableLocal.length}):\n` +
+        availableLocal.map(m => `  - ${m}`).join('\n') +
+        `\n\nTo run with an installed model:\n  termux-vision vlm <IMAGE> --model ${availableLocal[0]}`;
+    } else if (catalogModels && catalogModels.length > 0) {
+      msg += `\n\nNo models installed locally. Available catalog presets:\n` +
+        catalogModels.map(m => `  - ${m}`).join('\n') +
+        `\n\nInstall an official model:\n  termux-vision model install ${catalogModels[0]}`;
+    }
+    super(msg);
+    this.modelId = modelId;
+    this.availableLocal = availableLocal;
+    this.catalogModels = catalogModels;
+  }
+}
+
+class NoInstalledModelsError extends ModelNotFoundError {
+  constructor(catalogModels = []) {
+    const catText = catalogModels.length > 0 ? catalogModels.map(m => `  - ${m}`).join('\n') : '  (none)';
+    const msg = 
+      "No installed VLM models were found in local cache (~/.cache/termux-vision/models).\n\n" +
+      "How to use VLM models:\n" +
+      "1. Install an official catalog model:\n" +
+      `${catText}\n` +
+      "   Example: termux-vision model install smolvlm-500m-q4\n\n" +
+      "2. Use custom/external ('싸제') models:\n" +
+      "   - Place your text model GGUF and vision projector mmproj GGUF in:\n" +
+      "     ~/.cache/termux-vision/models/<custom_model_name>/\n" +
+      "   - Or pass direct file paths:\n" +
+      "     termux-vision vlm <IMAGE> --model /path/to/model.gguf --mmproj /path/to/mmproj.gguf\n" +
+      "   (Note: VLM inference requires both a language model .gguf and a vision projector mmproj-*.gguf)";
+    super('all', [], catalogModels);
+    this.message = msg;
+  }
+}
+
+class ModelSelectionRequiredError extends ModelNotFoundError {
+  constructor(installedModels = []) {
+    const modelsText = installedModels.map(m => `  - ${m}`).join('\n');
+    const exampleModel = installedModels[0] || 'MODEL_ID';
+    const msg =
+      `Multiple models are installed. Please specify one with --model:\n` +
+      `${modelsText}\n\n` +
+      `Example:\n` +
+      `  termux-vision vlm <IMAGE> --model ${exampleModel} -p "Describe this image"`;
+    super('multiple', installedModels, []);
+    this.message = msg;
+  }
+}
+
+class VulkanNotAvailableError extends TermuxVisionError {
+  constructor(reason = '') {
+    const detail = reason ? `\nFailure detail: ${reason}` : '';
+    const msg =
+      `Vulkan GPU acceleration is unavailable or failed on this device.${detail}\n\n` +
+      `[Action Required] Explicit GPU mode cannot proceed. Please switch to CPU mode:\n` +
+      `  CLI: --device cpu\n` +
+      `  Node.js API: device: 'cpu'\n` +
+      `Or use automatic detection: --device auto (device: 'auto')`;
+    super(msg);
+    this.reason = reason;
+  }
+}
+
+class RuntimeNotFoundError extends TermuxVisionError {
+  constructor(searchedPaths = []) {
+    let searched = '';
+    if (searchedPaths.length > 0) {
+      searched = '\n\nSearched paths:\n' + searchedPaths.map(p => `  - ${p}`).join('\n');
+    }
+    const msg =
+      `Required runtime 'llama-cli' was not found.${searched}\n\n` +
+      `Please ensure llama.cpp is installed on Termux:\n` +
+      `  pkg install termux-llamacpp  (or place llama-cli in PATH / $PREFIX/bin)`;
+    super(msg);
+    this.searchedPaths = searchedPaths;
+  }
+}
+
+class ModelDownloadError extends TermuxVisionError {
+  constructor(source, reason = '', availableLocal = []) {
+    let msg = `Failed to download model from '${source}': ${reason}`;
+    if (availableLocal && availableLocal.length > 0) {
+      msg += `\n\nInstalled local models available:\n` +
+        availableLocal.map(m => `  - ${m}`).join('\n') +
+        `\nRun with local model: termux-vision vlm <IMAGE> --model ${availableLocal[0]}`;
+    }
+    super(msg);
+    this.source = source;
+    this.reason = reason;
+  }
+}
+
+module.exports = {
+  TermuxVisionError,
+  ModelNotFoundError,
+  NoInstalledModelsError,
+  ModelSelectionRequiredError,
+  VulkanNotAvailableError,
+  RuntimeNotFoundError,
+  ModelDownloadError
+};