pixmap-engine 1.0.0

package/dist/preview.js

@@ -0,0 +1,216 @@
+import sharp from 'sharp';
+function detectProtocol() {
+    const term = process.env.TERM_PROGRAM ?? '';
+    const termEnv = process.env.TERM ?? '';
+    // iTerm2, WezTerm, Hyper, Tabby, mintty all support iTerm2 inline images
+    if (term === 'iTerm.app' ||
+        term === 'WezTerm' ||
+        term === 'Hyper' ||
+        term === 'Tabby' ||
+        process.env.WEZTERM_PANE != null ||
+        process.env.MINTTY_SHORTCUT != null) {
+        return 'iterm';
+    }
+    // Kitty
+    if (term === 'kitty' || process.env.KITTY_PID != null) {
+        return 'kitty';
+    }
+    // VS Code terminal supports iTerm2 protocol
+    if (term === 'vscode') {
+        return 'iterm';
+    }
+    // Sixel support (xterm with sixel, foot, mlterm)
+    if (termEnv.includes('xterm') && process.env.SIXEL_SUPPORT === '1') {
+        return 'sixel';
+    }
+    return 'halfblock';
+}
+/** Get usable terminal width */
+function getTermCols() {
+    return (process.stdout.columns || 120) - 4;
+}
+// ── iTerm2 inline image protocol ────────────────────────────────────
+async function renderIterm(imagePath, widthCols, heightRows) {
+    // Resize with sharp first to control dimensions
+    const meta = await sharp(imagePath).metadata();
+    const origW = meta.width ?? 400;
+    const origH = meta.height ?? 400;
+    // Each terminal column ≈ 8px, each row ≈ 16px (common defaults)
+    // We target pixel dimensions for good quality
+    const targetPxW = widthCols * 8;
+    let targetPxH = Math.round(targetPxW * (origH / origW));
+    if (heightRows) {
+        const maxPxH = heightRows * 16;
+        if (targetPxH > maxPxH) {
+            targetPxH = maxPxH;
+        }
+    }
+    const buf = await sharp(imagePath)
+        .resize(targetPxW, targetPxH, { fit: 'inside', kernel: 'lanczos3' })
+        .png()
+        .toBuffer();
+    const b64 = buf.toString('base64');
+    const args = `inline=1;width=${widthCols};preserveAspectRatio=1`;
+    // OSC 1337 ; File=[args]:[base64 data] ST
+    return `  \x1b]1337;File=${args}:${b64}\x07`;
+}
+// ── Kitty graphics protocol ─────────────────────────────────────────
+async function renderKitty(imagePath, widthCols, heightRows) {
+    const meta = await sharp(imagePath).metadata();
+    const origW = meta.width ?? 400;
+    const origH = meta.height ?? 400;
+    const targetPxW = widthCols * 8;
+    let targetPxH = Math.round(targetPxW * (origH / origW));
+    if (heightRows) {
+        const maxPxH = heightRows * 16;
+        if (targetPxH > maxPxH)
+            targetPxH = maxPxH;
+    }
+    const buf = await sharp(imagePath)
+        .resize(targetPxW, targetPxH, { fit: 'inside', kernel: 'lanczos3' })
+        .png()
+        .toBuffer();
+    const b64 = buf.toString('base64');
+    const chunks = [];
+    const CHUNK_SIZE = 4096;
+    for (let i = 0; i < b64.length; i += CHUNK_SIZE) {
+        const chunk = b64.slice(i, i + CHUNK_SIZE);
+        const more = i + CHUNK_SIZE < b64.length ? 1 : 0;
+        if (i === 0) {
+            chunks.push(`\x1b_Ga=T,f=100,c=${widthCols},m=${more};${chunk}\x1b\\`);
+        }
+        else {
+            chunks.push(`\x1b_Gm=${more};${chunk}\x1b\\`);
+        }
+    }
+    return '  ' + chunks.join('');
+}
+// ── Half-block fallback (true-color ANSI) ───────────────────────────
+const UPPER_HALF = '\u2580';
+const RESET = '\x1b[0m';
+function fg(r, g, b) {
+    return `\x1b[38;2;${r};${g};${b}m`;
+}
+function bgc(r, g, b) {
+    return `\x1b[48;2;${r};${g};${b}m`;
+}
+async function loadPixels(imagePath, cols, maxTermRows) {
+    const meta = await sharp(imagePath).metadata();
+    const origW = meta.width ?? cols;
+    const origH = meta.height ?? cols;
+    const aspect = origH / origW;
+    let pxW = cols;
+    let pxH = Math.round(cols * aspect);
+    if (maxTermRows && pxH > maxTermRows * 2) {
+        pxH = maxTermRows * 2;
+        pxW = Math.round(pxH / aspect);
+    }
+    pxW = Math.max(pxW, 2);
+    pxH = Math.max(pxH, 2);
+    // Ensure even height for half-block pairing
+    if (pxH % 2 !== 0)
+        pxH += 1;
+    const { data, info } = await sharp(imagePath)
+        .resize(pxW, pxH, { fit: 'fill', kernel: 'lanczos3' })
+        .removeAlpha()
+        .raw()
+        .toBuffer({ resolveWithObject: true });
+    const pixels = [];
+    for (let y = 0; y < info.height; y++) {
+        const row = [];
+        for (let x = 0; x < info.width; x++) {
+            const i = (y * info.width + x) * 3;
+            row.push([data[i], data[i + 1], data[i + 2]]);
+        }
+        pixels.push(row);
+    }
+    return { pixels, width: info.width, height: info.height };
+}
+function pixelsToAnsi(pixels, indent = 2) {
+    const pad = ' '.repeat(indent);
+    const lines = [];
+    for (let y = 0; y < pixels.length; y += 2) {
+        let line = pad;
+        const topRow = pixels[y];
+        const botRow = y + 1 < pixels.length ? pixels[y + 1] : null;
+        for (let x = 0; x < topRow.length; x++) {
+            const [tr, tg, tb] = topRow[x];
+            const [br, bg, bb] = botRow ? botRow[x] : [0, 0, 0];
+            line += `${fg(tr, tg, tb)}${bgc(br, bg, bb)}${UPPER_HALF}`;
+        }
+        line += RESET;
+        lines.push(line);
+    }
+    return lines.join('\n');
+}
+async function renderHalfblock(imagePath, cols, maxTermRows) {
+    const { pixels } = await loadPixels(imagePath, cols, maxTermRows);
+    return pixelsToAnsi(pixels);
+}
+// ── Public API ──────────────────────────────────────────────────────
+const protocol = detectProtocol();
+/** Render a single image to terminal string */
+export async function renderImage(imagePath, cols, maxTermRows) {
+    const w = cols ?? getTermCols();
+    switch (protocol) {
+        case 'iterm':
+            return renderIterm(imagePath, w, maxTermRows);
+        case 'kitty':
+            return renderKitty(imagePath, w, maxTermRows);
+        default:
+            return renderHalfblock(imagePath, w, maxTermRows);
+    }
+}
+/** Render multiple images side-by-side in a row with labels on top */
+export async function renderImageRow(panels, panelCols, maxPanelRows = 24, gap = 2) {
+    const termW = getTermCols();
+    const cols = panelCols ?? Math.floor((termW - gap * (panels.length - 1)) / panels.length);
+    // For inline-image protocols, render each image stacked (side-by-side isn't
+    // possible with protocol images). Label + image + gap, repeated.
+    if (protocol === 'iterm' || protocol === 'kitty') {
+        const parts = [];
+        for (const panel of panels) {
+            parts.push(`  ${panel.label}`);
+            const img = await renderImage(panel.imagePath, cols, maxPanelRows);
+            parts.push(img);
+            parts.push('');
+        }
+        return parts.join('\n');
+    }
+    // Half-block: true side-by-side rendering
+    const allData = await Promise.all(panels.map((p) => loadPixels(p.imagePath, cols, maxPanelRows)));
+    const lines = [];
+    const indent = '  ';
+    const gapStr = ' '.repeat(gap);
+    // Labels
+    let labelLine = indent;
+    for (let i = 0; i < panels.length; i++) {
+        const label = panels[i].label;
+        const padded = label.length > cols ? label.slice(0, cols - 1) + '\u2026' : label.padEnd(cols);
+        labelLine += padded;
+        if (i < panels.length - 1)
+            labelLine += gapStr;
+    }
+    lines.push(labelLine);
+    const maxPxRows = Math.max(...allData.map((d) => d.height));
+    for (let y = 0; y < maxPxRows; y += 2) {
+        let line = indent;
+        for (let gi = 0; gi < allData.length; gi++) {
+            const { pixels, width } = allData[gi];
+            for (let x = 0; x < width; x++) {
+                const [tr, tg, tb] = y < pixels.length ? pixels[y][x] : [0, 0, 0];
+                const [br, bg, bb] = y + 1 < pixels.length ? pixels[y + 1][x] : [0, 0, 0];
+                line += `${fg(tr, tg, tb)}${bgc(br, bg, bb)}${UPPER_HALF}`;
+            }
+            line += RESET;
+            // Pad shorter panels
+            const padN = cols - width;
+            if (padN > 0)
+                line += ' '.repeat(padN);
+            if (gi < allData.length - 1)
+                line += gapStr;
+        }
+        lines.push(line);
+    }
+    return lines.join('\n');
+}

package/dist/searcher.js

@@ -0,0 +1,17 @@
+import { DEFAULT_TOP_K } from './types.js';
+export async function findSimilar(queryImagePath, embedder, store, db, topK = DEFAULT_TOP_K) {
+    const queryVec = await embedder.embed(queryImagePath);
+    const hits = store.search(queryVec, topK);
+    const results = [];
+    for (const hit of hits) {
+        const metadata = db.get(hit.id);
+        if (!metadata) {
+            continue;
+        }
+        results.push({
+            ...metadata,
+            score: hit.score,
+        });
+    }
+    return results;
+}

package/dist/types.js

@@ -0,0 +1,3 @@
+export const VECTOR_DIMENSIONS = 512;
+export const DEFAULT_TOP_K = 5;
+export const DEFAULT_MAX_ELEMENTS = 100_000;

package/dist/vectorStore.js

@@ -0,0 +1,47 @@
+import fs from 'node:fs';
+import hnswlib from 'hnswlib-node';
+import { DEFAULT_MAX_ELEMENTS, VECTOR_DIMENSIONS } from './types.js';
+const { HierarchicalNSW } = hnswlib;
+export class VectorStore {
+    index;
+    dim;
+    maxElements;
+    constructor(dim = VECTOR_DIMENSIONS, maxElements = DEFAULT_MAX_ELEMENTS) {
+        this.dim = dim;
+        this.maxElements = maxElements;
+        this.index = new HierarchicalNSW('cosine', dim);
+    }
+    initOrLoad(indexPath) {
+        if (fs.existsSync(indexPath)) {
+            this.index.readIndexSync(indexPath);
+            return;
+        }
+        this.index.initIndex(this.maxElements);
+    }
+    add(id, vector) {
+        this.ensureCapacity();
+        this.index.addPoint(Array.from(vector), id);
+    }
+    search(queryVector, topK) {
+        const result = this.index.searchKnn(Array.from(queryVector), topK);
+        return result.neighbors.map((id, i) => ({
+            id,
+            score: 1 - result.distances[i],
+        }));
+    }
+    save(path) {
+        this.index.writeIndex(path);
+    }
+    getCount() {
+        return this.index.getCurrentCount();
+    }
+    ensureCapacity() {
+        const count = this.index.getCurrentCount();
+        if (count < this.maxElements) {
+            return;
+        }
+        const next = Math.ceil(this.maxElements * 1.5);
+        this.index.resizeIndex(next);
+        this.maxElements = next;
+    }
+}

package/docs/HOW.md

@@ -0,0 +1,177 @@
+# How pixmap works
+
+This document walks through the internals — how images get indexed, how search works, and why the stack is what it is.
+
+## The pieces
+
+Pixmap has a pretty simple architecture. There are three core components and a thin engine that ties them together:
+
+```
+PixmapEngine (src/engine.ts)
+├── ImageEmbedder  (src/embedder.ts)   — image → 512-d vector
+├── VectorStore    (src/vectorStore.ts) — HNSW index (add/search/save)
+└── MetadataDb     (src/metadataDb.ts)  — SQLite (paths, ids, timestamps)
+```
+
+On top of these, `indexer.ts` and `searcher.ts` are small workflow modules that coordinate the add and search operations respectively. The engine itself just wires everything together and exposes the public API.
+
+## Indexing an image
+
+When you call `engine.add(imagePath)`, here's what actually happens:
+
+**1. Dedup check.** The path is upserted into SQLite. If it already exists with `indexed = 1`, we bail early — no work to do. If it exists with `indexed = 0` (maybe a previous run crashed halfway through), we re-process it.
+
+**2. Preprocessing.** Sharp resizes the image to 224x224 pixels using a cover fit (maintains aspect ratio, crops the excess). Alpha channels get stripped. The output is a raw PNG buffer — this is what the CLIP model expects.
+
+**3. Embedding.** The preprocessed image is fed through `clip-vit-base-patch32` running in ONNX via `@xenova/transformers`. This produces a 512-dimensional float vector. The vector gets L2-normalized so it sits on the unit hypersphere, which is needed for cosine similarity to work correctly with HNSW.
+
+**4. Indexing.** The normalized vector is inserted into the HNSW index with the SQLite row ID as its key. The index is saved to disk immediately.
+
+**5. Marking done.** The SQLite row gets `indexed = 1`. This is the commit point — if the process dies before this, the next run will re-process the image.
+
+## Searching
+
+Search follows the same embedding path, then diverges:
+
+1. The query image goes through the same Sharp → CLIP → normalize pipeline.
+2. The resulting vector is handed to HNSW, which does an approximate nearest-neighbor search and returns the top-K vector IDs along with their cosine distances.
+3. Distances are converted to similarity scores (`1 - distance`, so 1.0 = identical).
+4. Each ID is looked up in SQLite to get the file path and timestamp.
+5. Results come back as an array of `{ id, path, score, created, indexed }`.
+
+The search itself (step 2) takes under 2ms even with 100K vectors in the index. The bottleneck is always the CLIP embedding step, which runs ~200-500ms on CPU.
+
+## Embedding: CLIP in detail
+
+### Why CLIP
+
+CLIP understands images semantically. Two photos of a dog — one close-up, one from across a park — will have similar embeddings even though their pixels are completely different. This is what makes it useful for "find images like this one" rather than just pixel-matching.
+
+We use `clip-vit-base-patch32`, a ViT-B/32 variant. It's about 150MB, runs fine on CPU, and gives 512-d vectors. The model downloads automatically on first run and gets cached in `~/.cache/huggingface/`.
+
+### The preprocessing pipeline
+
+CLIP needs exactly 224x224x3 input. Here's how we get there:
+
+1. **Resize** to 224x224 with cover fit — aspect ratio is preserved, overflow is cropped.
+2. **Strip alpha** — drop down to 3 channels (RGB).
+3. **Export as PNG** — lossless buffer, no compression artifacts.
+4. **Build RawImage** — 150,528 pixel values as a Uint8Array, ready for the model.
+
+### Normalization
+
+After inference, we normalize the vector to unit length:
+
+```
+normalized[i] = v[i] / sqrt(v[0]² + v[1]² + ... + v[511]²)
+```
+
+This matters because HNSW's cosine distance mode assumes unit vectors. Without normalization, similarity scores would be inconsistent — vectors with larger magnitudes would distort the distance calculations.
+
+## Vector index: HNSW
+
+### How HNSW works
+
+HNSW (Hierarchical Navigable Small World) is a graph-based approximate nearest neighbor algorithm. Think of it like a skip list but in vector space:
+
+- The bottom layer has every vector, connected to its closest neighbors.
+- Each layer above has fewer and fewer vectors, with longer-range connections.
+- To search, you start at the top (where there are few nodes but long jumps) and work down, getting more precise at each layer.
+
+```
+Layer 2:  A ──────────── D              (express lanes)
+          │              │
+Layer 1:  A ── B ── C ── D ── E         (mid-range)
+          │   │   │   │   │
+Layer 0:  A  B  C  D  E  F  G  H  I    (all vectors, local connections)
+```
+
+This gives O(log n) search instead of O(n) brute force. At 100K images, that's the difference between ~2ms and ~500ms per query.
+
+### Configuration
+
+The index is set up with:
+
+- **512 dimensions** (matching CLIP output)
+- **Cosine distance** (since vectors are normalized)
+- **100K initial capacity** (auto-resizes to 1.5x when full)
+
+The resize creates a new index, copies everything over, and swaps it in. Not the fastest operation, but it's a one-time cost and it keeps things simple.
+
+### Persistence
+
+After every `add()` call, the index is written to `index.hnsw`. On startup, if this file exists, it's loaded back. The format is hnswlib's native binary — it includes the vectors, graph edges, and config in a single file.
+
+## Metadata: SQLite
+
+### Schema
+
+```sql
+CREATE TABLE IF NOT EXISTS images (
+  id       INTEGER PRIMARY KEY AUTOINCREMENT,
+  path     TEXT    NOT NULL UNIQUE,
+  indexed  INTEGER NOT NULL DEFAULT 0,
+  created  INTEGER NOT NULL
+);
+```
+
+The `id` column does double duty — it's both the primary key in SQLite and the vector ID in HNSW. When HNSW returns vector ID 42, we look up `images WHERE id = 42` to get the file path. Simple 1:1 mapping, no translation layer needed.
+
+The `indexed` column (0 or 1) tracks whether the embedding pipeline completed. The `UNIQUE` constraint on `path` prevents the same file from being indexed twice.
+
+### Why SQLite instead of just a JSON file
+
+A JSON file would work for small collections, but it falls apart quickly:
+
+- No atomic writes — a crash mid-save corrupts the whole thing.
+- No indexed lookups — finding a record by path means scanning the entire file.
+- No concurrent access — reading while writing is a recipe for data loss.
+
+SQLite handles all of this out of the box. `better-sqlite3` gives us synchronous calls (no callback juggling in the indexer), and WAL mode means reads don't block writes.
+
+## Crash recovery
+
+The indexing pipeline is designed so that a crash at any point leaves things in a recoverable state:
+
+- **Crash before embedding completes:** SQLite has the row with `indexed = 0`. Next time you add the same path, it picks up where it left off.
+- **Crash after embedding but before HNSW save:** The vector is lost (wasn't persisted). SQLite still shows `indexed = 0`, so the next add re-embeds and re-inserts.
+- **Crash after HNSW save but before marking indexed:** On next run, the image gets re-embedded and the vector is re-inserted into HNSW. HNSW handles duplicate IDs by overwriting, so no corruption.
+- **Crash after everything:** Both stores are consistent. `indexed = 1` prevents reprocessing.
+
+There's no transaction spanning both stores — instead, the two-phase design (insert metadata first, mark complete last) naturally handles partial failures.
+
+## Initialization
+
+When `engine.init()` is called, three things happen:
+
+1. The CLIP model loads. First run downloads ~150MB from Hugging Face; subsequent runs use the cache.
+2. The HNSW index loads from `index.hnsw` if it exists, otherwise a fresh empty index is created.
+3. SQLite opens `metadata.db`, running the CREATE TABLE statement if needed (it's idempotent).
+
+All three dependencies are loaded via dynamic `import()`. This means you can `import { PixmapEngine } from "pixmap"` without triggering any model downloads or native module loading — that only happens when you call `init()`.
+
+There's no explicit shutdown or cleanup. HNSW is saved to disk after each add, SQLite handles its own connection lifecycle, and the ONNX session gets garbage-collected normally.
+
+## Performance
+
+Some rough numbers to set expectations:
+
+**Indexing (per image):**
+
+| Step | Time |
+|------|------|
+| Sharp resize | ~20ms |
+| CLIP embedding | ~200-500ms |
+| HNSW insert | <1ms |
+| SQLite upsert | <1ms |
+| HNSW disk save | ~5-50ms |
+
+The CLIP step dominates. On a modern laptop, expect around 2-4 images per second.
+
+**Search:**
+
+The embedding step is the same ~200-500ms. The actual HNSW lookup is under 2ms even at 100K scale. So search latency is essentially just the time to embed the query.
+
+**Storage:**
+
+Each indexed image costs about 2KB (512 floats for the vector, plus graph edges, plus the SQLite row). At 100K images, you're looking at roughly 220MB for the index and 20MB for the database. The original images aren't copied or stored — pixmap only keeps the vectors and paths.

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "pixmap-engine",
+  "version": "1.0.0",
+  "type": "module",
+  "description": "On-device image similarity search",
+  "main": "./src/engine.ts",
+  "exports": {
+    ".": {
+      "import": "./src/engine.ts",
+      "default": "./dist/engine.js"
+    }
+  },
+  "bin": {
+    "pixmap": "dist/index.js"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "start": "node dist/index.js",
+    "dev": "tsx src/index.ts"
+  },
+  "dependencies": {
+    "@xenova/transformers": "^2.17.2",
+    "better-sqlite3": "^11.7.0",
+    "hnswlib-node": "^3.0.0",
+    "sharp": "^0.33.5"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
+    "@types/node": "^22.10.2",
+    "tsx": "^4.19.2",
+    "typescript": "^5.7.2"
+  }
+}

package/src/embedder.ts

@@ -0,0 +1,100 @@
+import sharp from 'sharp';
+import { RawImage, pipeline } from '@xenova/transformers';
+import { VECTOR_DIMENSIONS } from './types.js';
+
+type Extractor = (input: unknown) => Promise<unknown>;
+
+export class ImageEmbedder {
+  private extractor: Extractor | null = null;
+
+  async init(): Promise<void> {
+    this.extractor = (await pipeline(
+      'image-feature-extraction',
+      'Xenova/clip-vit-base-patch32'
+    )) as Extractor;
+  }
+
+  async embed(imagePath: string): Promise<Float32Array> {
+    if (!this.extractor) {
+      throw new Error('ImageEmbedder is not initialized. Call init() first.');
+    }
+
+    const preprocessed = await sharp(imagePath)
+      .resize(224, 224, { fit: 'cover' })
+      .removeAlpha()
+      .toFormat('png')
+      .toBuffer();
+
+    const image = await this.bufferToRawImage(preprocessed);
+    const output = await this.extractor(image);
+    const vector = this.extractVector(output);
+
+    if (vector.length !== VECTOR_DIMENSIONS) {
+      throw new Error(
+        `Unexpected embedding dimensions: ${vector.length}. Expected ${VECTOR_DIMENSIONS}.`
+      );
+    }
+
+    return this.l2Normalize(vector);
+  }
+
+  private async bufferToRawImage(buffer: Buffer): Promise<unknown> {
+    const raw = RawImage as unknown as {
+      fromBlob?: (blob: Blob) => Promise<unknown>;
+      fromBuffer?: (b: Buffer) => Promise<unknown>;
+      read?: (b: Buffer) => Promise<unknown>;
+    };
+
+    if (typeof raw.fromBlob === 'function') {
+      const view = Uint8Array.from(buffer);
+      return raw.fromBlob(new Blob([view], { type: 'image/png' }));
+    }
+    if (typeof raw.fromBuffer === 'function') {
+      return raw.fromBuffer(buffer);
+    }
+    if (typeof raw.read === 'function') {
+      return raw.read(buffer);
+    }
+
+    throw new Error('No compatible RawImage constructor found in @xenova/transformers.');
+  }
+
+  private extractVector(output: unknown): Float32Array {
+    let data: number[] | Float32Array | undefined;
+
+    if (output instanceof Float32Array) {
+      data = output;
+    } else if (Array.isArray(output) && output.every((x) => typeof x === 'number')) {
+      data = output as number[];
+    } else if (typeof output === 'object' && output !== null) {
+      const maybeObject = output as { data?: number[] | Float32Array };
+      if (maybeObject.data) {
+        data = maybeObject.data;
+      } else if (Array.isArray(output)) {
+        const first = output[0] as { data?: number[] | Float32Array } | undefined;
+        data = first?.data;
+      }
+    }
+
+    if (!data) {
+      throw new Error('Failed to extract vector data from model output.');
+    }
+
+    return data instanceof Float32Array ? data : new Float32Array(data);
+  }
+
+  private l2Normalize(vector: Float32Array): Float32Array {
+    let sumSquares = 0;
+    for (let i = 0; i < vector.length; i += 1) {
+      sumSquares += vector[i] * vector[i];
+    }
+
+    const norm = Math.sqrt(sumSquares) || 1;
+    const normalized = new Float32Array(vector.length);
+    for (let i = 0; i < vector.length; i += 1) {
+      normalized[i] = vector[i] / norm;
+    }
+
+    return normalized;
+  }
+}