pixmap-engine 1.0.0

package/src/engine.ts

@@ -0,0 +1,106 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { DEFAULT_TOP_K, ImageRecord, SearchResult } from './types.js';
+
+type EmbedderInstance = {
+  init: () => Promise<void>;
+  embed: (imagePath: string) => Promise<Float32Array>;
+};
+
+type StoreInstance = {
+  initOrLoad: (indexPath: string) => void;
+  save: (indexPath: string) => void;
+  getCount: () => number;
+};
+
+type DbInstance = {
+  get: (id: number) => ImageRecord | undefined;
+  list: (limit?: number) => ImageRecord[];
+};
+
+export type PixmapEngineOptions = {
+  dataDir: string;
+  indexFileName?: string;
+  dbFileName?: string;
+};
+
+export class PixmapEngine {
+  private readonly dataDir: string;
+  private readonly indexPath: string;
+  private readonly dbPath: string;
+
+  private embedder: EmbedderInstance | null = null;
+  private store: StoreInstance | null = null;
+  private db: DbInstance | null = null;
+
+  constructor(options: PixmapEngineOptions) {
+    this.dataDir = path.resolve(options.dataDir);
+    this.indexPath = path.join(this.dataDir, options.indexFileName ?? 'index.hnsw');
+    this.dbPath = path.join(this.dataDir, options.dbFileName ?? 'metadata.db');
+  }
+
+  async init(): Promise<void> {
+    fs.mkdirSync(this.dataDir, { recursive: true });
+
+    const [{ ImageEmbedder }, { VectorStore }, { MetadataDb }] = await Promise.all([
+      import('./embedder.js'),
+      import('./vectorStore.js'),
+      import('./metadataDb.js'),
+    ]);
+
+    this.embedder = new ImageEmbedder();
+    await this.embedder.init();
+
+    this.store = new VectorStore();
+    this.store.initOrLoad(this.indexPath);
+
+    this.db = new MetadataDb(this.dbPath);
+  }
+
+  async add(imagePath: string): Promise<{ id: number; skipped: boolean; record?: ImageRecord }> {
+    this.assertInitialized();
+
+    const absolute = path.resolve(imagePath);
+    const { addImage } = await import('./indexer.js');
+    const result = await (addImage as any)(absolute, this.embedder!, this.store!, this.db!);
+    this.store!.save(this.indexPath);
+
+    return {
+      ...result,
+      record: this.db!.get(result.id),
+    };
+  }
+
+  async search(queryImagePath: string, topK = DEFAULT_TOP_K): Promise<SearchResult[]> {
+    this.assertInitialized();
+    const absolute = path.resolve(queryImagePath);
+
+    if (this.store!.getCount() === 0) {
+      return [];
+    }
+
+    const { findSimilar } = await import('./searcher.js');
+    return (findSimilar as any)(absolute, this.embedder!, this.store!, this.db!, topK);
+  }
+
+  listImages(limit = 200): ImageRecord[] {
+    this.assertInitialized();
+    return this.db!.list(limit);
+  }
+
+  getImage(id: number): ImageRecord | undefined {
+    this.assertInitialized();
+    return this.db!.get(id);
+  }
+
+  getIndexedCount(): number {
+    this.assertInitialized();
+    return this.store!.getCount();
+  }
+
+  private assertInitialized(): void {
+    if (!this.embedder || !this.store || !this.db) {
+      throw new Error('PixmapEngine not initialized. Call init() first.');
+    }
+  }
+}

package/src/index.ts

@@ -0,0 +1,310 @@
+#!/usr/bin/env node
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { ImageEmbedder } from './embedder.js';
+import { addImage } from './indexer.js';
+import { MetadataDb } from './metadataDb.js';
+import { renderImage, renderImageRow } from './preview.js';
+import { findSimilar } from './searcher.js';
+import { DEFAULT_TOP_K } from './types.js';
+import { VectorStore } from './vectorStore.js';
+
+type CliOptions = {
+  dataDir: string;
+  topK: number;
+};
+
+const IMAGE_EXTENSIONS = new Set(['.jpg', '.jpeg', '.png', '.webp', '.bmp', '.gif', '.avif', '.tiff']);
+
+async function main(): Promise<void> {
+  const args = process.argv.slice(2);
+
+  if (args.length === 0 || args[0] === '--help' || args[0] === '-h') {
+    printUsage();
+    return;
+  }
+
+  const command = args[0];
+  const rest = args.slice(1);
+  const options = parseOptions(rest);
+
+  const dataDir = path.resolve(options.dataDir);
+  fs.mkdirSync(dataDir, { recursive: true });
+
+  const dbPath = path.join(dataDir, 'metadata.db');
+  const indexPath = path.join(dataDir, 'index.hnsw');
+
+  if (command === 'list') {
+    const db = new MetadataDb(dbPath);
+    const images = db.list(200);
+
+    if (images.length === 0) {
+      console.log('No indexed images.');
+      return;
+    }
+
+    console.log(`\n  Indexed images (${images.length}):\n`);
+    for (const img of images) {
+      const date = new Date(img.created * 1000).toISOString().slice(0, 19);
+      const name = path.basename(img.path);
+      console.log(`  [${img.id}]  ${date}  ${name}`);
+    }
+    console.log();
+    return;
+  }
+
+  if (command === 'show') {
+    const target = rest.find((x) => !x.startsWith('--'));
+    if (!target) {
+      console.error('Error: Provide an image path or id.\n');
+      process.exitCode = 1;
+      return;
+    }
+
+    let imagePath: string;
+    const asId = Number(target);
+    if (Number.isFinite(asId) && asId > 0) {
+      const db = new MetadataDb(dbPath);
+      const record = db.get(asId);
+      if (!record) {
+        console.error(`Error: No image with id ${asId}`);
+        process.exitCode = 1;
+        return;
+      }
+      imagePath = record.path;
+    } else {
+      imagePath = path.resolve(target);
+    }
+
+    assertImageExists(imagePath);
+    console.log();
+    const preview = await renderImage(imagePath);
+    console.log(preview);
+    console.log(`    ${path.basename(imagePath)}\n`);
+    return;
+  }
+
+  if (command === 'status') {
+    const db = new MetadataDb(dbPath);
+    const store = new VectorStore();
+    store.initOrLoad(indexPath);
+    const images = db.list(999999);
+
+    console.log(`\n  pixmap status`);
+    console.log(`  data dir:   ${dataDir}`);
+    console.log(`  vectors:    ${store.getCount()}`);
+    console.log(`  images:     ${images.length}`);
+    console.log();
+    return;
+  }
+
+  // Commands below require the embedder
+  console.log('Loading CLIP model...');
+  const embedder = new ImageEmbedder();
+  await embedder.init();
+
+  const store = new VectorStore();
+  store.initOrLoad(indexPath);
+
+  const db = new MetadataDb(dbPath);
+
+  if (command === 'add') {
+    const targets = rest.filter((x) => !x.startsWith('--'));
+    if (targets.length === 0) {
+      console.error('Error: No image path provided.\n');
+      printUsage();
+      process.exitCode = 1;
+      return;
+    }
+
+    let added = 0;
+    let skipped = 0;
+
+    for (const target of targets) {
+      const absolutePath = path.resolve(target);
+      const stat = safeStat(absolutePath);
+
+      if (!stat) {
+        console.error(`  skip: not found — ${absolutePath}`);
+        continue;
+      }
+
+      const files = stat.isDirectory() ? walkImages(absolutePath) : [absolutePath];
+
+      for (const file of files) {
+        const result = await addImage(file, embedder, store, db);
+        if (result.skipped) {
+          skipped += 1;
+          console.log(`  skip: already indexed — ${path.basename(file)}`);
+        } else {
+          added += 1;
+          console.log(`  added: id=${result.id} — ${path.basename(file)}`);
+        }
+      }
+    }
+
+    store.save(indexPath);
+    console.log(`\nDone. added=${added} skipped=${skipped}`);
+    return;
+  }
+
+  if (command === 'search' || command === 'similar') {
+    const imagePath = rest.find((x) => !x.startsWith('--'));
+    if (!imagePath) {
+      console.error('Error: No query image provided.\n');
+      printUsage();
+      process.exitCode = 1;
+      return;
+    }
+
+    const absolutePath = path.resolve(imagePath);
+    assertImageExists(absolutePath);
+
+    if (store.getCount() === 0) {
+      console.log('Index is empty. Add images first with: pixmap add <path>');
+      return;
+    }
+
+    // Show query image preview
+    console.log(`\n  Query:\n`);
+    const queryPreview = await renderImage(absolutePath, undefined, 20);
+    console.log(queryPreview);
+    console.log(`    ${path.basename(absolutePath)}\n`);
+
+    const results = await findSimilar(absolutePath, embedder, store, db, options.topK);
+
+    if (results.length === 0) {
+      console.log('  No similar images found.\n');
+      return;
+    }
+
+    console.log(`  ── Results (${results.length}) ──\n`);
+
+    const PER_ROW = 3;
+
+    for (let i = 0; i < results.length; i += PER_ROW) {
+      const batch = results.slice(i, i + PER_ROW);
+      const panels = batch
+        .map((r, j) => {
+          const pct = (r.score * 100).toFixed(1);
+          const name = path.basename(r.path);
+          return fs.existsSync(r.path)
+            ? { label: `#${i + j + 1}  ${pct}%  ${name}`, imagePath: r.path }
+            : null;
+        })
+        .filter((p): p is NonNullable<typeof p> => p !== null);
+
+      if (panels.length > 0) {
+        const row = await renderImageRow(panels);
+        console.log(row);
+        console.log();
+      }
+    }
+    return;
+  }
+
+  console.error(`Unknown command: ${command}\n`);
+  printUsage();
+  process.exitCode = 1;
+}
+
+function parseOptions(args: string[]): CliOptions {
+  let dataDir = 'data';
+  let topK = DEFAULT_TOP_K;
+
+  for (let i = 0; i < args.length; i += 1) {
+    const arg = args[i];
+
+    if (arg === '--data-dir' || arg === '-d') {
+      dataDir = args[i + 1] ?? dataDir;
+      i += 1;
+      continue;
+    }
+
+    if (arg === '--top-k' || arg === '-k') {
+      const parsed = Number(args[i + 1]);
+      if (!Number.isNaN(parsed) && parsed > 0) {
+        topK = Math.floor(parsed);
+      }
+      i += 1;
+      continue;
+    }
+  }
+
+  return { dataDir, topK };
+}
+
+function walkImages(rootDir: string): string[] {
+  const result: string[] = [];
+  const stack = [rootDir];
+
+  while (stack.length > 0) {
+    const current = stack.pop();
+    if (!current) continue;
+
+    const entries = fs.readdirSync(current, { withFileTypes: true });
+    for (const entry of entries) {
+      const fullPath = path.join(current, entry.name);
+      if (entry.isDirectory()) {
+        stack.push(fullPath);
+        continue;
+      }
+
+      const ext = path.extname(entry.name).toLowerCase();
+      if (IMAGE_EXTENSIONS.has(ext)) {
+        result.push(fullPath);
+      }
+    }
+  }
+
+  return result.sort();
+}
+
+function safeStat(p: string): fs.Stats | null {
+  try {
+    return fs.statSync(p);
+  } catch {
+    return null;
+  }
+}
+
+function assertImageExists(imagePath: string): void {
+  if (!fs.existsSync(imagePath)) {
+    throw new Error(`File not found: ${imagePath}`);
+  }
+  if (!fs.statSync(imagePath).isFile()) {
+    throw new Error(`Not a file: ${imagePath}`);
+  }
+}
+
+function printUsage(): void {
+  console.log(`
+  pixmap — local image similarity search
+
+  Usage:
+    pixmap add <image|dir> [...]     Add image(s) or directory to the index
+    pixmap search <image> [-k N]     Find similar indexed images
+    pixmap show <image|id>           Preview an image in the terminal
+    pixmap list                      Show all indexed images
+    pixmap status                    Show index stats
+
+  Options:
+    -d, --data-dir <path>   Data directory (default: ./data)
+    -k, --top-k <number>    Number of results (default: ${DEFAULT_TOP_K})
+    -h, --help              Show this help
+
+  Examples:
+    pixmap add photo.jpg
+    pixmap add ./photos/
+    pixmap search query.png -k 10
+    pixmap show 3
+    pixmap list
+`);
+}
+
+main().catch((error: unknown) => {
+  const message = error instanceof Error ? error.message : String(error);
+  console.error(`Error: ${message}`);
+  process.exitCode = 1;
+});

package/src/indexer.ts

@@ -0,0 +1,21 @@
+import { ImageEmbedder } from './embedder.js';
+import { MetadataDb } from './metadataDb.js';
+import { VectorStore } from './vectorStore.js';
+
+export async function addImage(
+  imagePath: string,
+  embedder: ImageEmbedder,
+  store: VectorStore,
+  db: MetadataDb
+): Promise<{ id: number; skipped: boolean }> {
+  const record = db.upsert(imagePath);
+  if (record.indexed) {
+    return { id: record.id, skipped: true };
+  }
+
+  const vector = await embedder.embed(imagePath);
+  store.add(record.id, vector);
+  db.markIndexed(record.id);
+
+  return { id: record.id, skipped: false };
+}

package/src/metadataDb.ts

@@ -0,0 +1,87 @@
+import Database from 'better-sqlite3';
+import { ImageRecord } from './types.js';
+
+export class MetadataDb {
+  private db: Database.Database;
+
+  constructor(path: string) {
+    this.db = new Database(path);
+    this.db.pragma('journal_mode = WAL');
+    this.migrateSchema();
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS images (
+        id       INTEGER PRIMARY KEY AUTOINCREMENT,
+        path     TEXT NOT NULL UNIQUE,
+        indexed  INTEGER NOT NULL DEFAULT 0,
+        created  INTEGER NOT NULL DEFAULT (strftime('%s','now'))
+      )
+    `);
+  }
+
+  upsert(path: string): { id: number; indexed: boolean } {
+    const existing = this.findByPath(path);
+    if (existing) {
+      return { id: existing.id, indexed: existing.indexed === 1 };
+    }
+
+    const result = this.db.prepare('INSERT INTO images (path, indexed) VALUES (?, 0)').run(path);
+
+    return { id: Number(result.lastInsertRowid), indexed: false };
+  }
+
+  markIndexed(id: number): void {
+    this.db.prepare('UPDATE images SET indexed = 1 WHERE id = ?').run(id);
+  }
+
+  get(id: number): ImageRecord | undefined {
+    return this.db.prepare('SELECT * FROM images WHERE id = ?').get(id) as ImageRecord | undefined;
+  }
+
+  list(limit = 200): ImageRecord[] {
+    return this.db
+      .prepare('SELECT * FROM images WHERE indexed = 1 ORDER BY created DESC, id DESC LIMIT ?')
+      .all(limit) as ImageRecord[];
+  }
+
+  findByPath(path: string): ImageRecord | undefined {
+    return this.db.prepare('SELECT * FROM images WHERE path = ?').get(path) as ImageRecord | undefined;
+  }
+
+  private migrateSchema(): void {
+    const exists = this.db
+      .prepare(
+        "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'images'"
+      )
+      .get() as { name: string } | undefined;
+
+    if (!exists) {
+      return;
+    }
+
+    const columns = this.db
+      .prepare("PRAGMA table_info('images')")
+      .all() as Array<{ name: string }>;
+    const allowed = new Set(['id', 'path', 'indexed', 'created']);
+    const isCurrentSchema =
+      columns.length === allowed.size && columns.every((c) => allowed.has(c.name));
+
+    if (isCurrentSchema) {
+      return;
+    }
+
+    this.db.exec(`
+      BEGIN;
+      CREATE TABLE images_new (
+        id       INTEGER PRIMARY KEY AUTOINCREMENT,
+        path     TEXT NOT NULL UNIQUE,
+        indexed  INTEGER NOT NULL DEFAULT 0,
+        created  INTEGER NOT NULL DEFAULT (strftime('%s','now'))
+      );
+      INSERT INTO images_new (id, path, indexed, created)
+      SELECT id, path, indexed, created FROM images;
+      DROP TABLE images;
+      ALTER TABLE images_new RENAME TO images;
+      COMMIT;
+    `);
+  }
+}