zubo 0.1.0

package/src/memory/engine.ts

@@ -0,0 +1,170 @@
+import { Database } from "bun:sqlite";
+import { readAllMemoryFiles, writeMemory } from "./store";
+import { chunkText, type Chunk } from "./chunker";
+import { initEmbedder, embed, isEmbedderReady } from "./embedder";
+import { storeEmbedding } from "./vector-index";
+import { hybridSearch, type SearchResult } from "./hybrid-search";
+import { ftsSearch } from "./fts-index";
+import { logger } from "../util/logger";
+
+const DEFAULT_MAX_CHUNKS = 10_000;
+
+/**
+ * Initialize the memory engine: load embedder, index existing files.
+ */
+export async function initMemory(db: Database): Promise<void> {
+  // Try to initialize the embedder (non-blocking if it fails)
+  const embedderOk = await initEmbedder();
+  if (!embedderOk) {
+    logger.warn("Running memory with FTS-only (no vector embeddings)");
+  }
+
+  // Index existing memory files
+  await indexAllFiles(db);
+}
+
+/**
+ * Index all memory files: chunk → store in DB → embed.
+ */
+export async function indexAllFiles(db: Database): Promise<number> {
+  const files = readAllMemoryFiles();
+  let totalChunks = 0;
+
+  for (const file of files) {
+    const chunks = chunkText(file.content, file.filePath);
+    for (const chunk of chunks) {
+      const existing = db
+        .query(
+          "SELECT id FROM memory_chunks WHERE source_file = ? AND chunk_index = ?"
+        )
+        .get(chunk.sourceFile, chunk.index) as { id: number } | null;
+
+      let chunkId: number;
+      if (existing) {
+        // Update existing chunk
+        db.prepare(
+          "UPDATE memory_chunks SET content = ? WHERE id = ?"
+        ).run(chunk.content, existing.id);
+        chunkId = existing.id;
+      } else {
+        // Insert new chunk
+        const result = db
+          .prepare(
+            "INSERT INTO memory_chunks (source_file, chunk_index, content) VALUES (?, ?, ?)"
+          )
+          .run(chunk.sourceFile, chunk.index, chunk.content);
+        chunkId = Number(result.lastInsertRowid);
+      }
+
+      // Generate and store embedding
+      if (isEmbedderReady()) {
+        const embedding = await embed(chunk.content);
+        if (embedding) {
+          await storeEmbedding(db, chunkId, embedding);
+        }
+      }
+
+      totalChunks++;
+    }
+  }
+
+  logger.info("Memory indexed", { files: files.length, chunks: totalChunks });
+  return totalChunks;
+}
+
+/**
+ * Write a memory and index it immediately.
+ */
+export async function writeAndIndexMemory(
+  db: Database,
+  content: string
+): Promise<void> {
+  const filePath = writeMemory(content);
+
+  // Atomically get next index and insert in a transaction
+  const chunkId = db.transaction(() => {
+    const existing = db
+      .query("SELECT MAX(chunk_index) as maxIdx FROM memory_chunks WHERE source_file = ?")
+      .get(filePath) as { maxIdx: number | null } | null;
+
+    const nextIdx = (existing?.maxIdx ?? -1) + 1;
+
+    const result = db
+      .prepare(
+        "INSERT INTO memory_chunks (source_file, chunk_index, content) VALUES (?, ?, ?)"
+      )
+      .run(filePath, nextIdx, content);
+
+    return Number(result.lastInsertRowid);
+  })();
+
+  // Embed outside transaction (async not allowed inside)
+  if (isEmbedderReady()) {
+    const embedding = await embed(content);
+    if (embedding) {
+      await storeEmbedding(db, chunkId, embedding);
+    }
+  }
+
+  logger.info("Memory written and indexed", { filePath, content: content.slice(0, 50) });
+
+  // Prune if over limit
+  pruneOldChunks(db);
+}
+
+/**
+ * Remove oldest memory chunks when the total exceeds the max limit.
+ */
+export function pruneOldChunks(db: Database, maxChunks: number = DEFAULT_MAX_CHUNKS): number {
+  try {
+    const row = db.query("SELECT COUNT(*) as c FROM memory_chunks").get() as { c: number } | null;
+    const count = row?.c ?? 0;
+    if (count <= maxChunks) return 0;
+
+    const excess = count - maxChunks;
+    db.prepare(
+      "DELETE FROM memory_chunks WHERE id IN (SELECT id FROM memory_chunks ORDER BY id ASC LIMIT ?)"
+    ).run(excess);
+    logger.info(`Pruned ${excess} old memory chunks (was ${count}, max ${maxChunks})`);
+    return excess;
+  } catch (err: any) {
+    logger.warn("Memory pruning failed", { error: err.message });
+    return 0;
+  }
+}
+
+/**
+ * Get memory statistics.
+ */
+export function getMemoryStats(db: Database): { totalChunks: number; oldestDate: string | null } {
+  try {
+    const row = db.query(
+      "SELECT COUNT(*) as c, MIN(rowid) as oldest FROM memory_chunks"
+    ).get() as { c: number; oldest: number | null } | null;
+    return { totalChunks: row?.c ?? 0, oldestDate: null };
+  } catch {
+    return { totalChunks: 0, oldestDate: null };
+  }
+}
+
+/**
+ * Search memory using FTS (synchronous, for use in router).
+ */
+export function searchMemory(
+  db: Database,
+  query: string,
+  limit: number = 5
+): SearchResult[] {
+  return ftsSearch(db, query, limit);
+}
+
+/**
+ * Async hybrid search (vector + FTS).
+ */
+export async function searchMemoryAsync(
+  db: Database,
+  query: string,
+  limit: number = 5
+): Promise<SearchResult[]> {
+  return hybridSearch(db, query, limit);
+}

package/src/memory/fts-index.ts

@@ -0,0 +1,55 @@
+import { Database } from "bun:sqlite";
+
+export interface FtsResult {
+  id: number;
+  content: string;
+  sourceFile: string;
+  score: number;
+}
+
+/**
+ * Full-text search using SQLite FTS5 with BM25 ranking.
+ */
+export function ftsSearch(
+  db: Database,
+  query: string,
+  limit: number = 5
+): FtsResult[] {
+  // Escape FTS5 special characters
+  const escaped = query.replace(/['"*()]/g, "");
+  if (!escaped.trim()) return [];
+
+  // Split into terms and join with OR for broader matching
+  const terms = escaped.split(/\s+/).filter(Boolean);
+  const ftsQuery = terms.join(" OR ");
+
+  try {
+    const rows = db
+      .query(
+        `SELECT mc.id, mc.content, mc.source_file,
+                rank as score
+         FROM memory_fts
+         JOIN memory_chunks mc ON mc.id = memory_fts.rowid
+         WHERE memory_fts MATCH ?
+         ORDER BY rank
+         LIMIT ?`
+      )
+      .all(ftsQuery, limit) as Array<{
+      id: number;
+      content: string;
+      source_file: string;
+      score: number;
+    }>;
+
+    // Normalize FTS scores (BM25 returns negative values, lower = better)
+    const maxScore = rows.length > 0 ? Math.abs(rows[0].score) : 1;
+    return rows.map((row) => ({
+      id: row.id,
+      content: row.content,
+      sourceFile: row.source_file,
+      score: maxScore > 0 ? Math.abs(row.score) / maxScore : 0,
+    }));
+  } catch {
+    return [];
+  }
+}

package/src/memory/hybrid-search.ts

@@ -0,0 +1,72 @@
+import { Database } from "bun:sqlite";
+import { vectorSearch, type VectorResult } from "./vector-index";
+import { ftsSearch, type FtsResult } from "./fts-index";
+import { isEmbedderReady } from "./embedder";
+
+const VECTOR_WEIGHT = 0.6;
+const FTS_WEIGHT = 0.4;
+
+export interface SearchResult {
+  id: number;
+  content: string;
+  sourceFile: string;
+  score: number;
+}
+
+/**
+ * Hybrid search: weighted union of vector similarity and BM25 keyword search.
+ * Falls back to FTS-only when embedder is not available.
+ */
+export async function hybridSearch(
+  db: Database,
+  query: string,
+  limit: number = 5
+): Promise<SearchResult[]> {
+  const scoreMap = new Map<
+    number,
+    { content: string; sourceFile: string; score: number }
+  >();
+
+  // FTS search (always available)
+  const ftsResults = ftsSearch(db, query, limit * 2);
+  for (const r of ftsResults) {
+    const existing = scoreMap.get(r.id);
+    const ftsWeight = isEmbedderReady() ? FTS_WEIGHT : 1.0;
+    const score = r.score * ftsWeight;
+    if (existing) {
+      existing.score += score;
+    } else {
+      scoreMap.set(r.id, {
+        content: r.content,
+        sourceFile: r.sourceFile,
+        score,
+      });
+    }
+  }
+
+  // Vector search (when embedder is ready)
+  if (isEmbedderReady()) {
+    const vecResults = await vectorSearch(db, query, limit * 2);
+    for (const r of vecResults) {
+      const existing = scoreMap.get(r.id);
+      const score = r.score * VECTOR_WEIGHT;
+      if (existing) {
+        existing.score += score;
+      } else {
+        scoreMap.set(r.id, {
+          content: r.content,
+          sourceFile: r.sourceFile,
+          score,
+        });
+      }
+    }
+  }
+
+  // Sort by combined score
+  const results = Array.from(scoreMap.entries())
+    .map(([id, data]) => ({ id, ...data }))
+    .sort((a, b) => b.score - a.score)
+    .slice(0, limit);
+
+  return results;
+}

package/src/memory/store.ts

@@ -0,0 +1,56 @@
+import { paths } from "../config/paths";
+import { join } from "path";
+import { existsSync, readdirSync, readFileSync, appendFileSync } from "fs";
+
+/**
+ * Read all markdown memory files from the workspace.
+ */
+export function readAllMemoryFiles(): Array<{
+  filePath: string;
+  content: string;
+}> {
+  const files: Array<{ filePath: string; content: string }> = [];
+
+  // Read MEMORY.md
+  if (existsSync(paths.memoryFile)) {
+    files.push({
+      filePath: paths.memoryFile,
+      content: readFileSync(paths.memoryFile, "utf-8"),
+    });
+  }
+
+  // Read dated memory files
+  if (existsSync(paths.memory)) {
+    const memFiles = readdirSync(paths.memory).filter((f) =>
+      f.endsWith(".md")
+    );
+    for (const f of memFiles) {
+      const fp = join(paths.memory, f);
+      files.push({
+        filePath: fp,
+        content: readFileSync(fp, "utf-8"),
+      });
+    }
+  }
+
+  return files;
+}
+
+/**
+ * Append a memory entry to today's dated memory file.
+ */
+export function writeMemory(content: string): string {
+  const today = new Date().toISOString().split("T")[0];
+  const filePath = join(paths.memory, `${today}.md`);
+
+  if (!existsSync(filePath)) {
+    Bun.spawnSync(["mkdir", "-p", paths.memory]);
+    const header = `# Memories — ${today}\n\n`;
+    appendFileSync(filePath, header);
+  }
+
+  const entry = `- ${content}\n`;
+  appendFileSync(filePath, entry);
+
+  return filePath;
+}

package/src/memory/vector-index.ts

@@ -0,0 +1,72 @@
+import { Database } from "bun:sqlite";
+import { embed, cosineSimilarity, isEmbedderReady } from "./embedder";
+import { logger } from "../util/logger";
+
+export interface VectorResult {
+  id: number;
+  content: string;
+  sourceFile: string;
+  score: number;
+}
+
+/**
+ * Store embedding as blob in memory_chunks table.
+ */
+export async function storeEmbedding(
+  db: Database,
+  chunkId: number,
+  embedding: Float32Array
+) {
+  const blob = Buffer.from(embedding.buffer);
+  db.prepare("UPDATE memory_chunks SET embedding = ? WHERE id = ?").run(
+    blob,
+    chunkId
+  );
+}
+
+/**
+ * Vector search using cosine similarity computed in JS.
+ * Since sqlite-vec can't load, we pull all embeddings and compute in memory.
+ * This is fine for personal memory (<10k chunks).
+ */
+export async function vectorSearch(
+  db: Database,
+  query: string,
+  limit: number = 5
+): Promise<VectorResult[]> {
+  if (!isEmbedderReady()) return [];
+
+  const queryEmbedding = await embed(query);
+  if (!queryEmbedding) return [];
+
+  const rows = db
+    .query(
+      "SELECT id, content, source_file, embedding FROM memory_chunks WHERE embedding IS NOT NULL ORDER BY id DESC LIMIT 5000"
+    )
+    .all() as Array<{
+    id: number;
+    content: string;
+    source_file: string;
+    embedding: Buffer;
+  }>;
+
+  const scored: VectorResult[] = [];
+  for (const row of rows) {
+    if (!row.embedding) continue;
+    const emb = new Float32Array(
+      row.embedding.buffer,
+      row.embedding.byteOffset,
+      row.embedding.byteLength / 4
+    );
+    const score = cosineSimilarity(queryEmbedding, emb);
+    scored.push({
+      id: row.id,
+      content: row.content,
+      sourceFile: row.source_file,
+      score,
+    });
+  }
+
+  scored.sort((a, b) => b.score - a.score);
+  return scored.slice(0, limit);
+}

package/src/model.ts

@@ -0,0 +1,118 @@
+import { existsSync, readFileSync, writeFileSync } from "fs";
+import { paths } from "./config/paths";
+import { configExists } from "./config/loader";
+
+function readConfig(): any {
+  if (!configExists()) {
+    console.log("Config not found. Run 'zubo setup' first.");
+    process.exit(1);
+  }
+  return JSON.parse(readFileSync(paths.config, "utf-8"));
+}
+
+function writeConfig(config: any) {
+  writeFileSync(paths.config, JSON.stringify(config, null, 2) + "\n");
+}
+
+function showCurrent() {
+  const config = readConfig();
+
+  if (config.providers && config.activeProvider) {
+    const active = config.providers[config.activeProvider];
+    console.log(`\n  Active: ${config.activeProvider}/${active?.model ?? "?"}`);
+
+    if (config.failover?.length) {
+      const fallbacks = config.failover
+        .map((name: string) => {
+          const p = config.providers?.[name];
+          return p ? `${name}/${p.model}` : `${name} (not configured)`;
+        })
+        .join(", ");
+      console.log(`  Failover: ${fallbacks}`);
+    }
+  } else if (config.anthropicApiKey) {
+    console.log(`\n  Active: anthropic/${config.model ?? "claude-sonnet-4-5-20250929"} (legacy config)`);
+  } else {
+    console.log("\n  No provider configured.");
+  }
+  console.log("");
+}
+
+function listProviders() {
+  const config = readConfig();
+
+  console.log("\n  Configured Providers\n");
+
+  if (config.providers) {
+    for (const [name, p] of Object.entries(config.providers) as [string, any][]) {
+      const active = name === config.activeProvider ? " ← active" : "";
+      const failover = config.failover?.includes(name) ? " (failover)" : "";
+      const baseUrl = p.baseUrl ? ` @ ${p.baseUrl}` : "";
+      console.log(`  ${name}/${p.model}${baseUrl}${active}${failover}`);
+    }
+  } else if (config.anthropicApiKey) {
+    console.log(`  anthropic/${config.model ?? "claude-sonnet-4-5-20250929"} ← active (legacy)`);
+  } else {
+    console.log("  None configured.");
+  }
+  console.log("");
+}
+
+function switchModel(target: string) {
+  const config = readConfig();
+
+  // Parse "provider/model" or just "provider"
+  const slashIdx = target.indexOf("/");
+  let providerName: string;
+  let modelName: string | null;
+
+  if (slashIdx > 0) {
+    providerName = target.substring(0, slashIdx);
+    modelName = target.substring(slashIdx + 1);
+  } else {
+    providerName = target;
+    modelName = null;
+  }
+
+  // Ensure providers object exists
+  if (!config.providers) {
+    config.providers = {};
+  }
+
+  // Check if this provider is already configured
+  if (config.providers[providerName]) {
+    config.activeProvider = providerName;
+    if (modelName) {
+      config.providers[providerName].model = modelName;
+    }
+    // Keep legacy field in sync
+    config.model = config.providers[providerName].model;
+
+    writeConfig(config);
+    console.log(`Switched to ${providerName}/${config.providers[providerName].model}`);
+    console.log("Restart Zubo for changes to take effect.");
+    return;
+  }
+
+  // Provider not configured — offer quick setup for known providers
+  console.log(`Provider "${providerName}" is not configured.`);
+  console.log(`Run 'zubo setup' to add it, or edit ~/.zubo/config.json directly.`);
+  process.exit(1);
+}
+
+export function runModelCommand(args: string[]) {
+  if (args.includes("--list") || args.includes("-l")) {
+    listProviders();
+    return;
+  }
+
+  // If a target is given, switch to it
+  const target = args.find((a) => !a.startsWith("-"));
+  if (target) {
+    switchModel(target);
+    return;
+  }
+
+  // Default: show current
+  showCurrent();
+}

package/src/registry/cli.ts

@@ -0,0 +1,43 @@
+import { searchRegistry, fetchRegistry } from "./client";
+import { installFromRegistry } from "./installer";
+
+export async function handleRegistrySearch(query: string): Promise<void> {
+  console.log(`Searching registry for "${query}"...`);
+  const results = await searchRegistry(query);
+  if (results.length === 0) {
+    console.log("No skills found matching your query.");
+    return;
+  }
+  console.log(`\nFound ${results.length} skill(s):\n`);
+  for (const entry of results) {
+    console.log(`  ${entry.name} — ${entry.description}`);
+    console.log(`    repo: ${entry.repo}  tags: ${entry.tags.join(", ")}`);
+    if (entry.secrets?.length) {
+      console.log(`    requires: ${entry.secrets.join(", ")}`);
+    }
+    console.log();
+  }
+}
+
+export async function handleRegistryInstall(name: string): Promise<void> {
+  console.log(`Installing skill "${name}" from registry...`);
+  const result = await installFromRegistry(name);
+  if (result.success) {
+    console.log(`Installed "${name}" to ${result.path}`);
+    if (result.requiredSecrets?.length) {
+      console.log(`\nThis skill requires secrets: ${result.requiredSecrets.join(", ")}`);
+      console.log("Use 'zubo start' and tell Zubo to set them, or use the dashboard.");
+    }
+  } else {
+    console.error(`Failed: ${result.error}`);
+  }
+}
+
+export async function handleRegistryPublish(name: string): Promise<void> {
+  console.log(`Publishing skills is done via Pull Request to the zubo-skills/registry repo.`);
+  console.log(`\nSteps:`);
+  console.log(`1. Create a GitHub repo with your skill (SKILL.md + handler.ts)`);
+  console.log(`2. Fork https://github.com/zubo-skills/registry`);
+  console.log(`3. Add your skill entry to registry.json`);
+  console.log(`4. Submit a Pull Request`);
+}

package/src/registry/client.ts

@@ -0,0 +1,54 @@
+import { logger } from "../util/logger";
+
+const REGISTRY_URL = "https://raw.githubusercontent.com/zubo-skills/registry/main/registry.json";
+const CACHE_TTL_MS = 5 * 60_000; // 5 minutes
+
+export interface RegistryEntry {
+  name: string;
+  description: string;
+  repo: string;
+  author: string;
+  version: string;
+  tags: string[];
+  secrets?: string[];
+}
+
+let cachedRegistry: RegistryEntry[] | null = null;
+let cacheTime = 0;
+
+export async function fetchRegistry(): Promise<RegistryEntry[]> {
+  const now = Date.now();
+  if (cachedRegistry && now - cacheTime < CACHE_TTL_MS) {
+    return cachedRegistry;
+  }
+
+  try {
+    const res = await fetch(REGISTRY_URL);
+    if (!res.ok) {
+      throw new Error(`Registry fetch failed: ${res.status}`);
+    }
+    const data = (await res.json()) as { skills: RegistryEntry[] };
+    cachedRegistry = data.skills ?? [];
+    cacheTime = now;
+    return cachedRegistry;
+  } catch (err: any) {
+    logger.warn("Failed to fetch skill registry", { error: err.message });
+    return cachedRegistry ?? [];
+  }
+}
+
+export async function searchRegistry(query: string): Promise<RegistryEntry[]> {
+  const registry = await fetchRegistry();
+  const q = query.toLowerCase();
+  return registry.filter(
+    (e) =>
+      e.name.toLowerCase().includes(q) ||
+      e.description.toLowerCase().includes(q) ||
+      e.tags.some((t) => t.toLowerCase().includes(q))
+  );
+}
+
+export async function getRegistryEntry(name: string): Promise<RegistryEntry | null> {
+  const registry = await fetchRegistry();
+  return registry.find((e) => e.name === name) ?? null;
+}