zubo 0.1.0

package/src/llm/smart-router.ts

@@ -0,0 +1,241 @@
+import type {
+  LlmProvider,
+  LlmRequest,
+  LlmResponse,
+  LlmStreamEvent,
+} from "./provider";
+import { logger } from "../util/logger";
+
+const CODE_MARKERS = [
+  "```",
+  "function ",
+  "const ",
+  "let ",
+  "var ",
+  "import ",
+  "export ",
+  "class ",
+  "interface ",
+  "=>",
+  "async ",
+  "await ",
+  "return ",
+  ".ts",
+  ".js",
+  ".py",
+  ".tsx",
+  ".jsx",
+  "/src/",
+  "/lib/",
+  "/bin/",
+  "node_modules",
+];
+
+const MULTI_STEP_INDICATORS = [
+  "step by step",
+  "analyze",
+  "compare",
+  "implement",
+  "refactor",
+  "debug",
+  "build",
+  "create a",
+  "write a",
+  "design",
+  "architect",
+  "optimize",
+  "migrate",
+  "convert",
+  "transform",
+];
+
+const REASONING_INDICATORS = [
+  "why",
+  "how does",
+  "explain in detail",
+  "trade-offs",
+  "tradeoffs",
+  "pros and cons",
+  "what are the differences",
+  "elaborate",
+  "break down",
+  "walk me through",
+  "reasoning",
+  "implications",
+];
+
+/**
+ * Classify whether a user message is "simple" (can be handled by a fast/cheap model)
+ * or "complex" (needs the primary/expensive model).
+ */
+export function classifyComplexity(text: string): "simple" | "complex" {
+  const lower = text.toLowerCase().trim();
+  const words = lower.split(/\s+/).filter(Boolean);
+  const wordCount = words.length;
+
+  // Long messages are complex
+  if (wordCount >= 50) {
+    return "complex";
+  }
+
+  // Check for code markers
+  for (const marker of CODE_MARKERS) {
+    if (lower.includes(marker)) {
+      return "complex";
+    }
+  }
+
+  // Check for multi-step indicators
+  for (const indicator of MULTI_STEP_INDICATORS) {
+    if (lower.includes(indicator)) {
+      return "complex";
+    }
+  }
+
+  // Check for reasoning indicators
+  for (const indicator of REASONING_INDICATORS) {
+    if (lower.includes(indicator)) {
+      return "complex";
+    }
+  }
+
+  // Short messages with no complexity markers are simple
+  return "simple";
+}
+
+export class SmartRouterProvider implements LlmProvider {
+  providerName: string;
+  model: string;
+  contextWindow: number;
+
+  constructor(
+    private primary: LlmProvider,
+    private fast: LlmProvider,
+    private enabled: boolean,
+  ) {
+    this.providerName = primary.providerName;
+    this.model = primary.model;
+    this.contextWindow = primary.contextWindow;
+  }
+
+  private selectProvider(request: LlmRequest): LlmProvider {
+    if (!this.enabled) {
+      return this.primary;
+    }
+
+    // Extract the last user message text for classification
+    const lastUserMsg = [...request.messages]
+      .reverse()
+      .find((m) => m.role === "user");
+
+    if (!lastUserMsg) {
+      return this.primary;
+    }
+
+    const text = typeof lastUserMsg.content === "string"
+      ? lastUserMsg.content
+      : lastUserMsg.content
+          .filter((b) => b.type === "text")
+          .map((b) => b.text ?? "")
+          .join(" ");
+
+    const complexity = classifyComplexity(text);
+
+    if (complexity === "simple") {
+      logger.info("Smart router: using fast model", {
+        provider: this.fast.providerName,
+        model: this.fast.model,
+        reason: "simple query",
+      });
+      this.providerName = this.fast.providerName;
+      this.model = this.fast.model;
+      return this.fast;
+    }
+
+    logger.info("Smart router: using primary model", {
+      provider: this.primary.providerName,
+      model: this.primary.model,
+      reason: "complex query",
+    });
+    this.providerName = this.primary.providerName;
+    this.model = this.primary.model;
+    return this.primary;
+  }
+
+  async chat(request: LlmRequest): Promise<LlmResponse> {
+    const provider = this.selectProvider(request);
+
+    if (provider === this.fast) {
+      try {
+        return await provider.chat(request);
+      } catch (err: any) {
+        logger.warn("Fast model failed, falling back to primary", {
+          error: err.message,
+        });
+        this.providerName = this.primary.providerName;
+        this.model = this.primary.model;
+        return this.primary.chat(request);
+      }
+    }
+
+    return provider.chat(request);
+  }
+
+  async *chatStream(request: LlmRequest): AsyncIterable<LlmStreamEvent> {
+    const MAX_STREAM_EVENTS = 50_000;
+    const provider = this.selectProvider(request);
+
+    if (provider === this.fast) {
+      // Try fast model with fallback to primary
+      if (provider.chatStream) {
+        const events: LlmStreamEvent[] = [];
+        let succeeded = false;
+        try {
+          for await (const event of provider.chatStream(request)) {
+            if (events.length >= MAX_STREAM_EVENTS) {
+              throw new Error(`Stream exceeded maximum event limit (${MAX_STREAM_EVENTS})`);
+            }
+            events.push(event);
+          }
+          succeeded = true;
+        } catch (err: any) {
+          logger.warn("Fast model stream failed, falling back to primary", {
+            error: err.message,
+          });
+        }
+
+        if (succeeded) {
+          for (const event of events) {
+            yield event;
+          }
+          return;
+        }
+
+        // Fallback to primary stream
+        this.providerName = this.primary.providerName;
+        this.model = this.primary.model;
+      } else {
+        // Fast model has no streaming, fall back to primary
+        logger.info("Fast model has no streaming support, using primary");
+        this.providerName = this.primary.providerName;
+        this.model = this.primary.model;
+      }
+    }
+
+    // Use primary model (streaming or non-streaming fallback)
+    if (this.primary.chatStream) {
+      yield* this.primary.chatStream(request);
+    } else {
+      const response = await this.primary.chat(request);
+      for (const block of response.content) {
+        if (block.type === "text" && block.text) {
+          yield { type: "text_delta", text: block.text };
+        } else if (block.type === "tool_use") {
+          yield { type: "tool_use_start", id: block.id!, name: block.name! };
+          yield { type: "tool_use_end", id: block.id! };
+        }
+      }
+      yield { type: "message_done", response };
+    }
+  }
+}

package/src/logs.ts

@@ -0,0 +1,53 @@
+import { existsSync, readFileSync, watchFile } from "fs";
+import { paths } from "./config/paths";
+import { logger } from "./util/logger";
+
+export async function showLogs(follow = false) {
+  if (!existsSync(paths.logFile)) {
+    console.log("No log file found. Start Zubo first.");
+    return;
+  }
+
+  if (follow) {
+    await tailFollow();
+  } else {
+    tailLast(50);
+  }
+}
+
+function tailLast(n: number) {
+  const content = readFileSync(paths.logFile, "utf-8");
+  const lines = content.trimEnd().split("\n");
+  const tail = lines.slice(-n);
+  console.log(tail.join("\n"));
+}
+
+async function tailFollow() {
+  // Print last 10 lines first
+  tailLast(10);
+  console.log("--- following logs (Ctrl+C to stop) ---\n");
+
+  let pos = readFileSync(paths.logFile).byteLength;
+
+  watchFile(paths.logFile, { interval: 500 }, () => {
+    try {
+      const buf = readFileSync(paths.logFile);
+      if (buf.byteLength > pos) {
+        const newData = buf.subarray(pos).toString();
+        process.stdout.write(newData);
+        pos = buf.byteLength;
+      } else if (buf.byteLength < pos) {
+        // File was truncated/rotated
+        pos = 0;
+        const newData = buf.toString();
+        process.stdout.write(newData);
+        pos = buf.byteLength;
+      }
+    } catch (err: any) {
+      logger.warn("Failed to read log file update", { error: (err as Error).message });
+    }
+  });
+
+  // Keep process alive
+  await new Promise(() => {});
+}

package/src/memory/chunker.ts

@@ -0,0 +1,58 @@
+import { estimateTokens } from "../util/tokens";
+
+const CHUNK_SIZE = 400; // tokens
+const CHUNK_OVERLAP = 80; // tokens
+const CHARS_PER_TOKEN = 4;
+
+export interface Chunk {
+  content: string;
+  index: number;
+  sourceFile: string;
+}
+
+/**
+ * Split text into overlapping chunks of ~400 tokens.
+ */
+export function chunkText(text: string, sourceFile: string): Chunk[] {
+  const chunks: Chunk[] = [];
+  const chunkChars = CHUNK_SIZE * CHARS_PER_TOKEN;
+  const overlapChars = CHUNK_OVERLAP * CHARS_PER_TOKEN;
+  const step = chunkChars - overlapChars;
+
+  if (text.length <= chunkChars) {
+    return [{ content: text.trim(), index: 0, sourceFile }];
+  }
+
+  let offset = 0;
+  let idx = 0;
+  while (offset < text.length) {
+    let end = offset + chunkChars;
+    if (end > text.length) end = text.length;
+
+    // Try to break at a paragraph or sentence boundary
+    if (end < text.length) {
+      const slice = text.slice(offset, end);
+      const lastPara = slice.lastIndexOf("\n\n");
+      const lastNewline = slice.lastIndexOf("\n");
+      const lastPeriod = slice.lastIndexOf(". ");
+
+      if (lastPara > chunkChars * 0.5) {
+        end = offset + lastPara + 2;
+      } else if (lastNewline > chunkChars * 0.5) {
+        end = offset + lastNewline + 1;
+      } else if (lastPeriod > chunkChars * 0.5) {
+        end = offset + lastPeriod + 2;
+      }
+    }
+
+    const chunk = text.slice(offset, end).trim();
+    if (chunk.length > 0) {
+      chunks.push({ content: chunk, index: idx++, sourceFile });
+    }
+
+    offset += step;
+    if (offset >= text.length) break;
+  }
+
+  return chunks;
+}

package/src/memory/document-parser.ts

@@ -0,0 +1,115 @@
+import { readFileSync } from "fs";
+import { extname } from "path";
+import { logger } from "../util/logger";
+
+export interface ParsedDocument {
+  text: string;
+  metadata: {
+    filename: string;
+    mimeType: string;
+    pages?: number;
+    wordCount: number;
+  };
+}
+
+export async function parseDocument(
+  filePath: string,
+  mimeType: string
+): Promise<ParsedDocument> {
+  const filename = filePath.split("/").pop() ?? "unknown";
+
+  switch (mimeType) {
+    case "text/plain":
+    case "text/markdown":
+    case "text/csv": {
+      const text = readFileSync(filePath, "utf-8");
+      return {
+        text,
+        metadata: { filename, mimeType, wordCount: countWords(text) },
+      };
+    }
+
+    case "application/pdf": {
+      try {
+        const pdfParse = (await import("pdf-parse")).default;
+        const buffer = readFileSync(filePath);
+        const MAX_TEXT_LENGTH = 5_000_000; // 5MB of text
+        const TIMEOUT_MS = 30_000;
+        const data = await Promise.race([
+          pdfParse(buffer) as Promise<{ text: string; numpages: number }>,
+          new Promise<never>((_, reject) => setTimeout(() => reject(new Error("PDF parsing timeout")), TIMEOUT_MS)),
+        ]);
+        const text = data.text.length > MAX_TEXT_LENGTH ? data.text.slice(0, MAX_TEXT_LENGTH) + "\n[Truncated]" : data.text;
+        return {
+          text,
+          metadata: {
+            filename,
+            mimeType,
+            pages: data.numpages,
+            wordCount: countWords(text),
+          },
+        };
+      } catch (err: any) {
+        logger.warn("PDF parsing failed — install pdf-parse for PDF support", { error: err.message });
+        return {
+          text: `[PDF file: ${filename} — install pdf-parse for content extraction]`,
+          metadata: { filename, mimeType, wordCount: 0 },
+        };
+      }
+    }
+
+    case "application/vnd.openxmlformats-officedocument.wordprocessingml.document": {
+      try {
+        const mammoth = await import("mammoth");
+        const result = await mammoth.extractRawText({ buffer: readFileSync(filePath) });
+        return {
+          text: result.value,
+          metadata: { filename, mimeType, wordCount: countWords(result.value) },
+        };
+      } catch (err: any) {
+        logger.warn("DOCX parsing failed — install mammoth for DOCX support", { error: err.message });
+        return {
+          text: `[DOCX file: ${filename} — install mammoth for content extraction]`,
+          metadata: { filename, mimeType, wordCount: 0 },
+        };
+      }
+    }
+
+    default: {
+      // Try to read as text
+      const ext = extname(filePath).toLowerCase();
+      if ([".txt", ".md", ".csv", ".json", ".xml", ".yaml", ".yml", ".ts", ".js", ".py", ".sh"].includes(ext)) {
+        const text = readFileSync(filePath, "utf-8");
+        return {
+          text,
+          metadata: { filename, mimeType, wordCount: countWords(text) },
+        };
+      }
+      return {
+        text: `[Unsupported file type: ${mimeType}]`,
+        metadata: { filename, mimeType, wordCount: 0 },
+      };
+    }
+  }
+}
+
+function countWords(text: string): number {
+  return text.split(/\s+/).filter(Boolean).length;
+}
+
+const MIME_MAP: Record<string, string> = {
+  ".txt": "text/plain",
+  ".md": "text/markdown",
+  ".csv": "text/csv",
+  ".pdf": "application/pdf",
+  ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+  ".json": "application/json",
+  ".xml": "application/xml",
+  ".yaml": "text/yaml",
+  ".yml": "text/yaml",
+};
+
+export function guessMimeType(filename: string): string {
+  const ext = extname(filename).toLowerCase();
+  return MIME_MAP[ext] ?? "application/octet-stream";
+}

package/src/memory/embedder.ts

@@ -0,0 +1,235 @@
+import { InferenceSession, Tensor } from "onnxruntime-node";
+import { paths } from "../config/paths";
+import { join } from "path";
+import { existsSync } from "fs";
+import { logger } from "../util/logger";
+
+const MODEL_NAME = "all-MiniLM-L6-v2";
+const MODEL_URL = `https://huggingface.co/sentence-transformers/${MODEL_NAME}/resolve/main/onnx/model.onnx`;
+const TOKENIZER_URL = `https://huggingface.co/sentence-transformers/${MODEL_NAME}/resolve/main/tokenizer.json`;
+const EMBEDDING_DIM = 384;
+
+let session: InferenceSession | null = null;
+let vocab: Map<string, number> | null = null;
+
+async function downloadFile(url: string, dest: string) {
+  if (existsSync(dest)) return;
+  logger.info(`Downloading ${url}...`);
+  const resp = await fetch(url, { redirect: "follow" });
+  if (!resp.ok) throw new Error(`Failed to download ${url}: ${resp.status}`);
+  const contentLength = resp.headers.get("content-length");
+  if (contentLength) {
+    logger.info(`Download size: ${(Number(contentLength) / 1024 / 1024).toFixed(1)} MB`);
+  }
+  const buffer = await resp.arrayBuffer();
+  await Bun.write(dest, buffer);
+  logger.info(`Saved to ${dest} (${(buffer.byteLength / 1024 / 1024).toFixed(1)} MB)`);
+}
+
+async function ensureModel(): Promise<string> {
+  const modelDir = join(paths.models, MODEL_NAME);
+  Bun.spawnSync(["mkdir", "-p", modelDir]);
+
+  const modelPath = join(modelDir, "model.onnx");
+  const tokenizerPath = join(modelDir, "tokenizer.json");
+
+  await Promise.all([
+    downloadFile(MODEL_URL, modelPath),
+    downloadFile(TOKENIZER_URL, tokenizerPath),
+  ]);
+
+  return modelDir;
+}
+
+/**
+ * WordPiece tokenization matching BERT/MiniLM expectations.
+ * 1. Lowercase + strip accents
+ * 2. Split on whitespace and punctuation
+ * 3. For each word, greedily match longest vocab prefix, then continue with ## prefixed subwords
+ */
+function wordPieceTokenize(
+  text: string,
+  vocabMap: Map<string, number>,
+  maxLen: number = 128
+): { inputIds: number[]; attentionMask: number[] } {
+  const CLS = vocabMap.get("[CLS]") ?? 101;
+  const SEP = vocabMap.get("[SEP]") ?? 102;
+  const UNK = vocabMap.get("[UNK]") ?? 100;
+  const PAD = vocabMap.get("[PAD]") ?? 0;
+
+  // Basic pre-tokenization: lowercase, split on whitespace and punctuation
+  const normalized = text
+    .toLowerCase()
+    .normalize("NFD")
+    .replace(/[\u0300-\u036f]/g, ""); // strip accents
+
+  // Split into words, keeping punctuation as separate tokens
+  const words = normalized.match(/[a-z0-9]+|[^\s\w]/g) || [];
+
+  const tokens: number[] = [CLS];
+
+  for (const word of words) {
+    if (tokens.length >= maxLen - 1) break;
+
+    // WordPiece: greedily match longest subword from vocab
+    let start = 0;
+    let isBad = false;
+    const subTokens: number[] = [];
+
+    while (start < word.length) {
+      let end = word.length;
+      let found = false;
+
+      while (start < end) {
+        let substr = word.slice(start, end);
+        if (start > 0) substr = "##" + substr;
+
+        const id = vocabMap.get(substr);
+        if (id !== undefined) {
+          subTokens.push(id);
+          found = true;
+          break;
+        }
+        end--;
+      }
+
+      if (!found) {
+        isBad = true;
+        break;
+      }
+      start = end;
+    }
+
+    if (isBad) {
+      tokens.push(UNK);
+    } else {
+      for (const st of subTokens) {
+        if (tokens.length >= maxLen - 1) break;
+        tokens.push(st);
+      }
+    }
+  }
+
+  tokens.push(SEP);
+
+  const attentionMask = new Array(maxLen).fill(0);
+  for (let i = 0; i < tokens.length; i++) attentionMask[i] = 1;
+
+  while (tokens.length < maxLen) tokens.push(PAD);
+
+  return { inputIds: tokens, attentionMask };
+}
+
+async function loadVocab(modelDir: string): Promise<Map<string, number>> {
+  const tokenizerPath = join(modelDir, "tokenizer.json");
+  const raw = await Bun.file(tokenizerPath).json();
+  const map = new Map<string, number>();
+
+  if (raw.model?.vocab) {
+    for (const [token, id] of Object.entries(raw.model.vocab)) {
+      map.set(token, id as number);
+    }
+  }
+
+  return map;
+}
+
+export async function initEmbedder(): Promise<boolean> {
+  try {
+    const modelDir = await ensureModel();
+    const modelPath = join(modelDir, "model.onnx");
+
+    // Dispose existing session to prevent memory leak on re-initialization
+    if (session) {
+      try { (session as any).dispose?.(); } catch { /* ignore */ }
+    }
+
+    session = await InferenceSession.create(modelPath, {
+      executionProviders: ["cpu"],
+    });
+    vocab = await loadVocab(modelDir);
+
+    logger.info("Embedder initialized", {
+      model: MODEL_NAME,
+      dim: EMBEDDING_DIM,
+      vocabSize: vocab.size,
+    });
+    return true;
+  } catch (err: any) {
+    logger.warn("Failed to initialize embedder, falling back to FTS-only", {
+      error: err.message,
+    });
+    return false;
+  }
+}
+
+export async function embed(text: string): Promise<Float32Array | null> {
+  if (!session || !vocab) return null;
+
+  const { inputIds, attentionMask } = wordPieceTokenize(text, vocab);
+  const seqLen = inputIds.length;
+
+  const inputIdsTensor = new Tensor(
+    "int64",
+    BigInt64Array.from(inputIds.map(BigInt)),
+    [1, seqLen]
+  );
+  const attentionMaskTensor = new Tensor(
+    "int64",
+    BigInt64Array.from(attentionMask.map(BigInt)),
+    [1, seqLen]
+  );
+  const tokenTypeIds = new Tensor(
+    "int64",
+    new BigInt64Array(seqLen),
+    [1, seqLen]
+  );
+
+  const feeds: Record<string, Tensor> = {
+    input_ids: inputIdsTensor,
+    attention_mask: attentionMaskTensor,
+    token_type_ids: tokenTypeIds,
+  };
+
+  const output = await session.run(feeds);
+
+  const lastHidden = output["last_hidden_state"];
+  if (!lastHidden) return null;
+
+  const data = lastHidden.data as Float32Array;
+  const embedding = new Float32Array(EMBEDDING_DIM);
+
+  // Mean pooling over non-padding tokens
+  let count = 0;
+  for (let i = 0; i < seqLen; i++) {
+    if (attentionMask[i] === 1) {
+      for (let j = 0; j < EMBEDDING_DIM; j++) {
+        embedding[j] += data[i * EMBEDDING_DIM + j];
+      }
+      count++;
+    }
+  }
+  for (let j = 0; j < EMBEDDING_DIM; j++) {
+    embedding[j] /= count;
+  }
+
+  // L2 normalize
+  let norm = 0;
+  for (let j = 0; j < EMBEDDING_DIM; j++) norm += embedding[j] * embedding[j];
+  norm = Math.sqrt(norm);
+  if (norm > 0) {
+    for (let j = 0; j < EMBEDDING_DIM; j++) embedding[j] /= norm;
+  }
+
+  return embedding;
+}
+
+export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
+  let dot = 0;
+  for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
+  return dot; // Already L2-normalized, so dot product = cosine similarity
+}
+
+export function isEmbedderReady(): boolean {
+  return session !== null && vocab !== null;
+}