skyloom 1.4.0

package/src/core/router.ts

@@ -0,0 +1,124 @@
+/**
+ * Complexity router: classify a user goal into direct / single / orchestrate.
+ *
+ * Rules-first (no LLM), so classification stays under 1ms. The router exists
+ * solely to keep simple goals from triggering Snow's full task-decomposition
+ * LLM call when a single agent could answer in one shot.
+ */
+
+export type Mode = 'direct' | 'single' | 'orchestrate';
+
+const MULTI_STEP_TOKENS = [
+  '先', '再', '然后', '接着', '之后', '其次', '最后',
+  '第一步', '第二步', '第三步', '步骤', '顺序', '依次', '首先',
+  'step 1', 'step 2', 'first,', 'then,', 'after that', 'finally',
+];
+
+const GREETING_TOKENS = [
+  '你好', '您好', 'hi', 'hello', 'hey', '在吗', '嗨',
+  '早上好', '晚安', '谢谢', 'thanks', 'thank you', '再见', 'bye',
+];
+
+const SINGLE_ACTION_HINTS = [
+  '解释', '什么是', '为什么', '如何', '怎么', '查询', '搜索',
+  '搜一下', '翻译', '总结', 'summarize', 'explain', 'what is', 'why', 'how do',
+];
+
+const ACTION_VERBS = [
+  '写', '帮我写', '生成', '创建', '实现', '做', '搜', '查',
+  '找', '审查', '审计', '翻译', '重构', '修改', '改', '部署',
+  '运行', '执行',
+  'write', 'create', 'generate', 'implement', 'search', 'find',
+  'review', 'translate', 'deploy', 'run',
+];
+
+const CODE_BLOCK = /```[\s\S]+?```/;
+const URL_PATTERN = /https?:\/\/\S+/;
+const PATH_PATTERN = /(?:[A-Za-z]:[\\/]|[\\/])[\w\-./\\]+/;
+const NUMBERED_LIST = /(?:^|\n)\s*(?:\d+[.)、]|[-*])\s+/gm;
+const INLINE_ENUMERATED = /\b\d+[.)、]\s*\S/g;
+
+/**
+ * Decide the execution mode for a user goal.
+ *
+ * direct: short greeting / single factual question, no tools needed.
+ * single: clear single-purpose task, one agent + tools.
+ * orchestrate: multi-step plan worth decomposing into sub-tasks.
+ */
+export function classify(goal: string): Mode {
+  if (!goal || !goal.trim()) return 'direct';
+
+  const text = goal.trim();
+  const lower = text.toLowerCase();
+  const length = text.length;
+
+  const hasCode = CODE_BLOCK.test(text);
+  const hasUrl = URL_PATTERN.test(text);
+  const hasPath = PATH_PATTERN.test(text);
+  const listMatches = (text.match(NUMBERED_LIST) || []).length;
+  const inlineEnumHits = (text.match(INLINE_ENUMERATED) || []).length;
+
+  const multiStepHits = MULTI_STEP_TOKENS.filter(t => lower.includes(t)).length;
+  const greetingHits = GREETING_TOKENS.filter(t => lower.includes(t)).length;
+  const singleHits = SINGLE_ACTION_HINTS.filter(t => lower.includes(t)).length;
+  const actionHits = ACTION_VERBS.filter(t => lower.includes(t)).length;
+
+  if (greetingHits >= 1 && length < 30 && multiStepHits === 0 && actionHits === 0) {
+    return 'direct';
+  }
+
+  if (multiStepHits >= 2 || listMatches >= 2 || inlineEnumHits >= 3) {
+    return 'orchestrate';
+  }
+
+  if (length > 200 && multiStepHits >= 1) {
+    return 'orchestrate';
+  }
+
+  if (hasCode && length > 150) {
+    return 'orchestrate';
+  }
+
+  // Tool-use signals push toward single, not direct
+  if (hasPath || hasUrl || actionHits >= 1) {
+    return 'single';
+  }
+
+  if (length < 50 && !hasCode) {
+    if (singleHits >= 1 || text.endsWith('?') || text.endsWith('？')) {
+      return 'direct';
+    }
+    if (multiStepHits === 0) {
+      return 'direct';
+    }
+  }
+
+  return 'single';
+}
+
+/**
+ * Pick a single agent for a non-orchestrate goal, by keyword routing.
+ *
+ * Returns an agent name guaranteed to be in available, falling back to
+ * rain (generation generalist) then to any available agent.
+ */
+export function pickAgentForGoal(goal: string, available: Set<string>): string {
+  const lower = goal.toLowerCase();
+
+  // More specific buckets first
+  const buckets: Array<[string, string[]]> = [
+    ['frost', ['审查', 'review', '漏洞', '安全', '审计', 'lint', '重构建议', 'code smell']],
+    ['dew', ['部署', '运行', '执行命令', 'shell', 'deploy', 'ci', 'cd', '环境变量', '运维']],
+    ['fog', ['研究', '调研', '搜一下', '搜索', '查一下', '查资料', 'research', 'search', '调查', '找一下', '找资料']],
+    ['rain', ['写', '生成', '实现', 'create', 'generate', '写一段', '写个', '代码', '函数', '实现一个']],
+    ['fair', ['陪我', '聊天', '心情', '难过', '开心', '孤独', '倾诉', '你好', 'hi', 'hello', '嗨']],
+  ];
+
+  for (const [agent, hints] of buckets) {
+    if (!available.has(agent)) continue;
+    if (hints.some(h => lower.includes(h))) return agent;
+  }
+
+  if (available.has('rain')) return 'rain';
+  return Array.from(available)[0];
+}

package/src/core/schemas.ts

@@ -0,0 +1,282 @@
+/**
+ * Lightweight structured output schemas for LLM response validation.
+ *
+ * Why: LLM JSON output is inherently fragile — models emit markdown fences,
+ * trailing commas, unquoted keys, or hallucinated fields. Rather than layering
+ * heuristic repair (which silently passes corrupted data), we define typed
+ * schemas and validate on ingress. Parsing failures surface immediately so the
+ * caller can retry with a corrected prompt instead of propagating garbage.
+ *
+ * Zero external dependencies: uses only JSON and TypeScript types.
+ */
+
+/**
+ * Error raised when an LLM response fails schema validation.
+ * Carries both a human-readable message and the raw text so callers
+ * can log / retry with full context.
+ */
+export class SchemaValidationError extends Error {
+  raw: string;
+
+  constructor(message: string, raw: string = "") {
+    super(message);
+    this.name = "SchemaValidationError";
+    this.raw = raw;
+  }
+}
+
+/**
+ * One step in a task plan (mirrors PipelineStep / Task)
+ */
+export interface TaskStepSchema {
+  id: string | number;
+  description: string;
+  agent?: string;
+  depends_on?: string[];
+  priority?: "low" | "medium" | "high";
+}
+
+/**
+ * Full task plan output from Snow's orchestrator
+ */
+export interface TaskPlanSchema {
+  goal: string;
+  steps: TaskStepSchema[];
+}
+
+/**
+ * A single extracted fact for long-term memory
+ */
+export interface FactSchema {
+  key: string;
+  value: string;
+  category?: string;
+}
+
+/**
+ * Structured fact-extraction output from the LLM
+ */
+export interface ExtractionResultSchema {
+  facts: FactSchema[];
+}
+
+/**
+ * Tool call schema for LLM responses
+ */
+export interface ToolCallSchema {
+  name: string;
+  arguments: Record<string, unknown>;
+}
+
+/**
+ * Message schema for agent communication
+ */
+export interface MessageSchema {
+  role: "user" | "assistant" | "tool";
+  content: string;
+  tool_calls?: ToolCallSchema[];
+  tool_call_id?: string;
+}
+
+/**
+ * Coerce a value to a target type with best-effort conversion
+ */
+function coerceType(value: unknown, targetType: string): unknown {
+  if (value === null || value === undefined) {
+    return value;
+  }
+
+  switch (targetType) {
+    case "string":
+      return String(value);
+    case "number":
+      return Number(value);
+    case "boolean":
+      return Boolean(value);
+    case "array":
+      return Array.isArray(value) ? value : [];
+    case "object":
+      return typeof value === "object" ? value : {};
+    default:
+      return value;
+  }
+}
+
+/**
+ * Extract JSON object/array from a potentially malformed string
+ */
+function extractJSON(text: string): string {
+  let cleaned = text.trim();
+
+  // Strip markdown code fences
+  if (cleaned.includes("```")) {
+    for (const fence of ["```json", "```"]) {
+      if (cleaned.includes(fence)) {
+        const after = cleaned.split(fence, 1)[1];
+        if (after && after.includes("```")) {
+          cleaned = after.split("```")[0].trim();
+          break;
+        }
+      }
+    }
+  }
+
+  // Find first JSON object or array
+  let objStart = -1;
+  let depth = 0;
+  for (let i = 0; i < cleaned.length; i++) {
+    const ch = cleaned[i];
+    if (ch === "{") {
+      if (objStart < 0) objStart = i;
+      depth++;
+    } else if (ch === "}") {
+      depth--;
+      if (depth === 0 && objStart >= 0) {
+        return cleaned.substring(objStart, i + 1);
+      }
+    }
+  }
+
+  // If still not closed, close with extra braces
+  if (objStart >= 0) {
+    return cleaned.substring(objStart) + "}".repeat(depth);
+  }
+
+  throw new SchemaValidationError("No valid JSON found in response", text);
+}
+
+/**
+ * Repair common JSON issues (trailing commas, unquoted keys, etc.)
+ */
+function repairJSON(text: string): string {
+  let repaired = text;
+
+  // Remove trailing commas
+  repaired = repaired.replace(/,\s*([}\]])/g, "$1");
+
+  // Quote unquoted keys
+  repaired = repaired.replace(/(?<!["\'\w])(\w[\w\d_]*)(\s*:)/g, '"$1"$2');
+
+  // Normalize quotes
+  repaired = repaired.replace(/'/g, '"').replace(/`/g, '"');
+
+  return repaired;
+}
+
+/**
+ * Parse a raw LLM response string into a typed JSON object.
+ * Handles markdown fences, leading/trailing text, and minor JSON quirks.
+ * Raises SchemaValidationError on failure.
+ */
+export function parseSchema<T extends Record<string, unknown>>(
+  raw: string,
+  schemaType?: new () => T
+): T {
+  if (!raw || !raw.trim()) {
+    throw new SchemaValidationError("empty response", raw);
+  }
+
+  try {
+    const extracted = extractJSON(raw);
+    let data = JSON.parse(extracted);
+    return data as T;
+  } catch (error) {
+    if (error instanceof SchemaValidationError) throw error;
+
+    try {
+      const repaired = repairJSON(raw);
+      const data = JSON.parse(repaired);
+      return data as T;
+    } catch (repairError) {
+      throw new SchemaValidationError(
+        `JSON parse failed: ${(error as Error).message}`,
+        raw
+      );
+    }
+  }
+}
+
+/**
+ * Validate task plan schema
+ */
+export function validateTaskPlan(data: unknown): TaskPlanSchema {
+  if (!data || typeof data !== "object") {
+    throw new SchemaValidationError("Invalid task plan: must be an object");
+  }
+
+  const plan = data as Record<string, unknown>;
+
+  if (typeof plan.goal !== "string") {
+    throw new SchemaValidationError("Invalid task plan: goal must be a string");
+  }
+
+  if (!Array.isArray(plan.steps)) {
+    throw new SchemaValidationError("Invalid task plan: steps must be an array");
+  }
+
+  const steps = (plan.steps as unknown[]).map((step: unknown) => {
+    if (!step || typeof step !== "object") {
+      throw new SchemaValidationError("Invalid task plan: step must be an object");
+    }
+
+    const s = step as Record<string, unknown>;
+    if (typeof s.id !== "string" && typeof s.id !== "number") {
+      throw new SchemaValidationError("Invalid task plan: step.id must be string or number");
+    }
+
+    if (typeof s.description !== "string") {
+      throw new SchemaValidationError("Invalid task plan: step.description must be a string");
+    }
+
+    return {
+      id: s.id,
+      description: s.description,
+      agent: typeof s.agent === "string" ? s.agent : "rain",
+      depends_on: Array.isArray(s.depends_on) ? (s.depends_on as string[]) : [],
+      priority: (["low", "medium", "high"].includes(s.priority as string)
+        ? s.priority
+        : "medium") as "low" | "medium" | "high",
+    };
+  });
+
+  return {
+    goal: plan.goal,
+    steps,
+  };
+}
+
+/**
+ * Validate extraction result schema
+ */
+export function validateExtractionResult(data: unknown): ExtractionResultSchema {
+  if (!data || typeof data !== "object") {
+    throw new SchemaValidationError("Invalid extraction result: must be an object");
+  }
+
+  const result = data as Record<string, unknown>;
+
+  if (!Array.isArray(result.facts)) {
+    throw new SchemaValidationError("Invalid extraction result: facts must be an array");
+  }
+
+  const facts = (result.facts as unknown[]).map((fact: unknown) => {
+    if (!fact || typeof fact !== "object") {
+      throw new SchemaValidationError("Invalid extraction result: fact must be an object");
+    }
+
+    const f = fact as Record<string, unknown>;
+    if (typeof f.key !== "string" || typeof f.value !== "string") {
+      throw new SchemaValidationError(
+        "Invalid extraction result: fact must have key and value strings"
+      );
+    }
+
+    return {
+      key: f.key,
+      value: f.value,
+      category: typeof f.category === "string" ? f.category : "auto_extracted",
+    };
+  });
+
+  return { facts };
+}

package/src/core/semantic.ts

@@ -0,0 +1,211 @@
+/**
+ * Lightweight semantic retrieval — zero external dependencies.
+ *
+ * Why: The existing `recall_for_injection` uses SQL LIKE on tokens, which
+ * misses semantically related facts that share few literal characters (e.g.
+ * query "deploy" vs. stored fact "release_command"). This module provides
+ * a character n-gram Jaccard similarity scorer that catches those cross-
+ * lingual and synonym relationships without adding PyTorch / sentence-transformers.
+ *
+ * Design:
+ * - Character n-grams (size 2-4) naturally handle CJK, mixed-language, and
+ *   code identifiers better than word-level tokenization.
+ * - Jaccard similarity on n-gram sets is fast (< 10 µs per pair) and
+ *   well-correlated with human relevance judgments for short text.
+ * - Zero dependencies beyond stdlib.
+ */
+
+/**
+ * Represents a candidate item for ranking
+ */
+export interface Candidate {
+  key?: string;
+  value?: string | Record<string, unknown>;
+  [key: string]: unknown;
+}
+
+/**
+ * Character n-gram semantic similarity scorer.
+ *
+ * Usage:
+ * ```
+ * const scorer = new SemanticScorer();
+ * const score = scorer.similarity("deploy to prod", "release_command");
+ * // score ≈ 0.15 (low but non-zero — catches partial overlap)
+ *
+ * const ranked = scorer.rank(
+ *   "search query",
+ *   [{value: "candidate A"}, {value: "candidate B"}]
+ * );
+ * // ranked → [[0.85, {value: "candidate A"}], [0.30, {value: "candidate B"}]]
+ * ```
+ */
+export class SemanticScorer {
+  private nRange: [number, number];
+  private cache: Map<string, Set<string>>;
+  private maxCacheSize = 512;
+
+  /**
+   * Initialize scorer with n-gram range.
+   * @param nRange - Tuple of [minSize, maxSize] for n-grams (default [2, 4])
+   */
+  constructor(nRange: [number, number] = [2, 4]) {
+    this.nRange = nRange;
+    this.cache = new Map();
+  }
+
+  /**
+   * Generate character n-gram fingerprint (cached).
+   * @param text - Input text to fingerprint
+   * @returns Set of n-grams for the text
+   */
+  private fingerprint(text: string): Set<string> {
+    // Return cached result if available
+    if (this.cache.has(text)) {
+      return this.cache.get(text)!;
+    }
+
+    const lowered = text.toLowerCase();
+    const ngrams = new Set<string>();
+
+    // Generate n-grams for each size in range
+    for (let n = this.nRange[0]; n <= this.nRange[1]; n++) {
+      if (lowered.length < n) {
+        continue;
+      }
+      for (let i = 0; i <= lowered.length - n; i++) {
+        ngrams.add(lowered.slice(i, i + n));
+      }
+    }
+
+    // Cache result if we haven't hit size limit
+    if (this.cache.size < this.maxCacheSize) {
+      this.cache.set(text, ngrams);
+    }
+
+    return ngrams;
+  }
+
+  /**
+   * Calculate Jaccard similarity between two texts using character n-grams.
+   * @param a - First text
+   * @param b - Second text
+   * @returns Similarity score between 0 and 1
+   */
+  similarity(a: string, b: string): number {
+    if (!a || !b) {
+      return 0.0;
+    }
+
+    const fpA = this.fingerprint(a);
+    const fpB = this.fingerprint(b);
+
+    if (fpA.size === 0 || fpB.size === 0) {
+      return 0.0;
+    }
+
+    // Calculate intersection
+    let intersection = 0;
+    for (const ngram of fpA) {
+      if (fpB.has(ngram)) {
+        intersection++;
+      }
+    }
+
+    // Calculate union
+    const union = fpA.size + fpB.size - intersection;
+
+    return union > 0 ? intersection / union : 0.0;
+  }
+
+  /**
+   * Rank candidates by semantic similarity to the query.
+   *
+   * Each candidate is a dict. The scorer reads `candidate[keyField]`
+   * for text to compare. Also scores the "key" field if present.
+   * Returns `[score, candidate]` tuples sorted descending, filtered by `minScore`.
+   *
+   * @param query - Query string to match against
+   * @param candidates - Array of candidate objects
+   * @param keyField - Field name to extract text from (default "value")
+   * @param topK - Maximum number of results to return (default 3)
+   * @param minScore - Minimum similarity threshold (default 0.02)
+   * @returns Array of [score, candidate] tuples, sorted by score descending
+   */
+  rank(
+    query: string,
+    candidates: Candidate[],
+    keyField: string = "value",
+    topK: number = 3,
+    minScore: number = 0.02
+  ): Array<[number, Candidate]> {
+    const scored: Array<[number, Candidate]> = [];
+
+    for (const candidate of candidates) {
+      let text = candidate[keyField];
+
+      // Skip if field is missing or empty
+      if (!text) {
+        continue;
+      }
+
+      // Convert objects to string representation
+      if (typeof text === "object") {
+        text = JSON.stringify(text);
+      } else {
+        text = String(text);
+      }
+
+      // Calculate similarity with the candidate's value field
+      let score = this.similarity(query, text as string);
+
+      // Also score the "key" field if present — often more discriminative
+      const keyText = candidate.key;
+      if (keyText) {
+        const keyScore = this.similarity(query, String(keyText));
+        score = Math.max(score, keyScore);
+      }
+
+      // Add to results if above threshold
+      if (score >= minScore) {
+        scored.push([score, candidate]);
+      }
+    }
+
+    // Sort descending by score
+    scored.sort((a, b) => b[0] - a[0]);
+
+    // Return top K results
+    return scored.slice(0, topK);
+  }
+
+  /**
+   * Clear the fingerprint cache.
+   */
+  clearCache(): void {
+    this.cache.clear();
+  }
+
+  /**
+   * Get current cache size.
+   */
+  getCacheSize(): number {
+    return this.cache.size;
+  }
+}
+
+/**
+ * Module-level singleton instance (lazy initialization).
+ */
+let scorer: SemanticScorer | null = null;
+
+/**
+ * Get or create the module-level semantic scorer singleton.
+ * @returns SemanticScorer instance
+ */
+export function getScorer(): SemanticScorer {
+  if (scorer === null) {
+    scorer = new SemanticScorer();
+  }
+  return scorer;
+}