skyloom 1.6.0 → 1.8.0

package/src/core/estimate.ts

@@ -0,0 +1,104 @@
+/**
+ * 资源估算模块 — Token & time budget estimation for task planning.
+ *
+ * Helps Snow and other planning agents estimate the cost of
+ * proposed sub-tasks before committing to execution.
+ */
+
+import type { Task } from "./agent";
+
+/* ═══════════════════════════════════════
+   Token estimation
+   ═══════════════════════════════════════ */
+const CJK_REGEX = /[一-鿿぀-ゟ가-힯㐀-䶿]/g;
+
+/** Estimate tokens for a given text (CJK ~2 each, ASCII ~4 chars each). */
+export function estimateTokens(text: string): number {
+  const cjk = (text.match(CJK_REGEX) || []).length;
+  return cjk * 2 + Math.ceil((text.length - cjk) / 4);
+}
+
+/* ═══════════════════════════════════════
+   Per-task-type cost estimates
+   ═══════════════════════════════════════ */
+const TASK_TYPE_PATTERNS: Array<[RegExp, number, number]> = [
+  // [pattern, estimated tokens, estimated tools]
+  [/read|read_file|grep|search|查|搜索|list/i, 2000, 2],
+  [/write|write_file|生成|写|create|implement/i, 4000, 5],
+  [/edit|edit_file|改|修改|fix|修复/i, 3000, 3],
+  [/delete|delete_file|删|rm/i, 1500, 2],
+  [/deploy|部署|publish|发布|release/i, 8000, 8],
+  [/review|审查|audit|审计|scan|扫描/i, 5000, 4],
+  [/test|测试|run_test|coverage/i, 3000, 3],
+  [/research|研究|调研|analyze|分析/i, 6000, 4],
+  [/orchestrate|编排|multi-step|多步/i, 12000, 10],
+];
+
+/** Estimate cost for a single task description. */
+export function estimateTaskCost(description: string): { tokens: number; tools: number; timeSeconds: number } {
+  let tokens = 2000; // base
+  let tools = 2;     // base
+  for (const [pattern, t, tc] of TASK_TYPE_PATTERNS) {
+    if (pattern.test(description)) { tokens = Math.max(tokens, t); tools = Math.max(tools, tc); }
+  }
+
+  // Time estimate: ~0.5s per tool call + 2s per 1k tokens
+  const timeSeconds = (tokens / 1000) * 2 + tools * 0.5 + 2;
+  return { tokens, tools, timeSeconds };
+}
+
+/* ═══════════════════════════════════════
+   Task plan cost summary
+   ═══════════════════════════════════════ */
+export interface PlanEstimate {
+  totalTokens: number;
+  totalTools: number;
+  totalTimeSeconds: number;
+  perTask: Array<{ id: string; tokens: number; tools: number; time: number }>;
+  warnings: string[];
+}
+
+export function estimateTaskPlan(tasks: Task[]): PlanEstimate {
+  const perTask: PlanEstimate["perTask"] = [];
+  let totalTokens = 500; // system prompt overhead
+  let totalTools = 0;
+  let totalTime = 5; // init overhead
+  const warnings: string[] = [];
+
+  for (const t of tasks) {
+    const est = estimateTaskCost(t.description);
+    perTask.push({ id: t.id, tokens: est.tokens, tools: est.tools, time: est.timeSeconds });
+    totalTokens += est.tokens;
+    totalTools += est.tools;
+    totalTime += est.timeSeconds;
+
+    if (est.timeSeconds > 60) warnings.push(`Task ${t.id} may take >${Math.round(est.timeSeconds)}s`);
+    if (est.tools > 10) warnings.push(`Task ${t.id} uses many tool calls (${est.tools})`);
+  }
+
+  if (totalTokens > 64000) warnings.push(`Total token estimate (${totalTokens}) exceeds typical context window`);
+  if (totalTime > 120) warnings.push(`Estimated total time (${Math.round(totalTime)}s) is significant`);
+  if (tasks.length > 6) warnings.push(`Large number of sub-tasks (${tasks.length}) — consider merging simpler ones`);
+
+  return { totalTokens, totalTools, totalTimeSeconds: Math.round(totalTime), perTask, warnings };
+}
+
+/* ═══════════════════════════════════════
+   Format estimate for display
+   ═══════════════════════════════════════ */
+export function formatPlanEstimate(est: PlanEstimate): string {
+  const lines: string[] = [
+    `## Plan Estimate`,
+    `| Task | Tokens | Tools | Time |`,
+    `|------|--------|-------|------|`,
+    ...est.perTask.map(t => `| ${t.id} | ${t.tokens} | ${t.tools} | ${t.time.toFixed(0)}s |`),
+    `| **Total** | **${est.totalTokens}** | **${est.totalTools}** | **${est.totalTimeSeconds}s** |`,
+  ];
+
+  if (est.warnings.length > 0) {
+    lines.push("", "### Warnings");
+    for (const w of est.warnings) lines.push(`- ⚠ ${w}`);
+  }
+
+  return lines.join("\n");
+}

package/src/core/evolve.ts

@@ -0,0 +1,191 @@
+/**
+ * 自我进化模块 — Prompt self-optimization via failure analysis.
+ *
+ * When an agent repeatedly fails at similar tasks, this module analyzes
+ * the failure patterns and suggests targeted improvements to the agent's
+ * System Prompt. The agent can then apply these suggestions to improve
+ * future performance.
+ *
+ * Architecture:
+ *    Failure log → Pattern analysis → Prompt diff → Agent.applyDiff()
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+import { USER_CONFIG_DIR } from "./config";
+import { getLogger } from "./logger";
+
+const log = getLogger("evolve");
+
+/* ═══════════════════════════════════════
+   Prompt diff — a suggested change
+   ═══════════════════════════════════════ */
+export interface PromptDiff {
+  id: string;
+  ts: string;
+  agent: string;
+  reason: string;           // Why this change is needed
+  before: string;           // Old prompt fragment
+  after: string;            // New prompt fragment
+  applied: boolean;
+  improvement?: string;     // Measured improvement after applying
+}
+
+/* ═══════════════════════════════════════
+   Failure analysis
+   ═══════════════════════════════════════ */
+export interface FailureAnalysis {
+  agent: string;
+  period: string;
+  totalCalls: number;
+  failureCount: number;
+  topFailures: Array<{ pattern: string; count: number }>;
+  suggestedDiffs: PromptDiff[];
+}
+
+const evolveDir = path.join(USER_CONFIG_DIR, "evolve");
+function ensureDir() { if (!fs.existsSync(evolveDir)) fs.mkdirSync(evolveDir, { recursive: true }); }
+
+/** Analyze recent failures from the learning module and suggest prompt improvements. */
+export function analyzeFailures(
+  agent: string,
+  experiences: Array<{ pattern: string; solution: string; frequency: number; lastSeen: string }>,
+  systemPrompt: string
+): FailureAnalysis {
+  const recent = experiences.filter(e => {
+    try { return new Date(e.lastSeen).getTime() > Date.now() - 7 * 86400000; }
+    catch { return false; }
+  });
+
+  const topFailures = recent
+    .sort((a, b) => b.frequency - a.frequency)
+    .slice(0, 5)
+    .map(e => ({ pattern: e.pattern, count: e.frequency }));
+
+  const suggestedDiffs: PromptDiff[] = [];
+
+  // Rule-based suggestions from failure patterns
+  for (const f of topFailures) {
+    const lower = f.pattern.toLowerCase();
+
+    // Search storm → add search budget rule
+    if ((lower.includes("search") || lower.includes("web_search")) && f.count >= 3) {
+      const rule = `- 搜索不超过 5 轮。5 轮后直接基于已有信息综合回答。`;
+      if (!systemPrompt.includes("搜索不超过")) {
+        suggestedDiffs.push({
+          id: Math.random().toString(36).slice(2, 8),
+          ts: new Date().toISOString(), agent,
+          reason: `搜索风暴 (${f.count} 次重复搜索)`,
+          before: "", after: rule, applied: false,
+        });
+      }
+    }
+
+    // Empty response → add deliverable checklist
+    if ((lower.includes("empty") || lower.includes("placeholder") || lower.includes("完成了")) && f.count >= 2) {
+      const rule = `- 完成任务后，必须输出实际产物（代码/文件路径/数据），禁止只说"完成了"而无产出。`;
+      if (!systemPrompt.includes("必须输出实际产物")) {
+        suggestedDiffs.push({
+          id: Math.random().toString(36).slice(2, 8),
+          ts: new Date().toISOString(), agent,
+          reason: `空响应/占位 (${f.count} 次)`,
+          before: "", after: rule, applied: false,
+        });
+      }
+    }
+
+    // Tool not found → add tool discovery to prompt
+    if (lower.includes("does not exist") || lower.includes("tool") && lower.includes("not found")) {
+      const rule = `- 使用不熟悉的工具前先调 list_skills 查看可用工具列表。`;
+      if (!systemPrompt.includes("list_skills")) {
+        suggestedDiffs.push({
+          id: Math.random().toString(36).slice(2, 8),
+          ts: new Date().toISOString(), agent,
+          reason: `工具不存在 (${f.count} 次)`,
+          before: "", after: rule, applied: false,
+        });
+      }
+    }
+
+    // File not found → add path verification rule
+    if (lower.includes("file not found") || lower.includes("directory not found")) {
+      const rule = `- 文件操作前先用 list_directory 或 read_file 确认路径存在。`;
+      if (!systemPrompt.includes("确认路径存在")) {
+        suggestedDiffs.push({
+          id: Math.random().toString(36).slice(2, 8),
+          ts: new Date().toISOString(), agent,
+          reason: `文件路径错误 (${f.count} 次)`,
+          before: "", after: rule, applied: false,
+        });
+      }
+    }
+  }
+
+  // General rule: if failure rate > 20%, suggest self-review
+  const recentExperiences = recent.filter(e => {
+    try { return new Date(e.lastSeen).getTime() > Date.now() - 3 * 86400000; }
+    catch { return false; }
+  });
+
+  // Deduplicate suggestions
+  const seen = new Set<string>();
+  const uniqueDiffs = suggestedDiffs.filter(d => {
+    const key = d.after.slice(0, 30);
+    if (seen.has(key)) return false;
+    seen.add(key); return true;
+  });
+
+  // Max 3 suggestions per analysis
+  return {
+    agent, period: "last 7 days",
+    totalCalls: 0, failureCount: 0,
+    topFailures,
+    suggestedDiffs: uniqueDiffs.slice(0, 3),
+  };
+}
+
+/* ═══════════════════════════════════════
+   Apply prompt diff to agent
+   ═══════════════════════════════════════ */
+export function applyPromptDiff(agent: any, diff: PromptDiff): boolean {
+  try {
+    const currentPrompt = agent.systemPrompt;
+    if (!diff.after || currentPrompt.includes(diff.after.slice(0, 20))) return false;
+
+    // Append the new rule after "## 行为守则" or "## Behavior" section
+    const marker = currentPrompt.includes("行为守则") ? "## 行为守则" : "## Behavior";
+    const idx = currentPrompt.indexOf(marker);
+    if (idx < 0) { agent.systemPrompt += "\n" + diff.after; }
+    else {
+      const insertPoint = currentPrompt.indexOf("\n", currentPrompt.indexOf("\n-", idx) + 1);
+      agent.systemPrompt = currentPrompt.slice(0, insertPoint) + "\n" + diff.after + "\n" + currentPrompt.slice(insertPoint);
+    }
+
+    diff.applied = true;
+    diff.improvement = "pending evaluation";
+
+    // Persist the diff
+    ensureDir();
+    const file = path.join(evolveDir, `${diff.agent}_diffs.jsonl`);
+    fs.appendFileSync(file, JSON.stringify(diff) + "\n");
+
+    agent.rebuildSystemPrompt();
+    return true;
+  } catch (e) {
+    log.warn("apply_prompt_diff_failed", { agent: diff.agent, error: String(e) });
+    return false;
+  }
+}
+
+/** Get all applied diffs for an agent. */
+export function getAppliedDiffs(agent: string): PromptDiff[] {
+  const diffs: PromptDiff[] = [];
+  try {
+    const file = path.join(evolveDir, `${agent}_diffs.jsonl`);
+    if (fs.existsSync(file)) {
+      const lines = fs.readFileSync(file, "utf-8").split("\n").filter(Boolean);
+      for (const line of lines) { try { diffs.push(JSON.parse(line)); } catch { } }
+    }
+  } catch { /* ignore */ }
+  return diffs;
+}

package/src/core/filter.ts

@@ -0,0 +1,103 @@
+/**
+ * 输出过滤模块 — sensitive information sanitization.
+ *
+ * Before agent responses reach the user (or are persisted),
+ * scan for and redact sensitive patterns like API keys,
+ * tokens, passwords, PII, and internal paths.
+ */
+
+/* ═══════════════════════════════════════
+   Detection patterns — compiled once at module load
+   ═══════════════════════════════════════ */
+const SENSITIVE_PATTERNS: Array<[RegExp, string]> = [
+  // API keys & tokens
+  [/sk-[a-zA-Z0-9]{32,}/g, "[REDACTED:API_KEY]"],
+  [/(?:api_key|apikey|secret_key|access_token|auth_token)\s*[:=]\s*["']?[^\s"']{8,}["']?/gi, "$1: [REDACTED]"],
+  [/ghp_[a-zA-Z0-9]{36}/g, "[REDACTED:GITHUB_TOKEN]"],
+  [/gho_[a-zA-Z0-9]{36}/g, "[REDACTED:GITHUB_TOKEN]"],
+
+  // AWS credentials
+  [/AKIA[0-9A-Z]{16}/g, "[REDACTED:AWS_KEY]"],
+  [/(?:aws_access_key_id|aws_secret_access_key)\s*[:=]\s*["']?[^\s"']+/gi, "$1: [REDACTED]"],
+
+  // Passwords
+  [/(?:password|passwd|pwd)\s*[:=]\s*["']?[^\s"']{4,}["']?/gi, "$1: [REDACTED]"],
+  [/(?:密码|口令)\s*[:=]\s*["']?[^\s"']{2,}["']?/g, "$1: [已脱敏]"],
+
+  // Connection strings
+  [/(?:mongodb|postgres|mysql|redis):\/\/[^\s]+/g, "[REDACTED:DB_URI]"],
+  [/(?:jdbc|odbc):[^\s]+/g, "[REDACTED:DB_URI]"],
+
+  // Private keys
+  [/-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----[\s\S]*?-----END .*?PRIVATE KEY-----/g, "[REDACTED:PRIVATE_KEY]"],
+
+  // IP addresses (local only)
+  [/192\.168\.\d{1,3}\.\d{1,3}/g, "[REDACTED:LAN_IP]"],
+  [/10\.\d{1,3}\.\d{1,3}\.\d{1,3}/g, "[REDACTED:LAN_IP]"],
+  [/172\.(1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3}/g, "[REDACTED:LAN_IP]"],
+
+  // File paths
+  [/(?:\/etc\/(?:passwd|shadow|hosts|sudoers))/g, "[REDACTED:SYSTEM_PATH]"],
+];
+
+/* Email masking (function-based, handled separately) */
+const EMAIL_RE = /([a-zA-Z0-9._%+-]{3,})@([a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/g;
+
+
+/* ═══════════════════════════════════════
+   Filter function
+   ═══════════════════════════════════════ */
+export interface FilterResult {
+  clean: string;
+  redacted: boolean;
+  count: number;
+  details: string[];
+}
+
+export function filterOutput(text: string): FilterResult {
+  if (!text) return { clean: "", redacted: false, count: 0, details: [] };
+
+  let clean = text;
+  let count = 0;
+  const details: string[] = [];
+
+  // Email masking (function-based replacement)
+  let emailCount = 0;
+  clean = clean.replace(EMAIL_RE, (full, user, domain) => {
+    emailCount++;
+    return (user as string).slice(0, 2) + "***@" + (domain as string);
+  });
+  if (emailCount > 0) {
+    count += emailCount;
+    details.push(`Masked ${emailCount}x email addresses`);
+  }
+
+  for (const [pattern, replacement] of SENSITIVE_PATTERNS) {
+    const matches = clean.match(pattern);
+    if (matches) {
+      count += matches.length;
+      if (typeof replacement === "string") {
+        details.push(`Redacted ${matches.length}x ${pattern.source.slice(0, 30)}`);
+      } else {
+        details.push(`Masked ${matches.length}x email addresses`);
+      }
+      clean = clean.replace(pattern, replacement as string);
+    }
+  }
+
+  return { clean, redacted: count > 0, count, details };
+}
+
+/* ═══════════════════════════════════════
+   Quick check — is filtering needed?
+   ═══════════════════════════════════════ */
+export function needsFiltering(text: string): boolean {
+  if (!text) return false;
+  // Quick scan with the most common patterns
+  if (/sk-[a-zA-Z0-9]{32,}/.test(text)) return true;
+  if (/api_key.*[:=]/.test(text)) return true;
+  if (/password.*[:=]/.test(text)) return true;
+  if (/-----BEGIN.*PRIVATE KEY-----/.test(text)) return true;
+  if (EMAIL_RE.test(text)) return true;
+  return false;
+}

package/src/core/index.ts

@@ -26,6 +26,12 @@ export * from './skill';
 export * from './router';
 export * from './agent';
 export * from './factory';
+export * from './security';
+export * from './learn';
+export * from './longdoc';
+export * from './filter';
+export * from './estimate';
+export * from './arbitrate';
 
-// Version
-export const VERSION = '1.4.0';
+// Version — read from package.json
+export const VERSION = (() => { try { return require('../../package.json').version; } catch { return '1.6.0'; } })();

package/src/core/longdoc.ts

@@ -0,0 +1,155 @@
+/**
+ * 长文档处理策略 — sliding window + summary chain.
+ *
+ * When an input exceeds the agent's effective context window,
+ * split into overlapping chunks, summarize each, then chain
+ * summaries into a final digest.
+ *
+ * Architecture:
+ *   Input → Chunk(sliding window) → Per-chunk Summary → Chain → Final Digest
+ *
+ * All summaries are generated by the calling agent's LLM, so quality
+ * depends on the model in use. The chunker is pure text processing
+ * and works without any LLM call.
+ */
+
+import type { BaseAgent } from "./agent";
+
+/* ═══════════════════════════════════════
+   Chunker — split text into overlapping windows
+   ═══════════════════════════════════════ */
+export interface ChunkOptions {
+  /** Target chunk size in characters (default 6000) */
+  chunkSize?: number;
+  /** Overlap between consecutive chunks in characters (default 800) */
+  overlap?: number;
+  /** Minimum chunk size before we stop splitting (default 500) */
+  minChunk?: number;
+}
+
+export function chunkText(text: string, opts?: ChunkOptions): string[] {
+  const cs = opts?.chunkSize ?? 6000;
+  const ol = opts?.overlap ?? 800;
+  const min = opts?.minChunk ?? 500;
+  const chunks: string[] = [];
+
+  if (text.length <= cs + min) { chunks.push(text); return chunks; }
+
+  let start = 0;
+  while (start < text.length) {
+    let end = start + cs;
+    if (end >= text.length) { end = text.length; }
+    else {
+      // Try to break at paragraph boundary
+      const searchEnd = Math.min(end + 400, text.length);
+      const paraBreak = text.lastIndexOf("\n\n", searchEnd);
+      if (paraBreak > start + min) end = paraBreak;
+      else {
+        const lineBreak = text.lastIndexOf("\n", searchEnd);
+        if (lineBreak > start + min) end = lineBreak;
+        else {
+          const space = text.lastIndexOf(" ", searchEnd);
+          if (space > start + min) end = space;
+        }
+      }
+    }
+
+    chunks.push(text.slice(start, end).trim());
+    if (end >= text.length) break;
+    start = end - ol;
+    if (start < 0) start = 0;
+  }
+
+  return chunks;
+}
+
+/* ═══════════════════════════════════════
+   Summary chain — ask agent to summarize chunks then chain
+   ═══════════════════════════════════════ */
+export interface SummaryOptions {
+  /** Max total chars for the final digest (default 3000) */
+  maxDigestChars?: number;
+  /** Custom summarization prompt for each chunk */
+  chunkPrompt?: string;
+  /** Custom chain prompt for combining summaries */
+  chainPrompt?: string;
+}
+
+const DEFAULT_CHUNK_PROMPT = `Summarize the following text concisely. Keep all key facts, names, numbers, and code snippets. Output the summary directly without preamble. Limit to 300 words.
+
+Text:
+{text}`;
+
+const DEFAULT_CHAIN_PROMPT = `Combine the following section summaries into a single coherent digest. Preserve all key facts, remove redundancy. Output directly without preamble.
+
+{summaries}`;
+
+export async function summarizeLongDoc(
+  agent: BaseAgent,
+  text: string,
+  opts?: SummaryOptions
+): Promise<string> {
+  const maxDigest = opts?.maxDigestChars ?? 3000;
+  const chunks = chunkText(text);
+
+  // Single chunk — no summarization needed
+  if (chunks.length <= 1) {
+    if (text.length <= maxDigest) return text;
+    const prompt = (opts?.chunkPrompt || DEFAULT_CHUNK_PROMPT).replace("{text}", text);
+    return agent.chatOneshot(prompt, { maxTokens: maxDigest });
+  }
+
+  // Multi-chunk: summarize each, then chain
+  const summaries: string[] = [];
+  for (let i = 0; i < chunks.length; i++) {
+    const prompt = (opts?.chunkPrompt || DEFAULT_CHUNK_PROMPT).replace("{text}", chunks[i]);
+    try {
+      const s = await agent.chatOneshot(prompt, { maxTokens: 600 });
+      summaries.push(s);
+    } catch {
+      summaries.push(chunks[i].slice(0, 400) + "...");
+    }
+  }
+
+  // Chain summaries
+  if (summaries.length === 1) return summaries[0].slice(0, maxDigest);
+
+  const joined = summaries.map((s, i) => `## Section ${i + 1}\n${s}`).join("\n\n");
+  if (joined.length <= maxDigest) return joined;
+
+  const chainPrompt = (opts?.chainPrompt || DEFAULT_CHAIN_PROMPT).replace("{summaries}", joined);
+  const final = await agent.chatOneshot(chainPrompt, { maxTokens: maxDigest });
+  return final.slice(0, maxDigest);
+}
+
+/* ═══════════════════════════════════════
+   Structured data parsing helpers
+   ═══════════════════════════════════════ */
+export function parseStructuredInput(input: string): {
+  hasTable: boolean; hasJSON: boolean; hasCSV: boolean;
+  extractedJSON: string | null; extractedTable: string[][] | null;
+} {
+  const result = { hasTable: false, hasJSON: false, hasCSV: false, extractedJSON: null as string | null, extractedTable: null as string[][] | null };
+
+  // Detect JSON
+  const jsonMatch = input.match(/\{[\s\S]*\}|\[[\s\S]*\]/);
+  if (jsonMatch) {
+    try { JSON.parse(jsonMatch[0]); result.hasJSON = true; result.extractedJSON = jsonMatch[0]; }
+    catch { /* not valid JSON */ }
+  }
+
+  // Detect markdown table
+  const tableMatch = input.match(/\|[\s\S]*?\|/);
+  if (tableMatch) {
+    result.hasTable = true;
+    const lines = input.split("\n").filter(l => l.includes("|") && !l.startsWith("|---") && !l.startsWith("| --"));
+    result.extractedTable = lines.map(l => l.split("|").filter(c => c.trim()).map(c => c.trim()));
+  }
+
+  // Detect CSV
+  if (input.includes(",") && input.split("\n").filter(l => l.includes(",")).length >= 2) {
+    result.hasCSV = true;
+  }
+
+  return result;
+}