skyloom 1.6.0 → 1.7.0

package/src/core/filter.ts

@@ -0,0 +1,103 @@
+/**
+ * 输出过滤模块 — sensitive information sanitization.
+ *
+ * Before agent responses reach the user (or are persisted),
+ * scan for and redact sensitive patterns like API keys,
+ * tokens, passwords, PII, and internal paths.
+ */
+
+/* ═══════════════════════════════════════
+   Detection patterns — compiled once at module load
+   ═══════════════════════════════════════ */
+const SENSITIVE_PATTERNS: Array<[RegExp, string]> = [
+  // API keys & tokens
+  [/sk-[a-zA-Z0-9]{32,}/g, "[REDACTED:API_KEY]"],
+  [/(?:api_key|apikey|secret_key|access_token|auth_token)\s*[:=]\s*["']?[^\s"']{8,}["']?/gi, "$1: [REDACTED]"],
+  [/ghp_[a-zA-Z0-9]{36}/g, "[REDACTED:GITHUB_TOKEN]"],
+  [/gho_[a-zA-Z0-9]{36}/g, "[REDACTED:GITHUB_TOKEN]"],
+
+  // AWS credentials
+  [/AKIA[0-9A-Z]{16}/g, "[REDACTED:AWS_KEY]"],
+  [/(?:aws_access_key_id|aws_secret_access_key)\s*[:=]\s*["']?[^\s"']+/gi, "$1: [REDACTED]"],
+
+  // Passwords
+  [/(?:password|passwd|pwd)\s*[:=]\s*["']?[^\s"']{4,}["']?/gi, "$1: [REDACTED]"],
+  [/(?:密码|口令)\s*[:=]\s*["']?[^\s"']{2,}["']?/g, "$1: [已脱敏]"],
+
+  // Connection strings
+  [/(?:mongodb|postgres|mysql|redis):\/\/[^\s]+/g, "[REDACTED:DB_URI]"],
+  [/(?:jdbc|odbc):[^\s]+/g, "[REDACTED:DB_URI]"],
+
+  // Private keys
+  [/-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----[\s\S]*?-----END .*?PRIVATE KEY-----/g, "[REDACTED:PRIVATE_KEY]"],
+
+  // IP addresses (local only)
+  [/192\.168\.\d{1,3}\.\d{1,3}/g, "[REDACTED:LAN_IP]"],
+  [/10\.\d{1,3}\.\d{1,3}\.\d{1,3}/g, "[REDACTED:LAN_IP]"],
+  [/172\.(1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3}/g, "[REDACTED:LAN_IP]"],
+
+  // File paths
+  [/(?:\/etc\/(?:passwd|shadow|hosts|sudoers))/g, "[REDACTED:SYSTEM_PATH]"],
+];
+
+/* Email masking (function-based, handled separately) */
+const EMAIL_RE = /([a-zA-Z0-9._%+-]{3,})@([a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/g;
+
+
+/* ═══════════════════════════════════════
+   Filter function
+   ═══════════════════════════════════════ */
+export interface FilterResult {
+  clean: string;
+  redacted: boolean;
+  count: number;
+  details: string[];
+}
+
+export function filterOutput(text: string): FilterResult {
+  if (!text) return { clean: "", redacted: false, count: 0, details: [] };
+
+  let clean = text;
+  let count = 0;
+  const details: string[] = [];
+
+  // Email masking (function-based replacement)
+  let emailCount = 0;
+  clean = clean.replace(EMAIL_RE, (full, user, domain) => {
+    emailCount++;
+    return (user as string).slice(0, 2) + "***@" + (domain as string);
+  });
+  if (emailCount > 0) {
+    count += emailCount;
+    details.push(`Masked ${emailCount}x email addresses`);
+  }
+
+  for (const [pattern, replacement] of SENSITIVE_PATTERNS) {
+    const matches = clean.match(pattern);
+    if (matches) {
+      count += matches.length;
+      if (typeof replacement === "string") {
+        details.push(`Redacted ${matches.length}x ${pattern.source.slice(0, 30)}`);
+      } else {
+        details.push(`Masked ${matches.length}x email addresses`);
+      }
+      clean = clean.replace(pattern, replacement as string);
+    }
+  }
+
+  return { clean, redacted: count > 0, count, details };
+}
+
+/* ═══════════════════════════════════════
+   Quick check — is filtering needed?
+   ═══════════════════════════════════════ */
+export function needsFiltering(text: string): boolean {
+  if (!text) return false;
+  // Quick scan with the most common patterns
+  if (/sk-[a-zA-Z0-9]{32,}/.test(text)) return true;
+  if (/api_key.*[:=]/.test(text)) return true;
+  if (/password.*[:=]/.test(text)) return true;
+  if (/-----BEGIN.*PRIVATE KEY-----/.test(text)) return true;
+  if (EMAIL_RE.test(text)) return true;
+  return false;
+}

package/src/core/index.ts

@@ -26,6 +26,12 @@ export * from './skill';
 export * from './router';
 export * from './agent';
 export * from './factory';
+export * from './security';
+export * from './learn';
+export * from './longdoc';
+export * from './filter';
+export * from './estimate';
+export * from './arbitrate';
 
-// Version
-export const VERSION = '1.4.0';
+// Version — read from package.json
+export const VERSION = (() => { try { return require('../../package.json').version; } catch { return '1.6.0'; } })();

package/src/core/longdoc.ts

@@ -0,0 +1,155 @@
+/**
+ * 长文档处理策略 — sliding window + summary chain.
+ *
+ * When an input exceeds the agent's effective context window,
+ * split into overlapping chunks, summarize each, then chain
+ * summaries into a final digest.
+ *
+ * Architecture:
+ *   Input → Chunk(sliding window) → Per-chunk Summary → Chain → Final Digest
+ *
+ * All summaries are generated by the calling agent's LLM, so quality
+ * depends on the model in use. The chunker is pure text processing
+ * and works without any LLM call.
+ */
+
+import type { BaseAgent } from "./agent";
+
+/* ═══════════════════════════════════════
+   Chunker — split text into overlapping windows
+   ═══════════════════════════════════════ */
+export interface ChunkOptions {
+  /** Target chunk size in characters (default 6000) */
+  chunkSize?: number;
+  /** Overlap between consecutive chunks in characters (default 800) */
+  overlap?: number;
+  /** Minimum chunk size before we stop splitting (default 500) */
+  minChunk?: number;
+}
+
+export function chunkText(text: string, opts?: ChunkOptions): string[] {
+  const cs = opts?.chunkSize ?? 6000;
+  const ol = opts?.overlap ?? 800;
+  const min = opts?.minChunk ?? 500;
+  const chunks: string[] = [];
+
+  if (text.length <= cs + min) { chunks.push(text); return chunks; }
+
+  let start = 0;
+  while (start < text.length) {
+    let end = start + cs;
+    if (end >= text.length) { end = text.length; }
+    else {
+      // Try to break at paragraph boundary
+      const searchEnd = Math.min(end + 400, text.length);
+      const paraBreak = text.lastIndexOf("\n\n", searchEnd);
+      if (paraBreak > start + min) end = paraBreak;
+      else {
+        const lineBreak = text.lastIndexOf("\n", searchEnd);
+        if (lineBreak > start + min) end = lineBreak;
+        else {
+          const space = text.lastIndexOf(" ", searchEnd);
+          if (space > start + min) end = space;
+        }
+      }
+    }
+
+    chunks.push(text.slice(start, end).trim());
+    if (end >= text.length) break;
+    start = end - ol;
+    if (start < 0) start = 0;
+  }
+
+  return chunks;
+}
+
+/* ═══════════════════════════════════════
+   Summary chain — ask agent to summarize chunks then chain
+   ═══════════════════════════════════════ */
+export interface SummaryOptions {
+  /** Max total chars for the final digest (default 3000) */
+  maxDigestChars?: number;
+  /** Custom summarization prompt for each chunk */
+  chunkPrompt?: string;
+  /** Custom chain prompt for combining summaries */
+  chainPrompt?: string;
+}
+
+const DEFAULT_CHUNK_PROMPT = `Summarize the following text concisely. Keep all key facts, names, numbers, and code snippets. Output the summary directly without preamble. Limit to 300 words.
+
+Text:
+{text}`;
+
+const DEFAULT_CHAIN_PROMPT = `Combine the following section summaries into a single coherent digest. Preserve all key facts, remove redundancy. Output directly without preamble.
+
+{summaries}`;
+
+export async function summarizeLongDoc(
+  agent: BaseAgent,
+  text: string,
+  opts?: SummaryOptions
+): Promise<string> {
+  const maxDigest = opts?.maxDigestChars ?? 3000;
+  const chunks = chunkText(text);
+
+  // Single chunk — no summarization needed
+  if (chunks.length <= 1) {
+    if (text.length <= maxDigest) return text;
+    const prompt = (opts?.chunkPrompt || DEFAULT_CHUNK_PROMPT).replace("{text}", text);
+    return agent.chatOneshot(prompt, { maxTokens: maxDigest });
+  }
+
+  // Multi-chunk: summarize each, then chain
+  const summaries: string[] = [];
+  for (let i = 0; i < chunks.length; i++) {
+    const prompt = (opts?.chunkPrompt || DEFAULT_CHUNK_PROMPT).replace("{text}", chunks[i]);
+    try {
+      const s = await agent.chatOneshot(prompt, { maxTokens: 600 });
+      summaries.push(s);
+    } catch {
+      summaries.push(chunks[i].slice(0, 400) + "...");
+    }
+  }
+
+  // Chain summaries
+  if (summaries.length === 1) return summaries[0].slice(0, maxDigest);
+
+  const joined = summaries.map((s, i) => `## Section ${i + 1}\n${s}`).join("\n\n");
+  if (joined.length <= maxDigest) return joined;
+
+  const chainPrompt = (opts?.chainPrompt || DEFAULT_CHAIN_PROMPT).replace("{summaries}", joined);
+  const final = await agent.chatOneshot(chainPrompt, { maxTokens: maxDigest });
+  return final.slice(0, maxDigest);
+}
+
+/* ═══════════════════════════════════════
+   Structured data parsing helpers
+   ═══════════════════════════════════════ */
+export function parseStructuredInput(input: string): {
+  hasTable: boolean; hasJSON: boolean; hasCSV: boolean;
+  extractedJSON: string | null; extractedTable: string[][] | null;
+} {
+  const result = { hasTable: false, hasJSON: false, hasCSV: false, extractedJSON: null as string | null, extractedTable: null as string[][] | null };
+
+  // Detect JSON
+  const jsonMatch = input.match(/\{[\s\S]*\}|\[[\s\S]*\]/);
+  if (jsonMatch) {
+    try { JSON.parse(jsonMatch[0]); result.hasJSON = true; result.extractedJSON = jsonMatch[0]; }
+    catch { /* not valid JSON */ }
+  }
+
+  // Detect markdown table
+  const tableMatch = input.match(/\|[\s\S]*?\|/);
+  if (tableMatch) {
+    result.hasTable = true;
+    const lines = input.split("\n").filter(l => l.includes("|") && !l.startsWith("|---") && !l.startsWith("| --"));
+    result.extractedTable = lines.map(l => l.split("|").filter(c => c.trim()).map(c => c.trim()));
+  }
+
+  // Detect CSV
+  if (input.includes(",") && input.split("\n").filter(l => l.includes(",")).length >= 2) {
+    result.hasCSV = true;
+  }
+
+  return result;
+}