thatgfsj-code 2.2.9 → 3.0.1

package/src/cache/smartModel.ts

@@ -0,0 +1,52 @@
+/**
+ * Smart-model routing (P3 of the Reasonix plan).
+ *
+ * Idea: classify the user's input as "simple" or "complex" and route
+ * simple queries to a cheaper / faster model while keeping the main model
+ * for complex work. The classification is intentionally trivial — prompt
+ * length + presence of recent tool calls — because the cost of a
+ * misclassification (a slightly worse answer on a simple query) is much
+ * lower than the cost of running a frontier model on every greeting.
+ *
+ * Trade-off: each routing decision adds one model swap (different
+ * `model` field on the wire). If the cache prefix is keyed on the model
+ * string, the cache miss rate can spike on every simple query. To keep
+ * the prefix stable, callers should pass a *family* (e.g. "anthropic" or
+ * "deepseek") rather than a specific model name when forwarding the
+ * prompt to the downstream provider — that's outside the scope of this
+ * hook; this module only answers the question "should I downgrade?".
+ *
+ * v3.0.0: shipped as a stub. Real decision logic is conservative
+ * (downgrade only when prompt is short AND no recent tool activity).
+ * Future iterations can add heuristic or classifier-based routing.
+ */
+
+import type { ChatMessage } from '../types.js';
+
+export interface SmartModelDecision {
+  /** Should we route this turn to a cheaper model? */
+  downgrade: boolean;
+  /** Why we made this call — useful for `/cache` debug output. */
+  reason: string;
+}
+
+/**
+ * Heuristic: downgrade when
+ *   1. The latest user message is short (< 200 chars), AND
+ *   2. The last 3 messages contain no assistant tool_calls.
+ *
+ * This covers greetings, one-line questions, and simple follow-ups
+ * ("thanks", "ok", "explain this one line") while routing anything
+ * that triggered a tool to the main model.
+ */
+export function shouldDowngrade(messages: ChatMessage[], lastUserInput: string): SmartModelDecision {
+  if (lastUserInput.length > 200) {
+    return { downgrade: false, reason: 'prompt-too-long' };
+  }
+  const recent = messages.slice(-3);
+  const hasRecentToolCalls = recent.some(m => m.tool_calls && m.tool_calls.length > 0);
+  if (hasRecentToolCalls) {
+    return { downgrade: false, reason: 'recent-tool-call' };
+  }
+  return { downgrade: true, reason: 'short-no-tools' };
+}

package/src/cache/stats.ts

@@ -0,0 +1,199 @@
+/**
+ * Cache statistics store.
+ *
+ * Tracks per-round cache hit/miss token counts across the user's local
+ * sessions and persists them to `~/.thatgfsj/cache-stats.json`. The store
+ * is consumed by:
+ *   - the TUI Header (`⚡ 命中率 87%`)
+ *   - the /cache CLI command (detailed breakdown)
+ *   - the /cache reset command (clears the file)
+ *
+ * Provider-agnostic. Anthropic reports cache_creation_input_tokens /
+ * cache_read_input_tokens; DeepSeek reports prompt_cache_hit_tokens /
+ * prompt_cache_miss_tokens; OpenAI does not surface any cache stats at
+ * all (their automatic cache is invisible to the client). We normalize
+ * both shapes into a single internal model.
+ *
+ * Cost estimation is intentionally simplified:
+ *   - cache_read = 10% of normal input price
+ *   - cache_creation = 125% of normal input price (Anthropic charges extra
+ *     on first write)
+ *   - We compare against the baseline "what would this have cost without
+ *     caching" and report the delta as estimated savings.
+ *
+ * Prices are in CNY per million tokens. Defaults reflect Anthropic Claude
+ * Sonnet on the official API; users on cheaper providers may see savings
+ * proportional to their actual price floor. The numbers are *estimates*
+ * — the goal is to give the user a sense of order of magnitude, not a
+ * billable invoice.
+ */
+
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs';
+import { dirname, join } from 'path';
+import { homedir } from 'os';
+import type { Usage } from '../types.js';
+
+/** Per-million-token price in CNY. Adjust if your provider differs significantly. */
+const PRICE = {
+  normalInput: 3,
+  cacheRead: 0.3,
+  cacheCreation: 3.75,
+  output: 15,
+};
+
+/** Maximum number of recent rounds kept in the history array (rolling window). */
+const HISTORY_LIMIT = 50;
+
+export interface CacheStats {
+  /** Total tokens served from cache (Anthropic cache_read / DeepSeek hit). */
+  totalReadTokens: number;
+  /** Total tokens written into cache (Anthropic cache_creation / DeepSeek miss prefix written). */
+  totalCreationTokens: number;
+  /** Total input tokens (prompt_tokens). */
+  totalInputTokens: number;
+  /** Total output tokens. */
+  totalOutputTokens: number;
+  /** Number of completed rounds. */
+  totalRequests: number;
+  /** Last N round snapshots (oldest first). */
+  history: Array<{
+    ts: number;
+    read: number;
+    creation: number;
+    input: number;
+    output: number;
+    hitRate: number;
+  }>;
+}
+
+export interface CacheSnapshot extends CacheStats {
+  /** 0..1 ratio of cached tokens to total input tokens. */
+  hitRate: number;
+  /** Estimated CNY saved vs uncached baseline. */
+  estimatedSavingsCNY: number;
+}
+
+const EMPTY_STATS: CacheStats = {
+  totalReadTokens: 0,
+  totalCreationTokens: 0,
+  totalInputTokens: 0,
+  totalOutputTokens: 0,
+  totalRequests: 0,
+  history: [],
+};
+
+/**
+ * Estimate the CNY saved by cache_read vs a hypothetical uncached baseline.
+ * Negative values are clamped to zero (e.g. a relay station that returned
+ * bogus cache stats).
+ */
+export function estimateSavingsCNY(stats: CacheStats): number {
+  const baselineCost = (stats.totalReadTokens / 1e6) * PRICE.normalInput;
+  const cachedCost = (stats.totalReadTokens / 1e6) * PRICE.cacheRead;
+  return Math.max(0, baselineCost - cachedCost);
+}
+
+/**
+ * Persists cache stats to disk. Safe to call from a hot path — uses atomic
+ * write (write to .tmp, rename) to avoid corruption on crash.
+ */
+export class CacheStatsStore {
+  private stats: CacheStats;
+  private path: string;
+
+  constructor(path?: string) {
+    this.path = path ?? join(homedir(), '.thatgfsj', 'cache-stats.json');
+    this.stats = this.load();
+  }
+
+  /**
+   * Record a single round's usage. Provider-agnostic: accepts both Anthropic
+   * (cache_creation_input_tokens / cache_read_input_tokens) and DeepSeek
+   * (prompt_cache_hit_tokens / prompt_cache_miss_tokens) shapes. If a
+   * provider returned neither (e.g. OpenAI), the call is a no-op apart
+   * from updating totalRequest count.
+   */
+  record(usage: Usage): void {
+    const read = usage.cache_read_input_tokens ?? usage.prompt_cache_hit_tokens ?? 0;
+    const creation = usage.cache_creation_input_tokens
+      ?? (usage.prompt_cache_miss_tokens ?? 0);
+    const input = usage.prompt_tokens ?? 0;
+    const output = usage.completion_tokens ?? 0;
+
+    // Avoid double-counting: if the provider reported both cache_creation
+    // and prompt_cache_miss_tokens we'd otherwise sum them. The ?? in the
+    // chain above only uses prompt_cache_miss_tokens when
+    // cache_creation_input_tokens is undefined, which is what we want.
+
+    this.stats.totalReadTokens += read;
+    this.stats.totalCreationTokens += creation;
+    this.stats.totalInputTokens += input;
+    this.stats.totalOutputTokens += output;
+    this.stats.totalRequests += 1;
+
+    // Per-round snapshot (for /cache command history view).
+    const total = read + (input - read);
+    const hitRate = total > 0 ? read / total : 0;
+    this.stats.history.push({ ts: Date.now(), read, creation, input, output, hitRate });
+    if (this.stats.history.length > HISTORY_LIMIT) {
+      this.stats.history.splice(0, this.stats.history.length - HISTORY_LIMIT);
+    }
+
+    this.save();
+  }
+
+  /**
+   * Reset the stats. Used by `/cache reset`.
+   */
+  reset(): void {
+    this.stats = { ...EMPTY_STATS, history: [] };
+    this.save();
+  }
+
+  /**
+   * Read-only snapshot for rendering (TUI Header, /cache command).
+   */
+  snapshot(): CacheSnapshot {
+    const input = this.stats.totalInputTokens;
+    const read = this.stats.totalReadTokens;
+    const hitRate = input > 0 ? read / input : 0;
+    return {
+      ...this.stats,
+      hitRate,
+      estimatedSavingsCNY: estimateSavingsCNY(this.stats),
+    };
+  }
+
+  /** Direct accessor for tests. */
+  get raw(): CacheStats { return this.stats; }
+
+  // -- I/O --
+
+  private load(): CacheStats {
+    try {
+      if (!existsSync(this.path)) return { ...EMPTY_STATS, history: [] };
+      const txt = readFileSync(this.path, 'utf-8');
+      const parsed = JSON.parse(txt);
+      // Tolerate partial / older shapes.
+      return {
+        totalReadTokens: parsed.totalReadTokens ?? 0,
+        totalCreationTokens: parsed.totalCreationTokens ?? 0,
+        totalInputTokens: parsed.totalInputTokens ?? 0,
+        totalOutputTokens: parsed.totalOutputTokens ?? 0,
+        totalRequests: parsed.totalRequests ?? 0,
+        history: Array.isArray(parsed.history) ? parsed.history : [],
+      };
+    } catch {
+      return { ...EMPTY_STATS, history: [] };
+    }
+  }
+
+  private save(): void {
+    try {
+      mkdirSync(dirname(this.path), { recursive: true });
+      writeFileSync(this.path, JSON.stringify(this.stats, null, 2), 'utf-8');
+    } catch {
+      // best-effort persistence; do not crash the chat loop on disk errors
+    }
+  }
+}

package/src/cache/volatile.ts

@@ -0,0 +1,47 @@
+/**
+ * VolatileScratch — per-round reasoning / planning state that MUST NOT be
+ * uploaded to the API.
+ *
+ * Reasonix Principle #3 (Volatile Scratch):
+ *   The model's thinking content, intermediate plans, and other
+ *   intra-round state should never be appended to the conversation
+ *   history. They are useful for the UI (so users can debug / audit
+ *   reasoning) and for cross-iteration handoff within the same round
+ *   (e.g. retry-after-tool-failure), but they are NOT part of the cache
+ *   prefix and re-uploading them in the next round would:
+ *     1. waste tokens (potentially a lot — reasoning can dwarf the answer)
+ *     2. cause context-window pressure
+ *     3. risk leaking partial / contradictory reasoning back to the model
+ *
+ * Usage (in LLMService / useChat when receiving 'thinking' chunks):
+ *   scratch.write(chunk.content);
+ *   // ... later in the same round ...
+ *   scratch.read(); // → joined reasoning so far
+ *   scratch.reset(); // → call at end of round OR on /new
+ */
+export class VolatileScratch {
+  private current: string[] = [];
+
+  reset(): void {
+    this.current = [];
+  }
+
+  write(data: string): void {
+    if (!data) return;
+    this.current.push(data);
+  }
+
+  read(): string {
+    return this.current.join('\n');
+  }
+
+  /** True when no scratch has been written since the last reset. */
+  isEmpty(): boolean {
+    return this.current.length === 0;
+  }
+
+  /** Number of segments written. Useful for UI display ("thought for N lines"). */
+  size(): number {
+    return this.current.length;
+  }
+}