recallnest 1.4.1

package/src/event-segmenter.ts

@@ -0,0 +1,216 @@
+/**
+ * Event Segmentation (Tier 4.5)
+ *
+ * Replaces fixed-character chunking with semantic event boundaries for
+ * conversation text. Ensures retrieval gets complete semantic units
+ * (a full Q&A exchange, a complete list, an intact code block) rather
+ * than fragments cut mid-thought.
+ *
+ * Design: rule-based (zero LLM calls), CJK-aware, falls back to chunker
+ * for segments that exceed embedding model limits.
+ */
+
+// ============================================================================
+// Types
+// ============================================================================
+
+export interface EventSegment {
+  text: string;
+  /** Reason this boundary was detected */
+  boundaryType: "topic_shift" | "qa_boundary" | "list_end" | "code_block" | "time_jump" | "initial" | "max_size";
+}
+
+export interface EventSegmenterConfig {
+  /** Max chars per segment before forced split (default: 2000) */
+  maxSegmentSize: number;
+  /** Min chars per segment to avoid fragments (default: 200) */
+  minSegmentSize: number;
+}
+
+export const DEFAULT_EVENT_SEGMENTER_CONFIG: EventSegmenterConfig = {
+  maxSegmentSize: 2000,
+  minSegmentSize: 200,
+};
+
+// ============================================================================
+// Boundary Detection Signals
+// ============================================================================
+
+/**
+ * Topic shift markers — words/phrases that signal a new topic in conversation.
+ * Ordered roughly by strength of signal.
+ */
+const TOPIC_SHIFT_PATTERNS = [
+  // Strong signals (explicit topic change)
+  /^(?:\[用户\]|\[助手\])?\s*(?:另外|顺便|对了|换个话题|说到|关于另一个)/m,
+  /^(?:\[用户\]|\[助手\])?\s*(?:by the way|speaking of|on another note|moving on|anyway|also,? I wanted)/im,
+  // Medium signals (new question after discussion)
+  /^(?:\[用户\])\s*.{0,20}[？?]\s*$/m,
+];
+
+/** Detect if text at a given line index starts a new topic */
+function isTopicShift(line: string): boolean {
+  return TOPIC_SHIFT_PATTERNS.some(p => p.test(line));
+}
+
+/**
+ * Q&A boundary: a [用户] line after an [助手] block signals a new exchange.
+ */
+function isQABoundary(line: string, prevLine: string): boolean {
+  return /^\[用户\]/.test(line) && /^\[助手\]/.test(prevLine.trimStart());
+}
+
+/**
+ * Time jump: date/time markers that indicate a temporal discontinuity.
+ */
+const TIME_JUMP_PATTERN = /^(?:\[用户\]|\[助手\])?\s*(?:\d{4}[-/]\d{1,2}[-/]\d{1,2}|(?:今天|昨天|前天|上周|上个月|today|yesterday|last week))/i;
+
+function isTimeJump(line: string): boolean {
+  return TIME_JUMP_PATTERN.test(line);
+}
+
+// ============================================================================
+// Code Block Tracking
+// ============================================================================
+
+/** Track code fence state to avoid splitting inside code blocks */
+function isCodeFence(line: string): boolean {
+  return /^```/.test(line.trim());
+}
+
+// ============================================================================
+// List Completeness
+// ============================================================================
+
+/** Detect numbered or bulleted list items */
+const LIST_ITEM_PATTERN = /^\s*(?:\d+[.)]\s|[-*+]\s)/;
+
+function isListItem(line: string): boolean {
+  return LIST_ITEM_PATTERN.test(line);
+}
+
+// ============================================================================
+// Core Segmenter
+// ============================================================================
+
+/**
+ * Segment conversation text into semantic event units.
+ *
+ * Strategy:
+ * 1. Walk lines, tracking state (in-code-block, in-list, etc.)
+ * 2. At each potential boundary, decide: split or continue
+ * 3. Respect min/max size constraints
+ * 4. Never split inside code blocks or mid-list
+ */
+export function segmentEvents(
+  text: string,
+  config: EventSegmenterConfig = DEFAULT_EVENT_SEGMENTER_CONFIG,
+): EventSegment[] {
+  if (!text || text.trim().length === 0) return [];
+
+  // Very short text: no point in segmenting
+  if (text.length <= config.minSegmentSize) {
+    return [{ text: text.trim(), boundaryType: "initial" }];
+  }
+
+  const lines = text.split("\n");
+  const segments: EventSegment[] = [];
+  let currentLines: string[] = [];
+  let currentLen = 0;
+  let inCodeBlock = false;
+  let inList = false;
+  let pendingBoundary: EventSegment["boundaryType"] | null = null;
+
+  function flushSegment(boundaryType: EventSegment["boundaryType"]) {
+    const segText = currentLines.join("\n").trim();
+    if (segText.length > 0) {
+      segments.push({ text: segText, boundaryType });
+    }
+    currentLines = [];
+    currentLen = 0;
+    pendingBoundary = null;
+  }
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    const lineLen = line.length + 1; // +1 for \n
+
+    // Track code block state
+    if (isCodeFence(line)) {
+      inCodeBlock = !inCodeBlock;
+      currentLines.push(line);
+      currentLen += lineLen;
+      continue;
+    }
+
+    // Never split inside code blocks
+    if (inCodeBlock) {
+      currentLines.push(line);
+      currentLen += lineLen;
+      // Safety: if code block is absurdly long, force close
+      if (currentLen > config.maxSegmentSize * 2) {
+        inCodeBlock = false;
+        flushSegment("code_block");
+      }
+      continue;
+    }
+
+    // Track list state
+    if (isListItem(line)) {
+      inList = true;
+      currentLines.push(line);
+      currentLen += lineLen;
+      continue;
+    } else if (inList && line.trim().length > 0) {
+      // Non-list line after list items: list ended
+      inList = false;
+      // If accumulated enough, this is a good split point
+      if (currentLen >= config.minSegmentSize) {
+        pendingBoundary = "list_end";
+      }
+    }
+
+    // Check boundary signals (only if we have enough content)
+    if (currentLen >= config.minSegmentSize) {
+      const prevLine = currentLines.length > 0 ? currentLines[currentLines.length - 1] : "";
+
+      if (isTopicShift(line)) {
+        flushSegment("topic_shift");
+      } else if (isQABoundary(line, prevLine)) {
+        flushSegment("qa_boundary");
+      } else if (isTimeJump(line)) {
+        flushSegment("time_jump");
+      } else if (pendingBoundary) {
+        flushSegment(pendingBoundary);
+      }
+    }
+
+    // Force split if exceeding max size
+    if (currentLen + lineLen > config.maxSegmentSize && currentLen >= config.minSegmentSize) {
+      flushSegment("max_size");
+    }
+
+    currentLines.push(line);
+    currentLen += lineLen;
+  }
+
+  // Flush remaining
+  if (currentLines.length > 0) {
+    flushSegment(segments.length === 0 ? "initial" : "max_size");
+  }
+
+  // Merge tiny trailing segments into the previous one
+  if (segments.length > 1) {
+    const last = segments[segments.length - 1];
+    if (last.text.length < config.minSegmentSize) {
+      const prev = segments[segments.length - 2];
+      segments[segments.length - 2] = {
+        text: prev.text + "\n" + last.text,
+        boundaryType: prev.boundaryType,
+      };
+      segments.pop();
+    }
+  }
+
+  return segments;
+}