deepstrike 0.2.41__tar.gz → 0.2.43__tar.gz

{deepstrike-0.2.41 → deepstrike-0.2.43}/Cargo.lock

@@ -194,7 +194,7 @@ dependencies = [
 
 [[package]]
 name = "deepstrike-core"
-version = "0.2.41"
+version = "0.2.43"
 dependencies = [
  "compact_str",
  "pretty_assertions",
@@ -206,7 +206,7 @@ dependencies = [
 
 [[package]]
 name = "deepstrike-lab"
-version = "0.2.41"
+version = "0.2.43"
 dependencies = [
  "deepstrike-core",
  "serde",
@@ -216,7 +216,7 @@ dependencies = [
 
 [[package]]
 name = "deepstrike-node"
-version = "0.2.41"
+version = "0.2.43"
 dependencies = [
  "compact_str",
  "deepstrike-core",
@@ -228,7 +228,7 @@ dependencies = [
 
 [[package]]
 name = "deepstrike-py"
-version = "0.2.41"
+version = "0.2.43"
 dependencies = [
  "compact_str",
  "deepstrike-core",
@@ -239,7 +239,7 @@ dependencies = [
 
 [[package]]
 name = "deepstrike-sdk"
-version = "0.2.41"
+version = "0.2.43"
 dependencies = [
  "async-stream",
  "async-trait",
@@ -273,7 +273,7 @@ dependencies = [
 
 [[package]]
 name = "deepstrike-wasm"
-version = "0.2.41"
+version = "0.2.43"
 dependencies = [
  "compact_str",
  "deepstrike-core",

{deepstrike-0.2.41 → deepstrike-0.2.43}/Cargo.toml

@@ -3,13 +3,13 @@ resolver = "2"
 members = ["crates/deepstrike-core", "crates/deepstrike-py"]
 
 [workspace.package]
-version = "0.2.41"
+version = "0.2.43"
 edition = "2024"
 license = "MIT"
 repository = "https://github.com/kongusen/deepstrike"
 
 [workspace.dependencies]
-deepstrike-core = { path = "crates/deepstrike-core", version = "0.2.41" }
+deepstrike-core = { path = "crates/deepstrike-core", version = "0.2.43" }
 notify = "6"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"

{deepstrike-0.2.41 → deepstrike-0.2.43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepstrike
-Version: 0.2.41
+Version: 0.2.43
 Requires-Dist: httpx>=0.27
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: anyio>=4.0
@@ -206,7 +206,7 @@ A node's `kind` selects the control-flow shape; the same executor drives them al
 
 ### 0.2.11 capabilities
 
-- **Runtime fan-out** — give a node the `submit_workflow_nodes_tool` and its agent can append nodes to the live DAG mid-run (true loop-until-done; one verifier per claim it discovers). Recorded and replayed on `resume_workflow`.
+- **Runtime fan-out** — give a node the `submit_workflow_nodes_tool` and its agent can append nodes to the live DAG mid-run (true loop-until-done; one verifier per claim it discovers). Recorded and replayed on `resume_workflow`. Governance rejection fails the submitting node instead of acknowledging work that was never appended.
 - **Quarantine, no escape** — set `trust="quarantined"` on a node that reads untrusted content; it's denied write-capable isolation in-kernel, and any nodes it submits are coerced to quarantined too (no privilege escalation).
 - **Structured output** — set `output_schema` on a node; the runner instructs the agent, validates the result against the JSON-Schema subset, and re-runs once with the errors on mismatch. A node that never conforms fails (its dependents starve).
 - **Budget as signal** — with a `max_workflow_nodes` / `max_concurrent_subagents` quota installed, each spawned node's goal carries its remaining headroom so a coordinator can size its fan-out to fit.

{deepstrike-0.2.41 → deepstrike-0.2.43}/README.md

@@ -193,7 +193,7 @@ A node's `kind` selects the control-flow shape; the same executor drives them al
 
 ### 0.2.11 capabilities
 
-- **Runtime fan-out** — give a node the `submit_workflow_nodes_tool` and its agent can append nodes to the live DAG mid-run (true loop-until-done; one verifier per claim it discovers). Recorded and replayed on `resume_workflow`.
+- **Runtime fan-out** — give a node the `submit_workflow_nodes_tool` and its agent can append nodes to the live DAG mid-run (true loop-until-done; one verifier per claim it discovers). Recorded and replayed on `resume_workflow`. Governance rejection fails the submitting node instead of acknowledging work that was never appended.
 - **Quarantine, no escape** — set `trust="quarantined"` on a node that reads untrusted content; it's denied write-capable isolation in-kernel, and any nodes it submits are coerced to quarantined too (no privilege escalation).
 - **Structured output** — set `output_schema` on a node; the runner instructs the agent, validates the result against the JSON-Schema subset, and re-runs once with the errors on mismatch. A node that never conforms fails (its dependents starve).
 - **Budget as signal** — with a `max_workflow_nodes` / `max_concurrent_subagents` quota installed, each spawned node's goal carries its remaining headroom so a coordinator can size its fan-out to fit.

{deepstrike-0.2.41 → deepstrike-0.2.43}/crates/deepstrike-core/src/context/manager.rs

@@ -798,9 +798,9 @@ impl ContextManager {
         }
         Some(ToolSchema {
             name: CompactString::new(READ_RESULT_TOOL_NAME),
-            description: "Re-read a tool result that was evicted from context (you see a \
-                          placeholder like '[…tool result spooled…]' or a collapsed entry). \
-                          Pass the tool call's call_id; use offset/max_bytes to page through \
+            description: "Re-read the full content of a tool result that was truncated to save \
+                          context (marked '[Output truncated: … call the read_result tool …]'). \
+                          Pass that marker's call_id; use offset/max_bytes to page through \
                           large content."
                 .to_string(),
             parameters: serde_json::json!({

{deepstrike-0.2.41 → deepstrike-0.2.43}/crates/deepstrike-core/src/context/renderer.rs

@@ -256,7 +256,10 @@ fn normalize_turn_prefix(turns: &mut Vec<Message>) {
 /// preview, leaving a marker. Non-destructive — the full output stays in `partitions.history`;
 /// only the rendered copy shrinks. Un-collapse is boundary-only (P0-C): handles re-evaluate
 /// from Resident at the next compaction/renewal, never mid-generation (cache-safe monotonic).
-fn collapse_preview(output: &str) -> String {
+/// Trained-convention truncation marker: says what was cut and how to get it back (the
+/// `read_result` tool + this result's `call_id`), instead of kernel-internal vocabulary
+/// ("projected out of view") the model has never seen in training.
+fn collapse_preview(output: &str, call_id: &str) -> String {
     const PREVIEW_BYTES: usize = 160;
     let mut end = PREVIEW_BYTES.min(output.len());
     while end > 0 && !output.is_char_boundary(end) {
@@ -264,7 +267,7 @@ fn collapse_preview(output: &str) -> String {
     }
     let dropped = output.len().saturating_sub(end);
     format!(
-        "{}…\n[collapsed: {dropped} chars projected out of view; full result retained in history]",
+        "{}…\n[Output truncated: {dropped} bytes omitted. Call the read_result tool with call_id \"{call_id}\" to re-read the full result.]",
         &output[..end]
     )
 }
@@ -322,7 +325,7 @@ fn project_message(msg: &Message, handles: &HandleTable) -> Option<Message> {
                 changed = true;
                 ContentPart::ToolResult {
                     call_id: call_id.clone(),
-                    output: collapse_preview(output),
+                    output: collapse_preview(output, call_id.as_str()),
                     is_error: *is_error,
                 }
             }
@@ -656,8 +659,10 @@ mod tests {
                 _ => None,
             })
             .expect("tool result rendered");
-        // Rendered copy is a preview; original full output is retained in history.
-        assert!(rendered.contains("[collapsed:"));
+        // Rendered copy is a preview with the trained truncation phrasing + retrieval instruction.
+        assert!(rendered.contains("[Output truncated:"));
+        assert!(rendered.contains("read_result"));
+        assert!(rendered.contains("\"c1\""));
         assert!(rendered.len() < long.len());
         let stored = match &c.history.messages[0].content {
             Content::Parts(parts) => match &parts[0] {
@@ -700,7 +705,7 @@ mod tests {
             })
             .expect("tool result rendered");
         assert_eq!(rendered, body);
-        assert!(!rendered.contains("[collapsed:"));
+        assert!(!rendered.contains("[Output truncated:"));
     }
 
     // ── P1-F: state-turn recency footer ───────────────────────────────────

{deepstrike-0.2.41 → deepstrike-0.2.43}/crates/deepstrike-core/src/context/summarizer.rs

@@ -5,6 +5,11 @@ use crate::types::message::{Content, ContentPart, Message};
 /// Deterministic six-slot summariser used before archived units page out.
 pub struct RuleSummarizer;
 
+/// Items rendered per slot before an honest `(+N more)` line. Bounds each digest so the
+/// task-state compression history can afford to keep MANY digests visible instead of a few
+/// bloated ones.
+const SLOT_ITEM_CAP: usize = 6;
+
 impl RuleSummarizer {
     /// Produce a structured summary whose char-approx token count never exceeds
     /// `max_tokens`. Slot order is the deterministic truncation priority.
@@ -95,10 +100,18 @@ impl RuleSummarizer {
             if values.is_empty() {
                 push_line(&mut output, "- none", max_tokens, &engine);
             } else {
-                for value in values {
+                for value in values.iter().take(SLOT_ITEM_CAP) {
                     push_line(
                         &mut output,
-                        &format!("- {}", compact(&value, 240)),
+                        &format!("- {}", compact(value, 240)),
+                        max_tokens,
+                        &engine,
+                    );
+                }
+                if values.len() > SLOT_ITEM_CAP {
+                    push_line(
+                        &mut output,
+                        &format!("- (+{} more)", values.len() - SLOT_ITEM_CAP),
                         max_tokens,
                         &engine,
                     );
@@ -126,6 +139,9 @@ struct SummarySlots {
 
 fn classify_text(text: &str, slots: &mut SummarySlots) {
     for statement in statements(text) {
+        if is_diff_noise(&statement) {
+            continue;
+        }
         let folded = statement.to_lowercase();
         if contains_any(
             &folded,
@@ -199,11 +215,45 @@ fn classify_text(text: &str, slots: &mut SummarySlots) {
 }
 
 fn statements(text: &str) -> Vec<String> {
-    text.split(['\n', '.', '!', '?', ';', '。', '！', '？', '；'])
-        .map(str::trim)
-        .filter(|statement| !statement.is_empty())
-        .map(|statement| compact(statement, 240))
-        .collect()
+    let mut output = Vec::new();
+    let mut current = String::new();
+    let mut chars = text.chars().peekable();
+    while let Some(character) = chars.next() {
+        let boundary = match character {
+            '\n' | '!' | '?' | ';' | '。' | '！' | '？' | '；' => true,
+            // A '.' splits only at a sentence boundary (followed by whitespace or end of
+            // text). Splitting inside paths/versions shredded `src/auth.js` into useless
+            // `js b/src/auth` fragments that polluted every digest.
+            '.' => chars.peek().is_none_or(|next| next.is_whitespace()),
+            _ => false,
+        };
+        if boundary {
+            flush_statement(&mut current, &mut output);
+        } else {
+            current.push(character);
+        }
+    }
+    flush_statement(&mut current, &mut output);
+    output
+}
+
+fn flush_statement(current: &mut String, output: &mut Vec<String>) {
+    let statement = current.trim();
+    if !statement.is_empty() {
+        output.push(compact(statement, 240));
+    }
+    current.clear();
+}
+
+/// Structural diff/patch header lines carry no summarizable content — dropping them keeps
+/// digests dense so more real process state survives a given summary budget.
+fn is_diff_noise(statement: &str) -> bool {
+    let trimmed = statement.trim_start();
+    trimmed.starts_with("diff --git")
+        || trimmed.starts_with("+++")
+        || trimmed.starts_with("---")
+        || trimmed.starts_with("@@")
+        || trimmed.starts_with("index ")
 }
 
 fn contains_any(text: &str, markers: &[&str]) -> bool {

{deepstrike-0.2.41 → deepstrike-0.2.43}/crates/deepstrike-core/src/context/task_state.rs

@@ -46,10 +46,19 @@ pub struct TaskState {
     /// so updating it never churns the prompt cache. Bounded + recency-ordered; newest last.
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub recent_actions: Vec<String>,
-    /// Append-only log of all compression events. Never overwritten.
+    /// Rolling log of compression events, newest last. Bounded at [`MAX_COMPRESSION_LOG`]
+    /// (oldest dropped past the cap, counted in `compression_log_dropped`).
     /// Rendered into systemVolatile so the model always sees compression history.
     #[serde(default, skip_serializing_if = "Vec::is_empty")]
     pub compression_log: Vec<CompressionEntry>,
+    /// Entries dropped from `compression_log` past its cap — kept so the render stays honest
+    /// about how much history is no longer visible.
+    #[serde(default, skip_serializing_if = "u64_is_zero")]
+    pub compression_log_dropped: u64,
+}
+
+fn u64_is_zero(value: &u64) -> bool {
+    *value == 0
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -73,6 +82,9 @@ pub const MAX_DIRECTIVES: usize = 8;
 /// Maximum recent action-turns retained for the recency footer (bounded ring).
 pub const MAX_RECENT_ACTIONS: usize = 6;
 
+/// Compression-log entries retained in state (rolling window; render shows the last 3).
+pub const MAX_COMPRESSION_LOG: usize = 64;
+
 /// Partial update applied by the SDK or via `update_plan` meta-tool.
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct TaskUpdate {
@@ -136,7 +148,14 @@ impl TaskState {
             lines.push(format!("scratchpad: {}", self.scratchpad));
         }
 
-        // Render the most recent compression events (cap at 3 to limit token cost).
+        // Render only the most recent compression events (fixed cap of 3). A wider budgeted
+        // window was tried and withdrawn: live A/B at n≤4 (12-PR review @ 2048, DeepSeek)
+        // could not distinguish it from provider drift — a control run on the unmodified
+        // kernel showed the same failures — and a window full of old `tool X args` digest
+        // lines is plausible re-execution bait. Absent replay-lab evidence FOR widening, the
+        // proven shape stays. Older digests remain in `compression_log` (bounded at
+        // [`MAX_COMPRESSION_LOG`]); making their *content* durably useful to the model is
+        // semantic-summary (P2) territory, not a bigger raw window.
         if !self.compression_log.is_empty() {
             lines.push("compression_history:".to_string());
             let start = self.compression_log.len().saturating_sub(3);
@@ -184,12 +203,18 @@ impl TaskState {
         }
     }
 
-    /// Append a compression event to the log. Never overwrites existing entries.
+    /// Append a compression event to the log (bounded at [`MAX_COMPRESSION_LOG`]; the oldest
+    /// entries are dropped past the cap and counted so the render can report them).
     pub fn log_compression(&mut self, action: &str, summary: String) {
         self.compression_log.push(CompressionEntry {
             action: action.to_string(),
             summary,
         });
+        if self.compression_log.len() > MAX_COMPRESSION_LOG {
+            let overflow = self.compression_log.len() - MAX_COMPRESSION_LOG;
+            self.compression_log.drain(0..overflow);
+            self.compression_log_dropped += overflow as u64;
+        }
     }
 
     pub fn apply(&mut self, update: TaskUpdate) {
@@ -349,6 +374,33 @@ mod tests {
         assert_eq!(ts.directives.len(), MAX_DIRECTIVES);
     }
 
+    #[test]
+    fn compression_render_shows_exactly_the_last_three_digests() {
+        // Pin the 3-entry window (see format_compact for why widening was withdrawn) so a
+        // future widening must re-justify itself with replay-lab evidence.
+        let mut ts = TaskState::default();
+        ts.goal = "review PRs".into();
+        for n in 1..=10 {
+            ts.log_compression("auto_compact", format!("digest {n}"));
+        }
+        let rendered = ts.format_compact();
+        assert!(!rendered.contains("digest 7\n"));
+        assert!(rendered.contains("digest 8"));
+        assert!(rendered.contains("digest 9"));
+        assert!(rendered.contains("digest 10"));
+    }
+
+    #[test]
+    fn compression_log_is_bounded_and_counts_drops() {
+        let mut ts = TaskState::default();
+        for n in 0..(MAX_COMPRESSION_LOG + 5) {
+            ts.log_compression("micro_compact", format!("d{n}"));
+        }
+        assert_eq!(ts.compression_log.len(), MAX_COMPRESSION_LOG);
+        assert_eq!(ts.compression_log_dropped, 5);
+        assert_eq!(ts.compression_log[0].summary, "d5");
+    }
+
     #[test]
     fn apply_updates_fields() {
         let mut ts = TaskState::default();

{deepstrike-0.2.41 → deepstrike-0.2.43}/crates/deepstrike-core/src/context/utility.rs

@@ -6,6 +6,7 @@ use std::ops::Range;
 
 use super::token_engine::ContextTokenEngine;
 use super::units::unit_boundaries;
+use crate::lexical::{overlap_count, terms};
 use crate::types::message::{Content, ContentPart, Message};
 
 pub struct UtilitySelectionContext<'a> {
@@ -215,35 +216,6 @@ fn unit_text(messages: &[Message]) -> String {
     parts.join("\n")
 }
 
-fn terms(text: &str) -> BTreeSet<String> {
-    let mut output = BTreeSet::new();
-    let mut ascii = String::new();
-    for character in text.chars().flat_map(char::to_lowercase) {
-        if character.is_ascii_alphanumeric() || matches!(character, '_' | '-' | '/' | '.' | ':') {
-            ascii.push(character);
-            continue;
-        }
-        if !ascii.is_empty() {
-            if ascii.chars().count() > 1 {
-                output.insert(std::mem::take(&mut ascii));
-            } else {
-                ascii.clear();
-            }
-        }
-        if !character.is_whitespace() && !character.is_ascii_punctuation() {
-            output.insert(character.to_string());
-        }
-    }
-    if ascii.chars().count() > 1 {
-        output.insert(ascii);
-    }
-    output
-}
-
-fn overlap_count(left: &BTreeSet<String>, right: &BTreeSet<String>) -> u32 {
-    left.intersection(right).count() as u32
-}
-
 fn contains_folded(text: &str, pattern: &str) -> bool {
     !pattern.trim().is_empty() && text.to_lowercase().contains(&pattern.to_lowercase())
 }
@@ -367,6 +339,41 @@ mod tests {
         assert_eq!(plan.retained_tokens, 40);
     }
 
+    #[test]
+    fn chinese_directive_dependency_requires_bigram_overlap_not_shared_characters() {
+        // Under the old per-character CJK vocabulary, any Chinese unit sharing two
+        // common characters (中/文/回…) with an active directive was marked mandatory,
+        // so compression could never archive unrelated Chinese history.
+        let mut unrelated = Message::assistant("我们在文中回顾了天气");
+        unrelated.token_count = Some(30);
+        let mut on_topic = Message::user("已按要求保持中文回答");
+        on_topic.token_count = Some(30);
+        let mut recent = Message::user("recent");
+        recent.token_count = Some(10);
+        let messages = vec![unrelated, on_topic, recent];
+        let plan = plan_utility_archive(
+            &messages,
+            70,
+            10,
+            1,
+            &ContextTokenEngine::char_approx(),
+            &UtilitySelectionContext {
+                goal: "",
+                criteria: &[],
+                preserved_refs: &[],
+                active_directives: &["必须用中文回答".into()],
+            },
+        );
+        assert!(
+            !plan.scores[0].mandatory,
+            "unrelated Chinese text must not bind to the directive"
+        );
+        assert!(
+            plan.scores[1].mandatory,
+            "text restating the directive must stay mandatory"
+        );
+    }
+
     #[test]
     fn preserved_ref_keeps_complete_tool_unit() {
         let mut call = Message::assistant("read artifact");

deepstrike-0.2.43/crates/deepstrike-core/src/lexical.rs

@@ -0,0 +1,178 @@
+//! Shared lexical vocabulary for deterministic in-kernel value scoring.
+//!
+//! Compression utility (goal overlap, directive dependency) and memory curation
+//! (fuzzy dedupe) must speak ONE term vocabulary — and it must discriminate for
+//! CJK text, where there is no whitespace between words: per-character unigrams
+//! make every Chinese unit overlap every Chinese goal, and whitespace tokens
+//! make a whole Chinese sentence a single token. Terms here are lowercased word
+//! runs plus Han character bigrams, matching the host-side recall rankers
+//! (node `memory/ranking.ts`, python `memory/ranking.py`) so knowledge
+//! residency, compression, and memory recall share one notion of "relevant".
+
+use std::collections::BTreeSet;
+
+/// Deterministic term set of `text`.
+///
+/// - ASCII word runs (`[a-z0-9_\-/.:]`, lowercased) of more than one char are
+///   one term each — call ids, paths, and URLs stay whole.
+/// - Non-ASCII alphanumeric runs of more than one char are one term each; a
+///   lone non-ASCII char is kept as its own term.
+/// - Runs containing Han chars additionally emit every adjacent-char bigram,
+///   the standard whitespace-free segmentation proxy.
+///
+/// Punctuation (ASCII and fullwidth alike) and whitespace never become terms.
+pub fn terms(text: &str) -> BTreeSet<String> {
+    let mut output = BTreeSet::new();
+    let mut ascii_run = String::new();
+    let mut wide_run: Vec<char> = Vec::new();
+    for character in text.chars().flat_map(char::to_lowercase) {
+        if character.is_ascii_alphanumeric() || matches!(character, '_' | '-' | '/' | '.' | ':') {
+            flush_wide(&mut wide_run, &mut output);
+            ascii_run.push(character);
+        } else if !character.is_ascii() && character.is_alphanumeric() {
+            flush_ascii(&mut ascii_run, &mut output);
+            wide_run.push(character);
+        } else {
+            flush_ascii(&mut ascii_run, &mut output);
+            flush_wide(&mut wide_run, &mut output);
+        }
+    }
+    flush_ascii(&mut ascii_run, &mut output);
+    flush_wide(&mut wide_run, &mut output);
+    output
+}
+
+/// Number of terms shared by both sets.
+pub fn overlap_count(left: &BTreeSet<String>, right: &BTreeSet<String>) -> u32 {
+    left.intersection(right).count() as u32
+}
+
+/// Term-set Jaccard similarity over the shared vocabulary.
+///
+/// Inputs whose combined vocabulary is empty score 0.0 — no lexical evidence is
+/// treated as "not a duplicate", so degenerate content fails open (both records
+/// are kept) rather than merging on nothing.
+pub fn jaccard(left: &str, right: &str) -> f64 {
+    let left = terms(left);
+    let right = terms(right);
+    let union = left.union(&right).count();
+    if union == 0 {
+        return 0.0;
+    }
+    left.intersection(&right).count() as f64 / union as f64
+}
+
+fn flush_ascii(run: &mut String, output: &mut BTreeSet<String>) {
+    // All-ASCII, so byte length equals char count.
+    if run.len() > 1 {
+        output.insert(std::mem::take(run));
+    } else {
+        run.clear();
+    }
+}
+
+fn flush_wide(run: &mut Vec<char>, output: &mut BTreeSet<String>) {
+    match run.len() {
+        0 => return,
+        1 => {
+            output.insert(run[0].to_string());
+        }
+        _ => {
+            output.insert(run.iter().collect());
+            if run.iter().copied().any(is_han) {
+                for pair in run.windows(2) {
+                    output.insert(pair.iter().collect());
+                }
+            }
+        }
+    }
+    run.clear();
+}
+
+/// Han ranges mirrored from the host rankers (BMP unified + ext A, compat, ext B+).
+fn is_han(character: char) -> bool {
+    matches!(
+        u32::from(character),
+        0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xF900..=0xFAFF | 0x20000..=0x3134F
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn ascii_words_paths_and_ids_stay_whole() {
+        let output = terms("Read src/main.rs via call_abc-123, then STOP.");
+        assert!(output.contains("read"));
+        assert!(output.contains("src/main.rs"));
+        assert!(output.contains("call_abc-123"));
+        // Single ASCII chars never become terms; punctuation is stripped.
+        assert!(!output.contains("a"));
+        assert!(!output.contains(","));
+    }
+
+    #[test]
+    fn han_runs_emit_bigrams_not_unigrams() {
+        let output = terms("实现用户登录");
+        assert!(output.contains("实现"));
+        assert!(output.contains("用户"));
+        assert!(output.contains("户登"));
+        assert!(output.contains("实现用户登录"));
+        assert!(!output.contains("实"), "unigram noise must be gone");
+    }
+
+    #[test]
+    fn chinese_goal_overlap_discriminates() {
+        let goal = terms("实现用户登录功能");
+        let on_topic = terms("已完成用户登录表单");
+        let off_topic = terms("今天天气很好我们去公园");
+        assert!(overlap_count(&goal, &on_topic) >= 2);
+        assert_eq!(overlap_count(&goal, &off_topic), 0);
+    }
+
+    #[test]
+    fn fullwidth_punctuation_is_not_a_term() {
+        let output = terms("完成了。下一步：测试！");
+        assert!(!output.contains("。"));
+        assert!(!output.contains("："));
+        assert!(output.contains("完成"));
+    }
+
+    #[test]
+    fn lone_wide_char_is_kept() {
+        assert!(terms("改 a").contains("改"));
+    }
+
+    #[test]
+    fn mixed_ascii_and_han_split_into_both_vocabularies() {
+        let output = terms("部署v2服务到prod环境");
+        assert!(output.contains("v2"));
+        assert!(output.contains("prod"));
+        assert!(output.contains("部署"));
+        assert!(output.contains("服务"));
+        assert!(output.contains("环境"));
+    }
+
+    #[test]
+    fn jaccard_near_duplicate_chinese_scores_high() {
+        let near = jaccard("用户偏好深色模式界面", "用户偏好浅色模式界面");
+        let unrelated = jaccard("用户偏好深色模式界面", "周五之前完成部署上线");
+        assert!(near > 0.5, "near-duplicates must be detectable: {near}");
+        assert!(unrelated < 0.2, "unrelated must stay low: {unrelated}");
+    }
+
+    #[test]
+    fn jaccard_identical_english_is_one_and_empty_vocabulary_is_zero() {
+        assert_eq!(jaccard("prefer cargo nextest", "prefer cargo nextest"), 1.0);
+        assert_eq!(jaccard("", ""), 0.0);
+    }
+
+    #[test]
+    fn non_han_scripts_keep_whole_words_without_bigrams() {
+        let output = terms("привет мир");
+        assert!(output.contains("привет"));
+        assert!(output.contains("мир"));
+        assert!(!output.contains("пр"));
+    }
+}

{deepstrike-0.2.41 → deepstrike-0.2.43}/crates/deepstrike-core/src/lib.rs

@@ -28,6 +28,7 @@ pub mod benchmark;
 pub mod context;
 pub mod governance;
 pub mod harness;
+pub mod lexical;
 pub mod memory;
 pub mod mm;
 pub mod orchestration;