ultra-memory 3.2.0 → 4.1.0

package/scripts/conflict_detector.py

@@ -0,0 +1,320 @@
+#!/usr/bin/env python3
+"""
+ultra-memory: 矛盾检测核心模块
+检测 user_profile.json 和 knowledge_base.jsonl 中的时序矛盾，
+将旧记录标记为 superseded: true，召回时跳过失效记录。
+
+只使用 Python 标准库，无外部依赖。
+"""
+
+import os
+import re
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+# ── 停用词表 ────────────────────────────────────────────────────────────────
+
+STOPWORDS = {
+    "的", "了", "是", "在", "和", "与", "或", "以及",
+    "a", "an", "the", "is", "was", "are", "were",
+    "to", "of", "for", "with", "by", "from",
+}
+
+# ── 否定词表 ────────────────────────────────────────────────────────────────
+
+NEGATION_WORDS = {
+    "不", "没有", "无法", "不能", "不是", "别", "莫", "永不", "绝不",
+    "not", "no", "never", "none", "cannot", "don't", "doesn't",
+    "isn't", "aren't", "wasn't", "weren't", "won't", "wouldn't",
+}
+
+# ── 对立词对 ───────────────────────────────────────────────────────────────
+
+CONTRADICTORY_PAIRS = [
+    frozenset(["成功", "失败"]),
+    frozenset(["可以", "不能"]),
+    frozenset(["推荐", "不推荐"]),
+    frozenset(["启用", "禁用"]),
+    frozenset(["enable", "disable"]),
+    frozenset(["success", "failure"]),
+    frozenset(["yes", "no"]),
+    frozenset(["true", "false"]),
+]
+
+
+# ── 工具函数 ───────────────────────────────────────────────────────────────
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def _stopword_filter(words: list[str]) -> list[str]:
+    return [w for w in words if w.lower() not in STOPWORDS and len(w) > 1]
+
+
+# ── 画像冲突检测 ───────────────────────────────────────────────────────────
+
+
+def detect_profile_conflict(new_data: dict, home: Path) -> list[dict]:
+    """
+    检测 user_profile.json 中的字段冲突。
+    返回冲突列表，每条格式：
+      {"field": "...", "old_value": "...", "new_value": "...", "superseded_at": "..."}
+    """
+    profile_file = home / "semantic" / "user_profile.json"
+    if not profile_file.exists():
+        return []
+
+    try:
+        with open(profile_file, encoding="utf-8") as f:
+            profile = json.load(f)
+    except (json.JSONDecodeError, IOError):
+        return []
+
+    conflicts = []
+
+    for field, new_value in new_data.items():
+        # observed_patterns 只追加，不检测冲突
+        if field == "observed_patterns":
+            continue
+
+        # 跳过 superseded 标记的字段
+        if field.endswith("_superseded"):
+            continue
+
+        if field not in profile:
+            continue
+
+        old_value = profile[field]
+
+        # 字符串类型：旧值与新值不同 → 冲突
+        if isinstance(old_value, str) and isinstance(new_value, str):
+            if old_value != new_value:
+                conflicts.append({
+                    "field": field,
+                    "old_value": old_value,
+                    "new_value": new_value,
+                    "superseded_at": _now_iso(),
+                })
+
+        # 数组类型：完全替换（完全不同）→ 冲突；只新增元素 → 不冲突
+        elif isinstance(old_value, list) and isinstance(new_value, list):
+            # 新旧完全不同才算冲突；新值包含旧值的所有元素（新增元素）不算冲突
+            old_set = set(str(v) for v in old_value)
+            new_set = set(str(v) for v in new_value)
+            if old_set and new_set and old_set != new_set:
+                # 只有当旧值中的元素不在新值中，才算冲突
+                removed = old_set - new_set
+                if removed:  # 有元素被移除才算冲突
+                    conflicts.append({
+                        "field": field,
+                        "old_value": old_value,
+                        "new_value": new_value,
+                        "superseded_at": _now_iso(),
+                    })
+
+        # dict 类型：递归检测每个 key
+        elif isinstance(old_value, dict) and isinstance(new_value, dict):
+            for sub_key, sub_old in old_value.items():
+                if sub_key not in new_value:
+                    continue
+                sub_new = new_value[sub_key]
+                if isinstance(sub_old, str) and isinstance(sub_new, str):
+                    if sub_old != sub_new:
+                        full_key = f"{field}.{sub_key}"
+                        conflicts.append({
+                            "field": full_key,
+                            "old_value": sub_old,
+                            "new_value": "（已更新）" if sub_new else sub_new,
+                            "superseded_at": _now_iso(),
+                        })
+
+    return conflicts
+
+
+def mark_profile_superseded(home: Path, conflicts: list[dict]):
+    """
+    将 user_profile.json 中冲突的旧字段标记为 superseded。
+    在旧字段旁追加 "<field>_superseded" 记录旧值。
+    """
+    profile_file = home / "semantic" / "user_profile.json"
+    if not profile_file.exists() or not conflicts:
+        return
+
+    try:
+        with open(profile_file, encoding="utf-8") as f:
+            profile = json.load(f)
+    except (json.JSONDecodeError, IOError):
+        return
+
+    for conflict in conflicts:
+        field = conflict["field"]
+        # 记录旧值
+        superseded_key = f"{field}_superseded"
+        profile[superseded_key] = {
+            "old_value": conflict["old_value"],
+            "superseded_at": conflict["superseded_at"],
+        }
+        # 将新值写入主字段（已经在 profile_update 时更新了，这里只处理遗留情况）
+        if "." in field:
+            # 处理嵌套 dict key，如 work_style.confirm_before_implement
+            parts = field.split(".")
+            d = profile
+            for p in parts[:-1]:
+                d = d.setdefault(p, {})
+            d[parts[-1]] = conflict["new_value"]
+        else:
+            profile[field] = conflict["new_value"]
+
+    # 原子写入
+    tmp_file = profile_file.with_suffix(".tmp")
+    with open(tmp_file, "w", encoding="utf-8") as f:
+        json.dump(profile, f, ensure_ascii=False, indent=2)
+    tmp_file.replace(profile_file)
+
+
+# ── 知识库冲突检测 ────────────────────────────────────────────────────────
+
+
+def _extract_keywords(text: str) -> set[str]:
+    """提取关键词（去除停用词）"""
+    words = re.split(r'[\s,;.()\[\]{}]+', text.lower())
+    return set(_stopword_filter(words))
+
+
+def _title_similarity(title_a: str, title_b: str) -> float:
+    """计算标题相似度：关键词重叠数 / max(len_a, len_b)"""
+    keywords_a = _extract_keywords(title_a)
+    keywords_b = _extract_keywords(title_b)
+    if not keywords_a or not keywords_b:
+        return 0.0
+    overlap = len(keywords_a & keywords_b)
+    max_len = max(len(keywords_a), len(keywords_b))
+    return overlap / max_len
+
+
+def _has_negation(text: str) -> bool:
+    """检测文本中是否含否定词"""
+    text_lower = text.lower()
+    return any(nw in text_lower for nw in NEGATION_WORDS)
+
+
+def _has_number_change(text_old: str, text_new: str) -> bool:
+    """检测数值变化：两边都有数字但数字不同"""
+    nums_old = set(re.findall(r'\d+\.?\d*', text_old))
+    nums_new = set(re.findall(r'\d+\.?\d*', text_new))
+    if nums_old and nums_new and nums_old != nums_new:
+        return True
+    return False
+
+
+def _has_contradictory_pair(text_old: str, text_new: str) -> bool:
+    """检测对立词对：一边含 A 另一边含 B"""
+    text_lower_old = text_old.lower()
+    text_lower_new = text_new.lower()
+    for pair in CONTRADICTORY_PAIRS:
+        words_old = set(pair & set(text_lower_old.split()))
+        words_new = set(pair & set(text_lower_new.split()))
+        if words_old and words_new and words_old != words_new:
+            return True
+    return False
+
+
+def detect_knowledge_conflict(new_entry: dict, home: Path) -> list[dict]:
+    """
+    检测 knowledge_base.jsonl 中的内容矛盾。
+    new_entry 格式：{"title": "...", "content": "...", ...}
+    返回冲突列表： [{"seq": 行号, "title": "...", "reason": "规则1/2/3"}]
+    """
+    kb_file = home / "semantic" / "knowledge_base.jsonl"
+    if not kb_file.exists():
+        return []
+
+    new_title = new_entry.get("title", "")
+    new_content = new_entry.get("content", "")
+
+    conflicts = []
+    lines = []
+    with open(kb_file, encoding="utf-8") as f:
+        for i, line in enumerate(f):
+            if not line.strip():
+                continue
+            try:
+                entry = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            # 跳过已失效条目
+            if entry.get("superseded"):
+                continue
+            lines.append((i + 1, entry))  # 行号从1开始
+
+    for seq, old_entry in lines:
+        old_title = old_entry.get("title", "")
+        old_content = old_entry.get("content", "")
+
+        # 标题相似度 >= 0.5 才进入内容矛盾检测
+        sim = _title_similarity(new_title, old_title)
+        if sim < 0.5:
+            continue
+
+        # 矛盾规则检测
+        reason = None
+
+        # 规则1 — 否定词检测
+        has_neg_old = _has_negation(old_content)
+        has_neg_new = _has_negation(new_content)
+        if has_neg_old != has_neg_new:
+            reason = "规则1"
+
+        # 规则2 — 数值变化检测
+        elif _has_number_change(old_content, new_content):
+            reason = "规则2"
+
+        # 规则3 — 对立词检测
+        elif _has_contradictory_pair(old_content, new_content):
+            reason = "规则3"
+
+        if reason:
+            conflicts.append({
+                "seq": seq,
+                "title": old_title,
+                "reason": reason,
+            })
+
+    return conflicts
+
+
+def mark_superseded(home: Path, jsonl_path: Path, seq_list: list[int]):
+    """
+    将 knowledge_base.jsonl 中指定行号的条目标记为 superseded。
+    原子写入：先写 tmp 文件再 rename。
+    """
+    if not seq_list:
+        return
+
+    seq_set = set(seq_list)
+
+    tmp_file = jsonl_path.with_suffix(".tmp")
+    written_seq = set()
+
+    with open(jsonl_path, encoding="utf-8") as f_in, \
+         open(tmp_file, "w", encoding="utf-8") as f_out:
+        for i, line in enumerate(f_in):
+            if not line.strip():
+                continue
+            try:
+                entry = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+
+            line_num = i + 1  # 行号从1开始
+            if line_num in seq_set and line_num not in written_seq:
+                entry["superseded"] = True
+                entry["superseded_at"] = _now_iso()
+                written_seq.add(line_num)
+
+            f_out.write(json.dumps(entry, ensure_ascii=False) + "\n")
+
+    tmp_file.replace(jsonl_path)

package/scripts/log_op.py

@@ -8,10 +8,45 @@ import os
 import sys
 import json
 import re
+import time
+import logging
 import argparse
+import contextlib
 from datetime import datetime, timezone
 from pathlib import Path
 
+logging.basicConfig(
+    level=logging.WARNING,
+    format="[ultra-memory] %(levelname)s %(message)s",
+)
+_log = logging.getLogger("ultra-memory.log_op")
+
+
+@contextlib.contextmanager
+def _advisory_lock(lock_path: Path, timeout: float = 5.0):
+    """跨平台建议性文件锁（.lock 哨兵文件）"""
+    deadline = time.monotonic() + timeout
+    acquired = False
+    while True:
+        try:
+            fd = os.open(str(lock_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+            os.close(fd)
+            acquired = True
+            break
+        except FileExistsError:
+            if time.monotonic() >= deadline:
+                _log.warning("文件锁等待超时 %s，直接继续写入", lock_path)
+                break
+            time.sleep(0.05)
+    try:
+        yield
+    finally:
+        if acquired:
+            try:
+                lock_path.unlink(missing_ok=True)
+            except Exception:
+                pass
+
 if sys.stdout.encoding != "utf-8":
     sys.stdout.reconfigure(encoding="utf-8")
 if sys.stderr.encoding != "utf-8":
@@ -19,6 +54,20 @@ if sys.stderr.encoding != "utf-8":
 
 ULTRA_MEMORY_HOME = Path(os.environ.get("ULTRA_MEMORY_HOME", Path.home() / ".ultra-memory"))
 
+# 记忆注入标记（防止反馈环：AI 把自己的记忆输出又记录进去导致自引用积累）
+MEMORY_INJECTION_PATTERNS = [
+    r'\[ultra-memory\][^\n]*',           # 脚本自身的打印输出
+    r'MEMORY_READY[^\n]*',               # 初始化信号
+    r'COMPRESS_SUGGESTED[^\n]*',         # 压缩建议信号
+    r'SESSION_ID=sess_[A-Za-z0-9_]+',   # 会话 ID 注入
+    r'session_id:\s*sess_[A-Za-z0-9_]+',
+    r'\[RECALL\][^\n]*',                 # recall.py 的输出头
+    r'\[ops #\d+[^\]]*\][^\n]*',        # format_result 的 ops 格式
+    r'\[知识库[^\]]*\][^\n]*',           # format_result 的知识库格式
+    r'\[实体/[^\]]*\][^\n]*',           # format_result 的实体格式
+    r'\[摘要\][^\n]*',                   # format_result 的摘要格式
+]
+
 # 敏感词正则（防止记录密码/密钥）
 SENSITIVE_PATTERNS = [
     r'(?i)(password|passwd|pwd)\s*[=:]\s*\S+',
@@ -192,10 +241,20 @@ FILE_EXT_TAG_MAP = {
 }
 
 
+def filter_memory_markers(text: str) -> str:
+    """过滤记忆注入标记，防止反馈环（AI 将自身记忆输出再次记录导致自引用噪音积累）"""
+    if not text:
+        return text
+    for pattern in MEMORY_INJECTION_PATTERNS:
+        text = re.sub(pattern, "", text, flags=re.IGNORECASE)
+    return text.strip()
+
+
 def sanitize(text: str) -> str:
-    """过滤敏感信息"""
+    """过滤敏感信息 + 反馈环标记"""
     if not text:
         return text
+    text = filter_memory_markers(text)
     for pattern in SENSITIVE_PATTERNS:
         text = re.sub(pattern, "[REDACTED]", text)
     return text
@@ -279,17 +338,57 @@ def log_op(
         "compressed": False,
     }
 
-    # 追加写入（append-only，永不覆盖）
-    with open(ops_file, "a", encoding="utf-8") as f:
-        f.write(json.dumps(entry, ensure_ascii=False) + "\n")
-
-    # 更新 meta
-    meta["op_count"] = seq
-    meta["last_op_at"] = entry["ts"]
-    if op_type == "milestone":
-        meta["last_milestone"] = summary
-    with open(meta_file, "w", encoding="utf-8") as f:
-        json.dump(meta, f, ensure_ascii=False, indent=2)
+    # ── 矛盾检测（写入 ops.jsonl 之前）──────────────────────────────────────
+
+    # 2A：画像冲突检测（user_instruction/decision + profile_update）
+    if op_type in ("user_instruction", "decision") and detail.get("profile_update"):
+        try:
+            import sys as _sys
+            _scripts_dir_cd = Path(__file__).parent
+            if str(_scripts_dir_cd) not in _sys.path:
+                _sys.path.insert(0, str(_scripts_dir_cd))
+            from conflict_detector import detect_profile_conflict, mark_profile_superseded
+            conflicts = detect_profile_conflict(detail["profile_update"], ULTRA_MEMORY_HOME)
+            if conflicts:
+                entry["detail"]["profile_conflicts"] = conflicts
+                mark_profile_superseded(ULTRA_MEMORY_HOME, conflicts)
+                print(f"[ultra-memory] ⚡ 检测到 {len(conflicts)} 处画像矛盾，旧记录已标记失效")
+        except Exception as _e:
+            _log.debug("画像冲突检测失败（不影响主流程）: %s", _e)
+
+    # 2B：知识库冲突检测（milestone/decision + knowledge_entry）
+    if op_type in ("milestone", "decision") and detail.get("knowledge_entry"):
+        try:
+            import sys as _sys
+            _scripts_dir_cd = Path(__file__).parent
+            if str(_scripts_dir_cd) not in _sys.path:
+                _sys.path.insert(0, str(_scripts_dir_cd))
+            from conflict_detector import detect_knowledge_conflict, mark_superseded
+            conflicts = detect_knowledge_conflict(detail["knowledge_entry"], ULTRA_MEMORY_HOME)
+            if conflicts:
+                kb_path = ULTRA_MEMORY_HOME / "semantic" / "knowledge_base.jsonl"
+                seq_list = [c["seq"] for c in conflicts]
+                mark_superseded(ULTRA_MEMORY_HOME, kb_path, seq_list)
+                entry["detail"]["knowledge_conflicts"] = conflicts
+                print(f"[ultra-memory] ⚡ 检测到 {len(conflicts)} 条知识库矛盾，旧记录已标记失效")
+        except Exception as _e:
+            _log.debug("知识库冲突检测失败（不影响主流程）: %s", _e)
+
+    # 追加写入（append-only，永不覆盖）；文件锁保护并发写入
+    _lock_file = ops_file.with_suffix(".lock")
+    with _advisory_lock(_lock_file):
+        with open(ops_file, "a", encoding="utf-8") as f:
+            f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+
+        # 更新 meta（在锁内，保证 op_count 单调递增）
+        meta["op_count"] = seq
+        meta["last_op_at"] = entry["ts"]
+        if op_type == "milestone":
+            meta["last_milestone"] = summary
+        _tmp_meta = meta_file.with_suffix(".tmp")
+        with open(_tmp_meta, "w", encoding="utf-8") as f:
+            json.dump(meta, f, ensure_ascii=False, indent=2)
+        _tmp_meta.replace(meta_file)
 
     # 自动提取结构化实体（写入 semantic/entities.jsonl）
     try:
@@ -299,8 +398,8 @@ def log_op(
             _sys.path.insert(0, str(_scripts_dir))
         from extract_entities import extract_and_store
         extract_and_store(session_id, dict(entry))
-    except Exception:
-        pass  # 实体提取失败不影响主流程
+    except Exception as _e:
+        _log.debug("实体提取失败（不影响主流程）: %s", _e)
 
     # 自动提取结构化事实（写入 evolution/facts.jsonl，异步不阻塞）
     try:
@@ -322,8 +421,9 @@ def log_op(
 
     # 多模态处理：检测媒体文件并后台提取
     try:
-        _media_exts = {".pdf": "extract_from_pdf.py", ".png": "extract_from_image.py",
-                        ".jpg": "extract_from_image.py", ".jpeg": "extract_from_image.py",
+        _media_exts = {".pdf": "extract_from_pdf.py", ".docx": "extract_from_docx.py",
+                        ".png": "extract_from_image.py", ".jpg": "extract_from_image.py",
+                        ".jpeg": "extract_from_image.py",
                         ".mp4": "transcribe_video.py", ".avi": "transcribe_video.py",
                         ".mov": "transcribe_video.py"}
         _file_path = detail.get("path", "")