ultra-memory 3.1.0 → 4.0.0

package/scripts/auto_decay.py

@@ -0,0 +1,351 @@
+#!/usr/bin/env python3
+"""
+ultra-memory: 自动遗忘引擎 (Evolution Engine Phase 3)
+基于时间衰减 + 访问频率 + 重要性评分计算每条事实的 decay_score。
+decay_score < 0.05 的事实标记为遗忘（soft-delete，不删除原始记录）。
+
+衰减公式：
+  decay_score = importance_score × recency_weight × access_weight
+
+  recency_weight  = 0.5 ^ (age_days / half_life_days)   # half_life=30天
+  access_weight   = min(1.0, log2(access_count + 1) / log2(11))
+                    # access_count=0 → ~0, access_count=10 → ~0.95
+
+衰减等级：
+  ≥ 0.6  → "none"      (健康)
+  0.4–0.6 → "mild"     (轻度衰减)
+  0.2–0.4 → "moderate" (中度衰减)
+  0.05–0.2 → "severe"  (即将遗忘)
+  < 0.05 → "forgotten" (触发遗忘)
+
+被 cleanup.py --run-decay 调用，或每日定时触发。
+"""
+
+import os
+import sys
+import json
+import argparse
+import math
+from datetime import datetime, timezone
+from pathlib import Path
+
+if sys.stdout.encoding != "utf-8":
+    sys.stdout.reconfigure(encoding="utf-8")
+if sys.stderr.encoding != "utf-8":
+    sys.stderr.reconfigure(encoding="utf-8")
+
+ULTRA_MEMORY_HOME = Path(os.environ.get("ULTRA_MEMORY_HOME", Path.home() / ".ultra-memory"))
+
+# 默认衰减参数（可通过 config.json 覆盖）
+DEFAULT_HALF_LIFE_DAYS = 30
+DEFAULT_FORGET_THRESHOLD = 0.05
+
+# 衰减等级边界
+DECAY_LEVELS = [
+    (0.6, "none"),
+    (0.4, "mild"),
+    (0.2, "moderate"),
+    (0.05, "severe"),
+]
+
+
+# ── 工具函数 ───────────────────────────────────────────────────────────────
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def _parse_ts(ts_str: str) -> datetime:
+    """解析 ISO 时间字符串"""
+    if not ts_str:
+        return datetime.now(timezone.utc)
+    try:
+        # 移除 Z 后缀
+        ts_str = ts_str.replace("Z", "+00:00")
+        return datetime.fromisoformat(ts_str)
+    except ValueError:
+        return datetime.now(timezone.utc)
+
+
+def _load_config() -> dict:
+    """加载配置（如果有 decay 相关配置）"""
+    config_file = ULTRA_MEMORY_HOME / "config.json"
+    if config_file.exists():
+        try:
+            with open(config_file, encoding="utf-8") as f:
+                return json.load(f)
+        except (json.JSONDecodeError, IOError):
+            pass
+    return {}
+
+
+# ── 衰减计算 ───────────────────────────────────────────────────────────────
+
+
+def compute_decay_score(
+    fact_metadata: dict,
+    now: datetime,
+) -> float:
+    """
+    计算单条事实的衰减评分。
+
+    decay_score = importance_score × recency_weight × access_weight
+    """
+    age_days = (now - _parse_ts(fact_metadata.get("last_updated", ""))).days
+    half_life_days = fact_metadata.get("ttl_days", DEFAULT_HALF_LIFE_DAYS)
+
+    # 时间衰减权重
+    recency_weight = math.pow(0.5, age_days / half_life_days) if half_life_days > 0 else 0.0
+
+    # 访问频率权重
+    access_count = fact_metadata.get("access_count", 0)
+    access_weight = min(1.0, math.log2(access_count + 1) / math.log2(11))
+
+    # 重要性评分
+    importance_score = fact_metadata.get("importance_score", 0.5)
+
+    score = importance_score * recency_weight * access_weight
+
+    # 从未访问且较老的事实额外惩罚
+    if access_count == 0 and age_days > half_life_days:
+        score *= 0.5
+
+    return max(0.0, min(1.0, score))
+
+
+def compute_decay_level(score: float) -> str:
+    """根据衰减评分确定衰减等级"""
+    for threshold, level in DECAY_LEVELS:
+        if score >= threshold:
+            return level
+    return "forgotten"
+
+
+def compute_importance_score(fact: dict, meta: dict) -> float:
+    """
+    计算事实的重要性评分 (0.0–1.0)。
+    来源类型 × 矛盾抵抗 × 用户确认度 的加权组合。
+    """
+    base = 0.5
+
+    # 来源类型权重
+    source_weights = {
+        "milestone": 0.9,
+        "decision": 0.85,
+        "user_instruction": 0.8,
+        "file_write": 0.7,
+        "tool_call": 0.6,
+        "reasoning": 0.6,
+        "bash_exec": 0.55,
+        "error": 0.5,
+        "file_read": 0.4,
+    }
+    source_type = fact.get("source_type", "")
+    base = source_weights.get(source_type, 0.6)
+
+    # 矛盾抵抗：矛盾越少越稳定
+    contradiction_count = meta.get("contradiction_count", 0)
+    contradiction_penalty = min(0.3, contradiction_count * 0.1)
+    base = max(0.1, base - contradiction_penalty)
+
+    # 用户显式确认权重（如果 correction_history 有 manual 条目）
+    manual_corrections = [
+        c for c in meta.get("correction_history", [])
+        if c.get("source") == "manual"
+    ]
+    if manual_corrections:
+        base = min(1.0, base + 0.15)
+
+    return max(0.1, min(1.0, base))
+
+
+# ── 遗忘处理 ───────────────────────────────────────────────────────────────
+
+
+def _load_facts() -> list[dict]:
+    """加载所有 facts"""
+    facts_file = ULTRA_MEMORY_HOME / "evolution" / "facts.jsonl"
+    if not facts_file.exists():
+        return []
+    facts = []
+    with open(facts_file, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                facts.append(json.loads(line))
+            except json.JSONDecodeError:
+                continue
+    return facts
+
+
+def _load_metadata() -> dict:
+    """加载 fact_metadata.json"""
+    meta_file = ULTRA_MEMORY_HOME / "evolution" / "fact_metadata.json"
+    if not meta_file.exists():
+        return {"version": 1, "updated_at": _now_iso(), "facts": {}}
+    try:
+        with open(meta_file, encoding="utf-8") as f:
+            return json.load(f)
+    except (json.JSONDecodeError, IOError):
+        return {"version": 1, "updated_at": _now_iso(), "facts": {}}
+
+
+def _save_metadata(meta: dict):
+    """原子写入 metadata"""
+    evolution_dir = ULTRA_MEMORY_HOME / "evolution"
+    evolution_dir.mkdir(parents=True, exist_ok=True)
+    meta_file = evolution_dir / "fact_metadata.json"
+    meta["updated_at"] = _now_iso()
+
+    tmp_file = meta_file.with_suffix(".tmp")
+    with open(tmp_file, "w", encoding="utf-8") as f:
+        json.dump(meta, f, ensure_ascii=False, indent=2)
+    tmp_file.replace(meta_file)
+
+
+def append_decay_log(entry: dict):
+    """追加写入 decay_log.jsonl"""
+    evolution_dir = ULTRA_MEMORY_HOME / "evolution"
+    evolution_dir.mkdir(parents=True, exist_ok=True)
+    log_file = evolution_dir / "decay_log.jsonl"
+
+    with open(log_file, "a", encoding="utf-8") as f:
+        f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+
+
+# ── 全量衰减扫描 ───────────────────────────────────────────────────────────
+
+
+def run_decay_pass(session_id: str | None = None):
+    """
+    执行全量衰减扫描。
+    session_id 不为 None 时只扫描指定 session 的事实。
+    """
+    now = datetime.now(timezone.utc)
+    facts = _load_facts()
+    meta = _load_metadata()
+
+    config = _load_config()
+    forget_threshold = (
+        config.get("decay", {}).get("forget_threshold", DEFAULT_FORGET_THRESHOLD)
+    )
+
+    # 初始化所有 fact 的 metadata（如果不存在）
+    for fact in facts:
+        fid = fact.get("fact_id")
+        if not fid:
+            continue
+        if fid not in meta["facts"]:
+            meta["facts"][fid] = {
+                "confidence": fact.get("confidence", 0.7),
+                "access_count": fact.get("access_count", 1),
+                "last_accessed": fact.get("last_accessed", _now_iso()),
+                "last_updated": fact.get("ts", _now_iso()),
+                "importance_score": compute_importance_score(fact, {}),
+                "decay_level": "none",
+                "ttl_days": DEFAULT_HALF_LIFE_DAYS,
+                "expires_at": None,
+                "status": "active",
+                "contradiction_count": fact.get("contradiction_count", 0),
+                "correction_history": [],
+            }
+
+    # 计算每条事实的衰减
+    forgotten_ids = []
+    severe_ids = []
+    updated_ids = []
+
+    for fid, fact_meta in meta["facts"].items():
+        if fact_meta.get("status") in ("forgotten", "superseded"):
+            continue
+
+        # session_id 过滤
+        if session_id:
+            # 找到对应 fact
+            matching_fact = next(
+                (f for f in facts if f.get("fact_id") == fid), None
+            )
+            if not matching_fact:
+                continue
+            if matching_fact.get("session_id") != session_id:
+                continue
+
+        # TTL 过期检测
+        if fact_meta.get("expires_at"):
+            expires_at = _parse_ts(fact_meta["expires_at"])
+            if now >= expires_at:
+                fact_meta["decay_level"] = "forgotten"
+
+        # 重新计算重要性评分（基于最新事实数据）
+        matching_fact = next(
+            (f for f in facts if f.get("fact_id") == fid), None
+        )
+        if matching_fact:
+            fact_meta["importance_score"] = compute_importance_score(
+                matching_fact, fact_meta
+            )
+
+        # 计算衰减评分
+        decay_score = compute_decay_score(fact_meta, now)
+        old_level = fact_meta.get("decay_level", "none")
+        new_level = compute_decay_level(decay_score)
+        fact_meta["decay_level"] = new_level
+
+        # 更新 ttl_days 配置
+        ttl_days = fact_meta.get("ttl_days", DEFAULT_HALF_LIFE_DAYS)
+
+        # 触发遗忘
+        if new_level == "forgotten" and old_level != "forgotten":
+            fact_meta["status"] = "forgotten"
+            fact_meta["forgotten_at"] = _now_iso()
+            forgotten_ids.append(fid)
+
+            append_decay_log({
+                "ts": _now_iso(),
+                "fact_id": fid,
+                "reason": "ttl_expired" if fact_meta.get("expires_at") else "importance_decay",
+                "decay_level_before": old_level,
+                "action": "marked_forgotten",
+                "decay_score": round(decay_score, 3),
+                "session_id": session_id or "system",
+            })
+        elif new_level == "severe" and old_level in ("none", "mild", "moderate"):
+            severe_ids.append(fid)
+
+        if new_level != old_level:
+            updated_ids.append(fid)
+
+    _save_metadata(meta)
+
+    print(f"[ultra-memory] ✅ 衰减扫描完成 (session: {session_id or 'all'})")
+    print(f"  遗忘: {len(forgotten_ids)} 条")
+    print(f"  严重衰减: {len(severe_ids)} 条")
+    print(f"  等级变化: {len(updated_ids)} 条")
+
+    return {
+        "forgotten": forgotten_ids,
+        "severe": severe_ids,
+        "updated": updated_ids,
+    }
+
+
+# ── CLI ─────────────────────────────────────────────────────────────────────
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="执行事实衰减扫描")
+    parser.add_argument(
+        "--session", default=None,
+        help="会话 ID（省略则扫描所有事实）"
+    )
+    parser.add_argument(
+        "--run", action="store_true",
+        help="执行衰减扫描（配合 cleanup.py 使用）"
+    )
+    args = parser.parse_args()
+
+    result = run_decay_pass(args.session)
+    sys.exit(0)

package/scripts/cleanup.py

@@ -152,5 +152,26 @@ if __name__ == "__main__":
     parser.add_argument("--archive-only", action="store_true", help="只归档到 archive/ 目录，不删除")
     parser.add_argument("--dry-run", action="store_true", help="演习模式，只打印不执行")
     parser.add_argument("--project", default=None, help="只清理指定项目（默认所有项目）")
+    parser.add_argument(
+        "--run-decay", action="store_true",
+        help="执行事实衰减扫描（auto_decay.py），在清理前运行"
+    )
     args = parser.parse_args()
+
+    if args.run_decay:
+        import subprocess
+        scripts_dir = Path(__file__).parent
+        python = sys.executable
+        startupinfo = subprocess.STARTUPINFO()
+        startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+        print("[ultra-memory] 运行事实衰减扫描...")
+        result = subprocess.run(
+            [python, str(scripts_dir / "auto_decay.py"), "--session", args.project or ""],
+            capture_output=True, text=True, startupinfo=startupinfo,
+        )
+        if result.stdout:
+            print(result.stdout)
+        if result.stderr:
+            print(result.stderr, file=sys.stderr)
+
     cleanup(args.days, args.archive_only, args.dry_run, args.project)

package/scripts/conflict_detector.py

@@ -0,0 +1,319 @@
+#!/usr/bin/env python3
+"""
+ultra-memory: 矛盾检测核心模块
+检测 user_profile.json 和 knowledge_base.jsonl 中的时序矛盾，
+将旧记录标记为 superseded: true，召回时跳过失效记录。
+
+只使用 Python 标准库，无外部依赖。
+"""
+
+import os
+import re
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+# ── 停用词表 ────────────────────────────────────────────────────────────────
+
+STOPWORDS = {
+    "的", "了", "是", "在", "和", "与", "或", "以及",
+    "a", "an", "the", "is", "was", "are", "were",
+    "to", "of", "for", "with", "by", "from",
+}
+
+# ── 否定词表 ────────────────────────────────────────────────────────────────
+
+NEGATION_WORDS = {
+    "不", "没有", "无法", "不能", "不是", "别", "莫", "永不", "绝不",
+    "not", "no", "never", "none", "cannot", "don't", "doesn't",
+    "isn't", "aren't", "wasn't", "weren't", "won't", "wouldn't",
+}
+
+# ── 对立词对 ───────────────────────────────────────────────────────────────
+
+CONTRADICTORY_PAIRS = [
+    frozenset(["成功", "失败"]),
+    frozenset(["可以", "不能"]),
+    frozenset(["推荐", "不推荐"]),
+    frozenset(["启用", "禁用"]),
+    frozenset(["enable", "disable"]),
+    frozenset(["success", "failure"]),
+    frozenset(["yes", "no"]),
+    frozenset(["true", "false"]),
+]
+
+
+# ── 工具函数 ───────────────────────────────────────────────────────────────
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def _stopword_filter(words: list[str]) -> list[str]:
+    return [w for w in words if w.lower() not in STOPWORDS and len(w) > 1]
+
+
+# ── 画像冲突检测 ───────────────────────────────────────────────────────────
+
+
+def detect_profile_conflict(new_data: dict, home: Path) -> list[dict]:
+    """
+    检测 user_profile.json 中的字段冲突。
+    返回冲突列表，每条格式：
+      {"field": "...", "old_value": "...", "new_value": "...", "superseded_at": "..."}
+    """
+    profile_file = home / "semantic" / "user_profile.json"
+    if not profile_file.exists():
+        return []
+
+    try:
+        with open(profile_file, encoding="utf-8") as f:
+            profile = json.load(f)
+    except (json.JSONDecodeError, IOError):
+        return []
+
+    conflicts = []
+
+    for field, new_value in new_data.items():
+        # observed_patterns 只追加，不检测冲突
+        if field == "observed_patterns":
+            continue
+
+        # 跳过 superseded 标记的字段
+        if field.endswith("_superseded"):
+            continue
+
+        if field not in profile:
+            continue
+
+        old_value = profile[field]
+
+        # 字符串类型：旧值与新值不同 → 冲突
+        if isinstance(old_value, str) and isinstance(new_value, str):
+            if old_value != new_value:
+                conflicts.append({
+                    "field": field,
+                    "old_value": old_value,
+                    "new_value": new_value,
+                    "superseded_at": _now_iso(),
+                })
+
+        # 数组类型：完全替换（完全不同）→ 冲突；只新增元素 → 不冲突
+        elif isinstance(old_value, list) and isinstance(new_value, list):
+            # 新旧完全不同才算冲突；新值包含旧值的所有元素（新增元素）不算冲突
+            old_set = set(str(v) for v in old_value)
+            new_set = set(str(v) for v in new_value)
+            if old_set and new_set and old_set != new_set:
+                # 只有当旧值中的元素不在新值中，才算冲突
+                removed = old_set - new_set
+                if removed:  # 有元素被移除才算冲突
+                    conflicts.append({
+                        "field": field,
+                        "old_value": old_value,
+                        "new_value": new_value,
+                        "superseded_at": _now_iso(),
+                    })
+
+        # dict 类型：递归检测每个 key
+        elif isinstance(old_value, dict) and isinstance(new_value, dict):
+            for sub_key, sub_old in old_value.items():
+                if sub_key not in new_value:
+                    continue
+                sub_new = new_value[sub_key]
+                if isinstance(sub_old, str) and isinstance(sub_new, str):
+                    if sub_old != sub_new:
+                        full_key = f"{field}.{sub_key}"
+                        conflicts.append({
+                            "field": full_key,
+                            "old_value": sub_old,
+                            "new_value": "（已更新）" if sub_new else sub_new,
+                            "superseded_at": _now_iso(),
+                        })
+
+    return conflicts
+
+
+def mark_profile_superseded(home: Path, conflicts: list[dict]):
+    """
+    将 user_profile.json 中冲突的旧字段标记为 superseded。
+    在旧字段旁追加 "<field>_superseded" 记录旧值。
+    """
+    profile_file = home / "semantic" / "user_profile.json"
+    if not profile_file.exists() or not conflicts:
+        return
+
+    try:
+        with open(profile_file, encoding="utf-8") as f:
+            profile = json.load(f)
+    except (json.JSONDecodeError, IOError):
+        return
+
+    for conflict in conflicts:
+        field = conflict["field"]
+        # 记录旧值
+        superseded_key = f"{field}_superseded"
+        profile[superseded_key] = {
+            "old_value": conflict["old_value"],
+            "superseded_at": conflict["superseded_at"],
+        }
+        # 将新值写入主字段（已经在 profile_update 时更新了，这里只处理遗留情况）
+        if "." in field:
+            # 处理嵌套 dict key，如 work_style.confirm_before_implement
+            parts = field.split(".")
+            d = profile
+            for p in parts[:-1]:
+                d = d.setdefault(p, {})
+            d[parts[-1]] = conflict["new_value"]
+        else:
+            profile[field] = conflict["new_value"]
+
+    # 原子写入
+    tmp_file = profile_file.with_suffix(".tmp")
+    with open(tmp_file, "w", encoding="utf-8") as f:
+        json.dump(profile, f, ensure_ascii=False, indent=2)
+    tmp_file.replace(profile_file)
+
+
+# ── 知识库冲突检测 ────────────────────────────────────────────────────────
+
+
+def _extract_keywords(text: str) -> set[str]:
+    """提取关键词（去除停用词）"""
+    words = re.split(r'[\s,;.()\[\]{}]+', text.lower())
+    return set(_stopword_filter(words))
+
+
+def _title_similarity(title_a: str, title_b: str) -> float:
+    """计算标题相似度：关键词重叠数 / max(len_a, len_b)"""
+    keywords_a = _extract_keywords(title_a)
+    keywords_b = _extract_keywords(title_b)
+    if not keywords_a or not keywords_b:
+        return 0.0
+    overlap = len(keywords_a & keywords_b)
+    max_len = max(len(keywords_a), len(keywords_b))
+    return overlap / max_len
+
+
+def _has_negation(text: str) -> bool:
+    """检测文本中是否含否定词"""
+    text_lower = text.lower()
+    return any(nw in text_lower for nw in NEGATION_WORDS)
+
+
+def _has_number_change(text_old: str, text_new: str) -> bool:
+    """检测数值变化：两边都有数字但数字不同"""
+    nums_old = set(re.findall(r'\d+\.?\d*', text_old))
+    nums_new = set(re.findall(r'\d+\.?\d*', text_new))
+    if nums_old and nums_new and nums_old != nums_new:
+        return True
+    return False
+
+
+def _has_contradictory_pair(text_old: str, text_new: str) -> bool:
+    """检测对立词对：一边含 A 另一边含 B"""
+    text_lower_old = text_old.lower()
+    text_lower_new = text_new.lower()
+    for pair in CONTRADICTORY_PAIRS:
+        words_old = set(pair & set(text_lower_old.split()))
+        words_new = set(pair & set(text_lower_new.split()))
+        if words_old and words_new and words_old != words_new:
+            return True
+    return False
+
+
+def detect_knowledge_conflict(new_entry: dict, home: Path) -> list[dict]:
+    """
+    检测 knowledge_base.jsonl 中的内容矛盾。
+    new_entry 格式：{"title": "...", "content": "...", ...}
+    返回冲突列表： [{"seq": 行号, "title": "...", "reason": "规则1/2/3"}]
+    """
+    kb_file = home / "semantic" / "knowledge_base.jsonl"
+    if not kb_file.exists():
+        return []
+
+    new_title = new_entry.get("title", "")
+    new_content = new_entry.get("content", "")
+
+    conflicts = []
+    lines = []
+    with open(kb_file, encoding="utf-8") as f:
+        for i, line in enumerate(f):
+            if not line.strip():
+                continue
+            try:
+                entry = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            # 跳过已失效条目
+            if entry.get("superseded"):
+                continue
+            lines.append((i + 1, entry))  # 行号从1开始
+
+    for seq, old_entry in lines:
+        old_title = old_entry.get("title", "")
+        old_content = old_entry.get("content", "")
+
+        # 标题相似度 >= 0.5 才进入内容矛盾检测
+        sim = _title_similarity(new_title, old_title)
+        if sim < 0.5:
+            continue
+
+        # 矛盾规则检测
+        reason = None
+
+        # 规则1 — 否定词检测
+        has_neg_old = _has_negation(old_content)
+        has_neg_new = _has_negation(new_content)
+        if has_neg_old != has_neg_new:
+            reason = "规则1"
+
+        # 规则2 — 数值变化检测
+        elif _has_number_change(old_content, new_content):
+            reason = "规则2"
+
+        # 规则3 — 对立词检测
+        elif _has_contradictory_pair(old_content, new_content):
+            reason = "规则3"
+
+        if reason:
+            conflicts.append({
+                "seq": seq,
+                "title": old_title,
+                "reason": reason,
+            })
+
+    return conflicts
+
+
+def mark_superseded(home: Path, jsonl_path: Path, seq_list: list[int]):
+    """
+    将 knowledge_base.jsonl 中指定行号的条目标记为 superseded。
+    原子写入：先写 tmp 文件再 rename。
+    """
+    if not seq_list:
+        return
+
+    seq_set = set(seq_list)
+
+    tmp_file = jsonl_path.with_suffix(".tmp")
+    written_seq = set()
+
+    with open(jsonl_path, encoding="utf-8") as f:
+        for i, line in enumerate(f):
+            if not line.strip():
+                continue
+            try:
+                entry = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+
+            line_num = i + 1  # 行号从1开始
+            if line_num in seq_set and line_num not in written_seq:
+                entry["superseded"] = True
+                entry["superseded_at"] = _now_iso()
+                written_seq.add(line_num)
+
+            tmp_file.write(json.dumps(entry, ensure_ascii=False) + "\n")
+
+    tmp_file.replace(jsonl_path)