fragmented-memory 1.0.0__py3-none-any.whl

fragmented_memory/__init__.py

@@ -0,0 +1,495 @@
+"""
+fragmented-memory — 碎片化记忆系统 for Hermes Agent.
+
+每次对话自动检索相关记忆碎片注入上下文，支持：
+  - ✂️ 语义切分 — 按段落/句子边界自动拆分成独立碎片
+  - 🔍 向量搜索 — RediSearch KNN 语义检索
+  - ⏳ 时间衰减 — 新碎片权重高，旧碎片逐步降权
+  - 🔄 自动写入 — memory() 操作和对话轮次自动存档
+  - 🏷️ 标签过滤 — 可选按标签范围搜索
+
+安装: pip install fragmented-memory
+激活: config.yaml 中设置 memory.provider: fragmented
+
+配置优先级: 环境变量 > 配置文件 > 默认值
+配置文件: ~/.config/fragmented-memory/config.json (或 FRAGMENTED_MEMORY_CONFIG 自定义路径)
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from agent.memory_provider import MemoryProvider
+from tools.registry import tool_error
+
+from .embedder import create_embedder
+from .splitter import split_text
+from .storage import RedisStorage
+from .consolidator import Consolidator
+from .forgetter import Forgetter
+
+# ---------------------------------------------------------------------------
+# 工具扇区（供 Hermes MemoryProvider 注册）
+# ---------------------------------------------------------------------------
+
+FEEDBACK_SCHEMA = {
+    "name": "frag_memory_feedback",
+    "description": (
+        "记录用户对一条碎片的反馈 — 标记有用/没用。"
+        "正反馈让该碎片在未来搜索中排名更高，"
+        "负反馈大幅降权（标记为没用的碎片几乎不会再出现）。"
+    ),
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "fragment_key": {
+                "type": "string",
+                "description": "碎片的 Redis key（如 memory:frag:abc123），从相关碎片的 key 字段获得。",
+            },
+            "is_positive": {
+                "type": "boolean",
+                "description": "True = 这条记忆有用，False = 没用",
+            },
+        },
+        "required": ["fragment_key", "is_positive"],
+    },
+}
+
+HOT_TOPICS_SCHEMA = {
+    "name": "frag_hot_topics",
+    "description": (
+        "查询全局热门话题统计。返回跨会话出现最频繁的话题词。"
+        "可选日榜/周榜/全局。"
+    ),
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "limit": {
+                "type": "integer",
+                "description": "返回条数（默认 10，最大 30）",
+                "default": 10,
+            },
+            "period": {
+                "type": "string",
+                "enum": ["all", "daily", "weekly"],
+                "description": "统计周期：all=全局, daily=日榜, weekly=周榜",
+                "default": "all",
+            },
+        },
+        "required": [],
+    },
+}
+
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_CONFIG_PATH = "~/.config/fragmented-memory/config.json"
+
+
+def _load_json_config() -> dict:
+    """从 JSON 配置文件加载配置。
+
+    路径来源（优先级高到低）:
+      1. 环境变量 FRAGMENTED_MEMORY_CONFIG
+      2. ~/.config/fragmented-memory/config.json
+    文件不存在时返回空 dict。
+    """
+    path_str = os.environ.get("FRAGMENTED_MEMORY_CONFIG") or _DEFAULT_CONFIG_PATH
+    path = Path(path_str).expanduser()
+    if not path.exists():
+        logger.debug("fragmented: config file not found at %s", path)
+        return {}
+    try:
+        with open(path) as f:
+            cfg: dict = json.load(f)
+        logger.info("fragmented: loaded config from %s", path)
+        return cfg
+    except (json.JSONDecodeError, OSError) as e:
+        logger.warning("fragmented: failed to load config from %s: %s", path, e)
+        return {}
+
+
+def _deep_merge(base: dict, override: dict) -> dict:
+    """递归合并两个 dict，override 覆盖 base。"""
+    result = base.copy()
+    for key, val in override.items():
+        if key in result and isinstance(result[key], dict) and isinstance(val, dict):
+            result[key] = _deep_merge(result[key], val)
+        else:
+            result[key] = val
+    return result
+
+
+class FragmentedMemoryProvider(MemoryProvider):
+    """
+    碎片化记忆提供者。
+
+    和 Hermes builtin 内存共存，不冲突。每轮对话自动检索相关碎片
+    注入上下文，并自动将用户消息切分存档。
+
+    配置优先级（高→低）:
+      1. 环境变量 (FRAGMENTED_REDIS_HOST, FRAGMENTED_EMBEDDER 等)
+      2. JSON 配置文件 (~/.config/fragmented-memory/config.json)
+      3. config.yaml memory.fragmented 节（由 Hermes 传入）
+      4. 硬编码默认值
+    """
+
+    _initialized: bool = False
+    _storage: Optional[RedisStorage] = None
+    _tag_filter: str = ""
+    _consolidator: Optional[Consolidator] = None
+    _forgetter: Optional[Forgetter] = None
+    _last_maintenance: float = 0.0
+    _maintenance_interval: float = 7200.0  # 每 2h 跑一次维护
+
+    def __init__(self, **config):
+        """
+        参数（通过 config.yaml memory 节传入）:
+
+            memory:
+              provider: fragmented
+              fragmented:
+                redis_host: 127.0.0.1
+                redis_port: 6379
+                top_k: 5
+                candidate_k: 10
+                tag_filter: ""
+                embedder:
+                  provider: openai
+                  api_key: sk-xxx
+                  base_url: https://api.openai.com/v1
+                  model: text-embedding-3-small
+        """
+        super().__init__()
+        self._config = config
+
+    # ------------------------------------------------------------------
+    # 配置合并
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _resolve_config(inline_cfg: dict) -> dict:
+        """按优先级合并配置源，返回最终配置。
+
+        合并顺序（后覆盖前）: 默认值 ← JSON 文件 ← 环境变量 ← inline
+        inline = Hermes 的 config.yaml memory.fragmented 或 __init__ 传参
+        """
+        # 1. 硬编码默认值（不含 embedder — 由配置文件/环境变量按需开启）
+        cfg: dict = {
+            "redis_host": "127.0.0.1",
+            "redis_port": 6379,
+            "top_k": 5,
+            "candidate_k": 10,
+            "tag_filter": "",
+        }
+
+        # 2. JSON 配置文件覆盖
+        json_cfg = _load_json_config()
+        cfg = _deep_merge(cfg, json_cfg)
+
+        # 3. 环境变量覆盖
+        env_overrides = {
+            "redis_host": os.environ.get("FRAGMENTED_REDIS_HOST"),
+            "redis_port": os.environ.get("FRAGMENTED_REDIS_PORT"),
+            "top_k": os.environ.get("FRAGMENTED_TOP_K"),
+            "candidate_k": os.environ.get("FRAGMENTED_CANDIDATE_K"),
+            "tag_filter": os.environ.get("FRAGMENTED_TAG_FILTER"),
+        }
+        for key, val in env_overrides.items():
+            if val is not None:
+                cfg[key] = val
+
+        # 4. inline（Hermes 传入的 config.yaml 配置）覆盖
+        cfg = _deep_merge(cfg, inline_cfg)
+
+        return cfg
+
+    # ------------------------------------------------------------------
+    # MemoryProvider 接口
+    # ------------------------------------------------------------------
+
+    @property
+    def name(self) -> str:
+        return "fragmented"
+
+    def is_available(self) -> bool:
+        try:
+            import redis as _  # noqa: F401
+        except ImportError:
+            return False
+        return True
+
+    def initialize(self, session_id: str, **kwargs) -> None:
+        """初始化 — 加载配置、连接 Redis、自动创建 index。"""
+        cfg = self._resolve_config(self._config)
+
+        redis_host = cfg.get("redis_host", "127.0.0.1")
+        redis_port = int(cfg.get("redis_port", 6379))
+        top_k = int(cfg.get("top_k", 5))
+        candidate_k = int(cfg.get("candidate_k", 10))
+        self._tag_filter = cfg.get("tag_filter", "")
+
+        embed_cfg = cfg.get("embedder", {})
+        embed_provider = embed_cfg.get("provider", "").strip().lower()
+        # 只有显式配置了 embedder provider 才创建，否则走 BM25-only 模式
+        if embed_provider and embed_provider not in ("", "default", "none"):
+            embedder = create_embedder(
+                provider=embed_cfg.get("provider", ""),
+                api_key=embed_cfg.get("api_key", ""),
+                base_url=embed_cfg.get("base_url", ""),
+                model=embed_cfg.get("model", ""),
+            )
+            embed_dim = embedder.dimension
+            logger.info(
+                "fragmented: embedder enabled (%s, dim=%d)",
+                embed_provider, embed_dim,
+            )
+        else:
+            embedder = None
+            embed_dim = 1536
+            logger.info("fragmented: BM25-only mode (no embedder configured)")
+
+        self._storage = RedisStorage(
+            embedder=embedder,
+            host=redis_host,
+            port=redis_port,
+            candidate_count=candidate_k,
+            final_limit=top_k,
+            embed_dim=embed_dim,
+            bm25_limit=int(cfg.get("bm25_limit", 10)),
+            decay_half_days=int(cfg.get("decay_half_days", 60)),
+            embed_cache_ttl=int(cfg.get("embed_cache_ttl", 3600)),
+            sentiment_boost_positive=float(cfg.get("sentiment_boost_positive", 1.5)),
+            sentiment_boost_negative=float(cfg.get("sentiment_boost_negative", 1.3)),
+            feedback_positive_boost=float(cfg.get("feedback_positive_boost", 1.3)),
+            feedback_negative_penalty=float(cfg.get("feedback_negative_penalty", 0.5)),
+            hot_topic_boost=float(cfg.get("hot_topic_boost", 1.2)),
+            hot_topic_decay_half_days=int(cfg.get("hot_topic_decay_half_days", 30)),
+            emotion_intensity_factor=float(cfg.get("emotion_intensity_factor", 0.4)),
+            attention_boost_max=float(cfg.get("attention_boost_max", 1.5)),
+            attention_base_increment=float(cfg.get("attention_base_increment", 2.0)),
+            attention_emotion_factor=float(cfg.get("attention_emotion_factor", 1.5)),
+        )
+
+        # 自动创建/验证 index
+        if not self._storage.ensure_index():
+            logger.warning(
+                "fragmented: Redis / RediSearch not ready at %s:%s",
+                redis_host, redis_port,
+            )
+            return
+
+        self._initialized = True
+        logger.info(
+            "fragmented: connected (session=%s, top_k=%d, tag_filter=%s)",
+            session_id, top_k, self._tag_filter or "(none)",
+        )
+
+        # 初始化 Consolidator 和 Forgetter（守护模式）
+        self._consolidator = Consolidator(
+            storage=self._storage,
+            min_group_size=int(cfg.get("consolidate_min_group", 2)),
+            max_age_hours=int(cfg.get("consolidate_max_age_hours", 72)),
+        )
+        self._forgetter = Forgetter(
+            storage=self._storage,
+            max_age_days=int(cfg.get("forget_max_age_days", 30)),
+            dry_run=bool(cfg.get("forget_dry_run", True)),
+        )
+        logger.info("fragmented: maintenance engines initialized")
+
+    def system_prompt_block(self) -> str:
+        parts = [
+            "你有碎片化记忆系统（fragmented-memory），连接在 Redis + RediSearch 上。",
+            "每次对话或 memory(action='add') 操作时，系统会自动检索或存储相关碎片。",
+            "相关碎片就在下面「相关碎片」段落里，直接使用即可。",
+            "碎片综合排序 = BM25相似度 × 时间衰减 × 情感权重 × 反馈权重 × 热门话题权重。",
+            "正反馈用 frag_memory_feedback(key, positive=True) 标记有用，",
+            "负反馈用 frag_memory_feedback(key, positive=False) 标记没用。",
+            "热门话题用 frag_hot_topics() 查询。",
+        ]
+        return "\n".join(parts)
+
+    def prefetch(self, query: str, *, session_id: str = "") -> str:
+        """根据用户消息检索相关碎片，注入到上下文。"""
+        if not query or len(query.strip()) < 2 or not self._storage:
+            return ""
+
+        import time as _time
+        start = _time.time()
+        fragments = self._storage.search(
+            query.strip(),
+            tag_filter=self._tag_filter,
+        )
+        elapsed = _time.time() - start
+
+        if not fragments:
+            return ""
+
+        lines = ["<fragmented_memory>"]
+        lines.append(f"# 相关碎片 (检索耗时 {elapsed:.1f}s)")
+        lines.append("")
+        for i, frag in enumerate(fragments, 1):
+            lines.append(f"[{i}] {frag.get('content', '')}")
+            tags = frag.get("tags", "")
+            combined = frag.get("_combined_score", 0)
+            weights = frag.get("_weights", {})
+            info_parts = []
+            if tags:
+                info_parts.append(f"标签: {tags}")
+            info_parts.append(f"综合: {combined:.2f}")
+            if weights:
+                info_parts.append(f"w: sim={weights.get('sim',0):.2f} decay={weights.get('decay',0):.2f} "
+                                  f"emotion={weights.get('emotion',1):.1f} fb={weights.get('feedback',1):.1f} "
+                                  f"hot={weights.get('hot_topic',1):.1f}")
+            # 情感标签可视化
+            sent_label = frag.get("sentiment_label", "")
+            if sent_label and sent_label != "neutral":
+                sent_score = frag.get("sentiment_score", "0")
+                icon = "😊" if sent_label == "positive" else "😠"
+                info_parts.append(f"{icon} {sent_label}({sent_score})")
+            lines.append(f"    ({', '.join(info_parts)})")
+            lines.append("")
+
+        lines.append("</fragmented_memory>")
+        return "\n".join(lines)
+
+    def sync_turn(
+        self,
+        user_content: str,
+        assistant_content: str,
+        *,
+        session_id: str = "",
+        messages: Optional[List[Dict[str, Any]]] = None,
+    ) -> None:
+        """对话每轮结束后，将用户消息切分存档，并触发维护。"""
+        if not self._storage or not user_content or len(user_content.strip()) < 10:
+            return
+
+        segments = split_text(user_content.strip())
+        sid_short = session_id[:8] if session_id else "unknown"
+        for seg in segments:
+            self._storage.store(
+                text=seg,
+                tags=f"session:{sid_short}",
+                category="conversation",
+                source="sync_turn",
+                fragment_type="conversation",
+            )
+
+        # 定期触发维护（Consolidation + Forget）
+        self._maybe_maintain()
+
+    def _maybe_maintain(self) -> None:
+        """检查是否该执行维护，执行 Consolidation + Forget。"""
+        import time as _time
+        now = _time.time()
+        if now - self._last_maintenance < self._maintenance_interval:
+            return
+        self._last_maintenance = now
+        self.maintenance()
+
+    def maintenance(self) -> Dict[str, Any]:
+        """执行一轮完整维护：Consolidation → Forget。
+
+        返回:
+            维护统计
+        """
+        stats: Dict[str, Any] = {
+            "consolidator": {"status": "skipped"},
+            "forgetter": {"status": "skipped"},
+        }
+
+        # Step 1: Consolidation
+        if self._consolidator:
+            try:
+                result = self._consolidator.consolidate()
+                stats["consolidator"] = result
+                logger.info("fragmented: consolidation done — %s", result)
+            except Exception as e:
+                logger.warning("fragmented: consolidation error: %s", e)
+                stats["consolidator"] = {"status": "error", "reason": str(e)}
+
+        # Step 2: Selective Forgetting
+        if self._forgetter:
+            try:
+                result = self._forgetter.forget()
+                stats["forgetter"] = result
+                logger.info("fragmented: forgetting done — %s", result)
+            except Exception as e:
+                logger.warning("fragmented: forgetting error: %s", e)
+                stats["forgetter"] = {"status": "error", "reason": str(e)}
+
+        return stats
+
+    def get_tool_schemas(self) -> List[Dict[str, Any]]:
+        return [FEEDBACK_SCHEMA, HOT_TOPICS_SCHEMA]
+
+    def handle_tool_call(
+        self,
+        tool_name: str,
+        args: Dict[str, Any],
+        **kwargs,
+    ) -> str:
+        """Route tool calls to the appropriate handler."""
+        import json as _json
+
+        if tool_name == "frag_memory_feedback":
+            return self._handle_feedback(args, _json)
+        elif tool_name == "frag_hot_topics":
+            return self._handle_hot_topics(args, _json)
+        return tool_error(f"Unknown fragmented memory tool: '{tool_name}'")
+
+    # ------------------------------------------------------------------
+    # Tool handlers
+    # ------------------------------------------------------------------
+
+    def _handle_feedback(self, args: Dict[str, Any], _json) -> str:
+        key = args.get("fragment_key", "")
+        is_pos = bool(args.get("is_positive", True))
+        if not key:
+            return tool_error("fragment_key is required")
+        if not self._storage:
+            return tool_error("Memory storage not initialized")
+        ok = self._storage.record_feedback(key, is_pos)
+        if ok:
+            action = "有用 👍" if is_pos else "没用 👎"
+            return _json.dumps({"success": True, "action": action, "key": key})
+        return tool_error("Failed to record feedback")
+
+    def _handle_hot_topics(self, args: Dict[str, Any], _json) -> str:
+        limit = min(int(args.get("limit", 10)), 30)
+        period = args.get("period", "all")
+        if not self._storage:
+            return tool_error("Memory storage not initialized")
+        topics = self._storage.get_hot_topics(limit=limit, period=period)
+        return _json.dumps({"topics": topics, "count": len(topics)}, ensure_ascii=False)
+
+    def shutdown(self) -> None:
+        if self._storage:
+            self._storage.close()
+        logger.info("fragmented memory provider shutdown")
+
+    def on_memory_write(
+        self,
+        action: str,
+        target: str,
+        content: str,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        """builtin memory 写入时同步存到碎片库。"""
+        if action != "add" or not content or not self._storage:
+            return
+
+        for seg in split_text(content):
+            self._storage.store(
+                text=seg,
+                tags=target,
+                category="memory_tool",
+                source="hermes_agent",
+                fragment_type="memory",
+            )

fragmented_memory/attention.py

@@ -0,0 +1,146 @@
+"""注意力追踪 — 统计用户对各个话题的关注频率。
+
+用户反复提起某个话题 -> 该话题关注度上升 -> 相关碎片在搜索中权重更高。
+
+存储: Redis Sorted Set `fragmented:attention`
+  - member: 话题词（由 jieba/关键词提取来）
+  - score: 关注度累计值（每次提及 +2，情绪烈度加权）
+
+三套时间窗口（同 hot_topics 模式）:
+  - 全局: fractured:attention (7天)
+  - 日榜: fractured:attention:daily (2天)
+  - 周榜: fractured:attention:weekly (14天)
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional
+
+import redis
+
+logger = logging.getLogger(__name__)
+
+# Redis key 前缀
+ATTENTION_SET = "fragmented:attention"
+ATTENTION_DAILY = "fragmented:attention:daily"
+ATTENTION_WEEKLY = "fragmented:attention:weekly"
+
+# 过期时间
+_ATTENTION_TTL = {
+    ATTENTION_SET: 86400 * 7,        # 全局：7天
+    ATTENTION_DAILY: 86400 * 2,      # 日榜：2天
+    ATTENTION_WEEKLY: 86400 * 14,    # 周榜：14天
+}
+
+
+def record_attention(
+    client: redis.Redis,
+    text: str,
+    emotion_intensity: float = 0.0,
+    keywords: Optional[List[str]] = None,
+    base_increment: float = 2.0,
+    emotion_factor: float = 1.5,
+) -> None:
+    """记录用户对一段文本中话题的关注。"""
+    if not client or not text:
+        return
+
+    try:
+        # 没有关键词时跳过（正常路径由 store() 传入，不会走到这里）
+        if not keywords:
+            return
+
+        increment = base_increment + emotion_intensity * emotion_factor
+
+        for kw in keywords:
+            kw_lower = kw.lower().strip()
+            if len(kw_lower) < 2:
+                continue
+            for topic_set in (ATTENTION_SET, ATTENTION_DAILY, ATTENTION_WEEKLY):
+                client.zincrby(topic_set, increment, kw_lower)
+                client.expire(topic_set, _ATTENTION_TTL.get(topic_set, 86400))
+
+    except Exception as e:
+        logger.debug("attention: record_attention error: %s", e)
+
+
+def get_attention_score(client: redis.Redis, keyword: str) -> float:
+    """查某个词在当前注意力分数中的排名分。"""
+    if not client or not keyword:
+        return 0.0
+    try:
+        score = client.zscore(ATTENTION_SET, keyword.lower().strip())
+        return score if score is not None else 0.0
+    except Exception:
+        return 0.0
+
+
+def get_top_attention(client: redis.Redis, limit: int = 10,
+                      period: str = "all") -> List[Dict[str, Any]]:
+    """获取关注度最高的词。"""
+    key = {
+        "all": ATTENTION_SET,
+        "daily": ATTENTION_DAILY,
+        "weekly": ATTENTION_WEEKLY,
+    }.get(period, ATTENTION_SET)
+
+    if not client:
+        return []
+    try:
+        raw = client.zrevrange(key, 0, limit - 1, withscores=True)
+        results = []
+        for t, s_raw in raw:
+            topic = t.decode("utf-8") if isinstance(t, bytes) else t
+            if isinstance(s_raw, bytes):
+                s_raw = s_raw.decode("utf-8")
+            results.append({"topic": topic, "score": round(float(s_raw), 1)})
+        return results
+    except Exception as e:
+        logger.debug("attention: get_top_attention error: %s", e)
+        return []
+
+
+def match_attention_boost(
+    client: redis.Redis,
+    content: str,
+    top_n: int = 10,
+    boost_max: float = 1.5,
+) -> float:
+    """检查碎片内容的注意力关注度加权值。
+
+    从全局注意力取 top N 话题，看碎片内容命中几个。
+    命中越多权重越高，最高 boost_max。
+    """
+    if not client or not content:
+        return 1.0
+
+    try:
+        raw = client.zrevrange(ATTENTION_SET, 0, top_n - 1, withscores=True)
+        if not raw:
+            return 1.0
+
+        content_lower = content.lower()
+        total_score = 0.0
+        max_score = 0.0
+
+        for topic_b, score_raw in raw:
+            topic = topic_b.decode("utf-8") if isinstance(topic_b, bytes) else topic_b
+            if isinstance(score_raw, bytes):
+                score_raw = score_raw.decode("utf-8")
+            sc = float(score_raw)  # noqa: F841
+            if isinstance(sc, (int, float)):
+                if len(topic) >= 2 and topic in content_lower:
+                    total_score += sc
+                max_score += sc
+
+        if max_score <= 0:
+            return 1.0
+
+        # 归一化到 1.0~boost_max，命中越高越接近 boost_max
+        ratio = min(total_score / max_score, 1.0)
+        return 1.0 + (boost_max - 1.0) * ratio
+
+    except Exception as e:
+        logger.debug("attention: match_attention_boost error: %s", e)
+        return 1.0