keepsake-memory 1.0.0__py3-none-any.whl

keepsake/attention.py

@@ -0,0 +1,146 @@
+"""注意力追踪 — 统计用户对各个话题的关注频率。
+
+用户反复提起某个话题 -> 该话题关注度上升 -> 相关碎片在搜索中权重更高。
+
+存储: Redis Sorted Set `keepsake:attention`
+  - member: 话题词（由 jieba/关键词提取来）
+  - score: 关注度累计值（每次提及 +2，情绪烈度加权）
+
+三套时间窗口（同 hot_topics 模式）:
+  - 全局: fractured:attention (7天)
+  - 日榜: fractured:attention:daily (2天)
+  - 周榜: fractured:attention:weekly (14天)
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Optional
+
+import redis
+
+logger = logging.getLogger(__name__)
+
+# Redis key 前缀
+ATTENTION_SET = "keepsake:attention"
+ATTENTION_DAILY = "keepsake:attention:daily"
+ATTENTION_WEEKLY = "keepsake:attention:weekly"
+
+# 过期时间
+_ATTENTION_TTL = {
+    ATTENTION_SET: 86400 * 7,        # 全局：7天
+    ATTENTION_DAILY: 86400 * 2,      # 日榜：2天
+    ATTENTION_WEEKLY: 86400 * 14,    # 周榜：14天
+}
+
+
+def record_attention(
+    client: redis.Redis,
+    text: str,
+    emotion_intensity: float = 0.0,
+    keywords: Optional[List[str]] = None,
+    base_increment: float = 2.0,
+    emotion_factor: float = 1.5,
+) -> None:
+    """记录用户对一段文本中话题的关注。"""
+    if not client or not text:
+        return
+
+    try:
+        # 没有关键词时跳过（正常路径由 store() 传入，不会走到这里）
+        if not keywords:
+            return
+
+        increment = base_increment + emotion_intensity * emotion_factor
+
+        for kw in keywords:
+            kw_lower = kw.lower().strip()
+            if len(kw_lower) < 2:
+                continue
+            for topic_set in (ATTENTION_SET, ATTENTION_DAILY, ATTENTION_WEEKLY):
+                client.zincrby(topic_set, increment, kw_lower)
+                client.expire(topic_set, _ATTENTION_TTL.get(topic_set, 86400))
+
+    except Exception as e:
+        logger.debug("attention: record_attention error: %s", e)
+
+
+def get_attention_score(client: redis.Redis, keyword: str) -> float:
+    """查某个词在当前注意力分数中的排名分。"""
+    if not client or not keyword:
+        return 0.0
+    try:
+        score = client.zscore(ATTENTION_SET, keyword.lower().strip())
+        return score if score is not None else 0.0
+    except Exception:
+        return 0.0
+
+
+def get_top_attention(client: redis.Redis, limit: int = 10,
+                      period: str = "all") -> List[Dict[str, Any]]:
+    """获取关注度最高的词。"""
+    key = {
+        "all": ATTENTION_SET,
+        "daily": ATTENTION_DAILY,
+        "weekly": ATTENTION_WEEKLY,
+    }.get(period, ATTENTION_SET)
+
+    if not client:
+        return []
+    try:
+        raw = client.zrevrange(key, 0, limit - 1, withscores=True)
+        results = []
+        for t, s_raw in raw:
+            topic = t.decode("utf-8") if isinstance(t, bytes) else t
+            if isinstance(s_raw, bytes):
+                s_raw = s_raw.decode("utf-8")
+            results.append({"topic": topic, "score": round(float(s_raw), 1)})
+        return results
+    except Exception as e:
+        logger.debug("attention: get_top_attention error: %s", e)
+        return []
+
+
+def match_attention_boost(
+    client: redis.Redis,
+    content: str,
+    top_n: int = 10,
+    boost_max: float = 1.5,
+) -> float:
+    """检查碎片内容的注意力关注度加权值。
+
+    从全局注意力取 top N 话题，看碎片内容命中几个。
+    命中越多权重越高，最高 boost_max。
+    """
+    if not client or not content:
+        return 1.0
+
+    try:
+        raw = client.zrevrange(ATTENTION_SET, 0, top_n - 1, withscores=True)
+        if not raw:
+            return 1.0
+
+        content_lower = content.lower()
+        total_score = 0.0
+        max_score = 0.0
+
+        for topic_b, score_raw in raw:
+            topic = topic_b.decode("utf-8") if isinstance(topic_b, bytes) else topic_b
+            if isinstance(score_raw, bytes):
+                score_raw = score_raw.decode("utf-8")
+            sc = float(score_raw)  # noqa: F841
+            if isinstance(sc, (int, float)):
+                if len(topic) >= 2 and topic in content_lower:
+                    total_score += sc
+                max_score += sc
+
+        if max_score <= 0:
+            return 1.0
+
+        # 归一化到 1.0~boost_max，命中越高越接近 boost_max
+        ratio = min(total_score / max_score, 1.0)
+        return 1.0 + (boost_max - 1.0) * ratio
+
+    except Exception as e:
+        logger.debug("attention: match_attention_boost error: %s", e)
+        return 1.0

keepsake/consolidator.py

@@ -0,0 +1,395 @@
+"""Consolidation 引擎 — 将同主题碎片分层提炼为更高层记忆。
+
+工作流程:
+  1. 扫描所有未合并的碎片（fragment_type != "consolidated"）
+  2. 用 jieba 关键词提取做主题分组
+  3. 每组超过 min_group_size 条时，调 LLM 合并为一条高层次摘要
+  4. 存为新碎片（fragment_type="consolidated", level=N+1）
+  5. 删除原始碎片（或标记已合并）
+
+配置参数:
+  - min_group_size: 最少多少条碎片才触发合并（默认 3）
+  - max_age_hours: 只合并超过此年龄的碎片（给新碎片时间积累，默认 72h）
+  - llm_model: DashScope 模型名（默认 qwen-turbo，便宜够用）
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import time
+import urllib.error
+import urllib.request
+from collections import defaultdict
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+# DashScope API 端点
+DASHSCOPE_BASE = "https://dashscope.aliyuncs.com/compatible-mode/v1"
+
+# 默认参数
+DEFAULT_MIN_GROUP_SIZE = 2  # 有重复内容就合
+DEFAULT_MAX_AGE_HOURS = 72
+DEFAULT_LLM_MODEL = "qwen-plus"
+DEFAULT_BATCH_SIZE = 200  # 每次 consolidate 扫描的碎片数
+
+# LLM 超时
+LLM_TIMEOUT = 30
+
+# 合并提示词
+CONSOLIDATE_PROMPT = """你是一位知识提炼专家。以下是一组关于同一话题的对话片段。
+
+请将它们合并成一条简洁、信息完整的高层记忆条目，要求：
+1. 保留所有关键事实和结论，不丢信息
+2. 去掉重复内容
+3. 用陈述句表达，像一条知识条目
+4. 如果片段之间存在矛盾，指出矛盾但不选边
+5. 控制在 200 字以内
+
+对话片段：
+{segments}
+
+合并后的知识条目："""
+
+
+def _get_api_key() -> str:
+    """获取 DashScope API key。"""
+    key = os.environ.get("OPENAI_API_KEY", "") or os.environ.get("DASHSCOPE_API_KEY", "")
+    # fallback: 从 config.yaml 用 yaml 解析，优先取 providers.dashscope.api_key
+    if not key:
+        try:
+            import yaml
+            config_path = os.path.expanduser("~/.hermes/config.yaml")
+            if os.path.isfile(config_path):
+                with open(config_path) as f:
+                    cfg = yaml.safe_load(f)
+                if cfg:
+                    key = (
+                        cfg.get("providers", {}).get("dashscope", {}).get("api_key")
+                        or cfg.get("model", {}).get("api_key")
+                        or ""
+                    )
+                    key = key.strip().strip("'\"")
+        except Exception:
+            pass
+    return key
+
+
+def _call_llm(messages: List[Dict[str, str]], model: str = DEFAULT_LLM_MODEL,
+              max_retries: int = 2) -> Optional[str]:
+    """调用 DashScope chat API 获取 LLM 回复。带重试。"""
+    api_key = _get_api_key()
+    if not api_key:
+        logger.warning("consolidator: no API key for LLM calls")
+        return None
+
+    url = f"{DASHSCOPE_BASE}/chat/completions"
+    payload = json.dumps({
+        "model": model,
+        "messages": messages,
+        "max_tokens": 512,
+        "temperature": 0.3,
+    }).encode("utf-8")
+
+    for attempt in range(1 + max_retries):
+        if attempt > 0:
+            wait = 2.0 * (2 ** (attempt - 1))  # 2s, 4s
+            logger.debug("consolidator: retry %d/%d after %.0fs", attempt, max_retries, wait)
+            time.sleep(wait)
+
+        req = urllib.request.Request(
+            url, data=payload,
+            headers={
+                "Authorization": f"Bearer {api_key}",
+                "Content-Type": "application/json",
+            },
+            method="POST",
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=LLM_TIMEOUT) as resp:
+                data = json.loads(resp.read())
+                choices = data.get("choices", [])
+                if choices:
+                    return choices[0].get("message", {}).get("content", "").strip()
+            # Got response but no choices — don't retry
+            logger.warning("consolidator: LLM returned no choices (attempt %d)", attempt + 1)
+            return None
+        except (urllib.error.URLError, json.JSONDecodeError, OSError) as e:
+            if attempt < max_retries:
+                logger.debug("consolidator: LLM attempt %d failed: %s", attempt + 1, e)
+            else:
+                logger.warning("consolidator: LLM call failed after %d attempts: %s",
+                               attempt + 1, e)
+    return None
+
+
+class Consolidator:
+    """碎片分层提炼引擎。"""
+
+    def __init__(
+        self,
+        storage: Any,  # RedisStorage instance (avoid circular import)
+        min_group_size: int = DEFAULT_MIN_GROUP_SIZE,
+        max_age_hours: int = DEFAULT_MAX_AGE_HOURS,
+        llm_model: str = DEFAULT_LLM_MODEL,
+        batch_size: int = DEFAULT_BATCH_SIZE,
+    ):
+        self._storage = storage
+        self._min_group_size = min_group_size
+        self._max_age_hours = max_age_hours
+        self._llm_model = llm_model
+        self._batch_size = batch_size
+
+    def consolidate(self) -> Dict[str, Any]:
+        """执行一轮碎片合并。返回操作统计。"""
+        client = self._storage._get_client()
+        if not client:
+            return {"status": "error", "reason": "Redis not available"}
+
+        stats = {"scanned": 0, "groups_found": 0, "merged": 0, "skipped": 0, "errors": 0}
+
+        # 1. 扫描未合并的碎片
+        fragments = self._scan_unconsolidated(client)
+        stats["scanned"] = len(fragments)
+        if not fragments:
+            return stats
+
+        # 2. 按主题聚类
+        groups = self._cluster_by_topic(fragments)
+        stats["groups_found"] = len(groups)
+
+        # 3. 对每个符合条件的组执行合并
+        for group in groups:
+            if len(group) < self._min_group_size:
+                stats["skipped"] += len(group)
+                continue
+
+            result = self._merge_group(client, group)
+            if result:
+                stats["merged"] += len(group)
+            else:
+                stats["errors"] += len(group)
+
+        return stats
+
+    def _scan_unconsolidated(self, client) -> List[Dict[str, Any]]:
+        """扫描符合合并条件的碎片。
+
+        条件:
+          - fragment_type != "consumed"（未被更高层合并吞掉的）
+          - 创建时间 > max_age_hours（给新碎片时间积累）
+          - 已合并的（consolidated）也参与扫描，实现多级提炼
+        """
+        try:
+            cutoff = (datetime.now(timezone.utc).timestamp() - self._max_age_hours * 3600)
+            cursor = 0
+            fragments = []
+
+            while True:
+                cursor, keys = client.scan(
+                    cursor=cursor,
+                    match="memory:frag:*",
+                    count=self._batch_size,
+                )
+
+                if not keys:
+                    if cursor == 0:
+                        break
+                    continue
+
+                # 用 pipeline 批量 HMGETALL，减少网络往返
+                pipe = client.pipeline()
+                for key_b in keys:
+                    pipe.hgetall(key_b)
+                pipe_results = pipe.execute()
+
+                for key_b, data in zip(keys, pipe_results):
+                    key = key_b.decode("utf-8") if isinstance(key_b, bytes) else key_b
+                    if not data:
+                        continue
+
+                    # 解码
+                    doc = {}
+                    for k_b, v_b in data.items():
+                        k = k_b.decode("utf-8") if isinstance(k_b, bytes) else k_b
+                        v = v_b.decode("utf-8") if isinstance(v_b, bytes) else v_b
+                        doc[k] = v
+
+                    # 跳过已被更高层合并吞掉的
+                    if doc.get("fragment_type", "") == "consumed":
+                        continue
+
+                    # 检查年龄
+                    created_str = doc.get("created", "")
+                    if created_str:
+                        try:
+                            created_ts = datetime.fromisoformat(created_str).timestamp()
+                            if created_ts > cutoff:
+                                continue  # 太新，等下次
+                        except (ValueError, TypeError):
+                            pass
+
+                    doc["_key"] = key
+                    fragments.append(doc)
+
+                if cursor == 0:
+                    break
+
+            return fragments
+
+        except Exception as e:
+            logger.warning("consolidator: scan error: %s", e)
+            return []
+
+    def _cluster_by_topic(self, fragments: List[Dict]) -> List[List[Dict]]:
+        """按关键词重叠做简单聚类。
+
+        策略:
+          - 对每个碎片提取关键词（用 jieba）
+          - 关键词重叠 >= 2 的归为一组
+          - 贪心算法，不追求最优聚类
+        """
+        from .splitter import extract_keywords
+
+        # 提取每个碎片的关键词
+        frag_data = []
+        for f in fragments:
+            content = f.get("content", "")
+            if not content:
+                continue
+            kws = set(extract_keywords(content, max_keywords=5))
+            frag_data.append({"frag": f, "keywords": kws})
+
+        if not frag_data:
+            return []
+
+        # 贪心聚类
+        groups: List[List[Dict]] = []
+        assigned = set()
+
+        for i, data in enumerate(frag_data):
+            if i in assigned:
+                continue
+            group = [data["frag"]]
+            assigned.add(i)
+
+            for j, other in enumerate(frag_data):
+                if j in assigned:
+                    continue
+                # 重叠 >= 2 个关键词
+                overlap = len(data["keywords"] & other["keywords"])
+                if overlap >= 2:
+                    group.append(other["frag"])
+                    assigned.add(j)
+
+            groups.append(group)
+
+        return groups
+
+    def _merge_group(self, client, group: List[Dict]) -> bool:
+        """用 LLM 合并一组碎片。"""
+        # 计算新层级：取组内最高 level + 1
+        max_level = 1
+        for f in group:
+            try:
+                lv = int(f.get("level", "1"))
+                if lv > max_level:
+                    max_level = lv
+            except (ValueError, TypeError):
+                pass
+        new_level = max_level + 1
+
+        # 判断是否已有合并过的碎片
+        has_consolidated = any(
+            f.get("fragment_type") == "consolidated" or int(f.get("level", "1")) > 1
+            for f in group
+        )
+
+        # 准备片段文本
+        segments = []
+        tags_set = set()
+        for f in group:
+            content = f.get("content", "")
+            if content:
+                segments.append(f"• {content[:300]}")
+            tags = f.get("tags", "")
+            if tags:
+                for t in tags.split(","):
+                    t = t.strip()
+                    if t and not t.startswith("session:"):
+                        tags_set.add(t)
+
+        if len(segments) < self._min_group_size:
+            return False
+
+        # 根据是否已有提炼过的内容选择不同 prompt
+        if has_consolidated:
+            prompt = (
+                "以下是一组已经提炼过的记忆条目和相关的原始对话片段。"
+                "请将它们进一步提炼合并成一条更精炼的高层知识条目。\n\n"
+                + "\n".join(segments)
+                + "\n\n提炼后的高层知识条目："
+            )
+        else:
+            prompt = CONSOLIDATE_PROMPT.format(segments="\n".join(segments))
+        result = _call_llm([
+            {"role": "system", "content": "你是一位知识提炼专家，擅长从对话中提取核心信息。"},
+            {"role": "user", "content": prompt},
+        ], model=self._llm_model)
+
+        if not result:
+            return False
+
+        # 分析情绪
+        from .splitter import analyze_sentiment
+        sent_score, sent_label = analyze_sentiment(result)
+        now_str = datetime.now(timezone.utc).isoformat()
+
+        mapping = {
+            "content": result,
+            "tags": ",".join(sorted(tags_set)) if tags_set else "",
+            "category": "consolidated",
+            "source": "consolidator",
+            "created": now_str,
+            "fragment_type": "consolidated",
+            "level": str(new_level),  # 多级：原始=1，首次合并=2，二次合并=3...
+            "sentiment_score": str(sent_score),
+            "sentiment_label": sent_label,
+            "feedback_score": "0",
+        }
+
+        # 存 consolidated 碎片
+        import hashlib
+        content_hash = hashlib.sha256(result.encode()).hexdigest()[:12]
+        consolidated_key = f"memory:frag:{content_hash}"
+
+        # 如果没有相同 key（去重检查），就存
+        existing = client.exists(consolidated_key)
+        if existing:
+            logger.debug("consolidator: duplicate consolidated result, skipping")
+        else:
+            client.hset(consolidated_key, mapping=mapping)
+
+        # 软删除原始碎片（标记为已消费，不硬删）
+        from datetime import datetime as _dt
+        now_iso = _dt.now(timezone.utc).isoformat()
+        consumed_count = 0
+        for f in group:
+            key = f.get("_key")
+            if key:
+                try:
+                    client.hset(key, "consumed_by", consolidated_key)
+                    client.hset(key, "consumed_at", now_iso)
+                    client.hset(key, "fragment_type", "consumed")
+                    consumed_count += 1
+                except Exception:
+                    pass
+
+        logger.info(
+            "consolidator: merged %d fragments → '%s...' (marked %d as consumed)",
+            len(group), result[:60], consumed_count,
+        )
+        return True

keepsake/embedder.py

@@ -0,0 +1,155 @@
+"""Embedding 客户端 — 支持 OpenAI / DashScope / 自定义兼容端点。"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from abc import ABC, abstractmethod
+from typing import Optional
+from urllib.request import Request, urlopen
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# 模型 → 维度映射
+# ---------------------------------------------------------------------------
+
+_MODEL_DIMENSIONS: dict[str, int] = {
+    # OpenAI text-embedding-3 系列
+    "text-embedding-3-small": 1536,
+    "text-embedding-3-large": 3072,
+    "text-embedding-ada-002": 1536,
+    # DashScope
+    "text-embedding-v2": 1536,
+    "text-embedding-v3": 1024,
+}
+
+_DEFAULT_DIM = 1536
+
+
+def resolve_dimension(model: str) -> int:
+    """根据模型名返回向量维度，未知模型返回默认值 1536。"""
+    model_key = model.strip().lower()
+    return _MODEL_DIMENSIONS.get(model_key, _DEFAULT_DIM)
+
+
+# ---------------------------------------------------------------------------
+# 抽象基类
+# ---------------------------------------------------------------------------
+
+
+class Embedder(ABC):
+    @abstractmethod
+    def get_embedding(self, text: str) -> Optional[list[float]]:
+        """输入文本，返回 float 向量。"""
+        ...
+
+    @property
+    @abstractmethod
+    def dimension(self) -> int:
+        """返回当前模型输出的向量维度。"""
+        ...
+
+
+# ---------------------------------------------------------------------------
+# OpenAI 兼容
+# ---------------------------------------------------------------------------
+
+_DEFAULT_OPENAI_URL = "https://api.openai.com/v1/embeddings"
+_DEFAULT_OPENAI_MODEL = "text-embedding-3-small"
+
+
+class OpenAIEmbedder(Embedder):
+    """兼容 OpenAI Embedding API 的客户端。
+
+    也兼容 DashScope 等提供 /v1/embeddings 端点的服务。
+    """
+
+    def __init__(
+        self,
+        api_key: str = "",
+        base_url: str = _DEFAULT_OPENAI_URL,
+        model: str = _DEFAULT_OPENAI_MODEL,
+    ):
+        self._api_key = api_key or os.environ.get("OPENAI_API_KEY", "")
+        self._base_url = base_url.rstrip("/")
+        self._model = model
+        self._dim = resolve_dimension(model)
+
+    @property
+    def dimension(self) -> int:
+        return self._dim
+
+    def get_embedding(self, text: str) -> Optional[list[float]]:
+        if not self._api_key:
+            logger.warning("embedder: no API key configured")
+            return None
+
+        payload = json.dumps({
+            "model": self._model,
+            "input": text,
+        }).encode("utf-8")
+
+        req = Request(
+            f"{self._base_url}/embeddings",
+            data=payload,
+            headers={
+                "Authorization": f"Bearer {self._api_key}",
+                "Content-Type": "application/json",
+            },
+            method="POST",
+        )
+
+        try:
+            with urlopen(req, timeout=30) as resp:
+                data = json.loads(resp.read())
+                emb = data["data"][0]["embedding"]
+                # 如果服务返回的维度与预期不符，更新 self._dim
+                if len(emb) != self._dim:
+                    logger.info(
+                        "embedder: model %s returned %d dims (expected %d), updating",
+                        self._model, len(emb), self._dim,
+                    )
+                    self._dim = len(emb)
+                return emb
+        except Exception as e:
+            logger.debug("embedder: request failed: %s", e)
+            return None
+
+
+# ---------------------------------------------------------------------------
+# 工厂
+# ---------------------------------------------------------------------------
+
+_EMBEDDER_PROVIDERS: dict[str, type[Embedder]] = {
+    "openai": OpenAIEmbedder,
+    "dashscope": OpenAIEmbedder,  # DashScope 也走 /v1/embeddings
+}
+
+
+def create_embedder(
+    provider: str = "",
+    api_key: str = "",
+    base_url: str = "",
+    model: str = "",
+) -> Embedder:
+    """根据配置创建 Embedder 实例。
+
+    参数:
+        provider: "openai" | "dashscope" | 自定义
+        api_key: API 密钥
+        base_url: API 端点
+        model: 模型名
+    """
+    provider = provider or os.environ.get("FRAGMENTED_EMBEDDER", "openai").lower()
+    api_key = api_key or os.environ.get("OPENAI_API_KEY", "")
+    base_url = base_url or os.environ.get("FRAGMENTED_EMBEDDER_URL", _DEFAULT_OPENAI_URL)
+    model = model or os.environ.get("FRAGMENTED_EMBEDDER_MODEL", _DEFAULT_OPENAI_MODEL)
+
+    if provider == "dashscope":
+        base_url = base_url or "https://dashscope.aliyuncs.com/compatible-mode/v1"
+        model = model or "text-embedding-v2"
+
+    cls = _EMBEDDER_PROVIDERS.get(provider, OpenAIEmbedder)
+    return cls(api_key=api_key, base_url=base_url, model=model)