mindforge-memory 6.4.1__py3-none-any.whl

mindforge/modules/multimodal.py

@@ -0,0 +1,304 @@
+"""
+MindForge v5.0 多模态记忆模块
+支持文本、图像、音频、代码、结构化数据
+"""
+
+import base64
+import json
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Dict, List, Optional, Any, Tuple
+
+
+class MultimodalType(Enum):
+    """多模态类型"""
+    TEXT = "text"
+    IMAGE = "image"
+    AUDIO = "audio"
+    CODE = "code"
+    STRUCTURED = "structured"
+    MULTIMODAL = "multimodal"
+
+
+@dataclass
+class MultimodalContent:
+    """多模态内容"""
+    content_type: MultimodalType
+    data: str = ""
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    embedding: Optional[List[float]] = None
+    text_representation: str = ""
+
+    def to_dict(self) -> dict:
+        return {
+            "content_type": self.content_type.value,
+            "data": self.data,
+            "metadata": self.metadata,
+            "text_representation": self.text_representation,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "MultimodalContent":
+        return cls(
+            content_type=MultimodalType(data.get("content_type", "text")),
+            data=data.get("data", ""),
+            metadata=data.get("metadata", {}),
+            text_representation=data.get("text_representation", ""),
+        )
+
+
+@dataclass
+class ImageMetadata:
+    """图像元数据"""
+    width: int = 0
+    height: int = 0
+    format: str = ""
+    size_bytes: int = 0
+    caption: str = ""
+    objects_detected: List[str] = field(default_factory=list)
+    scene_description: str = ""
+
+    def to_dict(self) -> dict:
+        return self.__dict__
+
+
+@dataclass
+class AudioMetadata:
+    """音频元数据"""
+    duration_seconds: float = 0.0
+    format: str = ""
+    sample_rate: int = 0
+    transcript: str = ""
+    speaker: str = ""
+    language: str = ""
+
+    def to_dict(self) -> dict:
+        return self.__dict__
+
+
+@dataclass
+class CodeMetadata:
+    """代码元数据"""
+    language: str = ""
+    lines_of_code: int = 0
+    function_count: int = 0
+    class_count: int = 0
+    purpose: str = ""
+    dependencies: List[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        return self.__dict__
+
+
+class MultimodalMemory:
+    """多模态记忆管理器"""
+
+    def __init__(self, storage=None, index=None):
+        self.storage = storage
+        self.index = index
+        self._type_stats: Dict[str, int] = {}
+
+    def create_text_memory(self, text: str, **kwargs) -> MultimodalContent:
+        """创建文本记忆"""
+        return MultimodalContent(
+            content_type=MultimodalType.TEXT,
+            data=text,
+            text_representation=text,
+            metadata={"word_count": len(text.split()), "char_count": len(text)},
+        )
+
+    def create_image_memory(self,
+                            image_data: bytes,
+                            caption: str = "",
+                            width: int = 0,
+                            height: int = 0,
+                            format: str = "",
+                            objects: Optional[List[str]] = None) -> MultimodalContent:
+        """创建图像记忆"""
+        # v5.4.7 修复 L-9：限制图像大小（默认 50MB）
+        MAX_IMAGE_SIZE = 50 * 1024 * 1024
+        if len(image_data) > MAX_IMAGE_SIZE:
+            raise ValueError(f"Image too large: {len(image_data)} bytes (max {MAX_IMAGE_SIZE})")
+        encoded = base64.b64encode(image_data).decode()
+        metadata = ImageMetadata(
+            width=width,
+            height=height,
+            format=format,
+            size_bytes=len(image_data),
+            caption=caption,
+            objects_detected=objects or [],
+        )
+
+        text_repr = f"[图像] {caption} "
+        if objects:
+            text_repr += f"包含：{', '.join(objects)}"
+
+        return MultimodalContent(
+            content_type=MultimodalType.IMAGE,
+            data=encoded,
+            metadata=metadata.to_dict(),
+            text_representation=text_repr,
+        )
+
+    def create_audio_memory(self,
+                            audio_data: bytes,
+                            transcript: str = "",
+                            duration: float = 0.0,
+                            format: str = "",
+                            speaker: str = "",
+                            language: str = "") -> MultimodalContent:
+        """创建音频记忆"""
+        # v5.4.7 修复 L-9：限制音频大小（默认 100MB）
+        MAX_AUDIO_SIZE = 100 * 1024 * 1024
+        if len(audio_data) > MAX_AUDIO_SIZE:
+            raise ValueError(f"Audio too large: {len(audio_data)} bytes (max {MAX_AUDIO_SIZE})")
+        encoded = base64.b64encode(audio_data).decode()
+        metadata = AudioMetadata(
+            duration_seconds=duration,
+            format=format,
+            transcript=transcript,
+            speaker=speaker,
+            language=language,
+        )
+
+        text_repr = f"[音频] {speaker} 说：{transcript}" if transcript else "[音频]"
+
+        return MultimodalContent(
+            content_type=MultimodalType.AUDIO,
+            data=encoded,
+            metadata=metadata.to_dict(),
+            text_representation=text_repr,
+        )
+
+    def create_code_memory(self,
+                           code: str,
+                           language: str = "",
+                           purpose: str = "",
+                           dependencies: Optional[List[str]] = None) -> MultimodalContent:
+        """创建代码记忆"""
+        lines = code.strip().split('\n')
+        func_count = sum(1 for line in lines if line.strip().startswith('def ') or line.strip().startswith('function '))
+        class_count = sum(1 for line in lines if line.strip().startswith('class '))
+
+        metadata = CodeMetadata(
+            language=language,
+            lines_of_code=len(lines),
+            function_count=func_count,
+            class_count=class_count,
+            purpose=purpose,
+            dependencies=dependencies or [],
+        )
+
+        text_repr = f"[代码-{language}] {purpose}\n{code[:200]}"
+
+        return MultimodalContent(
+            content_type=MultimodalType.CODE,
+            data=code,
+            metadata=metadata.to_dict(),
+            text_representation=text_repr,
+        )
+
+    def create_structured_memory(self,
+                                  data: Dict,
+                                  schema_name: str = "",
+                                  description: str = "") -> MultimodalContent:
+        """创建结构化数据记忆"""
+        json_str = json.dumps(data, ensure_ascii=False, indent=2)
+        text_repr = f"[结构化数据-{schema_name}] {description}\n键：{', '.join(data.keys())}"
+
+        return MultimodalContent(
+            content_type=MultimodalType.STRUCTURED,
+            data=json_str,
+            metadata={"schema": schema_name, "key_count": len(data)},
+            text_representation=text_repr,
+        )
+
+    def create_multimodal_memory(self,
+                                  contents: List[MultimodalContent],
+                                  description: str = "") -> MultimodalContent:
+        """创建复合多模态记忆"""
+        all_text = "\n".join(c.text_representation for c in contents)
+        combined_data = json.dumps([c.to_dict() for c in contents], ensure_ascii=False)
+
+        return MultimodalContent(
+            content_type=MultimodalType.MULTIMODAL,
+            data=combined_data,
+            metadata={
+                "modality_count": len(contents),
+                "modalities": [c.content_type.value for c in contents],
+                "description": description,
+            },
+            text_representation=all_text,
+        )
+
+    def search_multimodal(self,
+                           query: str,
+                           content_types: Optional[List[MultimodalType]] = None,
+                           limit: int = 10) -> List[Tuple[MultimodalContent, float]]:
+        """跨模态搜索
+
+        v6.1.2 修复（两个方向都会出错）：`entry.memory_type` 是 MemoryType **枚举**，
+        原代码却拿它和 `[t.value for t in content_types]` 这个**字符串列表**比对——
+        枚举永远不等于字符串，所以只要传了 content_types，命中就全被丢弃、返回空；
+        而不传 content_types 时又走到 `MultimodalType(mtype)`，用枚举对象构造（应为
+        str），抛 ValueError。即「带过滤 = 静默漏检，不带过滤 = 直接崩溃」。
+        现在两侧统一归一到 .value 再比较，类型名无法识别时按 TEXT 暴露而非丢结果。
+        """
+        if not self.index:
+            return []
+
+        results = self.index.search(query, top_k=limit * 2)
+
+        # 归一过滤集合：既接受 MultimodalType 枚举，也接受裸字符串值
+        allowed = None
+        if content_types:
+            allowed = {getattr(t, "value", t) for t in content_types}
+
+        filtered = []
+        for doc_id, score in results:
+            if self.storage:
+                entry = self.storage.get_memory(doc_id)
+                if entry:
+                    mtype = getattr(entry, "memory_type", None)
+                    # 兼容 MemoryType 枚举与裸字符串两种落库形态
+                    type_value = getattr(mtype, "value", mtype)
+                    if allowed is not None and type_value not in allowed:
+                        continue
+                    try:
+                        ctype = MultimodalType(type_value)
+                    except ValueError:
+                        # 未知/缺失类型：按文本暴露，不让整条结果消失
+                        ctype = MultimodalType.TEXT
+                    content = MultimodalContent(
+                        content_type=ctype,
+                        data=entry.content,
+                        text_representation=entry.content,
+                    )
+                    filtered.append((content, score))
+
+        return filtered[:limit]
+
+    def get_type_stats(self) -> Dict[str, int]:
+        """获取各类型统计"""
+        if not self.storage:
+            return self._type_stats
+
+        stats = self.storage.get_stats()
+        return stats.get("by_type", {})
+
+    @staticmethod
+    def compute_similarity(content1: MultimodalContent, content2: MultimodalContent) -> float:
+        """计算两个多模态内容的相似度"""
+        text1 = content1.text_representation.lower()
+        text2 = content2.text_representation.lower()
+
+        words1 = set(text1.split())
+        words2 = set(text2.split())
+
+        if not words1 or not words2:
+            return 0.0
+
+        intersection = words1 & words2
+        union = words1 | words2
+
+        return len(intersection) / len(union)

mindforge/modules/outcome_learning.py

@@ -0,0 +1,186 @@
+"""
+MindForge v6.0.0 功能 5：按结果学习的记忆
+
+不止记"发生过什么"，还要记"哪段记忆带来了好结果"：
+- 成功的方法 → 记忆重要度上调，检索时优先
+- 失败的方法 → 自动降权
+- 可在记忆中标注 outcome 并给出解释
+
+实现：
+- memories 表新增 outcome / outcome_note / outcome_at 三列（migration）
+- apply_outcome(): 更新记忆的结果标记并调整 importance
+- search_weighted(): 检索时按 outcome 加权（success 优先）
+- outcome_stats(): 结果分布统计
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any, Dict, List, Optional
+
+_OUTCOME_IMPORTANCE_MAP = {
+    "success": {"HIGH": 0, "MEDIUM": 1, "LOW": 1},    # 成功 → 上调
+    "partial": {"HIGH": 0, "MEDIUM": 0, "LOW": 0},
+    "failure": {"HIGH": 0, "MEDIUM": -1, "LOW": -1},  # 失败 → 下调
+}
+
+_IMPORTANCE_ORDER = ["LOW", "MEDIUM", "HIGH"]
+
+
+def _shift_importance(importance: str, outcome: str) -> str:
+    """按结果调整重要度档位"""
+    imp = (importance or "MEDIUM").upper()
+    if imp not in _IMPORTANCE_ORDER:
+        imp = "MEDIUM"
+    shift = _OUTCOME_IMPORTANCE_MAP.get(outcome, {}).get(imp, 0)
+    idx = _IMPORTANCE_ORDER.index(imp)
+    idx = max(0, min(len(_IMPORTANCE_ORDER) - 1, idx + shift))
+    return _IMPORTANCE_ORDER[idx]
+
+
+def _ensure_columns(conn) -> None:
+    """幂等添加 outcome 列"""
+    for col, ddl in (
+        ("outcome", "TEXT DEFAULT ''"),
+        ("outcome_note", "TEXT DEFAULT ''"),
+        ("outcome_at", "REAL DEFAULT 0"),
+    ):
+        try:
+            conn.execute(f"ALTER TABLE memories ADD COLUMN {col} {ddl}")
+        except Exception:
+            pass  # 已存在
+
+
+def _normalize_memory_id(memory_id: Any) -> tuple:
+    """接受 str 或 MemoryEntry 对象，返回 (id_or_None, error_or_None)。
+
+    v6.0.0 只接受字符串；但 mf.add() 返回的是 MemoryEntry 对象，
+    把返回值直接传进来会在 len() 处抛 TypeError（对外表现为崩溃而非友好报错）。
+    v6.1.1 起统一收敛：对象取 .id，其余非法类型返回明确错误。
+    """
+    if memory_id is None:
+        return None, "记忆 ID 无效"
+    if not isinstance(memory_id, str):
+        memory_id = getattr(memory_id, "id", None)
+        if not isinstance(memory_id, str):
+            return None, "记忆 ID 必须是字符串或 add() 返回的 MemoryEntry 对象"
+    if not memory_id or len(memory_id) > 128:
+        return None, "记忆 ID 无效"
+    return memory_id, None
+
+
+def _chunk_score(chunk: Any) -> float:
+    """读取召回得分。
+
+    RecallEngine 写入的字段名是 relevance_score（见 modules/recall.py），
+    v6.0.0 的 search_weighted 误读为 score，getattr 取不到就落回 0.0，
+    导致按结果加权检索返回的 relevance 恒为 0.0。
+    """
+    for attr in ("relevance_score", "score", "similarity"):
+        val = getattr(chunk, attr, None)
+        if val is not None:
+            try:
+                return float(val)
+            except (TypeError, ValueError):
+                continue
+    if isinstance(chunk, dict):
+        for key in ("relevance_score", "score", "similarity"):
+            if chunk.get(key) is not None:
+                try:
+                    return float(chunk[key])
+                except (TypeError, ValueError):
+                    continue
+    return 0.0
+
+
+def apply_outcome(storage, memory_id: str, outcome: str = "success",
+                  note: str = "", actor: str = "",
+                  session_id: str = "") -> Dict[str, Any]:
+    """给记忆标记结果并调整重要度。outcome: success/partial/failure"""
+    outcome = (outcome or "success").lower()
+    if outcome not in ("success", "partial", "failure"):
+        return {"success": False, "error": f"无效 outcome: {outcome}"}
+    memory_id, id_error = _normalize_memory_id(memory_id)
+    if id_error:
+        return {"success": False, "error": id_error}
+
+    conn = storage._get_conn()
+    _ensure_columns(conn)
+    row = conn.execute(
+        "SELECT importance, outcome FROM memories WHERE id = ?", (memory_id,)).fetchone()
+    if not row:
+        return {"success": False, "error": "记忆不存在"}
+    old_importance, old_outcome = row[0] or "MEDIUM", row[1] or ""
+    new_importance = _shift_importance(old_importance, outcome)
+
+    now = time.time()
+    conn.execute(
+        "UPDATE memories SET outcome = ?, outcome_note = ?, outcome_at = ?, "
+        "importance = ? WHERE id = ?",
+        (outcome, (note or "")[:500], now, new_importance, memory_id))
+    conn.commit()
+    storage._add_audit("outcome_learn", memory_id, actor or "", session_id or "",
+                       "", {"outcome": outcome, "importance": new_importance,
+                            "note": (note or "")[:200]})
+    return {"success": True, "memory_id": memory_id, "outcome": outcome,
+            "importance_before": old_importance, "importance_after": new_importance,
+            "outcome_at": now}
+
+
+def outcome_stats(storage) -> Dict[str, Any]:
+    """结果标记分布统计"""
+    conn = storage._get_conn()
+    _ensure_columns(conn)
+    try:
+        rows = conn.execute(
+            "SELECT outcome, COUNT(*) FROM memories WHERE outcome != '' "
+            "GROUP BY outcome").fetchall()
+    except Exception:
+        rows = []
+    dist = {r[0]: r[1] for r in rows}
+    total = sum(dist.values())
+    return {
+        "total_marked": total,
+        "distribution": dist,
+        "success_rate": round(dist.get("success", 0) / total, 3) if total else 0.0,
+    }
+
+
+def search_weighted(mf, query: str, max_results: int = 10,
+                    outcome: str = "success",
+                    min_relevance: float = 0.0) -> List[Dict[str, Any]]:
+    """按结果加权检索：先用正常检索召回，再把命中 outcome 的记忆提到前面。
+
+    返回 [{memory_id, content_preview, outcome, importance, relevance}...]
+    """
+    from mindforge.modules.outcome_learning import _ensure_columns
+
+    results = mf.search(query=query, max_results=max(5, int(max_results) * 2),
+                        min_relevance=float(min_relevance or 0.0))
+    chunks = getattr(results, "chunks", None) or (results if isinstance(results, list) else [])
+    out = []
+    for r in chunks:
+        mid = getattr(r, "memory_id", None) or getattr(r, "id", None)
+        if not mid:
+            continue
+        row = None
+        try:
+            conn = mf._storage._get_conn()
+            _ensure_columns(conn)
+            row = conn.execute(
+                "SELECT outcome, importance, content FROM memories WHERE id = ?",
+                (mid,)).fetchone()
+        except Exception:
+            pass
+        out.append({
+            "memory_id": mid,
+            "content_preview": (getattr(r, "content", "") or "")[:200],
+            "outcome": row[0] if row else "",
+            "importance": row[1] if row else "",
+            "relevance": round(_chunk_score(r), 4),
+        })
+
+    # 目标 outcome 的记忆前置
+    out.sort(key=lambda x: (0 if x["outcome"] == outcome else 1,
+                            -x["relevance"]))
+    return out[:max(1, int(max_results))]