quantnodes 3.0.0__py3-none-any.whl

QuantNodes/core/knowledge/__init__.py

@@ -0,0 +1,69 @@
+"""Knowledge RAG — 因子知识库 + RAG prompt + 谱系展开 (Week 8)
++ 谱系压缩 (Week 9) + 评估指标 (Week 10)。
+
+公开 API:
+    - BaseRetriever (Protocol)
+    - TFIDFRetriever (sklearn 实现)
+    - IdentityRetriever (纯 Python fallback / 测试)
+    - make_retriever(kind)
+    - KnowledgeBase
+    - build_rag_prompt() (含谱系 RAG + 压缩)
+    - expand_lineage() / expand_lineage_batch() (Week 8)
+    - Compressor / compress_lineage() (Week 9)
+    - 5 个评估指标 + RAGEvaluator (Week 10)
+"""
+from .retriever import (
+    BaseRetriever,
+    IdentityRetriever,
+    TFIDFRetriever,
+    make_retriever,
+)
+from .knowledge_base import KnowledgeBase
+from .lineage_compress import Compressor, CompressedLineage, compress_lineage
+from .lineage_expand import expand_lineage, expand_lineage_batch
+from .rag_prompt import build_rag_prompt
+from .metrics import (
+    EvalReport,
+    QueryResult,
+    RAGEvaluator,
+    dcg_at_k,
+    hit_rate_at_k,
+    intra_list_diversity,
+    jaccard_similarity,
+    lineage_coverage,
+    mean_hit_rate_at_k,
+    mean_lineage_coverage,
+    mean_ndcg_at_k,
+    mean_reciprocal_rank,
+    ndcg_at_k,
+    reciprocal_rank,
+)
+
+__all__ = [
+    "BaseRetriever",
+    "TFIDFRetriever",
+    "IdentityRetriever",
+    "make_retriever",
+    "KnowledgeBase",
+    "expand_lineage",
+    "expand_lineage_batch",
+    "Compressor",
+    "CompressedLineage",
+    "compress_lineage",
+    "build_rag_prompt",
+    # Week 10 评估指标
+    "dcg_at_k",
+    "hit_rate_at_k",
+    "intra_list_diversity",
+    "jaccard_similarity",
+    "lineage_coverage",
+    "mean_hit_rate_at_k",
+    "mean_lineage_coverage",
+    "mean_ndcg_at_k",
+    "mean_reciprocal_rank",
+    "ndcg_at_k",
+    "reciprocal_rank",
+    "EvalReport",
+    "QueryResult",
+    "RAGEvaluator",
+]

QuantNodes/core/knowledge/knowledge_base.py

@@ -0,0 +1,217 @@
+"""KnowledgeBase — 因子知识库 (TrajectoryEntry + Retriever)。"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Optional
+
+import pandas as pd
+from pydantic import BaseModel, Field
+
+from ..trajectory import TrajectoryEntry, TrajectoryPool
+from .retriever import BaseRetriever, make_retriever
+from QuantNodes.core.path_utils import ensure_parent
+
+
+# H19: 默认字段权重 (外部可覆盖)
+DEFAULT_FIELD_WEIGHTS: dict[str, float] = {
+    "name": 3.0,
+    "expression": 2.0,
+    "hypothesis": 2.5,
+    "description": 2.0,
+    "summary": 1.0,
+}
+
+
+class KnowledgeBaseSetting(BaseModel):
+    """H19: KnowledgeBase 字段权重配置 (Pydantic)。"""
+    field_weights: dict[str, float] = Field(
+        default_factory=lambda: dict(DEFAULT_FIELD_WEIGHTS),
+        description="字段权重: 控制检索文本构造时各字段的重复次数",
+    )
+
+
+class KnowledgeBase:
+    """因子知识库。
+
+    Args:
+        retriever: BaseRetriever 实现 (默认 TFIDF)
+        pool: 可选 TrajectoryPool 同步源
+        setting: H19 字段权重配置 (None=默认)
+        field_weights: H19 字段权重 dict (None=默认, 兼容旧 API)
+    """
+
+    def __init__(
+        self,
+        retriever: Optional[BaseRetriever] = None,
+        pool: Optional[TrajectoryPool] = None,
+        setting: Optional[KnowledgeBaseSetting] = None,
+        field_weights: Optional[dict[str, float]] = None,
+    ):
+        self.retriever = retriever or make_retriever("tfidf")
+        self.pool = pool
+        self._entry_ids: set[str] = set()  # 已索引的 entry_id
+        # H19: 字段权重 — setting 优先, field_weights 其次, 默认最后
+        if setting is not None:
+            self._field_weights = dict(setting.field_weights)
+        elif field_weights is not None:
+            self._field_weights = dict(field_weights)
+        else:
+            self._field_weights = dict(DEFAULT_FIELD_WEIGHTS)
+
+    # ------------------------------------------------------------------
+    # CRUD
+    # ------------------------------------------------------------------
+
+    def add(self, entry: TrajectoryEntry) -> None:
+        """添加 entry 到知识库 (含 retriever indexing)。"""
+        text = self._entry_to_text(entry)
+        self.retriever.add(entry.entry_id, text)
+        self._entry_ids.add(entry.entry_id)
+
+    def add_many(self, entries) -> int:
+        """批量添加, 返回新加入数。"""
+        n = 0
+        for e in entries:
+            if e.entry_id not in self._entry_ids:
+                self.add(e)
+                n += 1
+        return n
+
+    def sync_from_pool(self) -> int:
+        """从 pool 同步所有未索引 entry。"""
+        if self.pool is None:
+            return 0
+        return self.add_many(self.pool.all())
+
+    # ------------------------------------------------------------------
+    # 检索
+    # ------------------------------------------------------------------
+
+    def query(
+        self,
+        text: str,
+        top_k: int = 5,
+        min_score: float = 0.0,
+    ) -> list[tuple[TrajectoryEntry, float]]:
+        """返回 top_k 检索结果, 包含 entry 对象 + 相似度。
+
+        排序: score 降序
+        """
+        results = self.retriever.query(text, top_k=top_k)
+        if not self.pool:
+            return [(None, score) for _, score in results]
+        out: list[tuple[TrajectoryEntry, float]] = []
+        for doc_id, score in results:
+            if score < min_score:
+                continue
+            try:
+                entry = self.pool.get(doc_id)
+            except KeyError:
+                continue
+            out.append((entry, score))
+        return out
+
+    def query_with_lineage(
+        self,
+        text: str,
+        top_k: int = 5,
+        max_ancestor_depth: int = 2,
+        max_descendant_depth: int = 2,
+    ) -> list[dict[str, Any]]:
+        """检索 + 附加谱系 (parents + children, 可配深度) 上下文。
+
+        Args:
+            text: 查询文本
+            top_k: 检索数量
+            max_ancestor_depth: 祖先深度 (1=parent, 2=grandparent, ...)
+            max_descendant_depth: 后裔深度
+        """
+        results = self.query(text, top_k=top_k)
+        out = []
+        for entry, score in results:
+            ctx = {
+                "entry": entry,
+                "score": score,
+                "ancestors": [],
+                "descendants": [],
+                "parents": [],
+                "children": [],
+            }
+            if self.pool and entry is not None:
+                from .lineage_expand import expand_lineage
+                expanded = expand_lineage(
+                    self.pool, entry.entry_id,
+                    max_ancestor_depth=max_ancestor_depth,
+                    max_descendant_depth=max_descendant_depth,
+                )
+                ctx["ancestors"] = [e for _, e in expanded["ancestors"]]
+                ctx["descendants"] = [e for _, e in expanded["descendants"]]
+                # 兼容旧字段 (depth=1)
+                ctx["parents"] = [e for d, e in expanded["ancestors"] if d == 1]
+                ctx["children"] = [e for d, e in expanded["descendants"] if d == 1]
+            out.append(ctx)
+        return out
+
+    # ------------------------------------------------------------------
+    # 持久化 (Parquet 索引 + JSON 单文件 - 与 TrajectoryPool 复用)
+    # ------------------------------------------------------------------
+
+    def save(self, path: Path | str) -> None:
+        """保存知识库索引 (entry_id 列表 + retriever 文本)。"""
+        path = Path(path)
+        ensure_parent(path)
+        rows = []
+        if self.pool:
+            for e in self.pool.all():
+                rows.append({
+                    "entry_id": e.entry_id,
+                    "text": self._entry_to_text(e),
+                })
+        df = pd.DataFrame(rows)
+        df.to_parquet(path, index=False)
+
+    @classmethod
+    def load(
+        cls,
+        path: Path | str,
+        pool: Optional[TrajectoryPool] = None,
+        retriever: Optional[BaseRetriever] = None,
+    ) -> "KnowledgeBase":
+        """从 Parquet 索引加载 (需配合 pool)。"""
+        path = Path(path)
+        df = pd.read_parquet(path)
+        kb = cls(retriever=retriever, pool=pool)
+        for _, row in df.iterrows():
+            kb.retriever.add(str(row["entry_id"]), str(row["text"]))
+            kb._entry_ids.add(str(row["entry_id"]))
+        return kb
+
+    # ------------------------------------------------------------------
+    # 内部
+    # ------------------------------------------------------------------
+
+    def _entry_to_text(self, entry: TrajectoryEntry) -> str:
+        """把 entry 转为可检索文本 (按字段权重重复关键 token)。"""
+        parts: list[str] = []
+        cfg = entry.config_snapshot or {}
+        factor_cfg = cfg.get("factor", {}) if isinstance(cfg, dict) else {}
+        for src, key in (
+            (factor_cfg.get("name", ""), "name"),
+            (factor_cfg.get("expression", ""), "expression"),
+            (factor_cfg.get("hypothesis", ""), "hypothesis"),
+            (factor_cfg.get("description", ""), "description"),
+        ):
+            weight = int(self._field_weights.get(key, 1))
+            parts.extend([str(src)] * weight)
+        if entry.feedback:
+            parts.extend(
+                [str(entry.feedback.summary or "")]
+                * int(self._field_weights.get("summary", 1))
+            )
+        if entry.metrics:
+            for k, v in entry.metrics.items():
+                parts.append(f"{k}={v}")
+        return " ".join(parts)
+
+    def __len__(self) -> int:
+        return len(self._entry_ids)

QuantNodes/core/knowledge/lineage_compress.py

@@ -0,0 +1,196 @@
+"""谱系压缩 — LLM 总结祖先/后裔链为 1 段简短描述, 减少 token。
+
+设计:
+    - Compressor 类 (类似 LLMJudge 的协议)
+    - compress_lineage(entries, relation) -> str
+    - 默认 mock 模式: 启发式拼接 (1 行 / entry, 含 name + operation + sharpe)
+    - 支持自定义 llm_callable (真实 LLM)
+    - 限制输出 token: max_tokens (默认 200 chars)
+
+用途:
+    build_rag_prompt(..., use_compress=True) 时,
+    每个示例的 ancestors / descendants 段先压缩为 1 行, 再注入 prompt。
+"""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Callable, Optional
+
+
+# 单个 entry 的压缩模板 (heuristic 模式)
+_HEURISTIC_ENTRY_TEMPLATE = "{name} ({op} r{round}, sharpe={sharpe:.2f})"
+
+
+@dataclass
+class CompressedLineage:
+    """压缩结果。"""
+    summary: str           # 1 段简短总结
+    original_count: int    # 压缩前 entry 数
+    compressed_chars: int  # 压缩后字符数
+    method: str            # "llm" / "heuristic"
+
+
+# ============================================================================
+# Compressor
+# ============================================================================
+
+class Compressor:
+    """谱系压缩器。
+
+    Args:
+        model: "mock" (默认启发式) / "deepseek-v3" / 其他 (需 llm_callable)
+        max_tokens: 输出最大字符数 (默认 200)
+        llm_callable: 自定义 LLM 调用函数, 接受 prompt 返回 string
+    """
+
+    def __init__(
+        self,
+        model: str = "mock",
+        max_tokens: int = 200,
+        llm_callable: Optional[Callable] = None,
+    ):
+        self.model = model
+        self.max_tokens = max_tokens
+        if llm_callable is None and model != "mock":
+            raise ValueError(
+                f"model={model!r} requires an explicit llm_callable. "
+                "Inject via get_llm_gateway() at the call site."
+            )
+        self._llm_callable = llm_callable
+
+    def compress(
+        self,
+        entries: list,  # list of (depth, TrajectoryEntry) 或 TrajectoryEntry
+        relation: str = "ancestors",  # "ancestors" / "descendants"
+    ) -> CompressedLineage:
+        """压缩一组 entry 为 1 段简短总结。
+
+        Args:
+            entries: lineage expand 后的 (depth, entry) 列表
+            relation: 关系类型 (ancestors / descendants), 仅用于 prompt 上下文
+
+        Returns:
+            CompressedLineage
+        """
+        # 规整化: (depth, entry)
+        normalized: list[tuple[int, object]] = []
+        for item in entries:
+            if isinstance(item, tuple) and len(item) == 2:
+                normalized.append(item)
+            else:
+                normalized.append((0, item))
+
+        if not normalized:
+            return CompressedLineage(
+                summary="", original_count=0, compressed_chars=0, method="heuristic"
+            )
+
+        if self._llm_callable is not None:
+            summary, method = self._llm_summarize(normalized, relation)
+        elif self.model == "mock":
+            summary = self._heuristic_summary(normalized, relation)
+            method = "heuristic"
+        else:
+            raise NotImplementedError(
+                f"真实 LLM '{self.model}' 未实现, 请提供 llm_callable 或使用 model='mock'"
+            )
+
+        # 截断到 max_tokens
+        if len(summary) > self.max_tokens:
+            summary = summary[: self.max_tokens - 3] + "..."
+
+        return CompressedLineage(
+            summary=summary,
+            original_count=len(normalized),
+            compressed_chars=len(summary),
+            method=method,
+        )
+
+    # ------------------------------------------------------------------
+    # 启发式 (无 LLM 依赖)
+    # ------------------------------------------------------------------
+
+    def _heuristic_summary(
+        self,
+        entries: list[tuple[int, object]],
+        relation: str,
+    ) -> str:
+        """启发式: 每 entry 1 行, name + operation + sharpe。"""
+        relation_arrow = "↑" if relation == "ancestors" else "↓"
+        parts: list[str] = []
+        for depth, entry in entries:
+            cfg = (entry.config_snapshot or {}).get("factor", {})
+            name = cfg.get("name", entry.entry_id[:8])
+            sharpe = (entry.metrics or {}).get("sharpe", 0)
+            entry_str = _HEURISTIC_ENTRY_TEMPLATE.format(
+                name=name, op=entry.operation,
+                round=entry.round_idx, sharpe=sharpe,
+            )
+            parts.append(f"{relation_arrow}d{depth} {entry_str}")
+        return " ; ".join(parts)
+
+    # ------------------------------------------------------------------
+    # LLM
+    # ------------------------------------------------------------------
+
+    def _llm_summarize(
+        self,
+        entries: list[tuple[int, object]],
+        relation: str,
+    ) -> tuple[str, str]:
+        """真实 LLM 总结。Returns (summary, effective_method)."""
+        prompt = self._build_prompt(entries, relation)
+        raw = self._llm_callable(prompt)
+        try:
+            data = json.loads(raw)
+            return str(data.get("summary", "")), "llm"
+        except (json.JSONDecodeError, TypeError, KeyError):
+            # 解析失败, fallback 到启发式
+            return self._heuristic_summary(entries, relation), "heuristic"
+
+    def _build_prompt(
+        self,
+        entries: list[tuple[int, object]],
+        relation: str,
+    ) -> str:
+        """构造 LLM prompt。"""
+        lines: list[str] = []
+        for depth, entry in entries:
+            cfg = (entry.config_snapshot or {}).get("factor", {})
+            sharpe = (entry.metrics or {}).get("sharpe", 0)
+            lines.append(
+                f"  depth={depth}, name={cfg.get('name', entry.entry_id[:8])}, "
+                f"op={entry.operation}, sharpe={sharpe}, "
+                f"desc={cfg.get('description', '')}"
+            )
+        entries_text = "\n".join(lines)
+        return (
+            f"请总结以下 {len(entries)} 个 {relation} 的核心设计思路 (限 {self.max_tokens} 字):\n"
+            f"{entries_text}\n\n"
+            f"返回 JSON: {{\"summary\": \"你的总结\"}}"
+        )
+
+
+# ============================================================================
+# 模块级便捷函数
+# ============================================================================
+
+def compress_lineage(
+    entries,
+    relation: str = "ancestors",
+    model: str = "mock",
+    max_tokens: int = 200,
+    llm_callable: Optional[Callable] = None,
+) -> CompressedLineage:
+    """便捷函数: 一次性压缩。
+
+    Args:
+        entries: (depth, entry) 列表
+        relation: "ancestors" / "descendants"
+        model: 同 Compressor
+        max_tokens: 同 Compressor
+        llm_callable: 同 Compressor
+    """
+    c = Compressor(model=model, max_tokens=max_tokens, llm_callable=llm_callable)
+    return c.compress(entries, relation=relation)

QuantNodes/core/knowledge/lineage_expand.py

@@ -0,0 +1,123 @@
+"""谱系展开 — 从根 entry 出发, BFS 收集 ancestors/descendants 到指定深度。
+
+输出格式:
+    {
+        'root': TrajectoryEntry,
+        'ancestors': [(depth, entry), ...],  # depth=1=parent, 2=grandparent, ...
+        'descendants': [(depth, entry), ...],
+    }
+"""
+from __future__ import annotations
+
+from collections import deque
+
+from ..trajectory import TrajectoryEntry, TrajectoryPool, children_of
+
+
+def expand_lineage(
+    pool: TrajectoryPool,
+    root_id: str,
+    max_ancestor_depth: int = 2,
+    max_descendant_depth: int = 2,
+    max_ancestors: int = 8,
+    max_descendants: int = 8,
+) -> dict:
+    """BFS 展开谱系, 返回 root + ancestors + descendants。
+
+    Args:
+        pool: TrajectoryPool
+        root_id: 中心 entry ID
+        max_ancestor_depth: 上溯深度 (1=parent, 2=grandparent, ...)
+        max_descendant_depth: 下探深度
+        max_ancestors: 最多收集多少 ancestor (避免 token 爆炸)
+        max_descendants: 最多收集多少 descendant
+
+    Returns:
+        dict: {'root', 'ancestors': [(depth, entry)], 'descendants': [(depth, entry)]}
+
+    H8 (2026-06-20): all_ids set is computed ONCE at function start
+    instead of inside every BFS step. Previously each BFS iteration
+    re-iterated pool.all() (O(N)) and rebuilt a set, making the total
+    cost O(K * N) where K is the BFS depth/width. Now: O(N) total.
+    """
+    all_ids = {e.entry_id for e in pool.all()}
+
+    if root_id not in all_ids:
+        return {"root": None, "ancestors": [], "descendants": []}
+
+    try:
+        root = pool.get(root_id)
+    except KeyError:
+        return {"root": None, "ancestors": [], "descendants": []}
+
+    ancestors: list[tuple[int, TrajectoryEntry]] = []
+    descendants: list[tuple[int, TrajectoryEntry]] = []
+
+    # ------------------------------------------------------------------
+    # Ancestors: BFS 上溯
+    # ------------------------------------------------------------------
+    visited_up: set[str] = {root_id}
+    queue: deque = deque([(root, 0)])
+    while queue and len(ancestors) < max_ancestors:
+        node, depth = queue.popleft()
+        if depth >= max_ancestor_depth:
+            continue
+        for pid in node.parent_ids:
+            if pid in visited_up or pid not in all_ids:
+                continue
+            visited_up.add(pid)
+            try:
+                parent = pool.get(pid)
+            except KeyError:
+                continue
+            ancestors.append((depth + 1, parent))
+            queue.append((parent, depth + 1))
+
+    # ------------------------------------------------------------------
+    # Descendants: BFS 下探
+    # ------------------------------------------------------------------
+    visited_down: set[str] = {root_id}
+    queue = deque([(root, 0)])
+    while queue and len(descendants) < max_descendants:
+        node, depth = queue.popleft()
+        if depth >= max_descendant_depth:
+            continue
+        for child in children_of(pool, node.entry_id):
+            if child.entry_id in visited_down:
+                continue
+            visited_down.add(child.entry_id)
+            descendants.append((depth + 1, child))
+            queue.append((child, depth + 1))
+
+    # 排序: 浅的在前, 同深度内按 round_idx 升序
+    ancestors.sort(key=lambda x: (x[0], x[1].round_idx))
+    descendants.sort(key=lambda x: (x[0], x[1].round_idx))
+
+    return {
+        "root": root,
+        "ancestors": ancestors,
+        "descendants": descendants,
+    }
+
+
+def expand_lineage_batch(
+    pool: TrajectoryPool,
+    root_ids: list[str],
+    max_ancestor_depth: int = 2,
+    max_descendant_depth: int = 2,
+    max_ancestors: int = 8,
+    max_descendants: int = 8,
+) -> list[dict]:
+    """批量展开, 去重。"""
+    seen: dict[str, dict] = {}
+    for rid in root_ids:
+        if rid in seen:
+            continue
+        seen[rid] = expand_lineage(
+            pool, rid,
+            max_ancestor_depth=max_ancestor_depth,
+            max_descendant_depth=max_descendant_depth,
+            max_ancestors=max_ancestors,
+            max_descendants=max_descendants,
+        )
+    return list(seen.values())

QuantNodes/core/knowledge/metrics/__init__.py

@@ -0,0 +1,43 @@
+"""RAG 评估指标 (Week 10)。
+
+公开 API:
+    - 5 个核心指标函数:
+        hit_rate_at_k / mean_hit_rate_at_k
+        ndcg_at_k / dcg_at_k / mean_ndcg_at_k
+        reciprocal_rank / mean_reciprocal_rank
+        lineage_coverage / mean_lineage_coverage
+        intra_list_diversity / jaccard_similarity
+    - RAGEvaluator: 汇总多 query 评估
+    - EvalReport / QueryResult: 结果数据类
+"""
+from .metrics import (
+    dcg_at_k,
+    hit_rate_at_k,
+    intra_list_diversity,
+    jaccard_similarity,
+    lineage_coverage,
+    mean_hit_rate_at_k,
+    mean_lineage_coverage,
+    mean_ndcg_at_k,
+    mean_reciprocal_rank,
+    ndcg_at_k,
+    reciprocal_rank,
+)
+from .evaluator import EvalReport, QueryResult, RAGEvaluator
+
+__all__ = [
+    "dcg_at_k",
+    "hit_rate_at_k",
+    "intra_list_diversity",
+    "jaccard_similarity",
+    "lineage_coverage",
+    "mean_hit_rate_at_k",
+    "mean_lineage_coverage",
+    "mean_ndcg_at_k",
+    "mean_reciprocal_rank",
+    "ndcg_at_k",
+    "reciprocal_rank",
+    "EvalReport",
+    "QueryResult",
+    "RAGEvaluator",
+]