quantnodes 3.0.0__py3-none-any.whl

QuantNodes/core/trajectory/pool.py

@@ -0,0 +1,211 @@
+"""TrajectoryPool — 演化轨迹池, 双层 Parquet + JSON 持久化。"""
+from __future__ import annotations
+
+import json
+import threading
+from pathlib import Path
+from typing import Iterator
+
+import pandas as pd
+
+from ..constants import PARQUET_COLUMNS as _PARQUET_COLUMNS
+from .entry import TrajectoryEntry
+from .lineage import children_of, descendants, lineage
+from QuantNodes.core.path_utils import ensure_dir
+
+
+_PARQUET_NAME = "trajectories.parquet"
+_ENTRIES_SUBDIR = "entries"
+
+
+class TrajectoryPool:
+    """演化轨迹池 — 持久化每轮实验的完整记录。
+
+    双层存储:
+        - Layer 1: {parquet_name}.parquet (元数据, append)
+        - Layer 2: entries/{entry_id}.json (完整记录, 独立子目录)
+
+    Phase H2 (2026-06-20):
+      - all()/_sorted_cache memoizes the timestamp-sorted list and
+        invalidates only on add() or reset(). Avoids repeated sorts when
+        many callers iterate the pool in tight loops (selector, loop,
+        cli/commands/factor.py, knowledge_base.py).
+      - _load() uses df.to_dict("records") instead of df.iterrows()
+        (5-10x faster bulk load on startup with large pools).
+
+    H1 (parquet O(n^2) append) deferred: ParquetWriter requires strict
+    schema uniformity per column, but the existing Parquet file holds
+    heterogeneous types (int/float/bool/str/list per column) and tests
+    assert type preservation. Re-implementing with native types would
+    require inferring the schema from existing data on every open.
+
+    Args:
+        base_dir: 池根目录 (自动创建)
+        parquet_name: Parquet 文件名 (默认 "trajectories.parquet"),
+                       允许不同实验共用 base_dir 各自存盘。
+    """
+
+    def __init__(
+        self,
+        base_dir: Path | str,
+        parquet_name: str = _PARQUET_NAME,
+    ):
+        self.base_dir = Path(base_dir)
+        ensure_dir(self.base_dir)
+        self._entries_dir = self.base_dir / _ENTRIES_SUBDIR
+        ensure_dir(self._entries_dir)
+        self._parquet_name = parquet_name
+        self._entries: dict[str, TrajectoryEntry] = {}
+        self._parquet_path = self.base_dir / self._parquet_name
+        self._lock = threading.Lock()
+        self._sorted_cache: list[TrajectoryEntry] | None = None
+        self._load()
+
+    def add(self, entry: TrajectoryEntry) -> None:
+        """添加一条 entry, 自动持久化。"""
+        with self._lock:
+            self._entries[entry.entry_id] = entry
+            self._persist(entry)
+            self._sorted_cache = None  # invalidate cache (H2)
+
+    def get(self, entry_id: str) -> TrajectoryEntry:
+        """按 ID 获取 entry, 不存在抛 KeyError。"""
+        if entry_id not in self._entries:
+            raise KeyError(f"entry_id 不存在: {entry_id}")
+        return self._entries[entry_id]
+
+    def all(self) -> list[TrajectoryEntry]:
+        """返回所有 entry 列表 (按时间排序)。
+
+        Cached: subsequent calls return the same list until add()/reset()
+        invalidates the cache.
+        """
+        if self._sorted_cache is None:
+            self._sorted_cache = sorted(self._entries.values(), key=lambda e: e.timestamp)
+        return list(self._sorted_cache)
+
+    def __iter__(self) -> Iterator[TrajectoryEntry]:
+        return iter(self.all())
+
+    def __len__(self) -> int:
+        return len(self._entries)
+
+    @property
+    def size(self) -> int:
+        return len(self._entries)
+
+    def reset(self) -> None:
+        """清空内存 + 删除所有持久化文件 (entries/ 子目录 + Parquet)。"""
+        with self._lock:
+            self._entries.clear()
+            self._sorted_cache = None
+            if self._parquet_path.exists():
+                self._parquet_path.unlink()
+            if self._entries_dir.exists():
+                for p in self._entries_dir.glob("*.json"):
+                    p.unlink()
+
+    # ------------------------------------------------------------------
+    # 过滤
+    # ------------------------------------------------------------------
+
+    def by_round(self, round_idx: int) -> list[TrajectoryEntry]:
+        return [e for e in self.all() if e.round_idx == round_idx]
+
+    def by_operation(self, operation: str) -> list[TrajectoryEntry]:
+        return [e for e in self.all() if e.operation == operation]
+
+    def filter(self, decision: bool | None = None) -> list[TrajectoryEntry]:
+        """按 feedback.decision 过滤 (None=不过滤)。"""
+        if decision is None:
+            return self.all()
+        return [
+            e for e in self.all()
+            if e.feedback is not None and e.feedback.decision == decision
+        ]
+
+    def best(self, top_n: int = 5, metric: str = "sharpe") -> list[TrajectoryEntry]:
+        """按 metric 降序, 返回 Top-N。"""
+        return sorted(
+            self.all(),
+            key=lambda e: float(e.metrics.get(metric, 0) or 0),
+            reverse=True,
+        )[:top_n]
+
+    def random(self, n: int, seed: int | None = None) -> list[TrajectoryEntry]:
+        """从池中随机抽 n 条。"""
+        import numpy as np
+        rng = np.random.default_rng(seed)
+        k = min(n, len(self._entries))
+        if k == 0:
+            return []
+        indices = rng.choice(len(self._entries), size=k, replace=False)
+        all_entries = self.all()
+        return [all_entries[int(i)] for i in indices]
+
+    # ------------------------------------------------------------------
+    # 谱系 (代理到 lineage.py)
+    # ------------------------------------------------------------------
+
+    def children_of(self, parent_id: str) -> list[TrajectoryEntry]:
+        return children_of(self._entries, parent_id)
+
+    def lineage(self, entry_id: str) -> list[TrajectoryEntry]:
+        return lineage(self._entries, entry_id)
+
+    def descendants(self, entry_id: str, max_depth: int | None = None) -> list[TrajectoryEntry]:
+        return descendants(self._entries, entry_id, max_depth=max_depth)
+
+    # ------------------------------------------------------------------
+    # 持久化
+    # ------------------------------------------------------------------
+
+    def _persist(self, entry: TrajectoryEntry) -> None:
+        """持久化单条 entry (Parquet append + JSON 单文件)。
+
+        保留原有 read+concat+rewrite 路径 (O(n^2) 但 schema 兼容)。
+        H1 的 ParquetWriter 增量追加方案因 schema 类型一致性限制暂未启用,
+        详见模块 docstring。H2 的 _sorted_cache 与 _load 优化已生效。
+        """
+        new_row = pd.DataFrame([entry.to_parquet_row()], columns=list(_PARQUET_COLUMNS))
+        if self._parquet_path.exists():
+            existing = pd.read_parquet(self._parquet_path)
+            for col in _PARQUET_COLUMNS:
+                if col not in existing.columns:
+                    existing[col] = None
+            existing = existing[list(_PARQUET_COLUMNS)].astype(object)
+            new_row = new_row.astype(object)
+            combined = pd.concat([existing, new_row], ignore_index=True)
+        else:
+            combined = new_row
+        combined.to_parquet(self._parquet_path, index=False)
+
+        json_path = self._entries_dir / f"{entry.entry_id}.json"
+        with json_path.open("w", encoding="utf-8") as f:
+            json.dump(entry.to_json_dict(), f, ensure_ascii=False, indent=2)
+
+    def _load(self) -> None:
+        """启动时从磁盘加载元数据 + JSON 详情。
+
+        H2: vectorized row->dict via to_dict('records') instead of iterrows.
+        """
+        if not self._parquet_path.exists():
+            return
+        try:
+            df = pd.read_parquet(self._parquet_path)
+        except Exception:
+            return
+        records = df.to_dict("records")
+        for row in records:
+            entry_id = str(row.get("entry_id", ""))
+            if not entry_id:
+                continue
+            json_path = self._entries_dir / f"{entry_id}.json"
+            if not json_path.exists():
+                continue
+            try:
+                with json_path.open("r", encoding="utf-8") as f:
+                    data = json.load(f)
+                self._entries[entry_id] = TrajectoryEntry.from_json_dict(data)
+            except Exception:
+                continue

QuantNodes/core/trajectory/selector.py

@@ -0,0 +1,140 @@
+"""ParentSelector — 5 种父辈选择策略。
+
+借鉴 QuantaAlpha `configs/experiment.yaml:73 parent_selection_strategy`。
+"""
+from __future__ import annotations
+
+from enum import Enum
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from .entry import TrajectoryEntry
+
+if TYPE_CHECKING:
+    from .pool import TrajectoryPool
+
+
+class SelectionStrategy(str, Enum):
+    """5 种选择策略。"""
+    BEST = "best"
+    RANDOM = "random"
+    WEIGHTED = "weighted"
+    WEIGHTED_INVERSE = "weighted_inverse"
+    TOP_PERCENT_PLUS_RANDOM = "top_percent_plus_random"
+
+
+class ParentSelector:
+    """父辈选择策略 — 从 TrajectoryPool 选 n 个 entry 作为下一轮 parent。
+
+    Args:
+        strategy: 策略名 (BEST/RANDOM/WEIGHTED/WEIGHTED_INVERSE/TOP_PERCENT_PLUS_RANDOM)
+        metric: 用于 best / weighted 的指标 (默认 'sharpe')
+        top_percent_threshold: top_percent_plus_random 中 top 比例 (默认 0.3)
+        seed: 随机种子 (None=不固定)
+        temperature: M4 weighted 策略的 softmax 温度 (默认 1.0,
+            > 1.0 更均匀采样, < 1.0 更集中于高分)
+    """
+
+    def __init__(
+        self,
+        strategy: str = "best",
+        metric: str = "sharpe",
+        top_percent_threshold: float = 0.3,
+        seed: int | None = None,
+        temperature: float = 1.0,
+    ):
+        valid = {s.value for s in SelectionStrategy}
+        if strategy not in valid:
+            raise ValueError(
+                f"未知 strategy: {strategy!r}, 应为 {sorted(valid)}"
+            )
+        self.strategy = strategy
+        self.metric = metric
+        self.top_percent_threshold = top_percent_threshold
+        self.temperature = temperature
+        self._rng = np.random.default_rng(seed)
+
+    def select(
+        self,
+        pool: list[TrajectoryEntry] | "TrajectoryPool",
+        n: int = 1,
+    ) -> list[TrajectoryEntry]:
+        """从 pool 选 n 个 entry (只选 feedback.decision=True 的)。"""
+        from .pool import TrajectoryPool  # 避免循环 import
+
+        if isinstance(pool, TrajectoryPool):
+            valid = [e for e in pool.all() if e.feedback and e.feedback.decision]
+        else:
+            valid = [e for e in pool if e.feedback and e.feedback.decision]
+        if not valid:
+            return []
+
+        if self.strategy == SelectionStrategy.BEST.value:
+            return self._best(valid, n)
+        if self.strategy == SelectionStrategy.RANDOM.value:
+            return self._random(valid, n)
+        if self.strategy == SelectionStrategy.WEIGHTED.value:
+            return self._weighted_sample(valid, n, inverse=False)
+        if self.strategy == SelectionStrategy.WEIGHTED_INVERSE.value:
+            return self._weighted_sample(valid, n, inverse=True)
+        if self.strategy == SelectionStrategy.TOP_PERCENT_PLUS_RANDOM.value:
+            return self._top_percent_plus_random(valid, n)
+        return []
+
+    # ------------------------------------------------------------------
+    # 5 个策略
+    # ------------------------------------------------------------------
+
+    def _best(self, valid: list[TrajectoryEntry], n: int) -> list[TrajectoryEntry]:
+        sorted_entries = sorted(
+            valid,
+            key=lambda e: float(e.metrics.get(self.metric, 0) or 0),
+            reverse=True,
+        )
+        return sorted_entries[:n]
+
+    def _random(self, valid: list[TrajectoryEntry], n: int) -> list[TrajectoryEntry]:
+        k = min(n, len(valid))
+        indices = self._rng.choice(len(valid), size=k, replace=False)
+        return [valid[int(i)] for i in indices]
+
+    def _weighted_sample(
+        self,
+        valid: list[TrajectoryEntry],
+        n: int,
+        inverse: bool,
+    ) -> list[TrajectoryEntry]:
+        scores = np.array(
+            [float(e.metrics.get(self.metric, 0) or 0) for e in valid],
+            dtype=float,
+        )
+        if inverse:
+            scores = -scores
+        scores = scores - scores.max()
+        # M4: temperature 调节 softmax 锐度
+        #   T → 0: 趋近 argmax; T → ∞: 趋近均匀; T=1: 标准 softmax
+        weights = np.exp(scores / max(self.temperature, 1e-9))
+        total = weights.sum()
+        if total == 0 or not np.isfinite(total):
+            return self._random(valid, n)
+        weights = weights / total
+        k = min(n, len(valid))
+        indices = self._rng.choice(len(valid), size=k, replace=False, p=weights)
+        return [valid[int(i)] for i in indices]
+
+    def _top_percent_plus_random(
+        self,
+        valid: list[TrajectoryEntry],
+        n: int,
+    ) -> list[TrajectoryEntry]:
+        top_n = max(1, int(len(valid) * self.top_percent_threshold))
+        top = self._best(valid, top_n)
+        if n <= top_n:
+            return top[:n]
+        rest = [e for e in valid if e.entry_id not in {t.entry_id for t in top}]
+        if not rest:
+            return top[:n]
+        k = min(n - top_n, len(rest))
+        extra_indices = self._rng.choice(len(rest), size=k, replace=False)
+        return top + [rest[int(i)] for i in extra_indices]

QuantNodes/core/visualization/__init__.py

@@ -0,0 +1,33 @@
+"""Visualization — 演化实验交互式 HTML 报告。
+
+公开 API:
+    - generate_report(entries, metric) -> dict[Figure]
+    - generate_html(entries, metric, output_path) -> str / 写文件
+    - ReportBuilder / Section / Report (Phase 1.3 fluent API)
+    - build_lineage_layout(entries, metric) -> {nodes, edges}
+    - lineage_dag_figure / metric_distribution_figure / metric_per_round_figure
+    - gate_breakdown_figure / operation_breakdown_figure
+"""
+from .lineage_dag import build_lineage_layout, lineage_dag_figure
+from .metric_distribution import (
+    metric_distribution_figure,
+    metric_per_round_figure,
+)
+from .gate_breakdown import gate_breakdown_figure, operation_breakdown_figure
+from .report import generate_html, generate_report
+from .builder import ReportBuilder, Section, Report
+
+__all__ = [
+    "build_lineage_layout",
+    "lineage_dag_figure",
+    "metric_distribution_figure",
+    "metric_per_round_figure",
+    "gate_breakdown_figure",
+    "operation_breakdown_figure",
+    "generate_report",
+    "generate_html",
+    # Phase 1.3
+    "ReportBuilder",
+    "Section",
+    "Report",
+]

QuantNodes/core/visualization/builder.py

@@ -0,0 +1,233 @@
+# coding=utf-8
+"""ReportBuilder — 流式构造演化报告 (Phase 1.3, Builder pattern).
+
+替代原来 generate_report() 内部硬编码的 dict 拼接逻辑。
+提供 .with_title().with_overview().add_section(...).build() 流式 API,
+并保证向后兼容 (旧 generate_report() 内部调用 ReportBuilder)。
+
+设计要点:
+  - Section 封装一个 fig + 可选标题
+  - 报告结构: title + overview table + 多个 section, 顺序可定制
+  - 与 plotly 解耦, 任何有 .to_html() 方法的对象都可作 section
+  - build() 返回 Report dataclass (含 sections 列表 + overview dict),
+    渲染时由 ReportRenderer 转换为 HTML
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Union
+
+from QuantNodes.core.path_utils import ensure_parent
+from QuantNodes.core.trajectory.entry import TrajectoryEntry
+
+
+# ============================================================================
+# Data classes
+# ============================================================================
+
+@dataclass
+class Section:
+    """报告中的一个 section, 含可选标题 + 可渲染对象 (有 .to_html() 方法)。"""
+    title: str
+    payload: Any  # plotly Figure 或其他有 to_html 的对象
+
+    def render(self, include_plotlyjs: bool = False, div_id: Optional[str] = None) -> str:
+        """渲染为 HTML fragment, 不含 <h2> 标题。
+
+        div_id: 传给 payload.to_html() 的 div id; 传 None 时用默认
+                 f"fig_{title.lower().replace(' ', '_')}" (与旧实现兼容)。
+
+        v3.0.0 graceful degradation: when ``payload`` is None (e.g. plotly
+        not installed), we emit a friendly install hint instead of
+        crashing. ``to_dict`` still records the None value so callers
+        can introspect the report structure.
+        """
+        if self.payload is None:
+            return (
+                "<p><em>(plotly not installed — install with "
+                "<code>pip install plotly</code> to render this chart)</em></p>"
+            )
+        if hasattr(self.payload, "to_html"):
+            kwargs = {"full_html": False, "include_plotlyjs": include_plotlyjs}
+            if div_id is None:
+                div_id = f"fig_{self.title.lower().replace(' ', '_')}"
+            kwargs["div_id"] = div_id
+            return self.payload.to_html(**kwargs)
+        return f"<pre>{self.payload!r}</pre>"
+
+
+@dataclass
+class Report:
+    """构建完成的报告。"""
+    title: str
+    overview: Dict[str, Any] = field(default_factory=dict)
+    sections: List[Section] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """与旧 generate_report() 返回格式兼容: {overview, *section_keys}。"""
+        out: Dict[str, Any] = {"overview": self.overview}
+        for i, s in enumerate(self.sections):
+            key = s.title.lower().replace(" ", "_")
+            out[key] = s.payload
+        return out
+
+
+# ============================================================================
+# ReportBuilder
+# ============================================================================
+
+class ReportBuilder:
+    """流式构造 Report (Builder pattern)。
+
+    用法:
+        >>> from QuantNodes.core.visualization import ReportBuilder
+        >>> report = (
+        ...     ReportBuilder()
+        ...     .with_title("演化报告")
+        ...     .with_overview({"size": 10, "passed": 8})
+        ...     .add_section("Lineage DAG", lineage_fig)
+        ...     .add_section("Metrics", metric_fig)
+        ...     .build()
+        ... )
+    """
+
+    def __init__(self) -> None:
+        self._title: str = "Report"
+        self._overview: Dict[str, Any] = {}
+        self._sections: List[Section] = []
+        self._preset_loaded = False
+
+    def with_title(self, title: str) -> "ReportBuilder":
+        """设置报告标题。"""
+        self._title = title
+        return self
+
+    def with_overview(self, overview: Dict[str, Any]) -> "ReportBuilder":
+        """设置概览 dict (会被原样放入 Report.overview)。"""
+        self._overview = dict(overview)
+        return self
+
+    def add_section(self, title: str, payload: Any) -> "ReportBuilder":
+        """追加一个 section。payload 通常是 plotly Figure, 任何有 to_html() 的对象都接受。"""
+        self._sections.append(Section(title=title, payload=payload))
+        return self
+
+    def with_evolve_preset(
+        self,
+        entries: Sequence[TrajectoryEntry],
+        metric: str = "sharpe",
+        figure_factories: Optional[Dict[str, Callable]] = None,
+    ) -> "ReportBuilder":
+        """预置: 加载 4-5 个演化实验常用图 + 概览 (Phase 1.3 兼容 generate_report)。
+
+        Args:
+            entries: TrajectoryEntry 列表或 Mapping
+            metric: 用于可视化的指标名
+            figure_factories: 覆盖默认 5 个图 builder 的 dict
+                e.g. {"lineage_dag": my_lineage_factory}
+        """
+        from .gate_breakdown import gate_breakdown_figure, operation_breakdown_figure
+        from .lineage_dag import lineage_dag_figure
+        from .metric_distribution import metric_distribution_figure, metric_per_round_figure
+
+        items = list(entries.values() if isinstance(entries, Mapping) else entries)
+        n = len(items)
+        n_passed = sum(1 for e in items if e.feedback and e.feedback.decision)
+        n_rejected = n - n_passed
+        rounds = sorted({e.round_idx for e in items}) if items else []
+        metrics_vals = [
+            float((e.metrics or {}).get(metric, 0) or 0)
+            for e in items
+            if (e.metrics or {}).get(metric) is not None
+        ]
+        best_metric = max(metrics_vals) if metrics_vals else 0.0
+
+        self._overview = {
+            "size": n,
+            "rounds": len(rounds),
+            "passed": n_passed,
+            "passed_pct": (n_passed / n) if n > 0 else 0.0,
+            "rejected": n_rejected,
+            "best_metric": best_metric,
+            "metric": metric,
+        }
+        self.with_title(self._title or "QuantNodes 演化实验报告")
+
+        factories = figure_factories or {
+            "lineage_dag": lambda: lineage_dag_figure(
+                items, metric=metric, title=f"演化谱系 DAG (按 {metric})"
+            ),
+            "metric_distribution": lambda: metric_distribution_figure(items, metric=metric),
+            "metric_per_round": lambda: metric_per_round_figure(items, metric=metric),
+            "gate_breakdown": lambda: gate_breakdown_figure(items),
+            "operation_breakdown": lambda: operation_breakdown_figure(items),
+        }
+        for title, factory in factories.items():
+            self._sections.append(Section(title=title, payload=factory()))
+        self._preset_loaded = True
+        return self
+
+    def build(self) -> Report:
+        """构建最终 Report 对象。"""
+        return Report(title=self._title, overview=self._overview, sections=list(self._sections))
+
+    def build_to_html(
+        self,
+        output_path: Optional[Union[str, Path]] = None,
+        plotly_cdn: str = '<script src="https://cdn.plot.ly/plotly-2.27.0.min.js"></script>',
+    ) -> str:
+        """直接构建并渲染为 HTML 字符串 (可选写入文件)。"""
+        report = self.build()
+        html = _render_html(report, plotly_cdn=plotly_cdn)
+        if output_path is not None:
+            output_path = Path(output_path)
+            ensure_parent(output_path)
+            output_path.write_text(html, encoding="utf-8")
+        return html
+
+
+# ============================================================================
+# HTML rendering
+# ============================================================================
+
+_OVERVIEW_TEMPLATE = """
+<h2>概览</h2>
+<table border="1" cellpadding="6" style="border-collapse:collapse;">
+  <tr><th>指标</th><th>值</th></tr>
+  <tr><td>总 entry 数</td><td>{size}</td></tr>
+  <tr><td>演化轮数</td><td>{rounds}</td></tr>
+  <tr><td>通过数</td><td>{passed} ({passed_pct:.1%})</td></tr>
+  <tr><td>拒绝数</td><td>{rejected}</td></tr>
+  <tr><td>Best {metric}</td><td>{best_metric:.4f}</td></tr>
+</table>
+"""
+
+
+def _render_html(report: Report, plotly_cdn: str) -> str:
+    overview_html = _OVERVIEW_TEMPLATE.format(**report.overview) if report.overview else ""
+    # 使用 title 派生 div_id (与旧 generate_html 行为兼容, div_id 格式 fig_<key>)
+    parts = [f"<h2>{s.title}</h2>\n{s.render(include_plotlyjs=False)}"
+             for s in report.sections]
+    return f"""<!DOCTYPE html>
+<html lang="zh-CN">
+<head>
+<meta charset="utf-8">
+<title>{report.title}</title>
+<style>
+body {{ font-family: -apple-system, sans-serif; margin: 20px; max-width: 1200px; }}
+h1 {{ border-bottom: 2px solid #4C78A8; padding-bottom: 8px; }}
+table {{ background: #fafafa; }}
+th {{ background: #4C78A8; color: white; }}
+</style>
+{plotly_cdn}
+</head>
+<body>
+<h1>{report.title}</h1>
+{overview_html}
+{"".join(parts)}
+<hr>
+<p style="color: #888; font-size: 12px;">Generated by QuantNodes ReportBuilder</p>
+</body>
+</html>
+"""