langchain-agentx-python 0.1__py3-none-any.whl

langchain_agentx/observability/evaluation/store.py

@@ -0,0 +1,258 @@
+"""
+evaluation/store.py — EvaluationStore，评估结果持久化。
+
+职责：
+    使用独立 SQLite 存储评估结果与诊断报告。
+
+链路位置：
+    由 EvaluationService 写入，供查询与诊断调度读取。
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import time
+import os
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from langchain_agentx.workspace import resolve_agent_workspace_config
+from .checkers.base import CheckResult
+
+
+@dataclass
+class DiagnosticReport:
+    session_id: str
+    checker_names: list[str]
+    report: str
+    model: str
+    created_at: float = field(default_factory=time.time)
+
+
+class EvaluationStore:
+    """评估结果 SQLite 存储。独立于 TraceStore。"""
+
+    def __init__(self, db_path: str | Path) -> None:
+        self.db_path = str(db_path)
+        self._init_db()
+
+    @classmethod
+    def from_env(cls) -> "EvaluationStore":
+        """从环境变量或 workspace 默认路径创建 EvaluationStore。"""
+        db_path = os.getenv("AGENTX_EVAL_DB_PATH")
+        if db_path:
+            return cls(db_path)
+        workspace_cfg = resolve_agent_workspace_config()
+        workspace_cfg.evaluation_db_path.parent.mkdir(parents=True, exist_ok=True)
+        return cls(workspace_cfg.evaluation_db_path)
+
+    @contextmanager
+    def _conn(self):  # type: ignore[no-untyped-def]
+        conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        conn.row_factory = sqlite3.Row
+        try:
+            yield conn
+            conn.commit()
+        finally:
+            conn.close()
+
+    def _init_db(self) -> None:
+        Path(self.db_path).parent.mkdir(parents=True, exist_ok=True)
+        with self._conn() as conn:
+            conn.execute("PRAGMA journal_mode=WAL")
+            conn.execute(
+                """
+                CREATE TABLE IF NOT EXISTS evaluation_results (
+                    id           INTEGER PRIMARY KEY AUTOINCREMENT,
+                    session_id   TEXT NOT NULL,
+                    checker_name TEXT NOT NULL,
+                    status       TEXT NOT NULL,
+                    message      TEXT NOT NULL,
+                    evidence     TEXT,
+                    created_at   REAL NOT NULL
+                )
+                """
+            )
+            conn.execute(
+                """
+                CREATE TABLE IF NOT EXISTS diagnostic_reports (
+                    id            INTEGER PRIMARY KEY AUTOINCREMENT,
+                    session_id    TEXT NOT NULL,
+                    checker_names TEXT NOT NULL,
+                    report        TEXT NOT NULL,
+                    model         TEXT NOT NULL,
+                    created_at    REAL NOT NULL
+                )
+                """
+            )
+            conn.execute("CREATE INDEX IF NOT EXISTS idx_eval_session ON evaluation_results(session_id)")
+            conn.execute("CREATE INDEX IF NOT EXISTS idx_eval_status ON evaluation_results(status)")
+            conn.execute("CREATE INDEX IF NOT EXISTS idx_diag_session ON diagnostic_reports(session_id)")
+
+    def save_results(self, session_id: str, results: list[CheckResult]) -> None:
+        now = time.time()
+        with self._conn() as conn:
+            conn.executemany(
+                (
+                    "INSERT INTO evaluation_results("
+                    "session_id,checker_name,status,message,evidence,created_at"
+                    ") VALUES(?,?,?,?,?,?)"
+                ),
+                [
+                    (
+                        session_id,
+                        result.checker_name,
+                        result.status,
+                        result.message,
+                        json.dumps(result.evidence),
+                        now,
+                    )
+                    for result in results
+                ],
+            )
+
+    def save_diagnostic_report(self, report: DiagnosticReport) -> None:
+        with self._conn() as conn:
+            conn.execute(
+                (
+                    "INSERT INTO diagnostic_reports("
+                    "session_id,checker_names,report,model,created_at"
+                    ") VALUES(?,?,?,?,?)"
+                ),
+                (
+                    report.session_id,
+                    json.dumps(report.checker_names),
+                    report.report,
+                    report.model,
+                    report.created_at,
+                ),
+            )
+
+    def get_results(self, session_id: str) -> list[CheckResult]:
+        with self._conn() as conn:
+            rows = conn.execute(
+                (
+                    "SELECT checker_name,status,message,evidence "
+                    "FROM evaluation_results WHERE session_id=? ORDER BY id"
+                ),
+                (session_id,),
+            ).fetchall()
+        return [
+            CheckResult(
+                checker_name=row["checker_name"],
+                status=row["status"],
+                message=row["message"],
+                evidence=json.loads(row["evidence"] or "{}"),
+            )
+            for row in rows
+        ]
+
+    def get_recent_failures(self, limit: int = 20) -> list[dict[str, Any]]:
+        with self._conn() as conn:
+            rows = conn.execute(
+                (
+                    "SELECT session_id,checker_name,message,evidence,created_at "
+                    "FROM evaluation_results WHERE status='fail' "
+                    "ORDER BY created_at DESC LIMIT ?"
+                ),
+                (limit,),
+            ).fetchall()
+        return [dict(row) for row in rows]
+
+    def get_failure_sessions(self, limit: int = 20) -> list[dict[str, Any]]:
+        """按 session 聚合返回失败会话摘要。"""
+        with self._conn() as conn:
+            rows = conn.execute(
+                """
+                SELECT
+                    e.session_id AS session_id,
+                    COUNT(*) AS fail_count,
+                    MIN(e.created_at) AS first_failed_at,
+                    MAX(e.created_at) AS last_failed_at,
+                    GROUP_CONCAT(DISTINCT e.checker_name) AS checker_names
+                FROM evaluation_results e
+                WHERE e.status='fail'
+                GROUP BY e.session_id
+                ORDER BY last_failed_at DESC
+                LIMIT ?
+                """,
+                (limit,),
+            ).fetchall()
+        items: list[dict[str, Any]] = []
+        for row in rows:
+            checker_names = str(row["checker_names"] or "")
+            items.append(
+                {
+                    "session_id": row["session_id"],
+                    "fail_count": int(row["fail_count"] or 0),
+                    "first_failed_at": row["first_failed_at"],
+                    "last_failed_at": row["last_failed_at"],
+                    "checker_names": [name for name in checker_names.split(",") if name],
+                }
+            )
+        return items
+
+    def get_diagnostic_report(self, session_id: str) -> DiagnosticReport | None:
+        with self._conn() as conn:
+            row = conn.execute(
+                "SELECT * FROM diagnostic_reports WHERE session_id=? ORDER BY created_at DESC LIMIT 1",
+                (session_id,),
+            ).fetchone()
+        if row is None:
+            return None
+        return DiagnosticReport(
+            session_id=row["session_id"],
+            checker_names=json.loads(row["checker_names"]),
+            report=row["report"],
+            model=row["model"],
+            created_at=row["created_at"],
+        )
+
+    def has_unanalyzed_failures(self, session_id: str) -> bool:
+        """判断指定 session 是否有 fail 且尚无诊断报告。"""
+        with self._conn() as conn:
+            fail_count = conn.execute(
+                "SELECT COUNT(*) FROM evaluation_results WHERE session_id=? AND status='fail'",
+                (session_id,),
+            ).fetchone()[0]
+            if fail_count == 0:
+                return False
+            report_count = conn.execute(
+                "SELECT COUNT(*) FROM diagnostic_reports WHERE session_id=?",
+                (session_id,),
+            ).fetchone()[0]
+            return report_count == 0
+
+    def get_sessions_needing_analysis(self, limit: int = 50) -> list[str]:
+        """返回有 fail 但无诊断报告的 session_id 列表。"""
+        with self._conn() as conn:
+            rows = conn.execute(
+                """
+                SELECT DISTINCT e.session_id
+                FROM evaluation_results e
+                WHERE e.status = 'fail'
+                  AND NOT EXISTS (
+                      SELECT 1 FROM diagnostic_reports d WHERE d.session_id = e.session_id
+                  )
+                ORDER BY e.created_at DESC
+                LIMIT ?
+                """,
+                (limit,),
+            ).fetchall()
+        return [row["session_id"] for row in rows]
+
+    def cleanup_old_reports(self, *, days: int = 180) -> int:
+        """按 ``created_at``（REAL，Unix 秒）删除早于 ``days`` 天的 ``diagnostic_reports`` 行。"""
+        cutoff = time.time() - float(days) * 24.0 * 3600.0
+        with self._conn() as conn:
+            cur = conn.execute(
+                "DELETE FROM diagnostic_reports WHERE created_at < ?",
+                (cutoff,),
+            )
+            return int(cur.rowcount or 0)
+
+
+__all__ = ["EvaluationStore", "DiagnosticReport"]

langchain_agentx/observability/events/__init__.py

@@ -0,0 +1,15 @@
+"""Agent utilities for LangChain AgentX."""
+
+from .langchain_agentx_event_adapter import (
+    LangchainAgentEventType,
+    LangchainAgentEvent,
+    EventAdapterState,
+    LangGraphToLangchainAgentEventAdapter,
+)
+
+__all__ = [
+    "LangchainAgentEventType",
+    "LangchainAgentEvent",
+    "EventAdapterState",
+    "LangGraphToLangchainAgentEventAdapter",
+]