langchain-agentx-python 0.1__py3-none-any.whl

langchain_agentx/observability/replay/store.py

@@ -0,0 +1,278 @@
+"""
+runtime/observability.py — 运行时可观测性总线与回放存储
+
+职责：
+    定义跨工具统一的可观测契约与存储策略，包括：
+    - schema 常量与归因字典（reason/layer）
+    - 分级脱敏与采样策略
+    - tool_call 级回放索引（内存 + 日期分片 JSONL）
+    - 历史分片 gzip 归档、TTL 与容量清理
+    - 查询过滤（tool_name/status/time range）
+
+OOP 边界：
+    - `ObservabilityTraceStore` 作为状态封装对象，负责持久化与查询；
+    - 顶层函数（sanitize/sample）保持纯函数，作为策略协作者供 store/adapter 复用；
+    - adapter/pipeline 只调用公共接口，不直接操作底层文件格式。
+
+与 CC 对比：
+    CC 有更完整的 tracing/decision 链路。本实现在 Runtime 层建立统一总线，
+    让不同工具共享同一观测模型，避免“每个工具各自打点”。
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+import gzip
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from langchain_agentx.workspace import resolve_agent_workspace_config
+
+OBS_SCHEMA_VERSION = "runtime-observability/v1"
+
+REASON_CODES = frozenset(
+    {
+        "long_timeout_policy",
+        "explicit_request",
+        "sandbox_not_requested",
+        "excluded_command",
+        "permission_denied",
+        "manual_approval_required",
+    }
+)
+
+DECISION_LAYERS = frozenset(
+    {
+        "mode_preflight",
+        "hardening_command",
+        "ast_guard",
+        "segment",
+        "mode_finalize",
+        "segment_single",
+    }
+)
+
+
+def _compile_extra_keywords() -> tuple[str, ...]:
+    base = ["token", "password", "secret", "key=", "authorization", "cookie", "sessionid"]
+    path = os.getenv("AGENTX_OBSERVABILITY_REDACTION_RULES_FILE")
+    if not path:
+        return tuple(base)
+    try:
+        with open(os.path.expanduser(path), encoding="utf-8") as f:
+            payload = json.load(f)
+    except Exception:
+        return tuple(base)
+    extra = payload.get("keywords", []) if isinstance(payload, dict) else []
+    for kw in extra:
+        if isinstance(kw, str) and kw:
+            base.append(kw.lower())
+    return tuple(dict.fromkeys(base))
+
+
+_REDACTION_KEYWORDS = _compile_extra_keywords()
+
+
+def sanitize_value(value: Any, *, level: str = "standard") -> Any:
+    """
+    level:
+      - strict: 对字符串更激进裁剪与脱敏
+      - standard: 默认
+      - dev: 仅截断，不脱敏（便于本地排障）
+    """
+    if isinstance(value, str):
+        lowered = value.lower()
+        if level != "dev" and any(k in lowered for k in _REDACTION_KEYWORDS):
+            return "<redacted>"
+        max_len = 120 if level == "strict" else 180
+        if len(value) > max_len:
+            return value[:max_len] + "...(truncated)"
+    if isinstance(value, dict):
+        return {k: sanitize_value(v, level=level) for k, v in value.items()}
+    if isinstance(value, list):
+        return [sanitize_value(v, level=level) for v in value]
+    return value
+
+
+def sample_events(events: list[dict[str, Any]], *, limit: int = 200) -> list[dict[str, Any]]:
+    if limit <= 0 or len(events) <= limit:
+        return events
+    head = max(1, limit // 2)
+    tail = max(1, limit - head)
+    sampled = events[:head] + events[-tail:]
+    sampled.insert(
+        head,
+        {"event": "events_sampled", "dropped_count": len(events) - len(sampled)},
+    )
+    return sampled
+
+
+@dataclass
+class ObservabilityTraceStore:
+    _records: dict[str, dict[str, Any]]
+
+    def __init__(self) -> None:
+        self._records = {}
+        workspace_cfg = resolve_agent_workspace_config()
+        self._root_dir = Path(
+            os.path.expanduser(
+                os.getenv(
+                    "AGENTX_OBSERVABILITY_ROOT",
+                    str(workspace_cfg.data_dir / "observability"),
+                )
+            )
+        )
+        self._max_records = int(os.getenv("AGENTX_OBSERVABILITY_MAX_RECORDS", "2000"))
+        self._ttl_sec = int(os.getenv("AGENTX_OBSERVABILITY_TTL_SEC", "86400"))
+        self._redaction_level = os.getenv("AGENTX_OBSERVABILITY_REDACTION_LEVEL", "standard")
+        self._archive_after_days = int(os.getenv("AGENTX_OBSERVABILITY_ARCHIVE_AFTER_DAYS", "3"))
+        self._root_dir.mkdir(parents=True, exist_ok=True)
+
+    def put(self, tool_call_id: str | None, record: dict[str, Any]) -> None:
+        if not tool_call_id:
+            return
+        now_ts = int(time.time())
+        safe_record = sanitize_value(
+            {
+            "saved_at_ts": now_ts,
+            **record,
+            },
+            level=self._redaction_level,
+        )
+        self._records[tool_call_id] = safe_record
+        self._append_sharded_jsonl(tool_call_id, safe_record)
+        self._prune()
+        self._archive_old_shards()
+
+    def get(self, tool_call_id: str) -> dict[str, Any] | None:
+        return self._records.get(tool_call_id)
+
+    def all_keys(self) -> list[str]:
+        return list(self._records.keys())
+
+    def tail_events(self, tool_call_id: str, offset: int = 0) -> tuple[list[dict[str, Any]], int]:
+        record = self._records.get(tool_call_id) or self._load_last_from_jsonl(tool_call_id)
+        if not record:
+            return [], offset
+        events = (((record.get("observability") or {}).get("events")) or [])
+        if not isinstance(events, list):
+            return [], offset
+        safe_offset = max(0, offset)
+        chunk = events[safe_offset:]
+        return chunk, safe_offset + len(chunk)
+
+    def query(
+        self,
+        *,
+        tool_name: str | None = None,
+        status: str | None = None,
+        start_ts: int | None = None,
+        end_ts: int | None = None,
+        limit: int = 200,
+    ) -> list[dict[str, Any]]:
+        rows = list(self._records.values())
+        rows.sort(key=lambda r: int(r.get("saved_at_ts", 0)), reverse=True)
+        out: list[dict[str, Any]] = []
+        for r in rows:
+            ts = int(r.get("saved_at_ts", 0))
+            if tool_name and r.get("tool_name") != tool_name:
+                continue
+            if status and r.get("status") != status:
+                continue
+            if start_ts is not None and ts < start_ts:
+                continue
+            if end_ts is not None and ts > end_ts:
+                continue
+            out.append(r)
+            if len(out) >= limit:
+                break
+        return out
+
+    def _current_shard_path(self) -> Path:
+        day = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+        return self._root_dir / f"replay-{day}.jsonl"
+
+    def _append_sharded_jsonl(self, tool_call_id: str, record: dict[str, Any]) -> None:
+        path = self._current_shard_path()
+        line = json.dumps(
+            {"tool_call_id": tool_call_id, "record": record},
+            ensure_ascii=False,
+        )
+        with open(path, "a", encoding="utf-8") as f:
+            f.write(line + "\n")
+
+    def _load_last_from_jsonl(self, tool_call_id: str) -> dict[str, Any] | None:
+        last: dict[str, Any] | None = None
+        for path in sorted(self._root_dir.glob("replay-*.jsonl")):
+            with open(path, encoding="utf-8") as f:
+                for line in f:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        data = json.loads(line)
+                    except json.JSONDecodeError:
+                        continue
+                    if data.get("tool_call_id") == tool_call_id:
+                        rec = data.get("record")
+                        if isinstance(rec, dict):
+                            last = rec
+        return last
+
+    def _prune(self) -> None:
+        if not self._records:
+            return
+        now_ts = int(time.time())
+        keys = list(self._records.keys())
+        for k in keys:
+            ts = int(self._records[k].get("saved_at_ts", now_ts))
+            if now_ts - ts > self._ttl_sec:
+                self._records.pop(k, None)
+        if len(self._records) <= self._max_records:
+            return
+        ordered = sorted(
+            self._records.items(),
+            key=lambda kv: int((kv[1] or {}).get("saved_at_ts", 0)),
+        )
+        drop_n = max(0, len(ordered) - self._max_records)
+        for i in range(drop_n):
+            self._records.pop(ordered[i][0], None)
+
+    def _archive_old_shards(self) -> None:
+        cutoff_days = self._archive_after_days
+        if cutoff_days <= 0:
+            return
+        now = datetime.now(timezone.utc).date()
+        for path in self._root_dir.glob("replay-*.jsonl"):
+            name = path.name.replace("replay-", "").replace(".jsonl", "")
+            try:
+                shard_day = datetime.strptime(name, "%Y-%m-%d").date()
+            except ValueError:
+                continue
+            if (now - shard_day).days < cutoff_days:
+                continue
+            gz_path = path.with_suffix(".jsonl.gz")
+            if gz_path.exists():
+                try:
+                    path.unlink()
+                except OSError:
+                    pass
+                continue
+            try:
+                with open(path, "rb") as src, gzip.open(gz_path, "wb") as dst:
+                    dst.write(src.read())
+                path.unlink()
+            except OSError:
+                continue
+
+
+_GLOBAL_TRACE_STORE = ObservabilityTraceStore()
+
+
+def get_global_trace_store() -> ObservabilityTraceStore:
+    return _GLOBAL_TRACE_STORE
+

langchain_agentx/observability/replay/ui.py

@@ -0,0 +1,47 @@
+"""
+runtime/observability_ui.py — 可观测性 UI 投影适配器
+
+职责：
+    将 replay 记录映射为前端可渲染的时间线结构（decision + event），
+    作为 UI 层与 runtime 观测数据之间的协议转换层。
+
+OOP 边界：
+    - 该模块只做数据投影，不参与存储与查询；
+    - 输入为 replay record，输出为 UI timeline DTO；
+    - 可被 Web/桌面/CLI TUI 多端复用。
+
+与 CC 对比：
+    对齐 CC 中“执行轨迹 + 决策归因”的可视化呈现方向，
+    保持视图层与存储层解耦。
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def to_ui_timeline(record: dict[str, Any] | None) -> list[dict[str, Any]]:
+    if not record:
+        return []
+    items: list[dict[str, Any]] = []
+    for d in record.get("decision_trace", []):
+        items.append(
+            {
+                "kind": "decision",
+                "title": f"{d.get('layer')} -> {d.get('behavior')}",
+                "policy_id": d.get("policy_id"),
+                "command": d.get("command"),
+            }
+        )
+    events = (((record.get("observability") or {}).get("events")) or [])
+    for ev in events:
+        items.append(
+            {
+                "kind": "event",
+                "title": ev.get("event"),
+                "seq": ev.get("seq"),
+                "payload": {k: v for k, v in ev.items() if k not in {"seq", "event"}},
+            }
+        )
+    return items
+

langchain_agentx/observability/trace/__init__.py

@@ -0,0 +1,25 @@
+"""
+Trace 系统：全链路可观测性。
+
+核心组件：
+- TraceCollector: 采集 + 查询
+- TraceLifecycleCollector / TraceCallbackHandler: 生命周期与 LLM/Tool 采集
+- SqliteTraceStore: 持久化存储
+"""
+
+from .models import TraceSession, TraceSpan
+from .collector import TraceCollector
+from .sqlite_store import SqliteTraceStore
+from .trace_lifecycle_collector import TraceLifecycleCollector
+from .trace_callback import TraceCallbackHandler
+from .hook_event_emitter import HookEventEmitter
+
+__all__ = [
+    "TraceSession",
+    "TraceSpan",
+    "TraceCollector",
+    "SqliteTraceStore",
+    "TraceLifecycleCollector",
+    "TraceCallbackHandler",
+    "HookEventEmitter",
+]