agentengine-sdk-python 0.8.4__py3-none-any.whl → 0.8.5__py3-none-any.whl

ksadk/harness/metrics.py

@@ -0,0 +1,90 @@
+"""长任务质量指标（长任务方案 §11，P2）。
+
+从 RuntimeEvent 流与召回/写入事实计算可重复的离线指标：
+
+- Goal/Constraint Retention：压缩后目标/约束是否保留；
+- Evidence Fidelity：关键事实（ID/金额/日期）在压缩摘要中的保留率；
+- Tool Pair Integrity：tool.call.begin/end 配对完整率；
+- Memory Precision / Recall：写入事实中有用且正确的比例 / 需要时成功召回的比例。
+
+指标函数是纯函数——绑定固定测试集调用（避免不同环境数据直接比较）。
+"""
+
+from __future__ import annotations
+
+from typing import Any, Iterable, Sequence
+
+from ksadk.harness.context_engine import extract_critical_facts
+from ksadk.harness.events import EventType, RuntimeEvent
+
+
+def compaction_fidelity(events: Sequence[RuntimeEvent]) -> dict[str, float]:
+    """从事件流计算压缩保真指标（无压缩事件时全部 1.0）。"""
+    completed = [e for e in events if e.event_type == EventType.CONTEXT_COMPACTION_COMPLETED]
+    tool_begins = [e for e in events if e.event_type == EventType.TOOL_CALL_BEGIN]
+    tool_ends = [e for e in events if e.event_type == EventType.TOOL_CALL_END]
+    return {
+        "goal_retention": _avg(
+            [_check(e, "goal_preserved") for e in completed]
+        ),
+        "constraint_retention": _avg(
+            [_check(e, "critical_facts_preserved") for e in completed]
+        ),
+        "tool_pair_integrity": (
+            len(tool_ends) / len(tool_begins) if tool_begins else 1.0
+        ),
+        "compaction_count": float(len(completed)),
+    }
+
+
+def evidence_fidelity(summary_text: str, original_text: str) -> float:
+    """摘要相对原文的关键事实保留率（Evidence Fidelity）。"""
+    expected = extract_critical_facts(original_text)
+    if not expected:
+        return 1.0
+    retained = extract_critical_facts(summary_text)
+    return len(expected & retained) / len(expected)
+
+
+def memory_precision_recall(
+    *,
+    written: Iterable[str],
+    relevant: Iterable[str],
+    recalled: Iterable[str],
+) -> dict[str, float]:
+    """Memory Precision / Recall（离线评测，绑定固定测试集）。
+
+    - precision = |relevant ∩ written| / |written|（写入中有用且正确的比例）；
+    - recall = |relevant ∩ recalled| / |relevant|（需要时成功召回的比例）。
+    """
+    written_set = {w.strip() for w in written if w.strip()}
+    relevant_set = {r.strip() for r in relevant if r.strip()}
+    recalled_set = {r.strip() for r in recalled if r.strip()}
+    precision = (
+        len(written_set & relevant_set) / len(written_set) if written_set else 1.0
+    )
+    recall = len(relevant_set & recalled_set) / len(relevant_set) if relevant_set else 1.0
+    return {"precision": precision, "recall": recall}
+
+
+def _avg(values: list[float]) -> float:
+    return sum(values) / len(values) if values else 1.0
+
+
+def _check(event: RuntimeEvent, key: str) -> float:
+    return float(event.payload.get("quality_checks", {}).get(key, True))
+
+
+def summarize_run(run_events: Sequence[RuntimeEvent], **extra: Any) -> dict[str, Any]:
+    """一次长任务 Run 的指标汇总（供评测门禁消费）。"""
+    report: dict[str, Any] = {"compaction": compaction_fidelity(run_events)}
+    report.update(extra)
+    return report
+
+
+__all__ = [
+    "compaction_fidelity",
+    "evidence_fidelity",
+    "memory_precision_recall",
+    "summarize_run",
+]

ksadk/harness/model_capability.py

@@ -0,0 +1,253 @@
+"""Provider 模型能力声明与运行时模式选择。
+
+外部兼容性验证的实测结果写入能力声明，而不是只留在
+报告里。Reasoner 在发起调用前查询声明：某模型被实测证明流式 Tool Call
+不可靠（例如 qwen3-8-max 流式/非流式行为不一致）时，运行时自动选择
+该模型受支持的模式，而不是把失败留给生产流量。
+
+声明来源（优先级从高到低）：
+
+1. 进程内 ``declare_model_capability``（宿主显式注入）；
+2. ``KSADK_MODEL_CAPABILITY_FILE`` 指向的 JSON 文件（由矩阵 CLI
+   ``--declare-out`` 产出，进发布工件）；
+3. 未声明 → 保持现有行为（环境变量/构造参数决定），不猜测。
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import threading
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+# 与 reasoner.py 保持一致的流式开关语义。
+_ENV_STREAMING = "KSADK_MODEL_STREAMING"
+_TRUTHY = {"1", "true", "yes", "on"}
+_ENV_CAPABILITY_FILE = "KSADK_MODEL_CAPABILITY_FILE"
+
+
+@dataclass(frozen=True)
+class ModelCapabilityDeclaration:
+    """一个模型标识的实测能力。未记录的维度保持 ``None``（不猜测）。"""
+
+    model_id: str
+    supports_basic_chat: bool | None = None
+    supports_tool_calling: bool | None = None
+    supports_streaming: bool | None = None
+    supports_streaming_tool_calls: bool | None = None
+    usage_reported: bool | None = None
+    #: 声明来源（矩阵端点/版本或 "manual"），用于审计，不含 Secret。
+    source: str = ""
+
+    def __post_init__(self) -> None:
+        if not self.model_id.strip():
+            raise ValueError("model_id must not be empty")
+
+
+@dataclass
+class ModelCapabilityStore:
+    """线程安全的能力声明注册表。"""
+
+    declarations: dict[str, ModelCapabilityDeclaration] = field(default_factory=dict)
+    _lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
+
+    def declare(self, declaration: ModelCapabilityDeclaration) -> None:
+        with self._lock:
+            self.declarations[declaration.model_id] = declaration
+
+    def lookup(self, model_id: str) -> ModelCapabilityDeclaration | None:
+        with self._lock:
+            return self.declarations.get(model_id)
+
+    def all(self) -> list[ModelCapabilityDeclaration]:  # noqa: A003 - registry API
+        with self._lock:
+            return list(self.declarations.values())
+
+    def to_dict(self) -> dict[str, Any]:
+        with self._lock:
+            return {
+                "schemaVersion": 1,
+                "declarations": [asdict(item) for item in self.declarations.values()],
+            }
+
+    @classmethod
+    def from_dict(cls, payload: dict[str, Any]) -> "ModelCapabilityStore":
+        raw = payload.get("declarations") or []
+        if not isinstance(raw, list):
+            raise ValueError("capability file 'declarations' must be a list")
+        store = cls()
+        for item in raw:
+            if not isinstance(item, dict):
+                raise ValueError("each declaration must be an object")
+            known = {f for f in ModelCapabilityDeclaration.__dataclass_fields__}
+            store.declare(
+                ModelCapabilityDeclaration(
+                    **{key: value for key, value in item.items() if key in known}
+                )
+            )
+        return store
+
+    def save(self, path: str | Path) -> None:
+        target = Path(path)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        temporary = target.with_suffix(".json.tmp")
+        temporary.write_text(
+            json.dumps(self.to_dict(), ensure_ascii=False, indent=2) + "\n",
+            encoding="utf-8",
+        )
+        os.replace(temporary, target)
+
+
+def capabilities_from_matrix_report(
+    report: dict[str, Any], *, source: str = ""
+) -> list[ModelCapabilityDeclaration]:
+    """把模型矩阵报告的结果行转成能力声明。
+
+    只读实测布尔值；矩阵的 errors/usage 明细留在报告里，声明保持最小。
+    """
+
+    results = report.get("results")
+    if not isinstance(results, list):
+        raise ValueError("matrix report has no 'results' list")
+    endpoint = report.get("endpoint") or ""
+    effective_source = source or f"model-matrix:{endpoint}"
+    declarations: list[ModelCapabilityDeclaration] = []
+    for item in results:
+        if not isinstance(item, dict) or not item.get("model_id"):
+            continue
+
+        def _flag(name: str) -> bool | None:
+            value = item.get(name)
+            return bool(value) if isinstance(value, bool) else None
+
+        declarations.append(
+            ModelCapabilityDeclaration(
+                model_id=str(item["model_id"]),
+                supports_basic_chat=_flag("basic_chat"),
+                supports_tool_calling=_flag("tool_calling"),
+                supports_streaming=_flag("stream_text"),
+                supports_streaming_tool_calls=_flag("stream_tool_calling"),
+                usage_reported=_flag("usage_reported"),
+                source=effective_source,
+            )
+        )
+    return declarations
+
+
+def apply_matrix_report(
+    report: dict[str, Any],
+    store: ModelCapabilityStore,
+    *,
+    source: str = "",
+) -> int:
+    count = 0
+    for declaration in capabilities_from_matrix_report(report, source=source):
+        store.declare(declaration)
+        count += 1
+    return count
+
+
+def _alias_keys(model_id: str) -> tuple[str, ...]:
+    """一个标识的查找别名：原始、去 provider 前缀、去版本引用。"""
+
+    keys = [model_id]
+    if "/" in model_id:
+        keys.append(model_id.rsplit("/", 1)[-1])
+    return tuple(dict.fromkeys(keys))
+
+
+def resolve_streaming_mode(
+    *,
+    requested: bool,
+    model_id: str,
+    store: ModelCapabilityStore | None,
+    allow_upgrade: bool = False,
+) -> bool:
+    """按能力声明裁决流式请求。
+
+    - 未声明该模型 → 返回 ``requested``（保持现有行为，不猜测）；
+    - 声明 ``supports_streaming_tool_calls=False`` / ``supports_streaming=False``
+      → 请求流式时降级为非流式；
+    - ``allow_upgrade`` 且声明 ``supports_tool_calling=False`` 而流式
+      Tool Call 可用（实测存在此类模型：非流式不发起 Tool Call，流式
+      正常）→ 把默认非流式升级为流式；
+    - 非流式请求（显式关闭流式）永不升级。
+    """
+
+    if store is None:
+        return requested
+    for key in _alias_keys(model_id):
+        declaration = store.lookup(key)
+        if declaration is None:
+            continue
+        if requested:
+            if declaration.supports_streaming_tool_calls is False:
+                return False
+            if declaration.supports_streaming is False:
+                return False
+            return True
+        if (
+            allow_upgrade
+            and declaration.supports_tool_calling is False
+            and declaration.supports_streaming_tool_calls is True
+            and declaration.supports_streaming is not False
+        ):
+            return True
+        return False
+    return requested
+
+
+_GLOBAL_STORE: ModelCapabilityStore | None = None
+_GLOBAL_LOCK = threading.Lock()
+
+
+def load_capability_store(
+    path: str | Path | None = None,
+) -> ModelCapabilityStore | None:
+    """从 JSON 文件装配全局能力声明（缺失/无效文件返回 ``None``）。"""
+
+    resolved = Path(path or os.getenv(_ENV_CAPABILITY_FILE, "").strip() or "")
+    if not str(resolved) or not resolved.is_file():
+        return None
+    try:
+        payload = json.loads(resolved.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise RuntimeError(
+            f"invalid model capability file: {resolved}: {exc}"
+        ) from exc
+    return ModelCapabilityStore.from_dict(payload)
+
+
+def global_capability_store() -> ModelCapabilityStore | None:
+    """惰性装配进程级能力声明（``KSADK_MODEL_CAPABILITY_FILE``）。"""
+
+    global _GLOBAL_STORE
+    with _GLOBAL_LOCK:
+        if _GLOBAL_STORE is None:
+            configured = os.getenv(_ENV_CAPABILITY_FILE, "").strip()
+            if not configured:
+                return None
+            _GLOBAL_STORE = load_capability_store(configured)
+        return _GLOBAL_STORE
+
+
+def reset_global_capability_store() -> None:
+    """测试钩子：清空进程级缓存。"""
+
+    global _GLOBAL_STORE
+    with _GLOBAL_LOCK:
+        _GLOBAL_STORE = None
+
+
+__all__ = [
+    "ModelCapabilityDeclaration",
+    "ModelCapabilityStore",
+    "apply_matrix_report",
+    "capabilities_from_matrix_report",
+    "global_capability_store",
+    "load_capability_store",
+    "reset_global_capability_store",
+    "resolve_streaming_mode",
+]

ksadk/harness/model_provider.py

@@ -0,0 +1,220 @@
+"""模型提供方失败分类与可审计执行决策。
+
+分类优先读取结构化 HTTP/异常字段，字符串匹配只作为不同 SDK 间的兼容兜底。
+本模块不依赖 LiteLLM，其他 Reasoner 也能遵守同一 Provider Policy。
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from enum import Enum
+
+from ksadk.harness.spec import ModelFailureCategory, ModelProviderPolicy
+
+
+class ModelFailureKind(str, Enum):
+    RATE_LIMIT = "rate_limit"
+    TIMEOUT = "timeout"
+    UNAVAILABLE = "unavailable"
+    TRANSPORT = "transport"
+    AUTHENTICATION = "authentication"
+    PERMISSION = "permission"
+    INVALID_REQUEST = "invalid_request"
+    CONTEXT_LENGTH = "context_length"
+    NOT_FOUND = "not_found"
+    UNKNOWN = "unknown"
+
+
+class ModelFailureAction(str, Enum):
+    RETRY = "retry_same_model"
+    FAILOVER = "failover"
+    RECOVER_CONTEXT = "recover_context"
+    ABORT = "abort"
+    BUDGET_EXHAUSTED = "budget_exhausted"
+
+
+@dataclass(frozen=True)
+class ClassifiedModelFailure:
+    kind: ModelFailureKind
+    status_code: int | None = None
+
+    @property
+    def policy_category(self) -> ModelFailureCategory | None:
+        try:
+            return ModelFailureCategory(self.kind.value)
+        except ValueError:
+            return None
+
+
+_SENSITIVE_ERROR_PATTERNS = (
+    re.compile(r"\bsk-[A-Za-z0-9_-]{8,}\b", re.IGNORECASE),
+    re.compile(r"\bbearer\s+[^\s,;]+", re.IGNORECASE),
+    re.compile(
+        r"\b(?:api[_-]?key|authorization|token|secret)\s*[:=]\s*[^\s,;]+",
+        re.IGNORECASE,
+    ),
+)
+
+
+def safe_model_error_message(error: Exception, *, limit: int = 256) -> str:
+    """返回可进入事件/异常链的有界脱敏摘要。"""
+
+    message = str(error)
+    for pattern in _SENSITIVE_ERROR_PATTERNS:
+        message = pattern.sub("***", message)
+    return message[:limit]
+
+
+def classify_model_failure(error: Exception) -> ClassifiedModelFailure:
+    """把厂商/SDK 异常归一化为稳定类别，不暴露异常对象或凭证。"""
+
+    status_code = _status_code(error)
+    if status_code == 429:
+        return ClassifiedModelFailure(ModelFailureKind.RATE_LIMIT, status_code)
+    if status_code in {408, 504}:
+        return ClassifiedModelFailure(ModelFailureKind.TIMEOUT, status_code)
+    if status_code in {500, 502, 503}:
+        return ClassifiedModelFailure(ModelFailureKind.UNAVAILABLE, status_code)
+    if status_code == 401:
+        return ClassifiedModelFailure(ModelFailureKind.AUTHENTICATION, status_code)
+    if status_code == 403:
+        return ClassifiedModelFailure(ModelFailureKind.PERMISSION, status_code)
+    if status_code == 404:
+        return ClassifiedModelFailure(ModelFailureKind.NOT_FOUND, status_code)
+    name = type(error).__name__.lower()
+    message = str(error).lower()
+    combined = f"{name} {message}"
+    # OpenAI-compatible providers commonly report context overflow as HTTP 400.
+    # Inspect the structured/message semantics before the generic 4xx bucket;
+    # otherwise the Harness cannot enter its one-shot emergency compaction path.
+    if any(
+        token in combined
+        for token in (
+            "context length",
+            "maximum context",
+            "context window",
+            "too many tokens",
+            "context_length_exceeded",
+            # 真实网关的上下文溢出错误：HTTP 400 "input token limit is N"。
+            "input token limit",
+            "token limit exceeded",
+            # 金山云 OpenAI-compatible 网关实测文案。
+            "prompt exceeds max length",
+            "prompt exceeds maximum length",
+            "prompt is too long",
+        )
+    ):
+        return ClassifiedModelFailure(ModelFailureKind.CONTEXT_LENGTH, status_code)
+    if status_code is not None and 400 <= status_code < 500:
+        return ClassifiedModelFailure(ModelFailureKind.INVALID_REQUEST, status_code)
+    if any(token in combined for token in ("ratelimit", "rate limit", "too many requests")):
+        kind = ModelFailureKind.RATE_LIMIT
+    elif any(token in combined for token in ("timeout", "timed out", "deadline exceeded")):
+        kind = ModelFailureKind.TIMEOUT
+    elif any(
+        token in combined
+        for token in (
+            "service unavailable",
+            "temporarily unavailable",
+            "unavailable",
+            "provider 500",
+            "overloaded",
+        )
+    ):
+        kind = ModelFailureKind.UNAVAILABLE
+    elif any(
+        token in combined
+        for token in ("connection", "transport", "network", "dns", "socket", "remoteprotocol")
+    ):
+        kind = ModelFailureKind.TRANSPORT
+    elif any(token in combined for token in ("authentication", "unauthorized", "invalid api key")):
+        kind = ModelFailureKind.AUTHENTICATION
+    elif any(token in combined for token in ("permission", "forbidden")):
+        kind = ModelFailureKind.PERMISSION
+    elif any(token in combined for token in ("bad request", "invalid request", "invalid json")):
+        kind = ModelFailureKind.INVALID_REQUEST
+    elif "not found" in combined:
+        kind = ModelFailureKind.NOT_FOUND
+    else:
+        kind = ModelFailureKind.UNKNOWN
+    return ClassifiedModelFailure(kind, status_code)
+
+
+def decide_model_failure_action(
+    failure: ClassifiedModelFailure,
+    *,
+    policy: ModelProviderPolicy,
+    model_attempt: int,
+    total_attempt: int,
+    has_fallback: bool,
+) -> ModelFailureAction:
+    """根据分类、当前尝试与预算作出唯一动作。"""
+
+    # 相同的超长输入切换 Provider/模型通常仍会失败。Context Engine 必须先
+    # 重新规划并紧急压缩；是否重试由默认 Agent Loop 的一次性门禁决定。
+    if failure.kind is ModelFailureKind.CONTEXT_LENGTH:
+        return ModelFailureAction.RECOVER_CONTEXT
+    category = failure.policy_category
+    if category is None:
+        return ModelFailureAction.ABORT
+    retryable = category in policy.retryable_categories
+    failover_allowed = category in policy.failover_categories
+    if total_attempt >= policy.total_attempt_budget:
+        return ModelFailureAction.BUDGET_EXHAUSTED
+    if retryable and model_attempt < policy.max_attempts_per_model:
+        return ModelFailureAction.RETRY
+    if failover_allowed and has_fallback:
+        return ModelFailureAction.FAILOVER
+    return ModelFailureAction.ABORT
+
+
+def retry_delay_ms(policy: ModelProviderPolicy, *, model_attempt: int) -> int:
+    """确定性渐进退避；最后一次重试达到上限，不引入随机抖动。
+
+    ``model_attempt`` 是刚失败的本模型尝试序号。默认十次尝试对应九次
+    等待，按几何曲线从 1 秒平滑增加到 60 秒。整秒策略会对齐到整秒，
+    让 Studio 的公开进度保持易读；毫秒级自定义策略仍保留其精度。
+    """
+
+    initial = policy.initial_backoff_ms
+    maximum = policy.max_backoff_ms
+    if initial <= 0 or maximum <= initial:
+        return min(initial, maximum)
+
+    retry_slots = max(1, policy.max_attempts_per_model - 1)
+    index = min(max(0, model_attempt - 1), retry_slots - 1)
+    if retry_slots == 1:
+        delay = initial
+    else:
+        progress = index / (retry_slots - 1)
+        delay = initial * ((maximum / initial) ** progress)
+
+    quantum = 1_000 if initial >= 1_000 and maximum >= 1_000 else 1
+    rounded = int(round(delay / quantum) * quantum)
+    return min(max(initial, rounded), maximum)
+
+
+def _status_code(error: Exception) -> int | None:
+    for value in (
+        getattr(error, "status_code", None),
+        getattr(error, "http_status", None),
+        getattr(getattr(error, "response", None), "status_code", None),
+    ):
+        try:
+            if value is not None:
+                return int(value)
+        except (TypeError, ValueError):
+            continue
+    return None
+
+
+__all__ = [
+    "ClassifiedModelFailure",
+    "ModelFailureAction",
+    "ModelFailureKind",
+    "classify_model_failure",
+    "decide_model_failure_action",
+    "retry_delay_ms",
+    "safe_model_error_message",
+]

ksadk/harness/observability/__init__.py

@@ -0,0 +1,29 @@
+"""Harness 可观测性适配层（OTel/OTLP 导出 + Context 检查）。
+
+不改变 RuntimeEvent 契约——在事件流之上加 OpenTelemetry Span 适配，
+按 ``KSADK_OTEL_ENDPOINT`` 环境变量开关。无配置时不导出（零侵入）。
+
+原 ``observability.py`` 的 context_trace / compaction_trace / token_report /
+context_inspection / capability_health 保留在 :mod:`inspection` 子模块。
+"""
+
+from ksadk.harness.observability.inspection import (
+    capability_health,
+    compaction_trace,
+    context_inspection,
+    context_trace,
+    token_report,
+)
+from ksadk.harness.observability.metrics import PerformanceMetrics, collect_metrics
+from ksadk.harness.observability.otel_exporter import OTelSpanAdapter
+
+__all__ = [
+    "OTelSpanAdapter",
+    "PerformanceMetrics",
+    "capability_health",
+    "collect_metrics",
+    "compaction_trace",
+    "context_inspection",
+    "context_trace",
+    "token_report",
+]