xg-cli 1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xg/__init__.py +3 -0
- xg/__main__.py +3 -0
- xg/adaptive/__init__.py +27 -0
- xg/adaptive/calibrate.py +191 -0
- xg/adaptive/feedback.py +170 -0
- xg/adaptive/learned_rules.py +284 -0
- xg/adaptive/signals.py +125 -0
- xg/adaptive/store.py +164 -0
- xg/agent/__init__.py +0 -0
- xg/agent/plan.py +631 -0
- xg/agent/react.py +268 -0
- xg/agent/team.py +1793 -0
- xg/assets/router.lgb +0 -0
- xg/assets/router_semantics.json +21278 -0
- xg/assets/router_semantics.onnx +0 -0
- xg/cli/__init__.py +0 -0
- xg/cli/app.py +1372 -0
- xg/cli/commands.py +921 -0
- xg/cli/completion.py +740 -0
- xg/cli/help.py +185 -0
- xg/cli/train.py +200 -0
- xg/config/__init__.py +0 -0
- xg/config/env_writer.py +121 -0
- xg/config/manager.py +446 -0
- xg/config/mcp.py +229 -0
- xg/config/provider_service.py +267 -0
- xg/config/providers.py +46 -0
- xg/config/settings.py +258 -0
- xg/config/skills.py +100 -0
- xg/config/smart_router_service.py +121 -0
- xg/config/web.py +140 -0
- xg/input_history/__init__.py +7 -0
- xg/input_history/models.py +28 -0
- xg/input_history/persistence.py +126 -0
- xg/input_history/policy.py +39 -0
- xg/input_history/prompt_toolkit.py +36 -0
- xg/input_history/store.py +118 -0
- xg/llm/__init__.py +0 -0
- xg/llm/client.py +49 -0
- xg/llm/factory.py +40 -0
- xg/llm/openai_compat.py +275 -0
- xg/llm/types.py +98 -0
- xg/mcp/__init__.py +4 -0
- xg/mcp/http.py +192 -0
- xg/mcp/manager.py +726 -0
- xg/mcp/models.py +86 -0
- xg/mcp/protocol.py +62 -0
- xg/mcp/resources.py +72 -0
- xg/mcp/schema.py +137 -0
- xg/mcp/stdio.py +210 -0
- xg/mcp/transport.py +66 -0
- xg/memory/__init__.py +15 -0
- xg/memory/context.py +327 -0
- xg/memory/manager.py +111 -0
- xg/memory/models.py +41 -0
- xg/memory/project.py +187 -0
- xg/memory/store.py +144 -0
- xg/router/__init__.py +124 -0
- xg/router/features.py +66 -0
- xg/router/keywords.py +50 -0
- xg/router/ml_router.py +178 -0
- xg/router/model_tiers.py +73 -0
- xg/router/postprocess.py +167 -0
- xg/router/rule_router.py +77 -0
- xg/router/semantic.py +138 -0
- xg/safety/__init__.py +0 -0
- xg/safety/audit.py +96 -0
- xg/safety/guards.py +106 -0
- xg/safety/hitl.py +73 -0
- xg/skill/__init__.py +9 -0
- xg/skill/errors.py +45 -0
- xg/skill/loader.py +42 -0
- xg/skill/models.py +57 -0
- xg/skill/parser.py +93 -0
- xg/skill/policy.py +40 -0
- xg/skill/prompt.py +45 -0
- xg/skill/registry.py +169 -0
- xg/tool/__init__.py +0 -0
- xg/tool/builtin.py +356 -0
- xg/tool/registry.py +228 -0
- xg/tui/__init__.py +34 -0
- xg/tui/app.py +612 -0
- xg/tui/controller.py +1296 -0
- xg/tui/diagrams/__init__.py +22 -0
- xg/tui/diagrams/layout.py +110 -0
- xg/tui/diagrams/markdown.py +39 -0
- xg/tui/diagrams/model.py +36 -0
- xg/tui/diagrams/parser.py +119 -0
- xg/tui/diagrams/renderer.py +551 -0
- xg/tui/i18n.py +169 -0
- xg/tui/messages.py +45 -0
- xg/tui/plan_renderables.py +147 -0
- xg/tui/reducer.py +1029 -0
- xg/tui/renderables.py +252 -0
- xg/tui/state.py +240 -0
- xg/tui/theme.tcss +208 -0
- xg/tui/widgets/__init__.py +1 -0
- xg/tui/widgets/action_card.py +94 -0
- xg/tui/widgets/agent_group_card.py +39 -0
- xg/tui/widgets/approval_modal.py +59 -0
- xg/tui/widgets/collapsible_card.py +40 -0
- xg/tui/widgets/command_suggestions.py +128 -0
- xg/tui/widgets/composer.py +151 -0
- xg/tui/widgets/config_panel.py +198 -0
- xg/tui/widgets/confirm_modal.py +31 -0
- xg/tui/widgets/footer.py +9 -0
- xg/tui/widgets/header.py +118 -0
- xg/tui/widgets/inspector.py +378 -0
- xg/tui/widgets/plan_modal.py +53 -0
- xg/tui/widgets/provider_form.py +141 -0
- xg/tui/widgets/queue_status.py +30 -0
- xg/tui/widgets/smart_router_form.py +95 -0
- xg/tui/widgets/transcript.py +354 -0
- xg/tui/workers.py +14 -0
- xg/web/__init__.py +21 -0
- xg/web/errors.py +57 -0
- xg/web/extract.py +127 -0
- xg/web/fetch.py +106 -0
- xg/web/markdown.py +77 -0
- xg/web/models.py +91 -0
- xg/web/providers.py +79 -0
- xg/web/search.py +118 -0
- xg/web/searxng.py +27 -0
- xg/web/serpapi.py +29 -0
- xg/web/url_policy.py +110 -0
- xg/web/zhipu.py +27 -0
- xg_cli-1.0.dist-info/METADATA +284 -0
- xg_cli-1.0.dist-info/RECORD +130 -0
- xg_cli-1.0.dist-info/WHEEL +4 -0
- xg_cli-1.0.dist-info/entry_points.txt +2 -0
xg/memory/store.py
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"""项目级 SQLite 长期记忆仓库。"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import sqlite3
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from xg.memory.models import MemoryEntry
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
SCHEMA_VERSION = 1
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def normalize_content(content: str) -> str:
|
|
16
|
+
return " ".join(content.split()).casefold()
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _parse_time(value: str) -> datetime:
|
|
20
|
+
return datetime.fromisoformat(value)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class SQLiteMemoryStore:
|
|
24
|
+
def __init__(self, db_path: str | Path) -> None:
|
|
25
|
+
self.db_path = Path(db_path)
|
|
26
|
+
self._initialize()
|
|
27
|
+
|
|
28
|
+
def _connect(self) -> sqlite3.Connection:
|
|
29
|
+
conn = sqlite3.connect(self.db_path, timeout=5.0)
|
|
30
|
+
conn.row_factory = sqlite3.Row
|
|
31
|
+
conn.execute("PRAGMA busy_timeout=5000")
|
|
32
|
+
conn.execute("PRAGMA foreign_keys=ON")
|
|
33
|
+
conn.execute("PRAGMA journal_mode=WAL")
|
|
34
|
+
return conn
|
|
35
|
+
|
|
36
|
+
def _initialize(self) -> None:
|
|
37
|
+
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
38
|
+
with self._connect() as conn:
|
|
39
|
+
version = int(conn.execute("PRAGMA user_version").fetchone()[0])
|
|
40
|
+
if version < 1:
|
|
41
|
+
conn.executescript(
|
|
42
|
+
"""
|
|
43
|
+
CREATE TABLE IF NOT EXISTS memories (
|
|
44
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
45
|
+
content TEXT NOT NULL CHECK (length(trim(content)) > 0),
|
|
46
|
+
normalized_content TEXT NOT NULL,
|
|
47
|
+
source TEXT NOT NULL DEFAULT 'manual',
|
|
48
|
+
created_at TEXT NOT NULL,
|
|
49
|
+
updated_at TEXT NOT NULL
|
|
50
|
+
);
|
|
51
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_memories_normalized
|
|
52
|
+
ON memories(normalized_content);
|
|
53
|
+
CREATE INDEX IF NOT EXISTS idx_memories_updated
|
|
54
|
+
ON memories(updated_at DESC, id DESC);
|
|
55
|
+
PRAGMA user_version = 1;
|
|
56
|
+
"""
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
@staticmethod
|
|
60
|
+
def _entry(row: sqlite3.Row) -> MemoryEntry:
|
|
61
|
+
return MemoryEntry(
|
|
62
|
+
id=int(row["id"]),
|
|
63
|
+
content=str(row["content"]),
|
|
64
|
+
source=str(row["source"]),
|
|
65
|
+
created_at=_parse_time(str(row["created_at"])),
|
|
66
|
+
updated_at=_parse_time(str(row["updated_at"])),
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
def save(self, content: str, source: str = "manual") -> tuple[MemoryEntry, bool]:
|
|
70
|
+
content = content.strip()
|
|
71
|
+
if not content:
|
|
72
|
+
raise ValueError("记忆内容不能为空")
|
|
73
|
+
if len(content) > 4000:
|
|
74
|
+
raise ValueError("单条记忆最多 4000 个字符")
|
|
75
|
+
normalized = normalize_content(content)
|
|
76
|
+
now = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
77
|
+
with self._connect() as conn:
|
|
78
|
+
row = conn.execute(
|
|
79
|
+
"SELECT * FROM memories WHERE normalized_content = ?", (normalized,)
|
|
80
|
+
).fetchone()
|
|
81
|
+
if row is not None:
|
|
82
|
+
conn.execute(
|
|
83
|
+
"UPDATE memories SET updated_at = ?, source = ? WHERE id = ?",
|
|
84
|
+
(now, source, int(row["id"])),
|
|
85
|
+
)
|
|
86
|
+
row = conn.execute(
|
|
87
|
+
"SELECT * FROM memories WHERE id = ?", (int(row["id"]),)
|
|
88
|
+
).fetchone()
|
|
89
|
+
assert row is not None
|
|
90
|
+
return self._entry(row), False
|
|
91
|
+
cursor = conn.execute(
|
|
92
|
+
"""
|
|
93
|
+
INSERT INTO memories(content, normalized_content, source, created_at, updated_at)
|
|
94
|
+
VALUES (?, ?, ?, ?, ?)
|
|
95
|
+
""",
|
|
96
|
+
(content, normalized, source, now, now),
|
|
97
|
+
)
|
|
98
|
+
row = conn.execute(
|
|
99
|
+
"SELECT * FROM memories WHERE id = ?", (cursor.lastrowid,)
|
|
100
|
+
).fetchone()
|
|
101
|
+
assert row is not None
|
|
102
|
+
return self._entry(row), True
|
|
103
|
+
|
|
104
|
+
def list(self, limit: int = 20, offset: int = 0) -> list[MemoryEntry]:
|
|
105
|
+
limit = max(1, min(int(limit), 100))
|
|
106
|
+
offset = max(0, int(offset))
|
|
107
|
+
with self._connect() as conn:
|
|
108
|
+
rows = conn.execute(
|
|
109
|
+
"SELECT * FROM memories ORDER BY updated_at DESC, id DESC LIMIT ? OFFSET ?",
|
|
110
|
+
(limit, offset),
|
|
111
|
+
).fetchall()
|
|
112
|
+
return [self._entry(row) for row in rows]
|
|
113
|
+
|
|
114
|
+
def search(self, query: str, limit: int = 20) -> list[MemoryEntry]:
|
|
115
|
+
query = normalize_content(query)
|
|
116
|
+
if not query:
|
|
117
|
+
return []
|
|
118
|
+
limit = max(1, min(int(limit), 100))
|
|
119
|
+
with self._connect() as conn:
|
|
120
|
+
rows = conn.execute(
|
|
121
|
+
"""
|
|
122
|
+
SELECT * FROM memories
|
|
123
|
+
WHERE instr(normalized_content, ?) > 0
|
|
124
|
+
ORDER BY updated_at DESC, id DESC
|
|
125
|
+
LIMIT ?
|
|
126
|
+
""",
|
|
127
|
+
(query, limit),
|
|
128
|
+
).fetchall()
|
|
129
|
+
return [self._entry(row) for row in rows]
|
|
130
|
+
|
|
131
|
+
def delete(self, memory_id: int) -> bool:
|
|
132
|
+
with self._connect() as conn:
|
|
133
|
+
cursor = conn.execute("DELETE FROM memories WHERE id = ?", (int(memory_id),))
|
|
134
|
+
return cursor.rowcount > 0
|
|
135
|
+
|
|
136
|
+
def clear(self) -> int:
|
|
137
|
+
with self._connect() as conn:
|
|
138
|
+
count = int(conn.execute("SELECT count(*) FROM memories").fetchone()[0])
|
|
139
|
+
conn.execute("DELETE FROM memories")
|
|
140
|
+
return count
|
|
141
|
+
|
|
142
|
+
def count(self) -> int:
|
|
143
|
+
with self._connect() as conn:
|
|
144
|
+
return int(conn.execute("SELECT count(*) FROM memories").fetchone()[0])
|
xg/router/__init__.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""SmartRouter 路由核心包。
|
|
2
|
+
|
|
3
|
+
按输入文本的规则特征判断复杂度档位(Basic / Enhanced / Superior / Ultimate),
|
|
4
|
+
并解析出该档位应使用的 provider / model。
|
|
5
|
+
|
|
6
|
+
第 1 期子步骤 A:仅提供纯函数路由能力,尚未接入主循环与命令开关
|
|
7
|
+
(见 XG-docs/smart-docs/states/phase-01-smart-router-core.md)。
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import time
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
|
|
15
|
+
from .features import extract
|
|
16
|
+
from .model_tiers import TIER_NAMES, TierTarget, resolve
|
|
17
|
+
from .postprocess import postprocess
|
|
18
|
+
from .rule_router import RuleDecision, confidence, rule_route, rule_score
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class RouteResult:
|
|
23
|
+
"""一次完整路由的结果。"""
|
|
24
|
+
|
|
25
|
+
tier: str # Basic / Enhanced / Superior / Ultimate
|
|
26
|
+
tier_idx: int # 0..3
|
|
27
|
+
provider: str
|
|
28
|
+
model: str
|
|
29
|
+
configured: bool # 档位是否来自用户显式配置
|
|
30
|
+
confidence: float # 0~1,规则路由的置信度估计
|
|
31
|
+
score: float # 规则加权总分
|
|
32
|
+
hard_rule: bool # 是否由硬规则直接决定
|
|
33
|
+
features: dict = field(default_factory=dict) # 特征快照(测试/调试/后续反馈采集用)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _tier_index(prev_tier: str | int | None) -> int | None:
|
|
37
|
+
"""把档位名(或索引)统一转成索引;未知名称返回 None。"""
|
|
38
|
+
if prev_tier is None:
|
|
39
|
+
return None
|
|
40
|
+
if isinstance(prev_tier, int):
|
|
41
|
+
return prev_tier
|
|
42
|
+
try:
|
|
43
|
+
return TIER_NAMES.index(prev_tier)
|
|
44
|
+
except ValueError:
|
|
45
|
+
return None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def route(text: str, *,
|
|
49
|
+
prev_tier: str | int | None = None,
|
|
50
|
+
prev_ts: float | None = None,
|
|
51
|
+
ts: float | None = None,
|
|
52
|
+
context_tokens: int = 0,
|
|
53
|
+
fallback_provider: str = "",
|
|
54
|
+
fallback_model: str = "",
|
|
55
|
+
tiers_config: dict | None = None,
|
|
56
|
+
manager=None,
|
|
57
|
+
calibration=None,
|
|
58
|
+
learned_rules=None,
|
|
59
|
+
hysteresis=None,
|
|
60
|
+
ml_router=None) -> RouteResult:
|
|
61
|
+
"""对一段用户输入做完整路由:特征 → 规则打分 → 校准/ML精判 → 后处理 → 档位解析。
|
|
62
|
+
|
|
63
|
+
``manager``(ConfigManager)可选,传入时对显式配置档做 provider/API Key 校验。
|
|
64
|
+
``calibration``(adaptive.Calibration)可选,phase-03 步骤 C 只读注入:
|
|
65
|
+
在规则打分后、安全后处理前应用档位偏置与置信门;不传即第 1 期纯规则行为。
|
|
66
|
+
``learned_rules``(adaptive.LearnedRules)可选,phase-04 步骤 A1 只读注入:
|
|
67
|
+
postprocess 末尾对其命中的 ±1 档微调,硬规则强制时无效;不传即第 3 期行为。
|
|
68
|
+
``hysteresis``(router.postprocess.Hysteresis)可选,phase-04 步骤 A2 只读注入:
|
|
69
|
+
作为最后一道闸抑制会话内短时跳档,硬规则强制时解冻;不传即 A1 行为。
|
|
70
|
+
``ml_router``(router.ml_router.MLRouter)可选,phase-05 步骤 B2 只读注入:
|
|
71
|
+
规则打分后参与精判(概率最高档 + 置信门 + 校准偏置),不可用/信心不足时
|
|
72
|
+
静默回落规则档位;硬规则决策不参与精判;不传即 A1+A2 行为。
|
|
73
|
+
"""
|
|
74
|
+
f = extract(text)
|
|
75
|
+
decision: RuleDecision = rule_route(f)
|
|
76
|
+
tier_idx = decision.tier_idx
|
|
77
|
+
hard_rule = decision.hard_rule
|
|
78
|
+
from xg.adaptive.calibrate import apply_calibration
|
|
79
|
+
|
|
80
|
+
# 优先:ML 精判(仅软规则决策、且产物可用时)→ 置信门 + 校准偏置
|
|
81
|
+
if ml_router is not None and not hard_rule and ml_router.available:
|
|
82
|
+
ml_tier = ml_router.decide(text, f, calibration)
|
|
83
|
+
if ml_tier is not None:
|
|
84
|
+
tier_idx = ml_tier
|
|
85
|
+
else:
|
|
86
|
+
if calibration is not None:
|
|
87
|
+
tier_idx = apply_calibration(
|
|
88
|
+
tier_idx, confidence(decision), hard_rule, calibration,
|
|
89
|
+
)
|
|
90
|
+
final_idx = postprocess(
|
|
91
|
+
tier_idx, text, f,
|
|
92
|
+
prev_tier=_tier_index(prev_tier), prev_ts=prev_ts,
|
|
93
|
+
ts=ts if ts is not None else time.time(),
|
|
94
|
+
context_tokens=context_tokens,
|
|
95
|
+
learned_rules=learned_rules,
|
|
96
|
+
hysteresis=hysteresis,
|
|
97
|
+
)
|
|
98
|
+
target: TierTarget = resolve(final_idx, fallback_provider, fallback_model,
|
|
99
|
+
tiers_config, manager)
|
|
100
|
+
return RouteResult(
|
|
101
|
+
tier=target.tier,
|
|
102
|
+
tier_idx=final_idx,
|
|
103
|
+
provider=target.provider,
|
|
104
|
+
model=target.model,
|
|
105
|
+
configured=target.configured,
|
|
106
|
+
confidence=confidence(decision),
|
|
107
|
+
score=decision.score,
|
|
108
|
+
hard_rule=decision.hard_rule,
|
|
109
|
+
features=f,
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
__all__ = [
|
|
114
|
+
"RouteResult",
|
|
115
|
+
"TierTarget",
|
|
116
|
+
"TIER_NAMES",
|
|
117
|
+
"extract",
|
|
118
|
+
"route",
|
|
119
|
+
"resolve",
|
|
120
|
+
"rule_route",
|
|
121
|
+
"rule_score",
|
|
122
|
+
"postprocess",
|
|
123
|
+
"KEYWORDS",
|
|
124
|
+
]
|
xg/router/features.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""SmartRouter 规则特征管线。
|
|
2
|
+
|
|
3
|
+
纯 Python 字符串统计,无需训练、零第三方依赖。
|
|
4
|
+
特征定义来源:XG-docs/smart-docs/ADAPTIVE_ROUTING.md §4.1。
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import re
|
|
10
|
+
|
|
11
|
+
from .keywords import KEYWORDS
|
|
12
|
+
|
|
13
|
+
# 附件/文件类提及词(has_attachment 特征用,不属于 KEYWORDS 打分类别)
|
|
14
|
+
_ATTACHMENT_WORDS = ("附件", "文件", "目录", "上传", "file", "attachment")
|
|
15
|
+
|
|
16
|
+
# 围栏代码块:``` 开头(可带语言标注)到 ``` 结束
|
|
17
|
+
_FENCE_RE = re.compile(r"```[^\n]*\n(.*?)```", re.S)
|
|
18
|
+
# 围栏块的语言标注(用于 num_json / num_xml 统计)
|
|
19
|
+
_FENCE_LANG_RE = re.compile(r"```(json|xml)[^\n]*\n", re.I)
|
|
20
|
+
# 列表项:数字编号(1. / 1、 / 1))或无序列表(- / * / +)
|
|
21
|
+
_LIST_ITEM_RE = re.compile(r"^\s*(?:\d+[.、)]|[-*+])\s+", re.M)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def code_blocks(text: str) -> list[str]:
|
|
25
|
+
"""提取 ``` 围栏代码块;无围栏时按行首缩进 >=4 空格的行聚合成块。"""
|
|
26
|
+
blocks = _FENCE_RE.findall(text)
|
|
27
|
+
if blocks:
|
|
28
|
+
return blocks
|
|
29
|
+
lines = [ln for ln in text.splitlines() if ln.startswith((" ", "\t"))]
|
|
30
|
+
return ["\n".join(lines)] if lines else []
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def extract(text: str) -> dict:
|
|
34
|
+
"""从用户输入文本提取规则特征,返回扁平特征字典。"""
|
|
35
|
+
lower = text.lower()
|
|
36
|
+
blocks = code_blocks(text)
|
|
37
|
+
code_chars = sum(len(b) for b in blocks)
|
|
38
|
+
|
|
39
|
+
f: dict = {
|
|
40
|
+
"len_chars": len(text),
|
|
41
|
+
"len_words": len(text.split()),
|
|
42
|
+
"num_code_blocks": len(blocks),
|
|
43
|
+
"code_chars_ratio": (code_chars / len(text)) if text else 0.0,
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
# JSON / XML 片段数:统计带语言标注的围栏块(```json / ```xml)
|
|
47
|
+
fence_langs = [m.lower() for m in _FENCE_LANG_RE.findall(text)]
|
|
48
|
+
f["num_json"] = fence_langs.count("json")
|
|
49
|
+
f["num_xml"] = fence_langs.count("xml")
|
|
50
|
+
|
|
51
|
+
# 列表项数
|
|
52
|
+
f["num_lists"] = len(_LIST_ITEM_RE.findall(text))
|
|
53
|
+
|
|
54
|
+
# 是否提及文件/附件
|
|
55
|
+
f["has_attachment"] = 1 if any(w in lower for w in _ATTACHMENT_WORDS) else 0
|
|
56
|
+
|
|
57
|
+
# 各类关键词命中数(英文转小写后子串匹配,按出现次数累计)
|
|
58
|
+
for cat, words in KEYWORDS.items():
|
|
59
|
+
f[f"num_{cat}_kw"] = sum(lower.count(w.lower()) for w in words)
|
|
60
|
+
|
|
61
|
+
# 闲聊标记:命中任一闲聊词
|
|
62
|
+
f["is_chatty"] = 1 if f["num_chatty_kw"] > 0 else 0
|
|
63
|
+
# 疑问句标记:含中/英文问号
|
|
64
|
+
f["question_mark"] = 1 if ("?" in text or "?" in text) else 0
|
|
65
|
+
|
|
66
|
+
return f
|
xg/router/keywords.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""SmartRouter 关键词表(中英双语)。
|
|
2
|
+
|
|
3
|
+
词表来源:XG-docs/smart-docs/ADAPTIVE_ROUTING.md §4.1。
|
|
4
|
+
命中判定用子串匹配(``kw in text``),中文词无需分词即可工作;
|
|
5
|
+
英文词统一转小写后匹配。可按业务增删,但不改变类别键名
|
|
6
|
+
(features/rule_router/postprocess 依赖类别名)。
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
KEYWORDS: dict[str, list[str]] = {
|
|
12
|
+
"arch": [ # 架构词(命中→高风险/高复杂度)
|
|
13
|
+
"架构", "重构", "扩展", "依赖", "分布式", "微服务", "模块化", "集成",
|
|
14
|
+
"迁移", "设计模式", "系统设计", "扩展性", "性能优化",
|
|
15
|
+
"architecture", "refactor", "microservice", "distributed",
|
|
16
|
+
"scalable", "modular", "integration",
|
|
17
|
+
],
|
|
18
|
+
"risk": [ # 风险词(命中→强制升档)
|
|
19
|
+
"部署", "回滚", "迁移", "生产", "客户", "法务", "上线", "宕机",
|
|
20
|
+
"故障", "安全", "合规", "数据丢失", "备份", "恢复", "资损",
|
|
21
|
+
"deploy", "rollback", "production", "compliance", "security",
|
|
22
|
+
"outage", "failure", "data-loss", "incident",
|
|
23
|
+
],
|
|
24
|
+
"planning": [ # 规划词(方案/设计类)
|
|
25
|
+
"设计", "方案", "对比", "取舍", "路线图", "规划", "roadmap",
|
|
26
|
+
"权衡", "tradeoff", "plan", "design", "compare", "evaluate",
|
|
27
|
+
],
|
|
28
|
+
"impl": [ # 实现词(具体动手任务)
|
|
29
|
+
"写", "实现", "改", "生成", "修复", "添加", "创建", "优化",
|
|
30
|
+
"帮我", "实现一个", "封装", "接入",
|
|
31
|
+
"write", "implement", "create", "fix", "build", "refactor",
|
|
32
|
+
],
|
|
33
|
+
"teach": [ # 教学词(解释/问答类)
|
|
34
|
+
"解释", "为什么", "原理", "区别", "对比", "是什么", "如何",
|
|
35
|
+
"什么是", "讲解", "说明",
|
|
36
|
+
"explain", "why", "difference", "how", "what is", "describe",
|
|
37
|
+
],
|
|
38
|
+
"constraint": [ # 约束词(有硬性要求→偏复杂)
|
|
39
|
+
"必须", "不能", "不允许", "性能", "安全", "精确", "严格",
|
|
40
|
+
"restriction", "constraint", "must", "cannot", "strict",
|
|
41
|
+
],
|
|
42
|
+
"chatty": [ # 闲聊词(命中→降档)
|
|
43
|
+
"你好", "谢谢", "在吗", "ok", "好的", "嗨", "嗯", "没问题", "再见",
|
|
44
|
+
"hello", "hi", "thanks", "bye", "okay", "sure", "fine",
|
|
45
|
+
],
|
|
46
|
+
"debug": [ # 调试词(命中→升一档)
|
|
47
|
+
"报错", "失败", "还是不行", "异常", "崩溃", "超时",
|
|
48
|
+
"error", "exception", "debug", "crash", "timeout", "bug",
|
|
49
|
+
],
|
|
50
|
+
}
|
xg/router/ml_router.py
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"""SmartRouter ML 精判接入与静默回落(第 5 期 B2)。
|
|
2
|
+
|
|
3
|
+
职责:加载离线训练的产物(tools/train_router.py 生成的 router.lgb),
|
|
4
|
+
在规则路由之后、安全后处理之前参与精判——取"概率最高档",受 置信门 +
|
|
5
|
+
校准偏置 约束;任何缺依赖 / 缺产物 / 产物损坏 情形一律**静默回落**,
|
|
6
|
+
行为退化为纯规则(与本模块不存在的第 4 期行为完全一致)。
|
|
7
|
+
|
|
8
|
+
导入保护(主进程零硬性 ML 依赖):
|
|
9
|
+
joblib / lightgbm / sklearn / numpy 仅在 onnx 运行期 import,
|
|
10
|
+
任一把 import 失败即在初始化时把本模块标记为不可用(available=False),
|
|
11
|
+
主进程绝不因缺依赖而报错。
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
from ..adaptive.calibrate import CONFIDENCE_BASE, apply_calibration
|
|
20
|
+
|
|
21
|
+
# 产物格式版本(与 tools/train_router.py 的 ARTIFACT_FORMAT 一致)
|
|
22
|
+
ARTIFACT_FORMAT = 1
|
|
23
|
+
_DEFAULT_ARTIFACT = "router.lgb" # 默认文件名,相对 data_dir
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class MLPrediction:
|
|
28
|
+
"""一次可用的 ML 精判结果。"""
|
|
29
|
+
|
|
30
|
+
tier: int # 概率最高档 0..3
|
|
31
|
+
prob: float # 该档预测概率 0..1
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class MLRouter:
|
|
35
|
+
"""产物加载与精判;不可用时全链路静默回落。"""
|
|
36
|
+
|
|
37
|
+
def __init__(self, artifact_path: Path | None = None,
|
|
38
|
+
semantic=None) -> None:
|
|
39
|
+
self._payload = None
|
|
40
|
+
self._vectorizer = None
|
|
41
|
+
self._model = None
|
|
42
|
+
self._feature_keys: tuple[str, ...] = ()
|
|
43
|
+
# 第 6 期 C2:可选语义编码器(bge 512 维)。训练产物带 sem_dim>0 时,
|
|
44
|
+
# 预测需语义列;语义不可用则整个静默回落(无法提供既定列宽)。
|
|
45
|
+
self._semantic = semantic
|
|
46
|
+
self._sem_dim = 0
|
|
47
|
+
# 随包兜底:默认产物缺失时首启自动落位(clone 后开箱可用)
|
|
48
|
+
if artifact_path is None:
|
|
49
|
+
self._ensure_bundled()
|
|
50
|
+
self._load(artifact_path)
|
|
51
|
+
|
|
52
|
+
def _ensure_bundled(self) -> None:
|
|
53
|
+
"""把随包默认产物(router.lgb 等)复制到数据目录;已存在则跳过。"""
|
|
54
|
+
from ..adaptive.store import ensure_default_artifacts
|
|
55
|
+
ensure_default_artifacts()
|
|
56
|
+
|
|
57
|
+
# -- 加载 -----------------------------------------------------------
|
|
58
|
+
def _load(self, artifact_path: Path | None) -> None:
|
|
59
|
+
"""加载产物;任何异常(缺依赖/缺文件/损坏/格式不符)→ 不可用。"""
|
|
60
|
+
path = artifact_path or self._default_path()
|
|
61
|
+
if path is None or not path.exists():
|
|
62
|
+
return
|
|
63
|
+
try:
|
|
64
|
+
import joblib # noqa: PLC0415
|
|
65
|
+
from sklearn.utils.validation import check_is_fitted # noqa: PLC0415
|
|
66
|
+
payload = joblib.load(path)
|
|
67
|
+
if not isinstance(payload, dict) or payload.get("format") != ARTIFACT_FORMAT:
|
|
68
|
+
return
|
|
69
|
+
model = payload.get("model")
|
|
70
|
+
vec = payload.get("vectorizer")
|
|
71
|
+
keys = payload.get("feature_keys") or ()
|
|
72
|
+
if not hasattr(model, "predict_proba"):
|
|
73
|
+
return
|
|
74
|
+
check_is_fitted(model)
|
|
75
|
+
sem_dim = payload.get("sem_dim", 0) or 0
|
|
76
|
+
# 产物声明需要语义列,但语义编码器不可用 → 整链静默回落
|
|
77
|
+
if sem_dim > 0 and (self._semantic is None or not self._semantic.available):
|
|
78
|
+
return
|
|
79
|
+
self._vectorizer = vec
|
|
80
|
+
self._model = model
|
|
81
|
+
self._feature_keys = tuple(keys)
|
|
82
|
+
self._sem_dim = int(sem_dim)
|
|
83
|
+
self._payload = payload
|
|
84
|
+
except Exception:
|
|
85
|
+
self._payload = None
|
|
86
|
+
self._vectorizer = None
|
|
87
|
+
self._model = None
|
|
88
|
+
self._feature_keys = ()
|
|
89
|
+
self._sem_dim = 0
|
|
90
|
+
|
|
91
|
+
def _default_path(self) -> Path | None:
|
|
92
|
+
from ..adaptive.store import data_dir
|
|
93
|
+
return data_dir() / _DEFAULT_ARTIFACT
|
|
94
|
+
|
|
95
|
+
@property
|
|
96
|
+
def available(self) -> bool:
|
|
97
|
+
"""产物已成功加载且模型可用。"""
|
|
98
|
+
return self._model is not None
|
|
99
|
+
|
|
100
|
+
@property
|
|
101
|
+
def n_samples(self) -> int | None:
|
|
102
|
+
return self._payload.get("n_samples") if self._payload else None
|
|
103
|
+
|
|
104
|
+
@property
|
|
105
|
+
def sem_dim(self) -> int:
|
|
106
|
+
"""训练时并入的语义特征维度(0=无语义,纯第 5 期行为)。"""
|
|
107
|
+
return self._sem_dim
|
|
108
|
+
|
|
109
|
+
@property
|
|
110
|
+
def semantic(self) -> object | None:
|
|
111
|
+
"""绑定的语义编码器(可能为 None);供 status 观测。"""
|
|
112
|
+
return self._semantic
|
|
113
|
+
|
|
114
|
+
# -- 预测 -----------------------------------------------------------
|
|
115
|
+
def _encode(self, text: str, features: dict | None):
|
|
116
|
+
"""把 text + 数值特征 拼成 (TF-IDF, 数值) 稀疏输入;无需义时行为同第 5 期。
|
|
117
|
+
|
|
118
|
+
语义列(sem_dim>0,第 6 期)追加在数值列之后,顺序为
|
|
119
|
+
[TF-IDF] + [数值] + [语义512],与 train_router 训练矩阵严格一致。
|
|
120
|
+
"""
|
|
121
|
+
import numpy as np # noqa: PLC0415
|
|
122
|
+
from scipy.sparse import csr_matrix, hstack # noqa: PLC0415
|
|
123
|
+
if self._vectorizer is not None:
|
|
124
|
+
x_text = self._vectorizer.transform([text or " "])
|
|
125
|
+
else:
|
|
126
|
+
x_text = csr_matrix((1, 0))
|
|
127
|
+
row = [float((features or {}).get(k, 0.0)) for k in self._feature_keys]
|
|
128
|
+
cols = [x_text, csr_matrix(np.array([row]))]
|
|
129
|
+
if self._sem_dim > 0:
|
|
130
|
+
import numpy as _np # noqa: PLC0415
|
|
131
|
+
sem = self._semantic.encode(text)
|
|
132
|
+
if sem is None:
|
|
133
|
+
# 语义列不可得(理论上 available 已兜底,防御万一)
|
|
134
|
+
sem = [0.0] * self._sem_dim
|
|
135
|
+
cols.append(csr_matrix(_np.array([sem[:self._sem_dim]])))
|
|
136
|
+
return hstack(cols).tocsr()
|
|
137
|
+
|
|
138
|
+
def predict(self, text: str, features: dict | None = None) -> MLPrediction | None:
|
|
139
|
+
"""训练产物预测:返回概率最高档;不可用返回 None(静默回落)。"""
|
|
140
|
+
if not self.available:
|
|
141
|
+
return None
|
|
142
|
+
x = self._encode(text, features)
|
|
143
|
+
probs = self._model.predict_proba(x)[0]
|
|
144
|
+
tier = int(probs.argmax())
|
|
145
|
+
return MLPrediction(tier=tier, prob=float(probs[tier]))
|
|
146
|
+
|
|
147
|
+
def decide(
|
|
148
|
+
self,
|
|
149
|
+
text: str,
|
|
150
|
+
features: dict | None = None,
|
|
151
|
+
calibration=None,
|
|
152
|
+
) -> int | None:
|
|
153
|
+
"""精判入口:置信门 + 校准偏置后返回档位;不采用返回 None。
|
|
154
|
+
|
|
155
|
+
与规则路径复用一个置信门/偏置逻辑(apply_calibration):
|
|
156
|
+
- 预测最高档概率 < 门(CONFIDENCE_BASE + threshold_adjust)→ 信心不足,
|
|
157
|
+
返回 None,由 route() 保留规则档位;
|
|
158
|
+
- 否则对最高档应用 apply_calibration(偏强档降/偏弱档升,最多一档)。
|
|
159
|
+
硬规则档位由 route() 在调用方排除(decision.hard_rule 时不走精判)。
|
|
160
|
+
"""
|
|
161
|
+
if not self.available:
|
|
162
|
+
return None
|
|
163
|
+
pred = self.predict(text, features)
|
|
164
|
+
if pred is None:
|
|
165
|
+
return None
|
|
166
|
+
gate = CONFIDENCE_BASE + (
|
|
167
|
+
calibration.threshold_adjust if calibration is not None else 0.0
|
|
168
|
+
)
|
|
169
|
+
if pred.prob < gate:
|
|
170
|
+
return None
|
|
171
|
+
if calibration is not None:
|
|
172
|
+
return apply_calibration(pred.tier, pred.prob, False, calibration)
|
|
173
|
+
return pred.tier
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def load_ml_router(artifact_path: Path | None = None) -> MLRouter:
|
|
177
|
+
"""便捷工厂:缺依赖/缺产物时返回一个 available=False 的 MLRouter。"""
|
|
178
|
+
return MLRouter(artifact_path)
|
xg/router/model_tiers.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""SmartRouter 档位→模型映射。
|
|
2
|
+
|
|
3
|
+
接入 ConfigManager(phase-01 子步骤 B):
|
|
4
|
+
- ``tiers_config`` 形如 ``{"Basic": {"provider": "glm", "model": "glm-4-flash"}, ...}``,
|
|
5
|
+
通常来自 ``ConfigManager.smart_router_config()["tiers"]``;
|
|
6
|
+
- 传入 ``manager`` 时启用完整校验链:provider 必须可解析、API Key 必须已配置,
|
|
7
|
+
否则该档回落 fallback(active 模型)并标记 ``configured=False``;
|
|
8
|
+
- 未配置的档位同样回落 fallback。
|
|
9
|
+
|
|
10
|
+
校验失败的档位在 UI 上应显示为不可用(阶段 2 的暗淡 + (x) 标记)。
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from typing import TYPE_CHECKING
|
|
17
|
+
|
|
18
|
+
from .postprocess import TIER
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
from xg.config.manager import ConfigManager
|
|
22
|
+
|
|
23
|
+
TIER_NAMES = TIER
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class TierTarget:
|
|
28
|
+
"""某档位解析出的模型目标。"""
|
|
29
|
+
|
|
30
|
+
tier: str # Basic / Enhanced / Superior / Ultimate
|
|
31
|
+
provider: str
|
|
32
|
+
model: str
|
|
33
|
+
configured: bool # 显式配置且校验通过(False 表示回落默认/校验失败)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def resolve(tier_idx: int, fallback_provider: str, fallback_model: str,
|
|
37
|
+
tiers_config: dict | None = None,
|
|
38
|
+
manager: "ConfigManager | None" = None) -> TierTarget:
|
|
39
|
+
"""把档位索引解析成 (provider, model)。
|
|
40
|
+
|
|
41
|
+
结算规则:
|
|
42
|
+
1. 档位指定了 ``provider``:model 缺省时用该 provider 的默认模型(D5),
|
|
43
|
+
需 ``manager`` 解析;传 None 时无法取默认模型,回落 fallback_model。
|
|
44
|
+
2. 档位未指定 ``provider``:整体回落 fallback(base provider 及其生效模型),
|
|
45
|
+
model 取档位显式 ``model`` 或 fallback_model。
|
|
46
|
+
3. 显式配置经 ``manager`` 校验:provider 必须可解析、API Key 必须已配置,
|
|
47
|
+
否则整档回落 fallback 并标记 ``configured=False``。
|
|
48
|
+
"""
|
|
49
|
+
tier_name = TIER_NAMES[tier_idx]
|
|
50
|
+
cfg = (tiers_config or {}).get(tier_name) or {}
|
|
51
|
+
provided_provider = str(cfg.get("provider") or "")
|
|
52
|
+
|
|
53
|
+
if provided_provider:
|
|
54
|
+
provider = provided_provider
|
|
55
|
+
default_model = fallback_model
|
|
56
|
+
if manager is not None:
|
|
57
|
+
prov = manager.resolve_provider(provider)
|
|
58
|
+
if prov is not None and prov.default_model:
|
|
59
|
+
default_model = prov.default_model
|
|
60
|
+
model = str(cfg.get("model") or "") or default_model
|
|
61
|
+
configured = bool(cfg.get("provider") or cfg.get("model"))
|
|
62
|
+
else:
|
|
63
|
+
provider = fallback_provider
|
|
64
|
+
model = str(cfg.get("model") or "") or fallback_model
|
|
65
|
+
configured = bool(cfg.get("model"))
|
|
66
|
+
|
|
67
|
+
if configured and manager is not None and (provider, model) != (fallback_provider, fallback_model):
|
|
68
|
+
prov = manager.resolve_provider(provider)
|
|
69
|
+
if prov is None or not manager.resolve_api_key(prov):
|
|
70
|
+
# provider 不存在或 API Key 未配置:整档回落 active
|
|
71
|
+
return TierTarget(tier=tier_name, provider=fallback_provider,
|
|
72
|
+
model=fallback_model, configured=False)
|
|
73
|
+
return TierTarget(tier=tier_name, provider=provider, model=model, configured=configured)
|