common-core 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- common_core-0.1.0/.env.example +77 -0
- common_core-0.1.0/.github/workflows/ci.yml +28 -0
- common_core-0.1.0/.github/workflows/publish.yml +33 -0
- common_core-0.1.0/.gitignore +29 -0
- common_core-0.1.0/PKG-INFO +19 -0
- common_core-0.1.0/docs/config.md +110 -0
- common_core-0.1.0/pyproject.toml +24 -0
- common_core-0.1.0/scripts/check_env_examples.py +155 -0
- common_core-0.1.0/src/common_core/__init__.py +54 -0
- common_core-0.1.0/src/common_core/auth.py +306 -0
- common_core-0.1.0/src/common_core/config.py +842 -0
- common_core-0.1.0/src/common_core/context.py +45 -0
- common_core-0.1.0/src/common_core/instrumentation.py +70 -0
- common_core-0.1.0/src/common_core/mcp_auth.py +319 -0
- common_core-0.1.0/src/common_core/observability.py +383 -0
- common_core-0.1.0/src/common_core/protocols.py +162 -0
- common_core-0.1.0/src/common_core/providers/__init__.py +21 -0
- common_core-0.1.0/src/common_core/providers/cache.py +266 -0
- common_core-0.1.0/src/common_core/providers/llm.py +272 -0
- common_core-0.1.0/src/common_core/providers/vector.py +440 -0
- common_core-0.1.0/src/common_core/rag/__init__.py +60 -0
- common_core-0.1.0/src/common_core/rag/assembly.py +496 -0
- common_core-0.1.0/src/common_core/rag/generation.py +177 -0
- common_core-0.1.0/src/common_core/rag/guard.py +302 -0
- common_core-0.1.0/src/common_core/security.py +124 -0
- common_core-0.1.0/src/common_core/telemetry.py +214 -0
- common_core-0.1.0/tests/conftest.py +1 -0
- common_core-0.1.0/tests/test_auth.py +137 -0
- common_core-0.1.0/tests/test_config.py +445 -0
- common_core-0.1.0/tests/test_context.py +23 -0
- common_core-0.1.0/tests/test_mcp_auth.py +108 -0
- common_core-0.1.0/tests/test_observability.py +70 -0
- common_core-0.1.0/tests/test_providers.py +151 -0
- common_core-0.1.0/tests/test_security.py +52 -0
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# common_core canonical config example.
|
|
2
|
+
# This is the shared contract' single source for key naming. Real secrets
|
|
3
|
+
# live in each skill's own .env (or process env) and must not be committed.
|
|
4
|
+
# Copy keys you need into a skill-specific .env; never load this file directly.
|
|
5
|
+
|
|
6
|
+
# --- LLM (OpenAI-compatible) --- type=str required
|
|
7
|
+
LLM_BASE_URL=https://api.deepseek.com/v1
|
|
8
|
+
LLM_API_KEY=replace-me
|
|
9
|
+
LLM_MODEL=deepseek-chat
|
|
10
|
+
LLM_TEMPERATURE=0
|
|
11
|
+
LLM_MAX_TOKENS=2048
|
|
12
|
+
LLM_TIMEOUT_SECONDS=30
|
|
13
|
+
|
|
14
|
+
# --- Embedding (global key, not LLM_ prefixed) --- type=str required-for-dense
|
|
15
|
+
EMBEDDING_MODEL=BAAI/bge-small-zh-v1.5
|
|
16
|
+
|
|
17
|
+
# --- Milvus --- type=str/int required-host+collection
|
|
18
|
+
MILVUS_HOST=127.0.0.1
|
|
19
|
+
MILVUS_PORT=19530
|
|
20
|
+
MILVUS_USER=
|
|
21
|
+
MILVUS_PASSWORD=
|
|
22
|
+
MILVUS_SECURE=false
|
|
23
|
+
MILVUS_DB=
|
|
24
|
+
MILVUS_TEXT_COLLECTION=knowledge_kb
|
|
25
|
+
MILVUS_IMAGE_COLLECTION=
|
|
26
|
+
MILVUS_DIM=0
|
|
27
|
+
MILVUS_OUTPUT_FIELDS=id,content,source
|
|
28
|
+
|
|
29
|
+
# --- Redis --- type=str/int
|
|
30
|
+
REDIS_HOST=127.0.0.1
|
|
31
|
+
REDIS_PORT=6379
|
|
32
|
+
REDIS_PASSWORD=
|
|
33
|
+
REDIS_DB=0
|
|
34
|
+
REDIS_DEFAULT_TTL=1800
|
|
35
|
+
REDIS_KEY_PREFIX=rag
|
|
36
|
+
REDIS_SOCKET_TIMEOUT=3
|
|
37
|
+
|
|
38
|
+
# --- Auth --- type=str mode=jwt|disabled
|
|
39
|
+
AUTH_MODE=jwt
|
|
40
|
+
AUTH_JWT_SECRET=
|
|
41
|
+
AUTH_JWT_PUBLIC_KEY=
|
|
42
|
+
# 与 AUTH_JWT_SECRET / AUTH_JWT_PUBLIC_KEY 三选一;
|
|
43
|
+
# 若同时配置,静态密钥优先于 JWKS,轮换时应移除静态键。
|
|
44
|
+
AUTH_JWT_ALGORITHMS=HS256
|
|
45
|
+
AUTH_JWT_ISSUER=
|
|
46
|
+
AUTH_JWT_AUDIENCE=
|
|
47
|
+
# 企业 IdP 的 JWKS 端点;配置后按 LIFESPAN 缓存公钥,
|
|
48
|
+
# 遇到未知 kid 自动刷新,签名密钥轮换无需重启服务。
|
|
49
|
+
AUTH_JWKS_URL=
|
|
50
|
+
AUTH_JWKS_LIFESPAN=300
|
|
51
|
+
AUTH_JWKS_TIMEOUT=30
|
|
52
|
+
AUTH_MCP_ISSUER_URL=
|
|
53
|
+
AUTH_MCP_RESOURCE_SERVER_URL=
|
|
54
|
+
|
|
55
|
+
# --- Metrics --- type=bool/int/str
|
|
56
|
+
METRICS_ENABLED=false
|
|
57
|
+
METRICS_PREFIX=app
|
|
58
|
+
METRICS_PORT=9090
|
|
59
|
+
METRICS_BIND=127.0.0.1
|
|
60
|
+
|
|
61
|
+
# --- Retrieval --- type=int/float
|
|
62
|
+
RETRIEVAL_TOP_K=20
|
|
63
|
+
RETRIEVAL_MIN_RELEVANCE=0.70
|
|
64
|
+
RETRIEVAL_HYBRID_WORKERS=16
|
|
65
|
+
RRF_TOP_K=20
|
|
66
|
+
RRF_K=60
|
|
67
|
+
RERANK_TOP_K=3
|
|
68
|
+
|
|
69
|
+
# --- Query rewrite --- type=str mode=off|identity|llm_rewrite|query_expansion
|
|
70
|
+
RETRIEVAL_QUERY_REWRITE_MODE=off
|
|
71
|
+
RETRIEVAL_QUERY_REWRITE_LLM_MODEL=
|
|
72
|
+
RETRIEVAL_QUERY_REWRITE_TEMPERATURE=0
|
|
73
|
+
RETRIEVAL_QUERY_REWRITE_MAX_TOKENS=256
|
|
74
|
+
RETRIEVAL_QUERY_REWRITE_EXPAND_COUNT=2
|
|
75
|
+
RETRIEVAL_QUERY_REWRITE_PROMPT=
|
|
76
|
+
RETRIEVAL_QUERY_REWRITE_EXPANSION_PROMPT=
|
|
77
|
+
RETRIEVAL_QUERY_REWRITE_SCOPES=
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
strategy:
|
|
13
|
+
matrix:
|
|
14
|
+
python-version: ["3.11", "3.12"]
|
|
15
|
+
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
|
|
19
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
20
|
+
uses: actions/setup-python@v5
|
|
21
|
+
with:
|
|
22
|
+
python-version: ${{ matrix.python-version }}
|
|
23
|
+
|
|
24
|
+
- name: Install package with test dependencies
|
|
25
|
+
run: pip install -e ".[test]"
|
|
26
|
+
|
|
27
|
+
- name: Run tests
|
|
28
|
+
run: pytest tests/ -v --tb=short
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*"]
|
|
6
|
+
workflow_dispatch:
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
publish:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
steps:
|
|
15
|
+
- name: Checkout
|
|
16
|
+
uses: actions/checkout@v4
|
|
17
|
+
|
|
18
|
+
- name: Set up Python
|
|
19
|
+
uses: actions/setup-python@v5
|
|
20
|
+
with:
|
|
21
|
+
python-version: "3.12"
|
|
22
|
+
|
|
23
|
+
- name: Install build tooling
|
|
24
|
+
run: python -m pip install --upgrade build twine
|
|
25
|
+
|
|
26
|
+
- name: Build package
|
|
27
|
+
run: python -m build
|
|
28
|
+
|
|
29
|
+
- name: Upload to PyPI
|
|
30
|
+
env:
|
|
31
|
+
TWINE_USERNAME: __token__
|
|
32
|
+
TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
|
|
33
|
+
run: python -m twine upload dist/*
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# IDE / editor local config
|
|
2
|
+
.idea/
|
|
3
|
+
.vscode/
|
|
4
|
+
*.iml
|
|
5
|
+
|
|
6
|
+
# Python
|
|
7
|
+
__pycache__/
|
|
8
|
+
*.py[cod]
|
|
9
|
+
*.egg-info/
|
|
10
|
+
build/
|
|
11
|
+
dist/
|
|
12
|
+
.mypy_cache/
|
|
13
|
+
.ruff_cache/
|
|
14
|
+
.pytest_cache/
|
|
15
|
+
.coverage
|
|
16
|
+
htmlcov/
|
|
17
|
+
|
|
18
|
+
# Virtual environments
|
|
19
|
+
.venv/
|
|
20
|
+
venv/
|
|
21
|
+
env/
|
|
22
|
+
|
|
23
|
+
# Environment files / secrets
|
|
24
|
+
.env
|
|
25
|
+
.env.*
|
|
26
|
+
!.env.example
|
|
27
|
+
|
|
28
|
+
# Misc
|
|
29
|
+
.DS_Store
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: common-core
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Shared runtime for AI skills: context, config, auth, observability, provider contracts.
|
|
5
|
+
Requires-Python: >=3.11
|
|
6
|
+
Requires-Dist: prometheus-client>=0.20
|
|
7
|
+
Requires-Dist: pyjwt[crypto]>=2.8
|
|
8
|
+
Requires-Dist: python-dotenv>=1.0
|
|
9
|
+
Provides-Extra: cache
|
|
10
|
+
Requires-Dist: redis>=5.2; extra == 'cache'
|
|
11
|
+
Provides-Extra: llm
|
|
12
|
+
Requires-Dist: openai>=1.60; extra == 'llm'
|
|
13
|
+
Requires-Dist: sentence-transformers>=3.3; extra == 'llm'
|
|
14
|
+
Provides-Extra: mcp
|
|
15
|
+
Requires-Dist: mcp>=1.2; extra == 'mcp'
|
|
16
|
+
Provides-Extra: test
|
|
17
|
+
Requires-Dist: pytest>=8.0; extra == 'test'
|
|
18
|
+
Provides-Extra: vector
|
|
19
|
+
Requires-Dist: pymilvus>=2.5; extra == 'vector'
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# common_core 配置契约
|
|
2
|
+
|
|
3
|
+
这是所有 AI skill 配置的**唯一契约入口**。契约约束键名、类型、必填/可选与敏感标记;真实值只存在于各 skill 的 `.env` 或进程环境,绝不进入本库或仓库。
|
|
4
|
+
|
|
5
|
+
## 加载顺序
|
|
6
|
+
|
|
7
|
+
配置来源优先级从高到低固定为:
|
|
8
|
+
|
|
9
|
+
1. 系统环境变量(`os.environ`)
|
|
10
|
+
2. 当前 skill 部署目录的 `.env`
|
|
11
|
+
3. 代码中的默认值
|
|
12
|
+
|
|
13
|
+
只由可执行入口(各 skill 的 `main()`)显式加载 `.env`(`override=False`),`common_core` 不主动读取自身安装目录下的 `.env`。每个 MCP 进程只加载一份 env 源;同名键(如 `LLM_MODEL`)在不同 skill 进程里允许不同值,互不冲突。
|
|
14
|
+
|
|
15
|
+
## 完整键表
|
|
16
|
+
|
|
17
|
+
标记说明:`必填` 列里的 `*` 表示所有 skill 都要;`rag`/`sql` 表示仅该 skill 要求。`敏感` 列标记为 `是` 的键必须脱敏,且 `.env.example` 只允许出现占位符。
|
|
18
|
+
|
|
19
|
+
| 键 | 类型 | 默认值 | 必填 | 敏感 |
|
|
20
|
+
| --- | --- | --- | --- | --- |
|
|
21
|
+
| `LLM_BASE_URL` | str | `""` | `*` | 否 |
|
|
22
|
+
| `LLM_API_KEY` | str | `""` | `*` | 是 |
|
|
23
|
+
| `LLM_MODEL` | str | `""` | `*` | 否 |
|
|
24
|
+
| `LLM_TEMPERATURE` | float | `0.0` | 可选 | 否 |
|
|
25
|
+
| `LLM_MAX_TOKENS` | int | `2048` | 可选 | 否 |
|
|
26
|
+
| `LLM_TIMEOUT_SECONDS` | float | `30.0` | 可选 | 否 |
|
|
27
|
+
| `EMBEDDING_MODEL` | str | `""` | `rag` | 否 |
|
|
28
|
+
| `MILVUS_HOST` | str | `localhost` | `rag` | 否 |
|
|
29
|
+
| `MILVUS_PORT` | int | `19530` | 可选 | 否 |
|
|
30
|
+
| `MILVUS_USER` | str | `""` | 可选 | 否 |
|
|
31
|
+
| `MILVUS_PASSWORD` | str | `""` | 可选 | 是 |
|
|
32
|
+
| `MILVUS_SECURE` | bool | `false` | 可选 | 否 |
|
|
33
|
+
| `MILVUS_DB` | str | `""` | 可选 | 否 |
|
|
34
|
+
| `MILVUS_TEXT_COLLECTION` | str | `""` | `rag` | 否 |
|
|
35
|
+
| `MILVUS_IMAGE_COLLECTION` | str | `""` | 可选 | 否 |
|
|
36
|
+
| `MILVUS_DIM` | int | `0` | 可选 | 否 |
|
|
37
|
+
| `MILVUS_OUTPUT_FIELDS` | csv | `[]` | 可选 | 否 |
|
|
38
|
+
| `REDIS_HOST` | str | `localhost` | `rag` | 否 |
|
|
39
|
+
| `REDIS_PORT` | int | `6379` | 可选 | 否 |
|
|
40
|
+
| `REDIS_PASSWORD` | str | `""` | 可选 | 是 |
|
|
41
|
+
| `REDIS_DB` | int | `0` | 可选 | 否 |
|
|
42
|
+
| `REDIS_DEFAULT_TTL` | int | `1800` | 可选 | 否 |
|
|
43
|
+
| `REDIS_KEY_PREFIX` | str | `rag` | 可选 | 否 |
|
|
44
|
+
| `REDIS_SOCKET_TIMEOUT` | float | `3.0` | 可选 | 否 |
|
|
45
|
+
| `AUTH_MODE` | enum | `jwt` | `*` | 否 |
|
|
46
|
+
| `AUTH_JWT_SECRET` | str | `""` | `*`(jwt 三选一) | 是 |
|
|
47
|
+
| `AUTH_JWT_PUBLIC_KEY` | str | `""` | `*`(jwt 三选一) | 是 |
|
|
48
|
+
| `AUTH_JWT_ALGORITHMS` | csv | `HS256`(配置 JWKS 时默认 `RS256/ES256`) | 可选 | 否 |
|
|
49
|
+
| `AUTH_JWT_ISSUER` | str | `""` | 可选 | 否 |
|
|
50
|
+
| `AUTH_JWT_AUDIENCE` | str | `""` | 可选 | 否 |
|
|
51
|
+
| `AUTH_JWKS_URL` | str | `""` | `*`(jwt 三选一) | 否 |
|
|
52
|
+
| `AUTH_JWKS_LIFESPAN` | float | `300` | 可选 | 否 |
|
|
53
|
+
| `AUTH_JWKS_TIMEOUT` | float | `30` | 可选 | 否 |
|
|
54
|
+
| `AUTH_MCP_ISSUER_URL` | str | `""` | 可选 | 否 |
|
|
55
|
+
| `AUTH_MCP_RESOURCE_SERVER_URL` | str | `""` | 可选 | 否 |
|
|
56
|
+
| `METRICS_ENABLED` | bool | `false` | 可选 | 否 |
|
|
57
|
+
| `METRICS_PREFIX` | str | `app` | 可选 | 否 |
|
|
58
|
+
| `METRICS_PORT` | int | `9090` | 可选 | 否 |
|
|
59
|
+
| `METRICS_BIND` | str | `127.0.0.1` | 可选 | 否 |
|
|
60
|
+
| `RETRIEVAL_TOP_K` | int | `20` | 可选 | 否 |
|
|
61
|
+
| `RETRIEVAL_MIN_RELEVANCE` | float | `0.70` | 可选 | 否 |
|
|
62
|
+
| `RETRIEVAL_HYBRID_WORKERS` | int | `16` | 可选 | 否 |
|
|
63
|
+
| `RETRIEVAL_ASSEMBLY_MAX_CHARS` | int | `8000` | 可选 | 否 |
|
|
64
|
+
| `RRF_TOP_K` | int | `20` | 可选 | 否 |
|
|
65
|
+
| `RRF_K` | int | `60` | 可选 | 否 |
|
|
66
|
+
| `RERANK_TOP_K` | int | `3` | 可选 | 否 |
|
|
67
|
+
| `RETRIEVAL_QUERY_REWRITE_MODE` | enum | `off` | 可选 | 否 |
|
|
68
|
+
| `RETRIEVAL_QUERY_REWRITE_LLM_MODEL` | str | `""` | 可选 | 否 |
|
|
69
|
+
| `RETRIEVAL_QUERY_REWRITE_TEMPERATURE` | float | `0.0` | 可选 | 否 |
|
|
70
|
+
| `RETRIEVAL_QUERY_REWRITE_MAX_TOKENS` | int | `256` | 可选 | 否 |
|
|
71
|
+
| `RETRIEVAL_QUERY_REWRITE_EXPAND_COUNT` | int | `2` | 可选 | 否 |
|
|
72
|
+
| `RETRIEVAL_QUERY_REWRITE_PROMPT` | str | `""` | 可选 | 否 |
|
|
73
|
+
| `RETRIEVAL_QUERY_REWRITE_EXPANSION_PROMPT` | str | `""` | 可选 | 否 |
|
|
74
|
+
| `RETRIEVAL_QUERY_REWRITE_SCOPES` | str | `""` | 可选 | 否 |
|
|
75
|
+
| `RERANKER_MODEL` | str | `BAAI/bge-reranker-base` | `rag` | 否 |
|
|
76
|
+
| `RERANKER_DEVICE` | str | `""` | 可选 | 否 |
|
|
77
|
+
| `RERANKER_CE_WEIGHT` | float | `0.6` | 可选 | 否 |
|
|
78
|
+
| `RERANKER_RETRIEVAL_WEIGHT` | float | `0.4` | 可选 | 否 |
|
|
79
|
+
| `SQL_HOST` | str | `localhost` | `sql` | 否 |
|
|
80
|
+
| `SQL_PORT` | int | `3306` | 可选 | 否 |
|
|
81
|
+
| `SQL_USER` | str | `""` | `sql` | 否 |
|
|
82
|
+
| `SQL_PASSWORD` | str | `""` | `sql` | 是 |
|
|
83
|
+
| `SQL_DATABASE` | str | `""` | `sql` | 否 |
|
|
84
|
+
| `SQL_CHARSET` | str | `utf8mb4` | 可选 | 否 |
|
|
85
|
+
| `SQL_CONNECT_TIMEOUT` | float | `3.0` | 可选 | 否 |
|
|
86
|
+
| `SQL_READ_TIMEOUT` | float | `5.0` | 可选 | 否 |
|
|
87
|
+
| `SQL_POOL_MAX_CONNECTIONS` | int | `10` | 可选 | 否 |
|
|
88
|
+
| `SQL_POOL_MIN_CACHED` | int | `2` | 可选 | 否 |
|
|
89
|
+
| `SQL_POOL_MAX_CACHED` | int | `5` | 可选 | 否 |
|
|
90
|
+
| `SQL_POOL_BLOCKING` | bool | `true` | 可选 | 否 |
|
|
91
|
+
| `SQL_DEFAULT_MAX_ROWS` | int | `20` | 可选 | 否 |
|
|
92
|
+
|
|
93
|
+
## 校验规则
|
|
94
|
+
|
|
95
|
+
- `RuntimeConfig.validate()` 启动时 fail-fast,缺任一必填键抛 `ConfigError`,附带完整缺失键清单。
|
|
96
|
+
- `AUTH_MODE` 只允许 `jwt` / `disabled`;`jwt` 模式必须提供
|
|
97
|
+
`AUTH_JWT_SECRET`、`AUTH_JWT_PUBLIC_KEY`、`AUTH_JWKS_URL` 三者之一。
|
|
98
|
+
- 配置 `AUTH_JWKS_URL` 后,`AUTH_JWKS_LIFESPAN` 必须大于 0;
|
|
99
|
+
不显式声明 `AUTH_JWT_ALGORITHMS` 时默认启用 `RS256` / `ES256`。
|
|
100
|
+
- 静态密钥优先于 JWKS:同时配置 `AUTH_JWT_SECRET` / `AUTH_JWT_PUBLIC_KEY`
|
|
101
|
+
与 `AUTH_JWKS_URL` 时仍走静态密钥路径,做轮换迁移时请移除静态键。
|
|
102
|
+
- 其余读取失败(整数、布尔解析失败)回退默认值,不中断启动。
|
|
103
|
+
|
|
104
|
+
## 一致性
|
|
105
|
+
|
|
106
|
+
CI 用 `common_core/scripts/check_env_examples.py` 对比每个 skill 的 `.env.example` 与上表:缺必填键、出现未声明键、或敏感键写成真实值即失败。修改键名/必填/敏感属性时,必须同步 `config.py`、`config.md` 与该脚本。
|
|
107
|
+
|
|
108
|
+
## 环境文件解析
|
|
109
|
+
|
|
110
|
+
`resolve_env_file()` 按“显式 `--env-file` 参数 > `*_ENV_FILE` 环境变量 > 当前目录 `.env`”解析,交给 `load_env_files()`(`override=False`)。启动日志会打印配置来源与脱敏配置指纹(见 `config_fingerprint()` / `log_config_audit()`)。
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "common-core"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Shared runtime for AI skills: context, config, auth, observability, provider contracts."
|
|
5
|
+
requires-python = ">=3.11"
|
|
6
|
+
dependencies = [
|
|
7
|
+
"prometheus-client>=0.20",
|
|
8
|
+
"PyJWT[crypto]>=2.8",
|
|
9
|
+
"python-dotenv>=1.0",
|
|
10
|
+
]
|
|
11
|
+
|
|
12
|
+
[build-system]
|
|
13
|
+
requires = ["hatchling"]
|
|
14
|
+
build-backend = "hatchling.build"
|
|
15
|
+
|
|
16
|
+
[tool.hatch.build.targets.wheel]
|
|
17
|
+
packages = ["src/common_core"]
|
|
18
|
+
|
|
19
|
+
[project.optional-dependencies]
|
|
20
|
+
llm = ["openai>=1.60", "sentence-transformers>=3.3"]
|
|
21
|
+
vector = ["pymilvus>=2.5"]
|
|
22
|
+
cache = ["redis>=5.2"]
|
|
23
|
+
mcp = ["mcp>=1.2"]
|
|
24
|
+
test = ["pytest>=8.0"]
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""CI guard: keep every skill ``.env.example`` aligned with the config contract.
|
|
2
|
+
|
|
3
|
+
The contract is defined here (Key name, required for which skill, sensitive,
|
|
4
|
+
expected type). It is the single source the examples and ``config.md`` must
|
|
5
|
+
agree with. Running this in CI fails the build when an example:
|
|
6
|
+
|
|
7
|
+
- omits a required shared key,
|
|
8
|
+
- declares an unknown key (typo / drift from the contract), or
|
|
9
|
+
- ships a real-looking secret instead of a placeholder.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import argparse
|
|
15
|
+
import sys
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
# Canonical contract. Keys used by skills beyond the common shared set are
|
|
19
|
+
# listed as optional so they are still spell-checked.
|
|
20
|
+
CONTRACT: dict[str, dict[str, str]] = {
|
|
21
|
+
# (required-for, sensitive, type)
|
|
22
|
+
"LLM_BASE_URL": {"required": {"common_core", "retrieve_skill"}, "sensitive": False, "type": "str"},
|
|
23
|
+
"LLM_API_KEY": {"required": {"common_core", "retrieve_skill"}, "sensitive": True, "type": "str"},
|
|
24
|
+
"LLM_MODEL": {"required": {"common_core", "retrieve_skill"}, "sensitive": False, "type": "str"},
|
|
25
|
+
"LLM_TEMPERATURE": {"required": set(), "sensitive": False, "type": "float"},
|
|
26
|
+
"LLM_MAX_TOKENS": {"required": set(), "sensitive": False, "type": "int"},
|
|
27
|
+
"LLM_TIMEOUT_SECONDS": {"required": set(), "sensitive": False, "type": "float"},
|
|
28
|
+
"EMBEDDING_MODEL": {"required": {"retrieve_skill"}, "sensitive": False, "type": "str"},
|
|
29
|
+
"MILVUS_HOST": {"required": {"retrieve_skill"}, "sensitive": False, "type": "str"},
|
|
30
|
+
"MILVUS_PORT": {"required": set(), "sensitive": False, "type": "int"},
|
|
31
|
+
"MILVUS_USER": {"required": set(), "sensitive": False, "type": "str"},
|
|
32
|
+
"MILVUS_PASSWORD": {"required": set(), "sensitive": True, "type": "str"},
|
|
33
|
+
"MILVUS_SECURE": {"required": set(), "sensitive": False, "type": "bool"},
|
|
34
|
+
"MILVUS_DB": {"required": set(), "sensitive": False, "type": "str"},
|
|
35
|
+
"MILVUS_TEXT_COLLECTION": {"required": {"retrieve_skill"}, "sensitive": False, "type": "str"},
|
|
36
|
+
"MILVUS_IMAGE_COLLECTION": {"required": set(), "sensitive": False, "type": "str"},
|
|
37
|
+
"MILVUS_DIM": {"required": set(), "sensitive": False, "type": "int"},
|
|
38
|
+
"MILVUS_OUTPUT_FIELDS": {"required": set(), "sensitive": False, "type": "csv"},
|
|
39
|
+
"REDIS_HOST": {"required": {"retrieve_skill"}, "sensitive": False, "type": "str"},
|
|
40
|
+
"REDIS_PORT": {"required": set(), "sensitive": False, "type": "int"},
|
|
41
|
+
"REDIS_PASSWORD": {"required": set(), "sensitive": True, "type": "str"},
|
|
42
|
+
"REDIS_DB": {"required": set(), "sensitive": False, "type": "int"},
|
|
43
|
+
"REDIS_DEFAULT_TTL": {"required": set(), "sensitive": False, "type": "int"},
|
|
44
|
+
"REDIS_KEY_PREFIX": {"required": set(), "sensitive": False, "type": "str"},
|
|
45
|
+
"REDIS_SOCKET_TIMEOUT": {"required": set(), "sensitive": False, "type": "float"},
|
|
46
|
+
"AUTH_MODE": {"required": {"common_core", "retrieve_skill"}, "sensitive": False, "type": "enum"},
|
|
47
|
+
"AUTH_JWT_SECRET": {"required": set(), "sensitive": True, "type": "str"},
|
|
48
|
+
"AUTH_JWT_PUBLIC_KEY": {"required": set(), "sensitive": True, "type": "str"},
|
|
49
|
+
"AUTH_JWT_ALGORITHMS": {"required": set(), "sensitive": False, "type": "csv"},
|
|
50
|
+
"AUTH_JWT_ISSUER": {"required": set(), "sensitive": False, "type": "str"},
|
|
51
|
+
"AUTH_JWT_AUDIENCE": {"required": set(), "sensitive": False, "type": "str"},
|
|
52
|
+
"AUTH_JWKS_URL": {"required": set(), "sensitive": False, "type": "str"},
|
|
53
|
+
"AUTH_JWKS_LIFESPAN": {"required": set(), "sensitive": False, "type": "float"},
|
|
54
|
+
"AUTH_JWKS_TIMEOUT": {"required": set(), "sensitive": False, "type": "float"},
|
|
55
|
+
"AUTH_MCP_ISSUER_URL": {"required": set(), "sensitive": False, "type": "str"},
|
|
56
|
+
"AUTH_MCP_RESOURCE_SERVER_URL": {"required": set(), "sensitive": False, "type": "str"},
|
|
57
|
+
"METRICS_ENABLED": {"required": set(), "sensitive": False, "type": "bool"},
|
|
58
|
+
"METRICS_PREFIX": {"required": set(), "sensitive": False, "type": "str"},
|
|
59
|
+
"METRICS_PORT": {"required": set(), "sensitive": False, "type": "int"},
|
|
60
|
+
"METRICS_BIND": {"required": set(), "sensitive": False, "type": "str"},
|
|
61
|
+
"RETRIEVAL_TOP_K": {"required": set(), "sensitive": False, "type": "int"},
|
|
62
|
+
"RETRIEVAL_MIN_RELEVANCE": {"required": set(), "sensitive": False, "type": "float"},
|
|
63
|
+
"RETRIEVAL_HYBRID_WORKERS": {"required": set(), "sensitive": False, "type": "int"},
|
|
64
|
+
"RETRIEVAL_ASSEMBLY_MAX_CHARS": {"required": set(), "sensitive": False, "type": "int"},
|
|
65
|
+
"RRF_TOP_K": {"required": set(), "sensitive": False, "type": "int"},
|
|
66
|
+
"RRF_K": {"required": set(), "sensitive": False, "type": "int"},
|
|
67
|
+
"RERANK_TOP_K": {"required": set(), "sensitive": False, "type": "int"},
|
|
68
|
+
"RETRIEVAL_QUERY_REWRITE_MODE": {"required": set(), "sensitive": False, "type": "enum"},
|
|
69
|
+
"RETRIEVAL_QUERY_REWRITE_LLM_MODEL": {"required": set(), "sensitive": False, "type": "str"},
|
|
70
|
+
"RETRIEVAL_QUERY_REWRITE_TEMPERATURE": {"required": set(), "sensitive": False, "type": "float"},
|
|
71
|
+
"RETRIEVAL_QUERY_REWRITE_MAX_TOKENS": {"required": set(), "sensitive": False, "type": "int"},
|
|
72
|
+
"RETRIEVAL_QUERY_REWRITE_EXPAND_COUNT": {"required": set(), "sensitive": False, "type": "int"},
|
|
73
|
+
"RETRIEVAL_QUERY_REWRITE_PROMPT": {"required": set(), "sensitive": False, "type": "str"},
|
|
74
|
+
"RETRIEVAL_QUERY_REWRITE_EXPANSION_PROMPT": {"required": set(), "sensitive": False, "type": "str"},
|
|
75
|
+
"RETRIEVAL_QUERY_REWRITE_SCOPES": {"required": set(), "sensitive": False, "type": "str"},
|
|
76
|
+
# skill-specific keys
|
|
77
|
+
"RERANKER_MODEL": {"required": {"retrieve_skill"}, "sensitive": False, "type": "str"},
|
|
78
|
+
"RERANKER_DEVICE": {"required": set(), "sensitive": False, "type": "str"},
|
|
79
|
+
"RERANKER_CE_WEIGHT": {"required": set(), "sensitive": False, "type": "float"},
|
|
80
|
+
"RERANKER_RETRIEVAL_WEIGHT": {"required": set(), "sensitive": False, "type": "float"},
|
|
81
|
+
"SQL_HOST": {"required": {"structured_query_skill"}, "sensitive": False, "type": "str"},
|
|
82
|
+
"SQL_PORT": {"required": set(), "sensitive": False, "type": "int"},
|
|
83
|
+
"SQL_USER": {"required": {"structured_query_skill"}, "sensitive": False, "type": "str"},
|
|
84
|
+
"SQL_PASSWORD": {"required": {"structured_query_skill"}, "sensitive": True, "type": "str"},
|
|
85
|
+
"SQL_DATABASE": {"required": {"structured_query_skill"}, "sensitive": False, "type": "str"},
|
|
86
|
+
"SQL_CHARSET": {"required": set(), "sensitive": False, "type": "str"},
|
|
87
|
+
"SQL_CONNECT_TIMEOUT": {"required": set(), "sensitive": False, "type": "float"},
|
|
88
|
+
"SQL_READ_TIMEOUT": {"required": set(), "sensitive": False, "type": "float"},
|
|
89
|
+
"SQL_POOL_MAX_CONNECTIONS": {"required": set(), "sensitive": False, "type": "int"},
|
|
90
|
+
"SQL_POOL_MIN_CACHED": {"required": set(), "sensitive": False, "type": "int"},
|
|
91
|
+
"SQL_POOL_MAX_CACHED": {"required": set(), "sensitive": False, "type": "int"},
|
|
92
|
+
"SQL_POOL_BLOCKING": {"required": set(), "sensitive": False, "type": "bool"},
|
|
93
|
+
"SQL_DEFAULT_MAX_ROWS": {"required": set(), "sensitive": False, "type": "int"},
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
PLACEHOLDER_MARKERS = ("replace-me", "changeme", "your-", "<", ">")
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def parse_env(path: Path) -> set[str]:
|
|
100
|
+
keys: set[str] = set()
|
|
101
|
+
for raw in path.read_text(encoding="utf-8").splitlines():
|
|
102
|
+
line = raw.strip()
|
|
103
|
+
if not line or line.startswith("#"):
|
|
104
|
+
continue
|
|
105
|
+
if "=" not in line:
|
|
106
|
+
continue
|
|
107
|
+
key = line.split("=", 1)[0].strip()
|
|
108
|
+
if key:
|
|
109
|
+
keys.add(key)
|
|
110
|
+
return keys
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def check_example(path: Path, skill: str, errors: list[str]) -> None:
|
|
114
|
+
keys = parse_env(path)
|
|
115
|
+
for required, meta in CONTRACT.items():
|
|
116
|
+
if skill in meta["required"] and required not in keys:
|
|
117
|
+
errors.append(f"{path.name}[{skill}]: missing required key {required}")
|
|
118
|
+
for key in sorted(keys):
|
|
119
|
+
if key not in CONTRACT:
|
|
120
|
+
errors.append(f"{path.name}: unknown key {key} not in contract")
|
|
121
|
+
continue
|
|
122
|
+
if CONTRACT[key]["sensitive"]:
|
|
123
|
+
value = ""
|
|
124
|
+
for raw in path.read_text(encoding="utf-8").splitlines():
|
|
125
|
+
if raw.startswith(key + "="):
|
|
126
|
+
value = raw.split("=", 1)[1].strip()
|
|
127
|
+
break
|
|
128
|
+
if value and not any(m in value.lower() for m in PLACEHOLDER_MARKERS):
|
|
129
|
+
errors.append(f"{path.name}: sensitive key {key} holds a real-looking value in an example")
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def main() -> int:
|
|
133
|
+
parser = argparse.ArgumentParser()
|
|
134
|
+
parser.add_argument("--root", type=Path, default=Path.cwd())
|
|
135
|
+
args = parser.parse_args()
|
|
136
|
+
errors: list[str] = []
|
|
137
|
+
examples = {
|
|
138
|
+
"common_core": args.root / "common_core" / ".env.example",
|
|
139
|
+
"retrieve_skill": args.root / "retrieve_skill" / ".env.example",
|
|
140
|
+
"structured_query_skill": args.root / "structured_query_skill" / ".env.example",
|
|
141
|
+
}
|
|
142
|
+
for skill, path in examples.items():
|
|
143
|
+
if not path.is_file():
|
|
144
|
+
errors.append(f"missing example for {skill}: {path}")
|
|
145
|
+
continue
|
|
146
|
+
check_example(path, skill, errors)
|
|
147
|
+
if errors:
|
|
148
|
+
print("\n".join(f"[fail] {e}" for e in errors))
|
|
149
|
+
return 1
|
|
150
|
+
print("env examples aligned with config contract")
|
|
151
|
+
return 0
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
if __name__ == "__main__":
|
|
155
|
+
sys.exit(main())
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""共用运行时(common_core):供多个 skill 共享的底层组件集合。
|
|
2
|
+
本包不依赖具体业务,提供配置、鉴权、上下文、可观测性、安全审查与数据提供者的基础组件,是各 skill 的地基。
|
|
3
|
+
导出名用于方便业务模块从包顶层直接引入常用类与函数。
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from .auth import AuthError, IdentityClaims, TokenVerifier
|
|
7
|
+
from .config import (
|
|
8
|
+
AuthConfig,
|
|
9
|
+
CacheConfig,
|
|
10
|
+
LLMConfig,
|
|
11
|
+
MetricsConfig,
|
|
12
|
+
RetrievalConfig,
|
|
13
|
+
RuntimeConfig,
|
|
14
|
+
VectorStoreConfig,
|
|
15
|
+
)
|
|
16
|
+
from .context import AgentContext
|
|
17
|
+
from .mcp_auth import (
|
|
18
|
+
MCPBearerTokenVerifier,
|
|
19
|
+
ToolAuthError,
|
|
20
|
+
ToolContextGuard,
|
|
21
|
+
build_mcp_auth,
|
|
22
|
+
resolve_tool_context,
|
|
23
|
+
)
|
|
24
|
+
from .observability import Observability
|
|
25
|
+
from .protocols import QueryRequest, QueryResult
|
|
26
|
+
from .security import INJECTION_PATTERNS, check_safety, mask_pii, normalize_query
|
|
27
|
+
|
|
28
|
+
__version__ = "0.1.0"
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
"AuthConfig",
|
|
32
|
+
"AuthError",
|
|
33
|
+
"AgentContext",
|
|
34
|
+
"CacheConfig",
|
|
35
|
+
"INJECTION_PATTERNS",
|
|
36
|
+
"IdentityClaims",
|
|
37
|
+
"LLMConfig",
|
|
38
|
+
"MCPBearerTokenVerifier",
|
|
39
|
+
"MetricsConfig",
|
|
40
|
+
"Observability",
|
|
41
|
+
"QueryRequest",
|
|
42
|
+
"QueryResult",
|
|
43
|
+
"RetrievalConfig",
|
|
44
|
+
"RuntimeConfig",
|
|
45
|
+
"ToolAuthError",
|
|
46
|
+
"ToolContextGuard",
|
|
47
|
+
"TokenVerifier",
|
|
48
|
+
"VectorStoreConfig",
|
|
49
|
+
"build_mcp_auth",
|
|
50
|
+
"check_safety",
|
|
51
|
+
"mask_pii",
|
|
52
|
+
"normalize_query",
|
|
53
|
+
"resolve_tool_context",
|
|
54
|
+
]
|