ue-knowledge-base 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ue_knowledge_base-0.1.0/LICENSE +21 -0
- ue_knowledge_base-0.1.0/PKG-INFO +105 -0
- ue_knowledge_base-0.1.0/README.md +91 -0
- ue_knowledge_base-0.1.0/pyproject.toml +26 -0
- ue_knowledge_base-0.1.0/setup.cfg +4 -0
- ue_knowledge_base-0.1.0/src/ue_knowledge/__init__.py +8 -0
- ue_knowledge_base-0.1.0/src/ue_knowledge/build.py +90 -0
- ue_knowledge_base-0.1.0/src/ue_knowledge/chunking.py +62 -0
- ue_knowledge_base-0.1.0/src/ue_knowledge/cli.py +131 -0
- ue_knowledge_base-0.1.0/src/ue_knowledge/config.py +33 -0
- ue_knowledge_base-0.1.0/src/ue_knowledge/query.py +60 -0
- ue_knowledge_base-0.1.0/src/ue_knowledge_base.egg-info/PKG-INFO +105 -0
- ue_knowledge_base-0.1.0/src/ue_knowledge_base.egg-info/SOURCES.txt +17 -0
- ue_knowledge_base-0.1.0/src/ue_knowledge_base.egg-info/dependency_links.txt +1 -0
- ue_knowledge_base-0.1.0/src/ue_knowledge_base.egg-info/entry_points.txt +2 -0
- ue_knowledge_base-0.1.0/src/ue_knowledge_base.egg-info/requires.txt +2 -0
- ue_knowledge_base-0.1.0/src/ue_knowledge_base.egg-info/top_level.txt +1 -0
- ue_knowledge_base-0.1.0/tests/test_chunking.py +31 -0
- ue_knowledge_base-0.1.0/tests/test_cli.py +19 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Dong
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ue-knowledge-base
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Offline semantic search over Unreal Engine development knowledge (BGE + ChromaDB)
|
|
5
|
+
Author: Dong
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: unreal-engine,rag,semantic-search,chromadb,knowledge-base
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Dist: sentence-transformers>=2.6
|
|
12
|
+
Requires-Dist: chromadb>=0.5
|
|
13
|
+
Dynamic: license-file
|
|
14
|
+
|
|
15
|
+
# UE Knowledge Base
|
|
16
|
+
|
|
17
|
+
Offline semantic search over Unreal Engine development knowledge — a curated
|
|
18
|
+
**Chinese-language knowledge corpus** (29 topics: GAS, animation, AI,
|
|
19
|
+
networking, UMG, Niagara, PCG, ...) plus a local RAG pipeline that indexes it
|
|
20
|
+
with `BAAI/bge-small-zh-v1.5` embeddings into ChromaDB.
|
|
21
|
+
|
|
22
|
+
**Zero API cost. Fully offline after a one-time model download.** Built for
|
|
23
|
+
Chinese-speaking UE developers, and ready to plug into any AI agent, IDE, or
|
|
24
|
+
CLI workflow.
|
|
25
|
+
|
|
26
|
+
```
|
|
27
|
+
knowledge/ (29 topics, ~84 markdown docs)
|
|
28
|
+
│ ue-kb build chunk + embed (BGE small, local)
|
|
29
|
+
▼
|
|
30
|
+
.chroma_db/ (vector index)
|
|
31
|
+
│ ue-kb query "GAS cooldown"
|
|
32
|
+
▼
|
|
33
|
+
top-k semantic hits with source + heading
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Features
|
|
37
|
+
|
|
38
|
+
- 📚 **Curated corpus**: hand-written UE development docs in Chinese, covering
|
|
39
|
+
Gameplay Ability System, character movement, animation, physics/collision,
|
|
40
|
+
AI navigation, networking/replication, UMG/Slate, Niagara, Mass Entity,
|
|
41
|
+
State Trees, PCG/procedural generation, materials/rendering, module build
|
|
42
|
+
system, editor tools, and more.
|
|
43
|
+
- 🧠 **Local semantic search**: `BAAI/bge-small-zh-v1.5` (multilingual, ~100MB)
|
|
44
|
+
+ ChromaDB — no cloud APIs, no cost, works on a laptop CPU.
|
|
45
|
+
- 🖥️ **Simple CLI** (`ue-kb`): `build`, `query`, `info`, `download-model`,
|
|
46
|
+
plus `--json` output for agent integration.
|
|
47
|
+
- 🔌 **Extensible**: index any extra UE docs with `--source`, or add engine
|
|
48
|
+
source-indexing scripts under `scripts/`.
|
|
49
|
+
|
|
50
|
+
## Quick start
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
# 1. Install
|
|
54
|
+
pip install -e . # or: pip install ue-knowledge-base
|
|
55
|
+
|
|
56
|
+
# 2. Download the embedding model once (~100MB)
|
|
57
|
+
ue-kb download-model
|
|
58
|
+
# In China: export HF_ENDPOINT=https://hf-mirror.com then retry
|
|
59
|
+
|
|
60
|
+
# 3. Build the index
|
|
61
|
+
ue-kb build
|
|
62
|
+
|
|
63
|
+
# 4. Search
|
|
64
|
+
ue-kb query "GAS ability cooldown" --top-k 5
|
|
65
|
+
ue-kb query "角色移动 速度衰减" --json # machine-readable for agents
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
### Example
|
|
69
|
+
|
|
70
|
+
```text
|
|
71
|
+
$ ue-kb query "GAS 技能冷却"
|
|
72
|
+
🔍 UE 知识库检索:GAS 技能冷却
|
|
73
|
+
|
|
74
|
+
[1] ue-gameplay-abilities/references/gas-input-integration.md › 问题 (匹配度: 21.1%)
|
|
75
|
+
UE 项目同时使用 GAS (GameplayAbilitySystem) 和 Enhanced Input 时,容易陷入
|
|
76
|
+
两个极端:- **全 GAS** → 所有输入走 GAS,但 WASD 轴输入不适合 GAS 的事件
|
|
77
|
+
模型,且 CommitAbility 的 GC 延迟影响跳跃手感 ...
|
|
78
|
+
[2] ue-gameplay-abilities/references/gas-input-integration.md › Jump — GAS 即时技能 (匹配度: 14.2%)
|
|
79
|
+
...
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## Python API
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
from ue_knowledge.query import query
|
|
86
|
+
|
|
87
|
+
for hit in query("GAS cooldown", top_k=5):
|
|
88
|
+
print(hit["source"], hit["heading"], hit["score"])
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Extending the corpus
|
|
92
|
+
|
|
93
|
+
1. Add a markdown file under `knowledge/<topic>/` (use `##`/`###` headings —
|
|
94
|
+
the indexer chunks on heading boundaries).
|
|
95
|
+
2. `ue-kb build --force` to rebuild.
|
|
96
|
+
|
|
97
|
+
For indexing **Unreal Engine C++ header comments** or **Epic official docs**,
|
|
98
|
+
see `scripts/index_engine_source.py` and `scripts/crawl_epic_docs.py`
|
|
99
|
+
(they expect local engine/UE paths — the extracted index data is generated
|
|
100
|
+
locally and is not redistributed, out of respect for Epic's copyright).
|
|
101
|
+
|
|
102
|
+
## License
|
|
103
|
+
|
|
104
|
+
MIT. The knowledge documents are original writing; no engine source code or
|
|
105
|
+
verbatim Epic documentation is included.
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# UE Knowledge Base
|
|
2
|
+
|
|
3
|
+
Offline semantic search over Unreal Engine development knowledge — a curated
|
|
4
|
+
**Chinese-language knowledge corpus** (29 topics: GAS, animation, AI,
|
|
5
|
+
networking, UMG, Niagara, PCG, ...) plus a local RAG pipeline that indexes it
|
|
6
|
+
with `BAAI/bge-small-zh-v1.5` embeddings into ChromaDB.
|
|
7
|
+
|
|
8
|
+
**Zero API cost. Fully offline after a one-time model download.** Built for
|
|
9
|
+
Chinese-speaking UE developers, and ready to plug into any AI agent, IDE, or
|
|
10
|
+
CLI workflow.
|
|
11
|
+
|
|
12
|
+
```
|
|
13
|
+
knowledge/ (29 topics, ~84 markdown docs)
|
|
14
|
+
│ ue-kb build chunk + embed (BGE small, local)
|
|
15
|
+
▼
|
|
16
|
+
.chroma_db/ (vector index)
|
|
17
|
+
│ ue-kb query "GAS cooldown"
|
|
18
|
+
▼
|
|
19
|
+
top-k semantic hits with source + heading
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
## Features
|
|
23
|
+
|
|
24
|
+
- 📚 **Curated corpus**: hand-written UE development docs in Chinese, covering
|
|
25
|
+
Gameplay Ability System, character movement, animation, physics/collision,
|
|
26
|
+
AI navigation, networking/replication, UMG/Slate, Niagara, Mass Entity,
|
|
27
|
+
State Trees, PCG/procedural generation, materials/rendering, module build
|
|
28
|
+
system, editor tools, and more.
|
|
29
|
+
- 🧠 **Local semantic search**: `BAAI/bge-small-zh-v1.5` (multilingual, ~100MB)
|
|
30
|
+
+ ChromaDB — no cloud APIs, no cost, works on a laptop CPU.
|
|
31
|
+
- 🖥️ **Simple CLI** (`ue-kb`): `build`, `query`, `info`, `download-model`,
|
|
32
|
+
plus `--json` output for agent integration.
|
|
33
|
+
- 🔌 **Extensible**: index any extra UE docs with `--source`, or add engine
|
|
34
|
+
source-indexing scripts under `scripts/`.
|
|
35
|
+
|
|
36
|
+
## Quick start
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
# 1. Install
|
|
40
|
+
pip install -e . # or: pip install ue-knowledge-base
|
|
41
|
+
|
|
42
|
+
# 2. Download the embedding model once (~100MB)
|
|
43
|
+
ue-kb download-model
|
|
44
|
+
# In China: export HF_ENDPOINT=https://hf-mirror.com then retry
|
|
45
|
+
|
|
46
|
+
# 3. Build the index
|
|
47
|
+
ue-kb build
|
|
48
|
+
|
|
49
|
+
# 4. Search
|
|
50
|
+
ue-kb query "GAS ability cooldown" --top-k 5
|
|
51
|
+
ue-kb query "角色移动 速度衰减" --json # machine-readable for agents
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
### Example
|
|
55
|
+
|
|
56
|
+
```text
|
|
57
|
+
$ ue-kb query "GAS 技能冷却"
|
|
58
|
+
🔍 UE 知识库检索:GAS 技能冷却
|
|
59
|
+
|
|
60
|
+
[1] ue-gameplay-abilities/references/gas-input-integration.md › 问题 (匹配度: 21.1%)
|
|
61
|
+
UE 项目同时使用 GAS (GameplayAbilitySystem) 和 Enhanced Input 时,容易陷入
|
|
62
|
+
两个极端:- **全 GAS** → 所有输入走 GAS,但 WASD 轴输入不适合 GAS 的事件
|
|
63
|
+
模型,且 CommitAbility 的 GC 延迟影响跳跃手感 ...
|
|
64
|
+
[2] ue-gameplay-abilities/references/gas-input-integration.md › Jump — GAS 即时技能 (匹配度: 14.2%)
|
|
65
|
+
...
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## Python API
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
from ue_knowledge.query import query
|
|
72
|
+
|
|
73
|
+
for hit in query("GAS cooldown", top_k=5):
|
|
74
|
+
print(hit["source"], hit["heading"], hit["score"])
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
## Extending the corpus
|
|
78
|
+
|
|
79
|
+
1. Add a markdown file under `knowledge/<topic>/` (use `##`/`###` headings —
|
|
80
|
+
the indexer chunks on heading boundaries).
|
|
81
|
+
2. `ue-kb build --force` to rebuild.
|
|
82
|
+
|
|
83
|
+
For indexing **Unreal Engine C++ header comments** or **Epic official docs**,
|
|
84
|
+
see `scripts/index_engine_source.py` and `scripts/crawl_epic_docs.py`
|
|
85
|
+
(they expect local engine/UE paths — the extracted index data is generated
|
|
86
|
+
locally and is not redistributed, out of respect for Epic's copyright).
|
|
87
|
+
|
|
88
|
+
## License
|
|
89
|
+
|
|
90
|
+
MIT. The knowledge documents are original writing; no engine source code or
|
|
91
|
+
verbatim Epic documentation is included.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "ue-knowledge-base"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Offline semantic search over Unreal Engine development knowledge (BGE + ChromaDB)"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Dong" }]
|
|
13
|
+
keywords = ["unreal-engine", "rag", "semantic-search", "chromadb", "knowledge-base"]
|
|
14
|
+
dependencies = [
|
|
15
|
+
"sentence-transformers>=2.6",
|
|
16
|
+
"chromadb>=0.5",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
[project.scripts]
|
|
20
|
+
ue-kb = "ue_knowledge.cli:main"
|
|
21
|
+
|
|
22
|
+
[tool.setuptools.packages.find]
|
|
23
|
+
where = ["src"]
|
|
24
|
+
|
|
25
|
+
[tool.pytest.ini_options]
|
|
26
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""ue_knowledge — offline semantic search over Unreal Engine development docs.
|
|
2
|
+
|
|
3
|
+
Build a local vector index from the bundled knowledge/ markdown corpus
|
|
4
|
+
(BAAI/bge-small-zh-v1.5 embeddings + ChromaDB), then query it with a CLI
|
|
5
|
+
or from Python. Zero API cost, fully offline after the one-time model download.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Index building — embed the corpus and store it in ChromaDB."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from . import config
|
|
6
|
+
from .chunking import collect_markdown
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def build_index(
|
|
10
|
+
source_dir: Path | None = None,
|
|
11
|
+
chroma_dir: Path | None = None,
|
|
12
|
+
model_name: str | None = None,
|
|
13
|
+
force: bool = False,
|
|
14
|
+
offline: bool = True,
|
|
15
|
+
) -> dict:
|
|
16
|
+
"""Build (or rebuild) the vector index from the markdown corpus.
|
|
17
|
+
|
|
18
|
+
Returns a summary dict: {files, chunks, collection, chroma_dir}.
|
|
19
|
+
"""
|
|
20
|
+
from sentence_transformers import SentenceTransformer
|
|
21
|
+
import chromadb
|
|
22
|
+
|
|
23
|
+
source = source_dir or config.source_dir()
|
|
24
|
+
chroma = chroma_dir or config.chroma_dir()
|
|
25
|
+
model_name = model_name or config.MODEL_NAME
|
|
26
|
+
|
|
27
|
+
if not source.is_dir():
|
|
28
|
+
raise FileNotFoundError(f"corpus directory not found: {source}")
|
|
29
|
+
|
|
30
|
+
print(f"[*] Loading model: {model_name}")
|
|
31
|
+
model = SentenceTransformer(
|
|
32
|
+
model_name,
|
|
33
|
+
local_files_only=offline,
|
|
34
|
+
)
|
|
35
|
+
model.max_seq_length = 512
|
|
36
|
+
print(f" Embedding dim: {model.get_sentence_embedding_dimension()}")
|
|
37
|
+
|
|
38
|
+
print(f"[*] Reading corpus: {source}")
|
|
39
|
+
documents = collect_markdown(source)
|
|
40
|
+
if not documents:
|
|
41
|
+
raise RuntimeError("no markdown documents found in corpus")
|
|
42
|
+
print(f" {len(documents)} chunks")
|
|
43
|
+
|
|
44
|
+
chroma.mkdir(parents=True, exist_ok=True)
|
|
45
|
+
client = chromadb.PersistentClient(path=str(chroma))
|
|
46
|
+
|
|
47
|
+
if force:
|
|
48
|
+
try:
|
|
49
|
+
client.delete_collection(config.COLLECTION_NAME)
|
|
50
|
+
except Exception:
|
|
51
|
+
pass
|
|
52
|
+
|
|
53
|
+
collection = client.get_or_create_collection(
|
|
54
|
+
name=config.COLLECTION_NAME,
|
|
55
|
+
metadata={
|
|
56
|
+
"description": "UE Game Development Knowledge Base",
|
|
57
|
+
"hnsw:space": "cosine",
|
|
58
|
+
},
|
|
59
|
+
)
|
|
60
|
+
if collection.count() > 0 and not force:
|
|
61
|
+
raise RuntimeError(
|
|
62
|
+
f"collection already has {collection.count()} docs; use --force to rebuild"
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
texts = [d["text"] for d in documents]
|
|
66
|
+
ids = [d["id"] for d in documents]
|
|
67
|
+
metadatas = [{"source": d["source"], "heading": d["heading"]} for d in documents]
|
|
68
|
+
|
|
69
|
+
print(f"[*] Embedding {len(texts)} chunks...")
|
|
70
|
+
batch_size = 64
|
|
71
|
+
for i in range(0, len(texts), batch_size):
|
|
72
|
+
batch_texts = texts[i:i + batch_size]
|
|
73
|
+
embeddings = model.encode(
|
|
74
|
+
batch_texts, show_progress_bar=False, normalize_embeddings=True
|
|
75
|
+
)
|
|
76
|
+
collection.add(
|
|
77
|
+
ids=ids[i:i + batch_size],
|
|
78
|
+
embeddings=embeddings.tolist(),
|
|
79
|
+
documents=batch_texts,
|
|
80
|
+
metadatas=metadatas[i:i + batch_size],
|
|
81
|
+
)
|
|
82
|
+
print(f" [{min(i + batch_size, len(texts)):4d}/{len(texts)}]")
|
|
83
|
+
|
|
84
|
+
print(f"[✓] Index ready: {chroma} ({collection.count()} docs)")
|
|
85
|
+
return {
|
|
86
|
+
"files": len(list(source.rglob("*.md"))),
|
|
87
|
+
"chunks": len(documents),
|
|
88
|
+
"collection": config.COLLECTION_NAME,
|
|
89
|
+
"chroma_dir": str(chroma),
|
|
90
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Markdown chunking — split docs on heading boundaries into semantic chunks."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def chunk_markdown(text: str, source: str, min_chars: int = 100, max_chars: int = 800) -> list[dict]:
|
|
7
|
+
"""Split markdown into chunks by heading boundaries.
|
|
8
|
+
|
|
9
|
+
Returns a list of dicts: {id, text, source, heading}.
|
|
10
|
+
"""
|
|
11
|
+
chunks = []
|
|
12
|
+
lines = text.split("\n")
|
|
13
|
+
buffer = []
|
|
14
|
+
buffer_heading = "前言"
|
|
15
|
+
|
|
16
|
+
def flush():
|
|
17
|
+
nonlocal buffer, buffer_heading
|
|
18
|
+
chunk_text = "\n".join(buffer).strip()
|
|
19
|
+
if len(chunk_text) >= min_chars:
|
|
20
|
+
chunks.append({
|
|
21
|
+
"text": chunk_text,
|
|
22
|
+
"source": source,
|
|
23
|
+
"heading": buffer_heading,
|
|
24
|
+
"id": hashlib.md5(
|
|
25
|
+
f"{source}:{buffer_heading}:{chunk_text[:50]}".encode()
|
|
26
|
+
).hexdigest() + f":{len(chunks)}",
|
|
27
|
+
})
|
|
28
|
+
|
|
29
|
+
for line in lines:
|
|
30
|
+
if line.startswith(("## ", "### ", "# ")):
|
|
31
|
+
flush()
|
|
32
|
+
buffer_heading = line.lstrip("#").strip()
|
|
33
|
+
buffer = [line]
|
|
34
|
+
else:
|
|
35
|
+
buffer.append(line)
|
|
36
|
+
|
|
37
|
+
flush()
|
|
38
|
+
|
|
39
|
+
# Merge tiny chunks into their predecessor so the index stays useful.
|
|
40
|
+
merged = []
|
|
41
|
+
for c in chunks:
|
|
42
|
+
if merged and len(c["text"]) < 200:
|
|
43
|
+
merged[-1]["text"] += "\n\n" + c["text"]
|
|
44
|
+
merged[-1]["heading"] += " / " + c["heading"]
|
|
45
|
+
else:
|
|
46
|
+
merged.append(c)
|
|
47
|
+
|
|
48
|
+
return merged
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def collect_markdown(source_dir, min_chars: int = 100) -> list[dict]:
|
|
52
|
+
"""Recursively read all .md files under source_dir into chunks."""
|
|
53
|
+
documents = []
|
|
54
|
+
files = sorted(source_dir.rglob("*.md"))
|
|
55
|
+
for fp in files:
|
|
56
|
+
try:
|
|
57
|
+
text = fp.read_text(encoding="utf-8")
|
|
58
|
+
except OSError:
|
|
59
|
+
continue
|
|
60
|
+
rel = fp.relative_to(source_dir)
|
|
61
|
+
documents.extend(chunk_markdown(text, source=str(rel), min_chars=min_chars))
|
|
62
|
+
return documents
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""ue-kb command-line interface: build / query / info / download-model."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
from . import __version__, config
|
|
8
|
+
from .build import build_index
|
|
9
|
+
from .query import format_results, query
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _model_unavailable_hint(exc: Exception) -> str:
|
|
13
|
+
return (
|
|
14
|
+
"Model 未找到或无法加载。首次使用请先运行:\n"
|
|
15
|
+
" ue-kb download-model\n"
|
|
16
|
+
"(中国大陆网络受限时: 设置 HF_ENDPOINT=https://hf-mirror.com 再执行)\n"
|
|
17
|
+
f"原始错误: {exc}"
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def cmd_build(args: argparse.Namespace) -> int:
|
|
22
|
+
try:
|
|
23
|
+
summary = build_index(
|
|
24
|
+
source_dir=config.source_dir(args.source),
|
|
25
|
+
chroma_dir=config.chroma_dir(args.db),
|
|
26
|
+
model_name=args.model,
|
|
27
|
+
force=args.force,
|
|
28
|
+
offline=not args.online,
|
|
29
|
+
)
|
|
30
|
+
except FileNotFoundError as e:
|
|
31
|
+
print(f"[!] {e}", file=sys.stderr)
|
|
32
|
+
return 1
|
|
33
|
+
except Exception as e:
|
|
34
|
+
print(f"[!] {_model_unavailable_hint(e)}", file=sys.stderr)
|
|
35
|
+
return 1
|
|
36
|
+
print(json.dumps(summary, ensure_ascii=False))
|
|
37
|
+
return 0
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def cmd_query(args: argparse.Namespace) -> int:
|
|
41
|
+
try:
|
|
42
|
+
results = query(
|
|
43
|
+
args.query,
|
|
44
|
+
top_k=args.top_k,
|
|
45
|
+
chroma_dir=config.chroma_dir(args.db),
|
|
46
|
+
model_name=args.model,
|
|
47
|
+
offline=not args.online,
|
|
48
|
+
)
|
|
49
|
+
except Exception as e:
|
|
50
|
+
print(f"[!] {_model_unavailable_hint(e)}", file=sys.stderr)
|
|
51
|
+
return 1
|
|
52
|
+
|
|
53
|
+
if args.json:
|
|
54
|
+
print(json.dumps(results, ensure_ascii=False, indent=2))
|
|
55
|
+
else:
|
|
56
|
+
print(format_results(results, args.query))
|
|
57
|
+
return 0
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def cmd_info(args: argparse.Namespace) -> int:
|
|
61
|
+
import chromadb
|
|
62
|
+
|
|
63
|
+
chroma = config.chroma_dir(args.db)
|
|
64
|
+
try:
|
|
65
|
+
client = chromadb.PersistentClient(path=str(chroma))
|
|
66
|
+
collection = client.get_collection(config.COLLECTION_NAME)
|
|
67
|
+
except Exception as e:
|
|
68
|
+
print(f"[!] 索引不存在: {chroma}(先运行 ue-kb build)\n{e}", file=sys.stderr)
|
|
69
|
+
return 1
|
|
70
|
+
meta = collection.metadata or {}
|
|
71
|
+
print(f"collection : {collection.name}")
|
|
72
|
+
print(f"documents : {collection.count()}")
|
|
73
|
+
print(f"description: {meta.get('description', '')}")
|
|
74
|
+
print(f"chroma_dir : {chroma}")
|
|
75
|
+
return 0
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def cmd_download_model(args: argparse.Namespace) -> int:
|
|
79
|
+
from sentence_transformers import SentenceTransformer
|
|
80
|
+
|
|
81
|
+
model_name = args.model or config.MODEL_NAME
|
|
82
|
+
print(f"[*] 下载模型: {model_name}(首次约 100MB,之后完全离线)")
|
|
83
|
+
print(" 网络受限时请设置: HF_ENDPOINT=https://hf-mirror.com")
|
|
84
|
+
try:
|
|
85
|
+
SentenceTransformer(model_name) # online download
|
|
86
|
+
except Exception as e:
|
|
87
|
+
print(f"[!] 下载失败: {e}", file=sys.stderr)
|
|
88
|
+
return 1
|
|
89
|
+
print("[✓] 模型已缓存。现在可以运行: ue-kb build && ue-kb query")
|
|
90
|
+
return 0
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def main(argv: list[str] | None = None) -> int:
|
|
94
|
+
parser = argparse.ArgumentParser(
|
|
95
|
+
prog="ue-kb",
|
|
96
|
+
description="UE Knowledge Base — offline semantic search over Unreal Engine dev docs",
|
|
97
|
+
)
|
|
98
|
+
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
99
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
100
|
+
|
|
101
|
+
p_build = sub.add_parser("build", help="build the vector index from knowledge/")
|
|
102
|
+
p_build.add_argument("--source", help="corpus dir (default: repo knowledge/)")
|
|
103
|
+
p_build.add_argument("--db", help="chroma dir (default: repo .chroma_db/)")
|
|
104
|
+
p_build.add_argument("--model", default=config.MODEL_NAME, help="embedding model")
|
|
105
|
+
p_build.add_argument("--force", action="store_true", help="rebuild even if index exists")
|
|
106
|
+
p_build.add_argument("--online", action="store_true", help="allow model download if missing")
|
|
107
|
+
p_build.set_defaults(func=cmd_build)
|
|
108
|
+
|
|
109
|
+
p_query = sub.add_parser("query", help="semantic search")
|
|
110
|
+
p_query.add_argument("query", help="search text (e.g. \"GAS cooldown\")")
|
|
111
|
+
p_query.add_argument("--top-k", type=int, default=5)
|
|
112
|
+
p_query.add_argument("--db", help="chroma dir")
|
|
113
|
+
p_query.add_argument("--model", default=config.MODEL_NAME, help="embedding model")
|
|
114
|
+
p_query.add_argument("--online", action="store_true", help="allow model download if missing")
|
|
115
|
+
p_query.add_argument("--json", action="store_true", help="raw JSON output")
|
|
116
|
+
p_query.set_defaults(func=cmd_query)
|
|
117
|
+
|
|
118
|
+
p_info = sub.add_parser("info", help="show index stats")
|
|
119
|
+
p_info.add_argument("--db", help="chroma dir")
|
|
120
|
+
p_info.set_defaults(func=cmd_info)
|
|
121
|
+
|
|
122
|
+
p_dl = sub.add_parser("download-model", help="download the embedding model once")
|
|
123
|
+
p_dl.add_argument("--model", default=config.MODEL_NAME)
|
|
124
|
+
p_dl.set_defaults(func=cmd_download_model)
|
|
125
|
+
|
|
126
|
+
args = parser.parse_args(argv)
|
|
127
|
+
return args.func(args)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
if __name__ == "__main__":
|
|
131
|
+
sys.exit(main())
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""Shared configuration: default paths and the embedding model name."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
# Default model — small, multilingual (Chinese-friendly), runs on CPU.
|
|
7
|
+
MODEL_NAME = "BAAI/bge-small-zh-v1.5"
|
|
8
|
+
|
|
9
|
+
# Repo root (…/ue-knowledge-base)
|
|
10
|
+
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
11
|
+
|
|
12
|
+
# Default corpus + vector store locations inside the repo.
|
|
13
|
+
DEFAULT_SOURCE_DIR = REPO_ROOT / "knowledge"
|
|
14
|
+
DEFAULT_CHROMA_DIR = REPO_ROOT / ".chroma_db"
|
|
15
|
+
|
|
16
|
+
# ChromaDB collection name.
|
|
17
|
+
COLLECTION_NAME = "ue_knowledge"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def source_dir(override: str | None = None) -> Path:
|
|
21
|
+
"""Resolve the markdown corpus directory (CLI flag > env > default)."""
|
|
22
|
+
if override:
|
|
23
|
+
return Path(override)
|
|
24
|
+
env = os.environ.get("UE_KB_SOURCE")
|
|
25
|
+
return Path(env) if env else DEFAULT_SOURCE_DIR
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def chroma_dir(override: str | None = None) -> Path:
|
|
29
|
+
"""Resolve the ChromaDB store directory (CLI flag > env > default)."""
|
|
30
|
+
if override:
|
|
31
|
+
return Path(override)
|
|
32
|
+
env = os.environ.get("UE_KB_CHROMA_DIR")
|
|
33
|
+
return Path(env) if env else DEFAULT_CHROMA_DIR
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Semantic querying against the built index."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from . import config
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def query(
|
|
9
|
+
query_text: str,
|
|
10
|
+
top_k: int = 5,
|
|
11
|
+
chroma_dir: Path | None = None,
|
|
12
|
+
model_name: str | None = None,
|
|
13
|
+
offline: bool = True,
|
|
14
|
+
) -> list[dict]:
|
|
15
|
+
"""Search the knowledge base. Returns [{source, heading, score, text}]."""
|
|
16
|
+
from sentence_transformers import SentenceTransformer
|
|
17
|
+
import chromadb
|
|
18
|
+
|
|
19
|
+
chroma = chroma_dir or config.chroma_dir()
|
|
20
|
+
model_name = model_name or config.MODEL_NAME
|
|
21
|
+
|
|
22
|
+
model = SentenceTransformer(model_name, local_files_only=offline)
|
|
23
|
+
client = chromadb.PersistentClient(path=str(chroma))
|
|
24
|
+
collection = client.get_collection(config.COLLECTION_NAME)
|
|
25
|
+
|
|
26
|
+
query_embedding = model.encode(
|
|
27
|
+
[query_text], normalize_embeddings=True
|
|
28
|
+
)[0]
|
|
29
|
+
|
|
30
|
+
results = collection.query(
|
|
31
|
+
query_embeddings=[query_embedding.tolist()],
|
|
32
|
+
n_results=top_k,
|
|
33
|
+
include=["documents", "metadatas", "distances"],
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
out = []
|
|
37
|
+
docs = results["documents"] or [[]]
|
|
38
|
+
metas = results["metadatas"] or [[]]
|
|
39
|
+
dists = results["distances"] or [[]]
|
|
40
|
+
for doc, meta, dist in zip(docs[0], metas[0], dists[0]):
|
|
41
|
+
out.append({
|
|
42
|
+
"source": meta.get("source", "?"),
|
|
43
|
+
"heading": meta.get("heading", "?"),
|
|
44
|
+
"score": round(1.0 - dist, 4),
|
|
45
|
+
"text": doc,
|
|
46
|
+
})
|
|
47
|
+
return out
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def format_results(results: list[dict], query_text: str) -> str:
|
|
51
|
+
"""Human-readable rendering of query results."""
|
|
52
|
+
if not results:
|
|
53
|
+
return "没有找到相关结果。"
|
|
54
|
+
lines = [f"🔍 UE 知识库检索:{query_text}", ""]
|
|
55
|
+
for i, r in enumerate(results, 1):
|
|
56
|
+
score = f"{r['score']:.1%}"
|
|
57
|
+
lines.append(f"[{i}] {r['source']} › {r['heading']} (匹配度: {score})")
|
|
58
|
+
lines.append(f" {r['text'][:200].replace(chr(10), ' ')}...")
|
|
59
|
+
lines.append("")
|
|
60
|
+
return "\n".join(lines)
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ue-knowledge-base
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Offline semantic search over Unreal Engine development knowledge (BGE + ChromaDB)
|
|
5
|
+
Author: Dong
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: unreal-engine,rag,semantic-search,chromadb,knowledge-base
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Dist: sentence-transformers>=2.6
|
|
12
|
+
Requires-Dist: chromadb>=0.5
|
|
13
|
+
Dynamic: license-file
|
|
14
|
+
|
|
15
|
+
# UE Knowledge Base
|
|
16
|
+
|
|
17
|
+
Offline semantic search over Unreal Engine development knowledge — a curated
|
|
18
|
+
**Chinese-language knowledge corpus** (29 topics: GAS, animation, AI,
|
|
19
|
+
networking, UMG, Niagara, PCG, ...) plus a local RAG pipeline that indexes it
|
|
20
|
+
with `BAAI/bge-small-zh-v1.5` embeddings into ChromaDB.
|
|
21
|
+
|
|
22
|
+
**Zero API cost. Fully offline after a one-time model download.** Built for
|
|
23
|
+
Chinese-speaking UE developers, and ready to plug into any AI agent, IDE, or
|
|
24
|
+
CLI workflow.
|
|
25
|
+
|
|
26
|
+
```
|
|
27
|
+
knowledge/ (29 topics, ~84 markdown docs)
|
|
28
|
+
│ ue-kb build chunk + embed (BGE small, local)
|
|
29
|
+
▼
|
|
30
|
+
.chroma_db/ (vector index)
|
|
31
|
+
│ ue-kb query "GAS cooldown"
|
|
32
|
+
▼
|
|
33
|
+
top-k semantic hits with source + heading
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Features
|
|
37
|
+
|
|
38
|
+
- 📚 **Curated corpus**: hand-written UE development docs in Chinese, covering
|
|
39
|
+
Gameplay Ability System, character movement, animation, physics/collision,
|
|
40
|
+
AI navigation, networking/replication, UMG/Slate, Niagara, Mass Entity,
|
|
41
|
+
State Trees, PCG/procedural generation, materials/rendering, module build
|
|
42
|
+
system, editor tools, and more.
|
|
43
|
+
- 🧠 **Local semantic search**: `BAAI/bge-small-zh-v1.5` (multilingual, ~100MB)
|
|
44
|
+
+ ChromaDB — no cloud APIs, no cost, works on a laptop CPU.
|
|
45
|
+
- 🖥️ **Simple CLI** (`ue-kb`): `build`, `query`, `info`, `download-model`,
|
|
46
|
+
plus `--json` output for agent integration.
|
|
47
|
+
- 🔌 **Extensible**: index any extra UE docs with `--source`, or add engine
|
|
48
|
+
source-indexing scripts under `scripts/`.
|
|
49
|
+
|
|
50
|
+
## Quick start
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
# 1. Install
|
|
54
|
+
pip install -e . # or: pip install ue-knowledge-base
|
|
55
|
+
|
|
56
|
+
# 2. Download the embedding model once (~100MB)
|
|
57
|
+
ue-kb download-model
|
|
58
|
+
# In China: export HF_ENDPOINT=https://hf-mirror.com then retry
|
|
59
|
+
|
|
60
|
+
# 3. Build the index
|
|
61
|
+
ue-kb build
|
|
62
|
+
|
|
63
|
+
# 4. Search
|
|
64
|
+
ue-kb query "GAS ability cooldown" --top-k 5
|
|
65
|
+
ue-kb query "角色移动 速度衰减" --json # machine-readable for agents
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
### Example
|
|
69
|
+
|
|
70
|
+
```text
|
|
71
|
+
$ ue-kb query "GAS 技能冷却"
|
|
72
|
+
🔍 UE 知识库检索:GAS 技能冷却
|
|
73
|
+
|
|
74
|
+
[1] ue-gameplay-abilities/references/gas-input-integration.md › 问题 (匹配度: 21.1%)
|
|
75
|
+
UE 项目同时使用 GAS (GameplayAbilitySystem) 和 Enhanced Input 时,容易陷入
|
|
76
|
+
两个极端:- **全 GAS** → 所有输入走 GAS,但 WASD 轴输入不适合 GAS 的事件
|
|
77
|
+
模型,且 CommitAbility 的 GC 延迟影响跳跃手感 ...
|
|
78
|
+
[2] ue-gameplay-abilities/references/gas-input-integration.md › Jump — GAS 即时技能 (匹配度: 14.2%)
|
|
79
|
+
...
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## Python API
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
from ue_knowledge.query import query
|
|
86
|
+
|
|
87
|
+
for hit in query("GAS cooldown", top_k=5):
|
|
88
|
+
print(hit["source"], hit["heading"], hit["score"])
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Extending the corpus
|
|
92
|
+
|
|
93
|
+
1. Add a markdown file under `knowledge/<topic>/` (use `##`/`###` headings —
|
|
94
|
+
the indexer chunks on heading boundaries).
|
|
95
|
+
2. `ue-kb build --force` to rebuild.
|
|
96
|
+
|
|
97
|
+
For indexing **Unreal Engine C++ header comments** or **Epic official docs**,
|
|
98
|
+
see `scripts/index_engine_source.py` and `scripts/crawl_epic_docs.py`
|
|
99
|
+
(they expect local engine/UE paths — the extracted index data is generated
|
|
100
|
+
locally and is not redistributed, out of respect for Epic's copyright).
|
|
101
|
+
|
|
102
|
+
## License
|
|
103
|
+
|
|
104
|
+
MIT. The knowledge documents are original writing; no engine source code or
|
|
105
|
+
verbatim Epic documentation is included.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/ue_knowledge/__init__.py
|
|
5
|
+
src/ue_knowledge/build.py
|
|
6
|
+
src/ue_knowledge/chunking.py
|
|
7
|
+
src/ue_knowledge/cli.py
|
|
8
|
+
src/ue_knowledge/config.py
|
|
9
|
+
src/ue_knowledge/query.py
|
|
10
|
+
src/ue_knowledge_base.egg-info/PKG-INFO
|
|
11
|
+
src/ue_knowledge_base.egg-info/SOURCES.txt
|
|
12
|
+
src/ue_knowledge_base.egg-info/dependency_links.txt
|
|
13
|
+
src/ue_knowledge_base.egg-info/entry_points.txt
|
|
14
|
+
src/ue_knowledge_base.egg-info/requires.txt
|
|
15
|
+
src/ue_knowledge_base.egg-info/top_level.txt
|
|
16
|
+
tests/test_chunking.py
|
|
17
|
+
tests/test_cli.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
ue_knowledge
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Unit tests for the markdown chunker (no external deps, runs anywhere)."""
|
|
2
|
+
|
|
3
|
+
from ue_knowledge.chunking import chunk_markdown
|
|
4
|
+
|
|
5
|
+
LONG = "word " * 60 # 300 chars — well above the 200-char merge threshold
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def test_chunks_by_heading():
|
|
9
|
+
doc = f"# Title\n\n{LONG}\n\n## Section A\n\n{LONG}\n\n### Sub\n\n{LONG}"
|
|
10
|
+
chunks = chunk_markdown(doc, source="test.md", min_chars=10)
|
|
11
|
+
assert len(chunks) == 3
|
|
12
|
+
headings = [c["heading"] for c in chunks]
|
|
13
|
+
assert headings[0] == "Title"
|
|
14
|
+
assert headings[1] == "Section A"
|
|
15
|
+
assert headings[2] == "Sub"
|
|
16
|
+
assert chunks[0]["source"] == "test.md"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def test_tiny_chunks_merged():
|
|
20
|
+
doc = f"# Big\n\n{LONG}\n\n## Tiny\n\nshort\n\n## Big2\n\n{LONG}"
|
|
21
|
+
chunks = chunk_markdown(doc, source="t.md", min_chars=100)
|
|
22
|
+
# "short" is below min_chars → dropped; only Big and Big2 become chunks
|
|
23
|
+
assert len(chunks) == 2
|
|
24
|
+
assert "Tiny" not in [c["heading"] for c in chunks]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_ids_unique():
|
|
28
|
+
doc = f"# A\n\n{LONG}\n\n## B\n\n{LONG}"
|
|
29
|
+
chunks = chunk_markdown(doc, source="u.md", min_chars=50)
|
|
30
|
+
ids = [c["id"] for c in chunks]
|
|
31
|
+
assert len(ids) == len(set(ids))
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""CLI smoke tests — argparse wiring only (no model/index required)."""
|
|
2
|
+
|
|
3
|
+
from ue_knowledge.cli import main
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def test_version(capsys):
|
|
7
|
+
try:
|
|
8
|
+
main(["--version"])
|
|
9
|
+
except SystemExit as e:
|
|
10
|
+
assert e.code == 0
|
|
11
|
+
out = capsys.readouterr().out
|
|
12
|
+
assert "ue-kb" in out
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_build_missing_source_errors(capsys):
|
|
16
|
+
rc = main(["build", "--source", "C:/definitely/not/here"])
|
|
17
|
+
assert rc == 1
|
|
18
|
+
err = capsys.readouterr().err
|
|
19
|
+
assert "not found" in err
|