virtudoc 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- virtudoc-0.1.0/LICENSE +21 -0
- virtudoc-0.1.0/PKG-INFO +224 -0
- virtudoc-0.1.0/README.md +192 -0
- virtudoc-0.1.0/pyproject.toml +62 -0
- virtudoc-0.1.0/setup.cfg +4 -0
- virtudoc-0.1.0/tests/test_chunker.py +76 -0
- virtudoc-0.1.0/tests/test_cli.py +263 -0
- virtudoc-0.1.0/tests/test_db.py +211 -0
- virtudoc-0.1.0/tests/test_doctor.py +68 -0
- virtudoc-0.1.0/tests/test_fetch.py +287 -0
- virtudoc-0.1.0/tests/test_indexer.py +207 -0
- virtudoc-0.1.0/tests/test_mcp.py +305 -0
- virtudoc-0.1.0/tests/test_notes.py +64 -0
- virtudoc-0.1.0/tests/test_parsers.py +60 -0
- virtudoc-0.1.0/tests/test_search.py +189 -0
- virtudoc-0.1.0/tests/test_tokens.py +40 -0
- virtudoc-0.1.0/tests/test_versions.py +215 -0
- virtudoc-0.1.0/tests/test_walker.py +120 -0
- virtudoc-0.1.0/virtudoc/__init__.py +3 -0
- virtudoc-0.1.0/virtudoc/__main__.py +4 -0
- virtudoc-0.1.0/virtudoc/cli.py +559 -0
- virtudoc-0.1.0/virtudoc/config.py +83 -0
- virtudoc-0.1.0/virtudoc/db.py +169 -0
- virtudoc-0.1.0/virtudoc/doctor.py +157 -0
- virtudoc-0.1.0/virtudoc/fetcher.py +241 -0
- virtudoc-0.1.0/virtudoc/indexer/__init__.py +0 -0
- virtudoc-0.1.0/virtudoc/indexer/chunker.py +139 -0
- virtudoc-0.1.0/virtudoc/indexer/indexer.py +310 -0
- virtudoc-0.1.0/virtudoc/indexer/parsers.py +248 -0
- virtudoc-0.1.0/virtudoc/indexer/walker.py +137 -0
- virtudoc-0.1.0/virtudoc/logging_setup.py +22 -0
- virtudoc-0.1.0/virtudoc/mcp_server.py +234 -0
- virtudoc-0.1.0/virtudoc/notes.py +76 -0
- virtudoc-0.1.0/virtudoc/search/__init__.py +0 -0
- virtudoc-0.1.0/virtudoc/search/engine.py +436 -0
- virtudoc-0.1.0/virtudoc/tokens.py +27 -0
- virtudoc-0.1.0/virtudoc/versions.py +243 -0
- virtudoc-0.1.0/virtudoc.egg-info/PKG-INFO +224 -0
- virtudoc-0.1.0/virtudoc.egg-info/SOURCES.txt +41 -0
- virtudoc-0.1.0/virtudoc.egg-info/dependency_links.txt +1 -0
- virtudoc-0.1.0/virtudoc.egg-info/entry_points.txt +2 -0
- virtudoc-0.1.0/virtudoc.egg-info/requires.txt +5 -0
- virtudoc-0.1.0/virtudoc.egg-info/top_level.txt +1 -0
virtudoc-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sergey Romanov
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
virtudoc-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: virtudoc
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Local documentation server for AI agents (MCP, SQLite FTS5)
|
|
5
|
+
Author-email: Sergey Romanov <xxsmotur@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/saromanov/virtudoc
|
|
8
|
+
Project-URL: Documentation, https://github.com/saromanov/virtudoc/tree/main/docs
|
|
9
|
+
Project-URL: Source, https://github.com/saromanov/virtudoc
|
|
10
|
+
Project-URL: Tracker, https://github.com/saromanov/virtudoc/issues
|
|
11
|
+
Keywords: mcp,llm,ai-agents,documentation,full-text-search,sqlite,fts5,offline
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Topic :: Database
|
|
22
|
+
Classifier: Topic :: Software Development :: Documentation
|
|
23
|
+
Classifier: Topic :: Text Processing :: Indexing
|
|
24
|
+
Requires-Python: >=3.11
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
Requires-Dist: markdown-it-py<4,>=3.0
|
|
28
|
+
Requires-Dist: mcp<3,>=2.3
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
# VirtuDoc
|
|
34
|
+
|
|
35
|
+
[](https://github.com/saromanov/virtudoc/actions/workflows/ci.yml)
|
|
36
|
+
[](https://pypi.org/project/virtudoc/)
|
|
37
|
+
[](https://pypi.org/project/virtudoc/)
|
|
38
|
+
[](https://github.com/saromanov/virtudoc/blob/main/LICENSE)
|
|
39
|
+
|
|
40
|
+
A local documentation server for AI agents. It indexes documentation from
|
|
41
|
+
local directories into SQLite with FTS5 full-text search and serves relevant
|
|
42
|
+
fragments to an agent through MCP tools. Fully offline: no network access
|
|
43
|
+
happens in any mode.
|
|
44
|
+
|
|
45
|
+
Think of it as [Upstash Context7](https://github.com/upstash/context7) running
|
|
46
|
+
entirely on your machine: private and internal docs never leave your computer,
|
|
47
|
+
there are no API keys, request limits, or cloud dependencies.
|
|
48
|
+
|
|
49
|
+
## Installation
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pip install virtudoc
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Requires Python 3.11+. The only runtime dependencies are `markdown-it-py`
|
|
56
|
+
(Markdown parsing) and the `mcp` SDK.
|
|
57
|
+
|
|
58
|
+
## Quick start
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
# Register a documentation directory and index it
|
|
62
|
+
virtudoc add /path/to/docs --name mylib --version 2.1.0
|
|
63
|
+
|
|
64
|
+
# Index statistics
|
|
65
|
+
virtudoc list
|
|
66
|
+
|
|
67
|
+
# Search from the terminal (same results an agent will get via MCP)
|
|
68
|
+
virtudoc search "how to handle authentication errors"
|
|
69
|
+
|
|
70
|
+
# Incremental reindexing after the docs changed
|
|
71
|
+
virtudoc update # all libraries
|
|
72
|
+
virtudoc update --name mylib
|
|
73
|
+
|
|
74
|
+
# Full rebuild of one library (applies changed chunk.* settings)
|
|
75
|
+
virtudoc reindex mylib
|
|
76
|
+
|
|
77
|
+
# Remove a library
|
|
78
|
+
virtudoc remove mylib
|
|
79
|
+
|
|
80
|
+
# Attach a team convention note to a library (agents receive it with docs)
|
|
81
|
+
virtudoc note add mylib --text "Use the retry helper for all outbound calls."
|
|
82
|
+
|
|
83
|
+
# Warn when the indexed docs version diverges from the project dependency
|
|
84
|
+
virtudoc link mylib --project /path/to/consuming/project
|
|
85
|
+
virtudoc doctor # config, sources, index integrity, version drift summary
|
|
86
|
+
|
|
87
|
+
# Pull public documentation from a git URL or archive (the only networked command)
|
|
88
|
+
virtudoc fetch --repo https://github.com/org/docs --name theirlib
|
|
89
|
+
|
|
90
|
+
# Move your whole setup to another machine (see docs/installation.md)
|
|
91
|
+
virtudoc export # dump the library list to ~/.virtudoc/libraries.toml
|
|
92
|
+
virtudoc import # register and index everything from the file
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## Connect your agent
|
|
96
|
+
|
|
97
|
+
Run the MCP server and register it in your agent's config. The server speaks
|
|
98
|
+
stdio by default and reindexes changed sources on every start, so the agent
|
|
99
|
+
always sees the current documentation. Agents spawn the server with a
|
|
100
|
+
default environment, so the data directory is `~/.virtudoc` — set
|
|
101
|
+
`VIRTUDOC_HOME` in the server `env` block if you need a custom location.
|
|
102
|
+
|
|
103
|
+
**Claude Code** — add to `.mcp.json` in your project (or run
|
|
104
|
+
`claude mcp add virtudoc --scope user -- virtudoc serve`):
|
|
105
|
+
|
|
106
|
+
```json
|
|
107
|
+
{
|
|
108
|
+
"mcpServers": {
|
|
109
|
+
"virtudoc": {
|
|
110
|
+
"command": "virtudoc",
|
|
111
|
+
"args": ["serve"]
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
**Cursor** — add to `.cursor/mcp.json`:
|
|
118
|
+
|
|
119
|
+
```json
|
|
120
|
+
{
|
|
121
|
+
"mcpServers": {
|
|
122
|
+
"virtudoc": {
|
|
123
|
+
"command": "virtudoc",
|
|
124
|
+
"args": ["serve"]
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
**OpenCode** — add to `opencode.json` (project root or
|
|
131
|
+
`~/.config/opencode/opencode.json`):
|
|
132
|
+
|
|
133
|
+
```json
|
|
134
|
+
{
|
|
135
|
+
"$schema": "https://opencode.ai/config.json",
|
|
136
|
+
"mcp": {
|
|
137
|
+
"virtudoc": {
|
|
138
|
+
"type": "local",
|
|
139
|
+
"command": ["virtudoc", "serve"],
|
|
140
|
+
"enabled": true
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
## MCP tools
|
|
147
|
+
|
|
148
|
+
- `find-library(libraryName, query)` — find the `local/<name>` id of a
|
|
149
|
+
library by its name and your question. Call it first.
|
|
150
|
+
- `get-library-docs(libraryId, query, tokens=5000, language, page, mode,
|
|
151
|
+
section)` — Markdown fragments with heading paths and source file paths,
|
|
152
|
+
assembled under the token budget. Relevant project notes are included
|
|
153
|
+
before the fragments. Empty results return the library's top-level
|
|
154
|
+
sections to help rephrase the query.
|
|
155
|
+
|
|
156
|
+
Optionally expose the server over streamable HTTP on 127.0.0.1 for clients
|
|
157
|
+
without stdio support: `virtudoc serve --http [port]` (default port 8765).
|
|
158
|
+
|
|
159
|
+
Supported formats: Markdown (`.md`, `.mdx`), reStructuredText (`.rst`),
|
|
160
|
+
plain text (`.txt`), HTML (`.html`, `.htm`, best-effort text extraction).
|
|
161
|
+
Hidden files and directories are always skipped; a `.virtudocignore` file at
|
|
162
|
+
the source root (`.gitignore`-compatible syntax: globs, `**`, `!` negation,
|
|
163
|
+
directory patterns) excludes generated or vendor trees from indexing.
|
|
164
|
+
|
|
165
|
+
## How it works
|
|
166
|
+
|
|
167
|
+
- The **chunker** splits documents along H2/H3 headings (target chunk size
|
|
168
|
+
600 tokens, hard limit 1200, 60-token overlap); code blocks are never split.
|
|
169
|
+
- **Search** is SQLite FTS5 with BM25 ranking. Queries cascade:
|
|
170
|
+
exact `AND` → prefix `AND` with ending stemming → `OR`, so Russian
|
|
171
|
+
inflections (`пакетная` finds `пакетной`) and fuzzy phrasing both resolve.
|
|
172
|
+
- Results are assembled under a token budget (default 5000) with a
|
|
173
|
+
topic boost for chunks whose heading path matches the query.
|
|
174
|
+
- **Indexing is incremental**: unchanged files are skipped by content hash.
|
|
175
|
+
- All data lives in `~/.virtudoc/` (WAL-mode `index.db`, config, log).
|
|
176
|
+
Deleting the directory removes the system completely.
|
|
177
|
+
|
|
178
|
+
## Configuration
|
|
179
|
+
|
|
180
|
+
`~/.virtudoc/config.toml` (all keys optional):
|
|
181
|
+
|
|
182
|
+
```toml
|
|
183
|
+
[chunk]
|
|
184
|
+
target_tokens = 600
|
|
185
|
+
max_tokens = 1200
|
|
186
|
+
overlap_tokens = 60
|
|
187
|
+
|
|
188
|
+
[search]
|
|
189
|
+
default_tokens = 5000
|
|
190
|
+
max_results = 20
|
|
191
|
+
|
|
192
|
+
[log]
|
|
193
|
+
level = "INFO"
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
Changing `chunk.*` only affects new indexing; the existing index is not
|
|
197
|
+
rebuilt automatically.
|
|
198
|
+
|
|
199
|
+
## Documentation
|
|
200
|
+
|
|
201
|
+
- [docs/](docs/index.md) — full documentation: installation, getting
|
|
202
|
+
started, CLI reference, MCP integration, configuration, indexing and
|
|
203
|
+
search internals, project notes, version drift warnings
|
|
204
|
+
- [ARCHITECTURE.md](ARCHITECTURE.md) — components, data flows, storage schema
|
|
205
|
+
|
|
206
|
+
## Development
|
|
207
|
+
|
|
208
|
+
```bash
|
|
209
|
+
pip install -e '.[dev]'
|
|
210
|
+
pytest
|
|
211
|
+
ruff check virtudoc tests
|
|
212
|
+
ruff format virtudoc tests
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
## Status
|
|
216
|
+
|
|
217
|
+
Milestones M1 (core: indexing, chunking, FTS5 search, CLI) and M2 (MCP
|
|
218
|
+
server over stdio/HTTP, project notes, outline mode, `.virtudocignore`) are
|
|
219
|
+
done, plus `link`/`doctor` version drift warnings and the `fetch` command.
|
|
220
|
+
Published on PyPI; a demo screencast is planned.
|
|
221
|
+
|
|
222
|
+
## License
|
|
223
|
+
|
|
224
|
+
MIT, see [LICENSE](LICENSE).
|
virtudoc-0.1.0/README.md
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
# VirtuDoc
|
|
2
|
+
|
|
3
|
+
[](https://github.com/saromanov/virtudoc/actions/workflows/ci.yml)
|
|
4
|
+
[](https://pypi.org/project/virtudoc/)
|
|
5
|
+
[](https://pypi.org/project/virtudoc/)
|
|
6
|
+
[](https://github.com/saromanov/virtudoc/blob/main/LICENSE)
|
|
7
|
+
|
|
8
|
+
A local documentation server for AI agents. It indexes documentation from
|
|
9
|
+
local directories into SQLite with FTS5 full-text search and serves relevant
|
|
10
|
+
fragments to an agent through MCP tools. Fully offline: no network access
|
|
11
|
+
happens in any mode.
|
|
12
|
+
|
|
13
|
+
Think of it as [Upstash Context7](https://github.com/upstash/context7) running
|
|
14
|
+
entirely on your machine: private and internal docs never leave your computer,
|
|
15
|
+
there are no API keys, request limits, or cloud dependencies.
|
|
16
|
+
|
|
17
|
+
## Installation
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
pip install virtudoc
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Requires Python 3.11+. The only runtime dependencies are `markdown-it-py`
|
|
24
|
+
(Markdown parsing) and the `mcp` SDK.
|
|
25
|
+
|
|
26
|
+
## Quick start
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
# Register a documentation directory and index it
|
|
30
|
+
virtudoc add /path/to/docs --name mylib --version 2.1.0
|
|
31
|
+
|
|
32
|
+
# Index statistics
|
|
33
|
+
virtudoc list
|
|
34
|
+
|
|
35
|
+
# Search from the terminal (same results an agent will get via MCP)
|
|
36
|
+
virtudoc search "how to handle authentication errors"
|
|
37
|
+
|
|
38
|
+
# Incremental reindexing after the docs changed
|
|
39
|
+
virtudoc update # all libraries
|
|
40
|
+
virtudoc update --name mylib
|
|
41
|
+
|
|
42
|
+
# Full rebuild of one library (applies changed chunk.* settings)
|
|
43
|
+
virtudoc reindex mylib
|
|
44
|
+
|
|
45
|
+
# Remove a library
|
|
46
|
+
virtudoc remove mylib
|
|
47
|
+
|
|
48
|
+
# Attach a team convention note to a library (agents receive it with docs)
|
|
49
|
+
virtudoc note add mylib --text "Use the retry helper for all outbound calls."
|
|
50
|
+
|
|
51
|
+
# Warn when the indexed docs version diverges from the project dependency
|
|
52
|
+
virtudoc link mylib --project /path/to/consuming/project
|
|
53
|
+
virtudoc doctor # config, sources, index integrity, version drift summary
|
|
54
|
+
|
|
55
|
+
# Pull public documentation from a git URL or archive (the only networked command)
|
|
56
|
+
virtudoc fetch --repo https://github.com/org/docs --name theirlib
|
|
57
|
+
|
|
58
|
+
# Move your whole setup to another machine (see docs/installation.md)
|
|
59
|
+
virtudoc export # dump the library list to ~/.virtudoc/libraries.toml
|
|
60
|
+
virtudoc import # register and index everything from the file
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## Connect your agent
|
|
64
|
+
|
|
65
|
+
Run the MCP server and register it in your agent's config. The server speaks
|
|
66
|
+
stdio by default and reindexes changed sources on every start, so the agent
|
|
67
|
+
always sees the current documentation. Agents spawn the server with a
|
|
68
|
+
default environment, so the data directory is `~/.virtudoc` — set
|
|
69
|
+
`VIRTUDOC_HOME` in the server `env` block if you need a custom location.
|
|
70
|
+
|
|
71
|
+
**Claude Code** — add to `.mcp.json` in your project (or run
|
|
72
|
+
`claude mcp add virtudoc --scope user -- virtudoc serve`):
|
|
73
|
+
|
|
74
|
+
```json
|
|
75
|
+
{
|
|
76
|
+
"mcpServers": {
|
|
77
|
+
"virtudoc": {
|
|
78
|
+
"command": "virtudoc",
|
|
79
|
+
"args": ["serve"]
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
**Cursor** — add to `.cursor/mcp.json`:
|
|
86
|
+
|
|
87
|
+
```json
|
|
88
|
+
{
|
|
89
|
+
"mcpServers": {
|
|
90
|
+
"virtudoc": {
|
|
91
|
+
"command": "virtudoc",
|
|
92
|
+
"args": ["serve"]
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
**OpenCode** — add to `opencode.json` (project root or
|
|
99
|
+
`~/.config/opencode/opencode.json`):
|
|
100
|
+
|
|
101
|
+
```json
|
|
102
|
+
{
|
|
103
|
+
"$schema": "https://opencode.ai/config.json",
|
|
104
|
+
"mcp": {
|
|
105
|
+
"virtudoc": {
|
|
106
|
+
"type": "local",
|
|
107
|
+
"command": ["virtudoc", "serve"],
|
|
108
|
+
"enabled": true
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
## MCP tools
|
|
115
|
+
|
|
116
|
+
- `find-library(libraryName, query)` — find the `local/<name>` id of a
|
|
117
|
+
library by its name and your question. Call it first.
|
|
118
|
+
- `get-library-docs(libraryId, query, tokens=5000, language, page, mode,
|
|
119
|
+
section)` — Markdown fragments with heading paths and source file paths,
|
|
120
|
+
assembled under the token budget. Relevant project notes are included
|
|
121
|
+
before the fragments. Empty results return the library's top-level
|
|
122
|
+
sections to help rephrase the query.
|
|
123
|
+
|
|
124
|
+
Optionally expose the server over streamable HTTP on 127.0.0.1 for clients
|
|
125
|
+
without stdio support: `virtudoc serve --http [port]` (default port 8765).
|
|
126
|
+
|
|
127
|
+
Supported formats: Markdown (`.md`, `.mdx`), reStructuredText (`.rst`),
|
|
128
|
+
plain text (`.txt`), HTML (`.html`, `.htm`, best-effort text extraction).
|
|
129
|
+
Hidden files and directories are always skipped; a `.virtudocignore` file at
|
|
130
|
+
the source root (`.gitignore`-compatible syntax: globs, `**`, `!` negation,
|
|
131
|
+
directory patterns) excludes generated or vendor trees from indexing.
|
|
132
|
+
|
|
133
|
+
## How it works
|
|
134
|
+
|
|
135
|
+
- The **chunker** splits documents along H2/H3 headings (target chunk size
|
|
136
|
+
600 tokens, hard limit 1200, 60-token overlap); code blocks are never split.
|
|
137
|
+
- **Search** is SQLite FTS5 with BM25 ranking. Queries cascade:
|
|
138
|
+
exact `AND` → prefix `AND` with ending stemming → `OR`, so Russian
|
|
139
|
+
inflections (`пакетная` finds `пакетной`) and fuzzy phrasing both resolve.
|
|
140
|
+
- Results are assembled under a token budget (default 5000) with a
|
|
141
|
+
topic boost for chunks whose heading path matches the query.
|
|
142
|
+
- **Indexing is incremental**: unchanged files are skipped by content hash.
|
|
143
|
+
- All data lives in `~/.virtudoc/` (WAL-mode `index.db`, config, log).
|
|
144
|
+
Deleting the directory removes the system completely.
|
|
145
|
+
|
|
146
|
+
## Configuration
|
|
147
|
+
|
|
148
|
+
`~/.virtudoc/config.toml` (all keys optional):
|
|
149
|
+
|
|
150
|
+
```toml
|
|
151
|
+
[chunk]
|
|
152
|
+
target_tokens = 600
|
|
153
|
+
max_tokens = 1200
|
|
154
|
+
overlap_tokens = 60
|
|
155
|
+
|
|
156
|
+
[search]
|
|
157
|
+
default_tokens = 5000
|
|
158
|
+
max_results = 20
|
|
159
|
+
|
|
160
|
+
[log]
|
|
161
|
+
level = "INFO"
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
Changing `chunk.*` only affects new indexing; the existing index is not
|
|
165
|
+
rebuilt automatically.
|
|
166
|
+
|
|
167
|
+
## Documentation
|
|
168
|
+
|
|
169
|
+
- [docs/](docs/index.md) — full documentation: installation, getting
|
|
170
|
+
started, CLI reference, MCP integration, configuration, indexing and
|
|
171
|
+
search internals, project notes, version drift warnings
|
|
172
|
+
- [ARCHITECTURE.md](ARCHITECTURE.md) — components, data flows, storage schema
|
|
173
|
+
|
|
174
|
+
## Development
|
|
175
|
+
|
|
176
|
+
```bash
|
|
177
|
+
pip install -e '.[dev]'
|
|
178
|
+
pytest
|
|
179
|
+
ruff check virtudoc tests
|
|
180
|
+
ruff format virtudoc tests
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
## Status
|
|
184
|
+
|
|
185
|
+
Milestones M1 (core: indexing, chunking, FTS5 search, CLI) and M2 (MCP
|
|
186
|
+
server over stdio/HTTP, project notes, outline mode, `.virtudocignore`) are
|
|
187
|
+
done, plus `link`/`doctor` version drift warnings and the `fetch` command.
|
|
188
|
+
Published on PyPI; a demo screencast is planned.
|
|
189
|
+
|
|
190
|
+
## License
|
|
191
|
+
|
|
192
|
+
MIT, see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "virtudoc"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Local documentation server for AI agents (MCP, SQLite FTS5)"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Sergey Romanov", email = "xxsmotur@gmail.com" }]
|
|
14
|
+
keywords = ["mcp", "llm", "ai-agents", "documentation", "full-text-search", "sqlite", "fts5", "offline"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Environment :: Console",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Programming Language :: Python :: 3.14",
|
|
25
|
+
"Topic :: Database",
|
|
26
|
+
"Topic :: Software Development :: Documentation",
|
|
27
|
+
"Topic :: Text Processing :: Indexing",
|
|
28
|
+
]
|
|
29
|
+
dependencies = [
|
|
30
|
+
"markdown-it-py>=3.0,<4",
|
|
31
|
+
"mcp>=2.3,<3",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
[project.optional-dependencies]
|
|
35
|
+
dev = [
|
|
36
|
+
"pytest>=8",
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
[project.scripts]
|
|
40
|
+
virtudoc = "virtudoc.cli:main"
|
|
41
|
+
|
|
42
|
+
[project.urls]
|
|
43
|
+
Homepage = "https://github.com/saromanov/virtudoc"
|
|
44
|
+
Documentation = "https://github.com/saromanov/virtudoc/tree/main/docs"
|
|
45
|
+
Source = "https://github.com/saromanov/virtudoc"
|
|
46
|
+
Tracker = "https://github.com/saromanov/virtudoc/issues"
|
|
47
|
+
|
|
48
|
+
[tool.setuptools.packages.find]
|
|
49
|
+
include = ["virtudoc*"]
|
|
50
|
+
|
|
51
|
+
[tool.ruff]
|
|
52
|
+
target-version = "py311"
|
|
53
|
+
line-length = 100
|
|
54
|
+
|
|
55
|
+
[tool.ruff.lint]
|
|
56
|
+
select = ["E", "F", "I", "UP", "RUF"]
|
|
57
|
+
# RUF001–003 flag intentional Cyrillic in user-facing strings and test data
|
|
58
|
+
# (Russian morphology support is a core feature).
|
|
59
|
+
ignore = ["RUF001", "RUF002", "RUF003"]
|
|
60
|
+
|
|
61
|
+
[tool.pytest.ini_options]
|
|
62
|
+
testpaths = ["tests"]
|
virtudoc-0.1.0/setup.cfg
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
from virtudoc.indexer.chunker import Chunker
|
|
2
|
+
from virtudoc.indexer.parsers import parse_markdown
|
|
3
|
+
from virtudoc.tokens import estimate_tokens
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def make_chunker(target=50, max_tokens=100, overlap=10):
|
|
7
|
+
return Chunker(target_tokens=target, max_tokens=max_tokens, overlap_tokens=overlap)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def para(text):
|
|
11
|
+
return text
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def test_chunks_split_by_headings():
|
|
15
|
+
text = "# Документ\n\n## Первый\n\n" + "слово " * 30 + "\n\n## Второй\n\n" + "другое " * 30
|
|
16
|
+
chunks = make_chunker().chunk(parse_markdown(text))
|
|
17
|
+
paths = [c.heading_path for c in chunks]
|
|
18
|
+
assert paths[0] == "Документ → Первый"
|
|
19
|
+
assert "Документ → Второй" in paths
|
|
20
|
+
assert paths == sorted(paths, key=paths.index)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_heading_path_nesting():
|
|
24
|
+
text = "# API\n\n## Middleware\n\n" + "текст " * 30 + "\n\n### Ошибки\n\n" + "ещё текст " * 20
|
|
25
|
+
chunks = make_chunker().chunk(parse_markdown(text))
|
|
26
|
+
assert chunks[0].heading_path == "API → Middleware"
|
|
27
|
+
assert chunks[-1].heading_path == "API → Middleware → Ошибки"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_target_budget_creates_multiple_chunks():
|
|
31
|
+
body = "\n\n".join(f"Параграф {i}: " + "а" * 120 for i in range(8))
|
|
32
|
+
text = f"# T\n\n## Раздел\n\n{body}\n"
|
|
33
|
+
chunks = make_chunker(target=100, max_tokens=200).chunk(parse_markdown(text))
|
|
34
|
+
assert len(chunks) >= 3
|
|
35
|
+
assert all(c.token_est <= 210 for c in chunks)
|
|
36
|
+
assert [c.ord for c in chunks] == list(range(len(chunks)))
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_overlap_between_adjacent_chunks():
|
|
40
|
+
body = "\n\n".join(f"Абзац номер {i} " + "x" * 100 for i in range(8))
|
|
41
|
+
text = f"# T\n\n## Раздел\n\n{body}\n"
|
|
42
|
+
chunks = make_chunker(target=100, overlap=10).chunk(parse_markdown(text))
|
|
43
|
+
assert len(chunks) >= 2
|
|
44
|
+
second = chunks[1]
|
|
45
|
+
assert second.text != chunks[0].text
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_code_block_not_split():
|
|
49
|
+
code = "\n".join(f"line_{i} = {i}" for i in range(40))
|
|
50
|
+
text = f"# T\n\n## Код\n\n```python\n{code}\n```\n"
|
|
51
|
+
chunks = make_chunker(target=30, max_tokens=200).chunk(parse_markdown(text))
|
|
52
|
+
code_chunks = [c for c in chunks if "line_39" in c.text]
|
|
53
|
+
assert len(code_chunks) == 1
|
|
54
|
+
assert "line_0" in code_chunks[0].text
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_oversized_paragraph_split_with_marker():
|
|
58
|
+
big = "слово " * 400
|
|
59
|
+
text = f"# T\n\n{big}\n"
|
|
60
|
+
chunks = make_chunker(target=100, max_tokens=150).chunk(parse_markdown(text))
|
|
61
|
+
assert len(chunks) >= 2
|
|
62
|
+
assert all(c.token_est <= 160 for c in chunks)
|
|
63
|
+
assert any("(continued)" in c.text for c in chunks[1:])
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_code_ratio():
|
|
67
|
+
code = "\n".join(f"x = {i}" for i in range(30))
|
|
68
|
+
text = f"# T\n\n## Раздел\n\nКороткий текст.\n\n```python\n{code}\n```\n"
|
|
69
|
+
chunks = make_chunker().chunk(parse_markdown(text))
|
|
70
|
+
assert any(c.code_ratio > 0.3 for c in chunks)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def test_token_est_matches_text():
|
|
74
|
+
text = "# T\n\n## Раздел\n\n" + "текст " * 50 + "\n"
|
|
75
|
+
chunks = make_chunker().chunk(parse_markdown(text))
|
|
76
|
+
assert all(c.token_est == estimate_tokens(c.text) for c in chunks)
|