virtudoc 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. virtudoc-0.1.0/LICENSE +21 -0
  2. virtudoc-0.1.0/PKG-INFO +224 -0
  3. virtudoc-0.1.0/README.md +192 -0
  4. virtudoc-0.1.0/pyproject.toml +62 -0
  5. virtudoc-0.1.0/setup.cfg +4 -0
  6. virtudoc-0.1.0/tests/test_chunker.py +76 -0
  7. virtudoc-0.1.0/tests/test_cli.py +263 -0
  8. virtudoc-0.1.0/tests/test_db.py +211 -0
  9. virtudoc-0.1.0/tests/test_doctor.py +68 -0
  10. virtudoc-0.1.0/tests/test_fetch.py +287 -0
  11. virtudoc-0.1.0/tests/test_indexer.py +207 -0
  12. virtudoc-0.1.0/tests/test_mcp.py +305 -0
  13. virtudoc-0.1.0/tests/test_notes.py +64 -0
  14. virtudoc-0.1.0/tests/test_parsers.py +60 -0
  15. virtudoc-0.1.0/tests/test_search.py +189 -0
  16. virtudoc-0.1.0/tests/test_tokens.py +40 -0
  17. virtudoc-0.1.0/tests/test_versions.py +215 -0
  18. virtudoc-0.1.0/tests/test_walker.py +120 -0
  19. virtudoc-0.1.0/virtudoc/__init__.py +3 -0
  20. virtudoc-0.1.0/virtudoc/__main__.py +4 -0
  21. virtudoc-0.1.0/virtudoc/cli.py +559 -0
  22. virtudoc-0.1.0/virtudoc/config.py +83 -0
  23. virtudoc-0.1.0/virtudoc/db.py +169 -0
  24. virtudoc-0.1.0/virtudoc/doctor.py +157 -0
  25. virtudoc-0.1.0/virtudoc/fetcher.py +241 -0
  26. virtudoc-0.1.0/virtudoc/indexer/__init__.py +0 -0
  27. virtudoc-0.1.0/virtudoc/indexer/chunker.py +139 -0
  28. virtudoc-0.1.0/virtudoc/indexer/indexer.py +310 -0
  29. virtudoc-0.1.0/virtudoc/indexer/parsers.py +248 -0
  30. virtudoc-0.1.0/virtudoc/indexer/walker.py +137 -0
  31. virtudoc-0.1.0/virtudoc/logging_setup.py +22 -0
  32. virtudoc-0.1.0/virtudoc/mcp_server.py +234 -0
  33. virtudoc-0.1.0/virtudoc/notes.py +76 -0
  34. virtudoc-0.1.0/virtudoc/search/__init__.py +0 -0
  35. virtudoc-0.1.0/virtudoc/search/engine.py +436 -0
  36. virtudoc-0.1.0/virtudoc/tokens.py +27 -0
  37. virtudoc-0.1.0/virtudoc/versions.py +243 -0
  38. virtudoc-0.1.0/virtudoc.egg-info/PKG-INFO +224 -0
  39. virtudoc-0.1.0/virtudoc.egg-info/SOURCES.txt +41 -0
  40. virtudoc-0.1.0/virtudoc.egg-info/dependency_links.txt +1 -0
  41. virtudoc-0.1.0/virtudoc.egg-info/entry_points.txt +2 -0
  42. virtudoc-0.1.0/virtudoc.egg-info/requires.txt +5 -0
  43. virtudoc-0.1.0/virtudoc.egg-info/top_level.txt +1 -0
virtudoc-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Sergey Romanov
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,224 @@
1
+ Metadata-Version: 2.4
2
+ Name: virtudoc
3
+ Version: 0.1.0
4
+ Summary: Local documentation server for AI agents (MCP, SQLite FTS5)
5
+ Author-email: Sergey Romanov <xxsmotur@gmail.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/saromanov/virtudoc
8
+ Project-URL: Documentation, https://github.com/saromanov/virtudoc/tree/main/docs
9
+ Project-URL: Source, https://github.com/saromanov/virtudoc
10
+ Project-URL: Tracker, https://github.com/saromanov/virtudoc/issues
11
+ Keywords: mcp,llm,ai-agents,documentation,full-text-search,sqlite,fts5,offline
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Environment :: Console
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3 :: Only
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Programming Language :: Python :: 3.14
21
+ Classifier: Topic :: Database
22
+ Classifier: Topic :: Software Development :: Documentation
23
+ Classifier: Topic :: Text Processing :: Indexing
24
+ Requires-Python: >=3.11
25
+ Description-Content-Type: text/markdown
26
+ License-File: LICENSE
27
+ Requires-Dist: markdown-it-py<4,>=3.0
28
+ Requires-Dist: mcp<3,>=2.3
29
+ Provides-Extra: dev
30
+ Requires-Dist: pytest>=8; extra == "dev"
31
+ Dynamic: license-file
32
+
33
+ # VirtuDoc
34
+
35
+ [![CI](https://github.com/saromanov/virtudoc/actions/workflows/ci.yml/badge.svg)](https://github.com/saromanov/virtudoc/actions/workflows/ci.yml)
36
+ [![PyPI](https://img.shields.io/pypi/v/virtudoc)](https://pypi.org/project/virtudoc/)
37
+ [![Python versions](https://img.shields.io/pypi/pyversions/virtudoc)](https://pypi.org/project/virtudoc/)
38
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/saromanov/virtudoc/blob/main/LICENSE)
39
+
40
+ A local documentation server for AI agents. It indexes documentation from
41
+ local directories into SQLite with FTS5 full-text search and serves relevant
42
+ fragments to an agent through MCP tools. Fully offline: no network access
43
+ happens in any mode.
44
+
45
+ Think of it as [Upstash Context7](https://github.com/upstash/context7) running
46
+ entirely on your machine: private and internal docs never leave your computer,
47
+ there are no API keys, request limits, or cloud dependencies.
48
+
49
+ ## Installation
50
+
51
+ ```bash
52
+ pip install virtudoc
53
+ ```
54
+
55
+ Requires Python 3.11+. The only runtime dependencies are `markdown-it-py`
56
+ (Markdown parsing) and the `mcp` SDK.
57
+
58
+ ## Quick start
59
+
60
+ ```bash
61
+ # Register a documentation directory and index it
62
+ virtudoc add /path/to/docs --name mylib --version 2.1.0
63
+
64
+ # Index statistics
65
+ virtudoc list
66
+
67
+ # Search from the terminal (same results an agent will get via MCP)
68
+ virtudoc search "how to handle authentication errors"
69
+
70
+ # Incremental reindexing after the docs changed
71
+ virtudoc update # all libraries
72
+ virtudoc update --name mylib
73
+
74
+ # Full rebuild of one library (applies changed chunk.* settings)
75
+ virtudoc reindex mylib
76
+
77
+ # Remove a library
78
+ virtudoc remove mylib
79
+
80
+ # Attach a team convention note to a library (agents receive it with docs)
81
+ virtudoc note add mylib --text "Use the retry helper for all outbound calls."
82
+
83
+ # Warn when the indexed docs version diverges from the project dependency
84
+ virtudoc link mylib --project /path/to/consuming/project
85
+ virtudoc doctor # config, sources, index integrity, version drift summary
86
+
87
+ # Pull public documentation from a git URL or archive (the only networked command)
88
+ virtudoc fetch --repo https://github.com/org/docs --name theirlib
89
+
90
+ # Move your whole setup to another machine (see docs/installation.md)
91
+ virtudoc export # dump the library list to ~/.virtudoc/libraries.toml
92
+ virtudoc import # register and index everything from the file
93
+ ```
94
+
95
+ ## Connect your agent
96
+
97
+ Run the MCP server and register it in your agent's config. The server speaks
98
+ stdio by default and reindexes changed sources on every start, so the agent
99
+ always sees the current documentation. Agents spawn the server with a
100
+ default environment, so the data directory is `~/.virtudoc` — set
101
+ `VIRTUDOC_HOME` in the server `env` block if you need a custom location.
102
+
103
+ **Claude Code** — add to `.mcp.json` in your project (or run
104
+ `claude mcp add virtudoc --scope user -- virtudoc serve`):
105
+
106
+ ```json
107
+ {
108
+ "mcpServers": {
109
+ "virtudoc": {
110
+ "command": "virtudoc",
111
+ "args": ["serve"]
112
+ }
113
+ }
114
+ }
115
+ ```
116
+
117
+ **Cursor** — add to `.cursor/mcp.json`:
118
+
119
+ ```json
120
+ {
121
+ "mcpServers": {
122
+ "virtudoc": {
123
+ "command": "virtudoc",
124
+ "args": ["serve"]
125
+ }
126
+ }
127
+ }
128
+ ```
129
+
130
+ **OpenCode** — add to `opencode.json` (project root or
131
+ `~/.config/opencode/opencode.json`):
132
+
133
+ ```json
134
+ {
135
+ "$schema": "https://opencode.ai/config.json",
136
+ "mcp": {
137
+ "virtudoc": {
138
+ "type": "local",
139
+ "command": ["virtudoc", "serve"],
140
+ "enabled": true
141
+ }
142
+ }
143
+ }
144
+ ```
145
+
146
+ ## MCP tools
147
+
148
+ - `find-library(libraryName, query)` — find the `local/<name>` id of a
149
+ library by its name and your question. Call it first.
150
+ - `get-library-docs(libraryId, query, tokens=5000, language, page, mode,
151
+ section)` — Markdown fragments with heading paths and source file paths,
152
+ assembled under the token budget. Relevant project notes are included
153
+ before the fragments. Empty results return the library's top-level
154
+ sections to help rephrase the query.
155
+
156
+ Optionally expose the server over streamable HTTP on 127.0.0.1 for clients
157
+ without stdio support: `virtudoc serve --http [port]` (default port 8765).
158
+
159
+ Supported formats: Markdown (`.md`, `.mdx`), reStructuredText (`.rst`),
160
+ plain text (`.txt`), HTML (`.html`, `.htm`, best-effort text extraction).
161
+ Hidden files and directories are always skipped; a `.virtudocignore` file at
162
+ the source root (`.gitignore`-compatible syntax: globs, `**`, `!` negation,
163
+ directory patterns) excludes generated or vendor trees from indexing.
164
+
165
+ ## How it works
166
+
167
+ - The **chunker** splits documents along H2/H3 headings (target chunk size
168
+ 600 tokens, hard limit 1200, 60-token overlap); code blocks are never split.
169
+ - **Search** is SQLite FTS5 with BM25 ranking. Queries cascade:
170
+ exact `AND` → prefix `AND` with ending stemming → `OR`, so Russian
171
+ inflections (`пакетная` finds `пакетной`) and fuzzy phrasing both resolve.
172
+ - Results are assembled under a token budget (default 5000) with a
173
+ topic boost for chunks whose heading path matches the query.
174
+ - **Indexing is incremental**: unchanged files are skipped by content hash.
175
+ - All data lives in `~/.virtudoc/` (WAL-mode `index.db`, config, log).
176
+ Deleting the directory removes the system completely.
177
+
178
+ ## Configuration
179
+
180
+ `~/.virtudoc/config.toml` (all keys optional):
181
+
182
+ ```toml
183
+ [chunk]
184
+ target_tokens = 600
185
+ max_tokens = 1200
186
+ overlap_tokens = 60
187
+
188
+ [search]
189
+ default_tokens = 5000
190
+ max_results = 20
191
+
192
+ [log]
193
+ level = "INFO"
194
+ ```
195
+
196
+ Changing `chunk.*` only affects new indexing; the existing index is not
197
+ rebuilt automatically.
198
+
199
+ ## Documentation
200
+
201
+ - [docs/](docs/index.md) — full documentation: installation, getting
202
+ started, CLI reference, MCP integration, configuration, indexing and
203
+ search internals, project notes, version drift warnings
204
+ - [ARCHITECTURE.md](ARCHITECTURE.md) — components, data flows, storage schema
205
+
206
+ ## Development
207
+
208
+ ```bash
209
+ pip install -e '.[dev]'
210
+ pytest
211
+ ruff check virtudoc tests
212
+ ruff format virtudoc tests
213
+ ```
214
+
215
+ ## Status
216
+
217
+ Milestones M1 (core: indexing, chunking, FTS5 search, CLI) and M2 (MCP
218
+ server over stdio/HTTP, project notes, outline mode, `.virtudocignore`) are
219
+ done, plus `link`/`doctor` version drift warnings and the `fetch` command.
220
+ Published on PyPI; a demo screencast is planned.
221
+
222
+ ## License
223
+
224
+ MIT, see [LICENSE](LICENSE).
@@ -0,0 +1,192 @@
1
+ # VirtuDoc
2
+
3
+ [![CI](https://github.com/saromanov/virtudoc/actions/workflows/ci.yml/badge.svg)](https://github.com/saromanov/virtudoc/actions/workflows/ci.yml)
4
+ [![PyPI](https://img.shields.io/pypi/v/virtudoc)](https://pypi.org/project/virtudoc/)
5
+ [![Python versions](https://img.shields.io/pypi/pyversions/virtudoc)](https://pypi.org/project/virtudoc/)
6
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/saromanov/virtudoc/blob/main/LICENSE)
7
+
8
+ A local documentation server for AI agents. It indexes documentation from
9
+ local directories into SQLite with FTS5 full-text search and serves relevant
10
+ fragments to an agent through MCP tools. Fully offline: no network access
11
+ happens in any mode.
12
+
13
+ Think of it as [Upstash Context7](https://github.com/upstash/context7) running
14
+ entirely on your machine: private and internal docs never leave your computer,
15
+ there are no API keys, request limits, or cloud dependencies.
16
+
17
+ ## Installation
18
+
19
+ ```bash
20
+ pip install virtudoc
21
+ ```
22
+
23
+ Requires Python 3.11+. The only runtime dependencies are `markdown-it-py`
24
+ (Markdown parsing) and the `mcp` SDK.
25
+
26
+ ## Quick start
27
+
28
+ ```bash
29
+ # Register a documentation directory and index it
30
+ virtudoc add /path/to/docs --name mylib --version 2.1.0
31
+
32
+ # Index statistics
33
+ virtudoc list
34
+
35
+ # Search from the terminal (same results an agent will get via MCP)
36
+ virtudoc search "how to handle authentication errors"
37
+
38
+ # Incremental reindexing after the docs changed
39
+ virtudoc update # all libraries
40
+ virtudoc update --name mylib
41
+
42
+ # Full rebuild of one library (applies changed chunk.* settings)
43
+ virtudoc reindex mylib
44
+
45
+ # Remove a library
46
+ virtudoc remove mylib
47
+
48
+ # Attach a team convention note to a library (agents receive it with docs)
49
+ virtudoc note add mylib --text "Use the retry helper for all outbound calls."
50
+
51
+ # Warn when the indexed docs version diverges from the project dependency
52
+ virtudoc link mylib --project /path/to/consuming/project
53
+ virtudoc doctor # config, sources, index integrity, version drift summary
54
+
55
+ # Pull public documentation from a git URL or archive (the only networked command)
56
+ virtudoc fetch --repo https://github.com/org/docs --name theirlib
57
+
58
+ # Move your whole setup to another machine (see docs/installation.md)
59
+ virtudoc export # dump the library list to ~/.virtudoc/libraries.toml
60
+ virtudoc import # register and index everything from the file
61
+ ```
62
+
63
+ ## Connect your agent
64
+
65
+ Run the MCP server and register it in your agent's config. The server speaks
66
+ stdio by default and reindexes changed sources on every start, so the agent
67
+ always sees the current documentation. Agents spawn the server with a
68
+ default environment, so the data directory is `~/.virtudoc` — set
69
+ `VIRTUDOC_HOME` in the server `env` block if you need a custom location.
70
+
71
+ **Claude Code** — add to `.mcp.json` in your project (or run
72
+ `claude mcp add virtudoc --scope user -- virtudoc serve`):
73
+
74
+ ```json
75
+ {
76
+ "mcpServers": {
77
+ "virtudoc": {
78
+ "command": "virtudoc",
79
+ "args": ["serve"]
80
+ }
81
+ }
82
+ }
83
+ ```
84
+
85
+ **Cursor** — add to `.cursor/mcp.json`:
86
+
87
+ ```json
88
+ {
89
+ "mcpServers": {
90
+ "virtudoc": {
91
+ "command": "virtudoc",
92
+ "args": ["serve"]
93
+ }
94
+ }
95
+ }
96
+ ```
97
+
98
+ **OpenCode** — add to `opencode.json` (project root or
99
+ `~/.config/opencode/opencode.json`):
100
+
101
+ ```json
102
+ {
103
+ "$schema": "https://opencode.ai/config.json",
104
+ "mcp": {
105
+ "virtudoc": {
106
+ "type": "local",
107
+ "command": ["virtudoc", "serve"],
108
+ "enabled": true
109
+ }
110
+ }
111
+ }
112
+ ```
113
+
114
+ ## MCP tools
115
+
116
+ - `find-library(libraryName, query)` — find the `local/<name>` id of a
117
+ library by its name and your question. Call it first.
118
+ - `get-library-docs(libraryId, query, tokens=5000, language, page, mode,
119
+ section)` — Markdown fragments with heading paths and source file paths,
120
+ assembled under the token budget. Relevant project notes are included
121
+ before the fragments. Empty results return the library's top-level
122
+ sections to help rephrase the query.
123
+
124
+ Optionally expose the server over streamable HTTP on 127.0.0.1 for clients
125
+ without stdio support: `virtudoc serve --http [port]` (default port 8765).
126
+
127
+ Supported formats: Markdown (`.md`, `.mdx`), reStructuredText (`.rst`),
128
+ plain text (`.txt`), HTML (`.html`, `.htm`, best-effort text extraction).
129
+ Hidden files and directories are always skipped; a `.virtudocignore` file at
130
+ the source root (`.gitignore`-compatible syntax: globs, `**`, `!` negation,
131
+ directory patterns) excludes generated or vendor trees from indexing.
132
+
133
+ ## How it works
134
+
135
+ - The **chunker** splits documents along H2/H3 headings (target chunk size
136
+ 600 tokens, hard limit 1200, 60-token overlap); code blocks are never split.
137
+ - **Search** is SQLite FTS5 with BM25 ranking. Queries cascade:
138
+ exact `AND` → prefix `AND` with ending stemming → `OR`, so Russian
139
+ inflections (`пакетная` finds `пакетной`) and fuzzy phrasing both resolve.
140
+ - Results are assembled under a token budget (default 5000) with a
141
+ topic boost for chunks whose heading path matches the query.
142
+ - **Indexing is incremental**: unchanged files are skipped by content hash.
143
+ - All data lives in `~/.virtudoc/` (WAL-mode `index.db`, config, log).
144
+ Deleting the directory removes the system completely.
145
+
146
+ ## Configuration
147
+
148
+ `~/.virtudoc/config.toml` (all keys optional):
149
+
150
+ ```toml
151
+ [chunk]
152
+ target_tokens = 600
153
+ max_tokens = 1200
154
+ overlap_tokens = 60
155
+
156
+ [search]
157
+ default_tokens = 5000
158
+ max_results = 20
159
+
160
+ [log]
161
+ level = "INFO"
162
+ ```
163
+
164
+ Changing `chunk.*` only affects new indexing; the existing index is not
165
+ rebuilt automatically.
166
+
167
+ ## Documentation
168
+
169
+ - [docs/](docs/index.md) — full documentation: installation, getting
170
+ started, CLI reference, MCP integration, configuration, indexing and
171
+ search internals, project notes, version drift warnings
172
+ - [ARCHITECTURE.md](ARCHITECTURE.md) — components, data flows, storage schema
173
+
174
+ ## Development
175
+
176
+ ```bash
177
+ pip install -e '.[dev]'
178
+ pytest
179
+ ruff check virtudoc tests
180
+ ruff format virtudoc tests
181
+ ```
182
+
183
+ ## Status
184
+
185
+ Milestones M1 (core: indexing, chunking, FTS5 search, CLI) and M2 (MCP
186
+ server over stdio/HTTP, project notes, outline mode, `.virtudocignore`) are
187
+ done, plus `link`/`doctor` version drift warnings and the `fetch` command.
188
+ Published on PyPI; a demo screencast is planned.
189
+
190
+ ## License
191
+
192
+ MIT, see [LICENSE](LICENSE).
@@ -0,0 +1,62 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "virtudoc"
7
+ version = "0.1.0"
8
+ description = "Local documentation server for AI agents (MCP, SQLite FTS5)"
9
+ readme = "README.md"
10
+ requires-python = ">=3.11"
11
+ license = "MIT"
12
+ license-files = ["LICENSE"]
13
+ authors = [{ name = "Sergey Romanov", email = "xxsmotur@gmail.com" }]
14
+ keywords = ["mcp", "llm", "ai-agents", "documentation", "full-text-search", "sqlite", "fts5", "offline"]
15
+ classifiers = [
16
+ "Development Status :: 4 - Beta",
17
+ "Environment :: Console",
18
+ "Intended Audience :: Developers",
19
+ "Operating System :: OS Independent",
20
+ "Programming Language :: Python :: 3 :: Only",
21
+ "Programming Language :: Python :: 3.11",
22
+ "Programming Language :: Python :: 3.12",
23
+ "Programming Language :: Python :: 3.13",
24
+ "Programming Language :: Python :: 3.14",
25
+ "Topic :: Database",
26
+ "Topic :: Software Development :: Documentation",
27
+ "Topic :: Text Processing :: Indexing",
28
+ ]
29
+ dependencies = [
30
+ "markdown-it-py>=3.0,<4",
31
+ "mcp>=2.3,<3",
32
+ ]
33
+
34
+ [project.optional-dependencies]
35
+ dev = [
36
+ "pytest>=8",
37
+ ]
38
+
39
+ [project.scripts]
40
+ virtudoc = "virtudoc.cli:main"
41
+
42
+ [project.urls]
43
+ Homepage = "https://github.com/saromanov/virtudoc"
44
+ Documentation = "https://github.com/saromanov/virtudoc/tree/main/docs"
45
+ Source = "https://github.com/saromanov/virtudoc"
46
+ Tracker = "https://github.com/saromanov/virtudoc/issues"
47
+
48
+ [tool.setuptools.packages.find]
49
+ include = ["virtudoc*"]
50
+
51
+ [tool.ruff]
52
+ target-version = "py311"
53
+ line-length = 100
54
+
55
+ [tool.ruff.lint]
56
+ select = ["E", "F", "I", "UP", "RUF"]
57
+ # RUF001–003 flag intentional Cyrillic in user-facing strings and test data
58
+ # (Russian morphology support is a core feature).
59
+ ignore = ["RUF001", "RUF002", "RUF003"]
60
+
61
+ [tool.pytest.ini_options]
62
+ testpaths = ["tests"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,76 @@
1
+ from virtudoc.indexer.chunker import Chunker
2
+ from virtudoc.indexer.parsers import parse_markdown
3
+ from virtudoc.tokens import estimate_tokens
4
+
5
+
6
+ def make_chunker(target=50, max_tokens=100, overlap=10):
7
+ return Chunker(target_tokens=target, max_tokens=max_tokens, overlap_tokens=overlap)
8
+
9
+
10
+ def para(text):
11
+ return text
12
+
13
+
14
+ def test_chunks_split_by_headings():
15
+ text = "# Документ\n\n## Первый\n\n" + "слово " * 30 + "\n\n## Второй\n\n" + "другое " * 30
16
+ chunks = make_chunker().chunk(parse_markdown(text))
17
+ paths = [c.heading_path for c in chunks]
18
+ assert paths[0] == "Документ → Первый"
19
+ assert "Документ → Второй" in paths
20
+ assert paths == sorted(paths, key=paths.index)
21
+
22
+
23
+ def test_heading_path_nesting():
24
+ text = "# API\n\n## Middleware\n\n" + "текст " * 30 + "\n\n### Ошибки\n\n" + "ещё текст " * 20
25
+ chunks = make_chunker().chunk(parse_markdown(text))
26
+ assert chunks[0].heading_path == "API → Middleware"
27
+ assert chunks[-1].heading_path == "API → Middleware → Ошибки"
28
+
29
+
30
+ def test_target_budget_creates_multiple_chunks():
31
+ body = "\n\n".join(f"Параграф {i}: " + "а" * 120 for i in range(8))
32
+ text = f"# T\n\n## Раздел\n\n{body}\n"
33
+ chunks = make_chunker(target=100, max_tokens=200).chunk(parse_markdown(text))
34
+ assert len(chunks) >= 3
35
+ assert all(c.token_est <= 210 for c in chunks)
36
+ assert [c.ord for c in chunks] == list(range(len(chunks)))
37
+
38
+
39
+ def test_overlap_between_adjacent_chunks():
40
+ body = "\n\n".join(f"Абзац номер {i} " + "x" * 100 for i in range(8))
41
+ text = f"# T\n\n## Раздел\n\n{body}\n"
42
+ chunks = make_chunker(target=100, overlap=10).chunk(parse_markdown(text))
43
+ assert len(chunks) >= 2
44
+ second = chunks[1]
45
+ assert second.text != chunks[0].text
46
+
47
+
48
+ def test_code_block_not_split():
49
+ code = "\n".join(f"line_{i} = {i}" for i in range(40))
50
+ text = f"# T\n\n## Код\n\n```python\n{code}\n```\n"
51
+ chunks = make_chunker(target=30, max_tokens=200).chunk(parse_markdown(text))
52
+ code_chunks = [c for c in chunks if "line_39" in c.text]
53
+ assert len(code_chunks) == 1
54
+ assert "line_0" in code_chunks[0].text
55
+
56
+
57
+ def test_oversized_paragraph_split_with_marker():
58
+ big = "слово " * 400
59
+ text = f"# T\n\n{big}\n"
60
+ chunks = make_chunker(target=100, max_tokens=150).chunk(parse_markdown(text))
61
+ assert len(chunks) >= 2
62
+ assert all(c.token_est <= 160 for c in chunks)
63
+ assert any("(continued)" in c.text for c in chunks[1:])
64
+
65
+
66
+ def test_code_ratio():
67
+ code = "\n".join(f"x = {i}" for i in range(30))
68
+ text = f"# T\n\n## Раздел\n\nКороткий текст.\n\n```python\n{code}\n```\n"
69
+ chunks = make_chunker().chunk(parse_markdown(text))
70
+ assert any(c.code_ratio > 0.3 for c in chunks)
71
+
72
+
73
+ def test_token_est_matches_text():
74
+ text = "# T\n\n## Раздел\n\n" + "текст " * 50 + "\n"
75
+ chunks = make_chunker().chunk(parse_markdown(text))
76
+ assert all(c.token_est == estimate_tokens(c.text) for c in chunks)