taskflow-doc-search 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- taskflow_doc_search-0.1.0/LICENSE +21 -0
- taskflow_doc_search-0.1.0/MANIFEST.in +3 -0
- taskflow_doc_search-0.1.0/PKG-INFO +137 -0
- taskflow_doc_search-0.1.0/PUBLISHING.md +84 -0
- taskflow_doc_search-0.1.0/README.md +120 -0
- taskflow_doc_search-0.1.0/doc_search/__init__.py +1 -0
- taskflow_doc_search-0.1.0/doc_search/embeddings.py +192 -0
- taskflow_doc_search-0.1.0/doc_search/server.py +336 -0
- taskflow_doc_search-0.1.0/pyproject.toml +30 -0
- taskflow_doc_search-0.1.0/requirements-embeddings.txt +2 -0
- taskflow_doc_search-0.1.0/requirements.txt +1 -0
- taskflow_doc_search-0.1.0/scripts/smoke_installed.py +54 -0
- taskflow_doc_search-0.1.0/server.json +36 -0
- taskflow_doc_search-0.1.0/server.py +6 -0
- taskflow_doc_search-0.1.0/setup.cfg +4 -0
- taskflow_doc_search-0.1.0/taskflow_doc_search.egg-info/PKG-INFO +137 -0
- taskflow_doc_search-0.1.0/taskflow_doc_search.egg-info/SOURCES.txt +22 -0
- taskflow_doc_search-0.1.0/taskflow_doc_search.egg-info/dependency_links.txt +1 -0
- taskflow_doc_search-0.1.0/taskflow_doc_search.egg-info/entry_points.txt +2 -0
- taskflow_doc_search-0.1.0/taskflow_doc_search.egg-info/requires.txt +7 -0
- taskflow_doc_search-0.1.0/taskflow_doc_search.egg-info/top_level.txt +1 -0
- taskflow_doc_search-0.1.0/tests/test_embeddings.py +114 -0
- taskflow_doc_search-0.1.0/tests/test_packaging.py +18 -0
- taskflow_doc_search-0.1.0/tests/test_server.py +160 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Cynthia Wang
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: taskflow-doc-search
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: MCP server for keyword search over the ai-workspace-kit documentation repos and demo tickets
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Repository, https://github.com/cynthiawang109-sudo/ai-workspace-kit
|
|
7
|
+
Project-URL: Documentation, https://github.com/cynthiawang109-sudo/ai-workspace-kit/tree/main/mcp/doc-search
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Dist: fastmcp>=4.1.7
|
|
12
|
+
Provides-Extra: dev
|
|
13
|
+
Requires-Dist: pytest>=9.2.0; extra == "dev"
|
|
14
|
+
Provides-Extra: embeddings
|
|
15
|
+
Requires-Dist: sentence-transformers>=3; extra == "embeddings"
|
|
16
|
+
Dynamic: license-file
|
|
17
|
+
|
|
18
|
+
# doc-search MCP server
|
|
19
|
+
|
|
20
|
+
A small [MCP](https://modelcontextprotocol.io) server written with FastMCP.
|
|
21
|
+
It gives an AI coding agent structured access to this workspace's
|
|
22
|
+
documentation and demo tickets — the local, public stand-in for the
|
|
23
|
+
Jira/Confluence side of the original production workspace. Five tools are
|
|
24
|
+
annotated read-only (embeddings maintains a derived local cache); the sixth can only write ticket *drafts* (see below).
|
|
25
|
+
|
|
26
|
+
## Tools
|
|
27
|
+
|
|
28
|
+
| Tool | What it does |
|
|
29
|
+
|---|---|
|
|
30
|
+
| `search_docs(query, limit=8)` | Keyword search across every Markdown doc under `projects/`; title hits ×3, heading hits ×2, body hits ×1; returns path, title, score, snippet |
|
|
31
|
+
| `search_docs_semantic(query, limit=8)` | TF-IDF vector-space search over the same corpus (cosine similarity); returns path, title, score (0–1), snippet |
|
|
32
|
+
| `search_docs_embeddings(query, limit=8)` | Sentence embeddings over heading chunks; returns path, title, cosine score, snippet, heading; caches in `.index/` |
|
|
33
|
+
| `get_doc(path)` | Fetch one document by workspace-relative path (only `projects/` and `tickets/` are readable) |
|
|
34
|
+
| `list_tickets(status?)` | List the pre-TF-115 archive from `tickets/*.md` front matter, optionally filtered by status |
|
|
35
|
+
| `propose_ticket(title, body, priority="medium")` | Write a ticket **draft** to `tickets/drafts/draft-<slug>.md` — the server's only source-content write; it cannot create or modify real tickets |
|
|
36
|
+
|
|
37
|
+
## Draft-first writes
|
|
38
|
+
|
|
39
|
+
`propose_ticket` mirrors the production workspace's rule for outward
|
|
40
|
+
writes (there: Jira changes start as drafts a human approves). The
|
|
41
|
+
boundary is enforced by the tool, not by politeness: it writes only
|
|
42
|
+
new files under `tickets/drafts/`, always with `status: draft`, it
|
|
43
|
+
refuses to overwrite an existing draft, and it is annotated as a
|
|
44
|
+
write, so clients ask the user before calling it. `tickets/` is the pre-TF-115
|
|
45
|
+
archive; `list_tickets` never shows drafts as tickets. A human reviews and
|
|
46
|
+
promotes a draft with `scripts/promote-draft.ps1` (or `.sh`), which files
|
|
47
|
+
an Issue instead of moving the file into `tickets/`. Set `AI_WORKSPACE_PROJECT`
|
|
48
|
+
to the workspace Project number, or pass `-Project N` / `--project N`, to
|
|
49
|
+
also add the Issue to the board. Without a Project, the script prints the
|
|
50
|
+
Issue URL and notes that it was not added to a board. The draft is preserved.
|
|
51
|
+
The doc-search server stays credential-free and never calls GitHub.
|
|
52
|
+
|
|
53
|
+
## Three rungs of document search
|
|
54
|
+
|
|
55
|
+
1. `search_docs` counts exact term hits with title/heading/body weights.
|
|
56
|
+
It is predictable for known terms, but repetition can dominate ranking.
|
|
57
|
+
2. `search_docs_semantic` uses TF-IDF cosine similarity over whole documents.
|
|
58
|
+
It down-weights common terms and balances multi-term queries. Both of
|
|
59
|
+
these first two rungs are bag-of-words models: neither understands synonyms.
|
|
60
|
+
They use only the standard library and rebuild their vectors per call.
|
|
61
|
+
3. `search_docs_embeddings` uses sentence-transformers over ATX heading
|
|
62
|
+
sections, embedding the document title, heading trail and body. It can
|
|
63
|
+
retrieve paraphrases and synonyms, but relevance is model-dependent.
|
|
64
|
+
Cosine ranking uses pure Python. A content-hash fingerprint invalidates
|
|
65
|
+
the derived `.index/` cache when documents or the model change.
|
|
66
|
+
|
|
67
|
+
- **Local documents**: searches never modify the source documents. Embeddings
|
|
68
|
+
writes a derived cache; `propose_ticket` writes drafts under the rules above.
|
|
69
|
+
- **No credentials**: keyword and TF-IDF need no network. Embeddings downloads
|
|
70
|
+
its model once, then runs fully locally.
|
|
71
|
+
- **Client-agnostic**: plain MCP over stdio.
|
|
72
|
+
|
|
73
|
+
## Setup
|
|
74
|
+
|
|
75
|
+
```powershell
|
|
76
|
+
cd mcp\doc-search
|
|
77
|
+
python -m venv .venv
|
|
78
|
+
.venv\Scripts\python -m pip install -r requirements.txt pytest
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Optional embeddings dependencies (includes torch):
|
|
82
|
+
|
|
83
|
+
```powershell
|
|
84
|
+
.venv\Scripts\python -m pip install -r requirements-embeddings.txt
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
`DOC_SEARCH_EMBEDDING_MODEL` overrides the default
|
|
88
|
+
`sentence-transformers/all-MiniLM-L6-v2`. `DOC_SEARCH_EMBEDDER=fake` selects
|
|
89
|
+
only a deterministic hashed-token test double, without semantic understanding.
|
|
90
|
+
|
|
91
|
+
Evaluate the synced corpus with `python scripts/run_eval.py`; add `--write`
|
|
92
|
+
to record the dated results in [EVAL.md](EVAL.md). The eval is not run in CI.
|
|
93
|
+
|
|
94
|
+
## Run the tests
|
|
95
|
+
|
|
96
|
+
```powershell
|
|
97
|
+
.venv\Scripts\python -m pytest
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
The tests build a throwaway fixture workspace, so they pass whether or not
|
|
101
|
+
`sync` has populated `projects/`.
|
|
102
|
+
|
|
103
|
+
## Register it
|
|
104
|
+
|
|
105
|
+
**Codex** — add to `~/.codex/config.toml`:
|
|
106
|
+
|
|
107
|
+
```toml
|
|
108
|
+
[mcp_servers.doc-search]
|
|
109
|
+
command = 'C:\path\to\ai-workspace-kit\mcp\doc-search\.venv\Scripts\python.exe'
|
|
110
|
+
args = ['C:\path\to\ai-workspace-kit\mcp\doc-search\server.py']
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
**Claude Code**:
|
|
114
|
+
|
|
115
|
+
```powershell
|
|
116
|
+
claude mcp add doc-search -- C:\path\to\ai-workspace-kit\mcp\doc-search\.venv\Scripts\python.exe C:\path\to\ai-workspace-kit\mcp\doc-search\server.py
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
The server finds the workspace root from its own location
|
|
120
|
+
(`<root>/mcp/doc-search/server.py`); set `AI_WORKSPACE_ROOT` to override.
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
## Install the distribution
|
|
124
|
+
|
|
125
|
+
Once published, install with `pip install taskflow-doc-search` (or
|
|
126
|
+
`pip install "taskflow-doc-search[embeddings]"` for sentence embeddings).
|
|
127
|
+
For a local checkout, `pip install .` from this directory works today.
|
|
128
|
+
Run `taskflow-doc-search` with `AI_WORKSPACE_ROOT` set to the absolute path
|
|
129
|
+
of the workspace to search. It serves MCP over stdio; no command arguments
|
|
130
|
+
are needed. The existing `python server.py` launcher and its automatic
|
|
131
|
+
workspace-root detection remain supported.
|
|
132
|
+
|
|
133
|
+
Release instructions: [PUBLISHING.md](PUBLISHING.md).
|
|
134
|
+
The included [server.json](server.json) is prepared for an optional MCP
|
|
135
|
+
registry listing and has not been published.
|
|
136
|
+
|
|
137
|
+
<!-- mcp-name: io.github.cynthiawang109-sudo/taskflow-doc-search -->
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# Publishing taskflow-doc-search
|
|
2
|
+
|
|
3
|
+
Version 0.1.0 is prepared locally; neither PyPI nor MCP registry publication
|
|
4
|
+
has been performed. The PyPI JSON endpoint returned HTTP 404 when preparing
|
|
5
|
+
this package; availability is not a reservation.
|
|
6
|
+
|
|
7
|
+
The project is licensed under MIT; see the root [LICENSE](../../LICENSE).
|
|
8
|
+
The distribution metadata in `pyproject.toml` declares `MIT`.
|
|
9
|
+
|
|
10
|
+
## One-time owner setup
|
|
11
|
+
|
|
12
|
+
1. Sign in to pypi.org and open
|
|
13
|
+
[Publishing](https://pypi.org/manage/account/publishing/). Add a **pending
|
|
14
|
+
publisher** using GitHub: project `taskflow-doc-search`, owner
|
|
15
|
+
`cynthiawang109-sudo`, repository `ai-workspace-kit`, workflow filename
|
|
16
|
+
`publish-doc-search.yml`, environment `pypi`.
|
|
17
|
+
2. Create the GitHub repository environment `pypi`. Configure release branch/tag
|
|
18
|
+
restrictions and any required reviewers appropriate for the repository.
|
|
19
|
+
The environment name must exactly match the PyPI publisher configuration.
|
|
20
|
+
|
|
21
|
+
The [Trusted Publishing workflow](../../.github/workflows/publish-doc-search.yml)
|
|
22
|
+
uses GitHub OIDC and `id-token: write` only in the publishing job. No PyPI API
|
|
23
|
+
tokens or stored credentials are needed. See the official
|
|
24
|
+
[PyPI Trusted Publishing guide](https://docs.pypi.org/trusted-publishers/using-a-publisher/).
|
|
25
|
+
The first successful upload creates the pending PyPI project.
|
|
26
|
+
|
|
27
|
+
## Local build and inspection
|
|
28
|
+
|
|
29
|
+
From `mcp/doc-search`, using a development virtual environment:
|
|
30
|
+
|
|
31
|
+
```sh
|
|
32
|
+
python -m pip install ".[dev]" build twine
|
|
33
|
+
python -m pytest -q
|
|
34
|
+
python -m build
|
|
35
|
+
python -m twine check dist/*
|
|
36
|
+
python -m zipfile -l dist/taskflow_doc_search-0.1.0-py3-none-any.whl
|
|
37
|
+
python -m tarfile -l dist/taskflow_doc_search-0.1.0.tar.gz
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Inspect metadata for name, version, license, Python requirement, base dependency,
|
|
41
|
+
`embeddings` extra, README, and `taskflow-doc-search` entry point. Confirm both
|
|
42
|
+
archives contain `doc_search/`; the sdist must contain README and tests.
|
|
43
|
+
Install the wheel into a fresh virtual environment outside the checkout. From
|
|
44
|
+
outside the repository, set `AI_WORKSPACE_ROOT` to the absolute workspace path,
|
|
45
|
+
start `taskflow-doc-search`, and use an MCP client to initialize, list all six
|
|
46
|
+
tools, and call `search_docs` with a known corpus term. Run the included
|
|
47
|
+
client with the fresh environment's Python:
|
|
48
|
+
|
|
49
|
+
```sh
|
|
50
|
+
python /path/to/mcp/doc-search/scripts/smoke_installed.py /path/to/venv/bin/taskflow-doc-search /path/to/workspace
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
On Windows the executable is `venv/Scripts/taskflow-doc-search.exe`.
|
|
54
|
+
The client checks all six advertised tools and calls the five read-only tools;
|
|
55
|
+
its embeddings call uses an empty-token query, avoiding downloads and cache
|
|
56
|
+
changes. Embeddings ranking itself is covered by the unit tests with a fake
|
|
57
|
+
embedder. No model download is required for keyword search. Remove the
|
|
58
|
+
temporary environment and build output.
|
|
59
|
+
|
|
60
|
+
## Cut a release
|
|
61
|
+
|
|
62
|
+
1. Update the version in `pyproject.toml` and both version fields in
|
|
63
|
+
`server.json`; complete the local checks above and merge the approved change.
|
|
64
|
+
2. Create a version tag on the intended commit (for example
|
|
65
|
+
`doc-search-v0.1.0`) and **publish a GitHub Release** for it.
|
|
66
|
+
3. The workflow runs tests, builds and checks sdist/wheel, then publishes via
|
|
67
|
+
OIDC after any environment review. Inspect the workflow and PyPI files.
|
|
68
|
+
Ordinary pushes do not publish. Any published GitHub Release triggers this
|
|
69
|
+
workflow, so coordinate releases for other workspace components accordingly.
|
|
70
|
+
4. `workflow_dispatch` is an explicit manual publishing path for the selected
|
|
71
|
+
ref; it runs the same checks. Use only for a version not already uploaded:
|
|
72
|
+
PyPI does not permit replacing an existing release file.
|
|
73
|
+
|
|
74
|
+
## Optional MCP registry listing
|
|
75
|
+
|
|
76
|
+
After PyPI publication, validate the included [server.json](server.json) against
|
|
77
|
+
its pinned schema and follow the official
|
|
78
|
+
[MCP registry publishing instructions](https://github.com/modelcontextprotocol/registry/tree/main/docs).
|
|
79
|
+
Authenticate ownership of `io.github.cynthiawang109-sudo`, confirm the package
|
|
80
|
+
README's `mcp-name` ownership marker, and publish with the registry publisher.
|
|
81
|
+
The manifest describes PyPI installation using `uvx`, stdio transport, and the
|
|
82
|
+
required workspace environment variable. Its publisher metadata currently marks
|
|
83
|
+
it `prepared-not-yet-published`; update that status only after actual publication.
|
|
84
|
+
Registry listing is separate from the PyPI workflow and is not automatic.
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# doc-search MCP server
|
|
2
|
+
|
|
3
|
+
A small [MCP](https://modelcontextprotocol.io) server written with FastMCP.
|
|
4
|
+
It gives an AI coding agent structured access to this workspace's
|
|
5
|
+
documentation and demo tickets — the local, public stand-in for the
|
|
6
|
+
Jira/Confluence side of the original production workspace. Five tools are
|
|
7
|
+
annotated read-only (embeddings maintains a derived local cache); the sixth can only write ticket *drafts* (see below).
|
|
8
|
+
|
|
9
|
+
## Tools
|
|
10
|
+
|
|
11
|
+
| Tool | What it does |
|
|
12
|
+
|---|---|
|
|
13
|
+
| `search_docs(query, limit=8)` | Keyword search across every Markdown doc under `projects/`; title hits ×3, heading hits ×2, body hits ×1; returns path, title, score, snippet |
|
|
14
|
+
| `search_docs_semantic(query, limit=8)` | TF-IDF vector-space search over the same corpus (cosine similarity); returns path, title, score (0–1), snippet |
|
|
15
|
+
| `search_docs_embeddings(query, limit=8)` | Sentence embeddings over heading chunks; returns path, title, cosine score, snippet, heading; caches in `.index/` |
|
|
16
|
+
| `get_doc(path)` | Fetch one document by workspace-relative path (only `projects/` and `tickets/` are readable) |
|
|
17
|
+
| `list_tickets(status?)` | List the pre-TF-115 archive from `tickets/*.md` front matter, optionally filtered by status |
|
|
18
|
+
| `propose_ticket(title, body, priority="medium")` | Write a ticket **draft** to `tickets/drafts/draft-<slug>.md` — the server's only source-content write; it cannot create or modify real tickets |
|
|
19
|
+
|
|
20
|
+
## Draft-first writes
|
|
21
|
+
|
|
22
|
+
`propose_ticket` mirrors the production workspace's rule for outward
|
|
23
|
+
writes (there: Jira changes start as drafts a human approves). The
|
|
24
|
+
boundary is enforced by the tool, not by politeness: it writes only
|
|
25
|
+
new files under `tickets/drafts/`, always with `status: draft`, it
|
|
26
|
+
refuses to overwrite an existing draft, and it is annotated as a
|
|
27
|
+
write, so clients ask the user before calling it. `tickets/` is the pre-TF-115
|
|
28
|
+
archive; `list_tickets` never shows drafts as tickets. A human reviews and
|
|
29
|
+
promotes a draft with `scripts/promote-draft.ps1` (or `.sh`), which files
|
|
30
|
+
an Issue instead of moving the file into `tickets/`. Set `AI_WORKSPACE_PROJECT`
|
|
31
|
+
to the workspace Project number, or pass `-Project N` / `--project N`, to
|
|
32
|
+
also add the Issue to the board. Without a Project, the script prints the
|
|
33
|
+
Issue URL and notes that it was not added to a board. The draft is preserved.
|
|
34
|
+
The doc-search server stays credential-free and never calls GitHub.
|
|
35
|
+
|
|
36
|
+
## Three rungs of document search
|
|
37
|
+
|
|
38
|
+
1. `search_docs` counts exact term hits with title/heading/body weights.
|
|
39
|
+
It is predictable for known terms, but repetition can dominate ranking.
|
|
40
|
+
2. `search_docs_semantic` uses TF-IDF cosine similarity over whole documents.
|
|
41
|
+
It down-weights common terms and balances multi-term queries. Both of
|
|
42
|
+
these first two rungs are bag-of-words models: neither understands synonyms.
|
|
43
|
+
They use only the standard library and rebuild their vectors per call.
|
|
44
|
+
3. `search_docs_embeddings` uses sentence-transformers over ATX heading
|
|
45
|
+
sections, embedding the document title, heading trail and body. It can
|
|
46
|
+
retrieve paraphrases and synonyms, but relevance is model-dependent.
|
|
47
|
+
Cosine ranking uses pure Python. A content-hash fingerprint invalidates
|
|
48
|
+
the derived `.index/` cache when documents or the model change.
|
|
49
|
+
|
|
50
|
+
- **Local documents**: searches never modify the source documents. Embeddings
|
|
51
|
+
writes a derived cache; `propose_ticket` writes drafts under the rules above.
|
|
52
|
+
- **No credentials**: keyword and TF-IDF need no network. Embeddings downloads
|
|
53
|
+
its model once, then runs fully locally.
|
|
54
|
+
- **Client-agnostic**: plain MCP over stdio.
|
|
55
|
+
|
|
56
|
+
## Setup
|
|
57
|
+
|
|
58
|
+
```powershell
|
|
59
|
+
cd mcp\doc-search
|
|
60
|
+
python -m venv .venv
|
|
61
|
+
.venv\Scripts\python -m pip install -r requirements.txt pytest
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Optional embeddings dependencies (includes torch):
|
|
65
|
+
|
|
66
|
+
```powershell
|
|
67
|
+
.venv\Scripts\python -m pip install -r requirements-embeddings.txt
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
`DOC_SEARCH_EMBEDDING_MODEL` overrides the default
|
|
71
|
+
`sentence-transformers/all-MiniLM-L6-v2`. `DOC_SEARCH_EMBEDDER=fake` selects
|
|
72
|
+
only a deterministic hashed-token test double, without semantic understanding.
|
|
73
|
+
|
|
74
|
+
Evaluate the synced corpus with `python scripts/run_eval.py`; add `--write`
|
|
75
|
+
to record the dated results in [EVAL.md](EVAL.md). The eval is not run in CI.
|
|
76
|
+
|
|
77
|
+
## Run the tests
|
|
78
|
+
|
|
79
|
+
```powershell
|
|
80
|
+
.venv\Scripts\python -m pytest
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
The tests build a throwaway fixture workspace, so they pass whether or not
|
|
84
|
+
`sync` has populated `projects/`.
|
|
85
|
+
|
|
86
|
+
## Register it
|
|
87
|
+
|
|
88
|
+
**Codex** — add to `~/.codex/config.toml`:
|
|
89
|
+
|
|
90
|
+
```toml
|
|
91
|
+
[mcp_servers.doc-search]
|
|
92
|
+
command = 'C:\path\to\ai-workspace-kit\mcp\doc-search\.venv\Scripts\python.exe'
|
|
93
|
+
args = ['C:\path\to\ai-workspace-kit\mcp\doc-search\server.py']
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
**Claude Code**:
|
|
97
|
+
|
|
98
|
+
```powershell
|
|
99
|
+
claude mcp add doc-search -- C:\path\to\ai-workspace-kit\mcp\doc-search\.venv\Scripts\python.exe C:\path\to\ai-workspace-kit\mcp\doc-search\server.py
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
The server finds the workspace root from its own location
|
|
103
|
+
(`<root>/mcp/doc-search/server.py`); set `AI_WORKSPACE_ROOT` to override.
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
## Install the distribution
|
|
107
|
+
|
|
108
|
+
Once published, install with `pip install taskflow-doc-search` (or
|
|
109
|
+
`pip install "taskflow-doc-search[embeddings]"` for sentence embeddings).
|
|
110
|
+
For a local checkout, `pip install .` from this directory works today.
|
|
111
|
+
Run `taskflow-doc-search` with `AI_WORKSPACE_ROOT` set to the absolute path
|
|
112
|
+
of the workspace to search. It serves MCP over stdio; no command arguments
|
|
113
|
+
are needed. The existing `python server.py` launcher and its automatic
|
|
114
|
+
workspace-root detection remain supported.
|
|
115
|
+
|
|
116
|
+
Release instructions: [PUBLISHING.md](PUBLISHING.md).
|
|
117
|
+
The included [server.json](server.json) is prepared for an optional MCP
|
|
118
|
+
registry listing and has not been published.
|
|
119
|
+
|
|
120
|
+
<!-- mcp-name: io.github.cynthiawang109-sudo/taskflow-doc-search -->
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Documentation search tools served over MCP stdio."""
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
"""Section embeddings with a content-validated, local derived index.
|
|
2
|
+
|
|
3
|
+
Importing this module does not import sentence-transformers or torch.
|
|
4
|
+
The fake embedder is a deterministic test double, not a semantic model.
|
|
5
|
+
"""
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import math
|
|
9
|
+
import os
|
|
10
|
+
import re
|
|
11
|
+
from functools import lru_cache
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from .server import ROOT
|
|
15
|
+
|
|
16
|
+
# Keep the checkout cache beside the compatibility launcher, even when the
|
|
17
|
+
# workspace is overridden. Installed packages cache under the workspace,
|
|
18
|
+
# avoiding writes to site-packages.
|
|
19
|
+
_SERVER_DIR = Path(__file__).resolve().parents[1]
|
|
20
|
+
INDEX_DIR = (
|
|
21
|
+
_SERVER_DIR if (_SERVER_DIR / "server.py").is_file()
|
|
22
|
+
else ROOT / "mcp" / "doc-search"
|
|
23
|
+
) / ".index"
|
|
24
|
+
DEFAULT_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class _FakeEmbedder:
|
|
28
|
+
def encode(self, texts):
|
|
29
|
+
from .server import _tokenize
|
|
30
|
+
|
|
31
|
+
vectors = []
|
|
32
|
+
for text in texts:
|
|
33
|
+
vector = [0.0] * 64
|
|
34
|
+
for token in _tokenize(text):
|
|
35
|
+
slot = int(hashlib.sha256(token.encode("utf-8")).hexdigest(), 16) % 64
|
|
36
|
+
vector[slot] += 1.0
|
|
37
|
+
norm = math.sqrt(sum(v * v for v in vector))
|
|
38
|
+
vectors.append([v / norm for v in vector] if norm else vector)
|
|
39
|
+
return vectors
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class _RealEmbedder:
|
|
43
|
+
def __init__(self, model):
|
|
44
|
+
try:
|
|
45
|
+
from sentence_transformers import SentenceTransformer
|
|
46
|
+
except ImportError as exc:
|
|
47
|
+
raise RuntimeError(
|
|
48
|
+
"sentence-transformers is not installed; "
|
|
49
|
+
"install the optional embeddings dependencies with: "
|
|
50
|
+
"pip install taskflow-doc-search[embeddings]"
|
|
51
|
+
) from exc
|
|
52
|
+
self.model = SentenceTransformer(model)
|
|
53
|
+
|
|
54
|
+
def encode(self, texts):
|
|
55
|
+
return self.model.encode(texts, normalize_embeddings=True).tolist()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@lru_cache(maxsize=2)
|
|
59
|
+
def _embedder(name, model):
|
|
60
|
+
if name == "fake":
|
|
61
|
+
return _FakeEmbedder()
|
|
62
|
+
if name != "sentence-transformers":
|
|
63
|
+
raise ValueError(f"unknown DOC_SEARCH_EMBEDDER: {name}")
|
|
64
|
+
return _RealEmbedder(model)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def get_embedder():
|
|
68
|
+
"""Return the configured embedder, loading the real model only on demand."""
|
|
69
|
+
return _embedder(
|
|
70
|
+
os.environ.get("DOC_SEARCH_EMBEDDER", "sentence-transformers"),
|
|
71
|
+
os.environ.get("DOC_SEARCH_EMBEDDING_MODEL", DEFAULT_MODEL),
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def chunk_document(path, text):
|
|
76
|
+
"""Split ATX sections into body chunks carrying their full heading trail.
|
|
77
|
+
|
|
78
|
+
The preamble uses the document title. Empty section bodies are retained
|
|
79
|
+
so a heading alone remains searchable. Fenced code is treated as body.
|
|
80
|
+
"""
|
|
81
|
+
from .server import ROOT, _doc_title
|
|
82
|
+
|
|
83
|
+
path = Path(path)
|
|
84
|
+
title = _doc_title(text, path)
|
|
85
|
+
relative = path.relative_to(ROOT).as_posix()
|
|
86
|
+
chunks, trail, body = [], [], []
|
|
87
|
+
heading = title
|
|
88
|
+
fence = None
|
|
89
|
+
|
|
90
|
+
def flush():
|
|
91
|
+
chunks.append({"path": relative, "title": title,
|
|
92
|
+
"heading": heading, "text": "\n".join(body).strip()})
|
|
93
|
+
|
|
94
|
+
for line in text.splitlines():
|
|
95
|
+
marker = re.match(r"^ {0,3}(`{3,}|~{3,})(.*)$", line)
|
|
96
|
+
if marker:
|
|
97
|
+
run, tail = marker.groups()
|
|
98
|
+
if fence is None:
|
|
99
|
+
fence = run
|
|
100
|
+
elif run[0] == fence[0] and len(run) >= len(fence) and not tail.strip():
|
|
101
|
+
fence = None
|
|
102
|
+
body.append(line)
|
|
103
|
+
continue
|
|
104
|
+
match = None if fence else re.match(r"^ {0,3}(#{1,6})(?:[ \t]+(.*)|$)", line)
|
|
105
|
+
if match:
|
|
106
|
+
if trail or any(part.strip() for part in body):
|
|
107
|
+
flush()
|
|
108
|
+
level = len(match[1])
|
|
109
|
+
label = re.sub(r"[ \t]+#+[ \t]*$", "", match[2] or "").strip()
|
|
110
|
+
trail = [(n, label_) for n, label_ in trail if n < level]
|
|
111
|
+
trail.append((level, label))
|
|
112
|
+
heading = " > ".join(label_ for _, label_ in trail)
|
|
113
|
+
body = []
|
|
114
|
+
else:
|
|
115
|
+
body.append(line)
|
|
116
|
+
if trail or any(part.strip() for part in body):
|
|
117
|
+
flush()
|
|
118
|
+
return chunks
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def build_index(documents, manifest):
|
|
122
|
+
"""Embed a byte snapshot of the corpus and persist its derived cache."""
|
|
123
|
+
chunks = []
|
|
124
|
+
for path, content in documents:
|
|
125
|
+
chunks.extend(chunk_document(path, content.decode("utf-8", errors="replace")))
|
|
126
|
+
if chunks:
|
|
127
|
+
vectors = get_embedder().encode([
|
|
128
|
+
f"{c['title']} > {c['heading']}\n{c['text']}" for c in chunks
|
|
129
|
+
])
|
|
130
|
+
for chunk, vector in zip(chunks, vectors):
|
|
131
|
+
chunk["embedding"] = vector
|
|
132
|
+
INDEX_DIR.mkdir(parents=True, exist_ok=True)
|
|
133
|
+
(INDEX_DIR / "chunks.jsonl").write_text(
|
|
134
|
+
"".join(json.dumps(c, ensure_ascii=False) + "\n" for c in chunks),
|
|
135
|
+
encoding="utf-8",
|
|
136
|
+
)
|
|
137
|
+
(INDEX_DIR / "manifest.json").write_text(
|
|
138
|
+
json.dumps({**manifest, "chunk_count": len(chunks)}, indent=2), encoding="utf-8"
|
|
139
|
+
)
|
|
140
|
+
return chunks
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def load_index():
|
|
144
|
+
"""Reuse the cache only when corpus bytes, model and embedder match."""
|
|
145
|
+
from .server import ROOT, _iter_docs
|
|
146
|
+
|
|
147
|
+
documents = [(p, p.read_bytes()) for p in _iter_docs()]
|
|
148
|
+
hashes = sorted((p.relative_to(ROOT).as_posix(), hashlib.sha256(b).hexdigest())
|
|
149
|
+
for p, b in documents)
|
|
150
|
+
manifest = {
|
|
151
|
+
"model": os.environ.get("DOC_SEARCH_EMBEDDING_MODEL", DEFAULT_MODEL),
|
|
152
|
+
"embedder": os.environ.get("DOC_SEARCH_EMBEDDER", "sentence-transformers"),
|
|
153
|
+
"fingerprint": hashlib.sha256(json.dumps(hashes).encode("utf-8")).hexdigest(),
|
|
154
|
+
}
|
|
155
|
+
try:
|
|
156
|
+
saved = json.loads((INDEX_DIR / "manifest.json").read_text(encoding="utf-8"))
|
|
157
|
+
if all(saved.get(k) == v for k, v in manifest.items()):
|
|
158
|
+
chunks = [json.loads(line) for line in
|
|
159
|
+
(INDEX_DIR / "chunks.jsonl").read_text(encoding="utf-8").splitlines()]
|
|
160
|
+
if len(chunks) == saved.get("chunk_count"):
|
|
161
|
+
return chunks
|
|
162
|
+
except (OSError, ValueError):
|
|
163
|
+
pass
|
|
164
|
+
return build_index(documents, manifest)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def search(query, limit=8):
|
|
168
|
+
"""Rank section chunks by pure-Python cosine; cache writes are local."""
|
|
169
|
+
from .server import _tokenize
|
|
170
|
+
|
|
171
|
+
if limit <= 0 or not _tokenize(query):
|
|
172
|
+
return []
|
|
173
|
+
chunks = load_index()
|
|
174
|
+
if not chunks:
|
|
175
|
+
return []
|
|
176
|
+
vector = get_embedder().encode([query])[0]
|
|
177
|
+
norm = math.sqrt(sum(v * v for v in vector))
|
|
178
|
+
results = []
|
|
179
|
+
for chunk in chunks:
|
|
180
|
+
other = chunk["embedding"]
|
|
181
|
+
denominator = norm * math.sqrt(sum(v * v for v in other))
|
|
182
|
+
score = round(sum(a * b for a, b in zip(vector, other)) / denominator, 4) if denominator else 0
|
|
183
|
+
if not score:
|
|
184
|
+
continue
|
|
185
|
+
results.append({
|
|
186
|
+
"path": chunk["path"], "title": chunk["title"], "score": score,
|
|
187
|
+
"snippet": next((line.strip()[:200] for line in chunk["text"].splitlines()
|
|
188
|
+
if line.strip()), ""),
|
|
189
|
+
"heading": chunk["heading"],
|
|
190
|
+
})
|
|
191
|
+
results.sort(key=lambda r: (-r["score"], r["path"], r["heading"]))
|
|
192
|
+
return results[:limit]
|