taskflow-doc-search 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Cynthia Wang
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,3 @@
1
+ include PUBLISHING.md server.json server.py requirements.txt requirements-embeddings.txt
2
+ recursive-include tests *.py
3
+ include scripts/smoke_installed.py
@@ -0,0 +1,137 @@
1
+ Metadata-Version: 2.4
2
+ Name: taskflow-doc-search
3
+ Version: 0.1.0
4
+ Summary: MCP server for keyword search over the ai-workspace-kit documentation repos and demo tickets
5
+ License-Expression: MIT
6
+ Project-URL: Repository, https://github.com/cynthiawang109-sudo/ai-workspace-kit
7
+ Project-URL: Documentation, https://github.com/cynthiawang109-sudo/ai-workspace-kit/tree/main/mcp/doc-search
8
+ Requires-Python: >=3.10
9
+ Description-Content-Type: text/markdown
10
+ License-File: LICENSE
11
+ Requires-Dist: fastmcp>=4.1.7
12
+ Provides-Extra: dev
13
+ Requires-Dist: pytest>=9.2.0; extra == "dev"
14
+ Provides-Extra: embeddings
15
+ Requires-Dist: sentence-transformers>=3; extra == "embeddings"
16
+ Dynamic: license-file
17
+
18
+ # doc-search MCP server
19
+
20
+ A small [MCP](https://modelcontextprotocol.io) server written with FastMCP.
21
+ It gives an AI coding agent structured access to this workspace's
22
+ documentation and demo tickets — the local, public stand-in for the
23
+ Jira/Confluence side of the original production workspace. Five tools are
24
+ annotated read-only (embeddings maintains a derived local cache); the sixth can only write ticket *drafts* (see below).
25
+
26
+ ## Tools
27
+
28
+ | Tool | What it does |
29
+ |---|---|
30
+ | `search_docs(query, limit=8)` | Keyword search across every Markdown doc under `projects/`; title hits ×3, heading hits ×2, body hits ×1; returns path, title, score, snippet |
31
+ | `search_docs_semantic(query, limit=8)` | TF-IDF vector-space search over the same corpus (cosine similarity); returns path, title, score (0–1), snippet |
32
+ | `search_docs_embeddings(query, limit=8)` | Sentence embeddings over heading chunks; returns path, title, cosine score, snippet, heading; caches in `.index/` |
33
+ | `get_doc(path)` | Fetch one document by workspace-relative path (only `projects/` and `tickets/` are readable) |
34
+ | `list_tickets(status?)` | List the pre-TF-115 archive from `tickets/*.md` front matter, optionally filtered by status |
35
+ | `propose_ticket(title, body, priority="medium")` | Write a ticket **draft** to `tickets/drafts/draft-<slug>.md` — the server's only source-content write; it cannot create or modify real tickets |
36
+
37
+ ## Draft-first writes
38
+
39
+ `propose_ticket` mirrors the production workspace's rule for outward
40
+ writes (there: Jira changes start as drafts a human approves). The
41
+ boundary is enforced by the tool, not by politeness: it writes only
42
+ new files under `tickets/drafts/`, always with `status: draft`, it
43
+ refuses to overwrite an existing draft, and it is annotated as a
44
+ write, so clients ask the user before calling it. `tickets/` is the pre-TF-115
45
+ archive; `list_tickets` never shows drafts as tickets. A human reviews and
46
+ promotes a draft with `scripts/promote-draft.ps1` (or `.sh`), which files
47
+ an Issue instead of moving the file into `tickets/`. Set `AI_WORKSPACE_PROJECT`
48
+ to the workspace Project number, or pass `-Project N` / `--project N`, to
49
+ also add the Issue to the board. Without a Project, the script prints the
50
+ Issue URL and notes that it was not added to a board. The draft is preserved.
51
+ The doc-search server stays credential-free and never calls GitHub.
52
+
53
+ ## Three rungs of document search
54
+
55
+ 1. `search_docs` counts exact term hits with title/heading/body weights.
56
+ It is predictable for known terms, but repetition can dominate ranking.
57
+ 2. `search_docs_semantic` uses TF-IDF cosine similarity over whole documents.
58
+ It down-weights common terms and balances multi-term queries. Both of
59
+ these first two rungs are bag-of-words models: neither understands synonyms.
60
+ They use only the standard library and rebuild their vectors per call.
61
+ 3. `search_docs_embeddings` uses sentence-transformers over ATX heading
62
+ sections, embedding the document title, heading trail and body. It can
63
+ retrieve paraphrases and synonyms, but relevance is model-dependent.
64
+ Cosine ranking uses pure Python. A content-hash fingerprint invalidates
65
+ the derived `.index/` cache when documents or the model change.
66
+
67
+ - **Local documents**: searches never modify the source documents. Embeddings
68
+ writes a derived cache; `propose_ticket` writes drafts under the rules above.
69
+ - **No credentials**: keyword and TF-IDF need no network. Embeddings downloads
70
+ its model once, then runs fully locally.
71
+ - **Client-agnostic**: plain MCP over stdio.
72
+
73
+ ## Setup
74
+
75
+ ```powershell
76
+ cd mcp\doc-search
77
+ python -m venv .venv
78
+ .venv\Scripts\python -m pip install -r requirements.txt pytest
79
+ ```
80
+
81
+ Optional embeddings dependencies (includes torch):
82
+
83
+ ```powershell
84
+ .venv\Scripts\python -m pip install -r requirements-embeddings.txt
85
+ ```
86
+
87
+ `DOC_SEARCH_EMBEDDING_MODEL` overrides the default
88
+ `sentence-transformers/all-MiniLM-L6-v2`. `DOC_SEARCH_EMBEDDER=fake` selects
89
+ only a deterministic hashed-token test double, without semantic understanding.
90
+
91
+ Evaluate the synced corpus with `python scripts/run_eval.py`; add `--write`
92
+ to record the dated results in [EVAL.md](EVAL.md). The eval is not run in CI.
93
+
94
+ ## Run the tests
95
+
96
+ ```powershell
97
+ .venv\Scripts\python -m pytest
98
+ ```
99
+
100
+ The tests build a throwaway fixture workspace, so they pass whether or not
101
+ `sync` has populated `projects/`.
102
+
103
+ ## Register it
104
+
105
+ **Codex** — add to `~/.codex/config.toml`:
106
+
107
+ ```toml
108
+ [mcp_servers.doc-search]
109
+ command = 'C:\path\to\ai-workspace-kit\mcp\doc-search\.venv\Scripts\python.exe'
110
+ args = ['C:\path\to\ai-workspace-kit\mcp\doc-search\server.py']
111
+ ```
112
+
113
+ **Claude Code**:
114
+
115
+ ```powershell
116
+ claude mcp add doc-search -- C:\path\to\ai-workspace-kit\mcp\doc-search\.venv\Scripts\python.exe C:\path\to\ai-workspace-kit\mcp\doc-search\server.py
117
+ ```
118
+
119
+ The server finds the workspace root from its own location
120
+ (`<root>/mcp/doc-search/server.py`); set `AI_WORKSPACE_ROOT` to override.
121
+
122
+
123
+ ## Install the distribution
124
+
125
+ Once published, install with `pip install taskflow-doc-search` (or
126
+ `pip install "taskflow-doc-search[embeddings]"` for sentence embeddings).
127
+ For a local checkout, `pip install .` from this directory works today.
128
+ Run `taskflow-doc-search` with `AI_WORKSPACE_ROOT` set to the absolute path
129
+ of the workspace to search. It serves MCP over stdio; no command arguments
130
+ are needed. The existing `python server.py` launcher and its automatic
131
+ workspace-root detection remain supported.
132
+
133
+ Release instructions: [PUBLISHING.md](PUBLISHING.md).
134
+ The included [server.json](server.json) is prepared for an optional MCP
135
+ registry listing and has not been published.
136
+
137
+ <!-- mcp-name: io.github.cynthiawang109-sudo/taskflow-doc-search -->
@@ -0,0 +1,84 @@
1
+ # Publishing taskflow-doc-search
2
+
3
+ Version 0.1.0 is prepared locally; neither PyPI nor MCP registry publication
4
+ has been performed. The PyPI JSON endpoint returned HTTP 404 when preparing
5
+ this package; availability is not a reservation.
6
+
7
+ The project is licensed under MIT; see the root [LICENSE](../../LICENSE).
8
+ The distribution metadata in `pyproject.toml` declares `MIT`.
9
+
10
+ ## One-time owner setup
11
+
12
+ 1. Sign in to pypi.org and open
13
+ [Publishing](https://pypi.org/manage/account/publishing/). Add a **pending
14
+ publisher** using GitHub: project `taskflow-doc-search`, owner
15
+ `cynthiawang109-sudo`, repository `ai-workspace-kit`, workflow filename
16
+ `publish-doc-search.yml`, environment `pypi`.
17
+ 2. Create the GitHub repository environment `pypi`. Configure release branch/tag
18
+ restrictions and any required reviewers appropriate for the repository.
19
+ The environment name must exactly match the PyPI publisher configuration.
20
+
21
+ The [Trusted Publishing workflow](../../.github/workflows/publish-doc-search.yml)
22
+ uses GitHub OIDC and `id-token: write` only in the publishing job. No PyPI API
23
+ tokens or stored credentials are needed. See the official
24
+ [PyPI Trusted Publishing guide](https://docs.pypi.org/trusted-publishers/using-a-publisher/).
25
+ The first successful upload creates the pending PyPI project.
26
+
27
+ ## Local build and inspection
28
+
29
+ From `mcp/doc-search`, using a development virtual environment:
30
+
31
+ ```sh
32
+ python -m pip install ".[dev]" build twine
33
+ python -m pytest -q
34
+ python -m build
35
+ python -m twine check dist/*
36
+ python -m zipfile -l dist/taskflow_doc_search-0.1.0-py3-none-any.whl
37
+ python -m tarfile -l dist/taskflow_doc_search-0.1.0.tar.gz
38
+ ```
39
+
40
+ Inspect metadata for name, version, license, Python requirement, base dependency,
41
+ `embeddings` extra, README, and `taskflow-doc-search` entry point. Confirm both
42
+ archives contain `doc_search/`; the sdist must contain README and tests.
43
+ Install the wheel into a fresh virtual environment outside the checkout. From
44
+ outside the repository, set `AI_WORKSPACE_ROOT` to the absolute workspace path,
45
+ start `taskflow-doc-search`, and use an MCP client to initialize, list all six
46
+ tools, and call `search_docs` with a known corpus term. Run the included
47
+ client with the fresh environment's Python:
48
+
49
+ ```sh
50
+ python /path/to/mcp/doc-search/scripts/smoke_installed.py /path/to/venv/bin/taskflow-doc-search /path/to/workspace
51
+ ```
52
+
53
+ On Windows the executable is `venv/Scripts/taskflow-doc-search.exe`.
54
+ The client checks all six advertised tools and calls the five read-only tools;
55
+ its embeddings call uses an empty-token query, avoiding downloads and cache
56
+ changes. Embeddings ranking itself is covered by the unit tests with a fake
57
+ embedder. No model download is required for keyword search. Remove the
58
+ temporary environment and build output.
59
+
60
+ ## Cut a release
61
+
62
+ 1. Update the version in `pyproject.toml` and both version fields in
63
+ `server.json`; complete the local checks above and merge the approved change.
64
+ 2. Create a version tag on the intended commit (for example
65
+ `doc-search-v0.1.0`) and **publish a GitHub Release** for it.
66
+ 3. The workflow runs tests, builds and checks sdist/wheel, then publishes via
67
+ OIDC after any environment review. Inspect the workflow and PyPI files.
68
+ Ordinary pushes do not publish. Any published GitHub Release triggers this
69
+ workflow, so coordinate releases for other workspace components accordingly.
70
+ 4. `workflow_dispatch` is an explicit manual publishing path for the selected
71
+ ref; it runs the same checks. Use only for a version not already uploaded:
72
+ PyPI does not permit replacing an existing release file.
73
+
74
+ ## Optional MCP registry listing
75
+
76
+ After PyPI publication, validate the included [server.json](server.json) against
77
+ its pinned schema and follow the official
78
+ [MCP registry publishing instructions](https://github.com/modelcontextprotocol/registry/tree/main/docs).
79
+ Authenticate ownership of `io.github.cynthiawang109-sudo`, confirm the package
80
+ README's `mcp-name` ownership marker, and publish with the registry publisher.
81
+ The manifest describes PyPI installation using `uvx`, stdio transport, and the
82
+ required workspace environment variable. Its publisher metadata currently marks
83
+ it `prepared-not-yet-published`; update that status only after actual publication.
84
+ Registry listing is separate from the PyPI workflow and is not automatic.
@@ -0,0 +1,120 @@
1
+ # doc-search MCP server
2
+
3
+ A small [MCP](https://modelcontextprotocol.io) server written with FastMCP.
4
+ It gives an AI coding agent structured access to this workspace's
5
+ documentation and demo tickets — the local, public stand-in for the
6
+ Jira/Confluence side of the original production workspace. Five tools are
7
+ annotated read-only (embeddings maintains a derived local cache); the sixth can only write ticket *drafts* (see below).
8
+
9
+ ## Tools
10
+
11
+ | Tool | What it does |
12
+ |---|---|
13
+ | `search_docs(query, limit=8)` | Keyword search across every Markdown doc under `projects/`; title hits ×3, heading hits ×2, body hits ×1; returns path, title, score, snippet |
14
+ | `search_docs_semantic(query, limit=8)` | TF-IDF vector-space search over the same corpus (cosine similarity); returns path, title, score (0–1), snippet |
15
+ | `search_docs_embeddings(query, limit=8)` | Sentence embeddings over heading chunks; returns path, title, cosine score, snippet, heading; caches in `.index/` |
16
+ | `get_doc(path)` | Fetch one document by workspace-relative path (only `projects/` and `tickets/` are readable) |
17
+ | `list_tickets(status?)` | List the pre-TF-115 archive from `tickets/*.md` front matter, optionally filtered by status |
18
+ | `propose_ticket(title, body, priority="medium")` | Write a ticket **draft** to `tickets/drafts/draft-<slug>.md` — the server's only source-content write; it cannot create or modify real tickets |
19
+
20
+ ## Draft-first writes
21
+
22
+ `propose_ticket` mirrors the production workspace's rule for outward
23
+ writes (there: Jira changes start as drafts a human approves). The
24
+ boundary is enforced by the tool, not by politeness: it writes only
25
+ new files under `tickets/drafts/`, always with `status: draft`, it
26
+ refuses to overwrite an existing draft, and it is annotated as a
27
+ write, so clients ask the user before calling it. `tickets/` is the pre-TF-115
28
+ archive; `list_tickets` never shows drafts as tickets. A human reviews and
29
+ promotes a draft with `scripts/promote-draft.ps1` (or `.sh`), which files
30
+ an Issue instead of moving the file into `tickets/`. Set `AI_WORKSPACE_PROJECT`
31
+ to the workspace Project number, or pass `-Project N` / `--project N`, to
32
+ also add the Issue to the board. Without a Project, the script prints the
33
+ Issue URL and notes that it was not added to a board. The draft is preserved.
34
+ The doc-search server stays credential-free and never calls GitHub.
35
+
36
+ ## Three rungs of document search
37
+
38
+ 1. `search_docs` counts exact term hits with title/heading/body weights.
39
+ It is predictable for known terms, but repetition can dominate ranking.
40
+ 2. `search_docs_semantic` uses TF-IDF cosine similarity over whole documents.
41
+ It down-weights common terms and balances multi-term queries. Both of
42
+ these first two rungs are bag-of-words models: neither understands synonyms.
43
+ They use only the standard library and rebuild their vectors per call.
44
+ 3. `search_docs_embeddings` uses sentence-transformers over ATX heading
45
+ sections, embedding the document title, heading trail and body. It can
46
+ retrieve paraphrases and synonyms, but relevance is model-dependent.
47
+ Cosine ranking uses pure Python. A content-hash fingerprint invalidates
48
+ the derived `.index/` cache when documents or the model change.
49
+
50
+ - **Local documents**: searches never modify the source documents. Embeddings
51
+ writes a derived cache; `propose_ticket` writes drafts under the rules above.
52
+ - **No credentials**: keyword and TF-IDF need no network. Embeddings downloads
53
+ its model once, then runs fully locally.
54
+ - **Client-agnostic**: plain MCP over stdio.
55
+
56
+ ## Setup
57
+
58
+ ```powershell
59
+ cd mcp\doc-search
60
+ python -m venv .venv
61
+ .venv\Scripts\python -m pip install -r requirements.txt pytest
62
+ ```
63
+
64
+ Optional embeddings dependencies (includes torch):
65
+
66
+ ```powershell
67
+ .venv\Scripts\python -m pip install -r requirements-embeddings.txt
68
+ ```
69
+
70
+ `DOC_SEARCH_EMBEDDING_MODEL` overrides the default
71
+ `sentence-transformers/all-MiniLM-L6-v2`. `DOC_SEARCH_EMBEDDER=fake` selects
72
+ only a deterministic hashed-token test double, without semantic understanding.
73
+
74
+ Evaluate the synced corpus with `python scripts/run_eval.py`; add `--write`
75
+ to record the dated results in [EVAL.md](EVAL.md). The eval is not run in CI.
76
+
77
+ ## Run the tests
78
+
79
+ ```powershell
80
+ .venv\Scripts\python -m pytest
81
+ ```
82
+
83
+ The tests build a throwaway fixture workspace, so they pass whether or not
84
+ `sync` has populated `projects/`.
85
+
86
+ ## Register it
87
+
88
+ **Codex** — add to `~/.codex/config.toml`:
89
+
90
+ ```toml
91
+ [mcp_servers.doc-search]
92
+ command = 'C:\path\to\ai-workspace-kit\mcp\doc-search\.venv\Scripts\python.exe'
93
+ args = ['C:\path\to\ai-workspace-kit\mcp\doc-search\server.py']
94
+ ```
95
+
96
+ **Claude Code**:
97
+
98
+ ```powershell
99
+ claude mcp add doc-search -- C:\path\to\ai-workspace-kit\mcp\doc-search\.venv\Scripts\python.exe C:\path\to\ai-workspace-kit\mcp\doc-search\server.py
100
+ ```
101
+
102
+ The server finds the workspace root from its own location
103
+ (`<root>/mcp/doc-search/server.py`); set `AI_WORKSPACE_ROOT` to override.
104
+
105
+
106
+ ## Install the distribution
107
+
108
+ Once published, install with `pip install taskflow-doc-search` (or
109
+ `pip install "taskflow-doc-search[embeddings]"` for sentence embeddings).
110
+ For a local checkout, `pip install .` from this directory works today.
111
+ Run `taskflow-doc-search` with `AI_WORKSPACE_ROOT` set to the absolute path
112
+ of the workspace to search. It serves MCP over stdio; no command arguments
113
+ are needed. The existing `python server.py` launcher and its automatic
114
+ workspace-root detection remain supported.
115
+
116
+ Release instructions: [PUBLISHING.md](PUBLISHING.md).
117
+ The included [server.json](server.json) is prepared for an optional MCP
118
+ registry listing and has not been published.
119
+
120
+ <!-- mcp-name: io.github.cynthiawang109-sudo/taskflow-doc-search -->
@@ -0,0 +1 @@
1
+ """Documentation search tools served over MCP stdio."""
@@ -0,0 +1,192 @@
1
+ """Section embeddings with a content-validated, local derived index.
2
+
3
+ Importing this module does not import sentence-transformers or torch.
4
+ The fake embedder is a deterministic test double, not a semantic model.
5
+ """
6
+ import hashlib
7
+ import json
8
+ import math
9
+ import os
10
+ import re
11
+ from functools import lru_cache
12
+ from pathlib import Path
13
+
14
+ from .server import ROOT
15
+
16
+ # Keep the checkout cache beside the compatibility launcher, even when the
17
+ # workspace is overridden. Installed packages cache under the workspace,
18
+ # avoiding writes to site-packages.
19
+ _SERVER_DIR = Path(__file__).resolve().parents[1]
20
+ INDEX_DIR = (
21
+ _SERVER_DIR if (_SERVER_DIR / "server.py").is_file()
22
+ else ROOT / "mcp" / "doc-search"
23
+ ) / ".index"
24
+ DEFAULT_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
25
+
26
+
27
+ class _FakeEmbedder:
28
+ def encode(self, texts):
29
+ from .server import _tokenize
30
+
31
+ vectors = []
32
+ for text in texts:
33
+ vector = [0.0] * 64
34
+ for token in _tokenize(text):
35
+ slot = int(hashlib.sha256(token.encode("utf-8")).hexdigest(), 16) % 64
36
+ vector[slot] += 1.0
37
+ norm = math.sqrt(sum(v * v for v in vector))
38
+ vectors.append([v / norm for v in vector] if norm else vector)
39
+ return vectors
40
+
41
+
42
+ class _RealEmbedder:
43
+ def __init__(self, model):
44
+ try:
45
+ from sentence_transformers import SentenceTransformer
46
+ except ImportError as exc:
47
+ raise RuntimeError(
48
+ "sentence-transformers is not installed; "
49
+ "install the optional embeddings dependencies with: "
50
+ "pip install taskflow-doc-search[embeddings]"
51
+ ) from exc
52
+ self.model = SentenceTransformer(model)
53
+
54
+ def encode(self, texts):
55
+ return self.model.encode(texts, normalize_embeddings=True).tolist()
56
+
57
+
58
+ @lru_cache(maxsize=2)
59
+ def _embedder(name, model):
60
+ if name == "fake":
61
+ return _FakeEmbedder()
62
+ if name != "sentence-transformers":
63
+ raise ValueError(f"unknown DOC_SEARCH_EMBEDDER: {name}")
64
+ return _RealEmbedder(model)
65
+
66
+
67
+ def get_embedder():
68
+ """Return the configured embedder, loading the real model only on demand."""
69
+ return _embedder(
70
+ os.environ.get("DOC_SEARCH_EMBEDDER", "sentence-transformers"),
71
+ os.environ.get("DOC_SEARCH_EMBEDDING_MODEL", DEFAULT_MODEL),
72
+ )
73
+
74
+
75
+ def chunk_document(path, text):
76
+ """Split ATX sections into body chunks carrying their full heading trail.
77
+
78
+ The preamble uses the document title. Empty section bodies are retained
79
+ so a heading alone remains searchable. Fenced code is treated as body.
80
+ """
81
+ from .server import ROOT, _doc_title
82
+
83
+ path = Path(path)
84
+ title = _doc_title(text, path)
85
+ relative = path.relative_to(ROOT).as_posix()
86
+ chunks, trail, body = [], [], []
87
+ heading = title
88
+ fence = None
89
+
90
+ def flush():
91
+ chunks.append({"path": relative, "title": title,
92
+ "heading": heading, "text": "\n".join(body).strip()})
93
+
94
+ for line in text.splitlines():
95
+ marker = re.match(r"^ {0,3}(`{3,}|~{3,})(.*)$", line)
96
+ if marker:
97
+ run, tail = marker.groups()
98
+ if fence is None:
99
+ fence = run
100
+ elif run[0] == fence[0] and len(run) >= len(fence) and not tail.strip():
101
+ fence = None
102
+ body.append(line)
103
+ continue
104
+ match = None if fence else re.match(r"^ {0,3}(#{1,6})(?:[ \t]+(.*)|$)", line)
105
+ if match:
106
+ if trail or any(part.strip() for part in body):
107
+ flush()
108
+ level = len(match[1])
109
+ label = re.sub(r"[ \t]+#+[ \t]*$", "", match[2] or "").strip()
110
+ trail = [(n, label_) for n, label_ in trail if n < level]
111
+ trail.append((level, label))
112
+ heading = " > ".join(label_ for _, label_ in trail)
113
+ body = []
114
+ else:
115
+ body.append(line)
116
+ if trail or any(part.strip() for part in body):
117
+ flush()
118
+ return chunks
119
+
120
+
121
+ def build_index(documents, manifest):
122
+ """Embed a byte snapshot of the corpus and persist its derived cache."""
123
+ chunks = []
124
+ for path, content in documents:
125
+ chunks.extend(chunk_document(path, content.decode("utf-8", errors="replace")))
126
+ if chunks:
127
+ vectors = get_embedder().encode([
128
+ f"{c['title']} > {c['heading']}\n{c['text']}" for c in chunks
129
+ ])
130
+ for chunk, vector in zip(chunks, vectors):
131
+ chunk["embedding"] = vector
132
+ INDEX_DIR.mkdir(parents=True, exist_ok=True)
133
+ (INDEX_DIR / "chunks.jsonl").write_text(
134
+ "".join(json.dumps(c, ensure_ascii=False) + "\n" for c in chunks),
135
+ encoding="utf-8",
136
+ )
137
+ (INDEX_DIR / "manifest.json").write_text(
138
+ json.dumps({**manifest, "chunk_count": len(chunks)}, indent=2), encoding="utf-8"
139
+ )
140
+ return chunks
141
+
142
+
143
+ def load_index():
144
+ """Reuse the cache only when corpus bytes, model and embedder match."""
145
+ from .server import ROOT, _iter_docs
146
+
147
+ documents = [(p, p.read_bytes()) for p in _iter_docs()]
148
+ hashes = sorted((p.relative_to(ROOT).as_posix(), hashlib.sha256(b).hexdigest())
149
+ for p, b in documents)
150
+ manifest = {
151
+ "model": os.environ.get("DOC_SEARCH_EMBEDDING_MODEL", DEFAULT_MODEL),
152
+ "embedder": os.environ.get("DOC_SEARCH_EMBEDDER", "sentence-transformers"),
153
+ "fingerprint": hashlib.sha256(json.dumps(hashes).encode("utf-8")).hexdigest(),
154
+ }
155
+ try:
156
+ saved = json.loads((INDEX_DIR / "manifest.json").read_text(encoding="utf-8"))
157
+ if all(saved.get(k) == v for k, v in manifest.items()):
158
+ chunks = [json.loads(line) for line in
159
+ (INDEX_DIR / "chunks.jsonl").read_text(encoding="utf-8").splitlines()]
160
+ if len(chunks) == saved.get("chunk_count"):
161
+ return chunks
162
+ except (OSError, ValueError):
163
+ pass
164
+ return build_index(documents, manifest)
165
+
166
+
167
+ def search(query, limit=8):
168
+ """Rank section chunks by pure-Python cosine; cache writes are local."""
169
+ from .server import _tokenize
170
+
171
+ if limit <= 0 or not _tokenize(query):
172
+ return []
173
+ chunks = load_index()
174
+ if not chunks:
175
+ return []
176
+ vector = get_embedder().encode([query])[0]
177
+ norm = math.sqrt(sum(v * v for v in vector))
178
+ results = []
179
+ for chunk in chunks:
180
+ other = chunk["embedding"]
181
+ denominator = norm * math.sqrt(sum(v * v for v in other))
182
+ score = round(sum(a * b for a, b in zip(vector, other)) / denominator, 4) if denominator else 0
183
+ if not score:
184
+ continue
185
+ results.append({
186
+ "path": chunk["path"], "title": chunk["title"], "score": score,
187
+ "snippet": next((line.strip()[:200] for line in chunk["text"].splitlines()
188
+ if line.strip()), ""),
189
+ "heading": chunk["heading"],
190
+ })
191
+ results.sort(key=lambda r: (-r["score"], r["path"], r["heading"]))
192
+ return results[:limit]