tokencut-proxy 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tokencut/__init__.py +0 -0
- tokencut/cli.py +127 -0
- tokencut/contracts.py +148 -0
- tokencut/gateway/__init__.py +0 -0
- tokencut/gateway/__main__.py +14 -0
- tokencut/gateway/adapters/__init__.py +1 -0
- tokencut/gateway/adapters/anthropic.py +119 -0
- tokencut/gateway/adapters/common.py +95 -0
- tokencut/gateway/adapters/gemini.py +107 -0
- tokencut/gateway/adapters/openai.py +82 -0
- tokencut/gateway/app.py +366 -0
- tokencut/gateway/brief.py +101 -0
- tokencut/gateway/ccr.py +83 -0
- tokencut/gateway/classify.py +41 -0
- tokencut/gateway/codeview.py +139 -0
- tokencut/gateway/history.py +433 -0
- tokencut/gateway/live.html +135 -0
- tokencut/gateway/media.py +825 -0
- tokencut/gateway/note.py +71 -0
- tokencut/gateway/prompt_router.py +214 -0
- tokencut/gateway/records.py +312 -0
- tokencut/gateway/routing.py +131 -0
- tokencut/gateway/session.py +76 -0
- tokencut/gateway/task_router.py +157 -0
- tokencut/gateway/textcut.py +91 -0
- tokencut/gateway/trimmer.py +256 -0
- tokencut/gateway/usage.py +120 -0
- tokencut/gateway/virtual.py +273 -0
- tokencut/graph/__init__.py +0 -0
- tokencut/graph/builder.py +307 -0
- tokencut/graph/modules.py +149 -0
- tokencut/graph/queries.py +106 -0
- tokencut/ingestion/__init__.py +0 -0
- tokencut/ingestion/base.py +18 -0
- tokencut/ingestion/parser.py +36 -0
- tokencut/ingestion/python_parser.py +275 -0
- tokencut/ingestion/walker.py +68 -0
- tokencut/launch.py +104 -0
- tokencut/litellm_proxy.py +134 -0
- tokencut/pdftool/__init__.py +9 -0
- tokencut/pdftool/__main__.py +115 -0
- tokencut/pdftool/reader.py +97 -0
- tokencut/repoindex/__init__.py +0 -0
- tokencut/repoindex/__main__.py +67 -0
- tokencut/repoindex/bm25.py +63 -0
- tokencut/repoindex/embed.py +93 -0
- tokencut/repoindex/index.py +455 -0
- tokencut/repoindex/mcp.py +145 -0
- tokencut/tokens.py +19 -0
- tokencut/trimming/__init__.py +0 -0
- tokencut/trimming/packing.py +317 -0
- tokencut/trimming/query.py +259 -0
- tokencut/trimming/render.py +65 -0
- tokencut/trimming/scoring.py +117 -0
- tokencut/trimming/variants.py +61 -0
- tokencut/userconfig.py +107 -0
- tokencut/wizard.py +94 -0
- tokencut_proxy-0.1.0.dist-info/METADATA +267 -0
- tokencut_proxy-0.1.0.dist-info/RECORD +62 -0
- tokencut_proxy-0.1.0.dist-info/WHEEL +5 -0
- tokencut_proxy-0.1.0.dist-info/entry_points.txt +2 -0
- tokencut_proxy-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Command line for read_pdf.
|
|
2
|
+
|
|
3
|
+
python -m tokencut.pdftool serve MCP server (stdio) with the read_pdf tool
|
|
4
|
+
python -m tokencut.pdftool hook Claude Code PreToolUse hook: Read of a .pdf -> read_pdf
|
|
5
|
+
python -m tokencut.pdftool read FILE [PAGES] print what the agent would get (no images)
|
|
6
|
+
|
|
7
|
+
Claude Code, for one session (nothing saved in its settings):
|
|
8
|
+
claude --mcp-config pdf_mcp.json --settings pdf_hook.json
|
|
9
|
+
with pdf_mcp.json {"mcpServers": {"pdf": {"command": "python", "args": ["-m",
|
|
10
|
+
"tokencut.pdftool", "serve"]}}} and pdf_hook.json {"hooks": {"PreToolUse": [{"matcher": "Read",
|
|
11
|
+
"hooks": [{"type": "command", "command": "python -m tokencut.pdftool hook"}]}]}}.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import sys
|
|
16
|
+
from typing import IO
|
|
17
|
+
|
|
18
|
+
from tokencut.pdftool.reader import read_pdf
|
|
19
|
+
|
|
20
|
+
SERVER_INFO = {"name": "tokencut-pdf", "version": "0.1.0"}
|
|
21
|
+
DEFAULT_PROTOCOL = "2025-06-18"
|
|
22
|
+
TOOL = {
|
|
23
|
+
"name": "read_pdf",
|
|
24
|
+
"description": (
|
|
25
|
+
"Read a PDF file (use this instead of Read for every .pdf). Pages with only text come "
|
|
26
|
+
"back as text; pages with charts, tables, images or scans come back as pictures. Up to "
|
|
27
|
+
"20 pages per call."
|
|
28
|
+
),
|
|
29
|
+
"inputSchema": {
|
|
30
|
+
"type": "object",
|
|
31
|
+
"properties": {
|
|
32
|
+
"file_path": {"type": "string", "description": "Absolute path of the PDF"},
|
|
33
|
+
"pages": {"type": "string",
|
|
34
|
+
"description": "Pages, e.g. '3', '1-5', '1-3,8' (default: from page 1)"},
|
|
35
|
+
"as_images": {"type": "boolean",
|
|
36
|
+
"description": "All selected pages as pictures (when looks matter)"},
|
|
37
|
+
},
|
|
38
|
+
"required": ["file_path"],
|
|
39
|
+
},
|
|
40
|
+
}
|
|
41
|
+
HOOK_MESSAGE = ("Read is not used for PDFs in this session. Use the read_pdf tool with the same "
|
|
42
|
+
"file_path (and pages, if you need a range).")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def handle(message: dict) -> dict | None:
|
|
46
|
+
"""Answer one JSON-RPC message (None for notifications)."""
|
|
47
|
+
method, msg_id = message.get("method"), message.get("id")
|
|
48
|
+
params = message.get("params") or {}
|
|
49
|
+
if msg_id is None:
|
|
50
|
+
return None
|
|
51
|
+
if method == "initialize":
|
|
52
|
+
result = {"protocolVersion": params.get("protocolVersion") or DEFAULT_PROTOCOL,
|
|
53
|
+
"capabilities": {"tools": {"listChanged": False}}, "serverInfo": SERVER_INFO}
|
|
54
|
+
elif method == "ping":
|
|
55
|
+
result = {}
|
|
56
|
+
elif method == "tools/list":
|
|
57
|
+
result = {"tools": [TOOL]}
|
|
58
|
+
elif method == "tools/call" and params.get("name") == "read_pdf":
|
|
59
|
+
args = params.get("arguments") or {}
|
|
60
|
+
content, is_error = read_pdf(str(args.get("file_path", "")), str(args.get("pages") or ""),
|
|
61
|
+
bool(args.get("as_images", False)))
|
|
62
|
+
result = {"content": content, "isError": is_error}
|
|
63
|
+
else:
|
|
64
|
+
return {"jsonrpc": "2.0", "id": msg_id,
|
|
65
|
+
"error": {"code": -32601, "message": f"Method not found: {method}"}}
|
|
66
|
+
return {"jsonrpc": "2.0", "id": msg_id, "result": result}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def serve(stdin: IO[str] = sys.stdin, stdout: IO[str] = sys.stdout) -> None:
|
|
70
|
+
for line in stdin:
|
|
71
|
+
if not line.strip():
|
|
72
|
+
continue
|
|
73
|
+
try:
|
|
74
|
+
message = json.loads(line)
|
|
75
|
+
except ValueError:
|
|
76
|
+
reply = {"jsonrpc": "2.0", "id": None,
|
|
77
|
+
"error": {"code": -32700, "message": "Parse error"}}
|
|
78
|
+
else:
|
|
79
|
+
reply = handle(message) if isinstance(message, dict) else None
|
|
80
|
+
if reply is not None:
|
|
81
|
+
stdout.write(json.dumps(reply) + "\n")
|
|
82
|
+
stdout.flush()
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def hook(stdin: IO[str] = sys.stdin, stderr: IO[str] = sys.stderr) -> int:
|
|
86
|
+
"""Exit code 2 blocks the tool call and shows stderr to the model; 0 lets it run."""
|
|
87
|
+
try:
|
|
88
|
+
event = json.load(stdin)
|
|
89
|
+
except ValueError:
|
|
90
|
+
return 0 # not our business: never block on a malformed event
|
|
91
|
+
path = str((event.get("tool_input") or {}).get("file_path", ""))
|
|
92
|
+
if event.get("tool_name") == "Read" and path.lower().endswith(".pdf"):
|
|
93
|
+
stderr.write(HOOK_MESSAGE + "\n")
|
|
94
|
+
return 2
|
|
95
|
+
return 0
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def main(argv: list[str] | None = None) -> int:
|
|
99
|
+
args = sys.argv[1:] if argv is None else argv
|
|
100
|
+
if args[:1] == ["serve"]:
|
|
101
|
+
serve()
|
|
102
|
+
return 0
|
|
103
|
+
if args[:1] == ["hook"]:
|
|
104
|
+
return hook()
|
|
105
|
+
if args[:1] == ["read"] and len(args) >= 2:
|
|
106
|
+
content, is_error = read_pdf(args[1], args[2] if len(args) > 2 else "")
|
|
107
|
+
for block in content:
|
|
108
|
+
print(block["text"] if block["type"] == "text" else f"<{block['mimeType']} picture>")
|
|
109
|
+
return 1 if is_error else 0
|
|
110
|
+
print(__doc__)
|
|
111
|
+
return 2
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
if __name__ == "__main__":
|
|
115
|
+
sys.exit(main())
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""Read selected PDF pages: text pages as text, the other pages as PNG pictures."""
|
|
2
|
+
|
|
3
|
+
import base64
|
|
4
|
+
import re
|
|
5
|
+
import shutil
|
|
6
|
+
import subprocess
|
|
7
|
+
import tempfile
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from pypdf import PdfReader
|
|
11
|
+
from pypdf.errors import PyPdfError
|
|
12
|
+
|
|
13
|
+
from tokencut.gateway.media import pdf_page_kind
|
|
14
|
+
|
|
15
|
+
MAX_PAGES = 20 # pages per call, as Claude Code's Read
|
|
16
|
+
PICTURE_PX = 1568 # long edge: what standard-tier Claude models receive anyway
|
|
17
|
+
_PDF_ERRORS = (PyPdfError, ValueError, KeyError, TypeError, AttributeError, OSError)
|
|
18
|
+
_RANGE = re.compile(r"^\s*(\d+)\s*(?:-\s*(\d+)\s*)?$")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def parse_pages(spec: str, total: int) -> list[int]:
|
|
22
|
+
"""1-based page numbers from "3", "1-5", "1-3,8"; empty = from page 1. At most MAX_PAGES."""
|
|
23
|
+
if not spec.strip():
|
|
24
|
+
return list(range(1, min(total, MAX_PAGES) + 1))
|
|
25
|
+
pages: list[int] = []
|
|
26
|
+
for part in spec.split(","):
|
|
27
|
+
m = _RANGE.match(part)
|
|
28
|
+
if not m:
|
|
29
|
+
raise ValueError(f"bad page range {part.strip()!r} (use e.g. '3', '1-5', '1-3,8')")
|
|
30
|
+
first, last = int(m.group(1)), int(m.group(2) or m.group(1))
|
|
31
|
+
pages += [n for n in range(first, last + 1) if 1 <= n <= total and n not in pages]
|
|
32
|
+
if not pages:
|
|
33
|
+
raise ValueError(f"no pages in {spec!r}; the PDF has {total} pages")
|
|
34
|
+
return pages[:MAX_PAGES]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def render_page(path: Path, number: int) -> bytes:
|
|
38
|
+
"""One page as PNG, long edge PICTURE_PX (Poppler's pdftoppm)."""
|
|
39
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
40
|
+
out = Path(tmp) / "page"
|
|
41
|
+
subprocess.run(["pdftoppm", "-png", "-singlefile", "-f", str(number), "-l", str(number),
|
|
42
|
+
"-scale-to", str(PICTURE_PX), str(path), str(out)],
|
|
43
|
+
check=True, capture_output=True, timeout=60)
|
|
44
|
+
return out.with_suffix(".png").read_bytes()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def read_pdf(path: str, pages: str = "", as_images: bool = False) -> tuple[list[dict], bool]:
|
|
48
|
+
"""(MCP content blocks, is_error). Pages in order; consecutive text pages share one block."""
|
|
49
|
+
file = Path(path).expanduser()
|
|
50
|
+
try:
|
|
51
|
+
reader = PdfReader(file)
|
|
52
|
+
if reader.is_encrypted:
|
|
53
|
+
return [_text(f"{file.name} is encrypted: use the Read tool.")], True
|
|
54
|
+
numbers = parse_pages(pages, len(reader.pages))
|
|
55
|
+
kinds = [("visual", "") if as_images else pdf_page_kind(reader.pages[n - 1])
|
|
56
|
+
for n in numbers]
|
|
57
|
+
except (*_PDF_ERRORS, FileNotFoundError) as e:
|
|
58
|
+
return [_text(f"Cannot read {path}: {e}")], True
|
|
59
|
+
if any(kind == "visual" for kind, _ in kinds) and shutil.which("pdftoppm") is None:
|
|
60
|
+
return [_text("Poppler's pdftoppm is not installed, so pages with charts or tables "
|
|
61
|
+
"cannot be shown. Install poppler-utils.")], True
|
|
62
|
+
|
|
63
|
+
text_pages = [n for n, (kind, _) in zip(numbers, kinds, strict=True) if kind == "text"]
|
|
64
|
+
picture_pages = [n for n in numbers if n not in text_pages]
|
|
65
|
+
content = [_text(_header(file.name, len(reader.pages), numbers, text_pages, picture_pages))]
|
|
66
|
+
for n, (kind, text) in zip(numbers, kinds, strict=True):
|
|
67
|
+
if kind == "text":
|
|
68
|
+
section = f"--- page {n} ---\n{text}"
|
|
69
|
+
if content[-1]["type"] == "text" and content[-1]["text"].startswith("--- page"):
|
|
70
|
+
content[-1]["text"] += "\n\n" + section
|
|
71
|
+
else:
|
|
72
|
+
content.append(_text(section))
|
|
73
|
+
else:
|
|
74
|
+
try:
|
|
75
|
+
png = render_page(file, n)
|
|
76
|
+
except (subprocess.SubprocessError, OSError) as e:
|
|
77
|
+
return [_text(f"Cannot render page {n} of {path}: {e}")], True
|
|
78
|
+
content.append(_text(f"--- page {n} (picture) ---"))
|
|
79
|
+
content.append({"type": "image", "data": base64.b64encode(png).decode(),
|
|
80
|
+
"mimeType": "image/png"})
|
|
81
|
+
return content, False
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _header(name: str, total: int, numbers: list[int], text_pages: list[int],
|
|
85
|
+
picture_pages: list[int]) -> str:
|
|
86
|
+
def join(ns: list[int]) -> str:
|
|
87
|
+
return ", ".join(map(str, ns)) or "none"
|
|
88
|
+
more = (f" Pages after {numbers[-1]}: call again with pages='{numbers[-1] + 1}-...'."
|
|
89
|
+
if numbers[-1] < total else "")
|
|
90
|
+
return (f"[read_pdf: {name}, {total} pages. Shown: {join(numbers)}. As text: "
|
|
91
|
+
f"{join(text_pages)}. As pictures (charts, tables, images or scans): "
|
|
92
|
+
f"{join(picture_pages)}. If how a text page looks matters, call again with "
|
|
93
|
+
f"as_images=true for that page.{more}]")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _text(text: str) -> dict:
|
|
97
|
+
return {"type": "text", "text": text}
|
|
File without changes
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Command line for the repo index.
|
|
2
|
+
|
|
3
|
+
python -m tokencut.repoindex serve --repo . [--embed] MCP server for the agent
|
|
4
|
+
python -m tokencut.repoindex build --repo . [--embed] index now (and embed chunks)
|
|
5
|
+
python -m tokencut.repoindex map --repo . [prefix]
|
|
6
|
+
python -m tokencut.repoindex find --repo . "issue text"
|
|
7
|
+
python -m tokencut.repoindex symbol --repo . QuerySet.only
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import logging
|
|
12
|
+
import sys
|
|
13
|
+
import threading
|
|
14
|
+
import time
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from tokencut.repoindex.index import DEFAULT_CACHE_DIR, RepoIndex
|
|
18
|
+
from tokencut.repoindex.mcp import serve
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def main(argv: list[str] | None = None) -> None:
|
|
22
|
+
parser = argparse.ArgumentParser(prog="python -m tokencut.repoindex")
|
|
23
|
+
parser.add_argument("command", choices=["serve", "build", "map", "find", "symbol"])
|
|
24
|
+
parser.add_argument("arg", nargs="?", default="")
|
|
25
|
+
parser.add_argument("--repo", default=".", help="working tree to index")
|
|
26
|
+
parser.add_argument("--cache-dir", type=Path, default=DEFAULT_CACHE_DIR)
|
|
27
|
+
parser.add_argument("--embed", action="store_true",
|
|
28
|
+
help="use dense embeddings too (slow to compute once, cached)")
|
|
29
|
+
args = parser.parse_args(argv)
|
|
30
|
+
logging.basicConfig(level=logging.INFO, stream=sys.stderr, format="%(message)s")
|
|
31
|
+
log = logging.getLogger("tokencut.repoindex")
|
|
32
|
+
|
|
33
|
+
start = time.perf_counter()
|
|
34
|
+
index = RepoIndex.open(args.repo, args.cache_dir)
|
|
35
|
+
log.info("indexed %d files, %d chunks in %.1fs", len(index.files), len(index.by_id),
|
|
36
|
+
time.perf_counter() - start)
|
|
37
|
+
if args.embed:
|
|
38
|
+
from tokencut.repoindex.embed import ChunkEmbedder
|
|
39
|
+
|
|
40
|
+
index.embedder = ChunkEmbedder(args.cache_dir / "embed")
|
|
41
|
+
|
|
42
|
+
if args.command == "serve":
|
|
43
|
+
index.save()
|
|
44
|
+
threading.Thread(target=index.graph, kwargs={"wait": True}, daemon=True).start()
|
|
45
|
+
if index.embedder is not None: # vectors fill in while the agent works
|
|
46
|
+
threading.Thread(target=index.embedder.embed_missing, args=(index.chunks(),),
|
|
47
|
+
daemon=True).start()
|
|
48
|
+
serve(index)
|
|
49
|
+
return
|
|
50
|
+
if args.command == "build":
|
|
51
|
+
index.graph(wait=True)
|
|
52
|
+
if index.embedder is not None:
|
|
53
|
+
index.embedder.embed_missing(index.chunks())
|
|
54
|
+
index.save()
|
|
55
|
+
return
|
|
56
|
+
index.graph(wait=True)
|
|
57
|
+
if args.command == "map":
|
|
58
|
+
print(index.repo_map(args.arg))
|
|
59
|
+
elif args.command == "find":
|
|
60
|
+
print(index.find_code(args.arg))
|
|
61
|
+
else:
|
|
62
|
+
print(index.symbol(args.arg))
|
|
63
|
+
index.save()
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
if __name__ == "__main__":
|
|
67
|
+
main()
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""BM25 over an inverted index that can add and remove documents (incremental updates).
|
|
2
|
+
|
|
3
|
+
rank_bm25 rebuilds everything for one changed file (1.7 s on django's 32k chunks); here a
|
|
4
|
+
changed file only re-indexes its own chunks, and a query touches only the postings of its
|
|
5
|
+
words instead of scanning every document.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import math
|
|
9
|
+
from collections import Counter
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Bm25Index:
|
|
13
|
+
def __init__(self, k1: float = 1.5, b: float = 0.75):
|
|
14
|
+
self.k1, self.b = k1, b
|
|
15
|
+
self.postings: dict[str, dict[str, int]] = {} # word -> {doc key: term frequency}
|
|
16
|
+
self.lengths: dict[str, int] = {} # doc key -> number of words
|
|
17
|
+
self._terms: dict[str, list[str]] = {} # doc key -> its distinct words (for removal)
|
|
18
|
+
self._total = 0
|
|
19
|
+
|
|
20
|
+
def __len__(self) -> int:
|
|
21
|
+
return len(self.lengths)
|
|
22
|
+
|
|
23
|
+
def add(self, key: str, words: list[str]) -> None:
|
|
24
|
+
if key in self.lengths:
|
|
25
|
+
self.remove(key)
|
|
26
|
+
counts = Counter(words)
|
|
27
|
+
for word, tf in counts.items():
|
|
28
|
+
self.postings.setdefault(word, {})[key] = tf
|
|
29
|
+
self.lengths[key] = len(words)
|
|
30
|
+
self._terms[key] = sorted(counts)
|
|
31
|
+
self._total += len(words)
|
|
32
|
+
|
|
33
|
+
def remove(self, key: str) -> None:
|
|
34
|
+
if key not in self.lengths:
|
|
35
|
+
return
|
|
36
|
+
for word in self._terms.pop(key):
|
|
37
|
+
docs = self.postings[word]
|
|
38
|
+
docs.pop(key, None)
|
|
39
|
+
if not docs:
|
|
40
|
+
del self.postings[word]
|
|
41
|
+
self._total -= self.lengths.pop(key)
|
|
42
|
+
|
|
43
|
+
def scores(self, query_words: list[str]) -> dict[str, float]:
|
|
44
|
+
"""Raw BM25 score of every document containing at least one query word."""
|
|
45
|
+
n = len(self.lengths)
|
|
46
|
+
if not n:
|
|
47
|
+
return {}
|
|
48
|
+
avg = self._total / n or 1.0
|
|
49
|
+
result: dict[str, float] = {}
|
|
50
|
+
for word, qtf in sorted(Counter(query_words).items()):
|
|
51
|
+
docs = self.postings.get(word)
|
|
52
|
+
if not docs:
|
|
53
|
+
continue
|
|
54
|
+
idf = math.log(1 + (n - len(docs) + 0.5) / (len(docs) + 0.5))
|
|
55
|
+
for key, tf in docs.items():
|
|
56
|
+
norm = tf + self.k1 * (1 - self.b + self.b * self.lengths[key] / avg)
|
|
57
|
+
result[key] = result.get(key, 0.0) + qtf * idf * tf * (self.k1 + 1) / norm
|
|
58
|
+
return result
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def normalised(scores: dict[str, float]) -> dict[str, float]:
|
|
62
|
+
best = max(scores.values(), default=0.0)
|
|
63
|
+
return {k: v / best for k, v in scores.items()} if best > 0 else {}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""Optional dense embeddings for the repo index (opt-in: `--embed`).
|
|
2
|
+
|
|
3
|
+
Model BAAI/bge-small-en-v1.5 through fastembed (ONNX, CPU), as in the offline benchmark.
|
|
4
|
+
Embedding a chunk is slow on a laptop CPU (~13 chunks/s), so vectors are computed ahead of
|
|
5
|
+
time (`python -m tokencut.repoindex build --embed`, or in a background thread of the server)
|
|
6
|
+
and cached by a hash of the chunk text: a changed chunk is the only one embedded again.
|
|
7
|
+
At search time only the short query is embedded; chunks without a vector yet are skipped.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import logging
|
|
12
|
+
import sqlite3
|
|
13
|
+
import threading
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
|
|
18
|
+
from tokencut.contracts import Chunk
|
|
19
|
+
|
|
20
|
+
log = logging.getLogger(__name__)
|
|
21
|
+
|
|
22
|
+
MODEL = "BAAI/bge-small-en-v1.5"
|
|
23
|
+
MAX_CHARS = 1_000 # same cut as the benchmark (~250 tokens; the head of a def says most)
|
|
24
|
+
BATCH = 32
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def chunk_text(chunk: Chunk) -> str:
|
|
28
|
+
return f"{chunk.file_path} {chunk.qualified_name}\n{chunk.code[:MAX_CHARS]}"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class ChunkEmbedder:
|
|
32
|
+
def __init__(self, cache_dir: Path, model: str = MODEL):
|
|
33
|
+
from fastembed import TextEmbedding # optional dependency, imported only when used
|
|
34
|
+
|
|
35
|
+
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
36
|
+
self.model_name = model
|
|
37
|
+
self.model = TextEmbedding(model, cache_dir=str(cache_dir / "models"))
|
|
38
|
+
self._db_path = cache_dir / "embeddings.sqlite"
|
|
39
|
+
self._lock = threading.Lock()
|
|
40
|
+
with self._connect() as db:
|
|
41
|
+
db.execute("CREATE TABLE IF NOT EXISTS vec (key TEXT PRIMARY KEY, v BLOB)")
|
|
42
|
+
self._vectors: dict[str, np.ndarray] = {} # key -> unit vector (loaded on demand)
|
|
43
|
+
|
|
44
|
+
def _connect(self) -> sqlite3.Connection:
|
|
45
|
+
return sqlite3.connect(self._db_path)
|
|
46
|
+
|
|
47
|
+
def _key(self, chunk: Chunk) -> str:
|
|
48
|
+
return hashlib.sha1(f"{self.model_name}\0{chunk_text(chunk)}".encode()).hexdigest()
|
|
49
|
+
|
|
50
|
+
def _load(self, keys: list[str]) -> None:
|
|
51
|
+
missing = [k for k in keys if k not in self._vectors]
|
|
52
|
+
with self._connect() as db:
|
|
53
|
+
for i in range(0, len(missing), 500):
|
|
54
|
+
batch = missing[i:i + 500]
|
|
55
|
+
marks = ",".join("?" * len(batch))
|
|
56
|
+
for key, blob in db.execute(f"SELECT key, v FROM vec WHERE key IN ({marks})", batch):
|
|
57
|
+
vec = np.frombuffer(blob, dtype=np.float32)
|
|
58
|
+
self._vectors[key] = vec / (np.linalg.norm(vec) or 1.0)
|
|
59
|
+
|
|
60
|
+
def embed_missing(self, chunks: list[Chunk], limit: int | None = None) -> int:
|
|
61
|
+
"""Compute vectors not cached yet (slow). Returns how many were computed."""
|
|
62
|
+
keys = {self._key(c): c for c in chunks}
|
|
63
|
+
with self._lock:
|
|
64
|
+
self._load(sorted(keys))
|
|
65
|
+
todo = [(k, c) for k, c in sorted(keys.items()) if k not in self._vectors][:limit]
|
|
66
|
+
done = 0
|
|
67
|
+
for i in range(0, len(todo), 256):
|
|
68
|
+
part = todo[i:i + 256]
|
|
69
|
+
vectors = self.model.passage_embed([chunk_text(c) for _, c in part], batch_size=BATCH)
|
|
70
|
+
rows = []
|
|
71
|
+
for (key, _), vec in zip(part, vectors):
|
|
72
|
+
vec = np.asarray(vec, dtype=np.float32)
|
|
73
|
+
rows.append((key, vec.tobytes()))
|
|
74
|
+
with self._lock:
|
|
75
|
+
self._vectors[key] = vec / (np.linalg.norm(vec) or 1.0)
|
|
76
|
+
with self._connect() as db:
|
|
77
|
+
db.executemany("INSERT OR REPLACE INTO vec VALUES (?, ?)", rows)
|
|
78
|
+
done += len(part)
|
|
79
|
+
log.info("embedded %d / %d chunks", done, len(todo))
|
|
80
|
+
return done
|
|
81
|
+
|
|
82
|
+
def similarities(self, query: str, chunks: list[Chunk]) -> dict[str, float]:
|
|
83
|
+
"""Cosine similarity of the query to every chunk that already has a vector."""
|
|
84
|
+
keys = [(c.id, self._key(c)) for c in chunks]
|
|
85
|
+
with self._lock:
|
|
86
|
+
self._load([k for _, k in keys])
|
|
87
|
+
known = [(cid, self._vectors[k]) for cid, k in keys if k in self._vectors]
|
|
88
|
+
if not known:
|
|
89
|
+
return {}
|
|
90
|
+
q = np.asarray(next(iter(self.model.query_embed([query]))), dtype=np.float32)
|
|
91
|
+
q = q / (np.linalg.norm(q) or 1.0)
|
|
92
|
+
sims = np.stack([v for _, v in known]) @ q
|
|
93
|
+
return {cid: float(s) for (cid, _), s in zip(known, sims)}
|