code-recycle 0.1.0__tar.gz

code_recycle-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,30 @@
+# Tests on every push and pull request: three operating systems, three Python versions.
+# Sources: GitHub Docs, "Building and testing Python" (version matrix, pytest step) and
+# "Workflow syntax" (strategy.matrix with os, fail-fast).
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false  # a failure on one machine doesn't stop the others, so every problem shows
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.11", "3.12", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v6
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Check this Python's SQLite has full-text search (FTS5)
+        run: python -c "import sqlite3; sqlite3.connect(':memory:').execute('CREATE VIRTUAL TABLE t USING fts5(x)'); print('FTS5 ok', sqlite3.sqlite_version)"
+      - name: Install the package and test tools
+        run: |
+          python -m pip install --upgrade pip
+          pip install ".[dev]"  # the embeddings tests use a fake model, so fastembed is not needed
+      - name: Test with pytest
+        run: pytest -q

code_recycle-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,54 @@
+# Publishes to PyPI when a version tag (for example v0.1.0) is pushed. Not used until the PyPI
+# project and its trusted publisher are set up. Sources: Python Packaging User Guide,
+# "Publishing package distribution releases using GitHub Actions CI/CD workflows".
+name: Publish
+
+on:
+  push:
+    tags: ["v*"]
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        persist-credentials: false
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.x"
+    - name: Install pypa/build
+      run: >-
+        python3 -m
+        pip install
+        build
+        --user
+    - name: Build a wheel and a source distribution
+      run: python3 -m build
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v5
+      with:
+        name: python-package-distributions
+        path: dist/
+
+  publish-to-pypi:
+    name: Publish to PyPI
+    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    needs:
+    - build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/code-recycle
+    permissions:
+      id-token: write  # mandatory for trusted publishing; no stored token
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v6
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

code_recycle-0.1.0/.gitignore

@@ -0,0 +1,11 @@
+.venv/
+dist/
+__pycache__/
+*.egg-info/
+.code_recycle/
+/code_recycle.toml
+docs/research_plan.md
+docs/reviews/
+docs/gpt_brief.md
+data/*.parquet
+docs/project_plan.md

code_recycle-0.1.0/CHANGELOG.md

@@ -0,0 +1,22 @@
+# Changelog
+
+## 0.1.0 (2026-11-06)
+
+First release.
+
+- Index folders: Python split by function, method and class with line numbers; Markdown research
+  notes read as structured entries (URL, where on the page, what it taught, copied code); other
+  Markdown by heading; other text in 60-line windows.
+- Keyword search with SQLite FTS5: stemming, camelCase splitting, per-field weights, identical
+  copies shown once.
+- MCP tool `search_own_work`, read-only, over stdio.
+- Command `code-recycle`: `init`, `index`, `search`, `serve`.
+- Settings file with typo checking, skip rules, `.gitignore` support and a file-type list.
+- Secrets: secret-named files never opened, every file scanned before indexing, every result
+  scanned before returning, and a self-check that stops indexing if the scanner can't see a
+  planted fake key.
+- Reuse rules: `no_suggest_from` and `no_suggest_into`. Rules apply before duplicate copies are
+  removed, and the MCP tool falls back to the folder it was started in when no `from_path` is given.
+- Accuracy published: CodeSearchNet Python MRR 0.797; own research notes MRR 0.308.
+- Experimental, off by default: meaning search with local embeddings (`pip install "code-recycle[embed]"`).
+  Tested and not adopted; see `results/prereg_embeddings.md`.

code_recycle-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Zee Mehmood
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

code_recycle-0.1.0/METHOD.md

@@ -0,0 +1,47 @@
+# How this was built
+
+## Sources first, then code
+
+Each design decision has a source, found and checked before that part was built. For each
+requirement there's a research note: the page's URL, where on the page, what it taught, and the
+code copied from it word for word. During the research, a script checked each copied snippet
+against the downloaded page, so a quote that didn't match the page was caught. Where no source
+exists (the settings layout, the reuse rules, the secrets policy, the research-note parser), the
+design is marked as my own and has its own tests.
+
+The full research plan stays private, because it holds code copied from other people's pages. The
+main sources:
+
+| Part | Source |
+|---|---|
+| MCP server and testing | MCP Python SDK docs and examples (v2, pinned to 2.3.0) |
+| Registering with clients | MCP SDK "Connect to a real host"; OpenAI Codex MCP docs |
+| Splitting Python code | Python docs, `ast` |
+| Keyword search | SQLite FTS5 docs (bm25 column weights, porter stemming, UNINDEXED columns) |
+| Combining rankings | Cormack, Clarke and Büttcher (2009), Reciprocal Rank Fusion |
+| Scoring | Mean reciprocal rank; CodeSearchNet (Husain et al. 2019) |
+| Settings | Python docs, `tomllib` |
+| Skip rules | `pathspec` (Git's own .gitignore behaviour) |
+| Secrets | `detect-secrets` (Yelp) |
+| Packaging and CI | Python Packaging User Guide; GitHub Actions docs |
+
+## A second opinion
+
+Before building, the plan was given to a different AI model (GPT, through the Codex CLI, read-only)
+and asked to attack it: what would make it look like a tutorial clone, what was missing, what to
+cut. Its points were treated as opinions and checked before anything changed. It cut the first
+release down, made the accuracy test fairer, and narrowed the secrets promise to what tests can
+prove.
+
+## Measuring accuracy
+
+- The test design was written down before the first run.
+- The keyword baseline was frozen before anything else was tried.
+- Embeddings were chosen on a validation split and then run once on the test sets, against a rule
+  set in advance. They didn't meet the rule, so they're not in v0.1. That result is published too.
+- One change was made to that plan before any accuracy run: this laptop embeds about 6 pieces a
+  second, so only code and research notes are embedded, cut to 400 characters. It's recorded with
+  the reason.
+- Leak checks: the CodeSearchNet test was re-run with the docstring deliberately left in the code
+  (MRR 0.988, against 0.797) to prove that removing it works, and with function names hidden (0.763)
+  to show the result doesn't rest on names alone.

code_recycle-0.1.0/PKG-INFO

@@ -0,0 +1,176 @@
+Metadata-Version: 2.5
+Name: code-recycle
+Version: 0.1.0
+Summary: Search your own code and verified sources before searching the web. Local, private, and usable from any MCP client.
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: detect-secrets==1.5.0
+Requires-Dist: mcp==2.3.0
+Requires-Dist: pathspec==1.1.1
+Provides-Extra: dev
+Requires-Dist: anyio; extra == 'dev'
+Requires-Dist: numpy; extra == 'dev'
+Requires-Dist: pyarrow==22.0.0; extra == 'dev'
+Requires-Dist: pytest==9.1.1; extra == 'dev'
+Provides-Extra: embed
+Requires-Dist: fastembed==0.9.1; extra == 'embed'
+Description-Content-Type: text/markdown
+
+# code_recycle
+
+Search your own code and your own verified sources before you search the web.
+
+Every new project I start, I end up looking for the same things again: how to retry a request,
+how to read a settings file, which page had that bootstrap example. Most of the time I've already
+solved it in another project, and I've already checked the source. I just can't find it.
+
+Companies have the same problem, with a twist. Pasting internal code or questions into outside
+tools is a risk, so the search needs to stay on the machine.
+
+code_recycle indexes your folders and answers "have I already solved this?" with the file and line
+numbers. If you keep research notes with a source URL and the code you copied from it, it finds
+those too, with the link and what you learned. It runs locally, needs no account and no API key,
+and plugs into Claude Code, Codex, Cursor or any MCP client.
+
+## Try it in 5 minutes
+
+Needs Python 3.11 or newer.
+
+```bash
+git clone https://github.com/Z33MScEssex/code_recycle
+cd code_recycle
+pip install .
+cd examples/demo
+code-recycle index
+code-recycle search "read a settings file"
+```
+
+The first result is a research note: the Python docs page for `tomllib`, where on the page, what it
+taught, and the code copied from it. Try `code-recycle search "try a request again with a wait"`
+and you get the `retry` function from `tools/http_utils.py`, lines 14 to 24.
+
+## Use it on your own folders
+
+```bash
+code-recycle init          # writes code_recycle.toml here; edit the folders list
+code-recycle index         # builds the index and prints what was skipped and why
+code-recycle search "parse an iso date"
+```
+
+Re-run `code-recycle index` after big changes. It reads everything again; on my laptop that's about
+40 seconds for 1,500 files.
+
+## Connect it to an AI coding assistant
+
+The tool is called `search_own_work`. It's read-only. Hosts start it with a near-empty environment,
+so give the settings file's full path.
+
+Claude Code:
+
+```bash
+claude mcp add code-recycle -e CODE_RECYCLE_SETTINGS=/full/path/to/code_recycle.toml -- code-recycle serve
+```
+
+Codex:
+
+```bash
+codex mcp add code-recycle --env CODE_RECYCLE_SETTINGS=/full/path/to/code_recycle.toml -- code-recycle serve
+```
+
+If the host can't find `code-recycle`, use the full path from `which code-recycle` (or
+`where code-recycle` on Windows).
+
+## Research notes it understands
+
+A Markdown section becomes a research entry when it has a URL field. These labels are read, bold
+or plain:
+
+```markdown
+## Topic 1: Read a TOML settings file
+
+- **URL:** https://docs.python.org/3/library/tomllib.html
+- **Where on the page:** "Examples", the first example.
+- **What it taught:** the file must be opened in binary mode.
+- **Code pattern from source:**
+    (a fenced code block)
+```
+
+Everything else is still indexed: Python by function, method and class with exact line numbers,
+other Markdown by heading, and other text files in overlapping windows of 60 lines.
+
+## How well it works
+
+Measured, not assumed. The test design was written down before the first run, and nothing was
+tuned on the test sets. Full write-up: [results/accuracy_2026-11-05.md](results/accuracy_2026-11-05.md).
+
+| Test | What it asks | Result |
+|---|---|---|
+| CodeSearchNet, Python | Find a function from its docstring among 999 others (Husain et al. 2019) | MRR 0.797, right function in the top 5 87% of the time |
+| My own research notes | Find another note that used the same source, from a topic heading | MRR 0.308, top 5 39% of the time, 51 cases |
+
+The second test is the real job, and it's much harder. The same source often gets described in
+different words in different projects, and keyword search can't bridge that.
+
+I tried meaning-based search with a small local embedding model. It lifted CodeSearchNet to 0.907,
+but it didn't help on my own notes (0.250 against 0.280 at the time), and the rule set beforehand
+said it had to improve both. So v0.1 ships keyword search. The embeddings code is in the repo,
+switched off and marked experimental: [results/prereg_embeddings.md](results/prereg_embeddings.md).
+
+## Privacy and secrets
+
+- Nothing leaves your machine. There are no network calls, unless you switch on the experimental
+  embeddings, which download a model once.
+- Files named like secrets (`.env`, `*.pem`, `id_rsa` and others) are never opened.
+- Every other file is scanned with [detect-secrets](https://github.com/Yelp/detect-secrets) before
+  it's indexed. A flagged file is skipped and listed in the index report. If you've checked it's
+  safe, add it to `[secrets] allow`.
+- Every result is scanned again before it's returned, for common live key formats (AWS, GitHub,
+  Slack, OpenAI-style, Telegram bot tokens, private key blocks). If one matches, the content is
+  withheld and only the location is shown.
+- If the scanner fails its own self-check (it must spot a planted fake key), nothing is indexed.
+
+Each of these behaviours has a test with fake keys. That isn't a guarantee: a pattern scanner can't
+catch every possible secret, so don't point the tool at folders you know hold live credentials.
+
+## Reuse rules
+
+Some folders shouldn't borrow from others. I have one project where every requirement must be
+researched fresh, so code from other projects must never be suggested into it.
+
+```toml
+[rules]
+no_suggest_from = ["private/"]    # never return results from here
+no_suggest_into = ["clean-room/"] # searches made from here only see this folder's own code,
+                                  # plus a note that other matches exist
+```
+
+The MCP tool takes a `from_path`, so the assistant can say where it's working. If it doesn't, the
+folder the server was started in is used. The rules are a guard against accidents, not a security
+boundary: they trust the path they're given.
+
+## Settings
+
+`code-recycle init` writes a commented example. The settings file is found in this order:
+`--settings`, then `$CODE_RECYCLE_SETTINGS`, then `./code_recycle.toml`, then
+`~/.config/code_recycle/code_recycle.toml`. Typos are refused with a clear message, because a
+misspelt rule silently ignored would be worse.
+
+## Limits
+
+- Python is the only language split by function. Others are searched in windows of lines.
+- The index is rebuilt in full each time. Fine for a few thousand files; not built for huge monorepos.
+- Search is keyword-based (SQLite FTS5 with stemming), so it finds shared words, not shared ideas.
+- So far only I have used it. It has had no outside users yet.
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+87 tests. The CI workflow runs them on Linux, macOS and Windows with Python 3.11 to 3.13 on every push. How sources were
+checked and how accuracy was measured: [METHOD.md](METHOD.md). Benchmarks: [bench/README.md](bench/README.md).
+
+MIT licence.

code_recycle-0.1.0/README.md

@@ -0,0 +1,157 @@
+# code_recycle
+
+Search your own code and your own verified sources before you search the web.
+
+Every new project I start, I end up looking for the same things again: how to retry a request,
+how to read a settings file, which page had that bootstrap example. Most of the time I've already
+solved it in another project, and I've already checked the source. I just can't find it.
+
+Companies have the same problem, with a twist. Pasting internal code or questions into outside
+tools is a risk, so the search needs to stay on the machine.
+
+code_recycle indexes your folders and answers "have I already solved this?" with the file and line
+numbers. If you keep research notes with a source URL and the code you copied from it, it finds
+those too, with the link and what you learned. It runs locally, needs no account and no API key,
+and plugs into Claude Code, Codex, Cursor or any MCP client.
+
+## Try it in 5 minutes
+
+Needs Python 3.11 or newer.
+
+```bash
+git clone https://github.com/Z33MScEssex/code_recycle
+cd code_recycle
+pip install .
+cd examples/demo
+code-recycle index
+code-recycle search "read a settings file"
+```
+
+The first result is a research note: the Python docs page for `tomllib`, where on the page, what it
+taught, and the code copied from it. Try `code-recycle search "try a request again with a wait"`
+and you get the `retry` function from `tools/http_utils.py`, lines 14 to 24.
+
+## Use it on your own folders
+
+```bash
+code-recycle init          # writes code_recycle.toml here; edit the folders list
+code-recycle index         # builds the index and prints what was skipped and why
+code-recycle search "parse an iso date"
+```
+
+Re-run `code-recycle index` after big changes. It reads everything again; on my laptop that's about
+40 seconds for 1,500 files.
+
+## Connect it to an AI coding assistant
+
+The tool is called `search_own_work`. It's read-only. Hosts start it with a near-empty environment,
+so give the settings file's full path.
+
+Claude Code:
+
+```bash
+claude mcp add code-recycle -e CODE_RECYCLE_SETTINGS=/full/path/to/code_recycle.toml -- code-recycle serve
+```
+
+Codex:
+
+```bash
+codex mcp add code-recycle --env CODE_RECYCLE_SETTINGS=/full/path/to/code_recycle.toml -- code-recycle serve
+```
+
+If the host can't find `code-recycle`, use the full path from `which code-recycle` (or
+`where code-recycle` on Windows).
+
+## Research notes it understands
+
+A Markdown section becomes a research entry when it has a URL field. These labels are read, bold
+or plain:
+
+```markdown
+## Topic 1: Read a TOML settings file
+
+- **URL:** https://docs.python.org/3/library/tomllib.html
+- **Where on the page:** "Examples", the first example.
+- **What it taught:** the file must be opened in binary mode.
+- **Code pattern from source:**
+    (a fenced code block)
+```
+
+Everything else is still indexed: Python by function, method and class with exact line numbers,
+other Markdown by heading, and other text files in overlapping windows of 60 lines.
+
+## How well it works
+
+Measured, not assumed. The test design was written down before the first run, and nothing was
+tuned on the test sets. Full write-up: [results/accuracy_2026-11-05.md](results/accuracy_2026-11-05.md).
+
+| Test | What it asks | Result |
+|---|---|---|
+| CodeSearchNet, Python | Find a function from its docstring among 999 others (Husain et al. 2019) | MRR 0.797, right function in the top 5 87% of the time |
+| My own research notes | Find another note that used the same source, from a topic heading | MRR 0.308, top 5 39% of the time, 51 cases |
+
+The second test is the real job, and it's much harder. The same source often gets described in
+different words in different projects, and keyword search can't bridge that.
+
+I tried meaning-based search with a small local embedding model. It lifted CodeSearchNet to 0.907,
+but it didn't help on my own notes (0.250 against 0.280 at the time), and the rule set beforehand
+said it had to improve both. So v0.1 ships keyword search. The embeddings code is in the repo,
+switched off and marked experimental: [results/prereg_embeddings.md](results/prereg_embeddings.md).
+
+## Privacy and secrets
+
+- Nothing leaves your machine. There are no network calls, unless you switch on the experimental
+  embeddings, which download a model once.
+- Files named like secrets (`.env`, `*.pem`, `id_rsa` and others) are never opened.
+- Every other file is scanned with [detect-secrets](https://github.com/Yelp/detect-secrets) before
+  it's indexed. A flagged file is skipped and listed in the index report. If you've checked it's
+  safe, add it to `[secrets] allow`.
+- Every result is scanned again before it's returned, for common live key formats (AWS, GitHub,
+  Slack, OpenAI-style, Telegram bot tokens, private key blocks). If one matches, the content is
+  withheld and only the location is shown.
+- If the scanner fails its own self-check (it must spot a planted fake key), nothing is indexed.
+
+Each of these behaviours has a test with fake keys. That isn't a guarantee: a pattern scanner can't
+catch every possible secret, so don't point the tool at folders you know hold live credentials.
+
+## Reuse rules
+
+Some folders shouldn't borrow from others. I have one project where every requirement must be
+researched fresh, so code from other projects must never be suggested into it.
+
+```toml
+[rules]
+no_suggest_from = ["private/"]    # never return results from here
+no_suggest_into = ["clean-room/"] # searches made from here only see this folder's own code,
+                                  # plus a note that other matches exist
+```
+
+The MCP tool takes a `from_path`, so the assistant can say where it's working. If it doesn't, the
+folder the server was started in is used. The rules are a guard against accidents, not a security
+boundary: they trust the path they're given.
+
+## Settings
+
+`code-recycle init` writes a commented example. The settings file is found in this order:
+`--settings`, then `$CODE_RECYCLE_SETTINGS`, then `./code_recycle.toml`, then
+`~/.config/code_recycle/code_recycle.toml`. Typos are refused with a clear message, because a
+misspelt rule silently ignored would be worse.
+
+## Limits
+
+- Python is the only language split by function. Others are searched in windows of lines.
+- The index is rebuilt in full each time. Fine for a few thousand files; not built for huge monorepos.
+- Search is keyword-based (SQLite FTS5 with stemming), so it finds shared words, not shared ideas.
+- So far only I have used it. It has had no outside users yet.
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+87 tests. The CI workflow runs them on Linux, macOS and Windows with Python 3.11 to 3.13 on every push. How sources were
+checked and how accuracy was measured: [METHOD.md](METHOD.md). Benchmarks: [bench/README.md](bench/README.md).
+
+MIT licence.

code_recycle-0.1.0/bench/README.md

@@ -0,0 +1,21 @@
+# Benchmarks
+
+## CodeSearchNet (public)
+
+Download the Python test split (28.7 MB) from the Hugging Face copy of CodeSearchNet. The original
+S3 links in the CodeSearchNet README no longer work.
+
+```bash
+curl -L -o data/csn_python_test.parquet \
+  https://huggingface.co/datasets/code-search-net/code_search_net/resolve/main/python/test-00000-of-00001.parquet
+pip install pyarrow
+python bench/codesearchnet.py
+```
+
+Each function keeps its own repository's licence, so the data is never committed here.
+
+## Own research entries
+
+`python bench/own_entries.py path/to/code_recycle.toml --detail somewhere_private.txt`
+
+Uses your own index. The per-case detail names your files, so keep it out of public repos.

code_recycle-0.1.0/bench/codesearchnet.py

@@ -0,0 +1,127 @@
+"""CodeSearchNet benchmark: find a Python function from its docstring among 999 others.
+
+Method: research plan Topic 7, "a public benchmark as well as my own entries". Follows the
+paper's test (Husain et al. 2019, Table 3): the docstring is the query, the right function must
+rank among 999 distractors, scored by mean reciprocal rank (MRR).
+
+Uses the same index columns, weights and query building as code_recycle's real search, so the
+number describes the shipped tool, not a tuned copy of it.
+
+Usage: python bench/codesearchnet.py [PARQUET] [--mode keyword|meaning|both] [--model NAME]
+Default PARQUET: data/csn_python_test.parquet. The model is chosen on the validation split only
+(results/prereg_embeddings.md); the test split is for the final number.
+Get the data: see bench/README.md.
+"""
+import ast  # removing the docstring from each function
+import random  # the fixed shuffle that builds the groups of 1,000
+import sqlite3  # an in-memory FTS5 table per group
+import argparse  # command-line options
+from pathlib import Path
+
+import pyarrow.parquet as pq  # reads the dataset file
+
+from code_recycle import embed, index, search
+
+SEED = 2026  # fixed before the first run, never changed
+GROUP = 1000  # 1 right answer + 999 distractors, as in the paper
+TOP = 5  # "hit rate at 5": is the right function in the first five?
+
+
+def strip_docstring(code):
+    """The function's code without its docstring, so the query isn't copied inside the answer.
+
+    Returns (code, how) where how is "ast" or "unparsed" (the code doesn't parse; kept as is but counted).
+    """
+    try:
+        tree = ast.parse(code)
+    except SyntaxError:
+        return code, "unparsed"
+    fn = tree.body[0] if tree.body else None
+    if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)) or not fn.body:
+        return code, "unparsed"
+    first = fn.body[0]
+    if not (isinstance(first, ast.Expr) and isinstance(first.value, ast.Constant) and isinstance(first.value.value, str)):
+        return code, "ast"  # no docstring to remove
+    lines = code.splitlines()
+    del lines[first.lineno - 1:first.end_lineno]  # drop exactly the docstring's lines
+    return "\n".join(lines), "ast"
+
+
+def run(path, mode="keyword", model_name=None, progress=lambda m: None):
+    rows = pq.read_table(path, columns=["func_name", "func_code_string", "func_documentation_string"]).to_pylist()
+    order = list(range(len(rows)))
+    random.Random(SEED).shuffle(order)
+    groups = [order[i:i + GROUP] for i in range(0, len(order) - GROUP + 1, GROUP)]  # leftover dropped
+
+    stripped = [strip_docstring(r["func_code_string"]) for r in rows]
+    names = [index.split_name(r["func_name"].rsplit(".", 1)[-1]) for r in rows]  # "YouTube.get_vid" -> "get_vid"
+    model = vectors = None
+    if mode != "keyword":
+        model = embed.load_model(model_name)
+        used = sorted({i for g in groups for i in g})
+        progress(f"Embedding {len(used)} functions with {model_name}...")
+        vecs = embed.embed_pieces(model, [embed.piece_text(names[i], stripped[i][0]) for i in used])
+        vectors = dict(zip(used, vecs))
+
+    modes = ["keyword", "meaning", "both"] if mode == "all" else [mode]
+    rr = {m: [] for m in modes}
+    hits = {m: 0 for m in modes}
+    no_words = 0
+    weights = ", ".join(map(str, search.WEIGHTS))
+    for gi, g in enumerate(groups, start=1):
+        con = sqlite3.connect(":memory:")
+        con.execute(index.SCHEMA)
+        for rowid, i in enumerate(g, start=1):
+            con.execute(
+                f"INSERT INTO chunks (rowid, {', '.join(index.COLUMNS + index.STORED)}) VALUES (?, {', '.join('?' * (len(index.COLUMNS) + len(index.STORED)))})",
+                (rowid, names[i], "", "", "", "", "", stripped[i][0], "function", "", 0, 0, rows[i]["func_name"]),
+            )
+        if vectors is not None:
+            import numpy as np
+            matrix = np.stack([vectors[i] for i in g])  # row r-1 is rowid r
+            ids = list(range(1, len(g) + 1))
+        for rowid, i in enumerate(g, start=1):
+            doc = rows[i]["func_documentation_string"]
+            q = search.to_query(doc)
+            keyword = [x for (x,) in con.execute(
+                f"SELECT rowid FROM chunks WHERE chunks MATCH ? ORDER BY bm25(chunks, {weights})", (q,))] if q else []
+            ranked = {"keyword": keyword}
+            if mode != "keyword":
+                meaning = embed.nearest(matrix, ids, embed.embed_query(model, doc), len(ids))
+                ranked["meaning"] = meaning
+                ranked["both"] = embed.fuse(keyword, meaning)
+            if not keyword:
+                no_words += 1
+            for m in modes:
+                if rowid in ranked[m]:
+                    pos = ranked[m].index(rowid) + 1
+                    rr[m].append(1 / pos)
+                    hits[m] += pos <= TOP
+                else:
+                    rr[m].append(0.0)  # not returned at all counts as a miss
+        con.close()
+        progress(f"group {gi} of {len(groups)}")
+
+    n = len(rr[modes[0]])
+    return {
+        "queries": n,
+        "groups": len(groups),
+        "modes": {m: {"mrr": sum(rr[m]) / n, "hit_at_5": hits[m] / n} for m in modes},
+        "unparsed_code": sum(how == "unparsed" for _, how in (stripped[i] for g in groups for i in g)),
+        "queries_with_no_words": no_words,
+    }
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser()
+    p.add_argument("parquet", nargs="?", default="data/csn_python_test.parquet")
+    p.add_argument("--mode", choices=["keyword", "meaning", "both", "all"], default="keyword")
+    p.add_argument("--model", default="BAAI/bge-small-en-v1.5")
+    a = p.parse_args()
+    r = run(Path(a.parquet), a.mode, a.model, progress=lambda m: print(m, flush=True))
+    print(f"CodeSearchNet Python, {Path(a.parquet).name}" + (f", model {a.model}" if a.mode != "keyword" else ""))
+    print(f"Queries: {r['queries']} in {r['groups']} groups of {GROUP} (1 right answer, 999 distractors), seed {SEED}")
+    for m, v in r["modes"].items():
+        print(f"{m:8} MRR {v['mrr']:.4f}   hit rate at {TOP}: {v['hit_at_5']:.4f}")
+    print(f"Functions that didn't parse (docstring left in): {r['unparsed_code']}")
+    print(f"Queries with no keyword matches: {r['queries_with_no_words']}")