whensaid 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. whensaid-0.1.0/.github/workflows/ci.yml +57 -0
  2. whensaid-0.1.0/.github/workflows/publish.yml +20 -0
  3. whensaid-0.1.0/.gitignore +14 -0
  4. whensaid-0.1.0/LICENSE +21 -0
  5. whensaid-0.1.0/PKG-INFO +146 -0
  6. whensaid-0.1.0/README.md +114 -0
  7. whensaid-0.1.0/assets/demo.gif +0 -0
  8. whensaid-0.1.0/pyproject.toml +45 -0
  9. whensaid-0.1.0/scripts/bench_rerank.py +120 -0
  10. whensaid-0.1.0/scripts/compare_models.py +122 -0
  11. whensaid-0.1.0/scripts/eval.py +66 -0
  12. whensaid-0.1.0/scripts/make_demo.py +57 -0
  13. whensaid-0.1.0/scripts/smoke.py +33 -0
  14. whensaid-0.1.0/src/whensaid/__init__.py +1 -0
  15. whensaid-0.1.0/src/whensaid/chunk.py +46 -0
  16. whensaid-0.1.0/src/whensaid/cli.py +246 -0
  17. whensaid-0.1.0/src/whensaid/cudalibs.py +43 -0
  18. whensaid-0.1.0/src/whensaid/db.py +120 -0
  19. whensaid-0.1.0/src/whensaid/doctor.py +121 -0
  20. whensaid-0.1.0/src/whensaid/embed.py +30 -0
  21. whensaid-0.1.0/src/whensaid/export.py +50 -0
  22. whensaid-0.1.0/src/whensaid/fetch.py +196 -0
  23. whensaid-0.1.0/src/whensaid/indexer.py +335 -0
  24. whensaid-0.1.0/src/whensaid/models.py +260 -0
  25. whensaid-0.1.0/src/whensaid/paths.py +18 -0
  26. whensaid-0.1.0/src/whensaid/rerank.py +80 -0
  27. whensaid-0.1.0/src/whensaid/search.py +170 -0
  28. whensaid-0.1.0/src/whensaid/server.py +214 -0
  29. whensaid-0.1.0/src/whensaid/subtitles.py +99 -0
  30. whensaid-0.1.0/src/whensaid/transcribe.py +136 -0
  31. whensaid-0.1.0/src/whensaid/urls.py +104 -0
  32. whensaid-0.1.0/src/whensaid/util.py +55 -0
  33. whensaid-0.1.0/tests/data/README.md +25 -0
  34. whensaid-0.1.0/tests/data/jfk.wav +0 -0
  35. whensaid-0.1.0/tests/fixtures/autocaptions.vtt +43 -0
  36. whensaid-0.1.0/tests/golden.json +185 -0
  37. whensaid-0.1.0/tests/golden_noisy.json +220 -0
  38. whensaid-0.1.0/tests/golden_scale.json +262 -0
  39. whensaid-0.1.0/tests/test_chunk.py +48 -0
  40. whensaid-0.1.0/tests/test_doctor.py +49 -0
  41. whensaid-0.1.0/tests/test_export.py +69 -0
  42. whensaid-0.1.0/tests/test_indexer.py +287 -0
  43. whensaid-0.1.0/tests/test_models.py +272 -0
  44. whensaid-0.1.0/tests/test_search.py +111 -0
  45. whensaid-0.1.0/tests/test_server.py +132 -0
  46. whensaid-0.1.0/tests/test_split.py +58 -0
  47. whensaid-0.1.0/tests/test_subtitles.py +129 -0
  48. whensaid-0.1.0/tests/test_urls.py +351 -0
@@ -0,0 +1,57 @@
1
+ name: ci
2
+
3
+ on:
4
+ push:
5
+ pull_request:
6
+ workflow_dispatch:
7
+
8
+ concurrency:
9
+ group: ci-${{ github.ref }}
10
+ cancel-in-progress: true
11
+
12
+ jobs:
13
+ test:
14
+ runs-on: ${{ matrix.os }}
15
+ strategy:
16
+ fail-fast: false
17
+ matrix:
18
+ os: [ubuntu-latest, macos-latest]
19
+ python: ["3.12"]
20
+ include:
21
+ - os: ubuntu-latest
22
+ python: "3.11"
23
+ steps:
24
+ - uses: actions/checkout@v4
25
+ - uses: actions/setup-python@v5
26
+ with:
27
+ python-version: ${{ matrix.python }}
28
+ cache: pip
29
+ - run: pip install -e ".[dev]"
30
+ - run: pytest -q
31
+
32
+ # Installs the package as a user would (no GPU extras), downloads the real models and
33
+ # searches a real clip. Slow and heavy, so only on main and by hand. macOS minutes count
34
+ # ten times on a private repo.
35
+ smoke:
36
+ needs: test
37
+ if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
38
+ runs-on: ${{ matrix.os }}
39
+ strategy:
40
+ fail-fast: false
41
+ matrix:
42
+ os: [ubuntu-latest, macos-latest]
43
+ env:
44
+ WHENSAID_HOME: ${{ github.workspace }}/.smoke-home
45
+ WHENSAID_WHISPER_MODEL: small.en # the default model is 1.5 GB; this one is enough to find a phrase
46
+ steps:
47
+ - uses: actions/checkout@v4
48
+ - uses: actions/setup-python@v5
49
+ with:
50
+ python-version: "3.12"
51
+ cache: pip
52
+ - uses: actions/cache@v4
53
+ with:
54
+ path: .smoke-home/models
55
+ key: models-${{ runner.os }}-small.en-v1
56
+ - run: pip install .
57
+ - run: python scripts/smoke.py tests/data
@@ -0,0 +1,20 @@
1
+ name: publish
2
+
3
+ on:
4
+ release:
5
+ types: [published]
6
+ workflow_dispatch:
7
+
8
+ jobs:
9
+ pypi:
10
+ runs-on: ubuntu-latest
11
+ environment: pypi
12
+ permissions:
13
+ id-token: write # trusted publishing, no API token stored anywhere
14
+ steps:
15
+ - uses: actions/checkout@v4
16
+ - uses: actions/setup-python@v5
17
+ with:
18
+ python-version: "3.12"
19
+ - run: pip install build && python -m build
20
+ - uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,14 @@
1
+ .venv/
2
+ __pycache__/
3
+ *.egg-info/
4
+ dist/
5
+ *.db
6
+ *.db-wal
7
+ *.db-shm
8
+ models/
9
+ tests/data/**/*.mp4
10
+ tests/data/**/*.mkv
11
+ tests/data/**/*.wav
12
+ tests/data/**/*.mp3
13
+ !tests/data/jfk.wav
14
+ docs/
whensaid-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 VishnupiriyanV
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,146 @@
1
+ Metadata-Version: 2.5
2
+ Name: whensaid
3
+ Version: 0.1.0
4
+ Summary: Local, offline search over your video and audio recordings, exposed to AI agents over MCP.
5
+ Project-URL: Homepage, https://github.com/VishnupiriyanV/whensaid
6
+ Project-URL: Issues, https://github.com/VishnupiriyanV/whensaid/issues
7
+ Author: VishnupiriyanV
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Keywords: audio,local,mcp,offline,search,transcription,video,whisper
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Environment :: Console
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Operating System :: POSIX :: Linux
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Topic :: Multimedia :: Sound/Audio :: Speech
17
+ Classifier: Topic :: Text Processing :: Indexing
18
+ Requires-Python: >=3.11
19
+ Requires-Dist: av<17,>=14
20
+ Requires-Dist: fastembed<1,>=0.9
21
+ Requires-Dist: faster-whisper<2,>=1.2
22
+ Requires-Dist: mcp<3,>=2
23
+ Requires-Dist: numpy>=1.26
24
+ Provides-Extra: cuda
25
+ Requires-Dist: nvidia-cublas-cu12; extra == 'cuda'
26
+ Requires-Dist: nvidia-cudnn-cu12; extra == 'cuda'
27
+ Provides-Extra: dev
28
+ Requires-Dist: pytest; extra == 'dev'
29
+ Provides-Extra: url
30
+ Requires-Dist: yt-dlp; extra == 'url'
31
+ Description-Content-Type: text/markdown
32
+
33
+ # whensaid
34
+
35
+ Search what was *said* in your local video and audio files, from your AI agent.
36
+
37
+ Point it at folders of recordings (talks, courses, meetings, interviews, podcasts) or at a YouTube playlist.
38
+ Your agent (Claude Code, Cursor, ...) calls `search("when did they talk about X")` and gets back the file, the exact
39
+ timestamp and the passage; for web videos, a link that opens the video at that moment. Everything runs on your
40
+ machine, and nothing is uploaded.
41
+
42
+ **Status: pre-alpha.** Tested on Linux (CI also runs on macOS). Runs on an NVIDIA GPU or on CPU. English only.
43
+
44
+ ## A worked example
45
+
46
+ ![terminal demo: two searches that find the passage and one off-topic question that returns nothing](https://raw.githubusercontent.com/VishnupiriyanV/whensaid/main/assets/demo.gif)
47
+
48
+ The recording above is real `whensaid` output (rendered by `scripts/make_demo.py`), run in a terminal rather than from an agent session. Text version:
49
+
50
+ The index here is 198 public-domain recordings (65 hours of audiobooks and old radio). The question is paraphrased,
51
+ not a quote:
52
+
53
+ ```
54
+ $ whensaid search "a mouse tribe claiming rights over everyone else" -n 1
55
+ strong the_consolation_of_philosophy_..._boethius_64kb.mp3 0:01:38 and over whom ye exercise your fancied
56
+ lordship suppose now that in the mouse tribe there should rise up one claiming rights and powers for himself above the rest ...
57
+ ```
58
+
59
+ Through MCP the agent gets the same thing as JSON (`file`, `path`, `start`, `at`, `text`, `match`, `relevance`, and
60
+ `link` for web videos) and can call `get_transcript` to read around the hit. About half a second per search on CPU.
61
+
62
+ ## Use it
63
+
64
+ ```bash
65
+ pip install "whensaid[url]" # add cuda on an NVIDIA GPU: "whensaid[cuda,url]" (or: uv tool install "whensaid[url]")
66
+ whensaid doctor # checks the GPU, models, yt-dlp and the index, says how to fix problems
67
+ whensaid add ~/Videos/talks # register a folder of recordings
68
+ whensaid add-url https://www.youtube.com/playlist?list=... # or a video, playlist or channel
69
+ whensaid index # transcribe and index what is new (models download on first use)
70
+ whensaid index --watch # or keep running and pick up new files every 30 s
71
+ whensaid index --subtitles # use a .srt/.vtt next to a recording instead of transcribing it
72
+ whensaid remove ~/Videos/old # stop watching a folder, playlist or video and drop it from the index
73
+ whensaid status
74
+ ```
75
+
76
+ The models (about 1.6 GB for the default set) are downloaded the first time they are needed. `whensaid models pull`
77
+ fetches them ahead of time, and `WHENSAID_OFFLINE=1` forbids any download.
78
+
79
+ Then give your agent the MCP server (stdio):
80
+
81
+ ```bash
82
+ claude mcp add whensaid -- whensaid serve
83
+ ```
84
+
85
+ Tools the agent gets (all read-only): `search`, `get_transcript`, `list_sources`, `index_status`.
86
+ Indexing is only ever done by the `whensaid index` command, never by the agent.
87
+
88
+ ## How good is it
89
+
90
+ Measured on the 65-hour public-domain index above, with questions written as paraphrases (the question sets are in `tests/`, the scripts in `scripts/`):
91
+
92
+ - The right passage is the top result for 83-97% of questions and in the top 5 for 96-100%, depending on the question set.
93
+ - Off-topic questions: 33 of 40 return nothing, and 6 of the other 7 come back marked `weak`.
94
+ - Against an agent that greps plain transcript files (`whensaid export`), with Claude Sonnet agents answering 36
95
+ questions: grep got 33 right and whensaid 31. That is a tie within noise for one run each, **so whensaid is not
96
+ more accurate than grep at this size**. What it adds is speed (an agent finished a batch of 8 questions in about a
97
+ minute instead of about nine), no export step, timestamp links for web videos, and a library that persists between
98
+ sessions. One run per arm, questions written by
99
+ someone who had read the transcripts. An earlier run looked much worse (25 of 36) only because the CLI cut each
100
+ passage to 200 characters; that is fixed.
101
+ - Not measured: word error rate against human transcripts, noisy or multi-speaker recordings beyond old radio, and any
102
+ hardware other than one Linux laptop (12 cores, RTX 3050).
103
+
104
+ ## Limits
105
+
106
+ - English only. Videos in other languages, and recordings the model is unsure of, are skipped (see below).
107
+ - Linux is the tested platform; macOS runs in CI but nobody has used it daily. Windows is untried.
108
+ - Speech only. It does not look at the picture.
109
+ - Transcription is the slow part: about 14x realtime on the GPU it was tested on, about 6x on 12 CPU cores with the
110
+ default model. `WHENSAID_WHISPER_MODEL=small.en` trades accuracy for speed.
111
+ - Web videos are for videos you are allowed to download; each site's terms still apply and are your responsibility.
112
+
113
+ ## Details
114
+
115
+ `whensaid index` also drops files that were deleted or moved (unless their folder is missing or holds no media at all,
116
+ which looks like an unplugged drive). With `--watch`, files are indexed once they have stopped changing for 15 s, and
117
+ a file that failed is retried only after it changes.
118
+
119
+ Web videos use their captions when there are any (seconds, no GPU) and are transcribed from the audio otherwise.
120
+ Playlists and channels are checked for new videos at most once an hour.
121
+
122
+ Things that will never work are skipped once and not asked for again: live streams, private or removed videos, videos
123
+ that are not in English, and recordings whose transcript the model itself was unsure of (usually another language, or
124
+ music). `whensaid status` lists them with the reason; `whensaid index --retry-skipped` tries them again, and
125
+ `WHENSAID_MIN_CONFIDENCE=0` turns the transcript check off.
126
+
127
+ Set `WHENSAID_HOME` to keep the index somewhere other than `~/.whensaid`.
128
+
129
+ ## How it works
130
+
131
+ ffmpeg-free decode (PyAV) -> faster-whisper with VAD and word timestamps -> ~45 s overlapping
132
+ chunks -> bge-small embeddings -> one SQLite file (FTS5 keyword index + vectors).
133
+ Search fuses keyword and vector ranks (reciprocal rank fusion), then a small cross-encoder reranks the top 20
134
+ (`WHENSAID_RERANK=0` skips it). Passages it scores as unrelated are dropped, so off-topic questions usually return
135
+ nothing, and borderline hits come back marked `weak` (`WHENSAID_MIN_RELEVANCE` tunes this, `off` disables).
136
+
137
+ ## Develop
138
+
139
+ ```bash
140
+ uv pip install -e ".[dev]" && pytest
141
+ WHENSAID_HOME=<index with tests/data indexed> python scripts/eval.py --verbose
142
+ ```
143
+
144
+ Test media is public domain; see [tests/data/README.md](tests/data/README.md).
145
+
146
+ License: MIT
@@ -0,0 +1,114 @@
1
+ # whensaid
2
+
3
+ Search what was *said* in your local video and audio files, from your AI agent.
4
+
5
+ Point it at folders of recordings (talks, courses, meetings, interviews, podcasts) or at a YouTube playlist.
6
+ Your agent (Claude Code, Cursor, ...) calls `search("when did they talk about X")` and gets back the file, the exact
7
+ timestamp and the passage; for web videos, a link that opens the video at that moment. Everything runs on your
8
+ machine, and nothing is uploaded.
9
+
10
+ **Status: pre-alpha.** Tested on Linux (CI also runs on macOS). Runs on an NVIDIA GPU or on CPU. English only.
11
+
12
+ ## A worked example
13
+
14
+ ![terminal demo: two searches that find the passage and one off-topic question that returns nothing](https://raw.githubusercontent.com/VishnupiriyanV/whensaid/main/assets/demo.gif)
15
+
16
+ The recording above is real `whensaid` output (rendered by `scripts/make_demo.py`), run in a terminal rather than from an agent session. Text version:
17
+
18
+ The index here is 198 public-domain recordings (65 hours of audiobooks and old radio). The question is paraphrased,
19
+ not a quote:
20
+
21
+ ```
22
+ $ whensaid search "a mouse tribe claiming rights over everyone else" -n 1
23
+ strong the_consolation_of_philosophy_..._boethius_64kb.mp3 0:01:38 and over whom ye exercise your fancied
24
+ lordship suppose now that in the mouse tribe there should rise up one claiming rights and powers for himself above the rest ...
25
+ ```
26
+
27
+ Through MCP the agent gets the same thing as JSON (`file`, `path`, `start`, `at`, `text`, `match`, `relevance`, and
28
+ `link` for web videos) and can call `get_transcript` to read around the hit. About half a second per search on CPU.
29
+
30
+ ## Use it
31
+
32
+ ```bash
33
+ pip install "whensaid[url]" # add cuda on an NVIDIA GPU: "whensaid[cuda,url]" (or: uv tool install "whensaid[url]")
34
+ whensaid doctor # checks the GPU, models, yt-dlp and the index, says how to fix problems
35
+ whensaid add ~/Videos/talks # register a folder of recordings
36
+ whensaid add-url https://www.youtube.com/playlist?list=... # or a video, playlist or channel
37
+ whensaid index # transcribe and index what is new (models download on first use)
38
+ whensaid index --watch # or keep running and pick up new files every 30 s
39
+ whensaid index --subtitles # use a .srt/.vtt next to a recording instead of transcribing it
40
+ whensaid remove ~/Videos/old # stop watching a folder, playlist or video and drop it from the index
41
+ whensaid status
42
+ ```
43
+
44
+ The models (about 1.6 GB for the default set) are downloaded the first time they are needed. `whensaid models pull`
45
+ fetches them ahead of time, and `WHENSAID_OFFLINE=1` forbids any download.
46
+
47
+ Then give your agent the MCP server (stdio):
48
+
49
+ ```bash
50
+ claude mcp add whensaid -- whensaid serve
51
+ ```
52
+
53
+ Tools the agent gets (all read-only): `search`, `get_transcript`, `list_sources`, `index_status`.
54
+ Indexing is only ever done by the `whensaid index` command, never by the agent.
55
+
56
+ ## How good is it
57
+
58
+ Measured on the 65-hour public-domain index above, with questions written as paraphrases (the question sets are in `tests/`, the scripts in `scripts/`):
59
+
60
+ - The right passage is the top result for 83-97% of questions and in the top 5 for 96-100%, depending on the question set.
61
+ - Off-topic questions: 33 of 40 return nothing, and 6 of the other 7 come back marked `weak`.
62
+ - Against an agent that greps plain transcript files (`whensaid export`), with Claude Sonnet agents answering 36
63
+ questions: grep got 33 right and whensaid 31. That is a tie within noise for one run each, **so whensaid is not
64
+ more accurate than grep at this size**. What it adds is speed (an agent finished a batch of 8 questions in about a
65
+ minute instead of about nine), no export step, timestamp links for web videos, and a library that persists between
66
+ sessions. One run per arm, questions written by
67
+ someone who had read the transcripts. An earlier run looked much worse (25 of 36) only because the CLI cut each
68
+ passage to 200 characters; that is fixed.
69
+ - Not measured: word error rate against human transcripts, noisy or multi-speaker recordings beyond old radio, and any
70
+ hardware other than one Linux laptop (12 cores, RTX 3050).
71
+
72
+ ## Limits
73
+
74
+ - English only. Videos in other languages, and recordings the model is unsure of, are skipped (see below).
75
+ - Linux is the tested platform; macOS runs in CI but nobody has used it daily. Windows is untried.
76
+ - Speech only. It does not look at the picture.
77
+ - Transcription is the slow part: about 14x realtime on the GPU it was tested on, about 6x on 12 CPU cores with the
78
+ default model. `WHENSAID_WHISPER_MODEL=small.en` trades accuracy for speed.
79
+ - Web videos are for videos you are allowed to download; each site's terms still apply and are your responsibility.
80
+
81
+ ## Details
82
+
83
+ `whensaid index` also drops files that were deleted or moved (unless their folder is missing or holds no media at all,
84
+ which looks like an unplugged drive). With `--watch`, files are indexed once they have stopped changing for 15 s, and
85
+ a file that failed is retried only after it changes.
86
+
87
+ Web videos use their captions when there are any (seconds, no GPU) and are transcribed from the audio otherwise.
88
+ Playlists and channels are checked for new videos at most once an hour.
89
+
90
+ Things that will never work are skipped once and not asked for again: live streams, private or removed videos, videos
91
+ that are not in English, and recordings whose transcript the model itself was unsure of (usually another language, or
92
+ music). `whensaid status` lists them with the reason; `whensaid index --retry-skipped` tries them again, and
93
+ `WHENSAID_MIN_CONFIDENCE=0` turns the transcript check off.
94
+
95
+ Set `WHENSAID_HOME` to keep the index somewhere other than `~/.whensaid`.
96
+
97
+ ## How it works
98
+
99
+ ffmpeg-free decode (PyAV) -> faster-whisper with VAD and word timestamps -> ~45 s overlapping
100
+ chunks -> bge-small embeddings -> one SQLite file (FTS5 keyword index + vectors).
101
+ Search fuses keyword and vector ranks (reciprocal rank fusion), then a small cross-encoder reranks the top 20
102
+ (`WHENSAID_RERANK=0` skips it). Passages it scores as unrelated are dropped, so off-topic questions usually return
103
+ nothing, and borderline hits come back marked `weak` (`WHENSAID_MIN_RELEVANCE` tunes this, `off` disables).
104
+
105
+ ## Develop
106
+
107
+ ```bash
108
+ uv pip install -e ".[dev]" && pytest
109
+ WHENSAID_HOME=<index with tests/data indexed> python scripts/eval.py --verbose
110
+ ```
111
+
112
+ Test media is public domain; see [tests/data/README.md](tests/data/README.md).
113
+
114
+ License: MIT
Binary file
@@ -0,0 +1,45 @@
1
+ [project]
2
+ name = "whensaid"
3
+ version = "0.1.0"
4
+ description = "Local, offline search over your video and audio recordings, exposed to AI agents over MCP."
5
+ readme = "README.md"
6
+ license = "MIT"
7
+ license-files = ["LICENSE"]
8
+ authors = [{ name = "VishnupiriyanV" }]
9
+ keywords = ["mcp", "whisper", "transcription", "search", "video", "audio", "local", "offline"]
10
+ classifiers = [
11
+ "Development Status :: 3 - Alpha",
12
+ "Environment :: Console",
13
+ "Intended Audience :: Developers",
14
+ "Operating System :: POSIX :: Linux",
15
+ "Programming Language :: Python :: 3",
16
+ "Topic :: Multimedia :: Sound/Audio :: Speech",
17
+ "Topic :: Text Processing :: Indexing",
18
+ ]
19
+ requires-python = ">=3.11"
20
+ dependencies = [
21
+ "mcp>=2,<3",
22
+ "faster-whisper>=1.2,<2",
23
+ "av>=14,<17", # faster-whisper 1.2.1 breaks on av>=19 (verified 2026-11-10)
24
+ "fastembed>=0.9,<1", # ONNX embeddings, no torch
25
+ "numpy>=1.26",
26
+ ]
27
+
28
+ [project.optional-dependencies]
29
+ cuda = ["nvidia-cublas-cu12", "nvidia-cudnn-cu12"] # what ctranslate2 needs on an NVIDIA GPU
30
+ url = ["yt-dlp"] # for `whensaid add-url`
31
+ dev = ["pytest"]
32
+
33
+ [project.urls]
34
+ Homepage = "https://github.com/VishnupiriyanV/whensaid"
35
+ Issues = "https://github.com/VishnupiriyanV/whensaid/issues"
36
+
37
+ [project.scripts]
38
+ whensaid = "whensaid.cli:main"
39
+
40
+ [build-system]
41
+ requires = ["hatchling"]
42
+ build-backend = "hatchling.build"
43
+
44
+ [tool.hatch.build.targets.wheel]
45
+ packages = ["src/whensaid"]
@@ -0,0 +1,120 @@
1
+ """Compare reranker variants for speed and accuracy on the golden sets.
2
+
3
+ WHENSAID_HOME=<home with everything indexed> python scripts/bench_rerank.py
4
+
5
+ The fused keyword+semantic candidates are computed once per query, then each variant only
6
+ reorders them, so differences between rows come from the reranker alone. Latency is measured
7
+ with a query embedding run just before each rerank, the way a real search does it.
8
+ """
9
+ import argparse
10
+ import json
11
+ import os
12
+ import statistics
13
+ import time
14
+ from pathlib import Path
15
+
16
+ from fastembed.common.model_description import ModelSource
17
+ from fastembed.rerank.cross_encoder import TextCrossEncoder
18
+
19
+ from whensaid import db, paths
20
+ from whensaid.embed import Embedder
21
+ from whensaid.paths import models_dir
22
+ from whensaid.search import Searcher, rrf
23
+
24
+ TESTS = Path(__file__).resolve().parent.parent / "tests"
25
+ SETS = ["golden.json", "golden_noisy.json", "golden_scale.json"]
26
+
27
+ # name -> (hf repo, onnx file)
28
+ MODELS = {
29
+ "L12": ("Xenova/ms-marco-MiniLM-L-12-v2", "onnx/model.onnx"),
30
+ "L12-q": ("Xenova/ms-marco-MiniLM-L-12-v2", "onnx/model_quantized.onnx"),
31
+ "L6": ("Xenova/ms-marco-MiniLM-L-6-v2", "onnx/model.onnx"),
32
+ "L6-q": ("Xenova/ms-marco-MiniLM-L-6-v2", "onnx/model_quantized.onnx"),
33
+ }
34
+
35
+
36
+ def load(name, threads):
37
+ repo, file = MODELS[name]
38
+ custom = f"whensaid-bench/{name}"
39
+ try:
40
+ TextCrossEncoder.add_custom_model(model=custom, sources=ModelSource(hf=repo), model_file=file)
41
+ except ValueError:
42
+ pass # registered on an earlier call
43
+ return TextCrossEncoder(custom, cache_dir=str(models_dir() / "fastembed"), threads=threads)
44
+
45
+
46
+ def is_hit(row, q):
47
+ return any(
48
+ os.path.basename(row["path"]) == a["file"] and row["start"] < a["end"] and row["end"] > a["start"]
49
+ for a in [q, *q.get("alt", [])]
50
+ )
51
+
52
+
53
+ def first_hit(rows, q, limit=10):
54
+ kept = []
55
+ for r in rows:
56
+ if any(k["path"] == r["path"] and k["start"] < r["end"] and r["start"] < k["end"] for k in kept):
57
+ continue # same passage as a better hit, as the searcher does
58
+ kept.append(r)
59
+ if is_hit(r, q):
60
+ return len(kept)
61
+ if len(kept) >= limit:
62
+ break
63
+ return None
64
+
65
+
66
+ def main():
67
+ ap = argparse.ArgumentParser()
68
+ ap.add_argument("--variants", default="L12:0,L12:4,L12-q:4,L6:4,L6-q:4", help="model:threads, 0 = library default")
69
+ ap.add_argument("--depths", default="20,10")
70
+ args = ap.parse_args()
71
+
72
+ conn = db.connect_readonly(paths.db_path())
73
+ embedder = Embedder()
74
+ searcher = Searcher(conn, embedder)
75
+ queries = []
76
+ for name in SETS:
77
+ for q in json.load(open(TESTS / name))["queries"]:
78
+ queries.append((name, q))
79
+
80
+ # candidates once per query: fused order, with the rows needed to score a hit
81
+ cands = []
82
+ for _, q in queries:
83
+ qv = searcher.query_vector(q["query"])
84
+ fused = rrf({"keyword": searcher.keyword_ids(q["query"], 50), "semantic": searcher.semantic_ids(q["query"], 50, qv)})
85
+ ids = [cid for cid, _, _ in fused[:40]]
86
+ marks = ",".join("?" * len(ids))
87
+ rows = {r["id"]: r for r in conn.execute(
88
+ f"select c.id, c.start, c.end, c.text, s.path from chunks c join sources s on s.id = c.source_id where c.id in ({marks})", ids)}
89
+ cands.append([rows[i] for i in ids])
90
+
91
+ def report(label, ranked, ms=None):
92
+ cells = []
93
+ for name in SETS:
94
+ rs = [first_hit(r, q) for (n, q), r in zip(queries, ranked) if n == name]
95
+ cells.append(f"{sum(1 for r in rs if r == 1) / len(rs):.2f}/{sum(1 for r in rs if r and r <= 5) / len(rs):.2f}")
96
+ t = "" if ms is None else f"{statistics.median(ms):5.0f} {max(ms):5.0f}"
97
+ print(f"{label:16} {cells[0]:>10} {cells[1]:>10} {cells[2]:>10} {t}")
98
+
99
+ print(f"{len(queries)} queries, R@1/R@5 per set, then median and max rerank ms")
100
+ print(f"{'variant':16} {'clean':>10} {'noisy':>10} {'scale':>10} {'med':>5} {'max':>5}")
101
+ report("no reranker", cands)
102
+ for spec in args.variants.split(","):
103
+ name, threads = spec.split(":")
104
+ rr = load(name, int(threads) or None)
105
+ list(rr.rerank("warm up", ["warm up"]))
106
+ for depth in [int(d) for d in args.depths.split(",")]:
107
+ ranked, ms = [], []
108
+ for (_, q), rows in zip(queries, cands):
109
+ embedder.encode([q["query"]]) # a search embeds the query right before reranking
110
+ head = rows[:depth]
111
+ t = time.time()
112
+ scores = list(rr.rerank(q["query"], [r["text"] for r in head]))
113
+ ms.append((time.time() - t) * 1000)
114
+ order = [r for _, r in sorted(zip(scores, head), key=lambda x: -x[0])]
115
+ ranked.append(order + rows[depth:])
116
+ report(f"{name} t={threads} d={depth}", ranked, ms)
117
+
118
+
119
+ if __name__ == "__main__":
120
+ main()
@@ -0,0 +1,122 @@
1
+ """Try other embedding models and rerankers on the golden set, without touching the index.
2
+
3
+ WHENSAID_HOME=<home> python scripts/compare_models.py --embedders bge-base,gte-base --rerankers jina-turbo
4
+
5
+ Chunks are re-embedded in memory for each candidate. Hybrid is keyword + semantic fused with
6
+ rrf as in the real searcher, and a reranker (if given) reorders the top 20 of that.
7
+ """
8
+ import argparse
9
+ import json
10
+ import os
11
+ import time
12
+ from pathlib import Path
13
+
14
+ import numpy as np
15
+ from fastembed import TextEmbedding
16
+ from fastembed.rerank.cross_encoder import TextCrossEncoder
17
+
18
+ from whensaid import db, paths
19
+ from whensaid.search import Searcher, rrf
20
+
21
+ GOLDEN = Path(__file__).resolve().parent.parent / "tests" / "golden.json"
22
+ BGE = "Represent this sentence for searching relevant passages: "
23
+
24
+ # name -> (model id, query prefix, passage prefix)
25
+ EMBEDDERS = {
26
+ "bge-small": ("BAAI/bge-small-en-v1.5", BGE, ""),
27
+ "bge-base": ("BAAI/bge-base-en-v1.5", BGE, ""),
28
+ "bge-large": ("BAAI/bge-large-en-v1.5", BGE, ""),
29
+ "arctic-m": ("snowflake/snowflake-arctic-embed-m", BGE, ""),
30
+ "gte-base": ("thenlper/gte-base", "", ""),
31
+ "nomic-q": ("nomic-ai/nomic-embed-text-v1.5-Q", "search_query: ", "search_document: "),
32
+ "mxbai": ("mixedbread-ai/mxbai-embed-large-v1", BGE, ""),
33
+ }
34
+ RERANKERS = {
35
+ "minilm-12": "Xenova/ms-marco-MiniLM-L-12-v2",
36
+ "jina-turbo": "jinaai/jina-reranker-v1-turbo-en",
37
+ "bge-rr-base": "BAAI/bge-reranker-base",
38
+ }
39
+
40
+
41
+ def is_hit(path, start, end, q):
42
+ return any(os.path.basename(path) == a["file"] and start < a["end"] and end > a["start"] for a in [q, *q.get("alt", [])])
43
+
44
+
45
+ def score(ranked, golden):
46
+ """ranked: one list of (path, start, end) per query, best first."""
47
+ rows = []
48
+ for q, hits in zip(golden, ranked):
49
+ r = next((i for i, h in enumerate(hits, 1) if is_hit(*h[:3], q)), None)
50
+ rows.append(r)
51
+ n = len(rows)
52
+ at = lambda k: sum(1 for r in rows if r and r <= k) / n
53
+ return at(1), at(5), at(10), sum(1 / r for r in rows if r) / n
54
+
55
+
56
+ def dedupe(hits, limit=10):
57
+ out = []
58
+ for h in hits:
59
+ if any(o[0] == h[0] and o[1] < h[2] and h[1] < o[2] for o in out):
60
+ continue
61
+ out.append(h)
62
+ return out[:limit]
63
+
64
+
65
+ def main():
66
+ ap = argparse.ArgumentParser()
67
+ ap.add_argument("--embedders", default="bge-small,bge-base,gte-base,arctic-m,nomic-q")
68
+ ap.add_argument("--rerankers", default="minilm-12,jina-turbo,bge-rr-base")
69
+ ap.add_argument("--depth", type=int, default=20, help="how many fused hits the reranker sees")
70
+ args = ap.parse_args()
71
+
72
+ golden = [q for q in json.load(open(GOLDEN))["queries"]]
73
+ conn = db.connect(paths.db_path())
74
+ chunks = conn.execute(
75
+ "select c.id, c.start, c.end, c.text, s.path from chunks c join sources s on s.id = c.source_id order by c.id"
76
+ ).fetchall()
77
+ ids = [c["id"] for c in chunks]
78
+ by_id = {c["id"]: c for c in chunks}
79
+ print(f"{len(chunks)} chunks, {len(golden)} queries")
80
+
81
+ kw = Searcher(conn, None) # only used for its keyword side
82
+ keyword = [kw.keyword_ids(q["query"], 50) for q in golden]
83
+
84
+ def as_hits(chunk_ids):
85
+ return [(by_id[i]["path"], by_id[i]["start"], by_id[i]["end"], i) for i in chunk_ids]
86
+
87
+ rerank_models = {}
88
+ print(f"\n{'embedder':10} {'stage':22} {'R@1':>5} {'R@5':>5} {'R@10':>5} {'MRR':>5} {'embed s':>7}")
89
+ for name in args.embedders.split(","):
90
+ model_id, qp, pp = EMBEDDERS[name]
91
+ t0 = time.time()
92
+ emb = TextEmbedding(model_id)
93
+ mat = np.array(list(emb.embed([pp + c["text"] for c in chunks], batch_size=32)), dtype=np.float32)
94
+ mat /= np.linalg.norm(mat, axis=1, keepdims=True)
95
+ took = time.time() - t0
96
+ qv = np.array(list(emb.embed([qp + q["query"] for q in golden])), dtype=np.float32)
97
+ qv /= np.linalg.norm(qv, axis=1, keepdims=True)
98
+
99
+ sem = [[ids[j] for j in np.argsort(-(mat @ v))[:50]] for v in qv]
100
+ fused = [[cid for cid, _, _ in rrf({"keyword": k, "semantic": s})] for k, s in zip(keyword, sem)]
101
+
102
+ def show(stage, lists, t=None):
103
+ ranked = [dedupe(as_hits(l)) for l in lists]
104
+ r1, r5, r10, mrr = score(ranked, golden)
105
+ print(f"{name:10} {stage:22} {r1:5.2f} {r5:5.2f} {r10:5.2f} {mrr:5.2f} {t if t else '':>7}")
106
+
107
+ show("semantic", sem, f"{took:.0f}")
108
+ show("hybrid", fused)
109
+ for rname in filter(None, args.rerankers.split(",")):
110
+ if rname not in rerank_models:
111
+ rerank_models[rname] = TextCrossEncoder(RERANKERS[rname])
112
+ rr = rerank_models[rname]
113
+ out = []
114
+ for q, f in zip(golden, fused):
115
+ top = f[: args.depth]
116
+ scores = list(rr.rerank(q["query"], [by_id[i]["text"] for i in top]))
117
+ out.append([i for _, i in sorted(zip(scores, top), key=lambda x: -x[0])])
118
+ show(f"hybrid+{rname}", out)
119
+
120
+
121
+ if __name__ == "__main__":
122
+ main()