whensaid 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
whensaid/__init__.py ADDED
@@ -0,0 +1 @@
1
+ __version__ = "0.0.1"
whensaid/chunk.py ADDED
@@ -0,0 +1,46 @@
1
+ from dataclasses import dataclass
2
+
3
+
4
+ @dataclass
5
+ class Segment:
6
+ start: float
7
+ end: float
8
+ text: str
9
+
10
+
11
+ def make_chunks(
12
+ segments: list[Segment], target: float = 45.0, overlap: float = 6.0, max_gap: float = 8.0
13
+ ) -> list[Segment]:
14
+ """Group whisper segments into windows of roughly `target` seconds.
15
+
16
+ A new window starts with the last few segments of the previous one (anything
17
+ that started within `overlap` seconds of its end) so a sentence on the
18
+ boundary shows up in both. A silence longer than `max_gap` (music, a title
19
+ card) ends the window outright, with no overlap across it.
20
+ """
21
+ chunks: list[Segment] = []
22
+ cur: list[Segment] = []
23
+
24
+ for seg in segments:
25
+ if cur and seg.start - cur[-1].end > max_gap:
26
+ chunks.append(_join(cur))
27
+ cur = []
28
+ elif cur and seg.end - cur[0].start > target:
29
+ chunks.append(_join(cur))
30
+ cur = _tail(cur, overlap)
31
+ if cur and seg.end - cur[0].start > target:
32
+ cur = [] # the carried tail would blow the window right away
33
+ cur.append(seg)
34
+
35
+ if cur and (not chunks or cur[-1].end > chunks[-1].end):
36
+ chunks.append(_join(cur))
37
+ return chunks
38
+
39
+
40
+ def _tail(segs: list[Segment], overlap: float) -> list[Segment]:
41
+ cutoff = segs[-1].end - overlap
42
+ return [s for s in segs[1:] if s.start >= cutoff] # never carry the whole window
43
+
44
+
45
+ def _join(segs: list[Segment]) -> Segment:
46
+ return Segment(segs[0].start, segs[-1].end, " ".join(s.text.strip() for s in segs))
whensaid/cli.py ADDED
@@ -0,0 +1,246 @@
1
+ import argparse
2
+ import os
3
+ import sys
4
+ from pathlib import Path
5
+
6
+ from . import db, paths
7
+
8
+
9
+ def cmd_add(args):
10
+ folder = os.path.abspath(os.path.expanduser(args.folder))
11
+ if not os.path.isdir(folder):
12
+ sys.exit(f"not a folder: {folder}")
13
+ conn = db.connect(paths.db_path())
14
+ conn.execute("insert or ignore into roots (path) values (?)", (folder,))
15
+ conn.commit()
16
+ print(f"added {folder}")
17
+
18
+
19
+ def cmd_index(args):
20
+ from .embed import Embedder
21
+ from .fetch import YtDlp
22
+ from .indexer import AlreadyRunning, ModelChanged, index_lock, run, watch
23
+ from .transcribe import Transcriber
24
+
25
+ from . import models
26
+
27
+ try:
28
+ models.ensure(models.INDEX)
29
+ except models.ModelsMissing as e:
30
+ sys.exit(str(e))
31
+ conn = db.connect(paths.db_path())
32
+ try:
33
+ with index_lock():
34
+ if args.retry_skipped:
35
+ from .urls import retry_skipped
36
+
37
+ print(f"{retry_skipped(conn)} skipped items set to try again")
38
+ if args.watch:
39
+ try:
40
+ watch(conn, Transcriber(), Embedder(), interval=args.interval, use_subtitles=args.subtitles,
41
+ fetcher=YtDlp())
42
+ except KeyboardInterrupt:
43
+ print("stopped")
44
+ return
45
+ counts = run(conn, Transcriber(), Embedder(), use_subtitles=args.subtitles, fetcher=YtDlp())
46
+ except AlreadyRunning:
47
+ sys.exit("another whensaid index is already running")
48
+ except ModelChanged as e:
49
+ sys.exit(str(e))
50
+ print(counts)
51
+
52
+
53
+ def cmd_add_url(args):
54
+ from .fetch import FetchError, YtDlp
55
+ from .urls import add_url
56
+
57
+ conn = db.connect(paths.db_path())
58
+ try:
59
+ r = add_url(conn, YtDlp(), args.url, limit=args.limit, no_playlist=args.no_playlist)
60
+ except FetchError as e:
61
+ sys.exit(f"could not read {args.url}: {e}")
62
+ print(f"run `whensaid index` to fetch {r['new']} new videos" if r["new"] else "nothing new to fetch")
63
+
64
+
65
+ def cmd_remove(args):
66
+ from .indexer import AlreadyRunning, index_lock, remove_root
67
+ from .urls import remove as remove_url
68
+
69
+ conn = db.connect(paths.db_path())
70
+ try:
71
+ with index_lock():
72
+ n = remove_url(conn, args.folder) if args.folder.startswith(("http://", "https://")) else remove_root(conn, args.folder)
73
+ except AlreadyRunning:
74
+ sys.exit("another whensaid index is already running")
75
+ print(f"removed {n} files from the index (the recordings themselves are untouched)")
76
+
77
+
78
+ def cmd_rechunk(args):
79
+ from .embed import Embedder
80
+ from .indexer import AlreadyRunning, index_lock, rechunk
81
+
82
+ conn = db.connect(paths.db_path())
83
+ try:
84
+ with index_lock():
85
+ rechunk(conn, Embedder())
86
+ except AlreadyRunning:
87
+ sys.exit("another whensaid index is already running")
88
+
89
+
90
+ def cmd_export(args):
91
+ from .export import export_all
92
+
93
+ conn = db.connect(paths.db_path())
94
+ out = None if args.beside else (Path(args.out).expanduser() if args.out else paths.home() / "transcripts")
95
+ files, skipped = export_all(conn, out, args.format, overwrite=args.overwrite)
96
+ where = "next to each recording" if out is None else str(out)
97
+ print(f"wrote {len(files)} {args.format} files, {where}")
98
+ if skipped:
99
+ print(f"left {len(skipped)} existing files alone (pass --overwrite to replace them), e.g. {skipped[0]}")
100
+
101
+
102
+ def cmd_search(args):
103
+ import json
104
+
105
+ from .embed import Embedder
106
+ from .rerank import load_reranker, min_relevance, strength
107
+ from .search import Searcher
108
+ from .util import deep_link, timecode
109
+
110
+ from . import models
111
+
112
+ try:
113
+ conn = db.connect_readonly(paths.db_path())
114
+ except FileNotFoundError:
115
+ sys.exit("nothing indexed yet: whensaid add <folder>, then whensaid index")
116
+ try:
117
+ models.ensure(models.wanted_for_search())
118
+ except models.ModelsMissing as e:
119
+ sys.exit(str(e))
120
+ try:
121
+ hits = Searcher(conn, Embedder(), reranker=load_reranker(), min_relevance=min_relevance()).search(args.query, limit=args.limit)
122
+ except RuntimeError as e:
123
+ sys.exit(str(e))
124
+ if args.json:
125
+ print(json.dumps([
126
+ {"file": h.title or os.path.basename(h.path), "path": h.path, "url": h.url, "link": deep_link(h.url, h.start),
127
+ "start": round(h.start, 1),
128
+ "at": timecode(h.start), "similarity": h.similarity and round(h.similarity, 3),
129
+ "relevance": h.relevance and round(h.relevance, 1), "match": strength(h.relevance), "text": h.text}
130
+ for h in hits
131
+ ], indent=1))
132
+ return
133
+ if not hits:
134
+ print("nothing relevant found")
135
+ elif strength(hits[0].relevance) == "weak":
136
+ print("only weak matches, the recordings may not cover this:")
137
+ for h in hits:
138
+ mark = {"weak": "weak ", "strong": "strong "}.get(strength(h.relevance), " ")
139
+ where = deep_link(h.url, h.start) or timecode(h.start)
140
+ print(f"{mark}{h.title or os.path.basename(h.path)} {where} {h.text}")
141
+
142
+
143
+ def cmd_serve(args):
144
+ from .server import main as serve
145
+
146
+ serve()
147
+
148
+
149
+ def cmd_doctor(args):
150
+ from .doctor import run
151
+
152
+ sys.exit(run())
153
+
154
+
155
+ def cmd_status(args):
156
+ conn = db.connect(paths.db_path())
157
+ rows = conn.execute(
158
+ "select status, sum(kind = 'file') files, sum(kind = 'url') web, coalesce(sum(duration), 0) d "
159
+ "from sources where status != 'removed' group by status"
160
+ ).fetchall()
161
+ for r in rows:
162
+ print(f"{r['status']:10} {r['files']:5} files {r['web']:5} web videos {r['d'] / 3600:.2f} h")
163
+ n = conn.execute("select count(*) from chunks").fetchone()[0]
164
+ print(f"{n} chunks")
165
+ feeds = conn.execute("select url, title from feeds").fetchall()
166
+ if feeds:
167
+ print(f"following {len(feeds)} playlists or channels:")
168
+ for f in feeds:
169
+ print(f" {f['title'] or ''} {f['url']}")
170
+ for status, what in (("failed", "failed, will be tried again"), ("skipped", "skipped for good (whensaid index --retry-skipped to try anyway)")):
171
+ why = conn.execute(
172
+ "select coalesce(error, '?') e, count(*) n, min(coalesce(title, path)) eg from sources where status = ? group by e order by n desc limit 5",
173
+ (status,),
174
+ ).fetchall()
175
+ if why:
176
+ print(f"{what}:")
177
+ for w in why:
178
+ print(f" {w['n']:4} {w['e'][:90]} (e.g. {os.path.basename(w['eg'])[:50]})")
179
+
180
+
181
+ def cmd_models_pull(args):
182
+ from . import models
183
+
184
+ todo = [k for k in ("whisper", "embedder", "reranker") if not models.present(k)]
185
+ if not todo:
186
+ print("all models are already on disk")
187
+ return
188
+ try:
189
+ models.download(todo)
190
+ except Exception as e:
191
+ sys.exit(f"download failed: {e}")
192
+ print("ok")
193
+
194
+
195
+ def main(argv=None):
196
+ p = argparse.ArgumentParser(prog="whensaid")
197
+ sub = p.add_subparsers(dest="cmd", required=True)
198
+
199
+ a = sub.add_parser("add", help="register a folder of recordings")
200
+ a.add_argument("folder")
201
+ a.set_defaults(fn=cmd_add)
202
+
203
+ au = sub.add_parser("add-url", help="add a video, playlist or channel from the web (needs yt-dlp)")
204
+ au.add_argument("url")
205
+ au.add_argument("--limit", type=int, default=200, help="most videos to take from a playlist or channel")
206
+ au.add_argument("--no-playlist", action="store_true", help="for a video url that also names a playlist: just the video")
207
+ au.set_defaults(fn=cmd_add_url)
208
+
209
+ rm = sub.add_parser("remove", help="stop watching a folder, playlist or video and drop it from the index")
210
+ rm.add_argument("folder", metavar="folder-or-url")
211
+ rm.set_defaults(fn=cmd_remove)
212
+ ix = sub.add_parser("index", help="transcribe and index anything new or changed")
213
+ ix.add_argument("--watch", action="store_true", help="keep running and pick up new files as they appear")
214
+ ix.add_argument("--interval", type=float, default=30, help="seconds between scans with --watch")
215
+ ix.add_argument("--subtitles", action="store_true",
216
+ help="use a .srt/.vtt next to a recording instead of transcribing it (English assumed)")
217
+ ix.add_argument("--retry-skipped", action="store_true", help="try again what was skipped for good (live then, private then, ...)")
218
+ ix.set_defaults(fn=cmd_index)
219
+ sub.add_parser("rechunk", help="rebuild chunks and embeddings from stored transcripts").set_defaults(fn=cmd_rechunk)
220
+ e = sub.add_parser("export", help="write transcripts as text files an agent can grep")
221
+ e.add_argument("--format", choices=["txt", "srt"], default="txt")
222
+ e.add_argument("--out", help="folder to write into (default: ~/.whensaid/transcripts)")
223
+ e.add_argument("--beside", action="store_true", help="write next to each recording instead")
224
+ e.add_argument("--overwrite", action="store_true", help="with --beside: replace files that are already there")
225
+ e.set_defaults(fn=cmd_export)
226
+
227
+ q = sub.add_parser("search", help="search from the terminal")
228
+ q.add_argument("query")
229
+ q.add_argument("-n", "--limit", type=int, default=8)
230
+ q.add_argument("--json", action="store_true")
231
+ q.set_defaults(fn=cmd_search)
232
+
233
+ sub.add_parser("serve", help="run the MCP server on stdio").set_defaults(fn=cmd_serve)
234
+ sub.add_parser("status", help="show what is indexed").set_defaults(fn=cmd_status)
235
+ sub.add_parser("doctor", help="check models, GPU, yt-dlp and the index, and say how to fix what is wrong").set_defaults(fn=cmd_doctor)
236
+
237
+ m = sub.add_parser("models", help="model management")
238
+ msub = m.add_subparsers(dest="mcmd", required=True)
239
+ msub.add_parser("pull", help="download the models").set_defaults(fn=cmd_models_pull)
240
+
241
+ args = p.parse_args(argv)
242
+ args.fn(args)
243
+
244
+
245
+ if __name__ == "__main__":
246
+ main()
whensaid/cudalibs.py ADDED
@@ -0,0 +1,43 @@
1
+ """ctranslate2 wants libcublas/libcudnn from CUDA 12. The pip wheels
2
+ (nvidia-cublas-cu12, nvidia-cudnn-cu12) put them in site-packages where the
3
+ dynamic loader never looks, so load them ourselves before ctranslate2 starts."""
4
+ import ctypes
5
+ import glob
6
+ import importlib.util
7
+ import os
8
+ import sys
9
+
10
+
11
+ def _spec(pkg: str):
12
+ try:
13
+ return importlib.util.find_spec(pkg)
14
+ except (ModuleNotFoundError, ValueError): # "nvidia" itself is missing when the cuda extra isn't installed
15
+ return None
16
+
17
+
18
+ def preload() -> None:
19
+ if not sys.platform.startswith("linux"):
20
+ return
21
+ for pkg in ("nvidia.cublas", "nvidia.cudnn"):
22
+ spec = _spec(pkg)
23
+ if not spec or not spec.submodule_search_locations:
24
+ continue
25
+ libdir = os.path.join(list(spec.submodule_search_locations)[0], "lib")
26
+ # load in name order so e.g. libcudnn.so.9 pulls in its sub-libraries
27
+ for so in sorted(glob.glob(os.path.join(libdir, "lib*.so*"))):
28
+ try:
29
+ ctypes.CDLL(so, mode=ctypes.RTLD_GLOBAL)
30
+ except OSError:
31
+ pass
32
+
33
+
34
+ def libs_load() -> bool:
35
+ """Can the CUDA 12 libraries ctranslate2 needs be found, from the pip wheels or the system?"""
36
+ if not sys.platform.startswith("linux"):
37
+ return False
38
+ try:
39
+ for name in ("libcublas.so.12", "libcudnn.so.9"):
40
+ ctypes.CDLL(name)
41
+ return True
42
+ except OSError:
43
+ return False
whensaid/db.py ADDED
@@ -0,0 +1,120 @@
1
+ import sqlite3
2
+ from pathlib import Path
3
+
4
+ SCHEMA = """
5
+ create table if not exists roots (
6
+ path text primary key
7
+ );
8
+
9
+ create table if not exists sources (
10
+ id integer primary key,
11
+ path text unique not null,
12
+ size integer,
13
+ mtime_ns integer,
14
+ duration real,
15
+ status text not null default 'pending', -- pending, done, no_audio, failed
16
+ error text,
17
+ indexed_at text
18
+ );
19
+
20
+ create table if not exists segments (
21
+ id integer primary key,
22
+ source_id integer not null references sources(id) on delete cascade,
23
+ start real not null,
24
+ end real not null,
25
+ text text not null
26
+ );
27
+ create index if not exists segments_source on segments(source_id, start);
28
+
29
+ create table if not exists chunks (
30
+ id integer primary key,
31
+ source_id integer not null references sources(id) on delete cascade,
32
+ start real not null,
33
+ end real not null,
34
+ text text not null,
35
+ embedding blob
36
+ );
37
+ create index if not exists chunks_source on chunks(source_id, start);
38
+
39
+ create table if not exists feeds (
40
+ url text primary key, -- a playlist or channel, re-listed now and then for new videos
41
+ kind text,
42
+ title text,
43
+ limit_n integer,
44
+ checked_at real
45
+ );
46
+
47
+ create table if not exists meta (
48
+ key text primary key,
49
+ value text
50
+ );
51
+
52
+ create virtual table if not exists chunks_fts using fts5(text, tokenize='porter unicode61');
53
+ """
54
+
55
+
56
+ # added after the first release, so existing indexes get them through alter table
57
+ SOURCE_COLUMNS = {
58
+ "kind": "text not null default 'file'", # file or url; a url source's path is a stable key like youtube:ID
59
+ "url": "text",
60
+ "title": "text",
61
+ "captions": "text", # manual, auto, whisper or subtitle-file
62
+ "feed_url": "text",
63
+ "confidence": "real", # mean word probability when whisper made the transcript
64
+ }
65
+
66
+
67
+ def _migrate(conn: sqlite3.Connection) -> None:
68
+ have = {r[1] for r in conn.execute("pragma table_info(sources)")}
69
+ for col, decl in SOURCE_COLUMNS.items():
70
+ if col not in have:
71
+ conn.execute(f"alter table sources add column {col} {decl}")
72
+
73
+
74
+ def connect(path: Path) -> sqlite3.Connection:
75
+ conn = sqlite3.connect(path, timeout=30)
76
+ conn.row_factory = sqlite3.Row
77
+ conn.execute("pragma journal_mode = wal")
78
+ conn.execute("pragma foreign_keys = on")
79
+ conn.executescript(SCHEMA)
80
+ _migrate(conn)
81
+ return conn
82
+
83
+
84
+ def get_meta(conn: sqlite3.Connection, key: str) -> str | None:
85
+ try:
86
+ row = conn.execute("select value from meta where key = ?", (key,)).fetchone()
87
+ except sqlite3.OperationalError: # index from before the meta table, opened read-only
88
+ return None
89
+ return row[0] if row else None
90
+
91
+
92
+ def set_meta(conn: sqlite3.Connection, key: str, value: str) -> None:
93
+ conn.execute(
94
+ "insert into meta (key, value) values (?, ?) on conflict(key) do update set value = excluded.value",
95
+ (key, value),
96
+ )
97
+
98
+
99
+ def forget_chunks(conn: sqlite3.Connection, source_id: int) -> None:
100
+ # fts rows are not tied to chunks by a foreign key, so clear them by hand
101
+ conn.execute(
102
+ "delete from chunks_fts where rowid in (select id from chunks where source_id = ?)",
103
+ (source_id,),
104
+ )
105
+ conn.execute("delete from chunks where source_id = ?", (source_id,))
106
+
107
+
108
+ def forget_source(conn: sqlite3.Connection, source_id: int) -> None:
109
+ forget_chunks(conn, source_id)
110
+ conn.execute("delete from segments where source_id = ?", (source_id,))
111
+
112
+
113
+ def connect_readonly(path: Path) -> sqlite3.Connection:
114
+ """For the mcp server: never writes, never creates the file. The connection is
115
+ shared between the sdk's worker threads, so the caller has to serialise use."""
116
+ if not Path(path).exists():
117
+ raise FileNotFoundError(path)
118
+ conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=30, check_same_thread=False)
119
+ conn.row_factory = sqlite3.Row
120
+ return conn
whensaid/doctor.py ADDED
@@ -0,0 +1,121 @@
1
+ """`whensaid doctor`: what works, what doesn't, and what to do about it. Makes no network
2
+ calls and never loads a model onto the GPU."""
3
+ import os
4
+ import platform
5
+ import shutil
6
+ import subprocess
7
+ import sys
8
+ import time
9
+
10
+ OK, WARN, FAIL = "ok", "warn", "fail"
11
+
12
+
13
+ def _free_gb(path) -> float:
14
+ return shutil.disk_usage(path).free / 1e9
15
+
16
+
17
+ def checks() -> list[tuple[str, str, str, str]]:
18
+ """(status, name, detail, how to fix)"""
19
+ from . import cudalibs, db, models, paths
20
+ from .indexer import indexer_running
21
+
22
+ out = []
23
+ add = lambda status, name, detail, fix="": out.append((status, name, detail, fix))
24
+
25
+ add(OK, "python", f"{platform.python_version()} on {platform.system()} {platform.machine()}")
26
+
27
+ # compute
28
+ cudalibs.preload() # the pip wheels are found by path, not by the system loader
29
+ try:
30
+ import ctranslate2
31
+
32
+ gpus = ctranslate2.get_cuda_device_count()
33
+ except Exception as e:
34
+ add(FAIL, "ctranslate2", f"cannot be imported: {e}", "pip install --force-reinstall whensaid")
35
+ gpus = 0
36
+ if gpus and cudalibs.libs_load():
37
+ add(OK, "gpu", f"{gpus} NVIDIA GPU, CUDA 12 libraries found")
38
+ elif gpus:
39
+ add(WARN, "gpu", "an NVIDIA GPU is there but the CUDA 12 libraries are not: transcribing on the CPU (about 3x slower)",
40
+ "pip install 'whensaid[cuda]' (with uvx: uvx --from 'whensaid[cuda]' whensaid ...)")
41
+ else:
42
+ add(OK, "gpu", f"no NVIDIA GPU, transcribing on the CPU ({os.cpu_count()} cores)")
43
+
44
+ # models
45
+ free = _free_gb(paths.models_dir())
46
+ for kind in ("whisper", "embedder", "reranker"):
47
+ if models.present(kind):
48
+ add(OK, f"{kind} model", "on disk")
49
+ else:
50
+ need = {"whisper": 1.5, "embedder": 0.1, "reranker": 0.1}[kind]
51
+ if free < need * 1.5:
52
+ add(FAIL, f"{kind} model", f"missing, and only {free:.1f} GB free in {paths.models_dir()}", "free some disk space")
53
+ else:
54
+ add(WARN, f"{kind} model", f"not downloaded yet ({models.size_label(kind)}), fetched on first use",
55
+ "whensaid models pull" if not models.offline() else "offline mode is on: run `whensaid models pull` once online")
56
+ if (paths.home() / "ipv4").exists():
57
+ add(OK, "network", "downloads use IPv4 only (a stalled IPv6 route was found earlier)")
58
+
59
+ # yt-dlp, only needed for add-url
60
+ from .fetch import FetchError, ytdlp_cmd
61
+
62
+ try:
63
+ cmd = ytdlp_cmd()
64
+ v = subprocess.run(cmd + ["--version"], capture_output=True, text=True, timeout=30).stdout.strip()
65
+ age = None
66
+ try:
67
+ y, m, d = (int(x) for x in v.split(".")[:3])
68
+ age = (time.time() - time.mktime((y, m, d, 0, 0, 0, 0, 0, -1))) / 86400
69
+ except ValueError:
70
+ pass
71
+ if age is not None and age > 120:
72
+ add(WARN, "yt-dlp", f"{v}, {age:.0f} days old; old versions are the usual reason YouTube fails", "pip install -U yt-dlp")
73
+ else:
74
+ add(OK, "yt-dlp", v or "installed")
75
+ if not (shutil.which("deno") or shutil.which("node")):
76
+ add(WARN, "yt-dlp JS runtime", "no deno or node: some YouTube formats may be missing (captions usually still work)",
77
+ "install deno (https://deno.com)")
78
+ except FetchError:
79
+ add(OK, "yt-dlp", "not installed (only needed for `whensaid add-url`)", "pip install 'whensaid[url]'")
80
+ except Exception as e:
81
+ add(WARN, "yt-dlp", f"installed but did not run: {e}")
82
+
83
+ # the index
84
+ dbp = paths.db_path()
85
+ if not dbp.exists():
86
+ add(OK, "index", f"none yet at {dbp}", "whensaid add <folder> (or add-url <url>), then whensaid index")
87
+ else:
88
+ try:
89
+ conn = db.connect_readonly(dbp)
90
+ n = conn.execute("select count(*) from sources").fetchone()[0]
91
+ chunks = conn.execute("select count(*) from chunks").fetchone()[0]
92
+ size = dbp.stat().st_size / 1e6
93
+ add(OK, "index", f"{n} sources, {chunks} chunks, {size:.0f} MB at {dbp}")
94
+ built = db.get_meta(conn, "embed_model")
95
+ from .embed import MODEL
96
+
97
+ if built and built != MODEL:
98
+ add(FAIL, "embedding model", f"the index was built with {built}, this version uses {MODEL}", "whensaid rechunk")
99
+ bad = {r[0]: r[1] for r in conn.execute(
100
+ "select status, count(*) from sources where status in ('failed', 'skipped') group by status")} \
101
+ if n else {}
102
+ for status, count in bad.items():
103
+ add(WARN if status == "failed" else OK, f"{status} sources", f"{count}", "whensaid status (shows the reasons)")
104
+ except Exception as e:
105
+ add(FAIL, "index", f"cannot be read: {e}", f"move {dbp} aside and re-index")
106
+ if indexer_running():
107
+ add(OK, "indexer", "running right now (search keeps working meanwhile)")
108
+ return out
109
+
110
+
111
+ def run(out=sys.stdout) -> int:
112
+ marks = {OK: "ok ", WARN: "warn", FAIL: "FAIL"}
113
+ results = checks()
114
+ for status, name, detail, fix in results:
115
+ print(f"[{marks[status]}] {name:18} {detail}", file=out)
116
+ if fix and status != OK or (fix and status == OK and "none yet" in detail):
117
+ print(f" -> {fix}", file=out)
118
+ bad = sum(1 for r in results if r[0] == FAIL)
119
+ warn = sum(1 for r in results if r[0] == WARN)
120
+ print(f"\n{bad} problems, {warn} warnings" if bad or warn else "\neverything looks fine", file=out)
121
+ return 1 if bad else 0
whensaid/embed.py ADDED
@@ -0,0 +1,30 @@
1
+ import os
2
+
3
+ import numpy as np
4
+
5
+ from .paths import models_dir
6
+
7
+ MODEL = "BAAI/bge-small-en-v1.5"
8
+
9
+
10
+ class Embedder:
11
+ def __init__(self, model=MODEL, allow_download=False):
12
+ self.model_name = model
13
+ self.allow_download = allow_download
14
+ self._m = None
15
+
16
+ def _load(self):
17
+ from fastembed import TextEmbedding
18
+ self._m = TextEmbedding(
19
+ self.model_name,
20
+ cache_dir=str(models_dir() / "fastembed"),
21
+ local_files_only=not self.allow_download,
22
+ threads=min(4, os.cpu_count() or 1), # see rerank.THREADS
23
+ )
24
+
25
+ def encode(self, texts: list[str]) -> np.ndarray:
26
+ if self._m is None:
27
+ self._load()
28
+ vecs = np.array(list(self._m.embed(texts)), dtype=np.float32)
29
+ norms = np.linalg.norm(vecs, axis=1, keepdims=True)
30
+ return vecs / np.maximum(norms, 1e-9)
whensaid/export.py ADDED
@@ -0,0 +1,50 @@
1
+ import re
2
+ from pathlib import Path
3
+
4
+ from .util import timecode
5
+
6
+
7
+ def render_txt(segments) -> str:
8
+ # one segment per line with its start time, so `grep -n` output is already a citation
9
+ return "".join(f"[{timecode(s['start'])}] {s['text']}\n" for s in segments)
10
+
11
+
12
+ def render_srt(segments) -> str:
13
+ out = []
14
+ for i, s in enumerate(segments, 1):
15
+ out.append(f"{i}\n{timecode(s['start'], srt=True)} --> {timecode(s['end'], srt=True)}\n{s['text']}\n")
16
+ return "\n".join(out)
17
+
18
+
19
+ RENDERERS = {"txt": render_txt, "srt": render_srt}
20
+
21
+
22
+ def target_path(media_path: str, out_dir: Path | None, fmt: str, kind: str = "file") -> Path:
23
+ if kind == "url": # a key like youtube:ID, there is no file to sit next to
24
+ return out_dir / "urls" / (re.sub(r"[^\w.-]+", "_", media_path) + "." + fmt)
25
+ if out_dir is None: # next to the media file, same name
26
+ return Path(media_path).with_suffix("." + fmt)
27
+ # mirror the full path so two files with the same name never collide
28
+ return out_dir / (media_path.lstrip("/") + "." + fmt)
29
+
30
+
31
+ def export_all(conn, out_dir: Path | None, fmt: str = "txt", overwrite: bool = False) -> tuple[list[Path], list[Path]]:
32
+ """Returns (written, skipped). Next to the recordings (out_dir None) an existing file may
33
+ be someone's real subtitles, so it is left alone unless overwrite is set."""
34
+ render = RENDERERS[fmt]
35
+ written, skipped = [], []
36
+ for src in conn.execute("select id, path, kind from sources where status = 'done' order by path").fetchall():
37
+ if src["kind"] == "url" and out_dir is None:
38
+ continue # web videos have no recording to write beside
39
+ segs = conn.execute(
40
+ "select start, end, text from segments where source_id = ? order by start", (src["id"],)
41
+ ).fetchall()
42
+ dest = target_path(src["path"], out_dir, fmt, src["kind"])
43
+ text = render(segs)
44
+ if out_dir is None and dest.exists() and not overwrite and dest.read_text(encoding="utf-8", errors="replace") != text:
45
+ skipped.append(dest)
46
+ continue
47
+ dest.parent.mkdir(parents=True, exist_ok=True)
48
+ dest.write_text(text, encoding="utf-8")
49
+ written.append(dest)
50
+ return written, skipped