whensaid 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- whensaid/__init__.py +1 -0
- whensaid/chunk.py +46 -0
- whensaid/cli.py +246 -0
- whensaid/cudalibs.py +43 -0
- whensaid/db.py +120 -0
- whensaid/doctor.py +121 -0
- whensaid/embed.py +30 -0
- whensaid/export.py +50 -0
- whensaid/fetch.py +196 -0
- whensaid/indexer.py +335 -0
- whensaid/models.py +260 -0
- whensaid/paths.py +18 -0
- whensaid/rerank.py +80 -0
- whensaid/search.py +170 -0
- whensaid/server.py +214 -0
- whensaid/subtitles.py +99 -0
- whensaid/transcribe.py +136 -0
- whensaid/urls.py +104 -0
- whensaid/util.py +55 -0
- whensaid-0.1.0.dist-info/METADATA +146 -0
- whensaid-0.1.0.dist-info/RECORD +24 -0
- whensaid-0.1.0.dist-info/WHEEL +4 -0
- whensaid-0.1.0.dist-info/entry_points.txt +2 -0
- whensaid-0.1.0.dist-info/licenses/LICENSE +21 -0
whensaid/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.0.1"
|
whensaid/chunk.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
@dataclass
|
|
5
|
+
class Segment:
|
|
6
|
+
start: float
|
|
7
|
+
end: float
|
|
8
|
+
text: str
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def make_chunks(
|
|
12
|
+
segments: list[Segment], target: float = 45.0, overlap: float = 6.0, max_gap: float = 8.0
|
|
13
|
+
) -> list[Segment]:
|
|
14
|
+
"""Group whisper segments into windows of roughly `target` seconds.
|
|
15
|
+
|
|
16
|
+
A new window starts with the last few segments of the previous one (anything
|
|
17
|
+
that started within `overlap` seconds of its end) so a sentence on the
|
|
18
|
+
boundary shows up in both. A silence longer than `max_gap` (music, a title
|
|
19
|
+
card) ends the window outright, with no overlap across it.
|
|
20
|
+
"""
|
|
21
|
+
chunks: list[Segment] = []
|
|
22
|
+
cur: list[Segment] = []
|
|
23
|
+
|
|
24
|
+
for seg in segments:
|
|
25
|
+
if cur and seg.start - cur[-1].end > max_gap:
|
|
26
|
+
chunks.append(_join(cur))
|
|
27
|
+
cur = []
|
|
28
|
+
elif cur and seg.end - cur[0].start > target:
|
|
29
|
+
chunks.append(_join(cur))
|
|
30
|
+
cur = _tail(cur, overlap)
|
|
31
|
+
if cur and seg.end - cur[0].start > target:
|
|
32
|
+
cur = [] # the carried tail would blow the window right away
|
|
33
|
+
cur.append(seg)
|
|
34
|
+
|
|
35
|
+
if cur and (not chunks or cur[-1].end > chunks[-1].end):
|
|
36
|
+
chunks.append(_join(cur))
|
|
37
|
+
return chunks
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _tail(segs: list[Segment], overlap: float) -> list[Segment]:
|
|
41
|
+
cutoff = segs[-1].end - overlap
|
|
42
|
+
return [s for s in segs[1:] if s.start >= cutoff] # never carry the whole window
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _join(segs: list[Segment]) -> Segment:
|
|
46
|
+
return Segment(segs[0].start, segs[-1].end, " ".join(s.text.strip() for s in segs))
|
whensaid/cli.py
ADDED
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import os
|
|
3
|
+
import sys
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from . import db, paths
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def cmd_add(args):
|
|
10
|
+
folder = os.path.abspath(os.path.expanduser(args.folder))
|
|
11
|
+
if not os.path.isdir(folder):
|
|
12
|
+
sys.exit(f"not a folder: {folder}")
|
|
13
|
+
conn = db.connect(paths.db_path())
|
|
14
|
+
conn.execute("insert or ignore into roots (path) values (?)", (folder,))
|
|
15
|
+
conn.commit()
|
|
16
|
+
print(f"added {folder}")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def cmd_index(args):
|
|
20
|
+
from .embed import Embedder
|
|
21
|
+
from .fetch import YtDlp
|
|
22
|
+
from .indexer import AlreadyRunning, ModelChanged, index_lock, run, watch
|
|
23
|
+
from .transcribe import Transcriber
|
|
24
|
+
|
|
25
|
+
from . import models
|
|
26
|
+
|
|
27
|
+
try:
|
|
28
|
+
models.ensure(models.INDEX)
|
|
29
|
+
except models.ModelsMissing as e:
|
|
30
|
+
sys.exit(str(e))
|
|
31
|
+
conn = db.connect(paths.db_path())
|
|
32
|
+
try:
|
|
33
|
+
with index_lock():
|
|
34
|
+
if args.retry_skipped:
|
|
35
|
+
from .urls import retry_skipped
|
|
36
|
+
|
|
37
|
+
print(f"{retry_skipped(conn)} skipped items set to try again")
|
|
38
|
+
if args.watch:
|
|
39
|
+
try:
|
|
40
|
+
watch(conn, Transcriber(), Embedder(), interval=args.interval, use_subtitles=args.subtitles,
|
|
41
|
+
fetcher=YtDlp())
|
|
42
|
+
except KeyboardInterrupt:
|
|
43
|
+
print("stopped")
|
|
44
|
+
return
|
|
45
|
+
counts = run(conn, Transcriber(), Embedder(), use_subtitles=args.subtitles, fetcher=YtDlp())
|
|
46
|
+
except AlreadyRunning:
|
|
47
|
+
sys.exit("another whensaid index is already running")
|
|
48
|
+
except ModelChanged as e:
|
|
49
|
+
sys.exit(str(e))
|
|
50
|
+
print(counts)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def cmd_add_url(args):
|
|
54
|
+
from .fetch import FetchError, YtDlp
|
|
55
|
+
from .urls import add_url
|
|
56
|
+
|
|
57
|
+
conn = db.connect(paths.db_path())
|
|
58
|
+
try:
|
|
59
|
+
r = add_url(conn, YtDlp(), args.url, limit=args.limit, no_playlist=args.no_playlist)
|
|
60
|
+
except FetchError as e:
|
|
61
|
+
sys.exit(f"could not read {args.url}: {e}")
|
|
62
|
+
print(f"run `whensaid index` to fetch {r['new']} new videos" if r["new"] else "nothing new to fetch")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def cmd_remove(args):
|
|
66
|
+
from .indexer import AlreadyRunning, index_lock, remove_root
|
|
67
|
+
from .urls import remove as remove_url
|
|
68
|
+
|
|
69
|
+
conn = db.connect(paths.db_path())
|
|
70
|
+
try:
|
|
71
|
+
with index_lock():
|
|
72
|
+
n = remove_url(conn, args.folder) if args.folder.startswith(("http://", "https://")) else remove_root(conn, args.folder)
|
|
73
|
+
except AlreadyRunning:
|
|
74
|
+
sys.exit("another whensaid index is already running")
|
|
75
|
+
print(f"removed {n} files from the index (the recordings themselves are untouched)")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def cmd_rechunk(args):
|
|
79
|
+
from .embed import Embedder
|
|
80
|
+
from .indexer import AlreadyRunning, index_lock, rechunk
|
|
81
|
+
|
|
82
|
+
conn = db.connect(paths.db_path())
|
|
83
|
+
try:
|
|
84
|
+
with index_lock():
|
|
85
|
+
rechunk(conn, Embedder())
|
|
86
|
+
except AlreadyRunning:
|
|
87
|
+
sys.exit("another whensaid index is already running")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def cmd_export(args):
|
|
91
|
+
from .export import export_all
|
|
92
|
+
|
|
93
|
+
conn = db.connect(paths.db_path())
|
|
94
|
+
out = None if args.beside else (Path(args.out).expanduser() if args.out else paths.home() / "transcripts")
|
|
95
|
+
files, skipped = export_all(conn, out, args.format, overwrite=args.overwrite)
|
|
96
|
+
where = "next to each recording" if out is None else str(out)
|
|
97
|
+
print(f"wrote {len(files)} {args.format} files, {where}")
|
|
98
|
+
if skipped:
|
|
99
|
+
print(f"left {len(skipped)} existing files alone (pass --overwrite to replace them), e.g. {skipped[0]}")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def cmd_search(args):
|
|
103
|
+
import json
|
|
104
|
+
|
|
105
|
+
from .embed import Embedder
|
|
106
|
+
from .rerank import load_reranker, min_relevance, strength
|
|
107
|
+
from .search import Searcher
|
|
108
|
+
from .util import deep_link, timecode
|
|
109
|
+
|
|
110
|
+
from . import models
|
|
111
|
+
|
|
112
|
+
try:
|
|
113
|
+
conn = db.connect_readonly(paths.db_path())
|
|
114
|
+
except FileNotFoundError:
|
|
115
|
+
sys.exit("nothing indexed yet: whensaid add <folder>, then whensaid index")
|
|
116
|
+
try:
|
|
117
|
+
models.ensure(models.wanted_for_search())
|
|
118
|
+
except models.ModelsMissing as e:
|
|
119
|
+
sys.exit(str(e))
|
|
120
|
+
try:
|
|
121
|
+
hits = Searcher(conn, Embedder(), reranker=load_reranker(), min_relevance=min_relevance()).search(args.query, limit=args.limit)
|
|
122
|
+
except RuntimeError as e:
|
|
123
|
+
sys.exit(str(e))
|
|
124
|
+
if args.json:
|
|
125
|
+
print(json.dumps([
|
|
126
|
+
{"file": h.title or os.path.basename(h.path), "path": h.path, "url": h.url, "link": deep_link(h.url, h.start),
|
|
127
|
+
"start": round(h.start, 1),
|
|
128
|
+
"at": timecode(h.start), "similarity": h.similarity and round(h.similarity, 3),
|
|
129
|
+
"relevance": h.relevance and round(h.relevance, 1), "match": strength(h.relevance), "text": h.text}
|
|
130
|
+
for h in hits
|
|
131
|
+
], indent=1))
|
|
132
|
+
return
|
|
133
|
+
if not hits:
|
|
134
|
+
print("nothing relevant found")
|
|
135
|
+
elif strength(hits[0].relevance) == "weak":
|
|
136
|
+
print("only weak matches, the recordings may not cover this:")
|
|
137
|
+
for h in hits:
|
|
138
|
+
mark = {"weak": "weak ", "strong": "strong "}.get(strength(h.relevance), " ")
|
|
139
|
+
where = deep_link(h.url, h.start) or timecode(h.start)
|
|
140
|
+
print(f"{mark}{h.title or os.path.basename(h.path)} {where} {h.text}")
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def cmd_serve(args):
|
|
144
|
+
from .server import main as serve
|
|
145
|
+
|
|
146
|
+
serve()
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def cmd_doctor(args):
|
|
150
|
+
from .doctor import run
|
|
151
|
+
|
|
152
|
+
sys.exit(run())
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def cmd_status(args):
|
|
156
|
+
conn = db.connect(paths.db_path())
|
|
157
|
+
rows = conn.execute(
|
|
158
|
+
"select status, sum(kind = 'file') files, sum(kind = 'url') web, coalesce(sum(duration), 0) d "
|
|
159
|
+
"from sources where status != 'removed' group by status"
|
|
160
|
+
).fetchall()
|
|
161
|
+
for r in rows:
|
|
162
|
+
print(f"{r['status']:10} {r['files']:5} files {r['web']:5} web videos {r['d'] / 3600:.2f} h")
|
|
163
|
+
n = conn.execute("select count(*) from chunks").fetchone()[0]
|
|
164
|
+
print(f"{n} chunks")
|
|
165
|
+
feeds = conn.execute("select url, title from feeds").fetchall()
|
|
166
|
+
if feeds:
|
|
167
|
+
print(f"following {len(feeds)} playlists or channels:")
|
|
168
|
+
for f in feeds:
|
|
169
|
+
print(f" {f['title'] or ''} {f['url']}")
|
|
170
|
+
for status, what in (("failed", "failed, will be tried again"), ("skipped", "skipped for good (whensaid index --retry-skipped to try anyway)")):
|
|
171
|
+
why = conn.execute(
|
|
172
|
+
"select coalesce(error, '?') e, count(*) n, min(coalesce(title, path)) eg from sources where status = ? group by e order by n desc limit 5",
|
|
173
|
+
(status,),
|
|
174
|
+
).fetchall()
|
|
175
|
+
if why:
|
|
176
|
+
print(f"{what}:")
|
|
177
|
+
for w in why:
|
|
178
|
+
print(f" {w['n']:4} {w['e'][:90]} (e.g. {os.path.basename(w['eg'])[:50]})")
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def cmd_models_pull(args):
|
|
182
|
+
from . import models
|
|
183
|
+
|
|
184
|
+
todo = [k for k in ("whisper", "embedder", "reranker") if not models.present(k)]
|
|
185
|
+
if not todo:
|
|
186
|
+
print("all models are already on disk")
|
|
187
|
+
return
|
|
188
|
+
try:
|
|
189
|
+
models.download(todo)
|
|
190
|
+
except Exception as e:
|
|
191
|
+
sys.exit(f"download failed: {e}")
|
|
192
|
+
print("ok")
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def main(argv=None):
|
|
196
|
+
p = argparse.ArgumentParser(prog="whensaid")
|
|
197
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
198
|
+
|
|
199
|
+
a = sub.add_parser("add", help="register a folder of recordings")
|
|
200
|
+
a.add_argument("folder")
|
|
201
|
+
a.set_defaults(fn=cmd_add)
|
|
202
|
+
|
|
203
|
+
au = sub.add_parser("add-url", help="add a video, playlist or channel from the web (needs yt-dlp)")
|
|
204
|
+
au.add_argument("url")
|
|
205
|
+
au.add_argument("--limit", type=int, default=200, help="most videos to take from a playlist or channel")
|
|
206
|
+
au.add_argument("--no-playlist", action="store_true", help="for a video url that also names a playlist: just the video")
|
|
207
|
+
au.set_defaults(fn=cmd_add_url)
|
|
208
|
+
|
|
209
|
+
rm = sub.add_parser("remove", help="stop watching a folder, playlist or video and drop it from the index")
|
|
210
|
+
rm.add_argument("folder", metavar="folder-or-url")
|
|
211
|
+
rm.set_defaults(fn=cmd_remove)
|
|
212
|
+
ix = sub.add_parser("index", help="transcribe and index anything new or changed")
|
|
213
|
+
ix.add_argument("--watch", action="store_true", help="keep running and pick up new files as they appear")
|
|
214
|
+
ix.add_argument("--interval", type=float, default=30, help="seconds between scans with --watch")
|
|
215
|
+
ix.add_argument("--subtitles", action="store_true",
|
|
216
|
+
help="use a .srt/.vtt next to a recording instead of transcribing it (English assumed)")
|
|
217
|
+
ix.add_argument("--retry-skipped", action="store_true", help="try again what was skipped for good (live then, private then, ...)")
|
|
218
|
+
ix.set_defaults(fn=cmd_index)
|
|
219
|
+
sub.add_parser("rechunk", help="rebuild chunks and embeddings from stored transcripts").set_defaults(fn=cmd_rechunk)
|
|
220
|
+
e = sub.add_parser("export", help="write transcripts as text files an agent can grep")
|
|
221
|
+
e.add_argument("--format", choices=["txt", "srt"], default="txt")
|
|
222
|
+
e.add_argument("--out", help="folder to write into (default: ~/.whensaid/transcripts)")
|
|
223
|
+
e.add_argument("--beside", action="store_true", help="write next to each recording instead")
|
|
224
|
+
e.add_argument("--overwrite", action="store_true", help="with --beside: replace files that are already there")
|
|
225
|
+
e.set_defaults(fn=cmd_export)
|
|
226
|
+
|
|
227
|
+
q = sub.add_parser("search", help="search from the terminal")
|
|
228
|
+
q.add_argument("query")
|
|
229
|
+
q.add_argument("-n", "--limit", type=int, default=8)
|
|
230
|
+
q.add_argument("--json", action="store_true")
|
|
231
|
+
q.set_defaults(fn=cmd_search)
|
|
232
|
+
|
|
233
|
+
sub.add_parser("serve", help="run the MCP server on stdio").set_defaults(fn=cmd_serve)
|
|
234
|
+
sub.add_parser("status", help="show what is indexed").set_defaults(fn=cmd_status)
|
|
235
|
+
sub.add_parser("doctor", help="check models, GPU, yt-dlp and the index, and say how to fix what is wrong").set_defaults(fn=cmd_doctor)
|
|
236
|
+
|
|
237
|
+
m = sub.add_parser("models", help="model management")
|
|
238
|
+
msub = m.add_subparsers(dest="mcmd", required=True)
|
|
239
|
+
msub.add_parser("pull", help="download the models").set_defaults(fn=cmd_models_pull)
|
|
240
|
+
|
|
241
|
+
args = p.parse_args(argv)
|
|
242
|
+
args.fn(args)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
if __name__ == "__main__":
|
|
246
|
+
main()
|
whensaid/cudalibs.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""ctranslate2 wants libcublas/libcudnn from CUDA 12. The pip wheels
|
|
2
|
+
(nvidia-cublas-cu12, nvidia-cudnn-cu12) put them in site-packages where the
|
|
3
|
+
dynamic loader never looks, so load them ourselves before ctranslate2 starts."""
|
|
4
|
+
import ctypes
|
|
5
|
+
import glob
|
|
6
|
+
import importlib.util
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _spec(pkg: str):
|
|
12
|
+
try:
|
|
13
|
+
return importlib.util.find_spec(pkg)
|
|
14
|
+
except (ModuleNotFoundError, ValueError): # "nvidia" itself is missing when the cuda extra isn't installed
|
|
15
|
+
return None
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def preload() -> None:
|
|
19
|
+
if not sys.platform.startswith("linux"):
|
|
20
|
+
return
|
|
21
|
+
for pkg in ("nvidia.cublas", "nvidia.cudnn"):
|
|
22
|
+
spec = _spec(pkg)
|
|
23
|
+
if not spec or not spec.submodule_search_locations:
|
|
24
|
+
continue
|
|
25
|
+
libdir = os.path.join(list(spec.submodule_search_locations)[0], "lib")
|
|
26
|
+
# load in name order so e.g. libcudnn.so.9 pulls in its sub-libraries
|
|
27
|
+
for so in sorted(glob.glob(os.path.join(libdir, "lib*.so*"))):
|
|
28
|
+
try:
|
|
29
|
+
ctypes.CDLL(so, mode=ctypes.RTLD_GLOBAL)
|
|
30
|
+
except OSError:
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def libs_load() -> bool:
|
|
35
|
+
"""Can the CUDA 12 libraries ctranslate2 needs be found, from the pip wheels or the system?"""
|
|
36
|
+
if not sys.platform.startswith("linux"):
|
|
37
|
+
return False
|
|
38
|
+
try:
|
|
39
|
+
for name in ("libcublas.so.12", "libcudnn.so.9"):
|
|
40
|
+
ctypes.CDLL(name)
|
|
41
|
+
return True
|
|
42
|
+
except OSError:
|
|
43
|
+
return False
|
whensaid/db.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import sqlite3
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
SCHEMA = """
|
|
5
|
+
create table if not exists roots (
|
|
6
|
+
path text primary key
|
|
7
|
+
);
|
|
8
|
+
|
|
9
|
+
create table if not exists sources (
|
|
10
|
+
id integer primary key,
|
|
11
|
+
path text unique not null,
|
|
12
|
+
size integer,
|
|
13
|
+
mtime_ns integer,
|
|
14
|
+
duration real,
|
|
15
|
+
status text not null default 'pending', -- pending, done, no_audio, failed
|
|
16
|
+
error text,
|
|
17
|
+
indexed_at text
|
|
18
|
+
);
|
|
19
|
+
|
|
20
|
+
create table if not exists segments (
|
|
21
|
+
id integer primary key,
|
|
22
|
+
source_id integer not null references sources(id) on delete cascade,
|
|
23
|
+
start real not null,
|
|
24
|
+
end real not null,
|
|
25
|
+
text text not null
|
|
26
|
+
);
|
|
27
|
+
create index if not exists segments_source on segments(source_id, start);
|
|
28
|
+
|
|
29
|
+
create table if not exists chunks (
|
|
30
|
+
id integer primary key,
|
|
31
|
+
source_id integer not null references sources(id) on delete cascade,
|
|
32
|
+
start real not null,
|
|
33
|
+
end real not null,
|
|
34
|
+
text text not null,
|
|
35
|
+
embedding blob
|
|
36
|
+
);
|
|
37
|
+
create index if not exists chunks_source on chunks(source_id, start);
|
|
38
|
+
|
|
39
|
+
create table if not exists feeds (
|
|
40
|
+
url text primary key, -- a playlist or channel, re-listed now and then for new videos
|
|
41
|
+
kind text,
|
|
42
|
+
title text,
|
|
43
|
+
limit_n integer,
|
|
44
|
+
checked_at real
|
|
45
|
+
);
|
|
46
|
+
|
|
47
|
+
create table if not exists meta (
|
|
48
|
+
key text primary key,
|
|
49
|
+
value text
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
create virtual table if not exists chunks_fts using fts5(text, tokenize='porter unicode61');
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# added after the first release, so existing indexes get them through alter table
|
|
57
|
+
SOURCE_COLUMNS = {
|
|
58
|
+
"kind": "text not null default 'file'", # file or url; a url source's path is a stable key like youtube:ID
|
|
59
|
+
"url": "text",
|
|
60
|
+
"title": "text",
|
|
61
|
+
"captions": "text", # manual, auto, whisper or subtitle-file
|
|
62
|
+
"feed_url": "text",
|
|
63
|
+
"confidence": "real", # mean word probability when whisper made the transcript
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _migrate(conn: sqlite3.Connection) -> None:
|
|
68
|
+
have = {r[1] for r in conn.execute("pragma table_info(sources)")}
|
|
69
|
+
for col, decl in SOURCE_COLUMNS.items():
|
|
70
|
+
if col not in have:
|
|
71
|
+
conn.execute(f"alter table sources add column {col} {decl}")
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def connect(path: Path) -> sqlite3.Connection:
|
|
75
|
+
conn = sqlite3.connect(path, timeout=30)
|
|
76
|
+
conn.row_factory = sqlite3.Row
|
|
77
|
+
conn.execute("pragma journal_mode = wal")
|
|
78
|
+
conn.execute("pragma foreign_keys = on")
|
|
79
|
+
conn.executescript(SCHEMA)
|
|
80
|
+
_migrate(conn)
|
|
81
|
+
return conn
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def get_meta(conn: sqlite3.Connection, key: str) -> str | None:
|
|
85
|
+
try:
|
|
86
|
+
row = conn.execute("select value from meta where key = ?", (key,)).fetchone()
|
|
87
|
+
except sqlite3.OperationalError: # index from before the meta table, opened read-only
|
|
88
|
+
return None
|
|
89
|
+
return row[0] if row else None
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def set_meta(conn: sqlite3.Connection, key: str, value: str) -> None:
|
|
93
|
+
conn.execute(
|
|
94
|
+
"insert into meta (key, value) values (?, ?) on conflict(key) do update set value = excluded.value",
|
|
95
|
+
(key, value),
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def forget_chunks(conn: sqlite3.Connection, source_id: int) -> None:
|
|
100
|
+
# fts rows are not tied to chunks by a foreign key, so clear them by hand
|
|
101
|
+
conn.execute(
|
|
102
|
+
"delete from chunks_fts where rowid in (select id from chunks where source_id = ?)",
|
|
103
|
+
(source_id,),
|
|
104
|
+
)
|
|
105
|
+
conn.execute("delete from chunks where source_id = ?", (source_id,))
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def forget_source(conn: sqlite3.Connection, source_id: int) -> None:
|
|
109
|
+
forget_chunks(conn, source_id)
|
|
110
|
+
conn.execute("delete from segments where source_id = ?", (source_id,))
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def connect_readonly(path: Path) -> sqlite3.Connection:
|
|
114
|
+
"""For the mcp server: never writes, never creates the file. The connection is
|
|
115
|
+
shared between the sdk's worker threads, so the caller has to serialise use."""
|
|
116
|
+
if not Path(path).exists():
|
|
117
|
+
raise FileNotFoundError(path)
|
|
118
|
+
conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=30, check_same_thread=False)
|
|
119
|
+
conn.row_factory = sqlite3.Row
|
|
120
|
+
return conn
|
whensaid/doctor.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""`whensaid doctor`: what works, what doesn't, and what to do about it. Makes no network
|
|
2
|
+
calls and never loads a model onto the GPU."""
|
|
3
|
+
import os
|
|
4
|
+
import platform
|
|
5
|
+
import shutil
|
|
6
|
+
import subprocess
|
|
7
|
+
import sys
|
|
8
|
+
import time
|
|
9
|
+
|
|
10
|
+
OK, WARN, FAIL = "ok", "warn", "fail"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _free_gb(path) -> float:
|
|
14
|
+
return shutil.disk_usage(path).free / 1e9
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def checks() -> list[tuple[str, str, str, str]]:
|
|
18
|
+
"""(status, name, detail, how to fix)"""
|
|
19
|
+
from . import cudalibs, db, models, paths
|
|
20
|
+
from .indexer import indexer_running
|
|
21
|
+
|
|
22
|
+
out = []
|
|
23
|
+
add = lambda status, name, detail, fix="": out.append((status, name, detail, fix))
|
|
24
|
+
|
|
25
|
+
add(OK, "python", f"{platform.python_version()} on {platform.system()} {platform.machine()}")
|
|
26
|
+
|
|
27
|
+
# compute
|
|
28
|
+
cudalibs.preload() # the pip wheels are found by path, not by the system loader
|
|
29
|
+
try:
|
|
30
|
+
import ctranslate2
|
|
31
|
+
|
|
32
|
+
gpus = ctranslate2.get_cuda_device_count()
|
|
33
|
+
except Exception as e:
|
|
34
|
+
add(FAIL, "ctranslate2", f"cannot be imported: {e}", "pip install --force-reinstall whensaid")
|
|
35
|
+
gpus = 0
|
|
36
|
+
if gpus and cudalibs.libs_load():
|
|
37
|
+
add(OK, "gpu", f"{gpus} NVIDIA GPU, CUDA 12 libraries found")
|
|
38
|
+
elif gpus:
|
|
39
|
+
add(WARN, "gpu", "an NVIDIA GPU is there but the CUDA 12 libraries are not: transcribing on the CPU (about 3x slower)",
|
|
40
|
+
"pip install 'whensaid[cuda]' (with uvx: uvx --from 'whensaid[cuda]' whensaid ...)")
|
|
41
|
+
else:
|
|
42
|
+
add(OK, "gpu", f"no NVIDIA GPU, transcribing on the CPU ({os.cpu_count()} cores)")
|
|
43
|
+
|
|
44
|
+
# models
|
|
45
|
+
free = _free_gb(paths.models_dir())
|
|
46
|
+
for kind in ("whisper", "embedder", "reranker"):
|
|
47
|
+
if models.present(kind):
|
|
48
|
+
add(OK, f"{kind} model", "on disk")
|
|
49
|
+
else:
|
|
50
|
+
need = {"whisper": 1.5, "embedder": 0.1, "reranker": 0.1}[kind]
|
|
51
|
+
if free < need * 1.5:
|
|
52
|
+
add(FAIL, f"{kind} model", f"missing, and only {free:.1f} GB free in {paths.models_dir()}", "free some disk space")
|
|
53
|
+
else:
|
|
54
|
+
add(WARN, f"{kind} model", f"not downloaded yet ({models.size_label(kind)}), fetched on first use",
|
|
55
|
+
"whensaid models pull" if not models.offline() else "offline mode is on: run `whensaid models pull` once online")
|
|
56
|
+
if (paths.home() / "ipv4").exists():
|
|
57
|
+
add(OK, "network", "downloads use IPv4 only (a stalled IPv6 route was found earlier)")
|
|
58
|
+
|
|
59
|
+
# yt-dlp, only needed for add-url
|
|
60
|
+
from .fetch import FetchError, ytdlp_cmd
|
|
61
|
+
|
|
62
|
+
try:
|
|
63
|
+
cmd = ytdlp_cmd()
|
|
64
|
+
v = subprocess.run(cmd + ["--version"], capture_output=True, text=True, timeout=30).stdout.strip()
|
|
65
|
+
age = None
|
|
66
|
+
try:
|
|
67
|
+
y, m, d = (int(x) for x in v.split(".")[:3])
|
|
68
|
+
age = (time.time() - time.mktime((y, m, d, 0, 0, 0, 0, 0, -1))) / 86400
|
|
69
|
+
except ValueError:
|
|
70
|
+
pass
|
|
71
|
+
if age is not None and age > 120:
|
|
72
|
+
add(WARN, "yt-dlp", f"{v}, {age:.0f} days old; old versions are the usual reason YouTube fails", "pip install -U yt-dlp")
|
|
73
|
+
else:
|
|
74
|
+
add(OK, "yt-dlp", v or "installed")
|
|
75
|
+
if not (shutil.which("deno") or shutil.which("node")):
|
|
76
|
+
add(WARN, "yt-dlp JS runtime", "no deno or node: some YouTube formats may be missing (captions usually still work)",
|
|
77
|
+
"install deno (https://deno.com)")
|
|
78
|
+
except FetchError:
|
|
79
|
+
add(OK, "yt-dlp", "not installed (only needed for `whensaid add-url`)", "pip install 'whensaid[url]'")
|
|
80
|
+
except Exception as e:
|
|
81
|
+
add(WARN, "yt-dlp", f"installed but did not run: {e}")
|
|
82
|
+
|
|
83
|
+
# the index
|
|
84
|
+
dbp = paths.db_path()
|
|
85
|
+
if not dbp.exists():
|
|
86
|
+
add(OK, "index", f"none yet at {dbp}", "whensaid add <folder> (or add-url <url>), then whensaid index")
|
|
87
|
+
else:
|
|
88
|
+
try:
|
|
89
|
+
conn = db.connect_readonly(dbp)
|
|
90
|
+
n = conn.execute("select count(*) from sources").fetchone()[0]
|
|
91
|
+
chunks = conn.execute("select count(*) from chunks").fetchone()[0]
|
|
92
|
+
size = dbp.stat().st_size / 1e6
|
|
93
|
+
add(OK, "index", f"{n} sources, {chunks} chunks, {size:.0f} MB at {dbp}")
|
|
94
|
+
built = db.get_meta(conn, "embed_model")
|
|
95
|
+
from .embed import MODEL
|
|
96
|
+
|
|
97
|
+
if built and built != MODEL:
|
|
98
|
+
add(FAIL, "embedding model", f"the index was built with {built}, this version uses {MODEL}", "whensaid rechunk")
|
|
99
|
+
bad = {r[0]: r[1] for r in conn.execute(
|
|
100
|
+
"select status, count(*) from sources where status in ('failed', 'skipped') group by status")} \
|
|
101
|
+
if n else {}
|
|
102
|
+
for status, count in bad.items():
|
|
103
|
+
add(WARN if status == "failed" else OK, f"{status} sources", f"{count}", "whensaid status (shows the reasons)")
|
|
104
|
+
except Exception as e:
|
|
105
|
+
add(FAIL, "index", f"cannot be read: {e}", f"move {dbp} aside and re-index")
|
|
106
|
+
if indexer_running():
|
|
107
|
+
add(OK, "indexer", "running right now (search keeps working meanwhile)")
|
|
108
|
+
return out
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def run(out=sys.stdout) -> int:
|
|
112
|
+
marks = {OK: "ok ", WARN: "warn", FAIL: "FAIL"}
|
|
113
|
+
results = checks()
|
|
114
|
+
for status, name, detail, fix in results:
|
|
115
|
+
print(f"[{marks[status]}] {name:18} {detail}", file=out)
|
|
116
|
+
if fix and status != OK or (fix and status == OK and "none yet" in detail):
|
|
117
|
+
print(f" -> {fix}", file=out)
|
|
118
|
+
bad = sum(1 for r in results if r[0] == FAIL)
|
|
119
|
+
warn = sum(1 for r in results if r[0] == WARN)
|
|
120
|
+
print(f"\n{bad} problems, {warn} warnings" if bad or warn else "\neverything looks fine", file=out)
|
|
121
|
+
return 1 if bad else 0
|
whensaid/embed.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
from .paths import models_dir
|
|
6
|
+
|
|
7
|
+
MODEL = "BAAI/bge-small-en-v1.5"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Embedder:
|
|
11
|
+
def __init__(self, model=MODEL, allow_download=False):
|
|
12
|
+
self.model_name = model
|
|
13
|
+
self.allow_download = allow_download
|
|
14
|
+
self._m = None
|
|
15
|
+
|
|
16
|
+
def _load(self):
|
|
17
|
+
from fastembed import TextEmbedding
|
|
18
|
+
self._m = TextEmbedding(
|
|
19
|
+
self.model_name,
|
|
20
|
+
cache_dir=str(models_dir() / "fastembed"),
|
|
21
|
+
local_files_only=not self.allow_download,
|
|
22
|
+
threads=min(4, os.cpu_count() or 1), # see rerank.THREADS
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
def encode(self, texts: list[str]) -> np.ndarray:
|
|
26
|
+
if self._m is None:
|
|
27
|
+
self._load()
|
|
28
|
+
vecs = np.array(list(self._m.embed(texts)), dtype=np.float32)
|
|
29
|
+
norms = np.linalg.norm(vecs, axis=1, keepdims=True)
|
|
30
|
+
return vecs / np.maximum(norms, 1e-9)
|
whensaid/export.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
from .util import timecode
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def render_txt(segments) -> str:
|
|
8
|
+
# one segment per line with its start time, so `grep -n` output is already a citation
|
|
9
|
+
return "".join(f"[{timecode(s['start'])}] {s['text']}\n" for s in segments)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def render_srt(segments) -> str:
|
|
13
|
+
out = []
|
|
14
|
+
for i, s in enumerate(segments, 1):
|
|
15
|
+
out.append(f"{i}\n{timecode(s['start'], srt=True)} --> {timecode(s['end'], srt=True)}\n{s['text']}\n")
|
|
16
|
+
return "\n".join(out)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
RENDERERS = {"txt": render_txt, "srt": render_srt}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def target_path(media_path: str, out_dir: Path | None, fmt: str, kind: str = "file") -> Path:
|
|
23
|
+
if kind == "url": # a key like youtube:ID, there is no file to sit next to
|
|
24
|
+
return out_dir / "urls" / (re.sub(r"[^\w.-]+", "_", media_path) + "." + fmt)
|
|
25
|
+
if out_dir is None: # next to the media file, same name
|
|
26
|
+
return Path(media_path).with_suffix("." + fmt)
|
|
27
|
+
# mirror the full path so two files with the same name never collide
|
|
28
|
+
return out_dir / (media_path.lstrip("/") + "." + fmt)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def export_all(conn, out_dir: Path | None, fmt: str = "txt", overwrite: bool = False) -> tuple[list[Path], list[Path]]:
|
|
32
|
+
"""Returns (written, skipped). Next to the recordings (out_dir None) an existing file may
|
|
33
|
+
be someone's real subtitles, so it is left alone unless overwrite is set."""
|
|
34
|
+
render = RENDERERS[fmt]
|
|
35
|
+
written, skipped = [], []
|
|
36
|
+
for src in conn.execute("select id, path, kind from sources where status = 'done' order by path").fetchall():
|
|
37
|
+
if src["kind"] == "url" and out_dir is None:
|
|
38
|
+
continue # web videos have no recording to write beside
|
|
39
|
+
segs = conn.execute(
|
|
40
|
+
"select start, end, text from segments where source_id = ? order by start", (src["id"],)
|
|
41
|
+
).fetchall()
|
|
42
|
+
dest = target_path(src["path"], out_dir, fmt, src["kind"])
|
|
43
|
+
text = render(segs)
|
|
44
|
+
if out_dir is None and dest.exists() and not overwrite and dest.read_text(encoding="utf-8", errors="replace") != text:
|
|
45
|
+
skipped.append(dest)
|
|
46
|
+
continue
|
|
47
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
48
|
+
dest.write_text(text, encoding="utf-8")
|
|
49
|
+
written.append(dest)
|
|
50
|
+
return written, skipped
|