codetta 0.8.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. codetta/CHANGELOG.md +153 -0
  2. codetta/__init__.py +8 -0
  3. codetta/__main__.py +5 -0
  4. codetta/cli/__init__.py +0 -0
  5. codetta/cli/__main__.py +59 -0
  6. codetta/cli/run.py +129 -0
  7. codetta/core/__init__.py +0 -0
  8. codetta/core/acoustid.py +225 -0
  9. codetta/core/activity.py +161 -0
  10. codetta/core/artwork.py +238 -0
  11. codetta/core/auth.py +362 -0
  12. codetta/core/avatar.py +92 -0
  13. codetta/core/changelog.py +51 -0
  14. codetta/core/connections.py +159 -0
  15. codetta/core/describe.py +69 -0
  16. codetta/core/engine.py +470 -0
  17. codetta/core/health.py +520 -0
  18. codetta/core/inbox.py +195 -0
  19. codetta/core/index.py +672 -0
  20. codetta/core/library.py +177 -0
  21. codetta/core/musicbrainz.py +662 -0
  22. codetta/core/notifications.py +77 -0
  23. codetta/core/onboarding.py +39 -0
  24. codetta/core/organize.py +490 -0
  25. codetta/core/plugins.py +183 -0
  26. codetta/core/presets.py +452 -0
  27. codetta/core/providers.py +203 -0
  28. codetta/core/rawtags.py +135 -0
  29. codetta/core/slskd.py +650 -0
  30. codetta/core/tagmap.py +538 -0
  31. codetta/core/undo.py +186 -0
  32. codetta/core/updates.py +132 -0
  33. codetta/plugins/__init__.py +63 -0
  34. codetta/plugins/builtin/__init__.py +3 -0
  35. codetta/plugins/builtin/tidy.py +70 -0
  36. codetta/web/__init__.py +0 -0
  37. codetta/web/app.py +2909 -0
  38. codetta/web/security.py +170 -0
  39. codetta/web/static/app.css +2684 -0
  40. codetta/web/static/app.js +5694 -0
  41. codetta/web/static/apple-touch-icon.png +0 -0
  42. codetta/web/static/fonts/LICENSE-Inter.txt +93 -0
  43. codetta/web/static/fonts/LICENSE-SchibstedGrotesk.txt +93 -0
  44. codetta/web/static/fonts/inter-cyrillic.woff2 +0 -0
  45. codetta/web/static/fonts/schibsted-grotesk.woff2 +0 -0
  46. codetta/web/static/icon.svg +1 -0
  47. codetta/web/static/icons/codetta-dark.png +0 -0
  48. codetta/web/static/icons/codetta-dark.svg +1 -0
  49. codetta/web/static/icons/codetta-dark.webp +0 -0
  50. codetta/web/static/icons/codetta-light.png +0 -0
  51. codetta/web/static/icons/codetta-light.svg +1 -0
  52. codetta/web/static/icons/codetta-light.webp +0 -0
  53. codetta/web/static/icons/codetta.png +0 -0
  54. codetta/web/static/icons/codetta.svg +1 -0
  55. codetta/web/static/icons/codetta.webp +0 -0
  56. codetta/web/static/index.html +473 -0
  57. codetta/web/static/theme.js +12 -0
  58. codetta-0.8.1.dist-info/METADATA +282 -0
  59. codetta-0.8.1.dist-info/RECORD +63 -0
  60. codetta-0.8.1.dist-info/WHEEL +5 -0
  61. codetta-0.8.1.dist-info/entry_points.txt +2 -0
  62. codetta-0.8.1.dist-info/licenses/LICENSE +661 -0
  63. codetta-0.8.1.dist-info/top_level.txt +1 -0
codetta/core/index.py ADDED
@@ -0,0 +1,672 @@
1
+ """Library-wide tag index, so the whole library can be searched in an instant.
2
+
3
+ Browsing reads one folder at a time; searching every file that way would take
4
+ minutes on a big library. Instead, a background thread keeps a small SQLite
5
+ database (``library-index.sqlite3`` in the config folder) with each file's tags
6
+ and display details:
7
+
8
+ * The first scan reads every audio file once. Later scans only look at file
9
+ sizes and modification times, and re-read just the files that changed.
10
+ * There is no timer: after the startup scan, the library is only re-checked
11
+ when someone searches (at most every ``RESCAN_SECONDS``), so an idle Codetta
12
+ never wakes sleeping hard drives.
13
+ * Files Codetta edits itself are refreshed right away (``mark_dirty``).
14
+ * Searches never block on a scan; while a scan runs, results cover what has
15
+ been indexed so far.
16
+ * It stays small: fingerprints, IDs, ReplayGain values and very long text
17
+ (lyrics) aren't indexed, and the stored track details are compressed. The
18
+ editor reads full tags from the file itself.
19
+
20
+ Query syntax (see ``parse_query``)::
21
+
22
+ genre: rap, hip-hop tag "genre" contains Rap OR Hip-Hop
23
+ artist: drake genre: rap both must match
24
+ night drive every word in title/artist/album/file name
25
+
26
+ Matching is case- and accent-insensitive and works on word starts, so
27
+ ``genre: rap`` finds "Rap" and "Hip-Hop, Rap" but not "Trap".
28
+ """
29
+
30
+ from __future__ import annotations
31
+
32
+ import json
33
+ import logging
34
+ import os
35
+ import re
36
+ import sqlite3
37
+ import threading
38
+ import time
39
+ import unicodedata
40
+ import zlib
41
+ from collections.abc import Callable, Iterable
42
+ from dataclasses import dataclass, field
43
+ from pathlib import Path
44
+
45
+ from codetta.core.engine import SUPPORTED_EXTENSIONS
46
+ from codetta.core.library import _is_valid_text, _sort_text, natural_key
47
+ from codetta.core.tagmap import FIELDS
48
+
49
+ log = logging.getLogger("codetta.index")
50
+
51
+ SCHEMA_VERSION = 4 # 4: a stored natural sort key per file (search sorts in SQLite, not Python)
52
+ RESCAN_SECONDS = 15 * 60 # a search re-checks the library if the last scan is older than this
53
+ BATCH = 200 # files per database transaction while scanning
54
+ PAR_CHUNK = 128 # files per job when worker processes help with a big scan
55
+ PAR_START = 256 # use worker processes once this many files need reading (a first scan, a big import)
56
+ MAX_QUERY_CHARS = 500
57
+
58
+ # Tags searched by plain words (no "key:").
59
+ FREE_TEXT_KEYS = ("title", "artist", "album", "albumartist")
60
+
61
+ # Not worth searching, and often large: fingerprints, IDs, loudness data.
62
+ _SKIP_KEY = re.compile(r"^(lyrics$|unsyncedlyrics$|acoustid_|musicbrainz_|replaygain_|itunnorm|itunsmpb|isrc$|barcode$|tidal_|spotify_|discogs_|deezer_|qobuz_|.*_id$|.*fingerprint)")
63
+ # Searchable, but not worth an index entry: sort names and credits repeat the real names.
64
+ _DUPLICATE_KEY = re.compile(r"^(.*sort|.*_credit|.*sortorder)$")
65
+ MAX_INDEXED_VALUE = 300 # longer values (lyrics, essays in comments) aren't searchable
66
+ MAX_STORED_VALUE = 1000 # longer values are left out of the stored track details
67
+
68
+
69
+ def _searchable(key: str, value: str) -> bool:
70
+ return len(value) <= MAX_INDEXED_VALUE and not _SKIP_KEY.match(key) and not _DUPLICATE_KEY.match(key)
71
+
72
+
73
+ def _slim(track: dict) -> dict:
74
+ """The track details kept in the index: everything the track list shows,
75
+ without huge values. Marked ``partial`` so the editor reloads full tags."""
76
+ tags = {}
77
+ for key, values in (track.get("tags") or {}).items():
78
+ if _SKIP_KEY.match(key.lower()):
79
+ continue
80
+ kept = [v for v in values if len(v) <= MAX_STORED_VALUE]
81
+ if kept:
82
+ tags[key] = kept
83
+ return {**track, "tags": tags, "partial": True}
84
+
85
+
86
+ def _pack(track: dict) -> bytes:
87
+ return zlib.compress(json.dumps(_slim(track), ensure_ascii=False, separators=(",", ":")).encode(), 6)
88
+
89
+
90
+ def _unpack(blob: bytes) -> dict:
91
+ return json.loads(zlib.decompress(blob))
92
+
93
+ # Scripts written without spaces between words: match anywhere, not on word starts.
94
+ _NO_SPACES = re.compile(r"[฀-๿⺀-鿿가-힯豈-﫿぀-ヿ]")
95
+
96
+
97
+ def sfold(text: str) -> str:
98
+ """Search form of ``text``: no accents or case, punctuation as single spaces.
99
+
100
+ "Beyoncé" -> "beyonce", "Hip-Hop" -> "hip hop", "Don't" -> "dont".
101
+ """
102
+ text = unicodedata.normalize("NFKD", text)
103
+ text = "".join(ch for ch in text if not unicodedata.combining(ch)).casefold()
104
+ text = re.sub(r"['’`]", "", text)
105
+ return " ".join(re.split(r"[\W_]+", text)).strip()
106
+
107
+
108
+ def natural_sort_text(path: str) -> str:
109
+ """``natural_key`` as one string SQLite can sort by ("Disc 2" < "Disc 10", case and accents
110
+ ignored), stored per file so a search of 100,000 hits is ordered in C, not in Python."""
111
+ out = []
112
+ for part in re.split(r"(\d+)", path):
113
+ if not part:
114
+ continue
115
+ if part.isdecimal():
116
+ digits = str(int(part))
117
+ out.append("\x00" + digits.rjust(max(20, len(digits)), "0"))
118
+ else:
119
+ out.append("\x01" + _sort_text(part))
120
+ return "".join(out)
121
+
122
+
123
+ def word_match(haystack: str | None, needle: str) -> bool:
124
+ """True if ``needle`` (already folded) starts a word somewhere in ``haystack``.
125
+
126
+ ``haystack`` holds one or more folded values separated by "\\x1f".
127
+ """
128
+ if not haystack or not needle:
129
+ return False
130
+ if _NO_SPACES.search(needle):
131
+ return needle in haystack
132
+ return any((" " + part).find(" " + needle) >= 0 for part in haystack.split("\x1f"))
133
+
134
+
135
+ # --- Query parsing ---------------------------------------------------------------------------
136
+
137
+
138
+ @dataclass
139
+ class Clause:
140
+ key: str
141
+ label: str
142
+ values: list[str] # as typed (for display); any of them may match
143
+
144
+
145
+ @dataclass
146
+ class Query:
147
+ words: list[str] = field(default_factory=list) # as typed
148
+ clauses: list[Clause] = field(default_factory=list)
149
+
150
+ @property
151
+ def empty(self) -> bool:
152
+ return not self.words and not any(c.values for c in self.clauses)
153
+
154
+
155
+ def _norm_key(text: str) -> str:
156
+ return re.sub(r"[\W_]+", "", text.casefold())
157
+
158
+
159
+ _FIELD_NAMES: dict[str, tuple[str, str]] = {}
160
+ for _f in FIELDS:
161
+ _FIELD_NAMES[_norm_key(_f.key)] = (_f.key, _f.label)
162
+ _FIELD_NAMES[_norm_key(_f.label)] = (_f.key, _f.label)
163
+ # A few names people naturally type.
164
+ for _alias, _key in {"year": "date", "track": "tracknumber", "disc": "discnumber",
165
+ "band": "albumartist", "albumartist": "albumartist", "label": "label"}.items():
166
+ _label = next((f.label for f in FIELDS if f.key == _key), _key)
167
+ _FIELD_NAMES.setdefault(_norm_key(_alias), (_key, _label))
168
+ # "movement: allegro" means the movement's name (the number is "movement number").
169
+ _FIELD_NAMES[_norm_key("movement")] = ("movementname", "Movement")
170
+
171
+
172
+ def parse_query(text: str, is_custom_key: Callable[[str], bool] = lambda k: False) -> Query:
173
+ """Split a search into ``key: value, value`` clauses and plain words.
174
+
175
+ A key is a known field name or label ("genre", "Album artist", "year"), or
176
+ a single word that is the name of a tag found in the library (custom tags).
177
+ Anything else, colons included, is plain text.
178
+ """
179
+ text = text[:MAX_QUERY_CHARS]
180
+ markers: list[tuple[int, int, str, str]] = [] # (start, end, key, label)
181
+ for colon in (m.start() for m in re.finditer(":", text)):
182
+ before = text[: colon]
183
+ floor = markers[-1][1] if markers else 0
184
+ words = list(re.finditer(r"\S+", before))
185
+ for n in (3, 2, 1): # longest field name first ("album artist" before "artist")
186
+ if len(words) < n or words[-n].start() < floor:
187
+ continue
188
+ start = words[-n].start()
189
+ name = _norm_key(before[start:])
190
+ if name in _FIELD_NAMES:
191
+ key, label = _FIELD_NAMES[name]
192
+ elif n == 1 and re.fullmatch(r"[\w.-]{1,64}", before[start:].strip()) and is_custom_key(before[start:].strip().lower()):
193
+ key = label = before[start:].strip().lower()
194
+ else:
195
+ continue
196
+ markers.append((start, colon + 1, key, label))
197
+ break
198
+
199
+ query = Query()
200
+ head = text[: markers[0][0]] if markers else text
201
+ query.words = [w for w in head.split() if sfold(w)]
202
+ for i, (_, end, key, label) in enumerate(markers):
203
+ stop = markers[i + 1][0] if i + 1 < len(markers) else len(text)
204
+ values = [v.strip() for v in re.split(r"[,;]", text[end:stop]) if sfold(v)]
205
+ query.clauses.append(Clause(key, label, values))
206
+ return query
207
+
208
+
209
+ # --- The index ---------------------------------------------------------------------------------
210
+
211
+
212
+ @dataclass
213
+ class IndexStatus:
214
+ ready: bool # the first full scan has finished
215
+ scanning: bool
216
+ files: int # files currently in the index
217
+ seen: int # files looked at in the scan that's running
218
+
219
+
220
+ Describe = Callable[[Path, str], dict] # (absolute path, library-relative path) -> track JSON
221
+
222
+
223
+ class _Workers:
224
+ """Helper processes (``python -m codetta.core.describe``) that read files during a big scan.
225
+
226
+ Jobs go out as JSON lines, results come back the same way; each worker has at most one job.
227
+ A worker that dies hands its job back to be read here, so a scan never gets stuck."""
228
+
229
+ def __init__(self, n: int, fallback: Callable[[list], list]) -> None:
230
+ import subprocess
231
+ import sys
232
+ self.fallback = fallback
233
+ self.procs = []
234
+ for _ in range(n):
235
+ try:
236
+ # A packaged app (one .exe) has no "python -m": it starts itself in helper mode instead.
237
+ cmd = [sys.executable, "--codetta-describe-worker"] if getattr(sys, "frozen", False) else \
238
+ [sys.executable, "-m", "codetta.core.describe"]
239
+ self.procs.append(subprocess.Popen(cmd, stdin=subprocess.PIPE,
240
+ stdout=subprocess.PIPE, text=True, encoding="utf-8", bufsize=1))
241
+ except OSError as exc:
242
+ log.warning("Couldn't start an index helper: %s", exc)
243
+ self.started = list(self.procs) # every helper ever started: all are reaped in close()
244
+ self.busy: list[tuple[object, str, list]] = [] # (process, root, items), oldest first
245
+
246
+ def __bool__(self) -> bool:
247
+ return bool(self.procs)
248
+
249
+ def _collect(self) -> list:
250
+ proc, root, items = self.busy.pop(0)
251
+ line = proc.stdout.readline()
252
+ if not line: # the helper died: read its files here instead
253
+ log.warning("An index helper stopped; carrying on without it")
254
+ self.procs.remove(proc)
255
+ return self.fallback(items)
256
+ try:
257
+ return [tuple(r) for r in json.loads(line)]
258
+ except ValueError:
259
+ return self.fallback(items)
260
+
261
+ def submit(self, root: str, items: list) -> list:
262
+ """Hand ``items`` to a free helper; returns whatever finished meanwhile."""
263
+ done = []
264
+ busy_procs = {id(b[0]) for b in self.busy}
265
+ free = next((p for p in self.procs if id(p) not in busy_procs), None)
266
+ while free is None and self.busy:
267
+ proc = self.busy[0][0]
268
+ done.extend(self._collect())
269
+ free = proc if proc in self.procs else next((p for p in self.procs if id(p) not in {id(b[0]) for b in self.busy}), None)
270
+ if free is None: # every helper is gone
271
+ return done + self.fallback(items)
272
+ try:
273
+ free.stdin.write(json.dumps({"root": root, "items": items}, ensure_ascii=False) + "\n")
274
+ free.stdin.flush()
275
+ self.busy.append((free, root, items))
276
+ except OSError:
277
+ self.procs.remove(free)
278
+ done.extend(self.fallback(items))
279
+ return done
280
+
281
+ def finish(self) -> list:
282
+ done = []
283
+ while self.busy:
284
+ done.extend(self._collect())
285
+ return done
286
+
287
+ def close(self) -> None:
288
+ """Stop every helper and close its pipes (including any that died along the way)."""
289
+ for p in self.started:
290
+ try:
291
+ p.stdin.close()
292
+ except Exception: # noqa: BLE001 - already closed or broken
293
+ pass
294
+ try:
295
+ p.wait(timeout=5)
296
+ except Exception: # noqa: BLE001
297
+ p.kill()
298
+ p.wait()
299
+ try:
300
+ p.stdout.close()
301
+ except Exception: # noqa: BLE001
302
+ pass
303
+ self.procs, self.started = [], []
304
+
305
+
306
+ class LibraryIndex:
307
+ """Keeps the searchable copy of the library's tags up to date.
308
+
309
+ ``describe(path, rel)`` returns the JSON-able track dict (the same shape the
310
+ browse API sends: tags, format, duration, art…), or raises ``OSError`` /
311
+ any exception for unreadable files, which are indexed with an ``error``.
312
+ """
313
+
314
+ def __init__(self, root: Path, db_path: Path | None, describe: Describe, rescan_seconds: float = RESCAN_SECONDS,
315
+ parallel: Callable[[str, list[tuple[str, int, int]]], list] | None = None, workers: int = 0):
316
+ """``parallel`` (a picklable function: (root, [(rel, mtime, size)]) -> [(rel, mtime, size, track)])
317
+ lets big scans use ``workers`` processes; 0/1 workers reads everything here."""
318
+ self.root = Path(os.path.abspath(root))
319
+ self.describe = describe
320
+ self.parallel = parallel
321
+ self.workers = workers
322
+ self.rescan_seconds = rescan_seconds
323
+ self._lock = threading.Lock() # guards the connection
324
+ self.version = 0 # goes up whenever a file is added, changed or removed (for caches)
325
+ self._dirty: set[str] = set()
326
+ self._dirty_lock = threading.Lock()
327
+ self._stop = threading.Event()
328
+ self._wake = threading.Event()
329
+ self._thread: threading.Thread | None = None
330
+ self._scanning = False
331
+ self._seen = 0
332
+ self._ready = False
333
+ self._last_scan = 0.0 # monotonic time the last full scan finished
334
+ self._scan_requested = True # scan once at startup
335
+ self.db = self._open(db_path)
336
+
337
+ # -- database ------------------------------------------------------------------------------
338
+
339
+ def _open(self, db_path: Path | None) -> sqlite3.Connection:
340
+ db = None
341
+ if db_path is not None:
342
+ try:
343
+ db_path.parent.mkdir(parents=True, exist_ok=True)
344
+ db = sqlite3.connect(db_path, check_same_thread=False)
345
+ db.execute("PRAGMA journal_mode=WAL")
346
+ version = db.execute("PRAGMA user_version").fetchone()[0]
347
+ if version not in (0, SCHEMA_VERSION): # older layout: rebuild from scratch
348
+ db.executescript("DROP TABLE IF EXISTS files; DROP TABLE IF EXISTS tags; DROP TABLE IF EXISTS keys; VACUUM;")
349
+ except (OSError, sqlite3.Error) as exc:
350
+ log.warning("Library index can't be saved in the config folder (%s); keeping it in memory.", exc)
351
+ db = None
352
+ if db is None:
353
+ db = sqlite3.connect(":memory:", check_same_thread=False)
354
+ db.executescript(
355
+ """
356
+ CREATE TABLE IF NOT EXISTS files (
357
+ id INTEGER PRIMARY KEY, path TEXT NOT NULL UNIQUE, mtime INTEGER, size INTEGER,
358
+ text TEXT, track BLOB NOT NULL, nkey TEXT NOT NULL DEFAULT '');
359
+ CREATE TABLE IF NOT EXISTS keys (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE);
360
+ -- One row per file and tag; tag names are stored once, in "keys".
361
+ CREATE TABLE IF NOT EXISTS tags (
362
+ key INTEGER NOT NULL, file INTEGER NOT NULL, value TEXT NOT NULL,
363
+ PRIMARY KEY (key, file)) WITHOUT ROWID;
364
+ CREATE INDEX IF NOT EXISTS tags_file ON tags(file);
365
+ """
366
+ )
367
+ db.execute(f"PRAGMA user_version={SCHEMA_VERSION}")
368
+ db.create_function("wm", 2, word_match, deterministic=True)
369
+ db.commit()
370
+ self._key_ids = dict(db.execute("SELECT name, id FROM keys").fetchall())
371
+ return db
372
+
373
+ def _key_id(self, cur: sqlite3.Cursor, name: str) -> int:
374
+ if name not in self._key_ids:
375
+ cur.execute("INSERT OR IGNORE INTO keys(name) VALUES (?)", (name,))
376
+ self._key_ids[name] = cur.execute("SELECT id FROM keys WHERE name=?", (name,)).fetchone()[0]
377
+ return self._key_ids[name]
378
+
379
+ def _store(self, rows: list[tuple[str, int, int, dict]]) -> None:
380
+ with self._lock:
381
+ cur = self.db.cursor()
382
+ for rel, mtime, size, track in rows:
383
+ tags = track.get("tags") or {}
384
+ text = "\x1f".join(
385
+ [sfold(v) for k in FREE_TEXT_KEYS for v in tags.get(k, [])] + [sfold(Path(rel).stem)]
386
+ )
387
+ row = cur.execute("SELECT id FROM files WHERE path=?", (rel,)).fetchone()
388
+ if row:
389
+ file_id = row[0]
390
+ cur.execute("DELETE FROM tags WHERE file=?", (file_id,))
391
+ cur.execute("UPDATE files SET mtime=?, size=?, text=?, track=?, nkey=? WHERE id=?",
392
+ (mtime, size, text, _pack(track), natural_sort_text(rel), file_id))
393
+ else:
394
+ cur.execute("INSERT INTO files(path, mtime, size, text, track, nkey) VALUES (?,?,?,?,?,?)",
395
+ (rel, mtime, size, text, _pack(track), natural_sort_text(rel)))
396
+ file_id = cur.lastrowid
397
+ merged: dict[str, list[str]] = {} # "Genre" and "genre" are the same tag
398
+ for k, vals in tags.items():
399
+ kept = [sfold(v) for v in vals if _searchable(k.lower(), v)]
400
+ if kept:
401
+ merged.setdefault(k.lower(), []).extend(kept)
402
+ cur.executemany("INSERT INTO tags(key, file, value) VALUES (?,?,?)",
403
+ [(self._key_id(cur, k), file_id, "\x1f".join(v)) for k, v in merged.items()])
404
+ self.db.commit()
405
+ if rows:
406
+ self.version += 1
407
+
408
+ def _remove(self, rels: Iterable[str]) -> None:
409
+ rels = list(rels)
410
+ if not rels:
411
+ return
412
+ with self._lock:
413
+ for r in rels:
414
+ row = self.db.execute("SELECT id FROM files WHERE path=?", (r,)).fetchone()
415
+ if row:
416
+ self.db.execute("DELETE FROM tags WHERE file=?", (row[0],))
417
+ self.db.execute("DELETE FROM files WHERE id=?", (row[0],))
418
+ self.db.commit()
419
+ self.version += 1
420
+
421
+ def _read_item(self, item: tuple[str, int, int]) -> tuple[str, int, int, dict]:
422
+ rel, mtime, size = item
423
+ try:
424
+ track = self.describe(self.root / rel, rel)
425
+ except Exception as exc: # noqa: BLE001 - one bad file must not stop the scan
426
+ track = {"path": rel, "name": rel.rsplit("/", 1)[-1], "error": str(exc) or type(exc).__name__}
427
+ return rel, mtime, size, track
428
+
429
+ def _read(self, rel: str, st: os.stat_result) -> tuple[str, int, int, dict]:
430
+ try:
431
+ track = self.describe(self.root / rel, rel)
432
+ except Exception as exc: # noqa: BLE001 - one bad file must not stop the scan
433
+ track = {"path": rel, "name": rel.rsplit("/", 1)[-1], "error": str(exc) or type(exc).__name__}
434
+ return rel, st.st_mtime_ns, st.st_size, track
435
+
436
+ # -- scanning ----------------------------------------------------------------------------------
437
+
438
+ def _walk(self) -> Iterable[tuple[str, os.stat_result]]:
439
+ """Every audio file under the root (following folder links, once each)."""
440
+ seen_dirs: set[tuple[int, int]] = set()
441
+ stack = [""]
442
+ while stack and not self._stop.is_set():
443
+ rel_dir = stack.pop()
444
+ folder = self.root / rel_dir if rel_dir else self.root
445
+ try:
446
+ st = folder.stat()
447
+ if (st.st_dev, st.st_ino) in seen_dirs:
448
+ continue # a link loop, or the same folder linked twice
449
+ seen_dirs.add((st.st_dev, st.st_ino))
450
+ with os.scandir(folder) as entries:
451
+ items = list(entries)
452
+ except OSError:
453
+ continue
454
+ for entry in items:
455
+ if entry.name.startswith(".") or not _is_valid_text(entry.name):
456
+ continue
457
+ rel = f"{rel_dir}/{entry.name}" if rel_dir else entry.name
458
+ try:
459
+ if entry.is_dir():
460
+ stack.append(rel)
461
+ elif entry.is_file() and Path(entry.name).suffix.lower() in SUPPORTED_EXTENSIONS:
462
+ yield rel, entry.stat()
463
+ except OSError:
464
+ continue
465
+
466
+ def scan(self) -> None:
467
+ """Bring the index up to date with the files on disk."""
468
+ if not self.root.is_dir():
469
+ return
470
+ self._scanning, self._seen = True, 0
471
+ started = time.monotonic()
472
+ pool = None
473
+ try:
474
+ with self._lock:
475
+ known = {p: (m, s) for p, m, s in self.db.execute("SELECT path, mtime, size FROM files")}
476
+ present: set[str] = set()
477
+ batch: list[tuple[str, int, int, dict]] = []
478
+ changed = 0
479
+ pending: list[tuple[str, int, int]] = [] # waiting to be read (in a helper, once it's worth it)
480
+
481
+ def store(rows: list) -> None:
482
+ nonlocal changed, batch
483
+ batch.extend(rows)
484
+ if len(batch) >= BATCH:
485
+ self._store(batch)
486
+ changed += len(batch)
487
+ batch = []
488
+
489
+ def read_here(items: list) -> list:
490
+ return [self._read_item(i) for i in items]
491
+
492
+ for rel, st in self._walk():
493
+ present.add(rel)
494
+ self._seen += 1
495
+ if known.get(rel) == (st.st_mtime_ns, st.st_size):
496
+ continue
497
+ pending.append((rel, st.st_mtime_ns, st.st_size))
498
+ if pool is None and len(pending) >= PAR_START and self.parallel and self.workers > 1:
499
+ pool = _Workers(self.workers, read_here) or None
500
+ if pool is not None and len(pending) >= PAR_CHUNK:
501
+ store(pool.submit(str(self.root), pending)) # bounded: one job per helper at a time
502
+ pending = []
503
+ elif pool is None and len(pending) >= PAR_START:
504
+ store([self._read_item(i) for i in pending]) # no workers: read here, in order
505
+ pending = []
506
+ self._flush_dirty() # edits made during a long first scan show up quickly
507
+ if pool is not None:
508
+ if pending:
509
+ store(pool.submit(str(self.root), pending))
510
+ pending = []
511
+ store(pool.finish())
512
+ pool.close()
513
+ store([self._read_item(i) for i in pending])
514
+ if batch:
515
+ self._store(batch)
516
+ changed += len(batch)
517
+ if not self._stop.is_set():
518
+ gone = set(known) - present
519
+ self._remove(gone)
520
+ self._ready = True
521
+ self._last_scan = time.monotonic()
522
+ if changed or gone:
523
+ with self._lock: # fold the write-ahead log back in, so it doesn't linger on disk
524
+ self.db.execute("PRAGMA wal_checkpoint(TRUNCATE)")
525
+ if changed or gone:
526
+ log.info("Library index: %d files, %d updated, %d removed (%.1fs)",
527
+ len(present), changed, len(gone), time.monotonic() - started)
528
+ finally:
529
+ if pool is not None:
530
+ pool.close() # never leave helpers behind, even if the scan stopped half-way
531
+ self._scanning = False
532
+
533
+ def mark_dirty(self, rels: Iterable[str]) -> None:
534
+ """Files Codetta just changed: re-read them before the next search."""
535
+ with self._dirty_lock:
536
+ self._dirty.update(rels)
537
+ self._wake.set()
538
+
539
+ def _flush_dirty(self) -> None:
540
+ with self._dirty_lock:
541
+ dirty, self._dirty = self._dirty, set()
542
+ if not dirty:
543
+ return
544
+ rows, gone = [], []
545
+ for rel in dirty:
546
+ try:
547
+ st = (self.root / rel).stat()
548
+ except OSError:
549
+ gone.append(rel)
550
+ continue
551
+ rows.append(self._read(rel, st))
552
+ if rows:
553
+ self._store(rows)
554
+ self._remove(gone)
555
+
556
+ # -- background thread -------------------------------------------------------------------------
557
+
558
+ def start(self) -> None:
559
+ if self._thread is None:
560
+ self._thread = threading.Thread(target=self._run, name="codetta-index", daemon=True)
561
+ self._thread.start()
562
+
563
+ def stop(self) -> None:
564
+ self._stop.set()
565
+ self._wake.set()
566
+ if self._thread is not None:
567
+ self._thread.join(timeout=5)
568
+ self._thread = None
569
+
570
+ def request_rescan(self) -> None:
571
+ """Called on each search: re-check the library if the last scan is old."""
572
+ if not self._scanning and time.monotonic() - self._last_scan > self.rescan_seconds:
573
+ self._scan_requested = True
574
+ self._wake.set()
575
+
576
+ def _run(self) -> None:
577
+ # No timer: sleeps until something asks (startup, a search, an edit).
578
+ while not self._stop.is_set():
579
+ try:
580
+ if self._scan_requested:
581
+ self._scan_requested = False
582
+ self.scan()
583
+ self._flush_dirty()
584
+ except Exception: # noqa: BLE001 - keep the index alive; try again on the next request
585
+ log.exception("Library index scan failed")
586
+ self._last_scan = time.monotonic()
587
+ self._wake.wait()
588
+ self._wake.clear()
589
+
590
+ # -- searching ---------------------------------------------------------------------------------
591
+
592
+ def status(self) -> IndexStatus:
593
+ with self._lock:
594
+ files = self.db.execute("SELECT COUNT(*) FROM files").fetchone()[0]
595
+ return IndexStatus(ready=self._ready, scanning=self._scanning, files=files, seen=self._seen)
596
+
597
+ def all_tracks(self) -> list[tuple[str, dict]]:
598
+ """Every indexed track (path, details), in path order. ~5,000 tracks take
599
+ about a tenth of a second: no file is opened."""
600
+ with self._lock:
601
+ rows = self.db.execute("SELECT path, track FROM files ORDER BY path").fetchall()
602
+ return [(p, _unpack(t)) for p, t in rows]
603
+
604
+ def stored(self, rels: list[str]) -> dict[str, tuple[int, int, dict]]:
605
+ """What the index has for these files: path -> (mtime_ns, size, details)."""
606
+ out: dict[str, tuple[int, int, dict]] = {}
607
+ for i in range(0, len(rels), 900): # SQLite's limit on query parameters
608
+ chunk = rels[i:i + 900]
609
+ q = "SELECT path, mtime, size, track FROM files WHERE path IN (" + ",".join("?" * len(chunk)) + ")"
610
+ with self._lock:
611
+ rows = self.db.execute(q, chunk).fetchall()
612
+ for path, mtime, size, blob in rows:
613
+ out[path] = (mtime, size, _unpack(blob))
614
+ return out
615
+
616
+ def iter_tracks(self, chunk: int = 2000):
617
+ """Every indexed track (path, details) in path order, read a chunk at a time, so even a
618
+ huge library is never all in memory at once (and indexing isn't blocked meanwhile)."""
619
+ last = ""
620
+ while True:
621
+ with self._lock:
622
+ rows = self.db.execute("SELECT path, track FROM files WHERE path > ? ORDER BY path LIMIT ?",
623
+ (last, chunk)).fetchall()
624
+ if not rows:
625
+ return
626
+ for path, blob in rows:
627
+ yield path, _unpack(blob)
628
+ last = rows[-1][0]
629
+
630
+ def paths_with_any(self, keys: Iterable[str]) -> set[str]:
631
+ """Files that have at least one of these tags (by name, any case)."""
632
+ with self._lock:
633
+ ids = [self._key_ids[k] for k in {k.lower() for k in keys} if k in self._key_ids]
634
+ if not ids:
635
+ return set()
636
+ q = ("SELECT DISTINCT f.path FROM files f JOIN tags t ON t.file = f.id WHERE t.key IN ("
637
+ + ",".join("?" * len(ids)) + ")")
638
+ return {p for (p,) in self.db.execute(q, ids)}
639
+
640
+ def has_key(self, key: str) -> bool:
641
+ with self._lock:
642
+ return key in self._key_ids
643
+
644
+ def search(self, query: Query, limit: int = 500) -> tuple[list[dict], int]:
645
+ """Tracks matching ``query``, in library order (first ``limit``), and the total count."""
646
+ if query.empty:
647
+ return [], 0
648
+ self._flush_dirty()
649
+ sql = ["SELECT path FROM files WHERE 1"]
650
+ args: list[str] = []
651
+ # instr() (in C) rules most rows out before the Python word check runs on what's left.
652
+ for word in query.words:
653
+ sql.append("AND instr(text, ?) AND wm(text, ?)")
654
+ args += [sfold(word), sfold(word)]
655
+ for clause in query.clauses:
656
+ needles = [sfold(v) for v in clause.values]
657
+ if not needles:
658
+ continue
659
+ sql.append("AND id IN (SELECT file FROM tags WHERE key=(SELECT id FROM keys WHERE name=?) AND ("
660
+ + " OR ".join(["(instr(value, ?) AND wm(value, ?))"] * len(needles)) + "))")
661
+ args += [clause.key, *[x for n in needles for x in (n, n)]]
662
+ sql.append("ORDER BY nkey") # library order ("Disc 2" before "Disc 10"), sorted by SQLite
663
+ with self._lock:
664
+ paths = [p for (p,) in self.db.execute(" ".join(sql), args)]
665
+ shown = paths[:limit]
666
+ with self._lock:
667
+ tracks = {}
668
+ for i in range(0, len(shown), 500):
669
+ chunk = shown[i:i + 500]
670
+ q = "SELECT path, track FROM files WHERE path IN (" + ",".join("?" * len(chunk)) + ")"
671
+ tracks.update({p: _unpack(t) for p, t in self.db.execute(q, chunk)})
672
+ return [tracks[p] for p in shown if p in tracks], len(paths)