codetta 0.8.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codetta/CHANGELOG.md +153 -0
- codetta/__init__.py +8 -0
- codetta/__main__.py +5 -0
- codetta/cli/__init__.py +0 -0
- codetta/cli/__main__.py +59 -0
- codetta/cli/run.py +129 -0
- codetta/core/__init__.py +0 -0
- codetta/core/acoustid.py +225 -0
- codetta/core/activity.py +161 -0
- codetta/core/artwork.py +238 -0
- codetta/core/auth.py +362 -0
- codetta/core/avatar.py +92 -0
- codetta/core/changelog.py +51 -0
- codetta/core/connections.py +159 -0
- codetta/core/describe.py +69 -0
- codetta/core/engine.py +470 -0
- codetta/core/health.py +520 -0
- codetta/core/inbox.py +195 -0
- codetta/core/index.py +672 -0
- codetta/core/library.py +177 -0
- codetta/core/musicbrainz.py +662 -0
- codetta/core/notifications.py +77 -0
- codetta/core/onboarding.py +39 -0
- codetta/core/organize.py +490 -0
- codetta/core/plugins.py +183 -0
- codetta/core/presets.py +452 -0
- codetta/core/providers.py +203 -0
- codetta/core/rawtags.py +135 -0
- codetta/core/slskd.py +650 -0
- codetta/core/tagmap.py +538 -0
- codetta/core/undo.py +186 -0
- codetta/core/updates.py +132 -0
- codetta/plugins/__init__.py +63 -0
- codetta/plugins/builtin/__init__.py +3 -0
- codetta/plugins/builtin/tidy.py +70 -0
- codetta/web/__init__.py +0 -0
- codetta/web/app.py +2909 -0
- codetta/web/security.py +170 -0
- codetta/web/static/app.css +2684 -0
- codetta/web/static/app.js +5694 -0
- codetta/web/static/apple-touch-icon.png +0 -0
- codetta/web/static/fonts/LICENSE-Inter.txt +93 -0
- codetta/web/static/fonts/LICENSE-SchibstedGrotesk.txt +93 -0
- codetta/web/static/fonts/inter-cyrillic.woff2 +0 -0
- codetta/web/static/fonts/schibsted-grotesk.woff2 +0 -0
- codetta/web/static/icon.svg +1 -0
- codetta/web/static/icons/codetta-dark.png +0 -0
- codetta/web/static/icons/codetta-dark.svg +1 -0
- codetta/web/static/icons/codetta-dark.webp +0 -0
- codetta/web/static/icons/codetta-light.png +0 -0
- codetta/web/static/icons/codetta-light.svg +1 -0
- codetta/web/static/icons/codetta-light.webp +0 -0
- codetta/web/static/icons/codetta.png +0 -0
- codetta/web/static/icons/codetta.svg +1 -0
- codetta/web/static/icons/codetta.webp +0 -0
- codetta/web/static/index.html +473 -0
- codetta/web/static/theme.js +12 -0
- codetta-0.8.1.dist-info/METADATA +282 -0
- codetta-0.8.1.dist-info/RECORD +63 -0
- codetta-0.8.1.dist-info/WHEEL +5 -0
- codetta-0.8.1.dist-info/entry_points.txt +2 -0
- codetta-0.8.1.dist-info/licenses/LICENSE +661 -0
- codetta-0.8.1.dist-info/top_level.txt +1 -0
codetta/core/index.py
ADDED
|
@@ -0,0 +1,672 @@
|
|
|
1
|
+
"""Library-wide tag index, so the whole library can be searched in an instant.
|
|
2
|
+
|
|
3
|
+
Browsing reads one folder at a time; searching every file that way would take
|
|
4
|
+
minutes on a big library. Instead, a background thread keeps a small SQLite
|
|
5
|
+
database (``library-index.sqlite3`` in the config folder) with each file's tags
|
|
6
|
+
and display details:
|
|
7
|
+
|
|
8
|
+
* The first scan reads every audio file once. Later scans only look at file
|
|
9
|
+
sizes and modification times, and re-read just the files that changed.
|
|
10
|
+
* There is no timer: after the startup scan, the library is only re-checked
|
|
11
|
+
when someone searches (at most every ``RESCAN_SECONDS``), so an idle Codetta
|
|
12
|
+
never wakes sleeping hard drives.
|
|
13
|
+
* Files Codetta edits itself are refreshed right away (``mark_dirty``).
|
|
14
|
+
* Searches never block on a scan; while a scan runs, results cover what has
|
|
15
|
+
been indexed so far.
|
|
16
|
+
* It stays small: fingerprints, IDs, ReplayGain values and very long text
|
|
17
|
+
(lyrics) aren't indexed, and the stored track details are compressed. The
|
|
18
|
+
editor reads full tags from the file itself.
|
|
19
|
+
|
|
20
|
+
Query syntax (see ``parse_query``)::
|
|
21
|
+
|
|
22
|
+
genre: rap, hip-hop tag "genre" contains Rap OR Hip-Hop
|
|
23
|
+
artist: drake genre: rap both must match
|
|
24
|
+
night drive every word in title/artist/album/file name
|
|
25
|
+
|
|
26
|
+
Matching is case- and accent-insensitive and works on word starts, so
|
|
27
|
+
``genre: rap`` finds "Rap" and "Hip-Hop, Rap" but not "Trap".
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import json
|
|
33
|
+
import logging
|
|
34
|
+
import os
|
|
35
|
+
import re
|
|
36
|
+
import sqlite3
|
|
37
|
+
import threading
|
|
38
|
+
import time
|
|
39
|
+
import unicodedata
|
|
40
|
+
import zlib
|
|
41
|
+
from collections.abc import Callable, Iterable
|
|
42
|
+
from dataclasses import dataclass, field
|
|
43
|
+
from pathlib import Path
|
|
44
|
+
|
|
45
|
+
from codetta.core.engine import SUPPORTED_EXTENSIONS
|
|
46
|
+
from codetta.core.library import _is_valid_text, _sort_text, natural_key
|
|
47
|
+
from codetta.core.tagmap import FIELDS
|
|
48
|
+
|
|
49
|
+
log = logging.getLogger("codetta.index")
|
|
50
|
+
|
|
51
|
+
SCHEMA_VERSION = 4 # 4: a stored natural sort key per file (search sorts in SQLite, not Python)
|
|
52
|
+
RESCAN_SECONDS = 15 * 60 # a search re-checks the library if the last scan is older than this
|
|
53
|
+
BATCH = 200 # files per database transaction while scanning
|
|
54
|
+
PAR_CHUNK = 128 # files per job when worker processes help with a big scan
|
|
55
|
+
PAR_START = 256 # use worker processes once this many files need reading (a first scan, a big import)
|
|
56
|
+
MAX_QUERY_CHARS = 500
|
|
57
|
+
|
|
58
|
+
# Tags searched by plain words (no "key:").
|
|
59
|
+
FREE_TEXT_KEYS = ("title", "artist", "album", "albumartist")
|
|
60
|
+
|
|
61
|
+
# Not worth searching, and often large: fingerprints, IDs, loudness data.
|
|
62
|
+
_SKIP_KEY = re.compile(r"^(lyrics$|unsyncedlyrics$|acoustid_|musicbrainz_|replaygain_|itunnorm|itunsmpb|isrc$|barcode$|tidal_|spotify_|discogs_|deezer_|qobuz_|.*_id$|.*fingerprint)")
|
|
63
|
+
# Searchable, but not worth an index entry: sort names and credits repeat the real names.
|
|
64
|
+
_DUPLICATE_KEY = re.compile(r"^(.*sort|.*_credit|.*sortorder)$")
|
|
65
|
+
MAX_INDEXED_VALUE = 300 # longer values (lyrics, essays in comments) aren't searchable
|
|
66
|
+
MAX_STORED_VALUE = 1000 # longer values are left out of the stored track details
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _searchable(key: str, value: str) -> bool:
|
|
70
|
+
return len(value) <= MAX_INDEXED_VALUE and not _SKIP_KEY.match(key) and not _DUPLICATE_KEY.match(key)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _slim(track: dict) -> dict:
|
|
74
|
+
"""The track details kept in the index: everything the track list shows,
|
|
75
|
+
without huge values. Marked ``partial`` so the editor reloads full tags."""
|
|
76
|
+
tags = {}
|
|
77
|
+
for key, values in (track.get("tags") or {}).items():
|
|
78
|
+
if _SKIP_KEY.match(key.lower()):
|
|
79
|
+
continue
|
|
80
|
+
kept = [v for v in values if len(v) <= MAX_STORED_VALUE]
|
|
81
|
+
if kept:
|
|
82
|
+
tags[key] = kept
|
|
83
|
+
return {**track, "tags": tags, "partial": True}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _pack(track: dict) -> bytes:
|
|
87
|
+
return zlib.compress(json.dumps(_slim(track), ensure_ascii=False, separators=(",", ":")).encode(), 6)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _unpack(blob: bytes) -> dict:
|
|
91
|
+
return json.loads(zlib.decompress(blob))
|
|
92
|
+
|
|
93
|
+
# Scripts written without spaces between words: match anywhere, not on word starts.
|
|
94
|
+
_NO_SPACES = re.compile(r"[-⺀-鿿가-豈--ヿ]")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def sfold(text: str) -> str:
|
|
98
|
+
"""Search form of ``text``: no accents or case, punctuation as single spaces.
|
|
99
|
+
|
|
100
|
+
"Beyoncé" -> "beyonce", "Hip-Hop" -> "hip hop", "Don't" -> "dont".
|
|
101
|
+
"""
|
|
102
|
+
text = unicodedata.normalize("NFKD", text)
|
|
103
|
+
text = "".join(ch for ch in text if not unicodedata.combining(ch)).casefold()
|
|
104
|
+
text = re.sub(r"['’`]", "", text)
|
|
105
|
+
return " ".join(re.split(r"[\W_]+", text)).strip()
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def natural_sort_text(path: str) -> str:
|
|
109
|
+
"""``natural_key`` as one string SQLite can sort by ("Disc 2" < "Disc 10", case and accents
|
|
110
|
+
ignored), stored per file so a search of 100,000 hits is ordered in C, not in Python."""
|
|
111
|
+
out = []
|
|
112
|
+
for part in re.split(r"(\d+)", path):
|
|
113
|
+
if not part:
|
|
114
|
+
continue
|
|
115
|
+
if part.isdecimal():
|
|
116
|
+
digits = str(int(part))
|
|
117
|
+
out.append("\x00" + digits.rjust(max(20, len(digits)), "0"))
|
|
118
|
+
else:
|
|
119
|
+
out.append("\x01" + _sort_text(part))
|
|
120
|
+
return "".join(out)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def word_match(haystack: str | None, needle: str) -> bool:
|
|
124
|
+
"""True if ``needle`` (already folded) starts a word somewhere in ``haystack``.
|
|
125
|
+
|
|
126
|
+
``haystack`` holds one or more folded values separated by "\\x1f".
|
|
127
|
+
"""
|
|
128
|
+
if not haystack or not needle:
|
|
129
|
+
return False
|
|
130
|
+
if _NO_SPACES.search(needle):
|
|
131
|
+
return needle in haystack
|
|
132
|
+
return any((" " + part).find(" " + needle) >= 0 for part in haystack.split("\x1f"))
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
# --- Query parsing ---------------------------------------------------------------------------
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
@dataclass
|
|
139
|
+
class Clause:
|
|
140
|
+
key: str
|
|
141
|
+
label: str
|
|
142
|
+
values: list[str] # as typed (for display); any of them may match
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@dataclass
|
|
146
|
+
class Query:
|
|
147
|
+
words: list[str] = field(default_factory=list) # as typed
|
|
148
|
+
clauses: list[Clause] = field(default_factory=list)
|
|
149
|
+
|
|
150
|
+
@property
|
|
151
|
+
def empty(self) -> bool:
|
|
152
|
+
return not self.words and not any(c.values for c in self.clauses)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _norm_key(text: str) -> str:
|
|
156
|
+
return re.sub(r"[\W_]+", "", text.casefold())
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
_FIELD_NAMES: dict[str, tuple[str, str]] = {}
|
|
160
|
+
for _f in FIELDS:
|
|
161
|
+
_FIELD_NAMES[_norm_key(_f.key)] = (_f.key, _f.label)
|
|
162
|
+
_FIELD_NAMES[_norm_key(_f.label)] = (_f.key, _f.label)
|
|
163
|
+
# A few names people naturally type.
|
|
164
|
+
for _alias, _key in {"year": "date", "track": "tracknumber", "disc": "discnumber",
|
|
165
|
+
"band": "albumartist", "albumartist": "albumartist", "label": "label"}.items():
|
|
166
|
+
_label = next((f.label for f in FIELDS if f.key == _key), _key)
|
|
167
|
+
_FIELD_NAMES.setdefault(_norm_key(_alias), (_key, _label))
|
|
168
|
+
# "movement: allegro" means the movement's name (the number is "movement number").
|
|
169
|
+
_FIELD_NAMES[_norm_key("movement")] = ("movementname", "Movement")
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def parse_query(text: str, is_custom_key: Callable[[str], bool] = lambda k: False) -> Query:
|
|
173
|
+
"""Split a search into ``key: value, value`` clauses and plain words.
|
|
174
|
+
|
|
175
|
+
A key is a known field name or label ("genre", "Album artist", "year"), or
|
|
176
|
+
a single word that is the name of a tag found in the library (custom tags).
|
|
177
|
+
Anything else, colons included, is plain text.
|
|
178
|
+
"""
|
|
179
|
+
text = text[:MAX_QUERY_CHARS]
|
|
180
|
+
markers: list[tuple[int, int, str, str]] = [] # (start, end, key, label)
|
|
181
|
+
for colon in (m.start() for m in re.finditer(":", text)):
|
|
182
|
+
before = text[: colon]
|
|
183
|
+
floor = markers[-1][1] if markers else 0
|
|
184
|
+
words = list(re.finditer(r"\S+", before))
|
|
185
|
+
for n in (3, 2, 1): # longest field name first ("album artist" before "artist")
|
|
186
|
+
if len(words) < n or words[-n].start() < floor:
|
|
187
|
+
continue
|
|
188
|
+
start = words[-n].start()
|
|
189
|
+
name = _norm_key(before[start:])
|
|
190
|
+
if name in _FIELD_NAMES:
|
|
191
|
+
key, label = _FIELD_NAMES[name]
|
|
192
|
+
elif n == 1 and re.fullmatch(r"[\w.-]{1,64}", before[start:].strip()) and is_custom_key(before[start:].strip().lower()):
|
|
193
|
+
key = label = before[start:].strip().lower()
|
|
194
|
+
else:
|
|
195
|
+
continue
|
|
196
|
+
markers.append((start, colon + 1, key, label))
|
|
197
|
+
break
|
|
198
|
+
|
|
199
|
+
query = Query()
|
|
200
|
+
head = text[: markers[0][0]] if markers else text
|
|
201
|
+
query.words = [w for w in head.split() if sfold(w)]
|
|
202
|
+
for i, (_, end, key, label) in enumerate(markers):
|
|
203
|
+
stop = markers[i + 1][0] if i + 1 < len(markers) else len(text)
|
|
204
|
+
values = [v.strip() for v in re.split(r"[,;]", text[end:stop]) if sfold(v)]
|
|
205
|
+
query.clauses.append(Clause(key, label, values))
|
|
206
|
+
return query
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
# --- The index ---------------------------------------------------------------------------------
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
@dataclass
|
|
213
|
+
class IndexStatus:
|
|
214
|
+
ready: bool # the first full scan has finished
|
|
215
|
+
scanning: bool
|
|
216
|
+
files: int # files currently in the index
|
|
217
|
+
seen: int # files looked at in the scan that's running
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
Describe = Callable[[Path, str], dict] # (absolute path, library-relative path) -> track JSON
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
class _Workers:
|
|
224
|
+
"""Helper processes (``python -m codetta.core.describe``) that read files during a big scan.
|
|
225
|
+
|
|
226
|
+
Jobs go out as JSON lines, results come back the same way; each worker has at most one job.
|
|
227
|
+
A worker that dies hands its job back to be read here, so a scan never gets stuck."""
|
|
228
|
+
|
|
229
|
+
def __init__(self, n: int, fallback: Callable[[list], list]) -> None:
|
|
230
|
+
import subprocess
|
|
231
|
+
import sys
|
|
232
|
+
self.fallback = fallback
|
|
233
|
+
self.procs = []
|
|
234
|
+
for _ in range(n):
|
|
235
|
+
try:
|
|
236
|
+
# A packaged app (one .exe) has no "python -m": it starts itself in helper mode instead.
|
|
237
|
+
cmd = [sys.executable, "--codetta-describe-worker"] if getattr(sys, "frozen", False) else \
|
|
238
|
+
[sys.executable, "-m", "codetta.core.describe"]
|
|
239
|
+
self.procs.append(subprocess.Popen(cmd, stdin=subprocess.PIPE,
|
|
240
|
+
stdout=subprocess.PIPE, text=True, encoding="utf-8", bufsize=1))
|
|
241
|
+
except OSError as exc:
|
|
242
|
+
log.warning("Couldn't start an index helper: %s", exc)
|
|
243
|
+
self.started = list(self.procs) # every helper ever started: all are reaped in close()
|
|
244
|
+
self.busy: list[tuple[object, str, list]] = [] # (process, root, items), oldest first
|
|
245
|
+
|
|
246
|
+
def __bool__(self) -> bool:
|
|
247
|
+
return bool(self.procs)
|
|
248
|
+
|
|
249
|
+
def _collect(self) -> list:
|
|
250
|
+
proc, root, items = self.busy.pop(0)
|
|
251
|
+
line = proc.stdout.readline()
|
|
252
|
+
if not line: # the helper died: read its files here instead
|
|
253
|
+
log.warning("An index helper stopped; carrying on without it")
|
|
254
|
+
self.procs.remove(proc)
|
|
255
|
+
return self.fallback(items)
|
|
256
|
+
try:
|
|
257
|
+
return [tuple(r) for r in json.loads(line)]
|
|
258
|
+
except ValueError:
|
|
259
|
+
return self.fallback(items)
|
|
260
|
+
|
|
261
|
+
def submit(self, root: str, items: list) -> list:
|
|
262
|
+
"""Hand ``items`` to a free helper; returns whatever finished meanwhile."""
|
|
263
|
+
done = []
|
|
264
|
+
busy_procs = {id(b[0]) for b in self.busy}
|
|
265
|
+
free = next((p for p in self.procs if id(p) not in busy_procs), None)
|
|
266
|
+
while free is None and self.busy:
|
|
267
|
+
proc = self.busy[0][0]
|
|
268
|
+
done.extend(self._collect())
|
|
269
|
+
free = proc if proc in self.procs else next((p for p in self.procs if id(p) not in {id(b[0]) for b in self.busy}), None)
|
|
270
|
+
if free is None: # every helper is gone
|
|
271
|
+
return done + self.fallback(items)
|
|
272
|
+
try:
|
|
273
|
+
free.stdin.write(json.dumps({"root": root, "items": items}, ensure_ascii=False) + "\n")
|
|
274
|
+
free.stdin.flush()
|
|
275
|
+
self.busy.append((free, root, items))
|
|
276
|
+
except OSError:
|
|
277
|
+
self.procs.remove(free)
|
|
278
|
+
done.extend(self.fallback(items))
|
|
279
|
+
return done
|
|
280
|
+
|
|
281
|
+
def finish(self) -> list:
|
|
282
|
+
done = []
|
|
283
|
+
while self.busy:
|
|
284
|
+
done.extend(self._collect())
|
|
285
|
+
return done
|
|
286
|
+
|
|
287
|
+
def close(self) -> None:
|
|
288
|
+
"""Stop every helper and close its pipes (including any that died along the way)."""
|
|
289
|
+
for p in self.started:
|
|
290
|
+
try:
|
|
291
|
+
p.stdin.close()
|
|
292
|
+
except Exception: # noqa: BLE001 - already closed or broken
|
|
293
|
+
pass
|
|
294
|
+
try:
|
|
295
|
+
p.wait(timeout=5)
|
|
296
|
+
except Exception: # noqa: BLE001
|
|
297
|
+
p.kill()
|
|
298
|
+
p.wait()
|
|
299
|
+
try:
|
|
300
|
+
p.stdout.close()
|
|
301
|
+
except Exception: # noqa: BLE001
|
|
302
|
+
pass
|
|
303
|
+
self.procs, self.started = [], []
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
class LibraryIndex:
|
|
307
|
+
"""Keeps the searchable copy of the library's tags up to date.
|
|
308
|
+
|
|
309
|
+
``describe(path, rel)`` returns the JSON-able track dict (the same shape the
|
|
310
|
+
browse API sends: tags, format, duration, art…), or raises ``OSError`` /
|
|
311
|
+
any exception for unreadable files, which are indexed with an ``error``.
|
|
312
|
+
"""
|
|
313
|
+
|
|
314
|
+
def __init__(self, root: Path, db_path: Path | None, describe: Describe, rescan_seconds: float = RESCAN_SECONDS,
|
|
315
|
+
parallel: Callable[[str, list[tuple[str, int, int]]], list] | None = None, workers: int = 0):
|
|
316
|
+
"""``parallel`` (a picklable function: (root, [(rel, mtime, size)]) -> [(rel, mtime, size, track)])
|
|
317
|
+
lets big scans use ``workers`` processes; 0/1 workers reads everything here."""
|
|
318
|
+
self.root = Path(os.path.abspath(root))
|
|
319
|
+
self.describe = describe
|
|
320
|
+
self.parallel = parallel
|
|
321
|
+
self.workers = workers
|
|
322
|
+
self.rescan_seconds = rescan_seconds
|
|
323
|
+
self._lock = threading.Lock() # guards the connection
|
|
324
|
+
self.version = 0 # goes up whenever a file is added, changed or removed (for caches)
|
|
325
|
+
self._dirty: set[str] = set()
|
|
326
|
+
self._dirty_lock = threading.Lock()
|
|
327
|
+
self._stop = threading.Event()
|
|
328
|
+
self._wake = threading.Event()
|
|
329
|
+
self._thread: threading.Thread | None = None
|
|
330
|
+
self._scanning = False
|
|
331
|
+
self._seen = 0
|
|
332
|
+
self._ready = False
|
|
333
|
+
self._last_scan = 0.0 # monotonic time the last full scan finished
|
|
334
|
+
self._scan_requested = True # scan once at startup
|
|
335
|
+
self.db = self._open(db_path)
|
|
336
|
+
|
|
337
|
+
# -- database ------------------------------------------------------------------------------
|
|
338
|
+
|
|
339
|
+
def _open(self, db_path: Path | None) -> sqlite3.Connection:
|
|
340
|
+
db = None
|
|
341
|
+
if db_path is not None:
|
|
342
|
+
try:
|
|
343
|
+
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
344
|
+
db = sqlite3.connect(db_path, check_same_thread=False)
|
|
345
|
+
db.execute("PRAGMA journal_mode=WAL")
|
|
346
|
+
version = db.execute("PRAGMA user_version").fetchone()[0]
|
|
347
|
+
if version not in (0, SCHEMA_VERSION): # older layout: rebuild from scratch
|
|
348
|
+
db.executescript("DROP TABLE IF EXISTS files; DROP TABLE IF EXISTS tags; DROP TABLE IF EXISTS keys; VACUUM;")
|
|
349
|
+
except (OSError, sqlite3.Error) as exc:
|
|
350
|
+
log.warning("Library index can't be saved in the config folder (%s); keeping it in memory.", exc)
|
|
351
|
+
db = None
|
|
352
|
+
if db is None:
|
|
353
|
+
db = sqlite3.connect(":memory:", check_same_thread=False)
|
|
354
|
+
db.executescript(
|
|
355
|
+
"""
|
|
356
|
+
CREATE TABLE IF NOT EXISTS files (
|
|
357
|
+
id INTEGER PRIMARY KEY, path TEXT NOT NULL UNIQUE, mtime INTEGER, size INTEGER,
|
|
358
|
+
text TEXT, track BLOB NOT NULL, nkey TEXT NOT NULL DEFAULT '');
|
|
359
|
+
CREATE TABLE IF NOT EXISTS keys (id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE);
|
|
360
|
+
-- One row per file and tag; tag names are stored once, in "keys".
|
|
361
|
+
CREATE TABLE IF NOT EXISTS tags (
|
|
362
|
+
key INTEGER NOT NULL, file INTEGER NOT NULL, value TEXT NOT NULL,
|
|
363
|
+
PRIMARY KEY (key, file)) WITHOUT ROWID;
|
|
364
|
+
CREATE INDEX IF NOT EXISTS tags_file ON tags(file);
|
|
365
|
+
"""
|
|
366
|
+
)
|
|
367
|
+
db.execute(f"PRAGMA user_version={SCHEMA_VERSION}")
|
|
368
|
+
db.create_function("wm", 2, word_match, deterministic=True)
|
|
369
|
+
db.commit()
|
|
370
|
+
self._key_ids = dict(db.execute("SELECT name, id FROM keys").fetchall())
|
|
371
|
+
return db
|
|
372
|
+
|
|
373
|
+
def _key_id(self, cur: sqlite3.Cursor, name: str) -> int:
|
|
374
|
+
if name not in self._key_ids:
|
|
375
|
+
cur.execute("INSERT OR IGNORE INTO keys(name) VALUES (?)", (name,))
|
|
376
|
+
self._key_ids[name] = cur.execute("SELECT id FROM keys WHERE name=?", (name,)).fetchone()[0]
|
|
377
|
+
return self._key_ids[name]
|
|
378
|
+
|
|
379
|
+
def _store(self, rows: list[tuple[str, int, int, dict]]) -> None:
|
|
380
|
+
with self._lock:
|
|
381
|
+
cur = self.db.cursor()
|
|
382
|
+
for rel, mtime, size, track in rows:
|
|
383
|
+
tags = track.get("tags") or {}
|
|
384
|
+
text = "\x1f".join(
|
|
385
|
+
[sfold(v) for k in FREE_TEXT_KEYS for v in tags.get(k, [])] + [sfold(Path(rel).stem)]
|
|
386
|
+
)
|
|
387
|
+
row = cur.execute("SELECT id FROM files WHERE path=?", (rel,)).fetchone()
|
|
388
|
+
if row:
|
|
389
|
+
file_id = row[0]
|
|
390
|
+
cur.execute("DELETE FROM tags WHERE file=?", (file_id,))
|
|
391
|
+
cur.execute("UPDATE files SET mtime=?, size=?, text=?, track=?, nkey=? WHERE id=?",
|
|
392
|
+
(mtime, size, text, _pack(track), natural_sort_text(rel), file_id))
|
|
393
|
+
else:
|
|
394
|
+
cur.execute("INSERT INTO files(path, mtime, size, text, track, nkey) VALUES (?,?,?,?,?,?)",
|
|
395
|
+
(rel, mtime, size, text, _pack(track), natural_sort_text(rel)))
|
|
396
|
+
file_id = cur.lastrowid
|
|
397
|
+
merged: dict[str, list[str]] = {} # "Genre" and "genre" are the same tag
|
|
398
|
+
for k, vals in tags.items():
|
|
399
|
+
kept = [sfold(v) for v in vals if _searchable(k.lower(), v)]
|
|
400
|
+
if kept:
|
|
401
|
+
merged.setdefault(k.lower(), []).extend(kept)
|
|
402
|
+
cur.executemany("INSERT INTO tags(key, file, value) VALUES (?,?,?)",
|
|
403
|
+
[(self._key_id(cur, k), file_id, "\x1f".join(v)) for k, v in merged.items()])
|
|
404
|
+
self.db.commit()
|
|
405
|
+
if rows:
|
|
406
|
+
self.version += 1
|
|
407
|
+
|
|
408
|
+
def _remove(self, rels: Iterable[str]) -> None:
|
|
409
|
+
rels = list(rels)
|
|
410
|
+
if not rels:
|
|
411
|
+
return
|
|
412
|
+
with self._lock:
|
|
413
|
+
for r in rels:
|
|
414
|
+
row = self.db.execute("SELECT id FROM files WHERE path=?", (r,)).fetchone()
|
|
415
|
+
if row:
|
|
416
|
+
self.db.execute("DELETE FROM tags WHERE file=?", (row[0],))
|
|
417
|
+
self.db.execute("DELETE FROM files WHERE id=?", (row[0],))
|
|
418
|
+
self.db.commit()
|
|
419
|
+
self.version += 1
|
|
420
|
+
|
|
421
|
+
def _read_item(self, item: tuple[str, int, int]) -> tuple[str, int, int, dict]:
|
|
422
|
+
rel, mtime, size = item
|
|
423
|
+
try:
|
|
424
|
+
track = self.describe(self.root / rel, rel)
|
|
425
|
+
except Exception as exc: # noqa: BLE001 - one bad file must not stop the scan
|
|
426
|
+
track = {"path": rel, "name": rel.rsplit("/", 1)[-1], "error": str(exc) or type(exc).__name__}
|
|
427
|
+
return rel, mtime, size, track
|
|
428
|
+
|
|
429
|
+
def _read(self, rel: str, st: os.stat_result) -> tuple[str, int, int, dict]:
|
|
430
|
+
try:
|
|
431
|
+
track = self.describe(self.root / rel, rel)
|
|
432
|
+
except Exception as exc: # noqa: BLE001 - one bad file must not stop the scan
|
|
433
|
+
track = {"path": rel, "name": rel.rsplit("/", 1)[-1], "error": str(exc) or type(exc).__name__}
|
|
434
|
+
return rel, st.st_mtime_ns, st.st_size, track
|
|
435
|
+
|
|
436
|
+
# -- scanning ----------------------------------------------------------------------------------
|
|
437
|
+
|
|
438
|
+
def _walk(self) -> Iterable[tuple[str, os.stat_result]]:
|
|
439
|
+
"""Every audio file under the root (following folder links, once each)."""
|
|
440
|
+
seen_dirs: set[tuple[int, int]] = set()
|
|
441
|
+
stack = [""]
|
|
442
|
+
while stack and not self._stop.is_set():
|
|
443
|
+
rel_dir = stack.pop()
|
|
444
|
+
folder = self.root / rel_dir if rel_dir else self.root
|
|
445
|
+
try:
|
|
446
|
+
st = folder.stat()
|
|
447
|
+
if (st.st_dev, st.st_ino) in seen_dirs:
|
|
448
|
+
continue # a link loop, or the same folder linked twice
|
|
449
|
+
seen_dirs.add((st.st_dev, st.st_ino))
|
|
450
|
+
with os.scandir(folder) as entries:
|
|
451
|
+
items = list(entries)
|
|
452
|
+
except OSError:
|
|
453
|
+
continue
|
|
454
|
+
for entry in items:
|
|
455
|
+
if entry.name.startswith(".") or not _is_valid_text(entry.name):
|
|
456
|
+
continue
|
|
457
|
+
rel = f"{rel_dir}/{entry.name}" if rel_dir else entry.name
|
|
458
|
+
try:
|
|
459
|
+
if entry.is_dir():
|
|
460
|
+
stack.append(rel)
|
|
461
|
+
elif entry.is_file() and Path(entry.name).suffix.lower() in SUPPORTED_EXTENSIONS:
|
|
462
|
+
yield rel, entry.stat()
|
|
463
|
+
except OSError:
|
|
464
|
+
continue
|
|
465
|
+
|
|
466
|
+
def scan(self) -> None:
|
|
467
|
+
"""Bring the index up to date with the files on disk."""
|
|
468
|
+
if not self.root.is_dir():
|
|
469
|
+
return
|
|
470
|
+
self._scanning, self._seen = True, 0
|
|
471
|
+
started = time.monotonic()
|
|
472
|
+
pool = None
|
|
473
|
+
try:
|
|
474
|
+
with self._lock:
|
|
475
|
+
known = {p: (m, s) for p, m, s in self.db.execute("SELECT path, mtime, size FROM files")}
|
|
476
|
+
present: set[str] = set()
|
|
477
|
+
batch: list[tuple[str, int, int, dict]] = []
|
|
478
|
+
changed = 0
|
|
479
|
+
pending: list[tuple[str, int, int]] = [] # waiting to be read (in a helper, once it's worth it)
|
|
480
|
+
|
|
481
|
+
def store(rows: list) -> None:
|
|
482
|
+
nonlocal changed, batch
|
|
483
|
+
batch.extend(rows)
|
|
484
|
+
if len(batch) >= BATCH:
|
|
485
|
+
self._store(batch)
|
|
486
|
+
changed += len(batch)
|
|
487
|
+
batch = []
|
|
488
|
+
|
|
489
|
+
def read_here(items: list) -> list:
|
|
490
|
+
return [self._read_item(i) for i in items]
|
|
491
|
+
|
|
492
|
+
for rel, st in self._walk():
|
|
493
|
+
present.add(rel)
|
|
494
|
+
self._seen += 1
|
|
495
|
+
if known.get(rel) == (st.st_mtime_ns, st.st_size):
|
|
496
|
+
continue
|
|
497
|
+
pending.append((rel, st.st_mtime_ns, st.st_size))
|
|
498
|
+
if pool is None and len(pending) >= PAR_START and self.parallel and self.workers > 1:
|
|
499
|
+
pool = _Workers(self.workers, read_here) or None
|
|
500
|
+
if pool is not None and len(pending) >= PAR_CHUNK:
|
|
501
|
+
store(pool.submit(str(self.root), pending)) # bounded: one job per helper at a time
|
|
502
|
+
pending = []
|
|
503
|
+
elif pool is None and len(pending) >= PAR_START:
|
|
504
|
+
store([self._read_item(i) for i in pending]) # no workers: read here, in order
|
|
505
|
+
pending = []
|
|
506
|
+
self._flush_dirty() # edits made during a long first scan show up quickly
|
|
507
|
+
if pool is not None:
|
|
508
|
+
if pending:
|
|
509
|
+
store(pool.submit(str(self.root), pending))
|
|
510
|
+
pending = []
|
|
511
|
+
store(pool.finish())
|
|
512
|
+
pool.close()
|
|
513
|
+
store([self._read_item(i) for i in pending])
|
|
514
|
+
if batch:
|
|
515
|
+
self._store(batch)
|
|
516
|
+
changed += len(batch)
|
|
517
|
+
if not self._stop.is_set():
|
|
518
|
+
gone = set(known) - present
|
|
519
|
+
self._remove(gone)
|
|
520
|
+
self._ready = True
|
|
521
|
+
self._last_scan = time.monotonic()
|
|
522
|
+
if changed or gone:
|
|
523
|
+
with self._lock: # fold the write-ahead log back in, so it doesn't linger on disk
|
|
524
|
+
self.db.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
|
525
|
+
if changed or gone:
|
|
526
|
+
log.info("Library index: %d files, %d updated, %d removed (%.1fs)",
|
|
527
|
+
len(present), changed, len(gone), time.monotonic() - started)
|
|
528
|
+
finally:
|
|
529
|
+
if pool is not None:
|
|
530
|
+
pool.close() # never leave helpers behind, even if the scan stopped half-way
|
|
531
|
+
self._scanning = False
|
|
532
|
+
|
|
533
|
+
def mark_dirty(self, rels: Iterable[str]) -> None:
|
|
534
|
+
"""Files Codetta just changed: re-read them before the next search."""
|
|
535
|
+
with self._dirty_lock:
|
|
536
|
+
self._dirty.update(rels)
|
|
537
|
+
self._wake.set()
|
|
538
|
+
|
|
539
|
+
def _flush_dirty(self) -> None:
|
|
540
|
+
with self._dirty_lock:
|
|
541
|
+
dirty, self._dirty = self._dirty, set()
|
|
542
|
+
if not dirty:
|
|
543
|
+
return
|
|
544
|
+
rows, gone = [], []
|
|
545
|
+
for rel in dirty:
|
|
546
|
+
try:
|
|
547
|
+
st = (self.root / rel).stat()
|
|
548
|
+
except OSError:
|
|
549
|
+
gone.append(rel)
|
|
550
|
+
continue
|
|
551
|
+
rows.append(self._read(rel, st))
|
|
552
|
+
if rows:
|
|
553
|
+
self._store(rows)
|
|
554
|
+
self._remove(gone)
|
|
555
|
+
|
|
556
|
+
# -- background thread -------------------------------------------------------------------------
|
|
557
|
+
|
|
558
|
+
def start(self) -> None:
|
|
559
|
+
if self._thread is None:
|
|
560
|
+
self._thread = threading.Thread(target=self._run, name="codetta-index", daemon=True)
|
|
561
|
+
self._thread.start()
|
|
562
|
+
|
|
563
|
+
def stop(self) -> None:
|
|
564
|
+
self._stop.set()
|
|
565
|
+
self._wake.set()
|
|
566
|
+
if self._thread is not None:
|
|
567
|
+
self._thread.join(timeout=5)
|
|
568
|
+
self._thread = None
|
|
569
|
+
|
|
570
|
+
def request_rescan(self) -> None:
|
|
571
|
+
"""Called on each search: re-check the library if the last scan is old."""
|
|
572
|
+
if not self._scanning and time.monotonic() - self._last_scan > self.rescan_seconds:
|
|
573
|
+
self._scan_requested = True
|
|
574
|
+
self._wake.set()
|
|
575
|
+
|
|
576
|
+
def _run(self) -> None:
|
|
577
|
+
# No timer: sleeps until something asks (startup, a search, an edit).
|
|
578
|
+
while not self._stop.is_set():
|
|
579
|
+
try:
|
|
580
|
+
if self._scan_requested:
|
|
581
|
+
self._scan_requested = False
|
|
582
|
+
self.scan()
|
|
583
|
+
self._flush_dirty()
|
|
584
|
+
except Exception: # noqa: BLE001 - keep the index alive; try again on the next request
|
|
585
|
+
log.exception("Library index scan failed")
|
|
586
|
+
self._last_scan = time.monotonic()
|
|
587
|
+
self._wake.wait()
|
|
588
|
+
self._wake.clear()
|
|
589
|
+
|
|
590
|
+
# -- searching ---------------------------------------------------------------------------------
|
|
591
|
+
|
|
592
|
+
def status(self) -> IndexStatus:
|
|
593
|
+
with self._lock:
|
|
594
|
+
files = self.db.execute("SELECT COUNT(*) FROM files").fetchone()[0]
|
|
595
|
+
return IndexStatus(ready=self._ready, scanning=self._scanning, files=files, seen=self._seen)
|
|
596
|
+
|
|
597
|
+
def all_tracks(self) -> list[tuple[str, dict]]:
|
|
598
|
+
"""Every indexed track (path, details), in path order. ~5,000 tracks take
|
|
599
|
+
about a tenth of a second: no file is opened."""
|
|
600
|
+
with self._lock:
|
|
601
|
+
rows = self.db.execute("SELECT path, track FROM files ORDER BY path").fetchall()
|
|
602
|
+
return [(p, _unpack(t)) for p, t in rows]
|
|
603
|
+
|
|
604
|
+
def stored(self, rels: list[str]) -> dict[str, tuple[int, int, dict]]:
|
|
605
|
+
"""What the index has for these files: path -> (mtime_ns, size, details)."""
|
|
606
|
+
out: dict[str, tuple[int, int, dict]] = {}
|
|
607
|
+
for i in range(0, len(rels), 900): # SQLite's limit on query parameters
|
|
608
|
+
chunk = rels[i:i + 900]
|
|
609
|
+
q = "SELECT path, mtime, size, track FROM files WHERE path IN (" + ",".join("?" * len(chunk)) + ")"
|
|
610
|
+
with self._lock:
|
|
611
|
+
rows = self.db.execute(q, chunk).fetchall()
|
|
612
|
+
for path, mtime, size, blob in rows:
|
|
613
|
+
out[path] = (mtime, size, _unpack(blob))
|
|
614
|
+
return out
|
|
615
|
+
|
|
616
|
+
def iter_tracks(self, chunk: int = 2000):
|
|
617
|
+
"""Every indexed track (path, details) in path order, read a chunk at a time, so even a
|
|
618
|
+
huge library is never all in memory at once (and indexing isn't blocked meanwhile)."""
|
|
619
|
+
last = ""
|
|
620
|
+
while True:
|
|
621
|
+
with self._lock:
|
|
622
|
+
rows = self.db.execute("SELECT path, track FROM files WHERE path > ? ORDER BY path LIMIT ?",
|
|
623
|
+
(last, chunk)).fetchall()
|
|
624
|
+
if not rows:
|
|
625
|
+
return
|
|
626
|
+
for path, blob in rows:
|
|
627
|
+
yield path, _unpack(blob)
|
|
628
|
+
last = rows[-1][0]
|
|
629
|
+
|
|
630
|
+
def paths_with_any(self, keys: Iterable[str]) -> set[str]:
|
|
631
|
+
"""Files that have at least one of these tags (by name, any case)."""
|
|
632
|
+
with self._lock:
|
|
633
|
+
ids = [self._key_ids[k] for k in {k.lower() for k in keys} if k in self._key_ids]
|
|
634
|
+
if not ids:
|
|
635
|
+
return set()
|
|
636
|
+
q = ("SELECT DISTINCT f.path FROM files f JOIN tags t ON t.file = f.id WHERE t.key IN ("
|
|
637
|
+
+ ",".join("?" * len(ids)) + ")")
|
|
638
|
+
return {p for (p,) in self.db.execute(q, ids)}
|
|
639
|
+
|
|
640
|
+
def has_key(self, key: str) -> bool:
|
|
641
|
+
with self._lock:
|
|
642
|
+
return key in self._key_ids
|
|
643
|
+
|
|
644
|
+
def search(self, query: Query, limit: int = 500) -> tuple[list[dict], int]:
|
|
645
|
+
"""Tracks matching ``query``, in library order (first ``limit``), and the total count."""
|
|
646
|
+
if query.empty:
|
|
647
|
+
return [], 0
|
|
648
|
+
self._flush_dirty()
|
|
649
|
+
sql = ["SELECT path FROM files WHERE 1"]
|
|
650
|
+
args: list[str] = []
|
|
651
|
+
# instr() (in C) rules most rows out before the Python word check runs on what's left.
|
|
652
|
+
for word in query.words:
|
|
653
|
+
sql.append("AND instr(text, ?) AND wm(text, ?)")
|
|
654
|
+
args += [sfold(word), sfold(word)]
|
|
655
|
+
for clause in query.clauses:
|
|
656
|
+
needles = [sfold(v) for v in clause.values]
|
|
657
|
+
if not needles:
|
|
658
|
+
continue
|
|
659
|
+
sql.append("AND id IN (SELECT file FROM tags WHERE key=(SELECT id FROM keys WHERE name=?) AND ("
|
|
660
|
+
+ " OR ".join(["(instr(value, ?) AND wm(value, ?))"] * len(needles)) + "))")
|
|
661
|
+
args += [clause.key, *[x for n in needles for x in (n, n)]]
|
|
662
|
+
sql.append("ORDER BY nkey") # library order ("Disc 2" before "Disc 10"), sorted by SQLite
|
|
663
|
+
with self._lock:
|
|
664
|
+
paths = [p for (p,) in self.db.execute(" ".join(sql), args)]
|
|
665
|
+
shown = paths[:limit]
|
|
666
|
+
with self._lock:
|
|
667
|
+
tracks = {}
|
|
668
|
+
for i in range(0, len(shown), 500):
|
|
669
|
+
chunk = shown[i:i + 500]
|
|
670
|
+
q = "SELECT path, track FROM files WHERE path IN (" + ",".join("?" * len(chunk)) + ")"
|
|
671
|
+
tracks.update({p: _unpack(t) for p, t in self.db.execute(q, chunk)})
|
|
672
|
+
return [tracks[p] for p in shown if p in tracks], len(paths)
|