python-musefs 1.0.0__py3-none-any.whl → 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- musefs_common/__init__.py +11 -1
- musefs_common/scan.py +11 -7
- musefs_common/schema.py +66 -1
- musefs_common/store.py +94 -1
- musefs_common/sync.py +28 -13
- python_musefs-1.1.0.dist-info/METADATA +27 -0
- python_musefs-1.1.0.dist-info/RECORD +14 -0
- python_musefs-1.0.0.dist-info/METADATA +0 -274
- python_musefs-1.0.0.dist-info/RECORD +0 -14
- {python_musefs-1.0.0.dist-info → python_musefs-1.1.0.dist-info}/WHEEL +0 -0
- {python_musefs-1.0.0.dist-info → python_musefs-1.1.0.dist-info}/licenses/LICENSE +0 -0
- {python_musefs-1.0.0.dist-info → python_musefs-1.1.0.dist-info}/top_level.txt +0 -0
musefs_common/__init__.py
CHANGED
|
@@ -11,19 +11,24 @@ from .errors import ScanError, SchemaMismatch
|
|
|
11
11
|
from .paths import realpath_key
|
|
12
12
|
from .scan import run_scan
|
|
13
13
|
from .store import (
|
|
14
|
+
TagRow,
|
|
14
15
|
check_schema_version,
|
|
15
16
|
connect,
|
|
17
|
+
delete_tracks,
|
|
16
18
|
merge_tags,
|
|
17
19
|
prune_missing,
|
|
18
20
|
replace_tags,
|
|
19
21
|
replace_track_art,
|
|
20
22
|
sniff_mime,
|
|
23
|
+
tags_for_track,
|
|
21
24
|
track_id_for_path,
|
|
25
|
+
track_ids_by_tag,
|
|
26
|
+
track_ids_for_paths,
|
|
22
27
|
upsert_art,
|
|
23
28
|
)
|
|
24
29
|
from .sync import ArtImage, Record, SyncStats, sync_files, sync_one
|
|
25
30
|
|
|
26
|
-
__version__ = "1.
|
|
31
|
+
__version__ = "1.1.0"
|
|
27
32
|
|
|
28
33
|
__all__ = [
|
|
29
34
|
"EXPECTED_USER_VERSION",
|
|
@@ -36,6 +41,11 @@ __all__ = [
|
|
|
36
41
|
"connect",
|
|
37
42
|
"check_schema_version",
|
|
38
43
|
"track_id_for_path",
|
|
44
|
+
"track_ids_for_paths",
|
|
45
|
+
"track_ids_by_tag",
|
|
46
|
+
"tags_for_track",
|
|
47
|
+
"TagRow",
|
|
48
|
+
"delete_tracks",
|
|
39
49
|
"prune_missing",
|
|
40
50
|
"merge_tags",
|
|
41
51
|
"replace_tags",
|
musefs_common/scan.py
CHANGED
|
@@ -4,19 +4,23 @@ import subprocess
|
|
|
4
4
|
from .errors import ScanError
|
|
5
5
|
|
|
6
6
|
|
|
7
|
-
def run_scan(binary, db_path, target, *, timeout=None):
|
|
8
|
-
"""Run ``<binary> scan <target...> --db <db_path
|
|
9
|
-
path or an iterable of paths; all targets precede the ``--db``
|
|
10
|
-
scanned under one process (one DB open). Creates the DB if
|
|
11
|
-
the structural columns a plugin can't compute.
|
|
12
|
-
``
|
|
13
|
-
|
|
7
|
+
def run_scan(binary, db_path, target, *, revalidate=False, timeout=None):
|
|
8
|
+
"""Run ``<binary> scan <target...> --db <db_path> [--revalidate]``. ``target``
|
|
9
|
+
is a single path or an iterable of paths; all targets precede the ``--db``
|
|
10
|
+
flag and are scanned under one process (one DB open). Creates the DB if
|
|
11
|
+
absent and fills the structural columns a plugin can't compute. With
|
|
12
|
+
``revalidate``, the scanner re-checks stamps, prunes rows whose backing file
|
|
13
|
+
is gone, and GCs orphaned art. Raises ``ScanError`` (with ``kind`` in
|
|
14
|
+
``"not_found" | "timeout" | "failed"``) on failure; the caller formats its
|
|
15
|
+
own user-facing message from the exception attributes."""
|
|
14
16
|
if isinstance(target, (str, os.PathLike)):
|
|
15
17
|
targets = [target]
|
|
16
18
|
else:
|
|
17
19
|
targets = list(target)
|
|
18
20
|
display = str(targets[0]) if len(targets) == 1 else f"{len(targets)} target(s)"
|
|
19
21
|
argv = [binary, "scan", *(str(t) for t in targets), "--db", str(db_path)]
|
|
22
|
+
if revalidate:
|
|
23
|
+
argv.append("--revalidate")
|
|
20
24
|
try:
|
|
21
25
|
result = subprocess.run(argv, capture_output=True, timeout=timeout)
|
|
22
26
|
except FileNotFoundError as exc:
|
musefs_common/schema.py
CHANGED
|
@@ -200,6 +200,71 @@ CREATE TRIGGER structural_blocks_ad AFTER DELETE ON structural_blocks BEGIN
|
|
|
200
200
|
UPDATE tracks SET content_version = content_version + 1 WHERE id = OLD.track_id;
|
|
201
201
|
END;
|
|
202
202
|
PRAGMA user_version = 1;
|
|
203
|
+
|
|
204
|
+
-- ── MIGRATION_V2 ──
|
|
205
|
+
-- fingerprint/content_hash are scanner-owned content identities. Neither is
|
|
206
|
+
-- UNIQUE and the index is NON-unique BY DESIGN: duplicate-content tracks (same
|
|
207
|
+
-- album in two places, genuine dupes) legitimately share both values, and a
|
|
208
|
+
-- UNIQUE constraint would abort the scan batch on the second copy. Correctness
|
|
209
|
+
-- comes from the refind logic (unique-missing candidate + confirmation), not
|
|
210
|
+
-- from DB uniqueness. Both columns carry a length(x) = 64 CHECK locking them
|
|
211
|
+
-- to SHA-256 hex (Task E2 benchmark locked the hash to SHA-256: under a
|
|
212
|
+
-- realistic SSD/HDD I/O profile the fingerprint adds ~8.6%; the RAM
|
|
213
|
+
-- microbench's higher ratio is an I/O-elimination artifact — see
|
|
214
|
+
-- the benchmarks docs). Hash function is now fixed, so the CHECK is added here.
|
|
215
|
+
ALTER TABLE tracks ADD COLUMN fingerprint TEXT
|
|
216
|
+
CHECK (fingerprint IS NULL OR length(fingerprint) = 64);
|
|
217
|
+
ALTER TABLE tracks ADD COLUMN content_hash TEXT
|
|
218
|
+
CHECK (content_hash IS NULL OR length(content_hash) = 64);
|
|
219
|
+
CREATE INDEX tracks_fingerprint_idx ON tracks(fingerprint);
|
|
220
|
+
|
|
221
|
+
-- Rebuild `tags` with a byte-accurate value cap (#505). SQLite's length() on
|
|
222
|
+
-- TEXT counts characters, so the V1 `CHECK (length(value) <= 262144)` was up to
|
|
223
|
+
-- ~4x looser than the documented 256 KiB byte bound; length(CAST(value AS BLOB))
|
|
224
|
+
-- counts bytes. SQLite cannot alter a CHECK in place, so recreate the table
|
|
225
|
+
-- (V2 is unreleased — this is folded in rather than added as a new migration).
|
|
226
|
+
-- Pre-existing over-cap rows (only reachable on an upgraded store) are dropped:
|
|
227
|
+
-- the read-time guard already counts bytes, so they were unreadable anyway, and
|
|
228
|
+
-- carrying them would abort the rebuild on the new CHECK.
|
|
229
|
+
CREATE TABLE tags_new (
|
|
230
|
+
track_id INTEGER NOT NULL REFERENCES tracks(id) ON DELETE CASCADE,
|
|
231
|
+
key TEXT NOT NULL,
|
|
232
|
+
value TEXT NOT NULL,
|
|
233
|
+
ordinal INTEGER NOT NULL DEFAULT 0,
|
|
234
|
+
value_blob BLOB,
|
|
235
|
+
PRIMARY KEY (track_id, key, ordinal),
|
|
236
|
+
CHECK (ordinal >= 0),
|
|
237
|
+
CHECK (value_blob IS NULL OR value = ''),
|
|
238
|
+
CHECK (length(key) <= 256),
|
|
239
|
+
CHECK (length(key) >= 1
|
|
240
|
+
AND key NOT GLOB '*[' || char(1) || '-' || char(31) || ']*'),
|
|
241
|
+
CHECK (length(CAST(value AS BLOB)) <= 262144),
|
|
242
|
+
CHECK (value_blob IS NULL OR length(value_blob) <= 16711680)
|
|
243
|
+
);
|
|
244
|
+
INSERT INTO tags_new (track_id, key, value, ordinal, value_blob)
|
|
245
|
+
SELECT track_id, key, value, ordinal, value_blob FROM tags
|
|
246
|
+
WHERE length(CAST(value AS BLOB)) <= 262144;
|
|
247
|
+
DROP TABLE tags;
|
|
248
|
+
ALTER TABLE tags_new RENAME TO tags;
|
|
249
|
+
|
|
250
|
+
-- DROP TABLE tags dropped its INSERT/UPDATE/DELETE triggers; recreate them
|
|
251
|
+
-- verbatim so the content_version/updated_at bump contract is unchanged.
|
|
252
|
+
CREATE TRIGGER tags_ai AFTER INSERT ON tags BEGIN
|
|
253
|
+
UPDATE tracks SET content_version = content_version + 1,
|
|
254
|
+
updated_at = CAST(strftime('%s','now') AS INTEGER)
|
|
255
|
+
WHERE id = NEW.track_id;
|
|
256
|
+
END;
|
|
257
|
+
CREATE TRIGGER tags_au AFTER UPDATE ON tags BEGIN
|
|
258
|
+
UPDATE tracks SET content_version = content_version + 1,
|
|
259
|
+
updated_at = CAST(strftime('%s','now') AS INTEGER)
|
|
260
|
+
WHERE id = NEW.track_id;
|
|
261
|
+
END;
|
|
262
|
+
CREATE TRIGGER tags_ad AFTER DELETE ON tags BEGIN
|
|
263
|
+
UPDATE tracks SET content_version = content_version + 1,
|
|
264
|
+
updated_at = CAST(strftime('%s','now') AS INTEGER)
|
|
265
|
+
WHERE id = OLD.track_id;
|
|
266
|
+
END;
|
|
267
|
+
PRAGMA user_version = 2;
|
|
203
268
|
"""
|
|
204
269
|
|
|
205
|
-
USER_VERSION =
|
|
270
|
+
USER_VERSION = 2
|
musefs_common/store.py
CHANGED
|
@@ -2,10 +2,27 @@ import contextlib
|
|
|
2
2
|
import hashlib
|
|
3
3
|
import os
|
|
4
4
|
import sqlite3
|
|
5
|
+
from dataclasses import dataclass
|
|
5
6
|
|
|
6
7
|
from .constants import EXPECTED_USER_VERSION
|
|
7
8
|
from .errors import SchemaMismatch
|
|
8
9
|
|
|
10
|
+
# SQLite caps a statement's host parameters (SQLITE_MAX_VARIABLE_NUMBER: 999 on
|
|
11
|
+
# the <3.32 floor). Chunk bulk IN-lists below it so large lookups never trip it.
|
|
12
|
+
_MAX_SQL_VARS = 900
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class TagRow:
|
|
17
|
+
"""One tag row read back from the store: the key, the text value, and the
|
|
18
|
+
raw ``value_blob``. Plugin-owned text tags have ``value_blob is None``;
|
|
19
|
+
scanner-written binary tags have ``value == ""`` and ``value_blob`` bytes."""
|
|
20
|
+
|
|
21
|
+
key: str
|
|
22
|
+
value: str
|
|
23
|
+
value_blob: object = None # bytes | None
|
|
24
|
+
|
|
25
|
+
|
|
9
26
|
# sqlite3.LEGACY_TRANSACTION_CONTROL is 3.12+; it is == -1. Use getattr so this
|
|
10
27
|
# module still imports on the 3.8 floor (where the constant does not exist).
|
|
11
28
|
_LEGACY = getattr(sqlite3, "LEGACY_TRANSACTION_CONTROL", -1)
|
|
@@ -88,6 +105,77 @@ def track_id_for_path(conn, key):
|
|
|
88
105
|
return row[0] if row else None
|
|
89
106
|
|
|
90
107
|
|
|
108
|
+
def track_ids_for_paths(conn, keys):
|
|
109
|
+
"""Resolve many ``backing_path`` keys to track ids in one pass, returning a
|
|
110
|
+
``{key: id}`` dict that omits keys with no matching track row. The IN-list is
|
|
111
|
+
chunked under SQLite's host-parameter cap so arbitrarily large lookups work
|
|
112
|
+
(the bulk counterpart to ``track_id_for_path``)."""
|
|
113
|
+
# Deduplicate while preserving first-seen order: a key repeated across chunk
|
|
114
|
+
# boundaries would re-fetch its row in a later chunk and trip the duplicate
|
|
115
|
+
# guard below even on a conformant DB.
|
|
116
|
+
keys = list(dict.fromkeys(keys))
|
|
117
|
+
out = {}
|
|
118
|
+
for start in range(0, len(keys), _MAX_SQL_VARS):
|
|
119
|
+
chunk = keys[start : start + _MAX_SQL_VARS]
|
|
120
|
+
placeholders = ",".join("?" for _ in chunk)
|
|
121
|
+
rows = conn.execute(
|
|
122
|
+
f"SELECT backing_path, id FROM tracks WHERE backing_path IN ({placeholders})",
|
|
123
|
+
chunk,
|
|
124
|
+
)
|
|
125
|
+
for backing_path, track_id in rows:
|
|
126
|
+
if backing_path in out:
|
|
127
|
+
# backing_path is UNIQUE in the schema, so a duplicate means a
|
|
128
|
+
# non-conformant DB; collapsing it would silently hide a track
|
|
129
|
+
# from prune (#478). Fail loudly instead.
|
|
130
|
+
raise ValueError(
|
|
131
|
+
f"duplicate backing_path {backing_path!r} in tracks "
|
|
132
|
+
f"(ids {out[backing_path]} and {track_id})"
|
|
133
|
+
)
|
|
134
|
+
out[backing_path] = track_id
|
|
135
|
+
return out
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def track_ids_by_tag(conn, key, value):
|
|
139
|
+
"""Return a list of track ids whose plugin-owned text tag ``(key, value)``
|
|
140
|
+
matches (order unspecified, possibly empty).
|
|
141
|
+
|
|
142
|
+
Scoped to text rows (``value_blob IS NULL``); scanner-written binary tags
|
|
143
|
+
never match. The intent-based counterpart to ``prune_missing``'s
|
|
144
|
+
existence-based scoping: used to map a source's "I deleted this album/artist"
|
|
145
|
+
signal back to the rows it tagged.
|
|
146
|
+
"""
|
|
147
|
+
rows = conn.execute(
|
|
148
|
+
"SELECT track_id FROM tags WHERE key = ? AND value = ? AND value_blob IS NULL",
|
|
149
|
+
(key, value),
|
|
150
|
+
)
|
|
151
|
+
return [track_id for (track_id,) in rows]
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def delete_tracks(conn, track_ids):
|
|
155
|
+
"""Unconditionally delete the given track rows; return the count actually
|
|
156
|
+
deleted (an already-gone id contributes 0).
|
|
157
|
+
|
|
158
|
+
The intent-based delete: unlike ``prune_missing`` it does not check on-disk
|
|
159
|
+
existence. ``tags`` and ``track_art`` rows cascade away via the schema's
|
|
160
|
+
``ON DELETE CASCADE`` (``connect`` enables ``foreign_keys = ON``).
|
|
161
|
+
"""
|
|
162
|
+
deleted = 0
|
|
163
|
+
for track_id in track_ids:
|
|
164
|
+
deleted += conn.execute("DELETE FROM tracks WHERE id = ?", (track_id,)).rowcount
|
|
165
|
+
return deleted
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def tags_for_track(conn, track_id):
|
|
169
|
+
"""Read back a track's tag rows as an ordered ``list[TagRow]`` (by key, then
|
|
170
|
+
ordinal). Includes both plugin-owned text tags (``value_blob is None``) and
|
|
171
|
+
scanner-written binary tags (``value == ""``, ``value_blob`` bytes)."""
|
|
172
|
+
rows = conn.execute(
|
|
173
|
+
"SELECT key, value, value_blob FROM tags WHERE track_id = ? ORDER BY key, ordinal",
|
|
174
|
+
(track_id,),
|
|
175
|
+
)
|
|
176
|
+
return [TagRow(key, value, value_blob) for key, value, value_blob in rows]
|
|
177
|
+
|
|
178
|
+
|
|
91
179
|
def prune_missing(conn, track_ids=None):
|
|
92
180
|
"""Delete track rows whose backing file no longer exists on disk.
|
|
93
181
|
|
|
@@ -150,9 +238,14 @@ def merge_tags(conn, track_id, managed_pairs, delete_keys):
|
|
|
150
238
|
for key, value in managed_pairs:
|
|
151
239
|
by_key.setdefault(key, []).append(value)
|
|
152
240
|
|
|
241
|
+
# Case-fold the key match: a scan seeds an unmapped tag in the file's
|
|
242
|
+
# native case (e.g. Vorbis ``LABEL``) while the plugin canonicalises to
|
|
243
|
+
# lowercase (``label``). Vorbis keys render case-insensitively, so an
|
|
244
|
+
# exact-case delete would leave the scan row and render a duplicate (#407).
|
|
153
245
|
for key in set(by_key) | set(delete_keys or ()):
|
|
154
246
|
conn.execute(
|
|
155
|
-
"DELETE FROM tags WHERE track_id = ? AND key = ?
|
|
247
|
+
"DELETE FROM tags WHERE track_id = ? AND lower(key) = lower(?) "
|
|
248
|
+
"AND value_blob IS NULL",
|
|
156
249
|
(track_id, key),
|
|
157
250
|
)
|
|
158
251
|
|
musefs_common/sync.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import sqlite3
|
|
3
4
|
from dataclasses import dataclass, field
|
|
4
5
|
|
|
5
6
|
from .constants import MAX_ART_BYTES
|
|
@@ -42,11 +43,14 @@ class SyncStats:
|
|
|
42
43
|
skipped: int = 0 # path had no matching track row
|
|
43
44
|
art_linked: int = 0
|
|
44
45
|
skipped_art: int = 0 # art over the size cap (or, in the beets adapter, unreadable)
|
|
46
|
+
skipped_invalid: int = 0 # record violated a store CHECK constraint
|
|
47
|
+
invalid: list = field(default_factory=list) # (record key, sqlite error message)
|
|
45
48
|
|
|
46
49
|
def summary(self):
|
|
47
50
|
return (
|
|
48
51
|
f"synced={self.synced} skipped={self.skipped} "
|
|
49
|
-
f"art_linked={self.art_linked} skipped_art={self.skipped_art}"
|
|
52
|
+
f"art_linked={self.art_linked} skipped_art={self.skipped_art} "
|
|
53
|
+
f"skipped_invalid={self.skipped_invalid}"
|
|
50
54
|
)
|
|
51
55
|
|
|
52
56
|
|
|
@@ -58,7 +62,13 @@ def sync_one(conn, record, stats, *, dry_run=False, merge=False):
|
|
|
58
62
|
scanner-written binary tags survive. Art is replaced when at least one image is
|
|
59
63
|
within ``MAX_ART_BYTES``; each over-cap image bumps ``skipped_art``, and if
|
|
60
64
|
every provided image is over cap any scan-seeded ``track_art`` is left
|
|
61
|
-
untouched.
|
|
65
|
+
untouched.
|
|
66
|
+
|
|
67
|
+
A record whose tags or art violate a store CHECK constraint (key/value/mime
|
|
68
|
+
length, ``picture_type`` range, control chars, ...) is rolled back through its
|
|
69
|
+
own savepoint and skipped -- it bumps ``skipped_invalid`` and appends
|
|
70
|
+
``(record.key, message)`` to ``invalid`` rather than aborting the whole batch
|
|
71
|
+
with an opaque commit-time ``IntegrityError`` (#420)."""
|
|
62
72
|
track_id = track_id_for_path(conn, record.key)
|
|
63
73
|
if track_id is None:
|
|
64
74
|
stats.skipped += 1
|
|
@@ -73,17 +83,22 @@ def sync_one(conn, record, stats, *, dry_run=False, merge=False):
|
|
|
73
83
|
will_link_art = bool(kept)
|
|
74
84
|
|
|
75
85
|
if not dry_run:
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
86
|
+
try:
|
|
87
|
+
with _savepoint(conn, "musefs_sync_one"):
|
|
88
|
+
if merge:
|
|
89
|
+
merge_tags(conn, track_id, record.pairs, record.delete_keys or [])
|
|
90
|
+
else:
|
|
91
|
+
replace_tags(conn, track_id, record.pairs)
|
|
92
|
+
if will_link_art:
|
|
93
|
+
arts = [
|
|
94
|
+
(upsert_art(conn, img.data, img.mime), img.picture_type, img.description)
|
|
95
|
+
for img in kept
|
|
96
|
+
]
|
|
97
|
+
replace_track_art(conn, track_id, arts)
|
|
98
|
+
except sqlite3.IntegrityError as err:
|
|
99
|
+
stats.skipped_invalid += 1
|
|
100
|
+
stats.invalid.append((record.key, str(err)))
|
|
101
|
+
return
|
|
87
102
|
|
|
88
103
|
if will_link_art:
|
|
89
104
|
stats.art_linked += 1
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: python-musefs
|
|
3
|
+
Version: 1.1.0
|
|
4
|
+
Summary: Shared musefs SQLite-store contract for the beets and Picard plugins
|
|
5
|
+
Author: Conor Futro
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Sohex/musefs
|
|
8
|
+
Project-URL: Repository, https://github.com/Sohex/musefs
|
|
9
|
+
Project-URL: Issues, https://github.com/Sohex/musefs/issues
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Operating System :: POSIX
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Multimedia :: Sound/Audio
|
|
15
|
+
Requires-Python: >=3.8
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Provides-Extra: test
|
|
19
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
20
|
+
Dynamic: license-file
|
|
21
|
+
|
|
22
|
+
# python-musefs
|
|
23
|
+
|
|
24
|
+
The full guide now lives in the musefs documentation site:
|
|
25
|
+
|
|
26
|
+
- **Published:** <https://sohex.github.io/musefs/integrations/python-musefs.html>
|
|
27
|
+
- **In-repo source:** [`docs/src/integrations/python-musefs.md`](../../docs/src/integrations/python-musefs.md)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
musefs_common/__init__.py,sha256=fA6mE93fSYBjObMMeCjDKnOnmETrbc2xt1TrdMKJ5cE,1505
|
|
2
|
+
musefs_common/constants.py,sha256=M0zS6YCnYdE_JjIbvxg-GhbYdfbe30ccKZvbZ1WTZZ8,302
|
|
3
|
+
musefs_common/contract.py,sha256=tUflCxfFiaIzJiTqYL-o-bjKNH_s91EHkC2amk8sKxE,1472
|
|
4
|
+
musefs_common/errors.py,sha256=OUBXzpa4q7TPYvFFiTyoIfRC3SGRRgZcpXlr43RP0jk,1456
|
|
5
|
+
musefs_common/paths.py,sha256=dE3wwop5mRxoJ8RhdYwTIZpX4MPaUUw2LJhjcxApOIA,649
|
|
6
|
+
musefs_common/scan.py,sha256=C20WmDWhGJ3FqjNOssLsoxtx0KgsTg9PPlrGPhOkE2E,1662
|
|
7
|
+
musefs_common/schema.py,sha256=HFo2qG0xtTgBVgxm8hCjVQAjxTeCqaPvYklw08rhOoU,11853
|
|
8
|
+
musefs_common/store.py,sha256=pUNlad1FITJCKwBNy4EH98jQQTmb9Ao5ggLlgo99XwU,12769
|
|
9
|
+
musefs_common/sync.py,sha256=iXlekqpuRfBEoOrjvujnZQYfDgs3afENaPU0n3D59nQ,4200
|
|
10
|
+
python_musefs-1.1.0.dist-info/licenses/LICENSE,sha256=4VbfzhgMpuFrhFjBCtLptGV_bzCj_gML9y_9o2Tr1OQ,1068
|
|
11
|
+
python_musefs-1.1.0.dist-info/METADATA,sha256=R3v-Qu_1-kc0JKc_xPlgIcZPhNuzcjIo6rvuZ8gAjT8,1012
|
|
12
|
+
python_musefs-1.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
13
|
+
python_musefs-1.1.0.dist-info/top_level.txt,sha256=ejWexGk95-s11kZnTFwg1T3iG_dFcaE4vhcLnL3t3Ak,14
|
|
14
|
+
python_musefs-1.1.0.dist-info/RECORD,,
|
|
@@ -1,274 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: python-musefs
|
|
3
|
-
Version: 1.0.0
|
|
4
|
-
Summary: Shared musefs SQLite-store contract for the beets and Picard plugins
|
|
5
|
-
Author: Conor Futro
|
|
6
|
-
License-Expression: MIT
|
|
7
|
-
Project-URL: Homepage, https://github.com/Sohex/musefs
|
|
8
|
-
Project-URL: Repository, https://github.com/Sohex/musefs
|
|
9
|
-
Project-URL: Issues, https://github.com/Sohex/musefs/issues
|
|
10
|
-
Classifier: Development Status :: 4 - Beta
|
|
11
|
-
Classifier: Intended Audience :: Developers
|
|
12
|
-
Classifier: Operating System :: POSIX
|
|
13
|
-
Classifier: Programming Language :: Python :: 3
|
|
14
|
-
Classifier: Topic :: Multimedia :: Sound/Audio
|
|
15
|
-
Requires-Python: >=3.8
|
|
16
|
-
Description-Content-Type: text/markdown
|
|
17
|
-
License-File: LICENSE
|
|
18
|
-
Provides-Extra: test
|
|
19
|
-
Requires-Dist: pytest>=7; extra == "test"
|
|
20
|
-
Dynamic: license-file
|
|
21
|
-
|
|
22
|
-
# python-musefs
|
|
23
|
-
|
|
24
|
-
The shared store-contract library behind the [beets](../beets/README.md),
|
|
25
|
-
[Picard](../picard/README.md), and [Lidarr](../lidarr/README.md) musefs
|
|
26
|
-
plugins. It is the single source of truth for how a plugin writes the musefs
|
|
27
|
-
SQLite store: the schema-version check, the `tags` / `art` / `track_art`
|
|
28
|
-
writes, sha256 art content-addressing, the `realpath_key` path normalization,
|
|
29
|
-
the `musefs scan` shell-out (`run_scan`), and the per-file sync write-loop
|
|
30
|
-
(`Record` / `sync_files`).
|
|
31
|
-
|
|
32
|
-
Field mapping stays in each plugin — beets expands multi-valued
|
|
33
|
-
`genres`/`composers` into one tag each, Picard takes the first value — so this
|
|
34
|
-
library deliberately does not own it.
|
|
35
|
-
|
|
36
|
-
## Writing a plugin
|
|
37
|
-
|
|
38
|
-
A plugin turns host metadata (a beets item, a Picard track, a Lidarr release)
|
|
39
|
-
into musefs store writes. This library owns every store-touching step except the
|
|
40
|
-
field mapping: you supply the per-file tag and art values, and it handles the
|
|
41
|
-
schema check, the scan shell-out, content-addressing, and the write loop.
|
|
42
|
-
|
|
43
|
-
### The write flow
|
|
44
|
-
|
|
45
|
-
The canonical order is **connect → check_schema_version → run_scan → build
|
|
46
|
-
`Record`s → sync_files → commit → prune_missing**. The caller owns the
|
|
47
|
-
transaction — nothing here commits for you.
|
|
48
|
-
|
|
49
|
-
```python
|
|
50
|
-
from musefs_common import (
|
|
51
|
-
SCAN_TIMEOUT_SECONDS,
|
|
52
|
-
ArtImage,
|
|
53
|
-
Record,
|
|
54
|
-
check_schema_version,
|
|
55
|
-
connect,
|
|
56
|
-
prune_missing,
|
|
57
|
-
realpath_key,
|
|
58
|
-
run_scan,
|
|
59
|
-
sync_files,
|
|
60
|
-
)
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
def sync(db_path, files, *, musefs_bin="musefs"):
|
|
64
|
-
# `run_scan` creates the DB if absent and fills the structural columns a
|
|
65
|
-
# plugin cannot compute (format, audio offset/length, backing size/mtime).
|
|
66
|
-
# On a brand-new store it must precede `connect`, which has nothing to open
|
|
67
|
-
# until the scan has created the file.
|
|
68
|
-
run_scan(musefs_bin, db_path, files, timeout=SCAN_TIMEOUT_SECONDS)
|
|
69
|
-
|
|
70
|
-
conn = connect(db_path)
|
|
71
|
-
try:
|
|
72
|
-
check_schema_version(conn) # raises SchemaMismatch on a version skew
|
|
73
|
-
|
|
74
|
-
records = [
|
|
75
|
-
Record(
|
|
76
|
-
key=realpath_key(path), # MUST equal the scanned row's backing_path
|
|
77
|
-
pairs=[("artist", artist), ("title", title)],
|
|
78
|
-
art=[ArtImage(data=cover, mime="image/jpeg")] if cover else None,
|
|
79
|
-
)
|
|
80
|
-
for path, artist, title, cover in host_metadata(files)
|
|
81
|
-
]
|
|
82
|
-
|
|
83
|
-
stats = sync_files(conn, records) # full-replace of plugin text tags
|
|
84
|
-
conn.commit() # the caller commits
|
|
85
|
-
|
|
86
|
-
prune_missing(conn) # drop rows whose backing file vanished
|
|
87
|
-
conn.commit()
|
|
88
|
-
return stats
|
|
89
|
-
finally:
|
|
90
|
-
conn.close()
|
|
91
|
-
```
|
|
92
|
-
|
|
93
|
-
For a dry run, pass `dry_run=True` to `sync_files` and `conn.rollback()` instead
|
|
94
|
-
of committing — `SyncStats` still reports what *would* change.
|
|
95
|
-
|
|
96
|
-
`run_scan` raises `ScanError` (`kind` ∈ `{"not_found", "timeout", "failed"}`)
|
|
97
|
-
and `check_schema_version` raises `SchemaMismatch`; a host adapter formats its
|
|
98
|
-
own user-facing message from the exception attributes (see the beets plugin's
|
|
99
|
-
`_scan_user_error`).
|
|
100
|
-
|
|
101
|
-
### The `Record` shape
|
|
102
|
-
|
|
103
|
-
One `Record` per file is your primary output. Its fields:
|
|
104
|
-
|
|
105
|
-
| field | type | meaning |
|
|
106
|
-
| ----- | ---- | ------- |
|
|
107
|
-
| `key` | `str` | The file's identity in the store. **Must** be `realpath_key(path)` — the canonicalized absolute path the scanner stored as `backing_path`. A `key` that matches no scanned row is silently counted in `SyncStats.skipped`, not written. |
|
|
108
|
-
| `pairs` | `list[tuple[str, str]]` | Ordered `(tag_key, value)` text tags. Duplicate keys are allowed and get contiguous ordinals (multi-valued tags). |
|
|
109
|
-
| `art` | `list[ArtImage] \| None` | Embedded pictures, already resolved to bytes. `None`/`[]` leaves existing art untouched. |
|
|
110
|
-
| `delete_keys` | `list[str] \| None` | Merge mode only: keys to clear without rewriting (see below). Ignored in replace mode. |
|
|
111
|
-
|
|
112
|
-
`ArtImage(data, mime, picture_type=3, description="")` is one picture: `data` is
|
|
113
|
-
raw bytes, `picture_type` is the ID3/FLAC type (3 = front cover). Images larger
|
|
114
|
-
than `MAX_ART_BYTES` are dropped and counted in `SyncStats.skipped_art`.
|
|
115
|
-
|
|
116
|
-
If every record lands in `skipped`, the `key`s and the scan target disagree —
|
|
117
|
-
both must canonicalize the same way, so scan the *real* files (not a symlink
|
|
118
|
-
farm) and build keys with `realpath_key`.
|
|
119
|
-
|
|
120
|
-
### Merge vs. replace, and sticky deletes
|
|
121
|
-
|
|
122
|
-
`sync_files(..., merge=False)` (the default) **replaces** every plugin-owned
|
|
123
|
-
text tag on each track: it clears all `value_blob IS NULL` rows and rewrites
|
|
124
|
-
them from `record.pairs`. Scanner-written binary tags always survive.
|
|
125
|
-
|
|
126
|
-
`sync_files(..., merge=True)` **merges**: only the keys named in `record.pairs`
|
|
127
|
-
and `record.delete_keys` are touched; other scan-seeded text tags stay. Use
|
|
128
|
-
merge when your plugin owns a *subset* of the tags and must not clobber the
|
|
129
|
-
rest. The store does not remember which keys you manage — **you** track your
|
|
130
|
-
managed-key set out of band (the contract is explicit that the store is not the
|
|
131
|
-
place for plugin state).
|
|
132
|
-
|
|
133
|
-
When the user removes a tag in the host, merge mode needs to delete the
|
|
134
|
-
now-orphaned store row. The beets plugin solves this with an **accumulating
|
|
135
|
-
managed-key set** (the `musefs_managed` pattern), worth copying:
|
|
136
|
-
|
|
137
|
-
- Persist, per file, the set of keys you have *ever* written (beets uses a
|
|
138
|
-
flexattr; any per-file host metadata works).
|
|
139
|
-
- On each sync, `delete_keys = previous_managed − keys_written_now`, and the new
|
|
140
|
-
persisted set is `previous_managed ∪ keys_written_now`.
|
|
141
|
-
- A key you stop writing becomes a tombstone: it keeps getting deleted on every
|
|
142
|
-
sync until you write it again. Persist the managed set **only after** the store
|
|
143
|
-
commit succeeds, so a failed sync doesn't lose the record of what you owe.
|
|
144
|
-
|
|
145
|
-
See `contrib/beets/beetsplug/_core.py` (`build_records` / `persist_managed`) for
|
|
146
|
-
the reference implementation.
|
|
147
|
-
|
|
148
|
-
### Store invariants you must respect
|
|
149
|
-
|
|
150
|
-
The full external-writer contract is in
|
|
151
|
-
[ARCHITECTURE.md](../../ARCHITECTURE.md#the-external-writer-contract). The rules
|
|
152
|
-
that bite plugin authors:
|
|
153
|
-
|
|
154
|
-
- **Write only `tags`, `art`, and `track_art`.** The scanner owns the structural
|
|
155
|
-
columns of `tracks` and all of `structural_blocks`; never compute them — run
|
|
156
|
-
`musefs scan` (i.e. `run_scan`). `CHECK` constraints reject malformed
|
|
157
|
-
structural shapes at commit, so you cannot persist them anyway.
|
|
158
|
-
- **Binary tags survive a sync.** `merge_tags` / `replace_tags` scope their
|
|
159
|
-
deletes to text rows (`value_blob IS NULL`), so the write loop never wipes
|
|
160
|
-
scanner-written binary tags. You may write binary tags yourself too — a binary
|
|
161
|
-
row carries its payload in `value_blob` and must leave `value` empty (the only
|
|
162
|
-
`CHECK` on the row).
|
|
163
|
-
- **Content-address art** through `upsert_art` (sha256 de-dup) rather than
|
|
164
|
-
inserting `art` rows by hand; `sync_files` does this for you.
|
|
165
|
-
- **Art rows are immutable.** A trigger rejects in-place updates of an
|
|
166
|
-
`art` row's content columns (`data`, `sha256`, `mime`, `byte_len`, `width`,
|
|
167
|
-
`height`). To change a track's art, insert a new content-addressed row via
|
|
168
|
-
`upsert_art` and relink it via `replace_track_art`.
|
|
169
|
-
- **Path layout is just a tag.** To drive a reorganized mount, write your
|
|
170
|
-
computed relative path into a custom tag (e.g. `beets_path`) and mount with
|
|
171
|
-
`--template '$!{beets_path}'`. musefs sanitizes each path segment, so a writer
|
|
172
|
-
cannot inject traversal.
|
|
173
|
-
|
|
174
|
-
## API reference
|
|
175
|
-
|
|
176
|
-
Everything in `__all__`, imported from the top-level `musefs_common` package.
|
|
177
|
-
|
|
178
|
-
**Connection & schema**
|
|
179
|
-
|
|
180
|
-
- `connect(db_path)` → `sqlite3.Connection` — open with a 5s busy timeout and
|
|
181
|
-
`foreign_keys = ON`.
|
|
182
|
-
- `check_schema_version(conn)` — raise `SchemaMismatch` unless the store's
|
|
183
|
-
`user_version` equals `EXPECTED_USER_VERSION`.
|
|
184
|
-
|
|
185
|
-
**Scanning**
|
|
186
|
-
|
|
187
|
-
- `run_scan(binary, db_path, target, *, timeout=None)` — shell out to `musefs
|
|
188
|
-
scan`; `target` is one path or an iterable, all scanned under one process.
|
|
189
|
-
Creates the DB if absent. Raises `ScanError`.
|
|
190
|
-
|
|
191
|
-
**Building records**
|
|
192
|
-
|
|
193
|
-
- `Record(key, pairs=[], art=None, delete_keys=None)` — one file's sync inputs
|
|
194
|
-
(see *The `Record` shape*).
|
|
195
|
-
- `ArtImage(data, mime, picture_type=3, description="")` — one embedded picture.
|
|
196
|
-
- `realpath_key(path)` — canonical path string matching the scanner's
|
|
197
|
-
`backing_path`; accepts `str`/`bytes`, returns `str`.
|
|
198
|
-
|
|
199
|
-
**Writing**
|
|
200
|
-
|
|
201
|
-
- `sync_files(conn, records, *, dry_run=False, stats=None, merge=False)` →
|
|
202
|
-
`SyncStats` — the write loop; caller owns the transaction. Pass `stats` to
|
|
203
|
-
accumulate into a caller-seeded instance.
|
|
204
|
-
- `sync_one(conn, record, stats, *, dry_run=False, merge=False)` — sync a single
|
|
205
|
-
record into a caller-supplied `SyncStats`.
|
|
206
|
-
- `SyncStats` — `synced` / `skipped` / `art_linked` / `skipped_art` counters,
|
|
207
|
-
plus `.summary()`.
|
|
208
|
-
|
|
209
|
-
**Lower-level store helpers** (called for you by `sync_files`; use directly only
|
|
210
|
-
for a custom write loop)
|
|
211
|
-
|
|
212
|
-
- `track_id_for_path(conn, key)` → track id or `None`.
|
|
213
|
-
- `merge_tags(conn, track_id, managed_pairs, delete_keys)` — per-key replace of
|
|
214
|
-
plugin-managed text tags, leaving unmanaged text rows intact.
|
|
215
|
-
- `replace_tags(conn, track_id, pairs)` — replace all plugin-owned text tags.
|
|
216
|
-
- `upsert_art(conn, data, mime)` → art id — content-address `data` by sha256,
|
|
217
|
-
inserting only if new.
|
|
218
|
-
- `replace_track_art(conn, track_id, arts)` — replace a track's `track_art`
|
|
219
|
-
rows; `arts` is `[(art_id, picture_type, description), …]`.
|
|
220
|
-
- `sniff_mime(data, path)` — image mime from magic bytes, falling back to file
|
|
221
|
-
extension.
|
|
222
|
-
- `prune_missing(conn, track_ids=None)` → count — delete tracks whose backing
|
|
223
|
-
file no longer exists (every track, or just `track_ids`).
|
|
224
|
-
|
|
225
|
-
**Constants**
|
|
226
|
-
|
|
227
|
-
- `EXPECTED_USER_VERSION` — schema `user_version` this library targets.
|
|
228
|
-
- `MAX_ART_BYTES` — per-image art cap; larger images are skipped.
|
|
229
|
-
- `SCAN_TIMEOUT_SECONDS` — default wall-clock cap for one `run_scan`.
|
|
230
|
-
|
|
231
|
-
**Exceptions**
|
|
232
|
-
|
|
233
|
-
- `SchemaMismatch(found)` — schema-version skew; `.found` is the DB's version.
|
|
234
|
-
- `ScanError(kind, *, binary, target, …)` — a `musefs scan` failure; `.kind` ∈
|
|
235
|
-
`{"not_found", "timeout", "failed"}`, with context attributes for messaging.
|
|
236
|
-
|
|
237
|
-
## Consumers
|
|
238
|
-
|
|
239
|
-
- **beets** depends on this package via pip (`contrib/beets/pyproject.toml`).
|
|
240
|
-
- **Picard** cannot pip-install plugin dependencies, so the package is
|
|
241
|
-
**vendored** into `contrib/picard/musefs/_common/` by
|
|
242
|
-
`vendor_to_picard.py`. After any change here, re-run:
|
|
243
|
-
|
|
244
|
-
```bash
|
|
245
|
-
python contrib/python-musefs/vendor_to_picard.py
|
|
246
|
-
```
|
|
247
|
-
|
|
248
|
-
The Picard test `tests/test_vendor_sync.py` fails if the committed copy drifts.
|
|
249
|
-
- **Lidarr** depends on this package via pip (`contrib/lidarr/pyproject.toml`).
|
|
250
|
-
|
|
251
|
-
## Schema coupling
|
|
252
|
-
|
|
253
|
-
`musefs_common/schema.py` (`SCHEMA_SQL`, `USER_VERSION`) is **generated** from
|
|
254
|
-
the Rust migrations in `musefs-db/src/schema.rs` — do not edit it by hand.
|
|
255
|
-
`EXPECTED_USER_VERSION` (in `constants.py`) derives from it. When the Rust
|
|
256
|
-
schema bumps, regenerate and re-vendor:
|
|
257
|
-
|
|
258
|
-
```bash
|
|
259
|
-
MUSEFS_REGEN_SCHEMA_PY=1 cargo test -p musefs-db schema_py
|
|
260
|
-
python contrib/python-musefs/vendor_to_picard.py
|
|
261
|
-
```
|
|
262
|
-
|
|
263
|
-
A `musefs-db` unit test fails if the generated file drifts. This is all
|
|
264
|
-
independent of the package's own `__version__` (its release SemVer).
|
|
265
|
-
|
|
266
|
-
## Tests
|
|
267
|
-
|
|
268
|
-
```bash
|
|
269
|
-
cd contrib/python-musefs
|
|
270
|
-
python -m venv .venv && source .venv/bin/activate
|
|
271
|
-
pip install -e ".[test]"
|
|
272
|
-
python -m pytest -v
|
|
273
|
-
ruff check . && ruff format --check .
|
|
274
|
-
```
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
musefs_common/__init__.py,sha256=e8xfwhe770V9_gnoJEjNqk4lJmpHcqaLgojSo9-MXks,1299
|
|
2
|
-
musefs_common/constants.py,sha256=M0zS6YCnYdE_JjIbvxg-GhbYdfbe30ccKZvbZ1WTZZ8,302
|
|
3
|
-
musefs_common/contract.py,sha256=tUflCxfFiaIzJiTqYL-o-bjKNH_s91EHkC2amk8sKxE,1472
|
|
4
|
-
musefs_common/errors.py,sha256=OUBXzpa4q7TPYvFFiTyoIfRC3SGRRgZcpXlr43RP0jk,1456
|
|
5
|
-
musefs_common/paths.py,sha256=dE3wwop5mRxoJ8RhdYwTIZpX4MPaUUw2LJhjcxApOIA,649
|
|
6
|
-
musefs_common/scan.py,sha256=2rOFQQt0pE1uhF31huSwRtapU0QK3fFDGbscynwLtRE,1453
|
|
7
|
-
musefs_common/schema.py,sha256=Blab4CIiFIQaaqh5IjMpUR0jD01xU0CFrl4AQQM3yKM,8511
|
|
8
|
-
musefs_common/store.py,sha256=nk34ghVlqMsF__PozYZHCu8Xd0QsoChl9bGTFsrdbwE,8721
|
|
9
|
-
musefs_common/sync.py,sha256=Rh65b69d92FLxNwN4Z939GmfUiZ1Sfw7F_vL31i4VtM,3375
|
|
10
|
-
python_musefs-1.0.0.dist-info/licenses/LICENSE,sha256=4VbfzhgMpuFrhFjBCtLptGV_bzCj_gML9y_9o2Tr1OQ,1068
|
|
11
|
-
python_musefs-1.0.0.dist-info/METADATA,sha256=F2pO6iiY0rn4HLaefxYYo73hc7dbc_KSgWWzvooCXm4,11971
|
|
12
|
-
python_musefs-1.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
13
|
-
python_musefs-1.0.0.dist-info/top_level.txt,sha256=ejWexGk95-s11kZnTFwg1T3iG_dFcaE4vhcLnL3t3Ak,14
|
|
14
|
-
python_musefs-1.0.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|