rekfix 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rekfix/__init__.py +2 -0
- rekfix/__main__.py +5 -0
- rekfix/anlz.py +234 -0
- rekfix/check.py +279 -0
- rekfix/cli.py +382 -0
- rekfix/clone.py +357 -0
- rekfix/menu.py +71 -0
- rekfix/pdb.py +503 -0
- rekfix/prune.py +327 -0
- rekfix/readtest.py +146 -0
- rekfix/rebuild.py +419 -0
- rekfix/snapshot.py +72 -0
- rekfix-0.2.0.dist-info/METADATA +172 -0
- rekfix-0.2.0.dist-info/RECORD +18 -0
- rekfix-0.2.0.dist-info/WHEEL +5 -0
- rekfix-0.2.0.dist-info/entry_points.txt +2 -0
- rekfix-0.2.0.dist-info/licenses/LICENSE +674 -0
- rekfix-0.2.0.dist-info/top_level.txt +1 -0
rekfix/__init__.py
ADDED
rekfix/__main__.py
ADDED
rekfix/anlz.py
ADDED
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"""Read rekordbox's per-track analysis files: PIONEER/USBANLZ/.../ANLZ0000.DAT and .EXT.
|
|
2
|
+
|
|
3
|
+
Big-endian, unlike export.pdb. A file starts with "PMAI" and its header length; then tagged
|
|
4
|
+
sections follow, each starting with a fourcc, a u32 header length and a u32 section length.
|
|
5
|
+
What's read here:
|
|
6
|
+
- PPTH: the audio file's path on the stick (UTF-16BE). It ties the analysis to its track even
|
|
7
|
+
when export.pdb is damaged or gone.
|
|
8
|
+
- PQTZ (in the .DAT): the beat grid, one 8-byte entry per beat: beat in the bar (u16), tempo x 100
|
|
9
|
+
(u16), time in ms (u32). The beat count is at +20 of the section.
|
|
10
|
+
- PCOB (.DAT) and PCO2 (.EXT): two cue lists each, list type 0 = memory cues, 1 = hot cues.
|
|
11
|
+
- PCOB entries are PCPT, 56 bytes.
|
|
12
|
+
- PCO2 entries are PCP2. rekordbox writes them 88 bytes or longer, with a name (UTF-16BE) and
|
|
13
|
+
a colour; a deck writes 44 bytes, with neither.
|
|
14
|
+
|
|
15
|
+
In an entry, the hot cue number is a u32 at +12 (0 = memory cue, 1-8 = A-H).
|
|
16
|
+
|
|
17
|
+
Field offsets were checked against thousands of real files written by rekordbox 5 and 6 and by
|
|
18
|
+
XDJ-1000MK2 decks (docs/format.md). Damaged files are read as far as they make sense; if the cue
|
|
19
|
+
lists can't be walked, the entries are found by their signatures instead.
|
|
20
|
+
"""
|
|
21
|
+
import os
|
|
22
|
+
import struct
|
|
23
|
+
from dataclasses import dataclass, field
|
|
24
|
+
|
|
25
|
+
NO_LOOP = (0, 0xFFFFFFFF) # rekordbox writes 0xFFFFFFFF as "no loop end", a deck writes 0
|
|
26
|
+
MAX_MS = 24 * 3600 * 1000 # anything later than this is garbage
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class Cue:
|
|
31
|
+
hot: int # 0 = memory cue; 1-8 = hot cue A-H (up to 16 on newer gear)
|
|
32
|
+
start_ms: int
|
|
33
|
+
end_ms: int = None # set for loops
|
|
34
|
+
name: str = ""
|
|
35
|
+
rgb: tuple = None # hot cue colour, when the entry carries one
|
|
36
|
+
source: str = "" # "EXT" or "DAT"
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def is_loop(self):
|
|
40
|
+
return self.end_ms is not None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass
|
|
44
|
+
class Analysis:
|
|
45
|
+
path: str = "" # the audio file, as the stick knows it
|
|
46
|
+
beats: list = field(default_factory=list) # (beat in bar, tempo x 100, ms)
|
|
47
|
+
cues: list = field(default_factory=list)
|
|
48
|
+
problems: list = field(default_factory=list)
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def bpm(self):
|
|
52
|
+
"""The median tempo of the beat grid, or 0."""
|
|
53
|
+
if not self.beats:
|
|
54
|
+
return 0.0
|
|
55
|
+
tempos = sorted(t for _, t, _ in self.beats)
|
|
56
|
+
return tempos[len(tempos) // 2] / 100
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def sections(data, problems=None):
|
|
60
|
+
"""(position, fourcc, header length, section length) for each section, stopping at the first
|
|
61
|
+
one that doesn't make sense (and noting it in problems)."""
|
|
62
|
+
if len(data) < 12 or data[:4] != b"PMAI":
|
|
63
|
+
if problems is not None:
|
|
64
|
+
problems.append("not an analysis file (no PMAI header)")
|
|
65
|
+
return
|
|
66
|
+
pos = struct.unpack_from(">I", data, 4)[0]
|
|
67
|
+
while pos + 12 <= len(data):
|
|
68
|
+
fourcc = bytes(data[pos:pos + 4])
|
|
69
|
+
lh, lt = struct.unpack_from(">II", data, pos + 4)
|
|
70
|
+
if lt < 12 or lh > lt or pos + lt > len(data) or not fourcc.isalnum():
|
|
71
|
+
if problems is not None:
|
|
72
|
+
problems.append(f"section at {pos} doesn't make sense; read stopped there")
|
|
73
|
+
return
|
|
74
|
+
yield pos, fourcc, lh, lt
|
|
75
|
+
pos += lt
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def read_path(data):
|
|
79
|
+
"""The PPTH path. Works on just the start of a file too: sections before it are skipped by
|
|
80
|
+
their length without being read."""
|
|
81
|
+
if len(data) < 12 or data[:4] != b"PMAI":
|
|
82
|
+
return ""
|
|
83
|
+
pos = struct.unpack_from(">I", data, 4)[0]
|
|
84
|
+
while pos + 16 <= len(data):
|
|
85
|
+
lh, lt = struct.unpack_from(">II", data, pos + 4)
|
|
86
|
+
if lt < 12:
|
|
87
|
+
return ""
|
|
88
|
+
if data[pos:pos + 4] == b"PPTH":
|
|
89
|
+
n = struct.unpack_from(">I", data, pos + 12)[0]
|
|
90
|
+
return data[pos + lh:pos + lh + n].decode("utf-16-be", "replace").rstrip("\x00")
|
|
91
|
+
pos += lt
|
|
92
|
+
return ""
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def read_beats(data, problems=None):
|
|
96
|
+
for pos, fourcc, lh, lt in sections(data, problems):
|
|
97
|
+
if fourcc == b"PQTZ" and lh >= 24:
|
|
98
|
+
n = struct.unpack_from(">I", data, pos + 20)[0]
|
|
99
|
+
beats, p = [], pos + lh
|
|
100
|
+
for _ in range(min(n, (lt - lh) // 8)):
|
|
101
|
+
beat, tempo, ms = struct.unpack_from(">HHI", data, p)
|
|
102
|
+
if tempo and ms <= MAX_MS:
|
|
103
|
+
beats.append((beat, tempo, ms))
|
|
104
|
+
p += 8
|
|
105
|
+
return beats
|
|
106
|
+
return []
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _entry(data, e, fourcc, source):
|
|
110
|
+
"""One PCPT/PCP2 entry -> Cue, or None if it isn't a usable cue."""
|
|
111
|
+
le = struct.unpack_from(">I", data, e + 8)[0]
|
|
112
|
+
hot = struct.unpack_from(">I", data, e + 12)[0]
|
|
113
|
+
if fourcc == b"PCPT":
|
|
114
|
+
kind = data[e + 28]
|
|
115
|
+
start, end = struct.unpack_from(">II", data, e + 32)
|
|
116
|
+
name, rgb = "", None
|
|
117
|
+
else:
|
|
118
|
+
kind = data[e + 16]
|
|
119
|
+
start, end = struct.unpack_from(">II", data, e + 20)
|
|
120
|
+
name, rgb = "", None
|
|
121
|
+
if le > 44:
|
|
122
|
+
n = struct.unpack_from(">I", data, e + 40)[0]
|
|
123
|
+
if 0 < n <= le - 44:
|
|
124
|
+
name = data[e + 44:e + 44 + n].decode("utf-16-be", "replace").rstrip("\x00")
|
|
125
|
+
after = e + 44 + n
|
|
126
|
+
if 0 <= n and after + 4 <= e + le:
|
|
127
|
+
code, r, g, b = data[after:after + 4]
|
|
128
|
+
if code or r or g or b:
|
|
129
|
+
rgb = (r, g, b)
|
|
130
|
+
if hot > 16 or start > MAX_MS:
|
|
131
|
+
return None
|
|
132
|
+
loop = kind == 2 and end not in NO_LOOP and start < end <= MAX_MS
|
|
133
|
+
return Cue(hot, start, end if loop else None, "".join(c for c in name if c.isprintable()), rgb, source)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def read_cues(data, source, problems=None):
|
|
137
|
+
"""The cues in one file, by walking its cue lists. Falls back to finding the entries by
|
|
138
|
+
their signature when the lists can't be walked (a damaged file)."""
|
|
139
|
+
out, ok = [], True
|
|
140
|
+
for pos, fourcc, lh, lt in sections(data, problems):
|
|
141
|
+
if fourcc not in (b"PCOB", b"PCO2"):
|
|
142
|
+
continue
|
|
143
|
+
count = struct.unpack_from(">H", data, pos + (18 if fourcc == b"PCOB" else 16))[0]
|
|
144
|
+
e, end = pos + lh, pos + lt
|
|
145
|
+
for _ in range(count):
|
|
146
|
+
if e + 44 > end or data[e:e + 4] not in (b"PCPT", b"PCP2"):
|
|
147
|
+
ok = False
|
|
148
|
+
break
|
|
149
|
+
le = struct.unpack_from(">I", data, e + 8)[0]
|
|
150
|
+
if le < 44 or e + le > end:
|
|
151
|
+
ok = False
|
|
152
|
+
break
|
|
153
|
+
cue = _entry(data, e, data[e:e + 4], source)
|
|
154
|
+
if cue:
|
|
155
|
+
out.append(cue)
|
|
156
|
+
e += le
|
|
157
|
+
if ok and out:
|
|
158
|
+
return out
|
|
159
|
+
if not ok and problems is not None:
|
|
160
|
+
problems.append(f"{source}: cue lists damaged; entries found by signature instead")
|
|
161
|
+
found = []
|
|
162
|
+
for sig in (b"PCP2", b"PCPT"):
|
|
163
|
+
i = data.find(sig)
|
|
164
|
+
while 0 <= i:
|
|
165
|
+
if i + 44 <= len(data):
|
|
166
|
+
le = struct.unpack_from(">I", data, i + 8)[0]
|
|
167
|
+
if 44 <= le <= 0x400 and i + le <= len(data):
|
|
168
|
+
cue = _entry(data, i, sig, source)
|
|
169
|
+
if cue:
|
|
170
|
+
found.append(cue)
|
|
171
|
+
i = data.find(sig, i + 4)
|
|
172
|
+
return found or out
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def merge_cues(ext, dat):
|
|
176
|
+
"""One list from both files: the .EXT's (names and colours) first, then any the .DAT has that
|
|
177
|
+
the .EXT doesn't (same slot, within 2 ms)."""
|
|
178
|
+
out = list(ext)
|
|
179
|
+
for c in dat:
|
|
180
|
+
if not any(o.hot == c.hot and abs(o.start_ms - c.start_ms) <= 2 and o.is_loop == c.is_loop for o in out):
|
|
181
|
+
out.append(c)
|
|
182
|
+
seen, unique = set(), []
|
|
183
|
+
for c in out:
|
|
184
|
+
key = (c.hot, c.start_ms, c.end_ms)
|
|
185
|
+
if key not in seen:
|
|
186
|
+
seen.add(key)
|
|
187
|
+
unique.append(c)
|
|
188
|
+
return sorted(unique, key=lambda c: (c.hot == 0, c.hot, c.start_ms))
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _load(path):
|
|
192
|
+
try:
|
|
193
|
+
with open(path, "rb") as fh:
|
|
194
|
+
return fh.read()
|
|
195
|
+
except OSError:
|
|
196
|
+
return None
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def read(dat_path):
|
|
200
|
+
"""Everything usable from a track's ANLZ0000.DAT and the .EXT next to it."""
|
|
201
|
+
a = Analysis()
|
|
202
|
+
dat = _load(dat_path)
|
|
203
|
+
ext = _load(dat_path[:-4] + ".EXT") if dat_path.upper().endswith(".DAT") else None
|
|
204
|
+
if dat is None and ext is None:
|
|
205
|
+
a.problems.append("analysis files missing")
|
|
206
|
+
return a
|
|
207
|
+
if dat is not None:
|
|
208
|
+
a.path = read_path(dat)
|
|
209
|
+
a.beats = read_beats(dat, a.problems)
|
|
210
|
+
if ext is not None and not a.path:
|
|
211
|
+
a.path = read_path(ext)
|
|
212
|
+
a.cues = merge_cues(read_cues(ext, "EXT", a.problems) if ext is not None else [],
|
|
213
|
+
read_cues(dat, "DAT", a.problems) if dat is not None else [])
|
|
214
|
+
return a
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def index_paths(usbanlz):
|
|
218
|
+
"""{audio path, lower case: .DAT path} for every analysis folder under USBANLZ, from the path
|
|
219
|
+
each file names (PPTH). Only the start of each file is read."""
|
|
220
|
+
out = {}
|
|
221
|
+
for root, _dirs, files in os.walk(usbanlz):
|
|
222
|
+
for f in files:
|
|
223
|
+
if f.upper() != "ANLZ0000.DAT":
|
|
224
|
+
continue
|
|
225
|
+
p = os.path.join(root, f)
|
|
226
|
+
try:
|
|
227
|
+
with open(p, "rb") as fh:
|
|
228
|
+
head = fh.read(4096)
|
|
229
|
+
except OSError:
|
|
230
|
+
continue
|
|
231
|
+
path = read_path(head)
|
|
232
|
+
if path:
|
|
233
|
+
out.setdefault(path.lower(), p)
|
|
234
|
+
return out
|
rekfix/check.py
ADDED
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
"""Read-only health checks for an export.pdb, and optionally the stick it came from.
|
|
2
|
+
|
|
3
|
+
Every finding has a level:
|
|
4
|
+
- error: something a player may choke on, or data that is plainly inconsistent;
|
|
5
|
+
- warn: worth a look (e.g. a track whose audio file isn't on the stick);
|
|
6
|
+
- info: facts that help a diagnosis (sizes, counts).
|
|
7
|
+
"""
|
|
8
|
+
import os
|
|
9
|
+
import struct
|
|
10
|
+
from collections import Counter, defaultdict
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
|
|
13
|
+
from .pdb import GROUP, HEADER, LOOKUPS, TABLES, Pdb
|
|
14
|
+
|
|
15
|
+
# the root categories a player builds by scanning every track when you land on them; on a large
|
|
16
|
+
# library (~13-14k tracks on an XDJ-1000MK2) they can freeze the deck - see docs/playbook.md
|
|
17
|
+
HEAVY_CATEGORIES = {14: "TIME", 15: "BITRATE", 16: "FILE NAME", 18: "HOT CUE BANK"}
|
|
18
|
+
LARGE_LIBRARY = 10_000
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class Finding:
|
|
23
|
+
level: str
|
|
24
|
+
code: str
|
|
25
|
+
message: str
|
|
26
|
+
|
|
27
|
+
def __str__(self):
|
|
28
|
+
return f"{self.level.upper():5} {self.code:24} {self.message}"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def walk(pdb):
|
|
32
|
+
"""Follow every table's page chain the long way, reporting what a reader would trip over.
|
|
33
|
+
Returns ({table type: (pages, rows present)}, findings)."""
|
|
34
|
+
out, found = {}, []
|
|
35
|
+
if len(pdb.data) % pdb.page_size:
|
|
36
|
+
found.append(Finding("warn", "file.size", f"{len(pdb.data)} bytes isn't a whole number of "
|
|
37
|
+
f"{pdb.page_size}-byte pages"))
|
|
38
|
+
highest = max((t.last for t in pdb.tables.values()), default=0)
|
|
39
|
+
if pdb.next_unused < pdb.num_pages // 2 or pdb.next_unused <= highest:
|
|
40
|
+
found.append(Finding("error", "header.stale", f"the header says the next free page is {pdb.next_unused}, "
|
|
41
|
+
f"but the file has {pdb.num_pages} pages (tables end as far "
|
|
42
|
+
f"as page {highest}): the header doesn't describe this file"))
|
|
43
|
+
for typ, table in pdb.tables.items():
|
|
44
|
+
name = table.name
|
|
45
|
+
index, seen, pages, rows, variants = table.first, [], 0, 0, []
|
|
46
|
+
while True:
|
|
47
|
+
if not 0 < index < pdb.num_pages:
|
|
48
|
+
found.append(Finding("error", "chain.outside", f"{name}: next page {index} is outside the file "
|
|
49
|
+
f"({pdb.num_pages} pages)"))
|
|
50
|
+
break
|
|
51
|
+
if index in seen:
|
|
52
|
+
found.append(Finding("error", "chain.loop", f"{name}: page chain loops back to page {index}"))
|
|
53
|
+
break
|
|
54
|
+
seen.append(index)
|
|
55
|
+
page = pdb.page(index)
|
|
56
|
+
pages += 1
|
|
57
|
+
if page.index != index:
|
|
58
|
+
found.append(Finding("error", "page.index", f"{name}: page {index} says it is page {page.index}"))
|
|
59
|
+
if page.type != typ:
|
|
60
|
+
found.append(Finding("error", "page.type", f"{name}: page {index} belongs to table {page.type} "
|
|
61
|
+
f"({TABLES.get(page.type, '?')})"))
|
|
62
|
+
if page.is_data:
|
|
63
|
+
rows += _check_page(pdb, page, name, found, variants)
|
|
64
|
+
if index == table.last:
|
|
65
|
+
break
|
|
66
|
+
index = page.next
|
|
67
|
+
if index == 0:
|
|
68
|
+
found.append(Finding("error", "chain.short", f"{name}: chain ends before its last page {table.last}"))
|
|
69
|
+
break
|
|
70
|
+
# data the chain doesn't reach: past the recorded last page, or on pages no chain visits
|
|
71
|
+
unreached = [(pg.index, sum(pdb.is_present(pg, s) for s in range(pg.slots)))
|
|
72
|
+
for pg in pdb.scan_pages(typ) if pg.index not in seen]
|
|
73
|
+
unreached = [(i, n) for i, n in unreached if n]
|
|
74
|
+
if unreached:
|
|
75
|
+
n_rows = sum(n for _, n in unreached)
|
|
76
|
+
level = "error" if n_rows > rows else "warn"
|
|
77
|
+
found.append(Finding(level, "pages.unreached", f"{name}: {len(unreached)} pages hold {n_rows:,} rows the "
|
|
78
|
+
f"chain doesn't reach (it reaches {rows:,}); "
|
|
79
|
+
f"`rekfix check --scan` reads them"))
|
|
80
|
+
if pdb.legacy(typ):
|
|
81
|
+
found.append(Finding("info", "format.legacy-flags", f"{name}: present bits are in the older-export "
|
|
82
|
+
f"footer word (see docs/format.md)"))
|
|
83
|
+
if variants:
|
|
84
|
+
found.append(Finding("info", "page.flags-word", f"{name}: {len(variants)} row groups have no present bits "
|
|
85
|
+
f"but a non-zero second flags word (pages "
|
|
86
|
+
f"{', '.join(map(str, variants[:5]))}...); deleted rows, "
|
|
87
|
+
f"or the older-export word order - see docs/format.md"))
|
|
88
|
+
out[typ] = (pages, rows)
|
|
89
|
+
return out, found
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def footer_slots(pdb, page):
|
|
93
|
+
"""Row slots implied by the page's free and used sizes: what's left is the footer, 18 u16s
|
|
94
|
+
per full group of 16 and (rows + 2) u16s for the last one."""
|
|
95
|
+
footer = pdb.page_size - HEADER - page.used - page.free
|
|
96
|
+
full, rem = divmod(footer // 2, 18)
|
|
97
|
+
return full * 16 + (rem - 2 if rem >= 3 else 0)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _check_page(pdb, page, name, found, variants):
|
|
101
|
+
space = pdb.page_size - HEADER
|
|
102
|
+
groups = (page.slots + 15) // 16
|
|
103
|
+
if page.free + page.used + groups * GROUP > space + GROUP:
|
|
104
|
+
found.append(Finding("error", "page.space", f"{name}: page {page.index} free {page.free} + used "
|
|
105
|
+
f"{page.used} + index doesn't fit the page"))
|
|
106
|
+
implied = footer_slots(pdb, page)
|
|
107
|
+
if page.slots and implied != page.slots:
|
|
108
|
+
found.append(Finding("warn", "page.footer", f"{name}: page {page.index} header says {page.slots} row slots, "
|
|
109
|
+
f"free/used sizes imply {implied}"))
|
|
110
|
+
present = 0
|
|
111
|
+
for slot in range(page.slots):
|
|
112
|
+
if not pdb.is_present(page, slot):
|
|
113
|
+
continue
|
|
114
|
+
present += 1
|
|
115
|
+
off = pdb.row_offset(page, slot)
|
|
116
|
+
if off >= page.used:
|
|
117
|
+
found.append(Finding("error", "row.offset", f"{name}: page {page.index} row {slot} starts at {off}, "
|
|
118
|
+
f"outside the page's {page.used} used bytes"))
|
|
119
|
+
if present != page.present:
|
|
120
|
+
found.append(Finding("warn", "page.count", f"{name}: page {page.index} header says {page.present} rows "
|
|
121
|
+
f"present, the bitmasks say {present}"))
|
|
122
|
+
# an older-export variant: present bits in the other footer word (see docs/format.md)
|
|
123
|
+
if pdb.legacy(page.type):
|
|
124
|
+
return present
|
|
125
|
+
for g in range(groups):
|
|
126
|
+
base = page.pos + pdb.page_size - g * GROUP
|
|
127
|
+
first, second = struct.unpack_from("<2H", pdb.data, base - 4)
|
|
128
|
+
if first == 0 and second:
|
|
129
|
+
variants.append(page.index)
|
|
130
|
+
return present
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def content(pdb):
|
|
134
|
+
"""Cross-table consistency. Returns (tracks by id, findings)."""
|
|
135
|
+
found = []
|
|
136
|
+
by_id = pdb.tracks()
|
|
137
|
+
dups = {tid: ts for tid, ts in by_id.items() if len(ts) > 1}
|
|
138
|
+
if dups:
|
|
139
|
+
sample = ", ".join(f"{tid} x{len(ts)}" for tid, ts in list(dups.items())[:5])
|
|
140
|
+
found.append(Finding("error", "tracks.duplicate-id", f"{len(dups)} track ids appear more than once ({sample})"))
|
|
141
|
+
bad_strings = [t for ts in by_id.values() for t in ts if t.string_errors]
|
|
142
|
+
if bad_strings:
|
|
143
|
+
t = bad_strings[0]
|
|
144
|
+
found.append(Finding("error", "tracks.strings", f"{len(bad_strings)} track rows have unreadable text, "
|
|
145
|
+
f"e.g. id {t.id}: {t.string_errors[0]}"))
|
|
146
|
+
tids = set(by_id)
|
|
147
|
+
tree = pdb.playlist_tree()
|
|
148
|
+
for pid, (parent, sort, folder, name) in tree.items():
|
|
149
|
+
if parent and parent not in tree:
|
|
150
|
+
found.append(Finding("error", "playlists.parent", f"playlist {name!r} ({pid}) has a missing parent {parent}"))
|
|
151
|
+
siblings = defaultdict(list)
|
|
152
|
+
for pid, (parent, sort, folder, name) in tree.items():
|
|
153
|
+
siblings[parent].append(sort)
|
|
154
|
+
for parent, sorts in siblings.items():
|
|
155
|
+
dup = [s for s, c in Counter(sorts).items() if c > 1]
|
|
156
|
+
if dup:
|
|
157
|
+
found.append(Finding("warn", "playlists.sort", f"{len(dup)} duplicate sort positions under parent {parent}"))
|
|
158
|
+
entries = pdb.playlist_entries()
|
|
159
|
+
orphans = [(p, t) for p, i, t, r in entries if t not in tids or p not in tree]
|
|
160
|
+
if orphans:
|
|
161
|
+
found.append(Finding("error", "playlists.orphans", f"{len(orphans)} playlist entries point at a missing "
|
|
162
|
+
f"track or playlist"))
|
|
163
|
+
per_list = defaultdict(list)
|
|
164
|
+
for p, i, t, r in entries:
|
|
165
|
+
per_list[p].append(i)
|
|
166
|
+
gaps = [p for p, idx in per_list.items() if sorted(idx) != list(range(1, len(idx) + 1))]
|
|
167
|
+
if gaps:
|
|
168
|
+
found.append(Finding("warn", "playlists.index", f"{len(gaps)} playlists' entry numbers aren't 1..n"))
|
|
169
|
+
hists = pdb.history_playlists()
|
|
170
|
+
hent = pdb.history_entries()
|
|
171
|
+
horph = [t for p, i, t, r in hent if t not in tids or p not in hists]
|
|
172
|
+
if horph:
|
|
173
|
+
found.append(Finding("error", "history.orphans", f"{len(horph)} history entries point at a missing track "
|
|
174
|
+
f"or history list"))
|
|
175
|
+
found.append(Finding("info", "summary", f"{len(tids):,} tracks, {sum(1 for v in tree.values() if not v[2]):,} "
|
|
176
|
+
f"playlists, {len(entries):,} playlist entries, {len(hists)} play "
|
|
177
|
+
f"histories ({len(hent):,} plays)"))
|
|
178
|
+
found += leftovers(pdb, [ts[0] for ts in by_id.values()])
|
|
179
|
+
menu = pdb.menu()
|
|
180
|
+
names = pdb.columns()
|
|
181
|
+
shown = [m for m in sorted(menu, key=lambda m: m.sort) if m.visible]
|
|
182
|
+
heavy = [names.get(m.category, HEAVY_CATEGORIES[m.category]) for m in shown if m.category in HEAVY_CATEGORIES]
|
|
183
|
+
if heavy and len(tids) >= LARGE_LIBRARY:
|
|
184
|
+
found.append(Finding("warn", "menu.heavy", f"{len(tids):,} tracks and the root menu shows {', '.join(heavy)}: "
|
|
185
|
+
f"on older players (e.g. XDJ-1000MK2) these can freeze the deck. "
|
|
186
|
+
f"See `rekfix menu --hide`"))
|
|
187
|
+
return by_id, found
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def unused(pdb, tracks):
|
|
191
|
+
"""Rows of the lookup tables (genres, artists, albums, labels, artwork) that none of `tracks`
|
|
192
|
+
uses, directly or through an album: {table type: [rows]}. Exports rekordbox writes have none.
|
|
193
|
+
It's been seen deleting a removed track's artwork row along with the track; whether it also
|
|
194
|
+
deletes an artist (album, genre, label) whose last track goes hasn't been observed."""
|
|
195
|
+
used = {typ: set() for typ in LOOKUPS}
|
|
196
|
+
for t in tracks:
|
|
197
|
+
for typ, (_, fields) in LOOKUPS.items():
|
|
198
|
+
used[typ].update(getattr(t, f) for f in fields)
|
|
199
|
+
for album, artists in pdb.albums().items():
|
|
200
|
+
if album in used[3]:
|
|
201
|
+
used[2].update(artists)
|
|
202
|
+
out = {}
|
|
203
|
+
for typ in LOOKUPS:
|
|
204
|
+
if typ in pdb.tables:
|
|
205
|
+
rows = [r for i, r in pdb.lookup_rows(typ) if i not in used[typ]]
|
|
206
|
+
if rows:
|
|
207
|
+
out[typ] = rows
|
|
208
|
+
return out
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def leftovers(pdb, tracks):
|
|
212
|
+
"""What removing tracks without rekordbox leaves behind: a stale song count, artwork (and
|
|
213
|
+
artists...) no track uses, pages missing the has-deleted-rows flag."""
|
|
214
|
+
found = []
|
|
215
|
+
row, count = pdb.song_count()
|
|
216
|
+
if row is not None and count != len(tracks):
|
|
217
|
+
found.append(Finding("warn", "tracks.count", f"players show {count:,} songs on their USB info panel, but "
|
|
218
|
+
f"{len(tracks):,} tracks are present (left over from tracks "
|
|
219
|
+
f"removed without updating it); `rekfix prune` corrects it"))
|
|
220
|
+
left = unused(pdb, tracks)
|
|
221
|
+
artwork = left.pop(13, [])
|
|
222
|
+
if artwork:
|
|
223
|
+
found.append(Finding("info", "artwork.unused", f"{len(artwork):,} artwork entries no track uses (left over "
|
|
224
|
+
f"from removed tracks); `rekfix prune` removes them, as "
|
|
225
|
+
f"rekordbox does when it removes a track"))
|
|
226
|
+
if left:
|
|
227
|
+
what = ", ".join(f"{len(rows):,} {TABLES[typ]}" for typ, rows in left.items())
|
|
228
|
+
found.append(Finding("info", "rows.unused", f"{what} that no track uses. rekordbox's exports have none, so "
|
|
229
|
+
f"they're likely left over from tracks removed by another tool. "
|
|
230
|
+
f"Left alone for now: see docs/format.md"))
|
|
231
|
+
flags = pdb.unflagged_pages()
|
|
232
|
+
if flags:
|
|
233
|
+
found.append(Finding("info", "page.deleted-flag", f"{len(flags):,} pages have deleted rows without the flag "
|
|
234
|
+
f"rekordbox sets on such pages (an edit by another tool); "
|
|
235
|
+
f"`rekfix prune` sets it"))
|
|
236
|
+
return found
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def files(pdb, root, by_id=None):
|
|
240
|
+
"""Tracks whose audio or analysis file isn't on the stick at `root`, and rekordbox's
|
|
241
|
+
broken-library marker."""
|
|
242
|
+
found = []
|
|
243
|
+
if os.path.exists(os.path.join(root, "PIONEER", "rekordbox", "brokendb")):
|
|
244
|
+
found.append(Finding("warn", "rekordbox.broken-marker", "rekordbox has marked this stick's library as broken "
|
|
245
|
+
"(PIONEER\\rekordbox\\brokendb): it crashed or failed "
|
|
246
|
+
"reading it, and will say 'device library is corrupted' "
|
|
247
|
+
"while the marker is there. See docs/playbook.md"))
|
|
248
|
+
by_id = by_id if by_id is not None else pdb.tracks()
|
|
249
|
+
missing_audio, missing_anlz = [], []
|
|
250
|
+
for ts in by_id.values():
|
|
251
|
+
t = ts[0]
|
|
252
|
+
if t.path and not os.path.exists(on_stick(root, t.path)):
|
|
253
|
+
missing_audio.append(t)
|
|
254
|
+
anlz = t.strings.get("analyze_path")
|
|
255
|
+
if anlz and not os.path.exists(on_stick(root, anlz)):
|
|
256
|
+
missing_anlz.append(t)
|
|
257
|
+
if missing_audio:
|
|
258
|
+
found.append(Finding("warn", "files.audio", f"{len(missing_audio):,} tracks' audio files aren't on the stick "
|
|
259
|
+
f"(listed but won't load), e.g. {missing_audio[0].path}. "
|
|
260
|
+
f"`rekfix prune` removes them or copies them back"))
|
|
261
|
+
if missing_anlz:
|
|
262
|
+
found.append(Finding("warn", "files.analysis", f"{len(missing_anlz):,} tracks' analysis files aren't on the "
|
|
263
|
+
f"stick, e.g. {missing_anlz[0].strings['analyze_path']}"))
|
|
264
|
+
return found
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def on_stick(root, path):
|
|
268
|
+
"""A path from the database ("/Contents/...") as a file on the stick mounted at root."""
|
|
269
|
+
return os.path.join(root, *[p for p in path.replace("\\", "/").split("/") if p])
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def run(pdb_path, root=None):
|
|
273
|
+
pdb = Pdb.open(pdb_path)
|
|
274
|
+
counts, found = walk(pdb)
|
|
275
|
+
by_id, more = content(pdb)
|
|
276
|
+
found += more
|
|
277
|
+
if root:
|
|
278
|
+
found += files(pdb, root, by_id)
|
|
279
|
+
return pdb, counts, found
|