navidrome-scripts 0.14.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- navidrome_scripts/__init__.py +1 -0
- navidrome_scripts/artist_ratings.py +424 -0
- navidrome_scripts/catalog.py +45 -0
- navidrome_scripts/date_evidence.py +265 -0
- navidrome_scripts/dates.py +369 -0
- navidrome_scripts/duplicates.py +356 -0
- navidrome_scripts/indexed_duplicates.py +370 -0
- navidrome_scripts/library_review.py +281 -0
- navidrome_scripts/lidarr.py +43 -0
- navidrome_scripts/locking.py +88 -0
- navidrome_scripts/maintenance.py +308 -0
- navidrome_scripts/metadata.py +46 -0
- navidrome_scripts/navidrome_guard.py +202 -0
- navidrome_scripts/playlists.py +313 -0
- navidrome_scripts/provider_review.py +122 -0
- navidrome_scripts/reconciliation.py +210 -0
- navidrome_scripts/rename_rollback.py +273 -0
- navidrome_scripts/renames.py +360 -0
- navidrome_scripts/spotify.py +131 -0
- navidrome_scripts/subsonic.py +166 -0
- navidrome_scripts/tag_repair.py +382 -0
- navidrome_scripts/unmapped.py +317 -0
- navidrome_scripts-0.14.0.dist-info/METADATA +485 -0
- navidrome_scripts-0.14.0.dist-info/RECORD +27 -0
- navidrome_scripts-0.14.0.dist-info/WHEEL +5 -0
- navidrome_scripts-0.14.0.dist-info/entry_points.txt +11 -0
- navidrome_scripts-0.14.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
"""Exact-edition date evidence and date-only writes through Beets' MediaFile backend."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import hashlib
|
|
5
|
+
import html
|
|
6
|
+
import json
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
import re
|
|
9
|
+
|
|
10
|
+
import requests
|
|
11
|
+
from mutagen import File, MutagenError
|
|
12
|
+
|
|
13
|
+
from .catalog import MusicBrainzClient
|
|
14
|
+
from . import dates
|
|
15
|
+
from .metadata import identify
|
|
16
|
+
from .playlists import hash_json, normalized
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def partial_date(value):
|
|
20
|
+
if not re.fullmatch(r'\d{4}(?:-\d{2}(?:-\d{2})?)?', value) or dates.date_problem(value):
|
|
21
|
+
raise ValueError('source date is blank, invalid or has unsupported precision')
|
|
22
|
+
return tuple(int(x) for x in value.split('-'))
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def family(key):
|
|
26
|
+
key=str(key).casefold().removeprefix('txxx:').removeprefix('----:com.apple.itunes:').replace('_','').replace(' ','')
|
|
27
|
+
return 'original_date' if key.startswith('original') or key in ('tdor','tory','wm/originalreleasetime','wm/originalreleaseyear') else 'release_date'
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def artist_ids(credits):
|
|
31
|
+
return [str(x.get('artist',{}).get('id','')).casefold() for x in credits if x.get('artist',{}).get('id')]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def verified_evidence(payload, tags):
|
|
35
|
+
if not tags.get('title') or tags.get('tracknumber',0)<1 or tags.get('discnumber',0)<1 or tags.get('duration_ms',0)<=0:
|
|
36
|
+
raise ValueError('complete title, position and duration are required for date evidence')
|
|
37
|
+
release_id=tags.get('release_mbid','').casefold()
|
|
38
|
+
group_id=tags.get('group_mbid','').casefold()
|
|
39
|
+
group=payload.get('release-group') or {}
|
|
40
|
+
if not release_id or not group_id or str(payload.get('id','')).casefold()!=release_id or str(group.get('id','')).casefold()!=group_id:
|
|
41
|
+
raise ValueError('tagged exact release/release-group differs from catalog data')
|
|
42
|
+
release_artists=artist_ids(payload.get('artist-credit',[]))
|
|
43
|
+
group_artists=artist_ids(group.get('artist-credit',[]))
|
|
44
|
+
if not release_artists or not group_artists or release_artists!=group_artists:
|
|
45
|
+
raise ValueError('release and release-group artist credits conflict or are unavailable')
|
|
46
|
+
if tags.get('album_artist_mbid') and tags['album_artist_mbid'].casefold() not in release_artists:
|
|
47
|
+
raise ValueError('tagged album artist differs from exact catalog release')
|
|
48
|
+
if not tags.get('album_artist_mbid'):
|
|
49
|
+
credited=''.join((c.get('name') or c.get('artist',{}).get('name',''))+c.get('joinphrase','') for c in payload.get('artist-credit',[]))
|
|
50
|
+
if normalized(tags.get('albumartist') or tags.get('artist',''))!=normalized(credited):
|
|
51
|
+
raise ValueError('album artist name cannot be verified against the exact catalog release')
|
|
52
|
+
matches=[]
|
|
53
|
+
for medium in payload.get('media',[]):
|
|
54
|
+
if medium.get('position')!=tags['discnumber']:
|
|
55
|
+
continue
|
|
56
|
+
for row in medium.get('tracks',[]):
|
|
57
|
+
recording=row.get('recording') or {}
|
|
58
|
+
length=row.get('length') or recording.get('length')
|
|
59
|
+
if str(row.get('number'))!=str(tags['tracknumber']) or not length or abs(int(length)-tags['duration_ms'])>2500:
|
|
60
|
+
continue
|
|
61
|
+
if normalized(row.get('title') or recording.get('title',''))!=normalized(tags.get('title','')):
|
|
62
|
+
continue
|
|
63
|
+
if tags.get('recording_mbid') and tags['recording_mbid'].casefold()!=str(recording.get('id','')).casefold():
|
|
64
|
+
continue
|
|
65
|
+
if tags.get('release_track_mbid') and tags['release_track_mbid'].casefold()!=str(row.get('id','')).casefold():
|
|
66
|
+
continue
|
|
67
|
+
matches.append(dict(id=row.get('id'),recording_id=recording.get('id'),title=row.get('title') or recording.get('title'),
|
|
68
|
+
length=length,disc=medium['position'],number=str(row['number'])))
|
|
69
|
+
if len(matches)!=1:
|
|
70
|
+
raise ValueError('no unique exact-edition track/position/title/duration match')
|
|
71
|
+
result=dict(kind='musicbrainz-exact-release',release_id=release_id,group_id=group_id,
|
|
72
|
+
release_artists=release_artists,group_artists=group_artists,track=matches[0],
|
|
73
|
+
release_date=payload.get('date') or '',original_date=group.get('first-release-date') or '',
|
|
74
|
+
release_title=payload.get('title'),source='https://musicbrainz.org/release/'+release_id)
|
|
75
|
+
return result
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def choose_targets(current, evidence, fill_missing=False):
|
|
79
|
+
targets, unresolved={},[]
|
|
80
|
+
for kind in ('release_date','original_date'):
|
|
81
|
+
existing=[v for key,vv in current['dates'].items() if family(key)==kind for v in vv]
|
|
82
|
+
invalid=[v for v in existing if dates.date_problem(v)]
|
|
83
|
+
if not invalid and (existing or not fill_missing):
|
|
84
|
+
continue
|
|
85
|
+
try:
|
|
86
|
+
source=partial_date(evidence[kind])
|
|
87
|
+
except ValueError:
|
|
88
|
+
unresolved.append(kind+': no valid exact catalog date; no replacement invented')
|
|
89
|
+
continue
|
|
90
|
+
blocked=False
|
|
91
|
+
for value in existing:
|
|
92
|
+
problem=dates.date_problem(value)
|
|
93
|
+
if problem and problem not in ('invalid placeholder year','invalid calendar date'):
|
|
94
|
+
unresolved.append(kind+': ambiguous existing date requires review');blocked=True;break
|
|
95
|
+
if not problem:
|
|
96
|
+
try:
|
|
97
|
+
valid=partial_date(value)
|
|
98
|
+
except ValueError:
|
|
99
|
+
unresolved.append(kind+': existing timestamp/precision requires review');blocked=True;break
|
|
100
|
+
common=min(len(source),len(valid))
|
|
101
|
+
if source[:common]!=valid[:common]:
|
|
102
|
+
unresolved.append(kind+': valid existing date conflicts with the catalog');blocked=True;break
|
|
103
|
+
if not blocked:
|
|
104
|
+
targets[kind]=evidence[kind]
|
|
105
|
+
return targets,unresolved
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def make_plan(root, report, output, client=None, fill_missing=False, apple_review=False, country='us', apple_client=None):
|
|
109
|
+
client=client or MusicBrainzClient()
|
|
110
|
+
if apple_review:
|
|
111
|
+
from .provider_review import AppleReviewClient,review
|
|
112
|
+
apple_client=apple_client or AppleReviewClient(country)
|
|
113
|
+
rows=[json.loads(line) for line in report.read_text(encoding='utf8').splitlines()]
|
|
114
|
+
if not rows or rows[0].get('type')!='start' or rows[-1].get('type')!='end' or Path(rows[0]['root']).resolve()!=root.resolve():
|
|
115
|
+
raise ValueError('date audit is incomplete or belongs to a different root')
|
|
116
|
+
entries,unresolved,cache,providers=[],[],{},{}
|
|
117
|
+
for row in rows:
|
|
118
|
+
if row.get('type')!='file':
|
|
119
|
+
continue
|
|
120
|
+
try:
|
|
121
|
+
path=dates.contained(root,row['path'])
|
|
122
|
+
current=dates.inspect(path,root)
|
|
123
|
+
if current['dates']!=row['dates']:
|
|
124
|
+
raise ValueError('date tags changed since audit')
|
|
125
|
+
tags=identify(path)
|
|
126
|
+
release=tags.get('release_mbid','').casefold()
|
|
127
|
+
if not release or not tags.get('group_mbid'):
|
|
128
|
+
raise ValueError('exact release and release-group tags are required for date lookup')
|
|
129
|
+
if release not in cache:
|
|
130
|
+
print('Looking up exact release '+release,flush=True)
|
|
131
|
+
cache[release]=client.lookup_release(release)
|
|
132
|
+
evidence=verified_evidence(cache[release],tags)
|
|
133
|
+
if apple_review and release not in providers:
|
|
134
|
+
print('Reviewing Apple storefront context for '+release,flush=True)
|
|
135
|
+
providers[release]=review(cache[release],apple_client)
|
|
136
|
+
targets,notes=choose_targets(current,evidence,fill_missing)
|
|
137
|
+
unresolved.extend(dict(path=row['path'],reason=note) for note in notes)
|
|
138
|
+
if targets or current['remove_keys']:
|
|
139
|
+
entries.append(dict(path=row['path'],sha256=dates.digest(path),dates=current['dates'],
|
|
140
|
+
remove_keys=current['remove_keys'],targets=targets,identifiers=tags,evidence=evidence))
|
|
141
|
+
elif current['stale_folder']:
|
|
142
|
+
unresolved.append(dict(path=row['path'],reason='embedded dates need no verified change; folder rename is a separate operation'))
|
|
143
|
+
except (ValueError,OSError,MutagenError,requests.RequestException) as exc:
|
|
144
|
+
unresolved.append(dict(path=row['path'],reason=str(exc) if not isinstance(exc,requests.RequestException) else 'catalog lookup failed: '+type(exc).__name__))
|
|
145
|
+
payload=dict(schema=2,action='repair-dates-from-evidence',root=str(root.resolve()),created=dates.utc(),
|
|
146
|
+
entries=entries,unresolved=unresolved,provider_reviews=list(providers.values()),
|
|
147
|
+
note='Reviewed date-only edits. Edition and original dates remain distinct; unknown precision is never expanded.')
|
|
148
|
+
with output.open('x',encoding='utf8') as stream:
|
|
149
|
+
json.dump(payload,stream,indent=2,ensure_ascii=False)
|
|
150
|
+
write_html(payload,output.with_suffix('.html'))
|
|
151
|
+
print(f'Date plan: {len(entries)} files, {len(unresolved)} unresolved notes; review {output}',flush=True)
|
|
152
|
+
return payload
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def write_html(payload,output):
|
|
156
|
+
escape=lambda value:html.escape(str(value),quote=True)
|
|
157
|
+
parts=['<!doctype html><meta charset="utf-8"><title>Date repair review</title>',
|
|
158
|
+
'<style>body{font:16px sans-serif;max-width:1100px;margin:2em auto;padding:1em}pre,code{white-space:pre-wrap;overflow-wrap:anywhere}section{border-top:1px solid #aaa;padding:1em 0}</style>',
|
|
159
|
+
'<h1>Date repair review</h1><p>No files were changed. Edition dates and original dates stay separate; missing precision is never invented.</p>']
|
|
160
|
+
for entry in payload['entries']:
|
|
161
|
+
parts.append('<section><h2>'+escape(entry['path'])+'</h2><p>Current date tags:</p><pre>'+escape(json.dumps(entry['dates'],indent=2,ensure_ascii=False))+'</pre>')
|
|
162
|
+
parts.append('<p>Reviewed replacement values:</p><pre>'+escape(json.dumps(entry['targets'],indent=2))+'</pre>')
|
|
163
|
+
parts.append('<p>Invalid placeholder aliases eligible for cleanup: '+escape(', '.join(entry['remove_keys']))+'</p>')
|
|
164
|
+
parts.append('<p><a href="'+escape(entry['evidence']['source'])+'">Exact MusicBrainz release evidence</a></p></section>')
|
|
165
|
+
parts.append('<h2>Unresolved review</h2><pre>'+escape(json.dumps(payload['unresolved'],indent=2,ensure_ascii=False))+'</pre>')
|
|
166
|
+
for review in payload.get('provider_reviews',[]):
|
|
167
|
+
parts.append('<section><h2>Storefront context: '+escape(review['artist']+' — '+review['title'])+'</h2><p>'+escape(review['note'])+'</p>')
|
|
168
|
+
for label,key in [('Review existing release','edit_release'),('Review existing release group','edit_group'),('Apple search','apple_search')]:
|
|
169
|
+
parts.append('<p><a href="'+escape(review[key])+'">'+label+'</a></p>')
|
|
170
|
+
for candidate in review['candidates']:
|
|
171
|
+
parts.append('<p><a href="'+escape(candidate['url'])+'">'+escape(candidate['title'])+'</a> — '+escape(candidate['date'])+'</p><pre>'+escape(json.dumps(candidate['track_comparison'],indent=2))+'</pre>')
|
|
172
|
+
parts.append('<p>'+escape(review['status'])+'</p></section>')
|
|
173
|
+
with output.open('x',encoding='utf8') as stream:
|
|
174
|
+
stream.write('\n'.join(parts))
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def validate_entry(entry, client, cache):
|
|
178
|
+
if entry.get('evidence',{}).get('kind')!='musicbrainz-exact-release':
|
|
179
|
+
raise ValueError('exact-release date evidence is required')
|
|
180
|
+
if not set(entry.get('targets',{})).issubset({'release_date','original_date'}):
|
|
181
|
+
raise ValueError('unknown date target field')
|
|
182
|
+
for kind,value in entry.get('targets',{}).items():
|
|
183
|
+
partial_date(value)
|
|
184
|
+
if value!=entry['evidence'].get(kind):
|
|
185
|
+
raise ValueError('replacement date differs from reviewed source precision/value')
|
|
186
|
+
allowed,_=choose_targets({'dates':entry['dates']},entry['evidence'],fill_missing=True)
|
|
187
|
+
if any(allowed.get(kind)!=value for kind,value in entry.get('targets',{}).items()):
|
|
188
|
+
raise ValueError('plan would overwrite a valid or conflicting existing date')
|
|
189
|
+
if entry.get('targets'):
|
|
190
|
+
release=entry['identifiers']['release_mbid'].casefold()
|
|
191
|
+
if release not in cache:
|
|
192
|
+
cache[release]=client.lookup_release(release)
|
|
193
|
+
if verified_evidence(cache[release],entry['identifiers'])!=entry['evidence']:
|
|
194
|
+
raise ValueError('exact catalog evidence changed since review; generate a fresh plan')
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def protected_metadata(path):
|
|
198
|
+
audio=File(path)
|
|
199
|
+
if audio is None:
|
|
200
|
+
raise ValueError('unreadable audio during date verification')
|
|
201
|
+
return hash_json(dict(tags={str(k):repr(v) for k,v in (audio.tags or {}).items() if not dates.date_key(k)},
|
|
202
|
+
pictures=[hashlib.sha256(p.write()).hexdigest() for p in getattr(audio,'pictures',[])],
|
|
203
|
+
id3_version=getattr(audio.tags,'version',None),
|
|
204
|
+
unknown_frames=[hashlib.sha256(frame).hexdigest() for frame in getattr(audio.tags,'unknown_frames',[])]))
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def semantic_dates(path):
|
|
208
|
+
try:
|
|
209
|
+
from mediafile import MediaFile
|
|
210
|
+
except ImportError:
|
|
211
|
+
raise ValueError('date replacement requires the tags extra: install navidrome-scripts[tags]') from None
|
|
212
|
+
audio=MediaFile(str(path))
|
|
213
|
+
result={}
|
|
214
|
+
for name,prefix in [('release_date',''),('original_date','original_')]:
|
|
215
|
+
year=getattr(audio,prefix+'year') or 0
|
|
216
|
+
month=getattr(audio,prefix+'month') or 0
|
|
217
|
+
day=getattr(audio,prefix+'day') or 0
|
|
218
|
+
if not year:
|
|
219
|
+
result[name]='';continue
|
|
220
|
+
result[name]=f'{year:04d}'+(f'-{month:02d}' if month else '')+(f'-{day:02d}' if month and day else '')
|
|
221
|
+
return result
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def write_dates(path, entry):
|
|
225
|
+
if entry.get('targets'):
|
|
226
|
+
try:
|
|
227
|
+
from mediafile import MediaFile
|
|
228
|
+
except ImportError:
|
|
229
|
+
raise ValueError('date replacement requires the tags extra: install navidrome-scripts[tags]') from None
|
|
230
|
+
raw=File(path)
|
|
231
|
+
original_version=getattr(raw.tags,'version',None)
|
|
232
|
+
audio=MediaFile(str(path),id3v23=bool(original_version and original_version[1]==3))
|
|
233
|
+
updates={}
|
|
234
|
+
for kind,date in entry['targets'].items():
|
|
235
|
+
parts=partial_date(date)
|
|
236
|
+
year,month,day=(*parts,0,0)[:3]
|
|
237
|
+
prefix='original_' if kind=='original_date' else ''
|
|
238
|
+
updates.update({prefix+'year':year,prefix+'month':month,prefix+'day':day})
|
|
239
|
+
audio.update(updates);audio.save()
|
|
240
|
+
raw=File(path)
|
|
241
|
+
changed=False
|
|
242
|
+
# Clear remaining legacy placeholder aliases, but never remove a valid field
|
|
243
|
+
# that the backend just replaced with the reviewed value.
|
|
244
|
+
for key in entry['remove_keys']:
|
|
245
|
+
if key in (raw.tags or {}) and all(dates.date_problem(v)=='invalid placeholder year' for v in dates.values(raw.tags[key])):
|
|
246
|
+
del raw.tags[key];changed=True
|
|
247
|
+
if changed:
|
|
248
|
+
raw.save()
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def verify_write(path, entry, protected):
|
|
252
|
+
if protected_metadata(path)!=protected:
|
|
253
|
+
raise ValueError('date writer changed unrelated metadata')
|
|
254
|
+
actual=dates.inspect(path,path.parent)['dates']
|
|
255
|
+
untouched={k:v for k,v in entry['dates'].items() if k not in entry['remove_keys'] and family(k) not in entry.get('targets',{})}
|
|
256
|
+
observed={k:v for k,v in actual.items() if family(k) not in entry.get('targets',{})}
|
|
257
|
+
if observed!=untouched:
|
|
258
|
+
raise ValueError('date writer changed an unapproved date field')
|
|
259
|
+
if entry.get('targets'):
|
|
260
|
+
semantic=semantic_dates(path)
|
|
261
|
+
for kind,value in entry['targets'].items():
|
|
262
|
+
if semantic[kind]!=value:
|
|
263
|
+
raise ValueError('date value/precision did not round-trip through the tag backend')
|
|
264
|
+
if any(dates.date_problem(v) for k,vv in actual.items() if family(k)==kind for v in vv):
|
|
265
|
+
raise ValueError('a conflicting invalid date alias remains')
|
|
@@ -0,0 +1,369 @@
|
|
|
1
|
+
"""Audit embedded dates and apply reviewed cleanup or exact-release replacements.
|
|
2
|
+
|
|
3
|
+
Never changes names or filesystem dates. Unknown dates remain unknown.
|
|
4
|
+
"""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import argparse
|
|
8
|
+
import calendar
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
import re
|
|
14
|
+
import shutil
|
|
15
|
+
import time
|
|
16
|
+
import tempfile
|
|
17
|
+
from datetime import datetime, timezone
|
|
18
|
+
|
|
19
|
+
from mutagen import File
|
|
20
|
+
from dotenv import load_dotenv
|
|
21
|
+
from .locking import library_lock,begin_tag_write,finish_tag_write
|
|
22
|
+
|
|
23
|
+
AUDIO = {'.flac', '.mp3', '.m4a', '.aac', '.ogg', '.oga', '.opus', '.wav', '.wma', '.aiff', '.ape', '.wv', '.dsf'}
|
|
24
|
+
DATE_KEYS = {'date', 'year', 'originaldate', 'originalyear', 'tdrc', 'tdor', 'tyer', 'tory', 'tdat', '©day',
|
|
25
|
+
'wm/year','wm/originalreleasetime','wm/originalreleaseyear'}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def utc():
|
|
29
|
+
return datetime.now(timezone.utc).isoformat()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def digest(path):
|
|
33
|
+
result = hashlib.sha256()
|
|
34
|
+
with path.open('rb') as stream:
|
|
35
|
+
for block in iter(lambda: stream.read(1024 * 1024), b''):
|
|
36
|
+
result.update(block)
|
|
37
|
+
return result.hexdigest()
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def contained(root, relative):
|
|
41
|
+
relative = Path(relative)
|
|
42
|
+
if relative.is_absolute() or '..' in relative.parts:
|
|
43
|
+
raise ValueError('relative library path required')
|
|
44
|
+
path = root / relative
|
|
45
|
+
if any((root / Path(*relative.parts[:i])).is_symlink() for i in range(1, len(relative.parts) + 1)) or not path.resolve().is_relative_to(root.resolve()):
|
|
46
|
+
raise ValueError('symlink or path outside library root')
|
|
47
|
+
return path
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def date_key(key):
|
|
51
|
+
key = str(key).casefold().removeprefix('txxx:').removeprefix('----:com.apple.itunes:')
|
|
52
|
+
return key.replace('_', '').replace(' ', '') in DATE_KEYS
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def values(value):
|
|
56
|
+
value = getattr(value, 'text', value)
|
|
57
|
+
if not isinstance(value, (tuple, list)):
|
|
58
|
+
value = [value]
|
|
59
|
+
return [v.decode('utf8', errors='replace') if isinstance(v, bytes) else str(v) for v in value]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def date_problem(value):
|
|
63
|
+
"""Retain year-only precision; never construct a full date from a year."""
|
|
64
|
+
value = value.strip()
|
|
65
|
+
if not value:
|
|
66
|
+
return ''
|
|
67
|
+
match = re.match(r'^(\d{1,4})(?:$|[-/T ])', value)
|
|
68
|
+
if not match:
|
|
69
|
+
return 'unrecognized date; review manually'
|
|
70
|
+
year = int(match[1])
|
|
71
|
+
if year <= 1:
|
|
72
|
+
return 'invalid placeholder year'
|
|
73
|
+
if year < 1000:
|
|
74
|
+
return 'unusual early year; review manually'
|
|
75
|
+
parts = re.fullmatch(r'(\d{4})(?:-(\d{2})(?:-(\d{2}))?)?', value)
|
|
76
|
+
if parts:
|
|
77
|
+
month, day = parts[2], parts[3]
|
|
78
|
+
if month and not 1 <= int(month) <= 12:
|
|
79
|
+
return 'invalid calendar date'
|
|
80
|
+
if day and not 1 <= int(day) <= calendar.monthrange(year, int(month))[1]:
|
|
81
|
+
return 'invalid calendar date'
|
|
82
|
+
else:
|
|
83
|
+
try:
|
|
84
|
+
datetime.fromisoformat(value.replace('Z','+00:00'))
|
|
85
|
+
except ValueError:
|
|
86
|
+
return 'unrecognized or invalid date; review manually'
|
|
87
|
+
return ''
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def inspect(path, root):
|
|
91
|
+
audio = File(path)
|
|
92
|
+
if audio is None:
|
|
93
|
+
raise ValueError('unreadable or unsupported audio')
|
|
94
|
+
tags = audio.tags or {}
|
|
95
|
+
dates = {str(k): values(v) for k, v in tags.items() if date_key(k)}
|
|
96
|
+
problems = {k: [dict(value=v, reason=date_problem(v)) for v in vv if date_problem(v)]
|
|
97
|
+
for k, vv in dates.items()}
|
|
98
|
+
problems = {k: v for k, v in problems.items() if v}
|
|
99
|
+
# Never erase a valid value sharing a multi-value date field.
|
|
100
|
+
removable = [k for k, vv in dates.items() if vv and all(
|
|
101
|
+
date_problem(v) == 'invalid placeholder year' for v in vv)]
|
|
102
|
+
return dict(path=path.relative_to(root).as_posix(), dates=dates, problems=problems,
|
|
103
|
+
remove_keys=removable,
|
|
104
|
+
stale_folder=any(re.search(r'\(0*1\)$', part) for part in (root.name, *path.relative_to(root).parts[:-1])))
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def audit(root, output):
|
|
108
|
+
root = root.resolve(strict=True)
|
|
109
|
+
if not root.is_dir():
|
|
110
|
+
raise ValueError('library root must be a directory')
|
|
111
|
+
started, clock = utc(), time.monotonic()
|
|
112
|
+
counts = dict(scanned=0, flagged=0, errors=0)
|
|
113
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
114
|
+
with output.open('x', encoding='utf8') as stream:
|
|
115
|
+
stream.write(json.dumps(dict(type='start', started=started, root=str(root))) + '\n')
|
|
116
|
+
def walk_error(error):
|
|
117
|
+
counts['errors'] += 1
|
|
118
|
+
stream.write(json.dumps(dict(type='error', path=str(error.filename), error=str(error))) + '\n')
|
|
119
|
+
for current, dirs, files in os.walk(root, onerror=walk_error):
|
|
120
|
+
for directory in dirs:
|
|
121
|
+
candidate = Path(current) / directory
|
|
122
|
+
if candidate.is_symlink():
|
|
123
|
+
counts['errors'] += 1
|
|
124
|
+
stream.write(json.dumps(dict(type='error', path=str(candidate), error='symlink directory not scanned')) + '\n')
|
|
125
|
+
dirs[:] = [d for d in dirs if not (Path(current) / d).is_symlink()]
|
|
126
|
+
for name in files:
|
|
127
|
+
path = Path(current) / name
|
|
128
|
+
if path.suffix.lower() not in AUDIO:
|
|
129
|
+
continue
|
|
130
|
+
counts['scanned'] += 1
|
|
131
|
+
try:
|
|
132
|
+
contained(root, path.relative_to(root))
|
|
133
|
+
row = inspect(path, root)
|
|
134
|
+
if row['problems'] or row['stale_folder']:
|
|
135
|
+
counts['flagged'] += 1
|
|
136
|
+
stream.write(json.dumps(dict(type='file', **row), ensure_ascii=False) + '\n')
|
|
137
|
+
except Exception as exc:
|
|
138
|
+
counts['errors'] += 1
|
|
139
|
+
stream.write(json.dumps(dict(type='error', path=str(path), error=str(exc))) + '\n')
|
|
140
|
+
if counts['scanned'] % 1000 == 0:
|
|
141
|
+
stream.flush()
|
|
142
|
+
print(json.dumps(counts), flush=True)
|
|
143
|
+
summary = dict(type='end', started=started, ended=utc(), elapsed_seconds=round(time.monotonic() - clock, 2), **counts)
|
|
144
|
+
stream.write(json.dumps(summary) + '\n')
|
|
145
|
+
print(json.dumps(summary), flush=True)
|
|
146
|
+
return 1 if counts['errors'] else 0
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def make_plan(root, report, output):
|
|
150
|
+
rows = [json.loads(line) for line in report.read_text(encoding='utf8').splitlines()]
|
|
151
|
+
if not rows or rows[0].get('type') != 'start' or rows[-1].get('type') != 'end':
|
|
152
|
+
raise ValueError('audit is incomplete; rerun it before planning')
|
|
153
|
+
if Path(rows[0]['root']).resolve() != root.resolve():
|
|
154
|
+
raise ValueError('audit library root differs')
|
|
155
|
+
entries = []
|
|
156
|
+
for row in rows:
|
|
157
|
+
if row.get('type') != 'file' or not row['remove_keys']:
|
|
158
|
+
continue
|
|
159
|
+
path = contained(root, row['path'])
|
|
160
|
+
current = inspect(path, root)
|
|
161
|
+
if current['dates'] != row['dates']:
|
|
162
|
+
raise ValueError('date tags changed since audit: ' + row['path'])
|
|
163
|
+
entries.append(dict(path=row['path'], sha256=digest(path), dates=row['dates'], remove_keys=row['remove_keys']))
|
|
164
|
+
payload = dict(schema=1, root=str(root.resolve()), created=utc(), action='remove-invalid-placeholder-dates',
|
|
165
|
+
note='Only invalid year-0/1 fields are removed. Valid dates and file paths are untouched.', entries=entries)
|
|
166
|
+
with output.open('x', encoding='utf8') as stream:
|
|
167
|
+
json.dump(payload, stream, indent=2, ensure_ascii=False)
|
|
168
|
+
print(f'Plan: {len(entries)} files; review {output} before apply', flush=True)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def apply_plan(root, plan_path, journal_path, evidence_client=None):
|
|
172
|
+
with library_lock(root):
|
|
173
|
+
return _apply_plan(root, plan_path, journal_path, evidence_client)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _apply_plan(root, plan_path, journal_path, evidence_client=None):
|
|
177
|
+
plan = json.loads(plan_path.read_text(encoding='utf8'))
|
|
178
|
+
evidence = plan.get('schema') == 2 and plan.get('action') == 'repair-dates-from-evidence'
|
|
179
|
+
if not (evidence or (plan.get('schema') == 1 and plan.get('action') == 'remove-invalid-placeholder-dates')) or Path(plan['root']).resolve() != root.resolve():
|
|
180
|
+
raise ValueError('wrong plan schema, action or root')
|
|
181
|
+
if evidence:
|
|
182
|
+
from . import date_evidence
|
|
183
|
+
from .catalog import MusicBrainzClient
|
|
184
|
+
evidence_client=evidence_client or MusicBrainzClient()
|
|
185
|
+
evidence_cache={}
|
|
186
|
+
# Full preflight before the first write; never silently follow a moved file.
|
|
187
|
+
entries = plan['entries']
|
|
188
|
+
if not entries:
|
|
189
|
+
print('No verified date changes selected; no files were changed.')
|
|
190
|
+
return
|
|
191
|
+
paths = [contained(root, e['path']) for e in entries]
|
|
192
|
+
if len(set(paths)) != len(paths):
|
|
193
|
+
raise ValueError('duplicate plan paths')
|
|
194
|
+
previous = {}
|
|
195
|
+
if journal_path.exists():
|
|
196
|
+
for line in journal_path.read_text(encoding='utf8').splitlines():
|
|
197
|
+
row = json.loads(line)
|
|
198
|
+
if row.get('plan_sha256') != digest(plan_path):
|
|
199
|
+
raise ValueError('journal belongs to a different plan')
|
|
200
|
+
previous[row['path']] = row
|
|
201
|
+
if not set(previous).issubset(e['path'] for e in entries):
|
|
202
|
+
raise ValueError('journal belongs to a different plan')
|
|
203
|
+
for entry, path in zip(entries, paths):
|
|
204
|
+
prior = previous.get(entry['path'])
|
|
205
|
+
if prior and prior['original_sha256'] != entry['sha256']:
|
|
206
|
+
raise ValueError('journal belongs to a different plan')
|
|
207
|
+
if prior and prior['state'] == 'applied':
|
|
208
|
+
if digest(path) != prior['updated_sha256']:
|
|
209
|
+
raise ValueError('file changed after repair: ' + entry['path'])
|
|
210
|
+
continue
|
|
211
|
+
current = inspect(path, root)
|
|
212
|
+
if digest(path) != entry['sha256'] or current['dates'] != entry['dates']:
|
|
213
|
+
raise ValueError('file changed since plan: ' + entry['path'])
|
|
214
|
+
if not (entry['remove_keys'] or (evidence and entry.get('targets'))) or not set(entry['remove_keys']).issubset(current['remove_keys']):
|
|
215
|
+
raise ValueError('plan attempts to remove a valid or unverified date')
|
|
216
|
+
if evidence:
|
|
217
|
+
date_evidence.validate_entry(entry,evidence_client,evidence_cache)
|
|
218
|
+
if path.stat().st_nlink > 1:
|
|
219
|
+
raise ValueError('hardlinked file requires separate review: ' + entry['path'])
|
|
220
|
+
backups = journal_path.with_suffix(journal_path.suffix + '.backups')
|
|
221
|
+
backups.mkdir(parents=True, exist_ok=True)
|
|
222
|
+
staged={}
|
|
223
|
+
# Validate all replacement formats on temporary copies before touching any
|
|
224
|
+
# original. The files and SQLite-free tag backend remain local to this run.
|
|
225
|
+
if evidence:
|
|
226
|
+
for index,(entry,path) in enumerate(zip(entries,paths)):
|
|
227
|
+
if previous.get(entry['path'],{}).get('state')=='applied':
|
|
228
|
+
continue
|
|
229
|
+
backup=backups/str(index)
|
|
230
|
+
if backup.exists():
|
|
231
|
+
if backup.is_symlink() or digest(backup)!=entry['sha256']:
|
|
232
|
+
raise ValueError('original backup differs; refusing to overwrite it')
|
|
233
|
+
else:
|
|
234
|
+
shutil.copy2(path,backup)
|
|
235
|
+
fd,name=tempfile.mkstemp(prefix='date-stage-',suffix=path.suffix,dir=backups)
|
|
236
|
+
os.close(fd);stage=Path(name);shutil.copy2(backup,stage)
|
|
237
|
+
protected=date_evidence.protected_metadata(backup)
|
|
238
|
+
date_evidence.write_dates(stage,entry)
|
|
239
|
+
date_evidence.verify_write(stage,entry,protected)
|
|
240
|
+
staged[entry['path']]=(stage,protected)
|
|
241
|
+
with journal_path.open('a' if journal_path.exists() else 'x', encoding='utf8') as journal:
|
|
242
|
+
for index, (entry, path) in enumerate(zip(entries, paths)):
|
|
243
|
+
if previous.get(entry['path'], {}).get('state') == 'applied':
|
|
244
|
+
print('ALREADY_APPLIED ' + entry['path'], flush=True)
|
|
245
|
+
continue
|
|
246
|
+
before = path.stat()
|
|
247
|
+
backup = backups / str(index)
|
|
248
|
+
if backup.exists():
|
|
249
|
+
if backup.is_symlink() or digest(backup)!=entry['sha256']:
|
|
250
|
+
raise ValueError('original backup differs; refusing to overwrite it')
|
|
251
|
+
else:
|
|
252
|
+
shutil.copy2(path, backup)
|
|
253
|
+
if digest(backup) != entry['sha256'] or digest(path) != entry['sha256']:
|
|
254
|
+
raise ValueError('file changed while backing up: ' + entry['path'])
|
|
255
|
+
record = dict(path=entry['path'], backup=str(backup.resolve()), original_sha256=entry['sha256'],
|
|
256
|
+
root=str(root.resolve()), plan_sha256=digest(plan_path),
|
|
257
|
+
mtime_ns=before.st_mtime_ns, atime_ns=before.st_atime_ns, state='prepared')
|
|
258
|
+
begin_tag_write(root,journal_path,'embedded dates')
|
|
259
|
+
journal.write(json.dumps(record) + '\n'); journal.flush(); os.fsync(journal.fileno())
|
|
260
|
+
try:
|
|
261
|
+
if evidence:
|
|
262
|
+
_,protected=staged[entry['path']]
|
|
263
|
+
date_evidence.write_dates(path,entry)
|
|
264
|
+
os.utime(path,ns=(before.st_atime_ns,before.st_mtime_ns))
|
|
265
|
+
date_evidence.verify_write(path,entry,protected)
|
|
266
|
+
record.update(state='applied',updated_sha256=digest(path))
|
|
267
|
+
journal.write(json.dumps(record)+'\n');journal.flush();os.fsync(journal.fileno())
|
|
268
|
+
try:
|
|
269
|
+
staged[entry['path']][0].unlink()
|
|
270
|
+
except OSError:
|
|
271
|
+
print('Temporary staged copy retained for review: '+str(staged[entry['path']][0]),flush=True)
|
|
272
|
+
print('APPLIED reviewed date fields '+entry['path'],flush=True)
|
|
273
|
+
continue
|
|
274
|
+
audio = File(path)
|
|
275
|
+
unrelated = {str(k): repr(v) for k, v in audio.tags.items() if str(k) not in entry['remove_keys']}
|
|
276
|
+
for key in entry['remove_keys']:
|
|
277
|
+
del audio.tags[key]
|
|
278
|
+
audio.save()
|
|
279
|
+
os.utime(path, ns=(before.st_atime_ns, before.st_mtime_ns))
|
|
280
|
+
after = inspect(path, root)
|
|
281
|
+
expected = {k: v for k, v in entry['dates'].items() if k not in entry['remove_keys']}
|
|
282
|
+
if after['dates'] != expected:
|
|
283
|
+
raise ValueError('date verification failed')
|
|
284
|
+
saved = File(path)
|
|
285
|
+
if {str(k): repr(v) for k, v in (saved.tags or {}).items()} != unrelated:
|
|
286
|
+
raise ValueError('unrelated metadata changed; restoring backup')
|
|
287
|
+
except Exception:
|
|
288
|
+
shutil.copy2(backup, path)
|
|
289
|
+
raise
|
|
290
|
+
record.update(state='applied', updated_sha256=digest(path))
|
|
291
|
+
journal.write(json.dumps(record) + '\n'); journal.flush(); os.fsync(journal.fileno())
|
|
292
|
+
print('APPLIED ' + entry['path'], flush=True)
|
|
293
|
+
finish_tag_write(root,journal_path)
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def rollback(root, journal_path):
|
|
297
|
+
with library_lock(root):
|
|
298
|
+
return _rollback(root, journal_path)
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _rollback(root, journal_path):
|
|
302
|
+
latest = {}
|
|
303
|
+
for line in journal_path.read_text(encoding='utf8').splitlines():
|
|
304
|
+
row = json.loads(line)
|
|
305
|
+
if Path(row['root']).resolve() != root.resolve():
|
|
306
|
+
raise ValueError('journal library root differs')
|
|
307
|
+
latest[row['path']] = row
|
|
308
|
+
for relative, row in latest.items():
|
|
309
|
+
path = contained(root, relative)
|
|
310
|
+
if digest(path) == row['original_sha256']:
|
|
311
|
+
continue # Already rolled back.
|
|
312
|
+
if row['state'] != 'applied':
|
|
313
|
+
raise ValueError('interrupted write requires manual backup review: ' + relative)
|
|
314
|
+
if digest(path) != row['updated_sha256'] or path.stat().st_nlink > 1:
|
|
315
|
+
raise ValueError('file changed after repair: ' + relative)
|
|
316
|
+
backup = Path(row['backup'])
|
|
317
|
+
if digest(backup) != row['original_sha256']:
|
|
318
|
+
raise ValueError('backup changed: ' + relative)
|
|
319
|
+
begin_tag_write(root,journal_path,'tag rollback')
|
|
320
|
+
shutil.copy2(backup, path)
|
|
321
|
+
os.utime(path, ns=(row['atime_ns'], row['mtime_ns']))
|
|
322
|
+
print('RESTORED ' + relative, flush=True)
|
|
323
|
+
if latest:
|
|
324
|
+
finish_tag_write(root,journal_path)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def main(argv=None):
|
|
328
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
329
|
+
parser.add_argument('--env-file',type=Path,default=Path('.env'))
|
|
330
|
+
parser.add_argument('--library-root', type=Path, required=True)
|
|
331
|
+
sub = parser.add_subparsers(dest='command', required=True)
|
|
332
|
+
scan = sub.add_parser('audit'); scan.add_argument('--output', type=Path, required=True)
|
|
333
|
+
plan = sub.add_parser('plan'); plan.add_argument('--audit', type=Path, required=True); plan.add_argument('--output', type=Path, required=True)
|
|
334
|
+
plan.add_argument('--lookup-dates',action='store_true',help='Use exact-release catalog evidence for invalid date replacements')
|
|
335
|
+
plan.add_argument('--fill-missing',action='store_true',help='Also propose missing date fields from verified evidence (requires --lookup-dates)')
|
|
336
|
+
plan.add_argument('--apple-review',action='store_true',help='Include read-only storefront date/track-list context; never uses it as write authority')
|
|
337
|
+
plan.add_argument('--country',default='us',help='Apple storefront country (default: us)')
|
|
338
|
+
apply = sub.add_parser('apply'); apply.add_argument('--plan', type=Path, required=True); apply.add_argument('--journal', type=Path, required=True)
|
|
339
|
+
apply.add_argument('--check-lidarr',action='store_true',help='Require Lidarr tag writing disabled and File Date=None before applying')
|
|
340
|
+
undo = sub.add_parser('rollback'); undo.add_argument('--journal', type=Path, required=True)
|
|
341
|
+
args = parser.parse_args(argv)
|
|
342
|
+
load_dotenv(args.env_file)
|
|
343
|
+
root = args.library_root.resolve(strict=True)
|
|
344
|
+
if args.command == 'audit':
|
|
345
|
+
return audit(root, args.output)
|
|
346
|
+
if args.command == 'plan':
|
|
347
|
+
if args.fill_missing and not args.lookup_dates:
|
|
348
|
+
parser.error('--fill-missing requires --lookup-dates')
|
|
349
|
+
if args.apple_review and not args.lookup_dates:
|
|
350
|
+
parser.error('--apple-review requires --lookup-dates')
|
|
351
|
+
if args.lookup_dates:
|
|
352
|
+
from .date_evidence import make_plan as evidence_plan
|
|
353
|
+
evidence_plan(root,args.audit,args.output,fill_missing=args.fill_missing,apple_review=args.apple_review,country=args.country)
|
|
354
|
+
else:
|
|
355
|
+
make_plan(root, args.audit, args.output)
|
|
356
|
+
elif args.command == 'apply':
|
|
357
|
+
if args.check_lidarr:
|
|
358
|
+
from .lidarr import from_env
|
|
359
|
+
client=from_env()
|
|
360
|
+
if str(client.get('config/metadataprovider').get('writeAudioTags','')).casefold()!='no' or str(client.get('config/mediamanagement').get('fileDate','')).casefold()!='none':
|
|
361
|
+
parser.error('set Lidarr Tag Audio Files=Never and File Date=None before date repair')
|
|
362
|
+
apply_plan(root, args.plan, args.journal)
|
|
363
|
+
else:
|
|
364
|
+
rollback(root, args.journal)
|
|
365
|
+
return 0
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
if __name__ == '__main__':
|
|
369
|
+
raise SystemExit(main())
|