malojaserver 3.2.2__py3-none-any.whl → 3.2.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- maloja/__main__.py +1 -94
- maloja/__pkginfo__.py +1 -1
- maloja/apis/_base.py +26 -19
- maloja/apis/_exceptions.py +1 -1
- maloja/apis/audioscrobbler.py +35 -7
- maloja/apis/audioscrobbler_legacy.py +5 -5
- maloja/apis/listenbrainz.py +8 -5
- maloja/apis/native_v1.py +43 -26
- maloja/cleanup.py +9 -7
- maloja/data_files/config/rules/predefined/krateng_kpopgirlgroups.tsv +2 -2
- maloja/database/__init__.py +68 -23
- maloja/database/associated.py +10 -6
- maloja/database/exceptions.py +28 -3
- maloja/database/sqldb.py +216 -168
- maloja/dev/profiler.py +3 -4
- maloja/images.py +6 -0
- maloja/malojauri.py +2 -0
- maloja/pkg_global/conf.py +30 -28
- maloja/proccontrol/tasks/export.py +2 -1
- maloja/proccontrol/tasks/import_scrobbles.py +110 -47
- maloja/server.py +11 -10
- maloja/setup.py +29 -17
- maloja/web/jinja/abstracts/base.jinja +1 -1
- maloja/web/jinja/admin_albumless.jinja +2 -0
- maloja/web/jinja/admin_issues.jinja +2 -2
- maloja/web/jinja/admin_overview.jinja +3 -3
- maloja/web/jinja/admin_setup.jinja +3 -3
- maloja/web/jinja/partials/album_showcase.jinja +1 -1
- maloja/web/jinja/snippets/entityrow.jinja +2 -2
- maloja/web/jinja/snippets/links.jinja +3 -1
- maloja/web/static/css/maloja.css +8 -2
- maloja/web/static/css/startpage.css +2 -2
- maloja/web/static/js/manualscrobble.js +2 -2
- maloja/web/static/js/notifications.js +16 -8
- maloja/web/static/js/search.js +18 -12
- maloja/web/static/js/upload.js +1 -1
- {malojaserver-3.2.2.dist-info → malojaserver-3.2.4.dist-info}/METADATA +24 -72
- {malojaserver-3.2.2.dist-info → malojaserver-3.2.4.dist-info}/RECORD +42 -42
- {malojaserver-3.2.2.dist-info → malojaserver-3.2.4.dist-info}/WHEEL +1 -1
- /maloja/data_files/state/{scrobbles → import}/dummy +0 -0
- {malojaserver-3.2.2.dist-info → malojaserver-3.2.4.dist-info}/LICENSE +0 -0
- {malojaserver-3.2.2.dist-info → malojaserver-3.2.4.dist-info}/entry_points.txt +0 -0
maloja/dev/profiler.py
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
import os
|
2
2
|
|
3
3
|
import cProfile, pstats
|
4
|
+
import time
|
4
5
|
|
5
6
|
from doreah.logging import log
|
6
|
-
from doreah.timing import Clock
|
7
7
|
|
8
8
|
from ..pkg_global.conf import data_dir
|
9
9
|
|
@@ -27,8 +27,7 @@ def profile(func):
|
|
27
27
|
|
28
28
|
def newfunc(*args,**kwargs):
|
29
29
|
|
30
|
-
|
31
|
-
clock.start()
|
30
|
+
starttime = time.time()
|
32
31
|
|
33
32
|
if FULL_PROFILE:
|
34
33
|
benchmarkfolder = data_dir['logs']("benchmarks")
|
@@ -44,7 +43,7 @@ def profile(func):
|
|
44
43
|
if FULL_PROFILE:
|
45
44
|
localprofiler.disable()
|
46
45
|
|
47
|
-
seconds =
|
46
|
+
seconds = time.time() - starttime
|
48
47
|
|
49
48
|
if not SINGLE_CALLS:
|
50
49
|
times.setdefault(realfunc,[]).append(seconds)
|
maloja/images.py
CHANGED
@@ -284,6 +284,12 @@ def image_request(artist_id=None,track_id=None,album_id=None):
|
|
284
284
|
if result is not None:
|
285
285
|
# we got an entry, even if it's that there is no image (value None)
|
286
286
|
if result['value'] is None:
|
287
|
+
# fallback to album regardless of setting (because we have no image)
|
288
|
+
if track_id:
|
289
|
+
track = database.sqldb.get_track(track_id)
|
290
|
+
if track.get("album"):
|
291
|
+
album_id = database.sqldb.get_album_id(track["album"])
|
292
|
+
return image_request(album_id=album_id)
|
287
293
|
# use placeholder
|
288
294
|
if malojaconfig["FANCY_PLACEHOLDER_ART"]:
|
289
295
|
placeholder_url = "https://generative-placeholders.glitch.me/image?width=300&height=300&style="
|
maloja/malojauri.py
CHANGED
@@ -29,6 +29,8 @@ def uri_to_internal(keys,accepted_entities=('artist','track','album'),forceTrack
|
|
29
29
|
|
30
30
|
# 1
|
31
31
|
filterkeys = {}
|
32
|
+
# this only takes care of the logic - what kind of entity we're dealing with
|
33
|
+
# it does not check with the database if it exists or what the canonical name is!!!
|
32
34
|
if "track" in accepted_entities and "title" in keys:
|
33
35
|
filterkeys.update({"track":{"artists":keys.getall("trackartist"),"title":keys.get("title")}})
|
34
36
|
if "artist" in accepted_entities and "artist" in keys:
|
maloja/pkg_global/conf.py
CHANGED
@@ -1,4 +1,7 @@
|
|
1
1
|
import os
|
2
|
+
|
3
|
+
import doreah.auth
|
4
|
+
import doreah.logging
|
2
5
|
from doreah.configuration import Configuration
|
3
6
|
from doreah.configuration import types as tp
|
4
7
|
|
@@ -29,9 +32,7 @@ pthj = os.path.join
|
|
29
32
|
def is_dir_usable(pth):
|
30
33
|
try:
|
31
34
|
os.makedirs(pth,exist_ok=True)
|
32
|
-
os.
|
33
|
-
os.remove(pthj(pth,".test"))
|
34
|
-
return True
|
35
|
+
return os.access(pth,os.W_OK)
|
35
36
|
except Exception:
|
36
37
|
return False
|
37
38
|
|
@@ -179,7 +180,7 @@ malojaconfig = Configuration(
|
|
179
180
|
"name":(tp.String(), "Name", "Generic Maloja User")
|
180
181
|
},
|
181
182
|
"Third Party Services":{
|
182
|
-
"metadata_providers":(tp.List(tp.String()), "Metadata Providers", ['lastfm','spotify','deezer','audiodb','musicbrainz'], "
|
183
|
+
"metadata_providers":(tp.List(tp.String()), "Metadata Providers", ['lastfm','spotify','deezer','audiodb','musicbrainz'], "List of which metadata providers should be used in what order. Musicbrainz is rate-limited and should not be used first."),
|
183
184
|
"scrobble_lastfm":(tp.Boolean(), "Proxy-Scrobble to Last.fm", False),
|
184
185
|
"lastfm_api_key":(tp.String(), "Last.fm API Key", None),
|
185
186
|
"lastfm_api_secret":(tp.String(), "Last.fm API Secret", None),
|
@@ -297,6 +298,7 @@ data_directories = {
|
|
297
298
|
"auth":pthj(dir_settings['state'],"auth"),
|
298
299
|
"backups":pthj(dir_settings['state'],"backups"),
|
299
300
|
"images":pthj(dir_settings['state'],"images"),
|
301
|
+
"import":pthj(dir_settings['state'],"import"),
|
300
302
|
"scrobbles":pthj(dir_settings['state']),
|
301
303
|
"rules":pthj(dir_settings['config'],"rules"),
|
302
304
|
"clients":pthj(dir_settings['config']),
|
@@ -310,6 +312,12 @@ data_directories = {
|
|
310
312
|
}
|
311
313
|
|
312
314
|
for identifier,path in data_directories.items():
|
315
|
+
if path is None:
|
316
|
+
continue
|
317
|
+
|
318
|
+
if malojaconfig.readonly and (path == dir_settings['config'] or path.startswith(dir_settings['config']+'/')):
|
319
|
+
continue
|
320
|
+
|
313
321
|
try:
|
314
322
|
os.makedirs(path,exist_ok=True)
|
315
323
|
if not is_dir_usable(path): raise PermissionError(f"Directory {path} is not usable!")
|
@@ -320,41 +328,35 @@ for identifier,path in data_directories.items():
|
|
320
328
|
print("Cannot use",path,"for cache, finding new folder...")
|
321
329
|
data_directories['cache'] = dir_settings['cache'] = malojaconfig['DIRECTORY_CACHE'] = find_good_folder('cache')
|
322
330
|
else:
|
323
|
-
print("Directory
|
331
|
+
print(f"Directory for {identifier} ({path}) is not writeable.")
|
324
332
|
print("Please change permissions or settings!")
|
333
|
+
print("Make sure Maloja has write and execute access to this directory.")
|
325
334
|
raise
|
326
335
|
|
336
|
+
class DataDirs:
|
337
|
+
def __init__(self, dirs):
|
338
|
+
self.dirs = dirs
|
327
339
|
|
328
|
-
|
329
|
-
|
330
|
-
}
|
331
|
-
|
332
|
-
|
340
|
+
def __getitem__(self, key):
|
341
|
+
return lambda *x, k=key: pthj(self.dirs[k], *x)
|
333
342
|
|
334
|
-
|
343
|
+
data_dir = DataDirs(data_directories)
|
335
344
|
|
336
|
-
|
337
|
-
|
338
|
-
config(
|
339
|
-
auth={
|
340
|
-
"multiuser":False,
|
341
|
-
"cookieprefix":"maloja",
|
342
|
-
"stylesheets":["/maloja.css"],
|
343
|
-
"dbfile":data_dir['auth']("auth.ddb")
|
344
|
-
},
|
345
|
-
logging={
|
346
|
-
"logfolder": data_dir['logs']() if malojaconfig["LOGGING"] else None
|
347
|
-
},
|
348
|
-
regular={
|
349
|
-
"offset": malojaconfig["TIMEZONE"]
|
350
|
-
}
|
351
|
-
)
|
345
|
+
### DOREAH OBJECTS
|
352
346
|
|
347
|
+
auth = doreah.auth.AuthManager(singleuser=True,cookieprefix='maloja',stylesheets=("/maloja.css",),dbfile=data_dir['auth']("auth.sqlite"))
|
353
348
|
|
349
|
+
#logger = doreah.logging.Logger(logfolder=data_dir['logs']() if malojaconfig["LOGGING"] else None)
|
350
|
+
#log = logger.log
|
354
351
|
|
352
|
+
# this is not how its supposed to be done, but lets ease the transition
|
353
|
+
doreah.logging.defaultlogger.logfolder = data_dir['logs']() if malojaconfig["LOGGING"] else None
|
355
354
|
|
356
355
|
|
357
|
-
|
356
|
+
try:
|
357
|
+
custom_css_files = [f for f in os.listdir(data_dir['css']()) if f.lower().endswith('.css')]
|
358
|
+
except FileNotFoundError:
|
359
|
+
custom_css_files = []
|
358
360
|
|
359
361
|
from ..database.sqldb import set_maloja_info
|
360
362
|
set_maloja_info({'last_run_version':VERSION})
|
@@ -12,11 +12,12 @@ def export(targetfolder=None):
|
|
12
12
|
targetfolder = os.getcwd()
|
13
13
|
|
14
14
|
timestr = time.strftime("%Y_%m_%d_%H_%M_%S")
|
15
|
+
timestamp = int(time.time()) # ok this is technically a separate time get from above, but those ms are not gonna matter, and im too lazy to change it all to datetime
|
15
16
|
filename = f"maloja_export_{timestr}.json"
|
16
17
|
outputfile = os.path.join(targetfolder,filename)
|
17
18
|
assert not os.path.exists(outputfile)
|
18
19
|
|
19
|
-
data = {'scrobbles':get_scrobbles()}
|
20
|
+
data = {'maloja':{'export_time': timestamp },'scrobbles':get_scrobbles()}
|
20
21
|
with open(outputfile,'w') as outfd:
|
21
22
|
json.dump(data,outfd,indent=3)
|
22
23
|
|
@@ -32,43 +32,62 @@ def import_scrobbles(inputf):
|
|
32
32
|
}
|
33
33
|
|
34
34
|
filename = os.path.basename(inputf)
|
35
|
+
importfunc = None
|
35
36
|
|
36
|
-
if re.match(r"
|
37
|
-
typeid,typedesc = "lastfm","Last.fm"
|
37
|
+
if re.match(r"recenttracks-.*\.csv", filename):
|
38
|
+
typeid, typedesc = "lastfm", "Last.fm (ghan CSV)"
|
39
|
+
importfunc = parse_lastfm_ghan_csv
|
40
|
+
|
41
|
+
elif re.match(r".*\.csv", filename):
|
42
|
+
typeid,typedesc = "lastfm", "Last.fm (benjaminbenben CSV)"
|
38
43
|
importfunc = parse_lastfm
|
39
44
|
|
40
|
-
elif re.match(r"Streaming_History_Audio.+\.json",filename):
|
41
|
-
typeid,typedesc = "spotify","Spotify"
|
45
|
+
elif re.match(r"Streaming_History_Audio.+\.json", filename):
|
46
|
+
typeid,typedesc = "spotify", "Spotify"
|
42
47
|
importfunc = parse_spotify_lite
|
43
48
|
|
44
|
-
elif re.match(r"endsong_[0-9]+\.json",filename):
|
45
|
-
typeid,typedesc = "spotify","Spotify"
|
49
|
+
elif re.match(r"endsong_[0-9]+\.json", filename):
|
50
|
+
typeid,typedesc = "spotify", "Spotify"
|
46
51
|
importfunc = parse_spotify
|
47
52
|
|
48
|
-
elif re.match(r"StreamingHistory[0-9]+\.json",filename):
|
49
|
-
typeid,typedesc = "spotify","Spotify"
|
53
|
+
elif re.match(r"StreamingHistory[0-9]+\.json", filename):
|
54
|
+
typeid,typedesc = "spotify", "Spotify"
|
50
55
|
importfunc = parse_spotify_lite_legacy
|
51
56
|
|
52
|
-
elif re.match(r"maloja_export[_0-9]*\.json",filename):
|
53
|
-
typeid,typedesc = "maloja","Maloja"
|
57
|
+
elif re.match(r"maloja_export[_0-9]*\.json", filename):
|
58
|
+
typeid,typedesc = "maloja", "Maloja"
|
54
59
|
importfunc = parse_maloja
|
55
60
|
|
56
61
|
# username_lb-YYYY-MM-DD.json
|
57
|
-
elif re.match(r".*_lb-[0-9-]+\.json",filename):
|
58
|
-
typeid,typedesc = "listenbrainz","ListenBrainz"
|
62
|
+
elif re.match(r".*_lb-[0-9-]+\.json", filename):
|
63
|
+
typeid,typedesc = "listenbrainz", "ListenBrainz"
|
59
64
|
importfunc = parse_listenbrainz
|
60
65
|
|
61
|
-
elif re.match(r"\.scrobbler\.log",filename):
|
62
|
-
typeid,typedesc = "rockbox","Rockbox"
|
66
|
+
elif re.match(r"\.scrobbler\.log", filename):
|
67
|
+
typeid,typedesc = "rockbox", "Rockbox"
|
63
68
|
importfunc = parse_rockbox
|
64
69
|
|
65
|
-
|
70
|
+
elif re.match(r"recenttracks-.*\.json", filename):
|
71
|
+
typeid, typedesc = "lastfm", "Last.fm (ghan JSON)"
|
72
|
+
importfunc = parse_lastfm_ghan_json
|
73
|
+
|
74
|
+
elif re.match(r".*\.json",filename):
|
75
|
+
try:
|
76
|
+
with open(filename,'r') as fd:
|
77
|
+
data = json.load(fd)
|
78
|
+
if 'maloja' in data:
|
79
|
+
typeid,typedesc = "maloja","Maloja"
|
80
|
+
importfunc = parse_maloja
|
81
|
+
except Exception:
|
82
|
+
pass
|
83
|
+
|
84
|
+
if not importfunc:
|
66
85
|
print("File",inputf,"could not be identified as a valid import source.")
|
67
86
|
return result
|
68
87
|
|
69
88
|
|
70
|
-
print(f"Parsing {col['yellow'](inputf)} as {col['cyan'](typedesc)} export")
|
71
|
-
print("
|
89
|
+
print(f"Parsing {col['yellow'](inputf)} as {col['cyan'](typedesc)} export.")
|
90
|
+
print(col['red']("Please double-check if this is correct - if the import fails, the file might have been interpreted as the wrong type."))
|
72
91
|
|
73
92
|
timestamps = set()
|
74
93
|
scrobblebuffer = []
|
@@ -131,27 +150,29 @@ def import_scrobbles(inputf):
|
|
131
150
|
|
132
151
|
return result
|
133
152
|
|
153
|
+
|
134
154
|
def parse_spotify_lite_legacy(inputf):
|
135
155
|
pth = os.path
|
136
156
|
# use absolute paths internally for peace of mind. just change representation for console output
|
137
157
|
inputf = pth.abspath(inputf)
|
138
158
|
inputfolder = pth.dirname(inputf)
|
139
159
|
filenames = re.compile(r'StreamingHistory[0-9]+\.json')
|
140
|
-
inputfiles = [os.path.join(inputfolder,f) for f in os.listdir(inputfolder) if filenames.match(f)]
|
160
|
+
#inputfiles = [os.path.join(inputfolder,f) for f in os.listdir(inputfolder) if filenames.match(f)]
|
161
|
+
inputfiles = [inputf]
|
141
162
|
|
142
|
-
if len(inputfiles) == 0:
|
143
|
-
|
144
|
-
|
163
|
+
#if len(inputfiles) == 0:
|
164
|
+
# print("No files found!")
|
165
|
+
# return
|
145
166
|
|
146
|
-
if inputfiles != [inputf]:
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
167
|
+
#if inputfiles != [inputf]:
|
168
|
+
# print("Spotify files should all be imported together to identify duplicates across the whole dataset.")
|
169
|
+
# if not ask("Import " + ", ".join(col['yellow'](pth.basename(i)) for i in inputfiles) + "?",default=True):
|
170
|
+
# inputfiles = [inputf]
|
171
|
+
# print("Only importing", col['yellow'](pth.basename(inputf)))
|
151
172
|
|
152
173
|
for inputf in inputfiles:
|
153
174
|
|
154
|
-
print("Importing",col['yellow'](inputf),"...")
|
175
|
+
#print("Importing",col['yellow'](inputf),"...")
|
155
176
|
with open(inputf,'r') as inputfd:
|
156
177
|
data = json.load(inputfd)
|
157
178
|
|
@@ -190,21 +211,22 @@ def parse_spotify_lite(inputf):
|
|
190
211
|
inputf = pth.abspath(inputf)
|
191
212
|
inputfolder = pth.dirname(inputf)
|
192
213
|
filenames = re.compile(r'Streaming_History_Audio.+\.json')
|
193
|
-
inputfiles = [os.path.join(inputfolder,f) for f in os.listdir(inputfolder) if filenames.match(f)]
|
214
|
+
#inputfiles = [os.path.join(inputfolder,f) for f in os.listdir(inputfolder) if filenames.match(f)]
|
215
|
+
inputfiles = [inputf]
|
194
216
|
|
195
|
-
if len(inputfiles) == 0:
|
196
|
-
|
197
|
-
|
217
|
+
#if len(inputfiles) == 0:
|
218
|
+
# print("No files found!")
|
219
|
+
# return
|
198
220
|
|
199
|
-
if inputfiles != [inputf]:
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
221
|
+
#if inputfiles != [inputf]:
|
222
|
+
# print("Spotify files should all be imported together to identify duplicates across the whole dataset.")
|
223
|
+
# if not ask("Import " + ", ".join(col['yellow'](pth.basename(i)) for i in inputfiles) + "?",default=True):
|
224
|
+
# inputfiles = [inputf]
|
225
|
+
# print("Only importing", col['yellow'](pth.basename(inputf)))
|
204
226
|
|
205
227
|
for inputf in inputfiles:
|
206
228
|
|
207
|
-
print("Importing",col['yellow'](inputf),"...")
|
229
|
+
#print("Importing",col['yellow'](inputf),"...")
|
208
230
|
with open(inputf,'r') as inputfd:
|
209
231
|
data = json.load(inputfd)
|
210
232
|
|
@@ -243,23 +265,25 @@ def parse_spotify_lite(inputf):
|
|
243
265
|
|
244
266
|
print()
|
245
267
|
|
268
|
+
|
246
269
|
def parse_spotify(inputf):
|
247
270
|
pth = os.path
|
248
271
|
# use absolute paths internally for peace of mind. just change representation for console output
|
249
272
|
inputf = pth.abspath(inputf)
|
250
273
|
inputfolder = pth.dirname(inputf)
|
251
274
|
filenames = re.compile(r'endsong_[0-9]+\.json')
|
252
|
-
inputfiles = [os.path.join(inputfolder,f) for f in os.listdir(inputfolder) if filenames.match(f)]
|
275
|
+
#inputfiles = [os.path.join(inputfolder,f) for f in os.listdir(inputfolder) if filenames.match(f)]
|
276
|
+
inputfiles = [inputf]
|
253
277
|
|
254
|
-
if len(inputfiles) == 0:
|
255
|
-
|
256
|
-
|
278
|
+
#if len(inputfiles) == 0:
|
279
|
+
# print("No files found!")
|
280
|
+
# return
|
257
281
|
|
258
|
-
if inputfiles != [inputf]:
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
282
|
+
#if inputfiles != [inputf]:
|
283
|
+
# print("Spotify files should all be imported together to identify duplicates across the whole dataset.")
|
284
|
+
# if not ask("Import " + ", ".join(col['yellow'](pth.basename(i)) for i in inputfiles) + "?",default=True):
|
285
|
+
# inputfiles = [inputf]
|
286
|
+
# print("Only importing", col['yellow'](pth.basename(inputf)))
|
263
287
|
|
264
288
|
# we keep timestamps here as well to remove duplicates because spotify's export
|
265
289
|
# is messy - this is specific to this import type and should not be mixed with
|
@@ -270,7 +294,7 @@ def parse_spotify(inputf):
|
|
270
294
|
|
271
295
|
for inputf in inputfiles:
|
272
296
|
|
273
|
-
print("Importing",col['yellow'](inputf),"...")
|
297
|
+
#print("Importing",col['yellow'](inputf),"...")
|
274
298
|
with open(inputf,'r') as inputfd:
|
275
299
|
data = json.load(inputfd)
|
276
300
|
|
@@ -354,6 +378,7 @@ def parse_spotify(inputf):
|
|
354
378
|
|
355
379
|
print()
|
356
380
|
|
381
|
+
|
357
382
|
def parse_lastfm(inputf):
|
358
383
|
|
359
384
|
with open(inputf,'r',newline='') as inputfd:
|
@@ -388,6 +413,44 @@ def parse_lastfm(inputf):
|
|
388
413
|
yield ('FAIL',None,f"{row} (Line {line}) could not be parsed. Scrobble not imported. ({repr(e)})")
|
389
414
|
continue
|
390
415
|
|
416
|
+
|
417
|
+
def parse_lastfm_ghan_json(inputf):
|
418
|
+
with open(inputf, 'r') as inputfd:
|
419
|
+
data = json.load(inputfd)
|
420
|
+
|
421
|
+
skip = 50000
|
422
|
+
for entry in data:
|
423
|
+
for track in entry['track']:
|
424
|
+
skip -= 1
|
425
|
+
#if skip: continue
|
426
|
+
#print(track)
|
427
|
+
#input()
|
428
|
+
|
429
|
+
yield ('CONFIDENT_IMPORT', {
|
430
|
+
'track_title': track['name'],
|
431
|
+
'track_artists': track['artist']['#text'],
|
432
|
+
'track_length': None,
|
433
|
+
'album_name': track['album']['#text'],
|
434
|
+
'scrobble_time': int(track['date']['uts']),
|
435
|
+
'scrobble_duration': None
|
436
|
+
}, '')
|
437
|
+
|
438
|
+
|
439
|
+
def parse_lastfm_ghan_csv(inputf):
|
440
|
+
with open(inputf, 'r') as inputfd:
|
441
|
+
reader = csv.DictReader(inputfd)
|
442
|
+
|
443
|
+
for row in reader:
|
444
|
+
yield ('CONFIDENT_IMPORT', {
|
445
|
+
'track_title': row['track'],
|
446
|
+
'track_artists': row['artist'],
|
447
|
+
'track_length': None,
|
448
|
+
'album_name': row['album'],
|
449
|
+
'scrobble_time': int(row['uts']),
|
450
|
+
'scrobble_duration': None
|
451
|
+
}, '')
|
452
|
+
|
453
|
+
|
391
454
|
def parse_listenbrainz(inputf):
|
392
455
|
|
393
456
|
with open(inputf,'r') as inputfd:
|
maloja/server.py
CHANGED
@@ -3,6 +3,7 @@ import os
|
|
3
3
|
from threading import Thread
|
4
4
|
from importlib import resources
|
5
5
|
import time
|
6
|
+
from magic import from_file
|
6
7
|
|
7
8
|
|
8
9
|
# server stuff
|
@@ -12,14 +13,13 @@ from jinja2.exceptions import TemplateNotFound
|
|
12
13
|
|
13
14
|
# doreah toolkit
|
14
15
|
from doreah.logging import log
|
15
|
-
from doreah import auth
|
16
16
|
|
17
17
|
# rest of the project
|
18
18
|
from . import database
|
19
19
|
from .database.jinjaview import JinjaDBConnection
|
20
20
|
from .images import image_request
|
21
21
|
from .malojauri import uri_to_internal, remove_identical
|
22
|
-
from .pkg_global.conf import malojaconfig, data_dir
|
22
|
+
from .pkg_global.conf import malojaconfig, data_dir, auth
|
23
23
|
from .pkg_global import conf
|
24
24
|
from .jinjaenv.context import jinja_environment
|
25
25
|
from .apis import init_apis, apikeystore
|
@@ -97,7 +97,7 @@ aliases = {
|
|
97
97
|
|
98
98
|
### API
|
99
99
|
|
100
|
-
auth.authapi.mount(server=webserver)
|
100
|
+
conf.auth.authapi.mount(server=webserver)
|
101
101
|
init_apis(webserver)
|
102
102
|
|
103
103
|
# redirects for backwards compatibility
|
@@ -155,7 +155,8 @@ def static_image(pth):
|
|
155
155
|
|
156
156
|
@webserver.route("/cacheimages/<uuid>")
|
157
157
|
def static_proxied_image(uuid):
|
158
|
-
|
158
|
+
mimetype = from_file(os.path.join(data_dir['cache']('images'),uuid),True)
|
159
|
+
return static_file(uuid,root=data_dir['cache']('images'),mimetype=mimetype)
|
159
160
|
|
160
161
|
@webserver.route("/login")
|
161
162
|
def login():
|
@@ -166,16 +167,16 @@ def login():
|
|
166
167
|
@webserver.route("/media/<name>.<ext>")
|
167
168
|
def static(name,ext):
|
168
169
|
assert ext in ["txt","ico","jpeg","jpg","png","less","js","ttf","css"]
|
169
|
-
|
170
|
-
|
170
|
+
staticfolder = resources.files('maloja') / 'web' / 'static'
|
171
|
+
response = static_file(ext + "/" + name + "." + ext,root=staticfolder)
|
171
172
|
response.set_header("Cache-Control", "public, max-age=3600")
|
172
173
|
return response
|
173
174
|
|
174
175
|
# new, direct reference
|
175
176
|
@webserver.route("/static/<path:path>")
|
176
177
|
def static(path):
|
177
|
-
|
178
|
-
|
178
|
+
staticfolder = resources.files('maloja') / 'web' / 'static'
|
179
|
+
response = static_file(path,root=staticfolder)
|
179
180
|
response.set_header("Cache-Control", "public, max-age=3600")
|
180
181
|
return response
|
181
182
|
|
@@ -197,7 +198,7 @@ def jinja_page(name):
|
|
197
198
|
if name in aliases: redirect(aliases[name])
|
198
199
|
keys = remove_identical(FormsDict.decode(request.query))
|
199
200
|
|
200
|
-
adminmode = request.cookies.get("adminmode") == "true" and auth.
|
201
|
+
adminmode = request.cookies.get("adminmode") == "true" and auth.check_request(request)
|
201
202
|
|
202
203
|
with JinjaDBConnection() as conn:
|
203
204
|
|
@@ -222,7 +223,7 @@ def jinja_page(name):
|
|
222
223
|
return res
|
223
224
|
|
224
225
|
@webserver.route("/<name:re:admin.*>")
|
225
|
-
@auth.
|
226
|
+
@auth.authenticated_function()
|
226
227
|
def jinja_page_private(name):
|
227
228
|
return jinja_page(name)
|
228
229
|
|
maloja/setup.py
CHANGED
@@ -1,14 +1,12 @@
|
|
1
1
|
import os
|
2
|
+
import shutil
|
2
3
|
|
3
4
|
from importlib import resources
|
4
|
-
|
5
|
-
|
6
|
-
except ImportError:
|
7
|
-
import distutils
|
5
|
+
from pathlib import PosixPath
|
6
|
+
|
8
7
|
from doreah.io import col, ask, prompt
|
9
|
-
from doreah import auth
|
10
8
|
|
11
|
-
from .pkg_global.conf import data_dir, dir_settings, malojaconfig
|
9
|
+
from .pkg_global.conf import data_dir, dir_settings, malojaconfig, auth
|
12
10
|
|
13
11
|
|
14
12
|
|
@@ -23,22 +21,33 @@ ext_apikeys = [
|
|
23
21
|
|
24
22
|
|
25
23
|
def copy_initial_local_files():
|
26
|
-
|
27
|
-
|
28
|
-
|
24
|
+
data_file_source = resources.files("maloja") / 'data_files'
|
25
|
+
for cat in dir_settings:
|
26
|
+
if dir_settings[cat] is None:
|
27
|
+
continue
|
28
|
+
if cat == 'config' and malojaconfig.readonly:
|
29
|
+
continue
|
30
|
+
|
31
|
+
# to avoid permission problems with the root dir
|
32
|
+
for subfolder in os.listdir(data_file_source / cat):
|
33
|
+
src = data_file_source / cat / subfolder
|
34
|
+
dst = PosixPath(dir_settings[cat]) / subfolder
|
35
|
+
if os.path.isdir(src):
|
36
|
+
shutil.copytree(src, dst, dirs_exist_ok=True)
|
37
|
+
|
29
38
|
|
30
39
|
charset = list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
|
31
40
|
def randomstring(length=32):
|
32
41
|
import random
|
33
42
|
return "".join(str(random.choice(charset)) for _ in range(length))
|
34
43
|
|
44
|
+
|
35
45
|
def setup():
|
36
46
|
|
37
47
|
copy_initial_local_files()
|
38
48
|
SKIP = malojaconfig["SKIP_SETUP"]
|
39
49
|
|
40
50
|
try:
|
41
|
-
|
42
51
|
print("Various external services can be used to display images. If not enough of them are set up, only local images will be used.")
|
43
52
|
for k in ext_apikeys:
|
44
53
|
keyname = malojaconfig.get_setting_info(k)['name']
|
@@ -46,9 +55,12 @@ def setup():
|
|
46
55
|
if key is False:
|
47
56
|
print(f"\tCurrently not using a {col['red'](keyname)} for image display.")
|
48
57
|
elif key is None or key == "ASK":
|
49
|
-
|
50
|
-
|
51
|
-
|
58
|
+
if malojaconfig.readonly:
|
59
|
+
print(f"\tCurrently not using a {col['red'](keyname)} for image display - config is read only.")
|
60
|
+
else:
|
61
|
+
promptmsg = f"\tPlease enter your {col['gold'](keyname)}. If you do not want to use one at this moment, simply leave this empty and press Enter."
|
62
|
+
key = prompt(promptmsg,types=(str,),default=False,skip=SKIP)
|
63
|
+
malojaconfig[k] = key
|
52
64
|
else:
|
53
65
|
print(f"\t{col['lawngreen'](keyname)} found.")
|
54
66
|
|
@@ -67,10 +79,10 @@ def setup():
|
|
67
79
|
|
68
80
|
if forcepassword is not None:
|
69
81
|
# user has specified to force the pw, nothing else matters
|
70
|
-
auth.
|
82
|
+
auth.change_pw(password=forcepassword)
|
71
83
|
print("Password has been set.")
|
72
|
-
elif auth.
|
73
|
-
#
|
84
|
+
elif auth.still_has_factory_default_user():
|
85
|
+
# this means we've never set this up properly (eg first start after update)
|
74
86
|
while True:
|
75
87
|
newpw = prompt("Please set a password for web backend access. Leave this empty to generate a random password.",skip=SKIP,secret=True)
|
76
88
|
if newpw is None:
|
@@ -81,7 +93,7 @@ def setup():
|
|
81
93
|
newpw_repeat = prompt("Please type again to confirm.",skip=SKIP,secret=True)
|
82
94
|
if newpw != newpw_repeat: print("Passwords do not match!")
|
83
95
|
else: break
|
84
|
-
auth.
|
96
|
+
auth.change_pw(password=newpw)
|
85
97
|
|
86
98
|
except EOFError:
|
87
99
|
print("No user input possible. If you are running inside a container, set the environment variable",col['yellow']("MALOJA_SKIP_SETUP=yes"))
|
@@ -75,7 +75,7 @@
|
|
75
75
|
<a href="/"><img style="display:block;" src="/favicon.png" /></a>
|
76
76
|
</div>
|
77
77
|
<div id="right-side">
|
78
|
-
<span><input id="searchinput" placeholder="Search for an artist or track..." oninput="search(this)" onblur="clearresults()" /></span>
|
78
|
+
<span><input id="searchinput" placeholder="Search for an album, artist or track..." oninput="search(this)" onblur="clearresults()" /></span>
|
79
79
|
</div>
|
80
80
|
|
81
81
|
|
@@ -15,7 +15,7 @@
|
|
15
15
|
|
16
16
|
|
17
17
|
var xhttp = new XMLHttpRequest();
|
18
|
-
xhttp.open("POST","/
|
18
|
+
xhttp.open("POST","/apis/mlj_1/newrule?", true);
|
19
19
|
xhttp.send(keys);
|
20
20
|
e = arguments[0];
|
21
21
|
line = e.parentNode;
|
@@ -25,7 +25,7 @@
|
|
25
25
|
function fullrebuild() {
|
26
26
|
|
27
27
|
var xhttp = new XMLHttpRequest();
|
28
|
-
xhttp.open("POST","/
|
28
|
+
xhttp.open("POST","/apis/mlj_1/rebuild", true);
|
29
29
|
xhttp.send();
|
30
30
|
window.location = "/wait";
|
31
31
|
|
@@ -67,9 +67,9 @@
|
|
67
67
|
<li>manually scrobble from track pages</li>
|
68
68
|
<li>delete scrobbles</li>
|
69
69
|
<li>reparse scrobbles</li>
|
70
|
-
<li>edit tracks and artists</li>
|
71
|
-
<li>merge tracks and artists</li>
|
72
|
-
<li>upload artist and track art by dropping a file on the existing image on an artist or track page</li>
|
70
|
+
<li>edit tracks, albums and artists</li>
|
71
|
+
<li>merge tracks, albums and artists</li>
|
72
|
+
<li>upload artist, album and track art by dropping a file on the existing image on an artist or track page</li>
|
73
73
|
<li>see more detailed error pages</li>
|
74
74
|
</ul>
|
75
75
|
|