malojaserver 3.2.1__py3-none-any.whl → 3.2.3__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- maloja/__main__.py +1 -1
- maloja/__pkginfo__.py +1 -1
- maloja/apis/_base.py +26 -19
- maloja/apis/_exceptions.py +1 -1
- maloja/apis/audioscrobbler.py +35 -7
- maloja/apis/audioscrobbler_legacy.py +5 -5
- maloja/apis/listenbrainz.py +7 -5
- maloja/apis/native_v1.py +43 -26
- maloja/cleanup.py +9 -7
- maloja/data_files/config/rules/predefined/krateng_kpopgirlgroups.tsv +4 -2
- maloja/database/__init__.py +55 -23
- maloja/database/associated.py +10 -6
- maloja/database/exceptions.py +28 -3
- maloja/database/sqldb.py +216 -168
- maloja/dev/profiler.py +3 -4
- maloja/images.py +6 -0
- maloja/malojauri.py +2 -0
- maloja/pkg_global/conf.py +97 -72
- maloja/proccontrol/tasks/export.py +2 -1
- maloja/proccontrol/tasks/import_scrobbles.py +57 -15
- maloja/server.py +4 -5
- maloja/setup.py +56 -44
- maloja/thirdparty/lastfm.py +18 -17
- maloja/web/jinja/abstracts/base.jinja +1 -1
- maloja/web/jinja/admin_albumless.jinja +2 -0
- maloja/web/jinja/admin_overview.jinja +3 -3
- maloja/web/jinja/admin_setup.jinja +1 -1
- maloja/web/jinja/error.jinja +2 -2
- maloja/web/jinja/partials/album_showcase.jinja +1 -1
- maloja/web/jinja/partials/awards_album.jinja +1 -1
- maloja/web/jinja/partials/awards_artist.jinja +2 -2
- maloja/web/jinja/partials/charts_albums_tiles.jinja +4 -0
- maloja/web/jinja/partials/charts_artists_tiles.jinja +5 -1
- maloja/web/jinja/partials/charts_tracks_tiles.jinja +4 -0
- maloja/web/jinja/snippets/entityrow.jinja +2 -2
- maloja/web/jinja/snippets/links.jinja +3 -1
- maloja/web/static/css/maloja.css +14 -2
- maloja/web/static/css/startpage.css +2 -2
- maloja/web/static/js/manualscrobble.js +1 -1
- maloja/web/static/js/notifications.js +16 -8
- {malojaserver-3.2.1.dist-info → malojaserver-3.2.3.dist-info}/METADATA +10 -46
- {malojaserver-3.2.1.dist-info → malojaserver-3.2.3.dist-info}/RECORD +45 -45
- {malojaserver-3.2.1.dist-info → malojaserver-3.2.3.dist-info}/WHEEL +1 -1
- {malojaserver-3.2.1.dist-info → malojaserver-3.2.3.dist-info}/LICENSE +0 -0
- {malojaserver-3.2.1.dist-info → malojaserver-3.2.3.dist-info}/entry_points.txt +0 -0
maloja/dev/profiler.py
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
import os
|
2
2
|
|
3
3
|
import cProfile, pstats
|
4
|
+
import time
|
4
5
|
|
5
6
|
from doreah.logging import log
|
6
|
-
from doreah.timing import Clock
|
7
7
|
|
8
8
|
from ..pkg_global.conf import data_dir
|
9
9
|
|
@@ -27,8 +27,7 @@ def profile(func):
|
|
27
27
|
|
28
28
|
def newfunc(*args,**kwargs):
|
29
29
|
|
30
|
-
|
31
|
-
clock.start()
|
30
|
+
starttime = time.time()
|
32
31
|
|
33
32
|
if FULL_PROFILE:
|
34
33
|
benchmarkfolder = data_dir['logs']("benchmarks")
|
@@ -44,7 +43,7 @@ def profile(func):
|
|
44
43
|
if FULL_PROFILE:
|
45
44
|
localprofiler.disable()
|
46
45
|
|
47
|
-
seconds =
|
46
|
+
seconds = time.time() - starttime
|
48
47
|
|
49
48
|
if not SINGLE_CALLS:
|
50
49
|
times.setdefault(realfunc,[]).append(seconds)
|
maloja/images.py
CHANGED
@@ -284,6 +284,12 @@ def image_request(artist_id=None,track_id=None,album_id=None):
|
|
284
284
|
if result is not None:
|
285
285
|
# we got an entry, even if it's that there is no image (value None)
|
286
286
|
if result['value'] is None:
|
287
|
+
# fallback to album regardless of setting (because we have no image)
|
288
|
+
if track_id:
|
289
|
+
track = database.sqldb.get_track(track_id)
|
290
|
+
if track.get("album"):
|
291
|
+
album_id = database.sqldb.get_album_id(track["album"])
|
292
|
+
return image_request(album_id=album_id)
|
287
293
|
# use placeholder
|
288
294
|
if malojaconfig["FANCY_PLACEHOLDER_ART"]:
|
289
295
|
placeholder_url = "https://generative-placeholders.glitch.me/image?width=300&height=300&style="
|
maloja/malojauri.py
CHANGED
@@ -29,6 +29,8 @@ def uri_to_internal(keys,accepted_entities=('artist','track','album'),forceTrack
|
|
29
29
|
|
30
30
|
# 1
|
31
31
|
filterkeys = {}
|
32
|
+
# this only takes care of the logic - what kind of entity we're dealing with
|
33
|
+
# it does not check with the database if it exists or what the canonical name is!!!
|
32
34
|
if "track" in accepted_entities and "title" in keys:
|
33
35
|
filterkeys.update({"track":{"artists":keys.getall("trackartist"),"title":keys.get("title")}})
|
34
36
|
if "artist" in accepted_entities and "artist" in keys:
|
maloja/pkg_global/conf.py
CHANGED
@@ -1,4 +1,7 @@
|
|
1
1
|
import os
|
2
|
+
|
3
|
+
import doreah.auth
|
4
|
+
import doreah.logging
|
2
5
|
from doreah.configuration import Configuration
|
3
6
|
from doreah.configuration import types as tp
|
4
7
|
|
@@ -17,9 +20,11 @@ AUX_MODE = True
|
|
17
20
|
# DIRECRORY_CONFIG, DIRECRORY_STATE, DIRECTORY_LOGS and DIRECTORY_CACHE
|
18
21
|
# config can only be determined by environment variable, the others can be loaded
|
19
22
|
# from the config files
|
20
|
-
# explicit settings will always be respected, fallback to default
|
21
23
|
|
22
|
-
#
|
24
|
+
# we don't specify 'default' values in the normal sense of the config object
|
25
|
+
# the default is none, meaning the app should figure it out (depending on environment)
|
26
|
+
# the actual 'default' values of our folders are simply in code since they are dependent on environment (container?)
|
27
|
+
# and we need to actually distinguish them from the user having specified something
|
23
28
|
|
24
29
|
# USEFUL FUNCS
|
25
30
|
pthj = os.path.join
|
@@ -27,9 +32,7 @@ pthj = os.path.join
|
|
27
32
|
def is_dir_usable(pth):
|
28
33
|
try:
|
29
34
|
os.makedirs(pth,exist_ok=True)
|
30
|
-
os.
|
31
|
-
os.remove(pthj(pth,".test"))
|
32
|
-
return True
|
35
|
+
return os.access(pth,os.W_OK)
|
33
36
|
except Exception:
|
34
37
|
return False
|
35
38
|
|
@@ -40,7 +43,10 @@ def get_env_vars(key,pathsuffix=[]):
|
|
40
43
|
|
41
44
|
directory_info = {
|
42
45
|
"config":{
|
43
|
-
"sentinel":"
|
46
|
+
"sentinel":".maloja_config_sentinel",
|
47
|
+
"possible_folders_container":[
|
48
|
+
"/config/config"
|
49
|
+
],
|
44
50
|
"possible_folders":[
|
45
51
|
"/etc/maloja",
|
46
52
|
os.path.expanduser("~/.local/share/maloja")
|
@@ -48,15 +54,22 @@ directory_info = {
|
|
48
54
|
"setting":"directory_config"
|
49
55
|
},
|
50
56
|
"cache":{
|
51
|
-
"sentinel":"
|
57
|
+
"sentinel":".maloja_cache_sentinel",
|
58
|
+
"possible_folders_container":[
|
59
|
+
"/config/cache"
|
60
|
+
],
|
52
61
|
"possible_folders":[
|
53
62
|
"/var/cache/maloja",
|
54
|
-
os.path.expanduser("~/.local/share/maloja/cache")
|
63
|
+
os.path.expanduser("~/.local/share/maloja/cache"),
|
64
|
+
"/tmp/maloja"
|
55
65
|
],
|
56
66
|
"setting":"directory_cache"
|
57
67
|
},
|
58
68
|
"state":{
|
59
|
-
"sentinel":"
|
69
|
+
"sentinel":".maloja_state_sentinel",
|
70
|
+
"possible_folders_container":[
|
71
|
+
"/config/state"
|
72
|
+
],
|
60
73
|
"possible_folders":[
|
61
74
|
"/var/lib/maloja",
|
62
75
|
os.path.expanduser("~/.local/share/maloja")
|
@@ -64,7 +77,10 @@ directory_info = {
|
|
64
77
|
"setting":"directory_state"
|
65
78
|
},
|
66
79
|
"logs":{
|
67
|
-
"sentinel":"
|
80
|
+
"sentinel":".maloja_logs_sentinel",
|
81
|
+
"possible_folders_container":[
|
82
|
+
"/config/logs"
|
83
|
+
],
|
68
84
|
"possible_folders":[
|
69
85
|
"/var/log/maloja",
|
70
86
|
os.path.expanduser("~/.local/share/maloja/logs")
|
@@ -77,51 +93,51 @@ directory_info = {
|
|
77
93
|
# checks if one has been in use before and writes it to dict/config
|
78
94
|
# if not, determines which to use and writes it to dict/config
|
79
95
|
# returns determined folder
|
80
|
-
def find_good_folder(datatype
|
96
|
+
def find_good_folder(datatype):
|
81
97
|
info = directory_info[datatype]
|
82
98
|
|
99
|
+
possible_folders = info['possible_folders']
|
100
|
+
if os.environ.get("MALOJA_CONTAINER"):
|
101
|
+
possible_folders = info['possible_folders_container'] + possible_folders
|
102
|
+
|
83
103
|
# check each possible folder if its used
|
84
|
-
for p in
|
104
|
+
for p in possible_folders:
|
85
105
|
if os.path.exists(pthj(p,info['sentinel'])):
|
86
|
-
|
87
|
-
|
88
|
-
|
106
|
+
if is_dir_usable(p):
|
107
|
+
#print(p,"was apparently used as maloja's folder for",datatype,"- fixing in settings")
|
108
|
+
return p
|
109
|
+
else:
|
110
|
+
raise PermissionError(f"Can no longer use previously used {datatype} folder {p}")
|
89
111
|
|
90
112
|
#print("Could not find previous",datatype,"folder")
|
91
113
|
# check which one we can use
|
92
|
-
for p in
|
114
|
+
for p in possible_folders:
|
93
115
|
if is_dir_usable(p):
|
94
116
|
#print(p,"has been selected as maloja's folder for",datatype)
|
95
|
-
configobject[info['setting']] = p
|
96
117
|
return p
|
97
118
|
#print("No folder can be used for",datatype)
|
98
119
|
#print("This should not happen!")
|
120
|
+
raise PermissionError(f"No folder could be found for {datatype}")
|
99
121
|
|
100
122
|
|
101
123
|
|
102
124
|
|
103
125
|
|
104
126
|
### STEP 1 - find out where the settings file is
|
105
|
-
# environment variables
|
106
|
-
maloja_dir_config = os.environ.get("MALOJA_DATA_DIRECTORY") or os.environ.get("MALOJA_DIRECTORY_CONFIG")
|
107
127
|
|
128
|
+
maloja_dir_config = os.environ.get("MALOJA_DATA_DIRECTORY") or os.environ.get("MALOJA_DIRECTORY_CONFIG")
|
108
129
|
|
109
130
|
if maloja_dir_config is None:
|
110
|
-
|
111
|
-
|
131
|
+
# if nothing is set, we set our own
|
132
|
+
maloja_dir_config = find_good_folder('config')
|
112
133
|
else:
|
113
|
-
|
114
|
-
#
|
134
|
+
pass
|
135
|
+
# if there is an environment variable, this is 100% explicitly defined by the user, so we respect it
|
136
|
+
# the user might run more than one instances on the same machine, so we don't do any heuristics here
|
137
|
+
# if you define this, we believe it!
|
115
138
|
|
116
139
|
os.makedirs(maloja_dir_config,exist_ok=True)
|
117
|
-
|
118
|
-
oldsettingsfile = pthj(maloja_dir_config,"settings","settings.ini")
|
119
|
-
newsettingsfile = pthj(maloja_dir_config,"settings.ini")
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
if os.path.exists(oldsettingsfile):
|
124
|
-
os.rename(oldsettingsfile,newsettingsfile)
|
140
|
+
settingsfile = pthj(maloja_dir_config,"settings.ini")
|
125
141
|
|
126
142
|
|
127
143
|
### STEP 2 - create settings object
|
@@ -131,10 +147,10 @@ malojaconfig = Configuration(
|
|
131
147
|
settings={
|
132
148
|
"Setup":{
|
133
149
|
"data_directory":(tp.String(), "Data Directory", None, "Folder for all user data. Overwrites all choices for specific directories."),
|
134
|
-
"directory_config":(tp.String(), "Config Directory",
|
135
|
-
"directory_state":(tp.String(), "State Directory",
|
136
|
-
"directory_logs":(tp.String(), "Log Directory",
|
137
|
-
"directory_cache":(tp.String(), "Cache Directory",
|
150
|
+
"directory_config":(tp.String(), "Config Directory", None, "Folder for config data. Only applied when global data directory is not set."),
|
151
|
+
"directory_state":(tp.String(), "State Directory", None, "Folder for state data. Only applied when global data directory is not set."),
|
152
|
+
"directory_logs":(tp.String(), "Log Directory", None, "Folder for log data. Only applied when global data directory is not set."),
|
153
|
+
"directory_cache":(tp.String(), "Cache Directory", None, "Folder for cache data. Only applied when global data directory is not set."),
|
138
154
|
"skip_setup":(tp.Boolean(), "Skip Setup", False, "Make server setup process non-interactive. Vital for Docker."),
|
139
155
|
"force_password":(tp.String(), "Force Password", None, "On startup, overwrite admin password with this one. This should usually only be done via environment variable in Docker."),
|
140
156
|
"clean_output":(tp.Boolean(), "Avoid Mutable Console Output", False, "Use if console output will be redirected e.g. to a web interface.")
|
@@ -164,7 +180,7 @@ malojaconfig = Configuration(
|
|
164
180
|
"name":(tp.String(), "Name", "Generic Maloja User")
|
165
181
|
},
|
166
182
|
"Third Party Services":{
|
167
|
-
"metadata_providers":(tp.List(tp.String()), "Metadata Providers", ['lastfm','spotify','deezer','audiodb','musicbrainz'], "
|
183
|
+
"metadata_providers":(tp.List(tp.String()), "Metadata Providers", ['lastfm','spotify','deezer','audiodb','musicbrainz'], "List of which metadata providers should be used in what order. Musicbrainz is rate-limited and should not be used first."),
|
168
184
|
"scrobble_lastfm":(tp.Boolean(), "Proxy-Scrobble to Last.fm", False),
|
169
185
|
"lastfm_api_key":(tp.String(), "Last.fm API Key", None),
|
170
186
|
"lastfm_api_secret":(tp.String(), "Last.fm API Secret", None),
|
@@ -202,6 +218,7 @@ malojaconfig = Configuration(
|
|
202
218
|
"default_album_artist":(tp.String(), "Default Albumartist", "Various Artists"),
|
203
219
|
"use_album_artwork_for_tracks":(tp.Boolean(), "Use Album Artwork for tracks", True),
|
204
220
|
"fancy_placeholder_art":(tp.Boolean(), "Use fancy placeholder artwork",False),
|
221
|
+
"show_play_number_on_tiles":(tp.Boolean(), "Show amount of plays on tiles",False),
|
205
222
|
"discourage_cpu_heavy_stats":(tp.Boolean(), "Discourage CPU-heavy stats", False, "Prevent visitors from mindlessly clicking on CPU-heavy options. Does not actually disable them for malicious actors!"),
|
206
223
|
"use_local_images":(tp.Boolean(), "Use Local Images", True),
|
207
224
|
#"local_image_rotate":(tp.Integer(), "Local Image Rotate", 3600),
|
@@ -209,18 +226,15 @@ malojaconfig = Configuration(
|
|
209
226
|
"theme":(tp.String(), "Theme", "maloja")
|
210
227
|
}
|
211
228
|
},
|
212
|
-
configfile=
|
229
|
+
configfile=settingsfile,
|
213
230
|
save_endpoint="/apis/mlj_1/settings",
|
214
231
|
env_prefix="MALOJA_",
|
215
232
|
extra_files=["/run/secrets/maloja.yml","/run/secrets/maloja.ini"]
|
216
233
|
|
217
234
|
)
|
218
235
|
|
219
|
-
if
|
220
|
-
|
221
|
-
malojaconfig["DIRECTORY_CONFIG"] = maloja_dir_config
|
222
|
-
except PermissionError as e:
|
223
|
-
pass
|
236
|
+
if not malojaconfig.readonly:
|
237
|
+
malojaconfig["DIRECTORY_CONFIG"] = maloja_dir_config
|
224
238
|
# this really doesn't matter because when are we gonna load info about where
|
225
239
|
# the settings file is stored from the settings file
|
226
240
|
# but oh well
|
@@ -242,17 +256,17 @@ except PermissionError as e:
|
|
242
256
|
pass
|
243
257
|
|
244
258
|
|
245
|
-
### STEP 3 - check
|
246
|
-
|
259
|
+
### STEP 3 - now check the other directories
|
247
260
|
|
248
261
|
|
249
262
|
if not malojaconfig.readonly:
|
250
263
|
for datatype in ("state","cache","logs"):
|
251
|
-
#
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
264
|
+
# if the setting is specified in the file or via a user environment variable, we accept it (we'll check later if it's usable)
|
265
|
+
if malojaconfig[directory_info[datatype]['setting']] or malojaconfig['DATA_DIRECTORY']:
|
266
|
+
pass
|
267
|
+
# otherwise, find a good one
|
268
|
+
else:
|
269
|
+
malojaconfig[directory_info[datatype]['setting']] = find_good_folder(datatype)
|
256
270
|
|
257
271
|
|
258
272
|
|
@@ -280,7 +294,6 @@ else:
|
|
280
294
|
"logs":pthj(malojaconfig['DATA_DIRECTORY'],"logs"),
|
281
295
|
}
|
282
296
|
|
283
|
-
|
284
297
|
data_directories = {
|
285
298
|
"auth":pthj(dir_settings['state'],"auth"),
|
286
299
|
"backups":pthj(dir_settings['state'],"backups"),
|
@@ -298,39 +311,51 @@ data_directories = {
|
|
298
311
|
}
|
299
312
|
|
300
313
|
for identifier,path in data_directories.items():
|
301
|
-
|
302
|
-
|
314
|
+
if path is None:
|
315
|
+
continue
|
303
316
|
|
304
|
-
|
305
|
-
|
306
|
-
}
|
317
|
+
if malojaconfig.readonly and (path == dir_settings['config'] or path.startswith(dir_settings['config']+'/')):
|
318
|
+
continue
|
307
319
|
|
320
|
+
try:
|
321
|
+
os.makedirs(path,exist_ok=True)
|
322
|
+
if not is_dir_usable(path): raise PermissionError(f"Directory {path} is not usable!")
|
323
|
+
except PermissionError:
|
324
|
+
# special case: cache does not contain info that can't be refetched, so no need to require user intervention
|
325
|
+
# just move to the next one
|
326
|
+
if identifier in ['cache']:
|
327
|
+
print("Cannot use",path,"for cache, finding new folder...")
|
328
|
+
data_directories['cache'] = dir_settings['cache'] = malojaconfig['DIRECTORY_CACHE'] = find_good_folder('cache')
|
329
|
+
else:
|
330
|
+
print(f"Directory for {identifier} ({path}) is not writeable.")
|
331
|
+
print("Please change permissions or settings!")
|
332
|
+
print("Make sure Maloja has write and execute access to this directory.")
|
333
|
+
raise
|
308
334
|
|
335
|
+
class DataDirs:
|
336
|
+
def __init__(self, dirs):
|
337
|
+
self.dirs = dirs
|
309
338
|
|
310
|
-
|
339
|
+
def __getitem__(self, key):
|
340
|
+
return lambda *x, k=key: pthj(self.dirs[k], *x)
|
311
341
|
|
312
|
-
|
342
|
+
data_dir = DataDirs(data_directories)
|
313
343
|
|
314
|
-
|
315
|
-
auth={
|
316
|
-
"multiuser":False,
|
317
|
-
"cookieprefix":"maloja",
|
318
|
-
"stylesheets":["/maloja.css"],
|
319
|
-
"dbfile":data_dir['auth']("auth.ddb")
|
320
|
-
},
|
321
|
-
logging={
|
322
|
-
"logfolder": data_dir['logs']() if malojaconfig["LOGGING"] else None
|
323
|
-
},
|
324
|
-
regular={
|
325
|
-
"offset": malojaconfig["TIMEZONE"]
|
326
|
-
}
|
327
|
-
)
|
344
|
+
### DOREAH OBJECTS
|
328
345
|
|
346
|
+
auth = doreah.auth.AuthManager(singleuser=True,cookieprefix='maloja',stylesheets=("/maloja.css",),dbfile=data_dir['auth']("auth.sqlite"))
|
329
347
|
|
348
|
+
#logger = doreah.logging.Logger(logfolder=data_dir['logs']() if malojaconfig["LOGGING"] else None)
|
349
|
+
#log = logger.log
|
330
350
|
|
351
|
+
# this is not how its supposed to be done, but lets ease the transition
|
352
|
+
doreah.logging.defaultlogger.logfolder = data_dir['logs']() if malojaconfig["LOGGING"] else None
|
331
353
|
|
332
354
|
|
333
|
-
|
355
|
+
try:
|
356
|
+
custom_css_files = [f for f in os.listdir(data_dir['css']()) if f.lower().endswith('.css')]
|
357
|
+
except FileNotFoundError:
|
358
|
+
custom_css_files = []
|
334
359
|
|
335
360
|
from ..database.sqldb import set_maloja_info
|
336
361
|
set_maloja_info({'last_run_version':VERSION})
|
@@ -12,11 +12,12 @@ def export(targetfolder=None):
|
|
12
12
|
targetfolder = os.getcwd()
|
13
13
|
|
14
14
|
timestr = time.strftime("%Y_%m_%d_%H_%M_%S")
|
15
|
+
timestamp = int(time.time()) # ok this is technically a separate time get from above, but those ms are not gonna matter, and im too lazy to change it all to datetime
|
15
16
|
filename = f"maloja_export_{timestr}.json"
|
16
17
|
outputfile = os.path.join(targetfolder,filename)
|
17
18
|
assert not os.path.exists(outputfile)
|
18
19
|
|
19
|
-
data = {'scrobbles':get_scrobbles()}
|
20
|
+
data = {'maloja':{'export_time': timestamp },'scrobbles':get_scrobbles()}
|
20
21
|
with open(outputfile,'w') as outfd:
|
21
22
|
json.dump(data,outfd,indent=3)
|
22
23
|
|
@@ -32,37 +32,53 @@ def import_scrobbles(inputf):
|
|
32
32
|
}
|
33
33
|
|
34
34
|
filename = os.path.basename(inputf)
|
35
|
+
importfunc = None
|
35
36
|
|
36
|
-
|
37
|
-
|
37
|
+
|
38
|
+
if re.match(r".*\.csv", filename):
|
39
|
+
typeid,typedesc = "lastfm", "Last.fm (benjaminbenben export)"
|
38
40
|
importfunc = parse_lastfm
|
39
41
|
|
40
|
-
elif re.match(r"Streaming_History_Audio.+\.json",filename):
|
41
|
-
typeid,typedesc = "spotify","Spotify"
|
42
|
+
elif re.match(r"Streaming_History_Audio.+\.json", filename):
|
43
|
+
typeid,typedesc = "spotify", "Spotify"
|
42
44
|
importfunc = parse_spotify_lite
|
43
45
|
|
44
|
-
elif re.match(r"endsong_[0-9]+\.json",filename):
|
45
|
-
typeid,typedesc = "spotify","Spotify"
|
46
|
+
elif re.match(r"endsong_[0-9]+\.json", filename):
|
47
|
+
typeid,typedesc = "spotify", "Spotify"
|
46
48
|
importfunc = parse_spotify
|
47
49
|
|
48
|
-
elif re.match(r"StreamingHistory[0-9]+\.json",filename):
|
49
|
-
typeid,typedesc = "spotify","Spotify"
|
50
|
+
elif re.match(r"StreamingHistory[0-9]+\.json", filename):
|
51
|
+
typeid,typedesc = "spotify", "Spotify"
|
50
52
|
importfunc = parse_spotify_lite_legacy
|
51
53
|
|
52
|
-
elif re.match(r"maloja_export[_0-9]*\.json",filename):
|
53
|
-
typeid,typedesc = "maloja","Maloja"
|
54
|
+
elif re.match(r"maloja_export[_0-9]*\.json", filename):
|
55
|
+
typeid,typedesc = "maloja", "Maloja"
|
54
56
|
importfunc = parse_maloja
|
55
57
|
|
56
58
|
# username_lb-YYYY-MM-DD.json
|
57
|
-
elif re.match(r".*_lb-[0-9-]+\.json",filename):
|
58
|
-
typeid,typedesc = "listenbrainz","ListenBrainz"
|
59
|
+
elif re.match(r".*_lb-[0-9-]+\.json", filename):
|
60
|
+
typeid,typedesc = "listenbrainz", "ListenBrainz"
|
59
61
|
importfunc = parse_listenbrainz
|
60
62
|
|
61
|
-
elif re.match(r"\.scrobbler\.log",filename):
|
62
|
-
typeid,typedesc = "rockbox","Rockbox"
|
63
|
+
elif re.match(r"\.scrobbler\.log", filename):
|
64
|
+
typeid,typedesc = "rockbox", "Rockbox"
|
63
65
|
importfunc = parse_rockbox
|
64
66
|
|
65
|
-
|
67
|
+
elif re.match(r"recenttracks-.*\.json", filename):
|
68
|
+
typeid, typedesc = "lastfm", "Last.fm (ghan export)"
|
69
|
+
importfunc = parse_lastfm_ghan
|
70
|
+
|
71
|
+
elif re.match(r".*\.json",filename):
|
72
|
+
try:
|
73
|
+
with open(filename,'r') as fd:
|
74
|
+
data = json.load(fd)
|
75
|
+
if 'maloja' in data:
|
76
|
+
typeid,typedesc = "maloja","Maloja"
|
77
|
+
importfunc = parse_maloja
|
78
|
+
except Exception:
|
79
|
+
pass
|
80
|
+
|
81
|
+
if not importfunc:
|
66
82
|
print("File",inputf,"could not be identified as a valid import source.")
|
67
83
|
return result
|
68
84
|
|
@@ -131,6 +147,7 @@ def import_scrobbles(inputf):
|
|
131
147
|
|
132
148
|
return result
|
133
149
|
|
150
|
+
|
134
151
|
def parse_spotify_lite_legacy(inputf):
|
135
152
|
pth = os.path
|
136
153
|
# use absolute paths internally for peace of mind. just change representation for console output
|
@@ -243,6 +260,7 @@ def parse_spotify_lite(inputf):
|
|
243
260
|
|
244
261
|
print()
|
245
262
|
|
263
|
+
|
246
264
|
def parse_spotify(inputf):
|
247
265
|
pth = os.path
|
248
266
|
# use absolute paths internally for peace of mind. just change representation for console output
|
@@ -354,6 +372,7 @@ def parse_spotify(inputf):
|
|
354
372
|
|
355
373
|
print()
|
356
374
|
|
375
|
+
|
357
376
|
def parse_lastfm(inputf):
|
358
377
|
|
359
378
|
with open(inputf,'r',newline='') as inputfd:
|
@@ -388,6 +407,29 @@ def parse_lastfm(inputf):
|
|
388
407
|
yield ('FAIL',None,f"{row} (Line {line}) could not be parsed. Scrobble not imported. ({repr(e)})")
|
389
408
|
continue
|
390
409
|
|
410
|
+
|
411
|
+
def parse_lastfm_ghan(inputf):
|
412
|
+
with open(inputf, 'r') as inputfd:
|
413
|
+
data = json.load(inputfd)
|
414
|
+
|
415
|
+
skip = 50000
|
416
|
+
for entry in data:
|
417
|
+
for track in entry['track']:
|
418
|
+
skip -= 1
|
419
|
+
#if skip: continue
|
420
|
+
#print(track)
|
421
|
+
#input()
|
422
|
+
|
423
|
+
yield ('CONFIDENT_IMPORT', {
|
424
|
+
'track_title': track['name'],
|
425
|
+
'track_artists': track['artist']['#text'],
|
426
|
+
'track_length': None,
|
427
|
+
'album_name': track['album']['#text'],
|
428
|
+
'scrobble_time': int(track['date']['uts']),
|
429
|
+
'scrobble_duration': None
|
430
|
+
}, '')
|
431
|
+
|
432
|
+
|
391
433
|
def parse_listenbrainz(inputf):
|
392
434
|
|
393
435
|
with open(inputf,'r') as inputfd:
|
maloja/server.py
CHANGED
@@ -12,14 +12,13 @@ from jinja2.exceptions import TemplateNotFound
|
|
12
12
|
|
13
13
|
# doreah toolkit
|
14
14
|
from doreah.logging import log
|
15
|
-
from doreah import auth
|
16
15
|
|
17
16
|
# rest of the project
|
18
17
|
from . import database
|
19
18
|
from .database.jinjaview import JinjaDBConnection
|
20
19
|
from .images import image_request
|
21
20
|
from .malojauri import uri_to_internal, remove_identical
|
22
|
-
from .pkg_global.conf import malojaconfig, data_dir
|
21
|
+
from .pkg_global.conf import malojaconfig, data_dir, auth
|
23
22
|
from .pkg_global import conf
|
24
23
|
from .jinjaenv.context import jinja_environment
|
25
24
|
from .apis import init_apis, apikeystore
|
@@ -97,7 +96,7 @@ aliases = {
|
|
97
96
|
|
98
97
|
### API
|
99
98
|
|
100
|
-
auth.authapi.mount(server=webserver)
|
99
|
+
conf.auth.authapi.mount(server=webserver)
|
101
100
|
init_apis(webserver)
|
102
101
|
|
103
102
|
# redirects for backwards compatibility
|
@@ -197,7 +196,7 @@ def jinja_page(name):
|
|
197
196
|
if name in aliases: redirect(aliases[name])
|
198
197
|
keys = remove_identical(FormsDict.decode(request.query))
|
199
198
|
|
200
|
-
adminmode = request.cookies.get("adminmode") == "true" and auth.
|
199
|
+
adminmode = request.cookies.get("adminmode") == "true" and auth.check_request(request)
|
201
200
|
|
202
201
|
with JinjaDBConnection() as conn:
|
203
202
|
|
@@ -222,7 +221,7 @@ def jinja_page(name):
|
|
222
221
|
return res
|
223
222
|
|
224
223
|
@webserver.route("/<name:re:admin.*>")
|
225
|
-
@auth.
|
224
|
+
@auth.authenticated_function()
|
226
225
|
def jinja_page_private(name):
|
227
226
|
return jinja_page(name)
|
228
227
|
|
maloja/setup.py
CHANGED
@@ -6,9 +6,8 @@ try:
|
|
6
6
|
except ImportError:
|
7
7
|
import distutils
|
8
8
|
from doreah.io import col, ask, prompt
|
9
|
-
from doreah import auth
|
10
9
|
|
11
|
-
from .pkg_global.conf import data_dir, dir_settings, malojaconfig
|
10
|
+
from .pkg_global.conf import data_dir, dir_settings, malojaconfig, auth
|
12
11
|
|
13
12
|
|
14
13
|
|
@@ -25,6 +24,12 @@ ext_apikeys = [
|
|
25
24
|
def copy_initial_local_files():
|
26
25
|
with resources.files("maloja") / 'data_files' as folder:
|
27
26
|
for cat in dir_settings:
|
27
|
+
if dir_settings[cat] is None:
|
28
|
+
continue
|
29
|
+
|
30
|
+
if cat == 'config' and malojaconfig.readonly:
|
31
|
+
continue
|
32
|
+
|
28
33
|
distutils.dir_util.copy_tree(os.path.join(folder,cat),dir_settings[cat],update=False)
|
29
34
|
|
30
35
|
charset = list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
|
@@ -37,46 +42,53 @@ def setup():
|
|
37
42
|
copy_initial_local_files()
|
38
43
|
SKIP = malojaconfig["SKIP_SETUP"]
|
39
44
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
# OWN API KEY
|
55
|
-
from .apis import apikeystore
|
56
|
-
if len(apikeystore) == 0:
|
57
|
-
answer = ask("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database.",default=True,skip=SKIP)
|
58
|
-
if answer:
|
59
|
-
key = apikeystore.generate_key('default')
|
60
|
-
print("Your API Key: " + col["yellow"](key))
|
61
|
-
|
62
|
-
# PASSWORD
|
63
|
-
forcepassword = malojaconfig["FORCE_PASSWORD"]
|
64
|
-
# this is mainly meant for docker, supply password via environment variable
|
65
|
-
|
66
|
-
if forcepassword is not None:
|
67
|
-
# user has specified to force the pw, nothing else matters
|
68
|
-
auth.defaultuser.setpw(forcepassword)
|
69
|
-
print("Password has been set.")
|
70
|
-
elif auth.defaultuser.checkpw("admin"):
|
71
|
-
# if the actual pw is admin, it means we've never set this up properly (eg first start after update)
|
72
|
-
while True:
|
73
|
-
newpw = prompt("Please set a password for web backend access. Leave this empty to generate a random password.",skip=SKIP,secret=True)
|
74
|
-
if newpw is None:
|
75
|
-
newpw = randomstring(32)
|
76
|
-
print("Generated password:",col["yellow"](newpw))
|
77
|
-
break
|
45
|
+
try:
|
46
|
+
print("Various external services can be used to display images. If not enough of them are set up, only local images will be used.")
|
47
|
+
for k in ext_apikeys:
|
48
|
+
keyname = malojaconfig.get_setting_info(k)['name']
|
49
|
+
key = malojaconfig[k]
|
50
|
+
if key is False:
|
51
|
+
print(f"\tCurrently not using a {col['red'](keyname)} for image display.")
|
52
|
+
elif key is None or key == "ASK":
|
53
|
+
if malojaconfig.readonly:
|
54
|
+
continue
|
55
|
+
promptmsg = f"\tPlease enter your {col['gold'](keyname)}. If you do not want to use one at this moment, simply leave this empty and press Enter."
|
56
|
+
key = prompt(promptmsg,types=(str,),default=False,skip=SKIP)
|
57
|
+
malojaconfig[k] = key
|
78
58
|
else:
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
59
|
+
print(f"\t{col['lawngreen'](keyname)} found.")
|
60
|
+
|
61
|
+
|
62
|
+
# OWN API KEY
|
63
|
+
from .apis import apikeystore
|
64
|
+
if len(apikeystore) == 0:
|
65
|
+
answer = ask("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database.",default=True,skip=SKIP)
|
66
|
+
if answer:
|
67
|
+
key = apikeystore.generate_key('default')
|
68
|
+
print("Your API Key: " + col["yellow"](key))
|
69
|
+
|
70
|
+
# PASSWORD
|
71
|
+
forcepassword = malojaconfig["FORCE_PASSWORD"]
|
72
|
+
# this is mainly meant for docker, supply password via environment variable
|
73
|
+
|
74
|
+
if forcepassword is not None:
|
75
|
+
# user has specified to force the pw, nothing else matters
|
76
|
+
auth.change_pw(password=forcepassword)
|
77
|
+
print("Password has been set.")
|
78
|
+
elif auth.still_has_factory_default_user():
|
79
|
+
# this means we've never set this up properly (eg first start after update)
|
80
|
+
while True:
|
81
|
+
newpw = prompt("Please set a password for web backend access. Leave this empty to generate a random password.",skip=SKIP,secret=True)
|
82
|
+
if newpw is None:
|
83
|
+
newpw = randomstring(32)
|
84
|
+
print("Generated password:",col["yellow"](newpw))
|
85
|
+
break
|
86
|
+
else:
|
87
|
+
newpw_repeat = prompt("Please type again to confirm.",skip=SKIP,secret=True)
|
88
|
+
if newpw != newpw_repeat: print("Passwords do not match!")
|
89
|
+
else: break
|
90
|
+
auth.change_pw(password=newpw)
|
91
|
+
|
92
|
+
except EOFError:
|
93
|
+
print("No user input possible. If you are running inside a container, set the environment variable",col['yellow']("MALOJA_SKIP_SETUP=yes"))
|
94
|
+
raise SystemExit
|