malojaserver 3.2.1__py3-none-any.whl → 3.2.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- maloja/__main__.py +1 -1
- maloja/__pkginfo__.py +1 -1
- maloja/apis/_base.py +26 -19
- maloja/apis/_exceptions.py +1 -1
- maloja/apis/audioscrobbler.py +35 -7
- maloja/apis/audioscrobbler_legacy.py +5 -5
- maloja/apis/listenbrainz.py +7 -5
- maloja/apis/native_v1.py +43 -26
- maloja/cleanup.py +9 -7
- maloja/data_files/config/rules/predefined/krateng_kpopgirlgroups.tsv +4 -2
- maloja/database/__init__.py +55 -23
- maloja/database/associated.py +10 -6
- maloja/database/exceptions.py +28 -3
- maloja/database/sqldb.py +216 -168
- maloja/dev/profiler.py +3 -4
- maloja/images.py +6 -0
- maloja/malojauri.py +2 -0
- maloja/pkg_global/conf.py +97 -72
- maloja/proccontrol/tasks/export.py +2 -1
- maloja/proccontrol/tasks/import_scrobbles.py +57 -15
- maloja/server.py +4 -5
- maloja/setup.py +56 -44
- maloja/thirdparty/lastfm.py +18 -17
- maloja/web/jinja/abstracts/base.jinja +1 -1
- maloja/web/jinja/admin_albumless.jinja +2 -0
- maloja/web/jinja/admin_overview.jinja +3 -3
- maloja/web/jinja/admin_setup.jinja +1 -1
- maloja/web/jinja/error.jinja +2 -2
- maloja/web/jinja/partials/album_showcase.jinja +1 -1
- maloja/web/jinja/partials/awards_album.jinja +1 -1
- maloja/web/jinja/partials/awards_artist.jinja +2 -2
- maloja/web/jinja/partials/charts_albums_tiles.jinja +4 -0
- maloja/web/jinja/partials/charts_artists_tiles.jinja +5 -1
- maloja/web/jinja/partials/charts_tracks_tiles.jinja +4 -0
- maloja/web/jinja/snippets/entityrow.jinja +2 -2
- maloja/web/jinja/snippets/links.jinja +3 -1
- maloja/web/static/css/maloja.css +14 -2
- maloja/web/static/css/startpage.css +2 -2
- maloja/web/static/js/manualscrobble.js +1 -1
- maloja/web/static/js/notifications.js +16 -8
- {malojaserver-3.2.1.dist-info → malojaserver-3.2.3.dist-info}/METADATA +10 -46
- {malojaserver-3.2.1.dist-info → malojaserver-3.2.3.dist-info}/RECORD +45 -45
- {malojaserver-3.2.1.dist-info → malojaserver-3.2.3.dist-info}/WHEEL +1 -1
- {malojaserver-3.2.1.dist-info → malojaserver-3.2.3.dist-info}/LICENSE +0 -0
- {malojaserver-3.2.1.dist-info → malojaserver-3.2.3.dist-info}/entry_points.txt +0 -0
maloja/dev/profiler.py
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
import os
|
2
2
|
|
3
3
|
import cProfile, pstats
|
4
|
+
import time
|
4
5
|
|
5
6
|
from doreah.logging import log
|
6
|
-
from doreah.timing import Clock
|
7
7
|
|
8
8
|
from ..pkg_global.conf import data_dir
|
9
9
|
|
@@ -27,8 +27,7 @@ def profile(func):
|
|
27
27
|
|
28
28
|
def newfunc(*args,**kwargs):
|
29
29
|
|
30
|
-
|
31
|
-
clock.start()
|
30
|
+
starttime = time.time()
|
32
31
|
|
33
32
|
if FULL_PROFILE:
|
34
33
|
benchmarkfolder = data_dir['logs']("benchmarks")
|
@@ -44,7 +43,7 @@ def profile(func):
|
|
44
43
|
if FULL_PROFILE:
|
45
44
|
localprofiler.disable()
|
46
45
|
|
47
|
-
seconds =
|
46
|
+
seconds = time.time() - starttime
|
48
47
|
|
49
48
|
if not SINGLE_CALLS:
|
50
49
|
times.setdefault(realfunc,[]).append(seconds)
|
maloja/images.py
CHANGED
@@ -284,6 +284,12 @@ def image_request(artist_id=None,track_id=None,album_id=None):
|
|
284
284
|
if result is not None:
|
285
285
|
# we got an entry, even if it's that there is no image (value None)
|
286
286
|
if result['value'] is None:
|
287
|
+
# fallback to album regardless of setting (because we have no image)
|
288
|
+
if track_id:
|
289
|
+
track = database.sqldb.get_track(track_id)
|
290
|
+
if track.get("album"):
|
291
|
+
album_id = database.sqldb.get_album_id(track["album"])
|
292
|
+
return image_request(album_id=album_id)
|
287
293
|
# use placeholder
|
288
294
|
if malojaconfig["FANCY_PLACEHOLDER_ART"]:
|
289
295
|
placeholder_url = "https://generative-placeholders.glitch.me/image?width=300&height=300&style="
|
maloja/malojauri.py
CHANGED
@@ -29,6 +29,8 @@ def uri_to_internal(keys,accepted_entities=('artist','track','album'),forceTrack
|
|
29
29
|
|
30
30
|
# 1
|
31
31
|
filterkeys = {}
|
32
|
+
# this only takes care of the logic - what kind of entity we're dealing with
|
33
|
+
# it does not check with the database if it exists or what the canonical name is!!!
|
32
34
|
if "track" in accepted_entities and "title" in keys:
|
33
35
|
filterkeys.update({"track":{"artists":keys.getall("trackartist"),"title":keys.get("title")}})
|
34
36
|
if "artist" in accepted_entities and "artist" in keys:
|
maloja/pkg_global/conf.py
CHANGED
@@ -1,4 +1,7 @@
|
|
1
1
|
import os
|
2
|
+
|
3
|
+
import doreah.auth
|
4
|
+
import doreah.logging
|
2
5
|
from doreah.configuration import Configuration
|
3
6
|
from doreah.configuration import types as tp
|
4
7
|
|
@@ -17,9 +20,11 @@ AUX_MODE = True
|
|
17
20
|
# DIRECRORY_CONFIG, DIRECRORY_STATE, DIRECTORY_LOGS and DIRECTORY_CACHE
|
18
21
|
# config can only be determined by environment variable, the others can be loaded
|
19
22
|
# from the config files
|
20
|
-
# explicit settings will always be respected, fallback to default
|
21
23
|
|
22
|
-
#
|
24
|
+
# we don't specify 'default' values in the normal sense of the config object
|
25
|
+
# the default is none, meaning the app should figure it out (depending on environment)
|
26
|
+
# the actual 'default' values of our folders are simply in code since they are dependent on environment (container?)
|
27
|
+
# and we need to actually distinguish them from the user having specified something
|
23
28
|
|
24
29
|
# USEFUL FUNCS
|
25
30
|
pthj = os.path.join
|
@@ -27,9 +32,7 @@ pthj = os.path.join
|
|
27
32
|
def is_dir_usable(pth):
|
28
33
|
try:
|
29
34
|
os.makedirs(pth,exist_ok=True)
|
30
|
-
os.
|
31
|
-
os.remove(pthj(pth,".test"))
|
32
|
-
return True
|
35
|
+
return os.access(pth,os.W_OK)
|
33
36
|
except Exception:
|
34
37
|
return False
|
35
38
|
|
@@ -40,7 +43,10 @@ def get_env_vars(key,pathsuffix=[]):
|
|
40
43
|
|
41
44
|
directory_info = {
|
42
45
|
"config":{
|
43
|
-
"sentinel":"
|
46
|
+
"sentinel":".maloja_config_sentinel",
|
47
|
+
"possible_folders_container":[
|
48
|
+
"/config/config"
|
49
|
+
],
|
44
50
|
"possible_folders":[
|
45
51
|
"/etc/maloja",
|
46
52
|
os.path.expanduser("~/.local/share/maloja")
|
@@ -48,15 +54,22 @@ directory_info = {
|
|
48
54
|
"setting":"directory_config"
|
49
55
|
},
|
50
56
|
"cache":{
|
51
|
-
"sentinel":"
|
57
|
+
"sentinel":".maloja_cache_sentinel",
|
58
|
+
"possible_folders_container":[
|
59
|
+
"/config/cache"
|
60
|
+
],
|
52
61
|
"possible_folders":[
|
53
62
|
"/var/cache/maloja",
|
54
|
-
os.path.expanduser("~/.local/share/maloja/cache")
|
63
|
+
os.path.expanduser("~/.local/share/maloja/cache"),
|
64
|
+
"/tmp/maloja"
|
55
65
|
],
|
56
66
|
"setting":"directory_cache"
|
57
67
|
},
|
58
68
|
"state":{
|
59
|
-
"sentinel":"
|
69
|
+
"sentinel":".maloja_state_sentinel",
|
70
|
+
"possible_folders_container":[
|
71
|
+
"/config/state"
|
72
|
+
],
|
60
73
|
"possible_folders":[
|
61
74
|
"/var/lib/maloja",
|
62
75
|
os.path.expanduser("~/.local/share/maloja")
|
@@ -64,7 +77,10 @@ directory_info = {
|
|
64
77
|
"setting":"directory_state"
|
65
78
|
},
|
66
79
|
"logs":{
|
67
|
-
"sentinel":"
|
80
|
+
"sentinel":".maloja_logs_sentinel",
|
81
|
+
"possible_folders_container":[
|
82
|
+
"/config/logs"
|
83
|
+
],
|
68
84
|
"possible_folders":[
|
69
85
|
"/var/log/maloja",
|
70
86
|
os.path.expanduser("~/.local/share/maloja/logs")
|
@@ -77,51 +93,51 @@ directory_info = {
|
|
77
93
|
# checks if one has been in use before and writes it to dict/config
|
78
94
|
# if not, determines which to use and writes it to dict/config
|
79
95
|
# returns determined folder
|
80
|
-
def find_good_folder(datatype
|
96
|
+
def find_good_folder(datatype):
|
81
97
|
info = directory_info[datatype]
|
82
98
|
|
99
|
+
possible_folders = info['possible_folders']
|
100
|
+
if os.environ.get("MALOJA_CONTAINER"):
|
101
|
+
possible_folders = info['possible_folders_container'] + possible_folders
|
102
|
+
|
83
103
|
# check each possible folder if its used
|
84
|
-
for p in
|
104
|
+
for p in possible_folders:
|
85
105
|
if os.path.exists(pthj(p,info['sentinel'])):
|
86
|
-
|
87
|
-
|
88
|
-
|
106
|
+
if is_dir_usable(p):
|
107
|
+
#print(p,"was apparently used as maloja's folder for",datatype,"- fixing in settings")
|
108
|
+
return p
|
109
|
+
else:
|
110
|
+
raise PermissionError(f"Can no longer use previously used {datatype} folder {p}")
|
89
111
|
|
90
112
|
#print("Could not find previous",datatype,"folder")
|
91
113
|
# check which one we can use
|
92
|
-
for p in
|
114
|
+
for p in possible_folders:
|
93
115
|
if is_dir_usable(p):
|
94
116
|
#print(p,"has been selected as maloja's folder for",datatype)
|
95
|
-
configobject[info['setting']] = p
|
96
117
|
return p
|
97
118
|
#print("No folder can be used for",datatype)
|
98
119
|
#print("This should not happen!")
|
120
|
+
raise PermissionError(f"No folder could be found for {datatype}")
|
99
121
|
|
100
122
|
|
101
123
|
|
102
124
|
|
103
125
|
|
104
126
|
### STEP 1 - find out where the settings file is
|
105
|
-
# environment variables
|
106
|
-
maloja_dir_config = os.environ.get("MALOJA_DATA_DIRECTORY") or os.environ.get("MALOJA_DIRECTORY_CONFIG")
|
107
127
|
|
128
|
+
maloja_dir_config = os.environ.get("MALOJA_DATA_DIRECTORY") or os.environ.get("MALOJA_DIRECTORY_CONFIG")
|
108
129
|
|
109
130
|
if maloja_dir_config is None:
|
110
|
-
|
111
|
-
|
131
|
+
# if nothing is set, we set our own
|
132
|
+
maloja_dir_config = find_good_folder('config')
|
112
133
|
else:
|
113
|
-
|
114
|
-
#
|
134
|
+
pass
|
135
|
+
# if there is an environment variable, this is 100% explicitly defined by the user, so we respect it
|
136
|
+
# the user might run more than one instances on the same machine, so we don't do any heuristics here
|
137
|
+
# if you define this, we believe it!
|
115
138
|
|
116
139
|
os.makedirs(maloja_dir_config,exist_ok=True)
|
117
|
-
|
118
|
-
oldsettingsfile = pthj(maloja_dir_config,"settings","settings.ini")
|
119
|
-
newsettingsfile = pthj(maloja_dir_config,"settings.ini")
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
if os.path.exists(oldsettingsfile):
|
124
|
-
os.rename(oldsettingsfile,newsettingsfile)
|
140
|
+
settingsfile = pthj(maloja_dir_config,"settings.ini")
|
125
141
|
|
126
142
|
|
127
143
|
### STEP 2 - create settings object
|
@@ -131,10 +147,10 @@ malojaconfig = Configuration(
|
|
131
147
|
settings={
|
132
148
|
"Setup":{
|
133
149
|
"data_directory":(tp.String(), "Data Directory", None, "Folder for all user data. Overwrites all choices for specific directories."),
|
134
|
-
"directory_config":(tp.String(), "Config Directory",
|
135
|
-
"directory_state":(tp.String(), "State Directory",
|
136
|
-
"directory_logs":(tp.String(), "Log Directory",
|
137
|
-
"directory_cache":(tp.String(), "Cache Directory",
|
150
|
+
"directory_config":(tp.String(), "Config Directory", None, "Folder for config data. Only applied when global data directory is not set."),
|
151
|
+
"directory_state":(tp.String(), "State Directory", None, "Folder for state data. Only applied when global data directory is not set."),
|
152
|
+
"directory_logs":(tp.String(), "Log Directory", None, "Folder for log data. Only applied when global data directory is not set."),
|
153
|
+
"directory_cache":(tp.String(), "Cache Directory", None, "Folder for cache data. Only applied when global data directory is not set."),
|
138
154
|
"skip_setup":(tp.Boolean(), "Skip Setup", False, "Make server setup process non-interactive. Vital for Docker."),
|
139
155
|
"force_password":(tp.String(), "Force Password", None, "On startup, overwrite admin password with this one. This should usually only be done via environment variable in Docker."),
|
140
156
|
"clean_output":(tp.Boolean(), "Avoid Mutable Console Output", False, "Use if console output will be redirected e.g. to a web interface.")
|
@@ -164,7 +180,7 @@ malojaconfig = Configuration(
|
|
164
180
|
"name":(tp.String(), "Name", "Generic Maloja User")
|
165
181
|
},
|
166
182
|
"Third Party Services":{
|
167
|
-
"metadata_providers":(tp.List(tp.String()), "Metadata Providers", ['lastfm','spotify','deezer','audiodb','musicbrainz'], "
|
183
|
+
"metadata_providers":(tp.List(tp.String()), "Metadata Providers", ['lastfm','spotify','deezer','audiodb','musicbrainz'], "List of which metadata providers should be used in what order. Musicbrainz is rate-limited and should not be used first."),
|
168
184
|
"scrobble_lastfm":(tp.Boolean(), "Proxy-Scrobble to Last.fm", False),
|
169
185
|
"lastfm_api_key":(tp.String(), "Last.fm API Key", None),
|
170
186
|
"lastfm_api_secret":(tp.String(), "Last.fm API Secret", None),
|
@@ -202,6 +218,7 @@ malojaconfig = Configuration(
|
|
202
218
|
"default_album_artist":(tp.String(), "Default Albumartist", "Various Artists"),
|
203
219
|
"use_album_artwork_for_tracks":(tp.Boolean(), "Use Album Artwork for tracks", True),
|
204
220
|
"fancy_placeholder_art":(tp.Boolean(), "Use fancy placeholder artwork",False),
|
221
|
+
"show_play_number_on_tiles":(tp.Boolean(), "Show amount of plays on tiles",False),
|
205
222
|
"discourage_cpu_heavy_stats":(tp.Boolean(), "Discourage CPU-heavy stats", False, "Prevent visitors from mindlessly clicking on CPU-heavy options. Does not actually disable them for malicious actors!"),
|
206
223
|
"use_local_images":(tp.Boolean(), "Use Local Images", True),
|
207
224
|
#"local_image_rotate":(tp.Integer(), "Local Image Rotate", 3600),
|
@@ -209,18 +226,15 @@ malojaconfig = Configuration(
|
|
209
226
|
"theme":(tp.String(), "Theme", "maloja")
|
210
227
|
}
|
211
228
|
},
|
212
|
-
configfile=
|
229
|
+
configfile=settingsfile,
|
213
230
|
save_endpoint="/apis/mlj_1/settings",
|
214
231
|
env_prefix="MALOJA_",
|
215
232
|
extra_files=["/run/secrets/maloja.yml","/run/secrets/maloja.ini"]
|
216
233
|
|
217
234
|
)
|
218
235
|
|
219
|
-
if
|
220
|
-
|
221
|
-
malojaconfig["DIRECTORY_CONFIG"] = maloja_dir_config
|
222
|
-
except PermissionError as e:
|
223
|
-
pass
|
236
|
+
if not malojaconfig.readonly:
|
237
|
+
malojaconfig["DIRECTORY_CONFIG"] = maloja_dir_config
|
224
238
|
# this really doesn't matter because when are we gonna load info about where
|
225
239
|
# the settings file is stored from the settings file
|
226
240
|
# but oh well
|
@@ -242,17 +256,17 @@ except PermissionError as e:
|
|
242
256
|
pass
|
243
257
|
|
244
258
|
|
245
|
-
### STEP 3 - check
|
246
|
-
|
259
|
+
### STEP 3 - now check the other directories
|
247
260
|
|
248
261
|
|
249
262
|
if not malojaconfig.readonly:
|
250
263
|
for datatype in ("state","cache","logs"):
|
251
|
-
#
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
264
|
+
# if the setting is specified in the file or via a user environment variable, we accept it (we'll check later if it's usable)
|
265
|
+
if malojaconfig[directory_info[datatype]['setting']] or malojaconfig['DATA_DIRECTORY']:
|
266
|
+
pass
|
267
|
+
# otherwise, find a good one
|
268
|
+
else:
|
269
|
+
malojaconfig[directory_info[datatype]['setting']] = find_good_folder(datatype)
|
256
270
|
|
257
271
|
|
258
272
|
|
@@ -280,7 +294,6 @@ else:
|
|
280
294
|
"logs":pthj(malojaconfig['DATA_DIRECTORY'],"logs"),
|
281
295
|
}
|
282
296
|
|
283
|
-
|
284
297
|
data_directories = {
|
285
298
|
"auth":pthj(dir_settings['state'],"auth"),
|
286
299
|
"backups":pthj(dir_settings['state'],"backups"),
|
@@ -298,39 +311,51 @@ data_directories = {
|
|
298
311
|
}
|
299
312
|
|
300
313
|
for identifier,path in data_directories.items():
|
301
|
-
|
302
|
-
|
314
|
+
if path is None:
|
315
|
+
continue
|
303
316
|
|
304
|
-
|
305
|
-
|
306
|
-
}
|
317
|
+
if malojaconfig.readonly and (path == dir_settings['config'] or path.startswith(dir_settings['config']+'/')):
|
318
|
+
continue
|
307
319
|
|
320
|
+
try:
|
321
|
+
os.makedirs(path,exist_ok=True)
|
322
|
+
if not is_dir_usable(path): raise PermissionError(f"Directory {path} is not usable!")
|
323
|
+
except PermissionError:
|
324
|
+
# special case: cache does not contain info that can't be refetched, so no need to require user intervention
|
325
|
+
# just move to the next one
|
326
|
+
if identifier in ['cache']:
|
327
|
+
print("Cannot use",path,"for cache, finding new folder...")
|
328
|
+
data_directories['cache'] = dir_settings['cache'] = malojaconfig['DIRECTORY_CACHE'] = find_good_folder('cache')
|
329
|
+
else:
|
330
|
+
print(f"Directory for {identifier} ({path}) is not writeable.")
|
331
|
+
print("Please change permissions or settings!")
|
332
|
+
print("Make sure Maloja has write and execute access to this directory.")
|
333
|
+
raise
|
308
334
|
|
335
|
+
class DataDirs:
|
336
|
+
def __init__(self, dirs):
|
337
|
+
self.dirs = dirs
|
309
338
|
|
310
|
-
|
339
|
+
def __getitem__(self, key):
|
340
|
+
return lambda *x, k=key: pthj(self.dirs[k], *x)
|
311
341
|
|
312
|
-
|
342
|
+
data_dir = DataDirs(data_directories)
|
313
343
|
|
314
|
-
|
315
|
-
auth={
|
316
|
-
"multiuser":False,
|
317
|
-
"cookieprefix":"maloja",
|
318
|
-
"stylesheets":["/maloja.css"],
|
319
|
-
"dbfile":data_dir['auth']("auth.ddb")
|
320
|
-
},
|
321
|
-
logging={
|
322
|
-
"logfolder": data_dir['logs']() if malojaconfig["LOGGING"] else None
|
323
|
-
},
|
324
|
-
regular={
|
325
|
-
"offset": malojaconfig["TIMEZONE"]
|
326
|
-
}
|
327
|
-
)
|
344
|
+
### DOREAH OBJECTS
|
328
345
|
|
346
|
+
auth = doreah.auth.AuthManager(singleuser=True,cookieprefix='maloja',stylesheets=("/maloja.css",),dbfile=data_dir['auth']("auth.sqlite"))
|
329
347
|
|
348
|
+
#logger = doreah.logging.Logger(logfolder=data_dir['logs']() if malojaconfig["LOGGING"] else None)
|
349
|
+
#log = logger.log
|
330
350
|
|
351
|
+
# this is not how its supposed to be done, but lets ease the transition
|
352
|
+
doreah.logging.defaultlogger.logfolder = data_dir['logs']() if malojaconfig["LOGGING"] else None
|
331
353
|
|
332
354
|
|
333
|
-
|
355
|
+
try:
|
356
|
+
custom_css_files = [f for f in os.listdir(data_dir['css']()) if f.lower().endswith('.css')]
|
357
|
+
except FileNotFoundError:
|
358
|
+
custom_css_files = []
|
334
359
|
|
335
360
|
from ..database.sqldb import set_maloja_info
|
336
361
|
set_maloja_info({'last_run_version':VERSION})
|
@@ -12,11 +12,12 @@ def export(targetfolder=None):
|
|
12
12
|
targetfolder = os.getcwd()
|
13
13
|
|
14
14
|
timestr = time.strftime("%Y_%m_%d_%H_%M_%S")
|
15
|
+
timestamp = int(time.time()) # ok this is technically a separate time get from above, but those ms are not gonna matter, and im too lazy to change it all to datetime
|
15
16
|
filename = f"maloja_export_{timestr}.json"
|
16
17
|
outputfile = os.path.join(targetfolder,filename)
|
17
18
|
assert not os.path.exists(outputfile)
|
18
19
|
|
19
|
-
data = {'scrobbles':get_scrobbles()}
|
20
|
+
data = {'maloja':{'export_time': timestamp },'scrobbles':get_scrobbles()}
|
20
21
|
with open(outputfile,'w') as outfd:
|
21
22
|
json.dump(data,outfd,indent=3)
|
22
23
|
|
@@ -32,37 +32,53 @@ def import_scrobbles(inputf):
|
|
32
32
|
}
|
33
33
|
|
34
34
|
filename = os.path.basename(inputf)
|
35
|
+
importfunc = None
|
35
36
|
|
36
|
-
|
37
|
-
|
37
|
+
|
38
|
+
if re.match(r".*\.csv", filename):
|
39
|
+
typeid,typedesc = "lastfm", "Last.fm (benjaminbenben export)"
|
38
40
|
importfunc = parse_lastfm
|
39
41
|
|
40
|
-
elif re.match(r"Streaming_History_Audio.+\.json",filename):
|
41
|
-
typeid,typedesc = "spotify","Spotify"
|
42
|
+
elif re.match(r"Streaming_History_Audio.+\.json", filename):
|
43
|
+
typeid,typedesc = "spotify", "Spotify"
|
42
44
|
importfunc = parse_spotify_lite
|
43
45
|
|
44
|
-
elif re.match(r"endsong_[0-9]+\.json",filename):
|
45
|
-
typeid,typedesc = "spotify","Spotify"
|
46
|
+
elif re.match(r"endsong_[0-9]+\.json", filename):
|
47
|
+
typeid,typedesc = "spotify", "Spotify"
|
46
48
|
importfunc = parse_spotify
|
47
49
|
|
48
|
-
elif re.match(r"StreamingHistory[0-9]+\.json",filename):
|
49
|
-
typeid,typedesc = "spotify","Spotify"
|
50
|
+
elif re.match(r"StreamingHistory[0-9]+\.json", filename):
|
51
|
+
typeid,typedesc = "spotify", "Spotify"
|
50
52
|
importfunc = parse_spotify_lite_legacy
|
51
53
|
|
52
|
-
elif re.match(r"maloja_export[_0-9]*\.json",filename):
|
53
|
-
typeid,typedesc = "maloja","Maloja"
|
54
|
+
elif re.match(r"maloja_export[_0-9]*\.json", filename):
|
55
|
+
typeid,typedesc = "maloja", "Maloja"
|
54
56
|
importfunc = parse_maloja
|
55
57
|
|
56
58
|
# username_lb-YYYY-MM-DD.json
|
57
|
-
elif re.match(r".*_lb-[0-9-]+\.json",filename):
|
58
|
-
typeid,typedesc = "listenbrainz","ListenBrainz"
|
59
|
+
elif re.match(r".*_lb-[0-9-]+\.json", filename):
|
60
|
+
typeid,typedesc = "listenbrainz", "ListenBrainz"
|
59
61
|
importfunc = parse_listenbrainz
|
60
62
|
|
61
|
-
elif re.match(r"\.scrobbler\.log",filename):
|
62
|
-
typeid,typedesc = "rockbox","Rockbox"
|
63
|
+
elif re.match(r"\.scrobbler\.log", filename):
|
64
|
+
typeid,typedesc = "rockbox", "Rockbox"
|
63
65
|
importfunc = parse_rockbox
|
64
66
|
|
65
|
-
|
67
|
+
elif re.match(r"recenttracks-.*\.json", filename):
|
68
|
+
typeid, typedesc = "lastfm", "Last.fm (ghan export)"
|
69
|
+
importfunc = parse_lastfm_ghan
|
70
|
+
|
71
|
+
elif re.match(r".*\.json",filename):
|
72
|
+
try:
|
73
|
+
with open(filename,'r') as fd:
|
74
|
+
data = json.load(fd)
|
75
|
+
if 'maloja' in data:
|
76
|
+
typeid,typedesc = "maloja","Maloja"
|
77
|
+
importfunc = parse_maloja
|
78
|
+
except Exception:
|
79
|
+
pass
|
80
|
+
|
81
|
+
if not importfunc:
|
66
82
|
print("File",inputf,"could not be identified as a valid import source.")
|
67
83
|
return result
|
68
84
|
|
@@ -131,6 +147,7 @@ def import_scrobbles(inputf):
|
|
131
147
|
|
132
148
|
return result
|
133
149
|
|
150
|
+
|
134
151
|
def parse_spotify_lite_legacy(inputf):
|
135
152
|
pth = os.path
|
136
153
|
# use absolute paths internally for peace of mind. just change representation for console output
|
@@ -243,6 +260,7 @@ def parse_spotify_lite(inputf):
|
|
243
260
|
|
244
261
|
print()
|
245
262
|
|
263
|
+
|
246
264
|
def parse_spotify(inputf):
|
247
265
|
pth = os.path
|
248
266
|
# use absolute paths internally for peace of mind. just change representation for console output
|
@@ -354,6 +372,7 @@ def parse_spotify(inputf):
|
|
354
372
|
|
355
373
|
print()
|
356
374
|
|
375
|
+
|
357
376
|
def parse_lastfm(inputf):
|
358
377
|
|
359
378
|
with open(inputf,'r',newline='') as inputfd:
|
@@ -388,6 +407,29 @@ def parse_lastfm(inputf):
|
|
388
407
|
yield ('FAIL',None,f"{row} (Line {line}) could not be parsed. Scrobble not imported. ({repr(e)})")
|
389
408
|
continue
|
390
409
|
|
410
|
+
|
411
|
+
def parse_lastfm_ghan(inputf):
|
412
|
+
with open(inputf, 'r') as inputfd:
|
413
|
+
data = json.load(inputfd)
|
414
|
+
|
415
|
+
skip = 50000
|
416
|
+
for entry in data:
|
417
|
+
for track in entry['track']:
|
418
|
+
skip -= 1
|
419
|
+
#if skip: continue
|
420
|
+
#print(track)
|
421
|
+
#input()
|
422
|
+
|
423
|
+
yield ('CONFIDENT_IMPORT', {
|
424
|
+
'track_title': track['name'],
|
425
|
+
'track_artists': track['artist']['#text'],
|
426
|
+
'track_length': None,
|
427
|
+
'album_name': track['album']['#text'],
|
428
|
+
'scrobble_time': int(track['date']['uts']),
|
429
|
+
'scrobble_duration': None
|
430
|
+
}, '')
|
431
|
+
|
432
|
+
|
391
433
|
def parse_listenbrainz(inputf):
|
392
434
|
|
393
435
|
with open(inputf,'r') as inputfd:
|
maloja/server.py
CHANGED
@@ -12,14 +12,13 @@ from jinja2.exceptions import TemplateNotFound
|
|
12
12
|
|
13
13
|
# doreah toolkit
|
14
14
|
from doreah.logging import log
|
15
|
-
from doreah import auth
|
16
15
|
|
17
16
|
# rest of the project
|
18
17
|
from . import database
|
19
18
|
from .database.jinjaview import JinjaDBConnection
|
20
19
|
from .images import image_request
|
21
20
|
from .malojauri import uri_to_internal, remove_identical
|
22
|
-
from .pkg_global.conf import malojaconfig, data_dir
|
21
|
+
from .pkg_global.conf import malojaconfig, data_dir, auth
|
23
22
|
from .pkg_global import conf
|
24
23
|
from .jinjaenv.context import jinja_environment
|
25
24
|
from .apis import init_apis, apikeystore
|
@@ -97,7 +96,7 @@ aliases = {
|
|
97
96
|
|
98
97
|
### API
|
99
98
|
|
100
|
-
auth.authapi.mount(server=webserver)
|
99
|
+
conf.auth.authapi.mount(server=webserver)
|
101
100
|
init_apis(webserver)
|
102
101
|
|
103
102
|
# redirects for backwards compatibility
|
@@ -197,7 +196,7 @@ def jinja_page(name):
|
|
197
196
|
if name in aliases: redirect(aliases[name])
|
198
197
|
keys = remove_identical(FormsDict.decode(request.query))
|
199
198
|
|
200
|
-
adminmode = request.cookies.get("adminmode") == "true" and auth.
|
199
|
+
adminmode = request.cookies.get("adminmode") == "true" and auth.check_request(request)
|
201
200
|
|
202
201
|
with JinjaDBConnection() as conn:
|
203
202
|
|
@@ -222,7 +221,7 @@ def jinja_page(name):
|
|
222
221
|
return res
|
223
222
|
|
224
223
|
@webserver.route("/<name:re:admin.*>")
|
225
|
-
@auth.
|
224
|
+
@auth.authenticated_function()
|
226
225
|
def jinja_page_private(name):
|
227
226
|
return jinja_page(name)
|
228
227
|
|
maloja/setup.py
CHANGED
@@ -6,9 +6,8 @@ try:
|
|
6
6
|
except ImportError:
|
7
7
|
import distutils
|
8
8
|
from doreah.io import col, ask, prompt
|
9
|
-
from doreah import auth
|
10
9
|
|
11
|
-
from .pkg_global.conf import data_dir, dir_settings, malojaconfig
|
10
|
+
from .pkg_global.conf import data_dir, dir_settings, malojaconfig, auth
|
12
11
|
|
13
12
|
|
14
13
|
|
@@ -25,6 +24,12 @@ ext_apikeys = [
|
|
25
24
|
def copy_initial_local_files():
|
26
25
|
with resources.files("maloja") / 'data_files' as folder:
|
27
26
|
for cat in dir_settings:
|
27
|
+
if dir_settings[cat] is None:
|
28
|
+
continue
|
29
|
+
|
30
|
+
if cat == 'config' and malojaconfig.readonly:
|
31
|
+
continue
|
32
|
+
|
28
33
|
distutils.dir_util.copy_tree(os.path.join(folder,cat),dir_settings[cat],update=False)
|
29
34
|
|
30
35
|
charset = list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
|
@@ -37,46 +42,53 @@ def setup():
|
|
37
42
|
copy_initial_local_files()
|
38
43
|
SKIP = malojaconfig["SKIP_SETUP"]
|
39
44
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
# OWN API KEY
|
55
|
-
from .apis import apikeystore
|
56
|
-
if len(apikeystore) == 0:
|
57
|
-
answer = ask("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database.",default=True,skip=SKIP)
|
58
|
-
if answer:
|
59
|
-
key = apikeystore.generate_key('default')
|
60
|
-
print("Your API Key: " + col["yellow"](key))
|
61
|
-
|
62
|
-
# PASSWORD
|
63
|
-
forcepassword = malojaconfig["FORCE_PASSWORD"]
|
64
|
-
# this is mainly meant for docker, supply password via environment variable
|
65
|
-
|
66
|
-
if forcepassword is not None:
|
67
|
-
# user has specified to force the pw, nothing else matters
|
68
|
-
auth.defaultuser.setpw(forcepassword)
|
69
|
-
print("Password has been set.")
|
70
|
-
elif auth.defaultuser.checkpw("admin"):
|
71
|
-
# if the actual pw is admin, it means we've never set this up properly (eg first start after update)
|
72
|
-
while True:
|
73
|
-
newpw = prompt("Please set a password for web backend access. Leave this empty to generate a random password.",skip=SKIP,secret=True)
|
74
|
-
if newpw is None:
|
75
|
-
newpw = randomstring(32)
|
76
|
-
print("Generated password:",col["yellow"](newpw))
|
77
|
-
break
|
45
|
+
try:
|
46
|
+
print("Various external services can be used to display images. If not enough of them are set up, only local images will be used.")
|
47
|
+
for k in ext_apikeys:
|
48
|
+
keyname = malojaconfig.get_setting_info(k)['name']
|
49
|
+
key = malojaconfig[k]
|
50
|
+
if key is False:
|
51
|
+
print(f"\tCurrently not using a {col['red'](keyname)} for image display.")
|
52
|
+
elif key is None or key == "ASK":
|
53
|
+
if malojaconfig.readonly:
|
54
|
+
continue
|
55
|
+
promptmsg = f"\tPlease enter your {col['gold'](keyname)}. If you do not want to use one at this moment, simply leave this empty and press Enter."
|
56
|
+
key = prompt(promptmsg,types=(str,),default=False,skip=SKIP)
|
57
|
+
malojaconfig[k] = key
|
78
58
|
else:
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
59
|
+
print(f"\t{col['lawngreen'](keyname)} found.")
|
60
|
+
|
61
|
+
|
62
|
+
# OWN API KEY
|
63
|
+
from .apis import apikeystore
|
64
|
+
if len(apikeystore) == 0:
|
65
|
+
answer = ask("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database.",default=True,skip=SKIP)
|
66
|
+
if answer:
|
67
|
+
key = apikeystore.generate_key('default')
|
68
|
+
print("Your API Key: " + col["yellow"](key))
|
69
|
+
|
70
|
+
# PASSWORD
|
71
|
+
forcepassword = malojaconfig["FORCE_PASSWORD"]
|
72
|
+
# this is mainly meant for docker, supply password via environment variable
|
73
|
+
|
74
|
+
if forcepassword is not None:
|
75
|
+
# user has specified to force the pw, nothing else matters
|
76
|
+
auth.change_pw(password=forcepassword)
|
77
|
+
print("Password has been set.")
|
78
|
+
elif auth.still_has_factory_default_user():
|
79
|
+
# this means we've never set this up properly (eg first start after update)
|
80
|
+
while True:
|
81
|
+
newpw = prompt("Please set a password for web backend access. Leave this empty to generate a random password.",skip=SKIP,secret=True)
|
82
|
+
if newpw is None:
|
83
|
+
newpw = randomstring(32)
|
84
|
+
print("Generated password:",col["yellow"](newpw))
|
85
|
+
break
|
86
|
+
else:
|
87
|
+
newpw_repeat = prompt("Please type again to confirm.",skip=SKIP,secret=True)
|
88
|
+
if newpw != newpw_repeat: print("Passwords do not match!")
|
89
|
+
else: break
|
90
|
+
auth.change_pw(password=newpw)
|
91
|
+
|
92
|
+
except EOFError:
|
93
|
+
print("No user input possible. If you are running inside a container, set the environment variable",col['yellow']("MALOJA_SKIP_SETUP=yes"))
|
94
|
+
raise SystemExit
|