fractal-server 2.12.0a1__py3-none-any.whl → 2.12.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/__main__.py +17 -63
- fractal_server/app/routes/api/v2/images.py +0 -12
- fractal_server/app/runner/executors/slurm/ssh/_slurm_job.py +0 -1
- fractal_server/app/runner/executors/slurm/ssh/executor.py +14 -23
- fractal_server/app/runner/executors/slurm/sudo/executor.py +52 -17
- fractal_server/app/security/__init__.py +6 -3
- fractal_server/app/security/signup_email.py +21 -12
- fractal_server/config.py +92 -51
- fractal_server/main.py +12 -9
- fractal_server/migrations/env.py +16 -63
- fractal_server/tasks/v2/local/collect.py +9 -8
- fractal_server/tasks/v2/local/deactivate.py +3 -0
- fractal_server/tasks/v2/local/reactivate.py +3 -0
- fractal_server/tasks/v2/ssh/collect.py +8 -8
- fractal_server/tasks/v2/ssh/deactivate.py +3 -0
- fractal_server/tasks/v2/ssh/reactivate.py +9 -6
- {fractal_server-2.12.0a1.dist-info → fractal_server-2.12.1.dist-info}/METADATA +2 -2
- {fractal_server-2.12.0a1.dist-info → fractal_server-2.12.1.dist-info}/RECORD +22 -22
- {fractal_server-2.12.0a1.dist-info → fractal_server-2.12.1.dist-info}/LICENSE +0 -0
- {fractal_server-2.12.0a1.dist-info → fractal_server-2.12.1.dist-info}/WHEEL +0 -0
- {fractal_server-2.12.0a1.dist-info → fractal_server-2.12.1.dist-info}/entry_points.txt +0 -0
fractal_server/__init__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__VERSION__ = "2.12.
|
1
|
+
__VERSION__ = "2.12.1"
|
fractal_server/__main__.py
CHANGED
@@ -63,45 +63,18 @@ update_db_data_parser = subparsers.add_parser(
|
|
63
63
|
description="Apply data-migration script to an existing database.",
|
64
64
|
)
|
65
65
|
|
66
|
-
# fractalctl email-
|
67
|
-
|
68
|
-
"email-
|
66
|
+
# fractalctl encrypt-email-password
|
67
|
+
encrypt_email_password_parser = subparsers.add_parser(
|
68
|
+
"encrypt-email-password",
|
69
69
|
description=(
|
70
70
|
"Generate valid values for environment variables "
|
71
|
-
"
|
71
|
+
"FRACTAL_EMAIL_PASSWORD and FRACTAL_EMAIL_PASSWORD_KEY."
|
72
72
|
),
|
73
73
|
)
|
74
|
-
email_settings_parser.add_argument(
|
75
|
-
"sender",
|
76
|
-
type=str,
|
77
|
-
help="Email of the sender",
|
78
|
-
)
|
79
|
-
email_settings_parser.add_argument(
|
80
|
-
"server",
|
81
|
-
type=str,
|
82
|
-
help="SMPT server used to send emails",
|
83
|
-
)
|
84
|
-
email_settings_parser.add_argument(
|
85
|
-
"port",
|
86
|
-
type=int,
|
87
|
-
help="Port of the SMPT server",
|
88
|
-
)
|
89
|
-
email_settings_parser.add_argument(
|
90
|
-
"instance",
|
91
|
-
type=str,
|
92
|
-
help="Name of the Fractal instance sending emails",
|
93
|
-
)
|
94
|
-
email_settings_parser.add_argument(
|
95
|
-
"--skip-starttls",
|
96
|
-
action="store_true",
|
97
|
-
default=False,
|
98
|
-
help="If set, skip the execution of `starttls` when sending emails",
|
99
|
-
)
|
100
74
|
|
101
75
|
|
102
76
|
def save_openapi(dest="openapi.json"):
|
103
77
|
from fractal_server.main import start_application
|
104
|
-
import json
|
105
78
|
|
106
79
|
app = start_application()
|
107
80
|
openapi_schema = app.openapi()
|
@@ -129,6 +102,11 @@ def set_db(skip_init_data: bool = False):
|
|
129
102
|
from pathlib import Path
|
130
103
|
import fractal_server
|
131
104
|
|
105
|
+
# Check settings
|
106
|
+
settings = Inject(get_settings)
|
107
|
+
settings.check_db()
|
108
|
+
|
109
|
+
# Perform migrations
|
132
110
|
alembic_ini = Path(fractal_server.__file__).parent / "alembic.ini"
|
133
111
|
alembic_args = ["-c", alembic_ini.as_posix(), "upgrade", "head"]
|
134
112
|
print(f"START: Run alembic.config, with argv={alembic_args}")
|
@@ -138,12 +116,10 @@ def set_db(skip_init_data: bool = False):
|
|
138
116
|
if skip_init_data:
|
139
117
|
return
|
140
118
|
|
141
|
-
#
|
119
|
+
# Create default group and user
|
142
120
|
print()
|
143
121
|
_create_first_group()
|
144
122
|
print()
|
145
|
-
# NOTE: It will be fixed with #1739
|
146
|
-
settings = Inject(get_settings)
|
147
123
|
asyncio.run(
|
148
124
|
_create_first_user(
|
149
125
|
email=settings.FRACTAL_DEFAULT_ADMIN_EMAIL,
|
@@ -224,31 +200,15 @@ def update_db_data():
|
|
224
200
|
current_update_db_data_module.fix_db()
|
225
201
|
|
226
202
|
|
227
|
-
def
|
228
|
-
sender: str,
|
229
|
-
server: str,
|
230
|
-
port: int,
|
231
|
-
instance: str,
|
232
|
-
skip_starttls: bool,
|
233
|
-
):
|
203
|
+
def print_encrypted_password():
|
234
204
|
from cryptography.fernet import Fernet
|
235
205
|
|
236
|
-
password = input(
|
206
|
+
password = input("Insert email password: ").encode("utf-8")
|
237
207
|
key = Fernet.generate_key().decode("utf-8")
|
238
|
-
|
239
|
-
dict(
|
240
|
-
sender=sender,
|
241
|
-
password=password,
|
242
|
-
smtp_server=server,
|
243
|
-
port=port,
|
244
|
-
instance_name=instance,
|
245
|
-
use_starttls=(not skip_starttls),
|
246
|
-
)
|
247
|
-
).encode("utf-8")
|
248
|
-
email_settings = Fernet(key).encrypt(fractal_mail_settings).decode("utf-8")
|
208
|
+
encrypted_password = Fernet(key).encrypt(password).decode("utf-8")
|
249
209
|
|
250
|
-
print(f"\
|
251
|
-
print(f"
|
210
|
+
print(f"\nFRACTAL_EMAIL_PASSWORD={encrypted_password}")
|
211
|
+
print(f"FRACTAL_EMAIL_PASSWORD_KEY={key}")
|
252
212
|
|
253
213
|
|
254
214
|
def run():
|
@@ -267,14 +227,8 @@ def run():
|
|
267
227
|
port=args.port,
|
268
228
|
reload=args.reload,
|
269
229
|
)
|
270
|
-
elif args.cmd == "email-
|
271
|
-
|
272
|
-
sender=args.sender,
|
273
|
-
server=args.server,
|
274
|
-
port=args.port,
|
275
|
-
instance=args.instance,
|
276
|
-
skip_starttls=args.skip_starttls,
|
277
|
-
)
|
230
|
+
elif args.cmd == "encrypt-email-password":
|
231
|
+
print_encrypted_password()
|
278
232
|
else:
|
279
233
|
sys.exit(f"Error: invalid command '{args.cmd}'.")
|
280
234
|
|
@@ -118,7 +118,6 @@ async def post_new_image(
|
|
118
118
|
async def query_dataset_images(
|
119
119
|
project_id: int,
|
120
120
|
dataset_id: int,
|
121
|
-
use_dataset_filters: bool = False, # query param
|
122
121
|
page: int = 1, # query param
|
123
122
|
page_size: Optional[int] = None, # query param
|
124
123
|
query: Optional[ImageQuery] = None, # body
|
@@ -138,17 +137,6 @@ async def query_dataset_images(
|
|
138
137
|
dataset = output["dataset"]
|
139
138
|
images = dataset.images
|
140
139
|
|
141
|
-
if use_dataset_filters is True:
|
142
|
-
images = [
|
143
|
-
image
|
144
|
-
for image in images
|
145
|
-
if match_filter(
|
146
|
-
image=image,
|
147
|
-
type_filters=dataset.type_filters,
|
148
|
-
attribute_filters=dataset.attribute_filters,
|
149
|
-
)
|
150
|
-
]
|
151
|
-
|
152
140
|
attributes = {}
|
153
141
|
for image in images:
|
154
142
|
for k, v in image["attributes"].items():
|
@@ -88,7 +88,6 @@ class SlurmJob:
|
|
88
88
|
self,
|
89
89
|
num_tasks_tot: int,
|
90
90
|
slurm_config: SlurmConfig,
|
91
|
-
workflow_task_file_prefix: Optional[str] = None,
|
92
91
|
slurm_file_prefix: Optional[str] = None,
|
93
92
|
wftask_file_prefixes: Optional[tuple[str, ...]] = None,
|
94
93
|
single_task_submission: bool = False,
|
@@ -62,7 +62,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
|
|
62
62
|
shutdown_file:
|
63
63
|
python_remote: Equal to `settings.FRACTAL_SLURM_WORKER_PYTHON`
|
64
64
|
wait_thread_cls: Class for waiting thread
|
65
|
-
keep_pickle_files:
|
66
65
|
workflow_dir_local:
|
67
66
|
Directory for both the cfut/SLURM and fractal-server files and logs
|
68
67
|
workflow_dir_remote:
|
@@ -84,7 +83,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
|
|
84
83
|
python_remote: str
|
85
84
|
|
86
85
|
wait_thread_cls = FractalSlurmWaitThread
|
87
|
-
keep_pickle_files: bool
|
88
86
|
|
89
87
|
common_script_lines: list[str]
|
90
88
|
slurm_account: Optional[str]
|
@@ -100,8 +98,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
|
|
100
98
|
# Folders and files
|
101
99
|
workflow_dir_local: Path,
|
102
100
|
workflow_dir_remote: Path,
|
103
|
-
# Runner options
|
104
|
-
keep_pickle_files: bool = False,
|
105
101
|
# Monitoring options
|
106
102
|
slurm_poll_interval: Optional[int] = None,
|
107
103
|
# SLURM submission script options
|
@@ -120,7 +116,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
|
|
120
116
|
fractal_ssh:
|
121
117
|
workflow_dir_local:
|
122
118
|
workflow_dir_remote:
|
123
|
-
keep_pickle_files:
|
124
119
|
slurm_poll_interval:
|
125
120
|
common_script_lines:
|
126
121
|
slurm_account:
|
@@ -194,7 +189,6 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
|
|
194
189
|
raise e
|
195
190
|
|
196
191
|
# Set/initialize some more options
|
197
|
-
self.keep_pickle_files = keep_pickle_files
|
198
192
|
self.map_jobid_to_slurm_files_local = {}
|
199
193
|
|
200
194
|
def _validate_common_script_lines(self):
|
@@ -901,12 +895,11 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
|
|
901
895
|
pass
|
902
896
|
for job_id in remaining_job_ids:
|
903
897
|
self._cleanup(job_id)
|
904
|
-
|
905
|
-
for
|
906
|
-
|
907
|
-
|
908
|
-
|
909
|
-
path.unlink()
|
898
|
+
for job in remaining_jobs:
|
899
|
+
for path in job.output_pickle_files_local:
|
900
|
+
path.unlink()
|
901
|
+
for path in job.input_pickle_files_local:
|
902
|
+
path.unlink()
|
910
903
|
|
911
904
|
def _completion(self, job_ids: list[str]) -> None:
|
912
905
|
"""
|
@@ -1001,8 +994,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
|
|
1001
994
|
f"Future {future} (SLURM job ID: {job_id}) "
|
1002
995
|
"was already cancelled."
|
1003
996
|
)
|
1004
|
-
|
1005
|
-
in_path.unlink()
|
997
|
+
in_path.unlink()
|
1006
998
|
self._cleanup(job_id)
|
1007
999
|
self._handle_remaining_jobs(
|
1008
1000
|
remaining_futures=remaining_futures,
|
@@ -1062,17 +1054,15 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
|
|
1062
1054
|
remaining_job_ids=remaining_job_ids,
|
1063
1055
|
)
|
1064
1056
|
return
|
1065
|
-
|
1066
|
-
out_path.unlink()
|
1057
|
+
out_path.unlink()
|
1067
1058
|
except InvalidStateError:
|
1068
1059
|
logger.warning(
|
1069
1060
|
f"Future {future} (SLURM job ID: {job_id}) was "
|
1070
1061
|
"already cancelled, exit from "
|
1071
1062
|
"FractalSlurmSSHExecutor._completion."
|
1072
1063
|
)
|
1073
|
-
|
1074
|
-
|
1075
|
-
in_path.unlink()
|
1064
|
+
out_path.unlink()
|
1065
|
+
in_path.unlink()
|
1076
1066
|
|
1077
1067
|
self._cleanup(job_id)
|
1078
1068
|
self._handle_remaining_jobs(
|
@@ -1082,8 +1072,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
|
|
1082
1072
|
return
|
1083
1073
|
|
1084
1074
|
# Clean up input pickle file
|
1085
|
-
|
1086
|
-
in_path.unlink()
|
1075
|
+
in_path.unlink()
|
1087
1076
|
self._cleanup(job_id)
|
1088
1077
|
if job.single_task_submission:
|
1089
1078
|
future.set_result(outputs[0])
|
@@ -1210,8 +1199,10 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
|
|
1210
1199
|
script_lines = slurm_config.sort_script_lines(script_lines)
|
1211
1200
|
logger.debug(script_lines)
|
1212
1201
|
|
1213
|
-
# Always print output of `pwd`
|
1214
|
-
script_lines.append(
|
1202
|
+
# Always print output of `uname -n` and `pwd`
|
1203
|
+
script_lines.append(
|
1204
|
+
'"Hostname: `uname -n`; current directory: `pwd`"\n'
|
1205
|
+
)
|
1215
1206
|
|
1216
1207
|
# Complete script preamble
|
1217
1208
|
script_lines.append("\n")
|
@@ -10,6 +10,7 @@
|
|
10
10
|
#
|
11
11
|
# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
|
12
12
|
# University of Zurich
|
13
|
+
import json
|
13
14
|
import math
|
14
15
|
import shlex
|
15
16
|
import subprocess # nosec
|
@@ -161,7 +162,6 @@ class SlurmJob:
|
|
161
162
|
self,
|
162
163
|
num_tasks_tot: int,
|
163
164
|
slurm_config: SlurmConfig,
|
164
|
-
workflow_task_file_prefix: Optional[str] = None,
|
165
165
|
slurm_file_prefix: Optional[str] = None,
|
166
166
|
wftask_file_prefixes: Optional[tuple[str, ...]] = None,
|
167
167
|
single_task_submission: bool = False,
|
@@ -219,7 +219,6 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
219
219
|
workflow_dir_local: Path
|
220
220
|
workflow_dir_remote: Path
|
221
221
|
map_jobid_to_slurm_files: dict[str, tuple[str, str, str]]
|
222
|
-
keep_pickle_files: bool
|
223
222
|
slurm_account: Optional[str]
|
224
223
|
jobs: dict[str, tuple[Future, SlurmJob]]
|
225
224
|
|
@@ -232,7 +231,6 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
232
231
|
user_cache_dir: Optional[str] = None,
|
233
232
|
common_script_lines: Optional[list[str]] = None,
|
234
233
|
slurm_poll_interval: Optional[int] = None,
|
235
|
-
keep_pickle_files: bool = False,
|
236
234
|
slurm_account: Optional[str] = None,
|
237
235
|
*args,
|
238
236
|
**kwargs,
|
@@ -253,11 +251,18 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
253
251
|
# raised within `__init__`).
|
254
252
|
self.wait_thread.shutdown_callback = self.shutdown
|
255
253
|
|
256
|
-
self.keep_pickle_files = keep_pickle_files
|
257
254
|
self.slurm_user = slurm_user
|
258
255
|
self.slurm_account = slurm_account
|
259
256
|
|
260
257
|
self.common_script_lines = common_script_lines or []
|
258
|
+
settings = Inject(get_settings)
|
259
|
+
|
260
|
+
if settings.FRACTAL_SLURM_WORKER_PYTHON is not None:
|
261
|
+
try:
|
262
|
+
self.check_remote_python_interpreter()
|
263
|
+
except Exception as e:
|
264
|
+
self._stop_and_join_wait_thread()
|
265
|
+
raise RuntimeError(f"Original error {str(e)}")
|
261
266
|
|
262
267
|
# Check that SLURM account is not set here
|
263
268
|
try:
|
@@ -289,7 +294,6 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
289
294
|
# Set the attribute slurm_poll_interval for self.wait_thread (see
|
290
295
|
# cfut.SlurmWaitThread)
|
291
296
|
if not slurm_poll_interval:
|
292
|
-
settings = Inject(get_settings)
|
293
297
|
slurm_poll_interval = settings.FRACTAL_SLURM_POLL_INTERVAL
|
294
298
|
self.wait_thread.slurm_poll_interval = slurm_poll_interval
|
295
299
|
self.wait_thread.slurm_user = self.slurm_user
|
@@ -608,7 +612,14 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
608
612
|
_prefixes = []
|
609
613
|
_subfolder_names = []
|
610
614
|
for component in components:
|
611
|
-
|
615
|
+
# In Fractal, `component` is a `dict` by construction (e.g.
|
616
|
+
# `component = {"zarr_url": "/something", "param": 1}``). The
|
617
|
+
# try/except covers the case of e.g. `executor.map([1, 2])`,
|
618
|
+
# which is useful for testing.
|
619
|
+
try:
|
620
|
+
actual_component = component.get(_COMPONENT_KEY_, None)
|
621
|
+
except AttributeError:
|
622
|
+
actual_component = str(component)
|
612
623
|
_task_file_paths = get_task_file_paths(
|
613
624
|
workflow_dir_local=task_files.workflow_dir_local,
|
614
625
|
workflow_dir_remote=task_files.workflow_dir_remote,
|
@@ -860,8 +871,7 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
860
871
|
" cancelled, exit from"
|
861
872
|
" FractalSlurmExecutor._completion."
|
862
873
|
)
|
863
|
-
|
864
|
-
in_path.unlink()
|
874
|
+
in_path.unlink()
|
865
875
|
self._cleanup(jobid)
|
866
876
|
return
|
867
877
|
|
@@ -903,23 +913,20 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
903
913
|
exc = TaskExecutionError(proxy.tb, **kwargs)
|
904
914
|
fut.set_exception(exc)
|
905
915
|
return
|
906
|
-
|
907
|
-
out_path.unlink()
|
916
|
+
out_path.unlink()
|
908
917
|
except InvalidStateError:
|
909
918
|
logger.warning(
|
910
919
|
f"Future {fut} (SLURM job ID: {jobid}) was already"
|
911
920
|
" cancelled, exit from"
|
912
921
|
" FractalSlurmExecutor._completion."
|
913
922
|
)
|
914
|
-
|
915
|
-
|
916
|
-
in_path.unlink()
|
923
|
+
out_path.unlink()
|
924
|
+
in_path.unlink()
|
917
925
|
self._cleanup(jobid)
|
918
926
|
return
|
919
927
|
|
920
928
|
# Clean up input pickle file
|
921
|
-
|
922
|
-
in_path.unlink()
|
929
|
+
in_path.unlink()
|
923
930
|
self._cleanup(jobid)
|
924
931
|
if job.single_task_submission:
|
925
932
|
fut.set_result(outputs[0])
|
@@ -1155,8 +1162,10 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
1155
1162
|
script_lines = slurm_config.sort_script_lines(script_lines)
|
1156
1163
|
logger.debug(script_lines)
|
1157
1164
|
|
1158
|
-
# Always print output of `pwd`
|
1159
|
-
script_lines.append(
|
1165
|
+
# Always print output of `uname -n` and `pwd`
|
1166
|
+
script_lines.append(
|
1167
|
+
'"Hostname: `uname -n`; current directory: `pwd`"\n'
|
1168
|
+
)
|
1160
1169
|
|
1161
1170
|
# Complete script preamble
|
1162
1171
|
script_lines.append("\n")
|
@@ -1243,3 +1252,29 @@ class FractalSlurmExecutor(SlurmExecutor):
|
|
1243
1252
|
)
|
1244
1253
|
self._stop_and_join_wait_thread()
|
1245
1254
|
logger.debug("[FractalSlurmExecutor.__exit__] End")
|
1255
|
+
|
1256
|
+
def check_remote_python_interpreter(self):
|
1257
|
+
"""
|
1258
|
+
Check fractal-server version on the _remote_ Python interpreter.
|
1259
|
+
"""
|
1260
|
+
settings = Inject(get_settings)
|
1261
|
+
output = _subprocess_run_or_raise(
|
1262
|
+
(
|
1263
|
+
f"{settings.FRACTAL_SLURM_WORKER_PYTHON} "
|
1264
|
+
"-m fractal_server.app.runner.versions"
|
1265
|
+
)
|
1266
|
+
)
|
1267
|
+
runner_version = json.loads(output.stdout.strip("\n"))[
|
1268
|
+
"fractal_server"
|
1269
|
+
]
|
1270
|
+
|
1271
|
+
if runner_version != __VERSION__:
|
1272
|
+
error_msg = (
|
1273
|
+
"Fractal-server version mismatch.\n"
|
1274
|
+
"Local interpreter: "
|
1275
|
+
f"({sys.executable}): {__VERSION__}.\n"
|
1276
|
+
"Remote interpreter: "
|
1277
|
+
f"({settings.FRACTAL_SLURM_WORKER_PYTHON}): {runner_version}."
|
1278
|
+
)
|
1279
|
+
logger.error(error_msg)
|
1280
|
+
raise ValueError(error_msg)
|
@@ -252,15 +252,18 @@ class UserManager(IntegerIDMixin, BaseUserManager[UserOAuth, int]):
|
|
252
252
|
# Send mail section
|
253
253
|
settings = Inject(get_settings)
|
254
254
|
|
255
|
-
if
|
255
|
+
if (
|
256
|
+
this_user.oauth_accounts
|
257
|
+
and settings.email_settings is not None
|
258
|
+
):
|
256
259
|
try:
|
257
260
|
logger.info(
|
258
261
|
"START sending email about new signup to "
|
259
|
-
f"{settings.
|
262
|
+
f"{settings.email_settings.recipients}."
|
260
263
|
)
|
261
264
|
mail_new_oauth_signup(
|
262
265
|
msg=f"New user registered: '{this_user.email}'.",
|
263
|
-
|
266
|
+
email_settings=settings.email_settings,
|
264
267
|
)
|
265
268
|
logger.info("END sending email about new signup.")
|
266
269
|
except Exception as e:
|
@@ -2,38 +2,47 @@ import smtplib
|
|
2
2
|
from email.message import EmailMessage
|
3
3
|
from email.utils import formataddr
|
4
4
|
|
5
|
+
from cryptography.fernet import Fernet
|
6
|
+
|
5
7
|
from fractal_server.config import MailSettings
|
6
8
|
|
7
9
|
|
8
|
-
def mail_new_oauth_signup(msg: str,
|
10
|
+
def mail_new_oauth_signup(msg: str, email_settings: MailSettings):
|
9
11
|
"""
|
10
12
|
Send an email using the specified settings to notify a new OAuth signup.
|
11
13
|
"""
|
12
14
|
|
13
15
|
mail_msg = EmailMessage()
|
14
16
|
mail_msg.set_content(msg)
|
15
|
-
mail_msg["From"] = formataddr(
|
17
|
+
mail_msg["From"] = formataddr(
|
18
|
+
(email_settings.sender, email_settings.sender)
|
19
|
+
)
|
16
20
|
mail_msg["To"] = ", ".join(
|
17
21
|
[
|
18
22
|
formataddr((recipient, recipient))
|
19
|
-
for recipient in
|
23
|
+
for recipient in email_settings.recipients
|
20
24
|
]
|
21
25
|
)
|
22
26
|
mail_msg[
|
23
27
|
"Subject"
|
24
|
-
] = f"[Fractal, {
|
28
|
+
] = f"[Fractal, {email_settings.instance_name}] New OAuth signup"
|
25
29
|
|
26
|
-
with smtplib.SMTP(
|
30
|
+
with smtplib.SMTP(
|
31
|
+
email_settings.smtp_server, email_settings.port
|
32
|
+
) as server:
|
27
33
|
server.ehlo()
|
28
|
-
if
|
34
|
+
if email_settings.use_starttls:
|
29
35
|
server.starttls()
|
30
36
|
server.ehlo()
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
37
|
+
if email_settings.use_login:
|
38
|
+
password = (
|
39
|
+
Fernet(email_settings.encryption_key)
|
40
|
+
.decrypt(email_settings.encrypted_password)
|
41
|
+
.decode("utf-8")
|
42
|
+
)
|
43
|
+
server.login(user=email_settings.sender, password=password)
|
35
44
|
server.sendmail(
|
36
|
-
from_addr=
|
37
|
-
to_addrs=
|
45
|
+
from_addr=email_settings.sender,
|
46
|
+
to_addrs=email_settings.recipients,
|
38
47
|
msg=mail_msg.as_string(),
|
39
48
|
)
|
fractal_server/config.py
CHANGED
@@ -11,7 +11,6 @@
|
|
11
11
|
# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
|
12
12
|
# Institute for Biomedical Research and Pelkmans Lab from the University of
|
13
13
|
# Zurich.
|
14
|
-
import json
|
15
14
|
import logging
|
16
15
|
import shutil
|
17
16
|
import sys
|
@@ -46,16 +45,19 @@ class MailSettings(BaseModel):
|
|
46
45
|
port: SMTP server port
|
47
46
|
password: Sender password
|
48
47
|
instance_name: Name of SMTP server instance
|
49
|
-
use_starttls:
|
48
|
+
use_starttls: Whether to use the security protocol
|
49
|
+
use_login: Whether to use login
|
50
50
|
"""
|
51
51
|
|
52
52
|
sender: EmailStr
|
53
53
|
recipients: list[EmailStr] = Field(min_items=1)
|
54
54
|
smtp_server: str
|
55
55
|
port: int
|
56
|
-
|
56
|
+
encrypted_password: Optional[str] = None
|
57
|
+
encryption_key: Optional[str] = None
|
57
58
|
instance_name: str
|
58
59
|
use_starttls: bool
|
60
|
+
use_login: bool
|
59
61
|
|
60
62
|
|
61
63
|
class FractalConfigurationError(RuntimeError):
|
@@ -406,7 +408,7 @@ class Settings(BaseSettings):
|
|
406
408
|
"""
|
407
409
|
|
408
410
|
@root_validator(pre=True)
|
409
|
-
def check_tasks_python(cls, values)
|
411
|
+
def check_tasks_python(cls, values):
|
410
412
|
"""
|
411
413
|
Perform multiple checks of the Python-interpreter variables.
|
412
414
|
|
@@ -416,7 +418,6 @@ class Settings(BaseSettings):
|
|
416
418
|
`sys.executable` and set the corresponding
|
417
419
|
`FRACTAL_TASKS_PYTHON_X_Y` (and unset all others).
|
418
420
|
"""
|
419
|
-
|
420
421
|
# `FRACTAL_TASKS_PYTHON_X_Y` variables can only be absolute paths
|
421
422
|
for version in ["3_9", "3_10", "3_11", "3_12"]:
|
422
423
|
key = f"FRACTAL_TASKS_PYTHON_{version}"
|
@@ -575,66 +576,107 @@ class Settings(BaseSettings):
|
|
575
576
|
###########################################################################
|
576
577
|
# SMTP SERVICE
|
577
578
|
###########################################################################
|
578
|
-
|
579
|
+
|
580
|
+
FRACTAL_EMAIL_SENDER: Optional[EmailStr] = None
|
581
|
+
"""
|
582
|
+
Address of the OAuth-signup email sender.
|
579
583
|
"""
|
580
|
-
|
581
|
-
`smtp_server`, `port`, `instance_name`, `use_starttls`.
|
584
|
+
FRACTAL_EMAIL_PASSWORD: Optional[str] = None
|
582
585
|
"""
|
583
|
-
|
586
|
+
Password for the OAuth-signup email sender.
|
587
|
+
"""
|
588
|
+
FRACTAL_EMAIL_PASSWORD_KEY: Optional[str] = None
|
584
589
|
"""
|
585
590
|
Key value for `cryptography.fernet` decrypt
|
586
591
|
"""
|
592
|
+
FRACTAL_EMAIL_SMTP_SERVER: Optional[str] = None
|
593
|
+
"""
|
594
|
+
SMPT server for the OAuth-signup emails.
|
595
|
+
"""
|
596
|
+
FRACTAL_EMAIL_SMTP_PORT: Optional[int] = None
|
597
|
+
"""
|
598
|
+
SMPT server port for the OAuth-signup emails.
|
599
|
+
"""
|
600
|
+
FRACTAL_EMAIL_INSTANCE_NAME: Optional[str] = None
|
601
|
+
"""
|
602
|
+
Fractal instance name, to be included in the OAuth-signup emails.
|
603
|
+
"""
|
587
604
|
FRACTAL_EMAIL_RECIPIENTS: Optional[str] = None
|
588
605
|
"""
|
589
|
-
|
606
|
+
Comma-separated list of recipients of the OAuth-signup emails.
|
607
|
+
"""
|
608
|
+
FRACTAL_EMAIL_USE_STARTTLS: Optional[bool] = True
|
609
|
+
"""
|
610
|
+
Whether to use StartTLS when using the SMTP server.
|
611
|
+
"""
|
612
|
+
FRACTAL_EMAIL_USE_LOGIN: Optional[bool] = True
|
590
613
|
"""
|
614
|
+
Whether to use login when using the SMTP server.
|
615
|
+
"""
|
616
|
+
email_settings: Optional[MailSettings] = None
|
591
617
|
|
592
|
-
@
|
593
|
-
def
|
594
|
-
|
595
|
-
|
596
|
-
|
597
|
-
|
598
|
-
|
599
|
-
|
600
|
-
|
601
|
-
|
602
|
-
|
603
|
-
)
|
604
|
-
|
605
|
-
|
606
|
-
|
607
|
-
)
|
608
|
-
|
609
|
-
|
610
|
-
|
611
|
-
|
612
|
-
|
613
|
-
|
614
|
-
|
615
|
-
|
616
|
-
|
617
|
-
|
618
|
-
|
619
|
-
|
620
|
-
|
618
|
+
@root_validator(pre=True)
|
619
|
+
def validate_email_settings(cls, values):
|
620
|
+
email_values = {
|
621
|
+
k: v for k, v in values.items() if k.startswith("FRACTAL_EMAIL")
|
622
|
+
}
|
623
|
+
if email_values:
|
624
|
+
|
625
|
+
def assert_key(key: str):
|
626
|
+
if key not in email_values:
|
627
|
+
raise ValueError(f"Missing '{key}'")
|
628
|
+
|
629
|
+
assert_key("FRACTAL_EMAIL_SENDER")
|
630
|
+
assert_key("FRACTAL_EMAIL_SMTP_SERVER")
|
631
|
+
assert_key("FRACTAL_EMAIL_SMTP_PORT")
|
632
|
+
assert_key("FRACTAL_EMAIL_INSTANCE_NAME")
|
633
|
+
assert_key("FRACTAL_EMAIL_RECIPIENTS")
|
634
|
+
|
635
|
+
if email_values.get("FRACTAL_EMAIL_USE_LOGIN", True):
|
636
|
+
if "FRACTAL_EMAIL_PASSWORD" not in email_values:
|
637
|
+
raise ValueError(
|
638
|
+
"'FRACTAL_EMAIL_USE_LOGIN' is True but "
|
639
|
+
"'FRACTAL_EMAIL_PASSWORD' is not provided."
|
640
|
+
)
|
641
|
+
elif "FRACTAL_EMAIL_PASSWORD_KEY" not in email_values:
|
642
|
+
raise ValueError(
|
643
|
+
"'FRACTAL_EMAIL_USE_LOGIN' is True but "
|
644
|
+
"'FRACTAL_EMAIL_PASSWORD_KEY' is not provided."
|
645
|
+
)
|
646
|
+
else:
|
647
|
+
try:
|
648
|
+
(
|
649
|
+
Fernet(email_values["FRACTAL_EMAIL_PASSWORD_KEY"])
|
650
|
+
.decrypt(email_values["FRACTAL_EMAIL_PASSWORD"])
|
651
|
+
.decode("utf-8")
|
652
|
+
)
|
653
|
+
except Exception as e:
|
654
|
+
raise ValueError(
|
655
|
+
"Invalid pair (FRACTAL_EMAIL_PASSWORD, "
|
656
|
+
"FRACTAL_EMAIL_PASSWORD_KEY). "
|
657
|
+
f"Original error: {str(e)}."
|
658
|
+
)
|
659
|
+
|
660
|
+
values["email_settings"] = MailSettings(
|
661
|
+
sender=email_values["FRACTAL_EMAIL_SENDER"],
|
662
|
+
recipients=email_values["FRACTAL_EMAIL_RECIPIENTS"].split(","),
|
663
|
+
smtp_server=email_values["FRACTAL_EMAIL_SMTP_SERVER"],
|
664
|
+
port=email_values["FRACTAL_EMAIL_SMTP_PORT"],
|
665
|
+
encrypted_password=email_values.get("FRACTAL_EMAIL_PASSWORD"),
|
666
|
+
encryption_key=email_values.get("FRACTAL_EMAIL_PASSWORD_KEY"),
|
667
|
+
instance_name=email_values["FRACTAL_EMAIL_INSTANCE_NAME"],
|
668
|
+
use_starttls=email_values.get(
|
669
|
+
"FRACTAL_EMAIL_USE_STARTTLS", True
|
670
|
+
),
|
671
|
+
use_login=email_values.get("FRACTAL_EMAIL_USE_LOGIN", True),
|
621
672
|
)
|
622
673
|
|
674
|
+
return values
|
675
|
+
|
623
676
|
###########################################################################
|
624
677
|
# BUSINESS LOGIC
|
625
678
|
###########################################################################
|
626
679
|
|
627
|
-
def check_fractal_mail_settings(self):
|
628
|
-
"""
|
629
|
-
Checks that the mail settings are properly set.
|
630
|
-
"""
|
631
|
-
try:
|
632
|
-
self.MAIL_SETTINGS
|
633
|
-
except Exception as e:
|
634
|
-
raise FractalConfigurationError(
|
635
|
-
f"Invalid email configuration settings. Original error: {e}"
|
636
|
-
)
|
637
|
-
|
638
680
|
def check_db(self) -> None:
|
639
681
|
"""
|
640
682
|
Checks that db environment variables are properly set.
|
@@ -740,7 +782,6 @@ class Settings(BaseSettings):
|
|
740
782
|
|
741
783
|
self.check_db()
|
742
784
|
self.check_runner()
|
743
|
-
self.check_fractal_mail_settings()
|
744
785
|
|
745
786
|
def get_sanitized(self) -> dict:
|
746
787
|
def _must_be_sanitized(string) -> bool:
|
fractal_server/main.py
CHANGED
@@ -28,6 +28,7 @@ from .logger import get_logger
|
|
28
28
|
from .logger import reset_logger_handlers
|
29
29
|
from .logger import set_logger
|
30
30
|
from .syringe import Inject
|
31
|
+
from fractal_server import __VERSION__
|
31
32
|
|
32
33
|
|
33
34
|
def collect_routers(app: FastAPI) -> None:
|
@@ -67,7 +68,7 @@ def check_settings() -> None:
|
|
67
68
|
logger = set_logger("fractal_server_settings")
|
68
69
|
logger.debug("Fractal Settings:")
|
69
70
|
for key, value in settings.dict().items():
|
70
|
-
if any(s in key.upper() for s in ["PASSWORD", "SECRET"]):
|
71
|
+
if any(s in key.upper() for s in ["PASSWORD", "SECRET", "KEY"]):
|
71
72
|
value = "*****"
|
72
73
|
logger.debug(f" {key}: {value}")
|
73
74
|
reset_logger_handlers(logger)
|
@@ -77,7 +78,7 @@ def check_settings() -> None:
|
|
77
78
|
async def lifespan(app: FastAPI):
|
78
79
|
app.state.jobsV2 = []
|
79
80
|
logger = set_logger("fractal_server.lifespan")
|
80
|
-
logger.info("
|
81
|
+
logger.info(f"[startup] START (fractal-server {__VERSION__})")
|
81
82
|
check_settings()
|
82
83
|
settings = Inject(get_settings)
|
83
84
|
|
@@ -88,31 +89,31 @@ async def lifespan(app: FastAPI):
|
|
88
89
|
app.state.fractal_ssh_list = FractalSSHList()
|
89
90
|
|
90
91
|
logger.info(
|
91
|
-
"Added empty FractalSSHList to app.state "
|
92
|
+
"[startup] Added empty FractalSSHList to app.state "
|
92
93
|
f"(id={id(app.state.fractal_ssh_list)})."
|
93
94
|
)
|
94
95
|
else:
|
95
96
|
app.state.fractal_ssh_list = None
|
96
97
|
|
97
98
|
config_uvicorn_loggers()
|
98
|
-
logger.info("
|
99
|
+
logger.info("[startup] END")
|
99
100
|
reset_logger_handlers(logger)
|
100
101
|
|
101
102
|
yield
|
102
103
|
|
103
104
|
logger = get_logger("fractal_server.lifespan")
|
104
|
-
logger.info("
|
105
|
+
logger.info("[teardown] START")
|
105
106
|
|
106
107
|
if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
|
107
108
|
logger.info(
|
108
|
-
"Close FractalSSH connections "
|
109
|
+
"[teardown] Close FractalSSH connections "
|
109
110
|
f"(current size: {app.state.fractal_ssh_list.size})."
|
110
111
|
)
|
111
112
|
|
112
113
|
app.state.fractal_ssh_list.close_all()
|
113
114
|
|
114
115
|
logger.info(
|
115
|
-
f"Current worker with pid {os.getpid()} is shutting down. "
|
116
|
+
f"[teardown] Current worker with pid {os.getpid()} is shutting down. "
|
116
117
|
f"Current jobs: {app.state.jobsV2=}"
|
117
118
|
)
|
118
119
|
if _backend_supports_shutdown(settings.FRACTAL_RUNNER_BACKEND):
|
@@ -128,9 +129,11 @@ async def lifespan(app: FastAPI):
|
|
128
129
|
f"Original error: {e}"
|
129
130
|
)
|
130
131
|
else:
|
131
|
-
logger.info(
|
132
|
+
logger.info(
|
133
|
+
"[teardown] Shutdown not available for this backend runner."
|
134
|
+
)
|
132
135
|
|
133
|
-
logger.info("
|
136
|
+
logger.info("[teardown] END")
|
134
137
|
reset_logger_handlers(logger)
|
135
138
|
|
136
139
|
|
fractal_server/migrations/env.py
CHANGED
@@ -1,16 +1,11 @@
|
|
1
|
-
import asyncio
|
2
1
|
from logging.config import fileConfig
|
3
2
|
|
4
3
|
from alembic import context
|
5
|
-
from sqlalchemy.engine import Connection
|
6
4
|
from sqlmodel import SQLModel
|
7
5
|
|
8
|
-
from fractal_server.config import get_settings
|
9
6
|
from fractal_server.migrations.naming_convention import NAMING_CONVENTION
|
10
|
-
from fractal_server.syringe import Inject
|
11
7
|
|
12
|
-
#
|
13
|
-
# access to the values within the .ini file in use.
|
8
|
+
# Alembic Config object (provides access to the values within the .ini file)
|
14
9
|
config = context.config
|
15
10
|
|
16
11
|
|
@@ -20,77 +15,35 @@ if config.config_file_name is not None:
|
|
20
15
|
fileConfig(config.config_file_name)
|
21
16
|
|
22
17
|
|
23
|
-
# add your model's MetaData object here
|
24
|
-
# for 'autogenerate' support
|
25
|
-
# from myapp import mymodel
|
26
|
-
# target_metadata = mymodel.Base.metadata
|
27
18
|
target_metadata = SQLModel.metadata
|
28
19
|
target_metadata.naming_convention = NAMING_CONVENTION
|
29
|
-
# Importing `fractal_server.app.models` after defining
|
20
|
+
# Importing `fractal_server.app.models` *after* defining
|
30
21
|
# `SQLModel.metadata.naming_convention` in order to apply the naming convention
|
31
22
|
# when autogenerating migrations (see issue #1819).
|
32
23
|
from fractal_server.app import models # noqa
|
33
24
|
|
34
|
-
# other values from the config, defined by the needs of env.py,
|
35
|
-
# can be acquired:
|
36
|
-
# my_important_option = config.get_main_option("my_important_option")
|
37
|
-
# ... etc.
|
38
|
-
|
39
|
-
|
40
|
-
def run_migrations_offline() -> None:
|
41
|
-
"""Run migrations in 'offline' mode.
|
42
|
-
|
43
|
-
This configures the context with just a URL
|
44
|
-
and not an Engine, though an Engine is acceptable
|
45
|
-
here as well. By skipping the Engine creation
|
46
|
-
we don't even need a DBAPI to be available.
|
47
|
-
|
48
|
-
Calls to context.execute() here emit the given string to the
|
49
|
-
script output.
|
50
25
|
|
26
|
+
def run_migrations_online() -> None:
|
51
27
|
"""
|
52
|
-
|
53
|
-
settings.check_db()
|
54
|
-
context.configure(
|
55
|
-
url=settings.DATABASE_ASYNC_URL,
|
56
|
-
target_metadata=target_metadata,
|
57
|
-
literal_binds=True,
|
58
|
-
dialect_opts={"paramstyle": "named"},
|
59
|
-
render_as_batch=True,
|
60
|
-
)
|
61
|
-
|
62
|
-
with context.begin_transaction():
|
63
|
-
context.run_migrations()
|
64
|
-
|
65
|
-
|
66
|
-
def do_run_migrations(connection: Connection) -> None:
|
67
|
-
context.configure(
|
68
|
-
connection=connection,
|
69
|
-
target_metadata=target_metadata,
|
70
|
-
render_as_batch=True,
|
71
|
-
)
|
72
|
-
|
73
|
-
with context.begin_transaction():
|
74
|
-
context.run_migrations()
|
75
|
-
|
76
|
-
|
77
|
-
async def run_migrations_online() -> None:
|
78
|
-
"""Run migrations in 'online' mode.
|
28
|
+
Run migrations in 'online' mode.
|
79
29
|
|
80
30
|
In this scenario we need to create an Engine
|
81
31
|
and associate a connection with the context.
|
82
|
-
|
83
32
|
"""
|
84
33
|
from fractal_server.app.db import DB
|
85
34
|
|
86
|
-
engine = DB.
|
87
|
-
|
88
|
-
|
35
|
+
engine = DB.engine_sync()
|
36
|
+
with engine.connect() as connection:
|
37
|
+
context.configure(
|
38
|
+
connection=connection,
|
39
|
+
target_metadata=target_metadata,
|
40
|
+
render_as_batch=True,
|
41
|
+
)
|
42
|
+
|
43
|
+
with context.begin_transaction():
|
44
|
+
context.run_migrations()
|
89
45
|
|
90
|
-
|
46
|
+
engine.dispose()
|
91
47
|
|
92
48
|
|
93
|
-
|
94
|
-
run_migrations_offline()
|
95
|
-
else:
|
96
|
-
asyncio.run(run_migrations_online())
|
49
|
+
run_migrations_online()
|
@@ -15,6 +15,7 @@ from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
|
15
15
|
from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
|
16
16
|
from fractal_server.app.schemas.v2 import WheelFile
|
17
17
|
from fractal_server.app.schemas.v2.manifest import ManifestV2
|
18
|
+
from fractal_server.logger import reset_logger_handlers
|
18
19
|
from fractal_server.logger import set_logger
|
19
20
|
from fractal_server.tasks.utils import get_log_path
|
20
21
|
from fractal_server.tasks.v2.local._utils import check_task_files_exist
|
@@ -80,7 +81,7 @@ def collect_local(
|
|
80
81
|
return
|
81
82
|
|
82
83
|
# Log some info
|
83
|
-
logger.
|
84
|
+
logger.info("START")
|
84
85
|
for key, value in task_group.model_dump().items():
|
85
86
|
logger.debug(f"task_group.{key}: {value}")
|
86
87
|
|
@@ -101,7 +102,7 @@ def collect_local(
|
|
101
102
|
try:
|
102
103
|
# Create task_group.path folder
|
103
104
|
Path(task_group.path).mkdir(parents=True)
|
104
|
-
logger.
|
105
|
+
logger.info(f"Created {task_group.path}")
|
105
106
|
|
106
107
|
# Write wheel file and set task_group.wheel_path
|
107
108
|
if wheel_file is not None:
|
@@ -109,9 +110,7 @@ def collect_local(
|
|
109
110
|
wheel_path = (
|
110
111
|
Path(task_group.path) / wheel_file.filename
|
111
112
|
).as_posix()
|
112
|
-
logger.
|
113
|
-
f"Write wheel-file contents into {wheel_path}"
|
114
|
-
)
|
113
|
+
logger.info(f"Write wheel-file contents into {wheel_path}")
|
115
114
|
with open(wheel_path, "wb") as f:
|
116
115
|
f.write(wheel_file.contents)
|
117
116
|
task_group.wheel_path = wheel_path
|
@@ -256,12 +255,14 @@ def collect_local(
|
|
256
255
|
)
|
257
256
|
|
258
257
|
# Finalize (write metadata to DB)
|
259
|
-
logger.
|
258
|
+
logger.info("finalising - START")
|
260
259
|
activity.status = TaskGroupActivityStatusV2.OK
|
261
260
|
activity.timestamp_ended = get_timestamp()
|
262
261
|
activity = add_commit_refresh(obj=activity, db=db)
|
263
|
-
logger.
|
264
|
-
logger.
|
262
|
+
logger.info("finalising - END")
|
263
|
+
logger.info("END")
|
264
|
+
|
265
|
+
reset_logger_handlers(logger)
|
265
266
|
|
266
267
|
except Exception as collection_e:
|
267
268
|
# Delete corrupted package dir
|
@@ -14,6 +14,7 @@ from fractal_server.app.models.v2 import TaskGroupV2
|
|
14
14
|
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
15
15
|
from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
|
16
16
|
from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
|
17
|
+
from fractal_server.logger import reset_logger_handlers
|
17
18
|
from fractal_server.logger import set_logger
|
18
19
|
from fractal_server.tasks.utils import FORBIDDEN_DEPENDENCY_STRINGS
|
19
20
|
from fractal_server.tasks.utils import get_log_path
|
@@ -217,6 +218,8 @@ def deactivate_local(
|
|
217
218
|
activity.timestamp_ended = get_timestamp()
|
218
219
|
activity = add_commit_refresh(obj=activity, db=db)
|
219
220
|
|
221
|
+
reset_logger_handlers(logger)
|
222
|
+
|
220
223
|
except Exception as e:
|
221
224
|
fail_and_cleanup(
|
222
225
|
task_group=task_group,
|
@@ -13,6 +13,7 @@ from fractal_server.app.models.v2 import TaskGroupActivityV2
|
|
13
13
|
from fractal_server.app.models.v2 import TaskGroupV2
|
14
14
|
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
15
15
|
from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
|
16
|
+
from fractal_server.logger import reset_logger_handlers
|
16
17
|
from fractal_server.logger import set_logger
|
17
18
|
from fractal_server.tasks.utils import get_log_path
|
18
19
|
from fractal_server.tasks.v2.utils_background import get_current_log
|
@@ -134,6 +135,8 @@ def reactivate_local(
|
|
134
135
|
task_group = add_commit_refresh(obj=task_group, db=db)
|
135
136
|
logger.debug("END")
|
136
137
|
|
138
|
+
reset_logger_handlers(logger)
|
139
|
+
|
137
140
|
except Exception as reactivate_e:
|
138
141
|
# Delete corrupted venv_path
|
139
142
|
try:
|
@@ -14,6 +14,7 @@ from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
|
14
14
|
from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
|
15
15
|
from fractal_server.app.schemas.v2 import WheelFile
|
16
16
|
from fractal_server.app.schemas.v2.manifest import ManifestV2
|
17
|
+
from fractal_server.logger import reset_logger_handlers
|
17
18
|
from fractal_server.logger import set_logger
|
18
19
|
from fractal_server.ssh._fabric import FractalSSH
|
19
20
|
from fractal_server.tasks.v2.ssh._utils import _customize_and_run_template
|
@@ -85,7 +86,7 @@ def collect_ssh(
|
|
85
86
|
return
|
86
87
|
|
87
88
|
# Log some info
|
88
|
-
logger.
|
89
|
+
logger.info("START")
|
89
90
|
for key, value in task_group.model_dump().items():
|
90
91
|
logger.debug(f"task_group.{key}: {value}")
|
91
92
|
|
@@ -137,7 +138,7 @@ def collect_ssh(
|
|
137
138
|
Path(task_group.path) / wheel_filename
|
138
139
|
).as_posix()
|
139
140
|
tmp_wheel_path = (Path(tmpdir) / wheel_filename).as_posix()
|
140
|
-
logger.
|
141
|
+
logger.info(
|
141
142
|
f"Write wheel-file contents into {tmp_wheel_path}"
|
142
143
|
)
|
143
144
|
with open(tmp_wheel_path, "wb") as f:
|
@@ -171,7 +172,7 @@ def collect_ssh(
|
|
171
172
|
logger_name=LOGGER_NAME,
|
172
173
|
)
|
173
174
|
|
174
|
-
logger.
|
175
|
+
logger.info("installing - START")
|
175
176
|
|
176
177
|
# Set status to ONGOING and refresh logs
|
177
178
|
activity.status = TaskGroupActivityStatusV2.ONGOING
|
@@ -286,14 +287,13 @@ def collect_ssh(
|
|
286
287
|
)
|
287
288
|
|
288
289
|
# Finalize (write metadata to DB)
|
289
|
-
logger.
|
290
|
+
logger.info("finalising - START")
|
290
291
|
activity.status = TaskGroupActivityStatusV2.OK
|
291
292
|
activity.timestamp_ended = get_timestamp()
|
292
293
|
activity = add_commit_refresh(obj=activity, db=db)
|
293
|
-
logger.
|
294
|
-
logger.
|
295
|
-
|
296
|
-
logger.debug("END")
|
294
|
+
logger.info("finalising - END")
|
295
|
+
logger.info("END")
|
296
|
+
reset_logger_handlers(logger)
|
297
297
|
|
298
298
|
except Exception as collection_e:
|
299
299
|
# Delete corrupted package dir
|
@@ -14,6 +14,7 @@ from fractal_server.app.models.v2 import TaskGroupV2
|
|
14
14
|
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
15
15
|
from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
|
16
16
|
from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
|
17
|
+
from fractal_server.logger import reset_logger_handlers
|
17
18
|
from fractal_server.logger import set_logger
|
18
19
|
from fractal_server.ssh._fabric import FractalSSH
|
19
20
|
from fractal_server.tasks.utils import FORBIDDEN_DEPENDENCY_STRINGS
|
@@ -252,6 +253,8 @@ def deactivate_ssh(
|
|
252
253
|
activity.timestamp_ended = get_timestamp()
|
253
254
|
activity = add_commit_refresh(obj=activity, db=db)
|
254
255
|
|
256
|
+
reset_logger_handlers(logger)
|
257
|
+
|
255
258
|
except Exception as e:
|
256
259
|
fail_and_cleanup(
|
257
260
|
task_group=task_group,
|
@@ -12,6 +12,7 @@ from fractal_server.app.models.v2 import TaskGroupActivityV2
|
|
12
12
|
from fractal_server.app.models.v2 import TaskGroupV2
|
13
13
|
from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
|
14
14
|
from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
|
15
|
+
from fractal_server.logger import reset_logger_handlers
|
15
16
|
from fractal_server.logger import set_logger
|
16
17
|
from fractal_server.ssh._fabric import FractalSSH
|
17
18
|
from fractal_server.tasks.utils import get_log_path
|
@@ -69,7 +70,7 @@ def reactivate_ssh(
|
|
69
70
|
return
|
70
71
|
|
71
72
|
# Log some info
|
72
|
-
logger.
|
73
|
+
logger.info("START")
|
73
74
|
for key, value in task_group.model_dump().items():
|
74
75
|
logger.debug(f"task_group.{key}: {value}")
|
75
76
|
|
@@ -152,28 +153,30 @@ def reactivate_ssh(
|
|
152
153
|
# Create remote directory for scripts
|
153
154
|
fractal_ssh.mkdir(folder=script_dir_remote)
|
154
155
|
|
155
|
-
logger.
|
156
|
+
logger.info("start - create venv")
|
156
157
|
_customize_and_run_template(
|
157
158
|
template_filename="1_create_venv.sh",
|
158
159
|
**common_args,
|
159
160
|
)
|
160
|
-
logger.
|
161
|
+
logger.info("end - create venv")
|
161
162
|
activity.log = get_current_log(log_file_path)
|
162
163
|
activity = add_commit_refresh(obj=activity, db=db)
|
163
164
|
|
164
|
-
logger.
|
165
|
+
logger.info("start - install from pip freeze")
|
165
166
|
_customize_and_run_template(
|
166
167
|
template_filename="6_pip_install_from_freeze.sh",
|
167
168
|
**common_args,
|
168
169
|
)
|
169
|
-
logger.
|
170
|
+
logger.info("end - install from pip freeze")
|
170
171
|
activity.log = get_current_log(log_file_path)
|
171
172
|
activity.status = TaskGroupActivityStatusV2.OK
|
172
173
|
activity.timestamp_ended = get_timestamp()
|
173
174
|
activity = add_commit_refresh(obj=activity, db=db)
|
174
175
|
task_group.active = True
|
175
176
|
task_group = add_commit_refresh(obj=task_group, db=db)
|
176
|
-
logger.
|
177
|
+
logger.info("END")
|
178
|
+
|
179
|
+
reset_logger_handlers(logger)
|
177
180
|
|
178
181
|
except Exception as reactivate_e:
|
179
182
|
# Delete corrupted venv_path
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.3
|
2
2
|
Name: fractal-server
|
3
|
-
Version: 2.12.
|
3
|
+
Version: 2.12.1
|
4
4
|
Summary: Backend component of the Fractal analytics platform
|
5
5
|
Home-page: https://github.com/fractal-analytics-platform/fractal-server
|
6
6
|
License: BSD-3-Clause
|
@@ -27,7 +27,7 @@ Requires-Dist: pydantic (>=1.10.8,<2)
|
|
27
27
|
Requires-Dist: python-dotenv (>=1.0.0,<1.1.0)
|
28
28
|
Requires-Dist: sqlalchemy[asyncio] (>=2.0.23,<2.1)
|
29
29
|
Requires-Dist: sqlmodel (==0.0.21)
|
30
|
-
Requires-Dist: uvicorn (
|
30
|
+
Requires-Dist: uvicorn (>=0.29.0,<0.35.0)
|
31
31
|
Requires-Dist: uvicorn-worker (==0.2.0)
|
32
32
|
Project-URL: Documentation, https://fractal-analytics-platform.github.io/fractal-server
|
33
33
|
Project-URL: Repository, https://github.com/fractal-analytics-platform/fractal-server
|
@@ -1,5 +1,5 @@
|
|
1
|
-
fractal_server/__init__.py,sha256=
|
2
|
-
fractal_server/__main__.py,sha256=
|
1
|
+
fractal_server/__init__.py,sha256=xmnpQEc5CF2JQ6rKqQzxa_0Lpy_QA2cHgLGOu73fTGo,23
|
2
|
+
fractal_server/__main__.py,sha256=igfS2XL3e8JycuhASl2vsYuIPma0MG0cfPPFRuQfh14,6906
|
3
3
|
fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
|
4
4
|
fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
5
5
|
fractal_server/app/db/__init__.py,sha256=wup2wcOkyOh8Vd0Xm76PZn_naxeMqaL4eF8DHHXTGlI,2889
|
@@ -30,7 +30,7 @@ fractal_server/app/routes/api/v2/_aux_functions.py,sha256=NJ6_1biN_hhIEK1w8Vj6Xh
|
|
30
30
|
fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py,sha256=c8eqPXdMhc3nIixX50B1Ka5n7LgbOZm2JbEs7lICQ04,6767
|
31
31
|
fractal_server/app/routes/api/v2/_aux_functions_tasks.py,sha256=uhNSs-jcS7ndIUFKiOC1yrDiViw3uvKEXi9UL04BMks,11642
|
32
32
|
fractal_server/app/routes/api/v2/dataset.py,sha256=Y6uZz--YSEGgnPYu05rZ9sr1Ug08bNl2v1h3VeApBe8,9441
|
33
|
-
fractal_server/app/routes/api/v2/images.py,sha256=
|
33
|
+
fractal_server/app/routes/api/v2/images.py,sha256=LjAl0m4blWHda9rW_30MPYipEzU_JH6uhMf91X6CfWE,8478
|
34
34
|
fractal_server/app/routes/api/v2/job.py,sha256=m89FTh9Px25oXCeWj2k2NdGWQaO2oxMh-6lZppcsJOY,5551
|
35
35
|
fractal_server/app/routes/api/v2/project.py,sha256=eWYFJ7F2ZYQcpi-_n-rhPF-Q4gJhzYBsVGYFhHZZXAE,6653
|
36
36
|
fractal_server/app/routes/api/v2/status.py,sha256=_cDZW-ESYw6zpf-lLFFqko5bLpKhqKrCM6yv1OfqxN4,6300
|
@@ -67,13 +67,13 @@ fractal_server/app/runner/executors/slurm/_slurm_config.py,sha256=Qa5UgcMZfYEj95
|
|
67
67
|
fractal_server/app/runner/executors/slurm/remote.py,sha256=wLziIsGdSMiO-jIXM8x77JRK82g_2hx0iBKTiMghuIo,5852
|
68
68
|
fractal_server/app/runner/executors/slurm/ssh/__init__.py,sha256=Cjn1rYvljddi96tAwS-qqGkNfOcfPzjChdaEZEObCcM,65
|
69
69
|
fractal_server/app/runner/executors/slurm/ssh/_executor_wait_thread.py,sha256=bKo5Ja0IGxJWpPWyh9dN0AG-PwzTDZzD5LyaEHB3YU4,3742
|
70
|
-
fractal_server/app/runner/executors/slurm/ssh/_slurm_job.py,sha256=
|
71
|
-
fractal_server/app/runner/executors/slurm/ssh/executor.py,sha256=
|
70
|
+
fractal_server/app/runner/executors/slurm/ssh/_slurm_job.py,sha256=rWPE_sZUNCMw5IWvZo9N1hlzoiasYoddS-wlBj4wMU4,4449
|
71
|
+
fractal_server/app/runner/executors/slurm/ssh/executor.py,sha256=eX-HPmb6gDy2WLBn5llF-Pb6U10_gTtIE09wT7rHF7w,53862
|
72
72
|
fractal_server/app/runner/executors/slurm/sudo/__init__.py,sha256=Cjn1rYvljddi96tAwS-qqGkNfOcfPzjChdaEZEObCcM,65
|
73
73
|
fractal_server/app/runner/executors/slurm/sudo/_check_jobs_status.py,sha256=wAgwpVcr6JIslKHOuS0FhRa_6T1KCManyRJqA-fifzw,1909
|
74
74
|
fractal_server/app/runner/executors/slurm/sudo/_executor_wait_thread.py,sha256=uRRyVHQtK9McHCB6OsjYfDnQsu2E8At9K_UYb_pe2pg,4682
|
75
75
|
fractal_server/app/runner/executors/slurm/sudo/_subprocess_run_as_user.py,sha256=g8wqUjSicN17UZVXlfaMomYZ-xOIbBu1oE7HdJTzfvw,5218
|
76
|
-
fractal_server/app/runner/executors/slurm/sudo/executor.py,sha256=
|
76
|
+
fractal_server/app/runner/executors/slurm/sudo/executor.py,sha256=Z5lcyXOr9LNz1yFEHlu7StAIpZdK2BkiUZbX6Fh7QVY,48513
|
77
77
|
fractal_server/app/runner/executors/slurm/utils_executors.py,sha256=naPyJI0I3lD-sYHbSXbMFGUBK4h_SggA5V91Z1Ch1Xg,1416
|
78
78
|
fractal_server/app/runner/extract_archive.py,sha256=tLpjDrX47OjTNhhoWvm6iNukg8KoieWyTb7ZfvE9eWU,2483
|
79
79
|
fractal_server/app/runner/filenames.py,sha256=lPnxKHtdRizr6FqG3zOdjDPyWA7GoaJGTtiuJV0gA8E,70
|
@@ -122,10 +122,10 @@ fractal_server/app/schemas/v2/task_collection.py,sha256=9c_yyFcVBXdAZpQQniy1bROh
|
|
122
122
|
fractal_server/app/schemas/v2/task_group.py,sha256=zZfvAH7c3MZC4_An09TMuOkNE_e1Z9XsYEnmN-axHdU,3217
|
123
123
|
fractal_server/app/schemas/v2/workflow.py,sha256=-KWvXnbHBFA3pj5n7mfSyLKJQSqkJmoziIEe7mpLl3M,1875
|
124
124
|
fractal_server/app/schemas/v2/workflowtask.py,sha256=xjFTmnKuHSetJvN-9k_GTMbPKwPg-J00zUkc_96QO7E,7726
|
125
|
-
fractal_server/app/security/__init__.py,sha256=
|
126
|
-
fractal_server/app/security/signup_email.py,sha256=
|
125
|
+
fractal_server/app/security/__init__.py,sha256=UyNjQc19cTNl-NwGyWwQs_-Yd22niHo86rIE-s-3czg,14087
|
126
|
+
fractal_server/app/security/signup_email.py,sha256=CR1VbsGFNshxsWquLDZPbUAYnGzkCHeJChtncq63RBc,1434
|
127
127
|
fractal_server/app/user_settings.py,sha256=OP1yiYKtPadxwM51_Q0hdPk3z90TCN4z1BLpQsXyWiU,1316
|
128
|
-
fractal_server/config.py,sha256=
|
128
|
+
fractal_server/config.py,sha256=kK6mf4uPNd95N-oKlkc6eSsjLkBQHv57jGYuGvZKmko,28495
|
129
129
|
fractal_server/data_migrations/README.md,sha256=_3AEFvDg9YkybDqCLlFPdDmGJvr6Tw7HRI14aZ3LOIw,398
|
130
130
|
fractal_server/data_migrations/tools.py,sha256=LeMeASwYGtEqd-3wOLle6WARdTGAimoyMmRbbJl-hAM,572
|
131
131
|
fractal_server/gunicorn_fractal.py,sha256=u6U01TLGlXgq1v8QmEpLih3QnsInZD7CqphgJ_GrGzc,1230
|
@@ -133,8 +133,8 @@ fractal_server/images/__init__.py,sha256=-_wjoKtSX02P1KjDxDP_EXKvmbONTRmbf7iGVTs
|
|
133
133
|
fractal_server/images/models.py,sha256=t4zcUFmWxhAzGgy7kkxs9Ctq8SAhVs0v910UcXcHIUw,3349
|
134
134
|
fractal_server/images/tools.py,sha256=4kfPAFJJnvg7fM-cL0JMx97Dc1Npva_0ghitEji3JUU,3407
|
135
135
|
fractal_server/logger.py,sha256=5Z3rfsFwl8UysVljTOaaIvt8Pyp6CVH492ez3jE8WAw,5113
|
136
|
-
fractal_server/main.py,sha256=
|
137
|
-
fractal_server/migrations/env.py,sha256=
|
136
|
+
fractal_server/main.py,sha256=yziIqi9AaecYYx9hVsaB5nzSpsgnOSSMvQOa0n30SBI,4555
|
137
|
+
fractal_server/migrations/env.py,sha256=nfyBpMIOT3kny6t-b-tUjyRjZ4k906bb1_wCQ7me1BI,1353
|
138
138
|
fractal_server/migrations/naming_convention.py,sha256=htbKrVdetx3pklowb_9Cdo5RqeF0fJ740DNecY5de_M,265
|
139
139
|
fractal_server/migrations/versions/034a469ec2eb_task_groups.py,sha256=vrPhC8hfFu1c4HmLHNZyCuqEfecFD8-bWc49bXMNes0,6199
|
140
140
|
fractal_server/migrations/versions/091b01f51f88_add_usergroup_and_linkusergroup_table.py,sha256=-BSS9AFTPcu3gYC-sYbawSy4MWQQx8TfMb5BW5EBKmQ,1450
|
@@ -175,14 +175,14 @@ fractal_server/tasks/utils.py,sha256=V7dj8o2AnoHhGSTYlqJHcRFhCIpmOrMOUhtiE_DvRVA
|
|
175
175
|
fractal_server/tasks/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
176
176
|
fractal_server/tasks/v2/local/__init__.py,sha256=9RVItnS7OyLsJOuJjWMCicaky4ASUPQEYD4SzDs0hOE,141
|
177
177
|
fractal_server/tasks/v2/local/_utils.py,sha256=EvhmVwYjqaNyDCUMEsTWYOUXLgEwR1xr6bu32apCEI8,2491
|
178
|
-
fractal_server/tasks/v2/local/collect.py,sha256=
|
179
|
-
fractal_server/tasks/v2/local/deactivate.py,sha256=
|
180
|
-
fractal_server/tasks/v2/local/reactivate.py,sha256=
|
178
|
+
fractal_server/tasks/v2/local/collect.py,sha256=gtntegkilH-zdZpMQaSWYA3DuUqKRAlsoYhs-iJLEKc,12194
|
179
|
+
fractal_server/tasks/v2/local/deactivate.py,sha256=94s_RDND8aR5Y8RxFrRx61rZBMPGqOmFnBFLVKK1HVY,10038
|
180
|
+
fractal_server/tasks/v2/local/reactivate.py,sha256=eBgFgq5xVKNr4DIDX7QU8xXerhwMrPaHDJ1wTth7aQc,6191
|
181
181
|
fractal_server/tasks/v2/ssh/__init__.py,sha256=aSQbVi6Ummt9QzcSLWNmSqYjfdxrn9ROmqgH6bDpI7k,135
|
182
182
|
fractal_server/tasks/v2/ssh/_utils.py,sha256=LjaEYVUJDChilu3YuhxuGWYRNnVJ_zqNE9SDHdRTIHY,2824
|
183
|
-
fractal_server/tasks/v2/ssh/collect.py,sha256
|
184
|
-
fractal_server/tasks/v2/ssh/deactivate.py,sha256=
|
185
|
-
fractal_server/tasks/v2/ssh/reactivate.py,sha256=
|
183
|
+
fractal_server/tasks/v2/ssh/collect.py,sha256=-wqqGMapjD1hQz1vphwXmZaD4qzziMetc6lV6fckDO4,13552
|
184
|
+
fractal_server/tasks/v2/ssh/deactivate.py,sha256=EAVH2HtyvmIFXqUwsGYhlJcAcVh_MvIOaKDY8AyBODw,11400
|
185
|
+
fractal_server/tasks/v2/ssh/reactivate.py,sha256=8Rnbbny7TjMEAHhboqfgxBVZZK5UNNmh4Ud-0y3jaVM,7970
|
186
186
|
fractal_server/tasks/v2/templates/1_create_venv.sh,sha256=PK0jdHKtQpda1zULebBaVPORt4t6V17wa4N1ohcj5ac,548
|
187
187
|
fractal_server/tasks/v2/templates/2_pip_install.sh,sha256=Gpk2io8u9YaflFUlQu2NgkDQw5AA4m4AOVG1sB4yrHQ,1822
|
188
188
|
fractal_server/tasks/v2/templates/3_pip_freeze.sh,sha256=JldREScEBI4cD_qjfX4UK7V4aI-FnX9ZvVNxgpSOBFc,168
|
@@ -197,8 +197,8 @@ fractal_server/tasks/v2/utils_templates.py,sha256=07TZpJ0Mh_A4lXVXrrH2o1VLFFGwxe
|
|
197
197
|
fractal_server/urls.py,sha256=QjIKAC1a46bCdiPMu3AlpgFbcv6a4l3ABcd5xz190Og,471
|
198
198
|
fractal_server/utils.py,sha256=PMwrxWFxRTQRl1b9h-NRIbFGPKqpH_hXnkAT3NfZdpY,3571
|
199
199
|
fractal_server/zip_tools.py,sha256=GjDgo_sf6V_DDg6wWeBlZu5zypIxycn_l257p_YVKGc,4876
|
200
|
-
fractal_server-2.12.
|
201
|
-
fractal_server-2.12.
|
202
|
-
fractal_server-2.12.
|
203
|
-
fractal_server-2.12.
|
204
|
-
fractal_server-2.12.
|
200
|
+
fractal_server-2.12.1.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
|
201
|
+
fractal_server-2.12.1.dist-info/METADATA,sha256=m7gJftwPSx3KP-HzIgYY_4a7m8zs-I8yGuQlW_HedKE,4570
|
202
|
+
fractal_server-2.12.1.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
|
203
|
+
fractal_server-2.12.1.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
|
204
|
+
fractal_server-2.12.1.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|