fractal-server 2.3.3__py3-none-any.whl → 2.3.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fractal_server/__init__.py +1 -1
- fractal_server/app/routes/api/__init__.py +9 -0
- fractal_server/app/routes/api/v1/project.py +10 -1
- fractal_server/app/runner/compress_folder.py +111 -99
- fractal_server/app/runner/executors/slurm/ssh/executor.py +8 -12
- fractal_server/app/runner/extract_archive.py +66 -19
- fractal_server/app/runner/run_subprocess.py +27 -0
- fractal_server/config.py +22 -1
- fractal_server/main.py +1 -1
- {fractal_server-2.3.3.dist-info → fractal_server-2.3.4.dist-info}/METADATA +1 -1
- {fractal_server-2.3.3.dist-info → fractal_server-2.3.4.dist-info}/RECORD +14 -13
- {fractal_server-2.3.3.dist-info → fractal_server-2.3.4.dist-info}/LICENSE +0 -0
- {fractal_server-2.3.3.dist-info → fractal_server-2.3.4.dist-info}/WHEEL +0 -0
- {fractal_server-2.3.3.dist-info → fractal_server-2.3.4.dist-info}/entry_points.txt +0 -0
fractal_server/__init__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__VERSION__ = "2.3.
|
1
|
+
__VERSION__ = "2.3.4"
|
@@ -2,9 +2,12 @@
|
|
2
2
|
`api` module
|
3
3
|
"""
|
4
4
|
from fastapi import APIRouter
|
5
|
+
from fastapi import Depends
|
5
6
|
|
6
7
|
from ....config import get_settings
|
7
8
|
from ....syringe import Inject
|
9
|
+
from ...models.security import UserOAuth
|
10
|
+
from ...security import current_active_superuser
|
8
11
|
|
9
12
|
|
10
13
|
router_api = APIRouter()
|
@@ -17,3 +20,9 @@ async def alive():
|
|
17
20
|
alive=True,
|
18
21
|
version=settings.PROJECT_VERSION,
|
19
22
|
)
|
23
|
+
|
24
|
+
|
25
|
+
@router_api.get("/settings/")
|
26
|
+
async def view_settings(user: UserOAuth = Depends(current_active_superuser)):
|
27
|
+
settings = Inject(get_settings)
|
28
|
+
return settings.get_sanitized()
|
@@ -250,9 +250,18 @@ async def apply_workflow(
|
|
250
250
|
db: AsyncSession = Depends(get_async_db),
|
251
251
|
) -> Optional[ApplyWorkflowReadV1]:
|
252
252
|
|
253
|
+
settings = Inject(get_settings)
|
254
|
+
if settings.FRACTAL_API_V1_MODE == "include_without_submission":
|
255
|
+
raise HTTPException(
|
256
|
+
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
257
|
+
detail=(
|
258
|
+
"Legacy API is still accessible, "
|
259
|
+
"but the submission of legacy jobs is not available."
|
260
|
+
),
|
261
|
+
)
|
262
|
+
|
253
263
|
# Remove non-submitted V1 jobs from the app state when the list grows
|
254
264
|
# beyond a threshold
|
255
|
-
settings = Inject(get_settings)
|
256
265
|
if (
|
257
266
|
len(request.app.state.jobsV1)
|
258
267
|
> settings.FRACTAL_API_MAX_JOB_LIST_LENGTH
|
@@ -1,120 +1,132 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
"""
|
2
|
+
Wrap `tar` compression command.
|
3
|
+
|
4
|
+
This module is used both locally (in the environment where `fractal-server`
|
5
|
+
is running) and remotely (as a standalon Python module, executed over SSH).
|
6
|
+
|
7
|
+
This is a twin-module of `extract_archive.py`.
|
8
|
+
|
9
|
+
The reason for using the `tar` command via `subprocess` rather than Python
|
10
|
+
built-in `tarfile` library has to do with performance issues we observed
|
11
|
+
when handling files which were just created within a SLURM job, and in the
|
12
|
+
context of a CephFS filesystem.
|
13
|
+
"""
|
14
|
+
import shutil
|
3
15
|
import sys
|
4
|
-
import tarfile
|
5
|
-
import time
|
6
16
|
from pathlib import Path
|
7
|
-
from typing import Optional
|
8
17
|
|
18
|
+
from fractal_server.app.runner.run_subprocess import run_subprocess
|
19
|
+
from fractal_server.logger import get_logger
|
20
|
+
from fractal_server.logger import set_logger
|
21
|
+
|
22
|
+
|
23
|
+
def copy_subfolder(src: Path, dest: Path, logger_name: str):
|
24
|
+
cmd_cp = f"cp -r {src.as_posix()} {dest.as_posix()}"
|
25
|
+
logger = get_logger(logger_name=logger_name)
|
26
|
+
logger.debug(f"{cmd_cp=}")
|
27
|
+
res = run_subprocess(cmd=cmd_cp, logger_name=logger_name)
|
28
|
+
return res
|
29
|
+
|
30
|
+
|
31
|
+
def create_tar_archive(
|
32
|
+
tarfile_path: Path,
|
33
|
+
subfolder_path_tmp_copy: Path,
|
34
|
+
logger_name: str,
|
35
|
+
remote_to_local: bool,
|
36
|
+
):
|
37
|
+
logger = get_logger(logger_name)
|
38
|
+
|
39
|
+
if remote_to_local:
|
40
|
+
exclude_options = "--exclude *sbatch --exclude *_in_*.pickle "
|
41
|
+
else:
|
42
|
+
exclude_options = ""
|
43
|
+
|
44
|
+
cmd_tar = (
|
45
|
+
f"tar czf {tarfile_path} "
|
46
|
+
f"{exclude_options} "
|
47
|
+
f"--directory={subfolder_path_tmp_copy.as_posix()} "
|
48
|
+
"."
|
49
|
+
)
|
50
|
+
logger.debug(f"cmd tar:\n{cmd_tar}")
|
51
|
+
run_subprocess(cmd=cmd_tar, logger_name=logger_name)
|
9
52
|
|
10
|
-
# COMPRESS_FOLDER_MODALITY = "python"
|
11
|
-
COMPRESS_FOLDER_MODALITY = "cp-tar-rmtree"
|
12
53
|
|
54
|
+
def remove_temp_subfolder(subfolder_path_tmp_copy: Path, logger_name: str):
|
55
|
+
logger = get_logger(logger_name)
|
56
|
+
try:
|
57
|
+
logger.debug(f"Now remove {subfolder_path_tmp_copy}")
|
58
|
+
shutil.rmtree(subfolder_path_tmp_copy)
|
59
|
+
except Exception as e:
|
60
|
+
logger.debug(f"ERROR during shutil.rmtree: {e}")
|
13
61
|
|
14
|
-
def _filter(info: tarfile.TarInfo) -> Optional[tarfile.TarInfo]:
|
15
|
-
if info.name.endswith(".pickle"):
|
16
|
-
filename = info.name.split("/")[-1]
|
17
|
-
parts = filename.split("_")
|
18
|
-
if len(parts) == 3 and parts[1] == "in":
|
19
|
-
return None
|
20
|
-
elif len(parts) == 5 and parts[3] == "in":
|
21
|
-
return None
|
22
|
-
elif info.name.endswith("slurm_submit.sbatch"):
|
23
|
-
return None
|
24
|
-
return info
|
25
62
|
|
63
|
+
def compress_folder(
|
64
|
+
subfolder_path: Path, remote_to_local: bool = False
|
65
|
+
) -> str:
|
66
|
+
"""
|
67
|
+
Compress e.g. `/path/archive` into `/path/archive.tar.gz`
|
26
68
|
|
27
|
-
|
28
|
-
|
29
|
-
"Expected use:\n"
|
30
|
-
"python -m fractal_server.app.runner.compress_folder "
|
31
|
-
"path/to/folder"
|
32
|
-
)
|
69
|
+
Note that `/path/archive.tar.gz` may already exist. In this case, it will
|
70
|
+
be overwritten.
|
33
71
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
72
|
+
Args:
|
73
|
+
subfolder_path: Absolute path to the folder to compress.
|
74
|
+
remote_to_local: If `True`, exclude some files from the tar.gz archive.
|
75
|
+
|
76
|
+
Returns:
|
77
|
+
Absolute path to the tar.gz archive.
|
78
|
+
"""
|
38
79
|
|
39
|
-
|
40
|
-
|
41
|
-
print("[compress_folder.py] START")
|
42
|
-
print(f"[compress_folder.py] {COMPRESS_FOLDER_MODALITY=}")
|
43
|
-
print(f"[compress_folder.py] {subfolder_path=}")
|
80
|
+
logger_name = "compress_folder"
|
81
|
+
logger = set_logger(logger_name)
|
44
82
|
|
45
|
-
|
83
|
+
logger.debug("START")
|
84
|
+
logger.debug(f"{subfolder_path=}")
|
85
|
+
parent_dir = subfolder_path.parent
|
46
86
|
subfolder_name = subfolder_path.name
|
47
|
-
tarfile_path = (
|
48
|
-
|
49
|
-
|
50
|
-
if COMPRESS_FOLDER_MODALITY == "python":
|
51
|
-
raise NotImplementedError()
|
52
|
-
with tarfile.open(tarfile_path, "w:gz") as tar:
|
53
|
-
tar.add(
|
54
|
-
subfolder_path,
|
55
|
-
arcname=".", # ????
|
56
|
-
recursive=True,
|
57
|
-
filter=_filter,
|
58
|
-
)
|
59
|
-
elif COMPRESS_FOLDER_MODALITY == "cp-tar-rmtree":
|
60
|
-
import shutil
|
61
|
-
import time
|
62
|
-
|
63
|
-
subfolder_path_tmp_copy = (
|
64
|
-
subfolder_path.parent / f"{subfolder_path.name}_copy"
|
65
|
-
)
|
87
|
+
tarfile_path = (parent_dir / f"{subfolder_name}.tar.gz").as_posix()
|
88
|
+
logger.debug(f"{tarfile_path=}")
|
66
89
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
)
|
74
|
-
res = subprocess.run( # nosec
|
75
|
-
shlex.split(cmd_cp),
|
76
|
-
check=True,
|
77
|
-
capture_output=True,
|
78
|
-
encoding="utf-8",
|
90
|
+
subfolder_path_tmp_copy = (
|
91
|
+
subfolder_path.parent / f"{subfolder_path.name}_copy"
|
92
|
+
)
|
93
|
+
try:
|
94
|
+
copy_subfolder(
|
95
|
+
subfolder_path, subfolder_path_tmp_copy, logger_name=logger_name
|
79
96
|
)
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
f"{tarfile_path} "
|
86
|
-
"--exclude *sbatch --exclude *_in_*.pickle "
|
87
|
-
f"--directory={subfolder_path_tmp_copy.as_posix()} "
|
88
|
-
"."
|
97
|
+
create_tar_archive(
|
98
|
+
tarfile_path,
|
99
|
+
subfolder_path_tmp_copy,
|
100
|
+
logger_name=logger_name,
|
101
|
+
remote_to_local=remote_to_local,
|
89
102
|
)
|
103
|
+
return tarfile_path
|
90
104
|
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
shlex.split(cmd_tar),
|
95
|
-
capture_output=True,
|
96
|
-
encoding="utf-8",
|
97
|
-
)
|
98
|
-
t1 = time.perf_counter()
|
99
|
-
t_1 = time.perf_counter()
|
100
|
-
print(f"[compress_folder.py] tar END - elapsed: {t1-t0:.3f} s")
|
105
|
+
except Exception as e:
|
106
|
+
logger.debug(f"ERROR: {e}")
|
107
|
+
sys.exit(1)
|
101
108
|
|
102
|
-
|
109
|
+
finally:
|
110
|
+
remove_temp_subfolder(subfolder_path_tmp_copy, logger_name=logger_name)
|
103
111
|
|
104
|
-
if res.returncode != 0:
|
105
|
-
print("[compress_folder.py] ERROR in tar")
|
106
|
-
print(f"[compress_folder.py] tar stdout:\n{res.stdout}")
|
107
|
-
print(f"[compress_folder.py] tar stderr:\n{res.stderr}")
|
108
112
|
|
109
|
-
|
110
|
-
|
113
|
+
def main(sys_argv: list[str]):
|
114
|
+
|
115
|
+
help_msg = (
|
116
|
+
"Expected use:\n"
|
117
|
+
"python -m fractal_server.app.runner.compress_folder "
|
118
|
+
"path/to/folder [--remote-to-local]\n"
|
119
|
+
)
|
120
|
+
num_args = len(sys_argv[1:])
|
121
|
+
if num_args == 0:
|
122
|
+
sys.exit(f"Invalid argument.\n{help_msg}\nProvided: {sys_argv[1:]=}")
|
123
|
+
elif num_args == 1:
|
124
|
+
compress_folder(subfolder_path=Path(sys_argv[1]))
|
125
|
+
elif num_args == 2 and sys_argv[2] == "--remote-to-local":
|
126
|
+
compress_folder(subfolder_path=Path(sys_argv[1]), remote_to_local=True)
|
127
|
+
else:
|
128
|
+
sys.exit(f"Invalid argument.\n{help_msg}\nProvided: {sys_argv[1:]=}")
|
111
129
|
|
112
|
-
t0 = time.perf_counter()
|
113
|
-
shutil.rmtree(subfolder_path_tmp_copy)
|
114
|
-
t1 = time.perf_counter()
|
115
|
-
print(
|
116
|
-
f"[compress_folder.py] shutil.rmtree END - elapsed: {t1-t0:.3f} s"
|
117
|
-
)
|
118
130
|
|
119
|
-
|
120
|
-
|
131
|
+
if __name__ == "__main__":
|
132
|
+
main(sys.argv)
|
@@ -13,7 +13,6 @@
|
|
13
13
|
import json
|
14
14
|
import math
|
15
15
|
import sys
|
16
|
-
import tarfile
|
17
16
|
import threading
|
18
17
|
import time
|
19
18
|
from concurrent.futures import Future
|
@@ -38,9 +37,11 @@ from ...slurm._slurm_config import SlurmConfig
|
|
38
37
|
from .._batching import heuristics
|
39
38
|
from ._executor_wait_thread import FractalSlurmWaitThread
|
40
39
|
from fractal_server.app.runner.components import _COMPONENT_KEY_
|
40
|
+
from fractal_server.app.runner.compress_folder import compress_folder
|
41
41
|
from fractal_server.app.runner.exceptions import JobExecutionError
|
42
42
|
from fractal_server.app.runner.exceptions import TaskExecutionError
|
43
43
|
from fractal_server.app.runner.executors.slurm.ssh._slurm_job import SlurmJob
|
44
|
+
from fractal_server.app.runner.extract_archive import extract_archive
|
44
45
|
from fractal_server.config import get_settings
|
45
46
|
from fractal_server.logger import set_logger
|
46
47
|
from fractal_server.ssh._fabric import FractalSSH
|
@@ -822,17 +823,12 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
|
|
822
823
|
|
823
824
|
# Create compressed subfolder archive (locally)
|
824
825
|
local_subfolder = self.workflow_dir_local / subfolder_name
|
825
|
-
|
826
|
-
|
827
|
-
|
828
|
-
).as_posix()
|
826
|
+
tarfile_path_local = compress_folder(local_subfolder)
|
827
|
+
tarfile_name = Path(tarfile_path_local).name
|
828
|
+
logger.info(f"Subfolder archive created at {tarfile_path_local}")
|
829
829
|
tarfile_path_remote = (
|
830
830
|
self.workflow_dir_remote / tarfile_name
|
831
831
|
).as_posix()
|
832
|
-
with tarfile.open(tarfile_path_local, "w:gz") as tar:
|
833
|
-
for this_file in local_subfolder.glob("*"):
|
834
|
-
tar.add(this_file, arcname=this_file.name)
|
835
|
-
logger.info(f"Subfolder archive created at {tarfile_path_local}")
|
836
832
|
|
837
833
|
# Transfer archive
|
838
834
|
t_0_put = time.perf_counter()
|
@@ -1222,7 +1218,8 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
|
|
1222
1218
|
tar_command = (
|
1223
1219
|
f"{self.python_remote} "
|
1224
1220
|
"-m fractal_server.app.runner.compress_folder "
|
1225
|
-
f"{(self.workflow_dir_remote / subfolder_name).as_posix()}"
|
1221
|
+
f"{(self.workflow_dir_remote / subfolder_name).as_posix()} "
|
1222
|
+
"--remote-to-local"
|
1226
1223
|
)
|
1227
1224
|
stdout = self.fractal_ssh.run_command(cmd=tar_command)
|
1228
1225
|
print(stdout)
|
@@ -1240,8 +1237,7 @@ class FractalSlurmSSHExecutor(SlurmExecutor):
|
|
1240
1237
|
)
|
1241
1238
|
|
1242
1239
|
# Extract tarfile locally
|
1243
|
-
|
1244
|
-
tar.extractall(path=(self.workflow_dir_local / subfolder_name))
|
1240
|
+
extract_archive(Path(tarfile_path_local))
|
1245
1241
|
|
1246
1242
|
t_1 = time.perf_counter()
|
1247
1243
|
logger.info("[_get_subfolder_sftp] End - " f"elapsed: {t_1-t_0:.3f} s")
|
@@ -1,7 +1,22 @@
|
|
1
|
+
"""
|
2
|
+
Wrap `tar` extraction command.
|
3
|
+
|
4
|
+
This module is used both locally (in the environment where `fractal-server`
|
5
|
+
is running) and remotely (as a standalon Python module, executed over SSH).
|
6
|
+
|
7
|
+
This is a twin-module of `compress_folder.py`.
|
8
|
+
|
9
|
+
The reason for using the `tar` command via `subprocess` rather than Python
|
10
|
+
built-in `tarfile` library has to do with performance issues we observed
|
11
|
+
when handling files which were just created within a SLURM job, and in the
|
12
|
+
context of a CephFS filesystem.
|
13
|
+
"""
|
1
14
|
import sys
|
2
|
-
import tarfile
|
3
15
|
from pathlib import Path
|
4
16
|
|
17
|
+
from .run_subprocess import run_subprocess
|
18
|
+
from fractal_server.logger import set_logger
|
19
|
+
|
5
20
|
|
6
21
|
def _remove_suffix(*, string: str, suffix: str) -> str:
|
7
22
|
if string.endswith(suffix):
|
@@ -10,29 +25,61 @@ def _remove_suffix(*, string: str, suffix: str) -> str:
|
|
10
25
|
raise ValueError(f"Cannot remove {suffix=} from {string=}.")
|
11
26
|
|
12
27
|
|
13
|
-
|
28
|
+
def extract_archive(archive_path: Path):
|
29
|
+
"""
|
30
|
+
Extract e.g. `/path/archive.tar.gz` archive into `/path/archive` folder
|
31
|
+
|
32
|
+
Note that `/path/archive` may already exist. In this case, files with
|
33
|
+
the same name are overwritten and new files are added.
|
34
|
+
|
35
|
+
Arguments:
|
36
|
+
archive_path: Absolute path to the archive file.
|
37
|
+
"""
|
38
|
+
|
39
|
+
logger_name = "extract_archive"
|
40
|
+
logger = set_logger(logger_name)
|
41
|
+
|
42
|
+
logger.debug("START")
|
43
|
+
logger.debug(f"{archive_path.as_posix()=}")
|
44
|
+
|
45
|
+
# Check archive_path is valid
|
46
|
+
if not archive_path.exists():
|
47
|
+
sys.exit(f"Missing file {archive_path.as_posix()}.")
|
48
|
+
|
49
|
+
# Prepare subfolder path
|
50
|
+
parent_dir = archive_path.parent
|
51
|
+
subfolder_name = _remove_suffix(string=archive_path.name, suffix=".tar.gz")
|
52
|
+
subfolder_path = parent_dir / subfolder_name
|
53
|
+
logger.debug(f"{subfolder_path.as_posix()=}")
|
54
|
+
|
55
|
+
# Create subfolder
|
56
|
+
subfolder_path.mkdir(exist_ok=True)
|
57
|
+
|
58
|
+
# Run tar command
|
59
|
+
cmd_tar = (
|
60
|
+
f"tar -xzvf {archive_path} "
|
61
|
+
f"--directory={subfolder_path.as_posix()} "
|
62
|
+
"."
|
63
|
+
)
|
64
|
+
logger.debug(f"{cmd_tar=}")
|
65
|
+
run_subprocess(cmd=cmd_tar, logger_name=logger_name)
|
66
|
+
|
67
|
+
logger.debug("END")
|
68
|
+
|
69
|
+
|
70
|
+
def main(sys_argv: list[str]):
|
14
71
|
help_msg = (
|
15
72
|
"Expected use:\n"
|
16
73
|
"python -m fractal_server.app.runner.extract_archive "
|
17
74
|
"path/to/archive.tar.gz"
|
18
75
|
)
|
19
76
|
|
20
|
-
if len(
|
21
|
-
|
22
|
-
|
23
|
-
)
|
24
|
-
|
25
|
-
raise ValueError(
|
26
|
-
f"Invalid argument.\n{help_msg}\nProvided: {sys.argv=}"
|
27
|
-
)
|
28
|
-
|
29
|
-
tarfile_path = Path(sys.argv[1])
|
30
|
-
|
31
|
-
print(f"[extract_archive.py] {tarfile_path=}")
|
77
|
+
if len(sys_argv[1:]) != 1 or not sys_argv[1].endswith(".tar.gz"):
|
78
|
+
sys.exit(f"Invalid argument.\n{help_msg}\nProvided: {sys_argv[1:]=}")
|
79
|
+
else:
|
80
|
+
tarfile_path = Path(sys_argv[1])
|
81
|
+
extract_archive(tarfile_path)
|
32
82
|
|
33
|
-
job_folder = tarfile_path.parent
|
34
|
-
subfolder_name = _remove_suffix(string=tarfile_path.name, suffix=".tar.gz")
|
35
|
-
with tarfile.open(tarfile_path) as tar:
|
36
|
-
tar.extractall(path=Path(job_folder, subfolder_name).as_posix())
|
37
83
|
|
38
|
-
|
84
|
+
if __name__ == "__main__":
|
85
|
+
main(sys.argv)
|
@@ -0,0 +1,27 @@
|
|
1
|
+
import shlex
|
2
|
+
import subprocess # nosec
|
3
|
+
from typing import Optional
|
4
|
+
|
5
|
+
from fractal_server.logger import get_logger
|
6
|
+
|
7
|
+
|
8
|
+
def run_subprocess(
|
9
|
+
cmd: str, logger_name: Optional[str] = None
|
10
|
+
) -> subprocess.CompletedProcess:
|
11
|
+
logger = get_logger(logger_name)
|
12
|
+
try:
|
13
|
+
res = subprocess.run( # nosec
|
14
|
+
shlex.split(cmd), check=True, capture_output=True, encoding="utf-8"
|
15
|
+
)
|
16
|
+
return res
|
17
|
+
except subprocess.CalledProcessError as e:
|
18
|
+
logger.debug(
|
19
|
+
f"Command '{e.cmd}' returned non-zero exit status {e.returncode}."
|
20
|
+
)
|
21
|
+
logger.debug(f"stdout: {e.stdout}")
|
22
|
+
logger.debug(f"stderr: {e.stderr}")
|
23
|
+
raise e
|
24
|
+
except Exception as e:
|
25
|
+
logger.debug(f"An error occurred while running command: {cmd}")
|
26
|
+
logger.debug(str(e))
|
27
|
+
raise e
|
fractal_server/config.py
CHANGED
@@ -546,7 +546,9 @@ class Settings(BaseSettings):
|
|
546
546
|
attribute in their input-arguments JSON file.
|
547
547
|
"""
|
548
548
|
|
549
|
-
FRACTAL_API_V1_MODE: Literal[
|
549
|
+
FRACTAL_API_V1_MODE: Literal[
|
550
|
+
"include", "include_without_submission", "exclude"
|
551
|
+
] = "include"
|
550
552
|
"""
|
551
553
|
Whether to include the v1 API.
|
552
554
|
"""
|
@@ -685,6 +687,25 @@ class Settings(BaseSettings):
|
|
685
687
|
self.check_db()
|
686
688
|
self.check_runner()
|
687
689
|
|
690
|
+
def get_sanitized(self) -> dict:
|
691
|
+
def _must_be_sanitized(string) -> bool:
|
692
|
+
if not string.upper().startswith("FRACTAL") or any(
|
693
|
+
s in string.upper()
|
694
|
+
for s in ["PASSWORD", "SECRET", "PWD", "TOKEN"]
|
695
|
+
):
|
696
|
+
return True
|
697
|
+
else:
|
698
|
+
return False
|
699
|
+
|
700
|
+
sanitized_settings = {}
|
701
|
+
for k, v in self.dict().items():
|
702
|
+
if _must_be_sanitized(k):
|
703
|
+
sanitized_settings[k] = "***"
|
704
|
+
else:
|
705
|
+
sanitized_settings[k] = v
|
706
|
+
|
707
|
+
return sanitized_settings
|
708
|
+
|
688
709
|
|
689
710
|
def get_settings(settings=Settings()) -> Settings:
|
690
711
|
return settings
|
fractal_server/main.py
CHANGED
@@ -49,7 +49,7 @@ def collect_routers(app: FastAPI) -> None:
|
|
49
49
|
settings = Inject(get_settings)
|
50
50
|
|
51
51
|
app.include_router(router_api, prefix="/api")
|
52
|
-
if settings.FRACTAL_API_V1_MODE
|
52
|
+
if settings.FRACTAL_API_V1_MODE.startswith("include"):
|
53
53
|
app.include_router(router_api_v1, prefix="/api/v1")
|
54
54
|
app.include_router(
|
55
55
|
router_admin_v1, prefix="/admin/v1", tags=["V1 Admin area"]
|
@@ -1,4 +1,4 @@
|
|
1
|
-
fractal_server/__init__.py,sha256=
|
1
|
+
fractal_server/__init__.py,sha256=eG9nilkh3fQGESwdGxiBx9bAq-mIR2ueRWgDXGopc1M,22
|
2
2
|
fractal_server/__main__.py,sha256=CocbzZooX1UtGqPi55GcHGNxnrJXFg5tUU5b3wyFCyo,4958
|
3
3
|
fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
|
4
4
|
fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -25,12 +25,12 @@ fractal_server/app/routes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJW
|
|
25
25
|
fractal_server/app/routes/admin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
26
26
|
fractal_server/app/routes/admin/v1.py,sha256=ShmsUFtfyRqP84QScqmRDVrJFpbR4p-8baLxAkI3n1U,13926
|
27
27
|
fractal_server/app/routes/admin/v2.py,sha256=JuG1qqVeQIgVJgPyqrB1053il22mGPGpKBiJi6zVsqQ,13687
|
28
|
-
fractal_server/app/routes/api/__init__.py,sha256=
|
28
|
+
fractal_server/app/routes/api/__init__.py,sha256=XlJUFd-0FossfyKyJti4dmwY6SMysQn1yiisMrNzgBE,615
|
29
29
|
fractal_server/app/routes/api/v1/__init__.py,sha256=Y2HQdG197J0a7DyQEE2jn53IfxD0EHGhzK1I2JZuEck,958
|
30
30
|
fractal_server/app/routes/api/v1/_aux_functions.py,sha256=CeaVrNVYs_lEbiJbu4uaTeeiajljeXfdq1iLkt5RoRo,12636
|
31
31
|
fractal_server/app/routes/api/v1/dataset.py,sha256=HRE-8vPmVkeXf7WFYkI19mDtbY-iJZeJ7PmMiV0LMgY,16923
|
32
32
|
fractal_server/app/routes/api/v1/job.py,sha256=217fGh7U37esib1JG8INpLhE0W88t9X0fFwCNVt2r_M,5313
|
33
|
-
fractal_server/app/routes/api/v1/project.py,sha256=
|
33
|
+
fractal_server/app/routes/api/v1/project.py,sha256=0DavnACBDr8-BHWGQ0YPfxVNJLsYmbuo-TeKJk1s3Hw,16436
|
34
34
|
fractal_server/app/routes/api/v1/task.py,sha256=udbKnenzc-Q10elYCVB9JmOPWATraa9tZi0AaByvWo0,6129
|
35
35
|
fractal_server/app/routes/api/v1/task_collection.py,sha256=82XBsJHlPiDPCbpLa-16ojKDpj2LYj9_jFSZt0t58bQ,8911
|
36
36
|
fractal_server/app/routes/api/v1/workflow.py,sha256=7r9IoIevg_rvYCrerMOsIsUabSOQatxdPCfLdkP0dRs,10942
|
@@ -57,7 +57,7 @@ fractal_server/app/runner/.gitignore,sha256=ytzN_oyHWXrGU7iFAtoHSTUbM6Rn6kG0Zkdd
|
|
57
57
|
fractal_server/app/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
58
58
|
fractal_server/app/runner/async_wrap.py,sha256=_O6f8jftKYXG_DozkmlrDBhoiK9QhE9MablOyECq2_M,829
|
59
59
|
fractal_server/app/runner/components.py,sha256=ZF8ct_Ky5k8IAcrmpYOZ-bc6OBgdELEighYVqFDEbZg,119
|
60
|
-
fractal_server/app/runner/compress_folder.py,sha256=
|
60
|
+
fractal_server/app/runner/compress_folder.py,sha256=zmxo2EFkSaO4h3GnMRi9DYaf62bxy4zznZZGfmq-n68,3975
|
61
61
|
fractal_server/app/runner/exceptions.py,sha256=_qZ_t8O4umAdJ1ikockiF5rDJuxnEskrGrLjZcnQl7A,4159
|
62
62
|
fractal_server/app/runner/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
63
63
|
fractal_server/app/runner/executors/slurm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -67,14 +67,15 @@ fractal_server/app/runner/executors/slurm/remote.py,sha256=wLziIsGdSMiO-jIXM8x77
|
|
67
67
|
fractal_server/app/runner/executors/slurm/ssh/__init__.py,sha256=Cjn1rYvljddi96tAwS-qqGkNfOcfPzjChdaEZEObCcM,65
|
68
68
|
fractal_server/app/runner/executors/slurm/ssh/_executor_wait_thread.py,sha256=jM4G-wiHynZhNERusVGLtDTepJDiYjCDloWZyflaMV0,3482
|
69
69
|
fractal_server/app/runner/executors/slurm/ssh/_slurm_job.py,sha256=rwlqZzoGo4SAb4nSlFjsQJdaCgfM1J6YGcjb8yYxlqc,4506
|
70
|
-
fractal_server/app/runner/executors/slurm/ssh/executor.py,sha256=
|
70
|
+
fractal_server/app/runner/executors/slurm/ssh/executor.py,sha256=O3BV7Qo7W8qEJmIeaajxtCCbkbYQDEXZbfS7GKCO5LQ,55338
|
71
71
|
fractal_server/app/runner/executors/slurm/sudo/__init__.py,sha256=Cjn1rYvljddi96tAwS-qqGkNfOcfPzjChdaEZEObCcM,65
|
72
72
|
fractal_server/app/runner/executors/slurm/sudo/_check_jobs_status.py,sha256=wAgwpVcr6JIslKHOuS0FhRa_6T1KCManyRJqA-fifzw,1909
|
73
73
|
fractal_server/app/runner/executors/slurm/sudo/_executor_wait_thread.py,sha256=z5LlhaiqAb8pHsF1WwdzXN39C5anQmwjo1rSQgtRAYE,4422
|
74
74
|
fractal_server/app/runner/executors/slurm/sudo/_subprocess_run_as_user.py,sha256=uZgmxP0ZneGpzTVt-GT-6EgNKUh1sW2-QH7LFYc1tNI,5132
|
75
75
|
fractal_server/app/runner/executors/slurm/sudo/executor.py,sha256=nu1mcg3veOEaDwqPLyrf9_pMUc34-uXhd8er3IsMQZw,48236
|
76
|
-
fractal_server/app/runner/extract_archive.py,sha256=
|
76
|
+
fractal_server/app/runner/extract_archive.py,sha256=tLpjDrX47OjTNhhoWvm6iNukg8KoieWyTb7ZfvE9eWU,2483
|
77
77
|
fractal_server/app/runner/filenames.py,sha256=9lwu3yB4C67yiijYw8XIKaLFn3mJUt6_TCyVFM_aZUQ,206
|
78
|
+
fractal_server/app/runner/run_subprocess.py,sha256=KTkJnWLrLQdR2WRJ3jGu0RBu4330L3mtCAE_B0wDx3M,818
|
78
79
|
fractal_server/app/runner/set_start_and_last_task_index.py,sha256=-q4zVybAj8ek2XlbENKlfOAJ39hT_zoJoZkqzDqiAMY,1254
|
79
80
|
fractal_server/app/runner/shutdown.py,sha256=I_o2iYKJwzku0L3E85ETjrve3QPECygR5xhhsAo5huM,2910
|
80
81
|
fractal_server/app/runner/task_files.py,sha256=sd_MpJ01C8c9QTO8GzGMidFGdlq_hXX_ARDRhd_YMnI,3762
|
@@ -138,14 +139,14 @@ fractal_server/app/schemas/v2/task_collection.py,sha256=8PG1bOqkfQqORMN0brWf6mHD
|
|
138
139
|
fractal_server/app/schemas/v2/workflow.py,sha256=Zzx3e-qgkH8le0FUmAx9UrV5PWd7bj14PPXUh_zgZXM,1827
|
139
140
|
fractal_server/app/schemas/v2/workflowtask.py,sha256=atVuVN4aXsVEOmSd-vyg-8_8OnPmqx-gT75rXcn_AlQ,6552
|
140
141
|
fractal_server/app/security/__init__.py,sha256=2-QbwuR-nsuHM_uwKS_WzYvkhnuhO5jUv8UVROetyVk,11169
|
141
|
-
fractal_server/config.py,sha256=
|
142
|
+
fractal_server/config.py,sha256=KOa2jrsbx0H6zG2ItNZkLiKqbuOkV3aUYKFuIN3FIyE,24921
|
142
143
|
fractal_server/data_migrations/README.md,sha256=_3AEFvDg9YkybDqCLlFPdDmGJvr6Tw7HRI14aZ3LOIw,398
|
143
144
|
fractal_server/gunicorn_fractal.py,sha256=2AOkgxu-oQ-XB578_voT0VuhmAXFTmb0c-nYn1XLy_Q,1231
|
144
145
|
fractal_server/images/__init__.py,sha256=xO6jTLE4EZKO6cTDdJsBmK9cdeh9hFTaSbSuWgQg7y4,196
|
145
146
|
fractal_server/images/models.py,sha256=9ipU5h4N6ogBChoB-2vHoqtL0TXOHCv6kRR-fER3mkM,4167
|
146
147
|
fractal_server/images/tools.py,sha256=gxeniYy4Z-cp_ToK2LHPJUTVVUUrdpogYdcBUvBuLiY,2209
|
147
148
|
fractal_server/logger.py,sha256=56wfka6fHaa3Rx5qO009nEs_y8gx5wZ2NUNZZ1I-uvc,5130
|
148
|
-
fractal_server/main.py,sha256=
|
149
|
+
fractal_server/main.py,sha256=Kmty1C9jPfH101nP3b82u9H9QvT-5Z-8Dd60wf9S5h0,5298
|
149
150
|
fractal_server/migrations/README,sha256=4rQvyDfqodGhpJw74VYijRmgFP49ji5chyEemWGHsuw,59
|
150
151
|
fractal_server/migrations/env.py,sha256=Bvg-FJzRJZIH_wqS_ZyZNXANIaathjo22_IY7c3fCjo,2636
|
151
152
|
fractal_server/migrations/script.py.mako,sha256=oMXw9LC3zRbinWWPPDgeZ4z9FJrV2zhRWiYdS5YgNbI,526
|
@@ -192,8 +193,8 @@ fractal_server/tasks/v2/templates/_5_pip_show.sh,sha256=GrJ19uHYQxANEy9JaeNJZVTq
|
|
192
193
|
fractal_server/tasks/v2/utils.py,sha256=JOyCacb6MNvrwfLNTyLwcz8y79J29YuJeJ2MK5kqXRM,1657
|
193
194
|
fractal_server/urls.py,sha256=5o_qq7PzKKbwq12NHSQZDmDitn5RAOeQ4xufu-2v9Zk,448
|
194
195
|
fractal_server/utils.py,sha256=b7WwFdcFZ8unyT65mloFToYuEDXpQoHRcmRNqrhd_dQ,2115
|
195
|
-
fractal_server-2.3.
|
196
|
-
fractal_server-2.3.
|
197
|
-
fractal_server-2.3.
|
198
|
-
fractal_server-2.3.
|
199
|
-
fractal_server-2.3.
|
196
|
+
fractal_server-2.3.4.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
|
197
|
+
fractal_server-2.3.4.dist-info/METADATA,sha256=mfO_uoRAUY2bfiBN0AU1exGu0orny30foijJx6SR1Xc,4425
|
198
|
+
fractal_server-2.3.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
199
|
+
fractal_server-2.3.4.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
|
200
|
+
fractal_server-2.3.4.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|