fractal-server 2.16.5__py3-none-any.whl → 2.17.0a0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. fractal_server/__init__.py +1 -1
  2. fractal_server/__main__.py +129 -22
  3. fractal_server/app/db/__init__.py +9 -11
  4. fractal_server/app/models/security.py +7 -3
  5. fractal_server/app/models/user_settings.py +0 -4
  6. fractal_server/app/models/v2/__init__.py +4 -0
  7. fractal_server/app/models/v2/job.py +3 -4
  8. fractal_server/app/models/v2/profile.py +16 -0
  9. fractal_server/app/models/v2/project.py +3 -0
  10. fractal_server/app/models/v2/resource.py +130 -0
  11. fractal_server/app/models/v2/task_group.py +3 -0
  12. fractal_server/app/routes/admin/v2/__init__.py +4 -0
  13. fractal_server/app/routes/admin/v2/_aux_functions.py +55 -0
  14. fractal_server/app/routes/admin/v2/profile.py +86 -0
  15. fractal_server/app/routes/admin/v2/resource.py +229 -0
  16. fractal_server/app/routes/admin/v2/task_group_lifecycle.py +48 -82
  17. fractal_server/app/routes/api/__init__.py +26 -7
  18. fractal_server/app/routes/api/v2/_aux_functions.py +27 -1
  19. fractal_server/app/routes/api/v2/_aux_functions_history.py +2 -2
  20. fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py +3 -3
  21. fractal_server/app/routes/api/v2/_aux_functions_tasks.py +7 -7
  22. fractal_server/app/routes/api/v2/project.py +5 -1
  23. fractal_server/app/routes/api/v2/submit.py +32 -24
  24. fractal_server/app/routes/api/v2/task.py +5 -0
  25. fractal_server/app/routes/api/v2/task_collection.py +36 -47
  26. fractal_server/app/routes/api/v2/task_collection_custom.py +11 -5
  27. fractal_server/app/routes/api/v2/task_collection_pixi.py +34 -40
  28. fractal_server/app/routes/api/v2/task_group_lifecycle.py +39 -82
  29. fractal_server/app/routes/api/v2/workflow_import.py +4 -3
  30. fractal_server/app/routes/auth/_aux_auth.py +3 -3
  31. fractal_server/app/routes/auth/current_user.py +45 -7
  32. fractal_server/app/routes/auth/oauth.py +1 -1
  33. fractal_server/app/routes/auth/users.py +9 -0
  34. fractal_server/app/routes/aux/_runner.py +2 -1
  35. fractal_server/app/routes/aux/validate_user_profile.py +62 -0
  36. fractal_server/app/routes/aux/validate_user_settings.py +12 -9
  37. fractal_server/app/schemas/user.py +20 -13
  38. fractal_server/app/schemas/user_settings.py +0 -4
  39. fractal_server/app/schemas/v2/__init__.py +11 -0
  40. fractal_server/app/schemas/v2/profile.py +72 -0
  41. fractal_server/app/schemas/v2/resource.py +117 -0
  42. fractal_server/app/security/__init__.py +6 -13
  43. fractal_server/app/security/signup_email.py +2 -2
  44. fractal_server/app/user_settings.py +2 -12
  45. fractal_server/config/__init__.py +23 -0
  46. fractal_server/config/_database.py +58 -0
  47. fractal_server/config/_email.py +170 -0
  48. fractal_server/config/_init_data.py +27 -0
  49. fractal_server/config/_main.py +216 -0
  50. fractal_server/config/_settings_config.py +7 -0
  51. fractal_server/images/tools.py +3 -3
  52. fractal_server/logger.py +3 -3
  53. fractal_server/main.py +14 -21
  54. fractal_server/migrations/versions/90f6508c6379_drop_useroauth_username.py +36 -0
  55. fractal_server/migrations/versions/a80ac5a352bf_resource_profile.py +195 -0
  56. fractal_server/runner/config/__init__.py +2 -0
  57. fractal_server/runner/config/_local.py +21 -0
  58. fractal_server/runner/config/_slurm.py +128 -0
  59. fractal_server/runner/config/slurm_mem_to_MB.py +63 -0
  60. fractal_server/runner/exceptions.py +4 -0
  61. fractal_server/runner/executors/base_runner.py +17 -7
  62. fractal_server/runner/executors/local/get_local_config.py +21 -86
  63. fractal_server/runner/executors/local/runner.py +48 -5
  64. fractal_server/runner/executors/slurm_common/_batching.py +2 -2
  65. fractal_server/runner/executors/slurm_common/base_slurm_runner.py +59 -25
  66. fractal_server/runner/executors/slurm_common/get_slurm_config.py +38 -54
  67. fractal_server/runner/executors/slurm_common/remote.py +1 -1
  68. fractal_server/runner/executors/slurm_common/{_slurm_config.py → slurm_config.py} +3 -254
  69. fractal_server/runner/executors/slurm_common/slurm_job_task_models.py +1 -1
  70. fractal_server/runner/executors/slurm_ssh/runner.py +12 -14
  71. fractal_server/runner/executors/slurm_sudo/_subprocess_run_as_user.py +2 -2
  72. fractal_server/runner/executors/slurm_sudo/runner.py +12 -12
  73. fractal_server/runner/v2/_local.py +36 -21
  74. fractal_server/runner/v2/_slurm_ssh.py +40 -4
  75. fractal_server/runner/v2/_slurm_sudo.py +41 -11
  76. fractal_server/runner/v2/db_tools.py +1 -1
  77. fractal_server/runner/v2/runner.py +3 -11
  78. fractal_server/runner/v2/runner_functions.py +42 -28
  79. fractal_server/runner/v2/submit_workflow.py +87 -108
  80. fractal_server/runner/versions.py +8 -3
  81. fractal_server/ssh/_fabric.py +6 -6
  82. fractal_server/tasks/config/__init__.py +3 -0
  83. fractal_server/tasks/config/_pixi.py +127 -0
  84. fractal_server/tasks/config/_python.py +51 -0
  85. fractal_server/tasks/v2/local/_utils.py +7 -7
  86. fractal_server/tasks/v2/local/collect.py +13 -5
  87. fractal_server/tasks/v2/local/collect_pixi.py +26 -10
  88. fractal_server/tasks/v2/local/deactivate.py +7 -1
  89. fractal_server/tasks/v2/local/deactivate_pixi.py +5 -1
  90. fractal_server/tasks/v2/local/delete.py +4 -0
  91. fractal_server/tasks/v2/local/reactivate.py +13 -5
  92. fractal_server/tasks/v2/local/reactivate_pixi.py +27 -9
  93. fractal_server/tasks/v2/ssh/_pixi_slurm_ssh.py +11 -10
  94. fractal_server/tasks/v2/ssh/_utils.py +6 -7
  95. fractal_server/tasks/v2/ssh/collect.py +19 -12
  96. fractal_server/tasks/v2/ssh/collect_pixi.py +34 -16
  97. fractal_server/tasks/v2/ssh/deactivate.py +12 -8
  98. fractal_server/tasks/v2/ssh/deactivate_pixi.py +14 -10
  99. fractal_server/tasks/v2/ssh/delete.py +12 -9
  100. fractal_server/tasks/v2/ssh/reactivate.py +18 -12
  101. fractal_server/tasks/v2/ssh/reactivate_pixi.py +36 -17
  102. fractal_server/tasks/v2/templates/4_pip_show.sh +4 -6
  103. fractal_server/tasks/v2/utils_database.py +2 -2
  104. fractal_server/tasks/v2/utils_python_interpreter.py +8 -16
  105. fractal_server/tasks/v2/utils_templates.py +7 -10
  106. fractal_server/utils.py +1 -1
  107. {fractal_server-2.16.5.dist-info → fractal_server-2.17.0a0.dist-info}/METADATA +5 -5
  108. {fractal_server-2.16.5.dist-info → fractal_server-2.17.0a0.dist-info}/RECORD +112 -90
  109. {fractal_server-2.16.5.dist-info → fractal_server-2.17.0a0.dist-info}/WHEEL +1 -1
  110. fractal_server/config.py +0 -906
  111. /fractal_server/{runner → app}/shutdown.py +0 -0
  112. {fractal_server-2.16.5.dist-info → fractal_server-2.17.0a0.dist-info}/entry_points.txt +0 -0
  113. {fractal_server-2.16.5.dist-info → fractal_server-2.17.0a0.dist-info/licenses}/LICENSE +0 -0
fractal_server/config.py DELETED
@@ -1,906 +0,0 @@
1
- # Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
2
- # University of Zurich
3
- #
4
- # Original authors:
5
- # Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
6
- # Tommaso Comparin <tommaso.comparin@exact-lab.it>
7
- # Yuri Chiucconi <yuri.chiucconi@exact-lab.it>
8
- # Marco Franzon <marco.franzon@exact-lab.it>
9
- #
10
- # This file is part of Fractal and was originally developed by eXact lab S.r.l.
11
- # <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
12
- # Institute for Biomedical Research and Pelkmans Lab from the University of
13
- # Zurich.
14
- import json
15
- import logging
16
- import shutil
17
- import sys
18
- from os import environ
19
- from os import getenv
20
- from pathlib import Path
21
- from typing import Annotated
22
- from typing import Literal
23
- from typing import TypeVar
24
-
25
- from cryptography.fernet import Fernet
26
- from dotenv import load_dotenv
27
- from pydantic import AfterValidator
28
- from pydantic import BaseModel
29
- from pydantic import EmailStr
30
- from pydantic import Field
31
- from pydantic import field_validator
32
- from pydantic import model_validator
33
- from pydantic import PositiveInt
34
- from pydantic import SecretStr
35
- from pydantic_settings import BaseSettings
36
- from pydantic_settings import SettingsConfigDict
37
- from sqlalchemy.engine import URL
38
-
39
- import fractal_server
40
- from fractal_server.types import AbsolutePathStr
41
- from fractal_server.types import DictStrStr
42
- from fractal_server.types import NonEmptyStr
43
-
44
-
45
- class MailSettings(BaseModel):
46
- """
47
- Schema for `MailSettings`
48
-
49
- Attributes:
50
- sender: Sender email address
51
- recipients: List of recipients email address
52
- smtp_server: SMTP server address
53
- port: SMTP server port
54
- password: Sender password
55
- instance_name: Name of SMTP server instance
56
- use_starttls: Whether to use the security protocol
57
- use_login: Whether to use login
58
- """
59
-
60
- sender: EmailStr
61
- recipients: list[EmailStr] = Field(min_length=1)
62
- smtp_server: str
63
- port: int
64
- encrypted_password: SecretStr | None = None
65
- encryption_key: SecretStr | None = None
66
- instance_name: str
67
- use_starttls: bool
68
- use_login: bool
69
-
70
-
71
- def _check_pixi_slurm_memory(mem: str) -> str:
72
- if mem[-1] not in ["K", "M", "G", "T"]:
73
- raise ValueError(
74
- f"Invalid memory requirement {mem=} for `pixi`, "
75
- "please set a K/M/G/T units suffix."
76
- )
77
- return mem
78
-
79
-
80
- class PixiSLURMConfig(BaseModel):
81
- """
82
- Parameters that are passed directly to a `sbatch` command.
83
-
84
- See https://slurm.schedmd.com/sbatch.html.
85
- """
86
-
87
- partition: NonEmptyStr
88
- """
89
- `-p, --partition=<partition_names>`
90
- """
91
- cpus: PositiveInt
92
- """
93
- `-c, --cpus-per-task=<ncpus>
94
- """
95
- mem: Annotated[NonEmptyStr, AfterValidator(_check_pixi_slurm_memory)]
96
- """
97
- `--mem=<size>[units]` (examples: `"10M"`, `"10G"`).
98
- From `sbatch` docs: Specify the real memory required per node. Default
99
- units are megabytes. Different units can be specified using the suffix
100
- [K|M|G|T].
101
- """
102
- time: NonEmptyStr
103
- """
104
- `-t, --time=<time>`.
105
- From `sbatch` docs: "A time limit of zero requests that no time limit be
106
- imposed. Acceptable time formats include "minutes", "minutes:seconds",
107
- "hours:minutes:seconds", "days-hours", "days-hours:minutes" and
108
- "days-hours:minutes:seconds".
109
- """
110
-
111
-
112
- class PixiSettings(BaseModel):
113
- """
114
- Configuration for Pixi Task collection.
115
-
116
- In order to use Pixi for Task collection, you must have one or more Pixi
117
- binaries in your machine
118
- (see
119
- [example/get_pixi.sh](https://github.com/fractal-analytics-platform/fractal-server/blob/main/example/get_pixi.sh)
120
- for installation example).
121
-
122
- To let Fractal Server use these binaries for Task collection, a JSON file
123
- must be prepared with the data to populate `PixiSettings` (arguments with
124
- default values may be omitted).
125
-
126
- The path to this JSON file must then be provided to Fractal via the
127
- environment variable `FRACTAL_PIXI_CONFIG_FILE`.
128
- """
129
-
130
- versions: DictStrStr
131
- """
132
- A dictionary with Pixi versions as keys and paths to the corresponding
133
- folder as values.
134
-
135
- E.g. let's assume that you have Pixi v0.47.0 at
136
- `/pixi-path/0.47.0/bin/pixi` and Pixi v0.48.2 at
137
- `/pixi-path/0.48.2/bin/pixi`, then
138
- ```json
139
- "versions": {
140
- "0.47.0": "/pixi-path/0.47.0",
141
- "0.48.2": "/pixi-path/0.48.2"
142
- }
143
- ```
144
- """
145
- default_version: str
146
- """
147
- Default Pixi version to be used for Task collection.
148
-
149
- Must be a key of the `versions` dictionary.
150
- """
151
- PIXI_CONCURRENT_SOLVES: int = 4
152
- """
153
- Value of
154
- [`--concurrent-solves`](https://pixi.sh/latest/reference/cli/pixi/install/#arg---concurrent-solves)
155
- for `pixi install`.
156
- """
157
- PIXI_CONCURRENT_DOWNLOADS: int = 4
158
- """
159
- Value of
160
- [`--concurrent-downloads`](https://pixi.sh/latest/reference/cli/pixi/install/#arg---concurrent-downloads)
161
- for `pixi install`.
162
- """
163
- TOKIO_WORKER_THREADS: int = 2
164
- """
165
- From
166
- [Tokio documentation](
167
- https://docs.rs/tokio/latest/tokio/#cpu-bound-tasks-and-blocking-code
168
- )
169
- :
170
-
171
- The core threads are where all asynchronous code runs,
172
- and Tokio will by default spawn one for each CPU core.
173
- You can use the environment variable `TOKIO_WORKER_THREADS` to override
174
- the default value.
175
- """
176
- DEFAULT_ENVIRONMENT: str = "default"
177
- """
178
- Default pixi environment name.
179
- """
180
- DEFAULT_PLATFORM: str = "linux-64"
181
- """
182
- Default platform for pixi.
183
- """
184
- SLURM_CONFIG: PixiSLURMConfig | None = None
185
- """
186
- Required when using pixi in a SSH/SLURM deployment.
187
- """
188
-
189
- @model_validator(mode="after")
190
- def check_pixi_settings(self):
191
- if self.default_version not in self.versions:
192
- raise ValueError(
193
- f"Default version '{self.default_version}' not in "
194
- f"available version {list(self.versions.keys())}."
195
- )
196
-
197
- pixi_base_dir = Path(self.versions[self.default_version]).parent
198
-
199
- for key, value in self.versions.items():
200
- pixi_path = Path(value)
201
-
202
- if pixi_path.parent != pixi_base_dir:
203
- raise ValueError(
204
- f"{pixi_path=} is not located within the {pixi_base_dir=}."
205
- )
206
- if pixi_path.name != key:
207
- raise ValueError(f"{pixi_path.name=} is not equal to {key=}")
208
-
209
- return self
210
-
211
-
212
- class FractalConfigurationError(RuntimeError):
213
- pass
214
-
215
-
216
- T = TypeVar("T")
217
-
218
-
219
- load_dotenv(".fractal_server.env")
220
-
221
-
222
- class OAuthClientConfig(BaseModel):
223
- """
224
- OAuth Client Config Model
225
-
226
- This model wraps the variables that define a client against an Identity
227
- Provider. As some providers are supported by the libraries used within the
228
- server, some attributes are optional.
229
-
230
- Attributes:
231
- CLIENT_NAME:
232
- The name of the client
233
- CLIENT_ID:
234
- ID of client
235
- CLIENT_SECRET:
236
- Secret to authorise against the identity provider
237
- OIDC_CONFIGURATION_ENDPOINT:
238
- OpenID configuration endpoint,
239
- allowing to discover the required endpoints automatically
240
- REDIRECT_URL:
241
- String to be used as `redirect_url` argument for
242
- `fastapi_users.get_oauth_router`, and then in
243
- `httpx_oauth.integrations.fastapi.OAuth2AuthorizeCallback`.
244
- """
245
-
246
- CLIENT_NAME: str
247
- CLIENT_ID: str
248
- CLIENT_SECRET: SecretStr
249
- OIDC_CONFIGURATION_ENDPOINT: str | None = None
250
- REDIRECT_URL: str | None = None
251
-
252
- @model_validator(mode="before")
253
- @classmethod
254
- def check_configuration(cls, values):
255
- if values.get("CLIENT_NAME") not in ["GOOGLE", "GITHUB"]:
256
- if not values.get("OIDC_CONFIGURATION_ENDPOINT"):
257
- raise FractalConfigurationError(
258
- f"Missing OAUTH_{values.get('CLIENT_NAME')}"
259
- "_OIDC_CONFIGURATION_ENDPOINT"
260
- )
261
- return values
262
-
263
-
264
- class Settings(BaseSettings):
265
- """
266
- Contains all the configuration variables for Fractal Server
267
-
268
- The attributes of this class are set from the environment.
269
- """
270
-
271
- model_config = SettingsConfigDict(case_sensitive=True)
272
-
273
- PROJECT_NAME: str = "Fractal Server"
274
- PROJECT_VERSION: str = fractal_server.__VERSION__
275
-
276
- ###########################################################################
277
- # AUTH
278
- ###########################################################################
279
-
280
- OAUTH_CLIENTS_CONFIG: list[OAuthClientConfig] = Field(default_factory=list)
281
-
282
- # JWT TOKEN
283
- JWT_EXPIRE_SECONDS: int = 180
284
- """
285
- JWT token lifetime, in seconds.
286
- """
287
-
288
- JWT_SECRET_KEY: SecretStr | None = None
289
- """
290
- JWT secret
291
-
292
- ⚠️ **IMPORTANT**: set this variable to a secure string, and do not disclose
293
- it.
294
- """
295
-
296
- # COOKIE TOKEN
297
- COOKIE_EXPIRE_SECONDS: int = 86400
298
- """
299
- Cookie token lifetime, in seconds.
300
- """
301
-
302
- @model_validator(mode="before")
303
- @classmethod
304
- def collect_oauth_clients(cls, values):
305
- """
306
- Automatic collection of OAuth Clients
307
-
308
- This method collects the environment variables relative to a single
309
- OAuth client and saves them within the `Settings` object in the form
310
- of an `OAuthClientConfig` instance.
311
-
312
- Fractal can support an arbitrary number of OAuth providers, which are
313
- automatically detected by parsing the environment variable names. In
314
- particular, to set the provider `FOO`, one must specify the variables
315
-
316
- OAUTH_FOO_CLIENT_ID
317
- OAUTH_FOO_CLIENT_SECRET
318
- ...
319
-
320
- etc (cf. OAuthClientConfig).
321
- """
322
- oauth_env_variable_keys = [
323
- key for key in environ.keys() if key.startswith("OAUTH_")
324
- ]
325
- clients_available = {
326
- var.split("_")[1] for var in oauth_env_variable_keys
327
- }
328
-
329
- values["OAUTH_CLIENTS_CONFIG"] = []
330
- for client in clients_available:
331
- prefix = f"OAUTH_{client}"
332
- oauth_client_config = OAuthClientConfig(
333
- CLIENT_NAME=client,
334
- CLIENT_ID=getenv(f"{prefix}_CLIENT_ID", None),
335
- CLIENT_SECRET=getenv(f"{prefix}_CLIENT_SECRET", None),
336
- OIDC_CONFIGURATION_ENDPOINT=getenv(
337
- f"{prefix}_OIDC_CONFIGURATION_ENDPOINT", None
338
- ),
339
- REDIRECT_URL=getenv(f"{prefix}_REDIRECT_URL", None),
340
- )
341
- values["OAUTH_CLIENTS_CONFIG"].append(oauth_client_config)
342
- return values
343
-
344
- ###########################################################################
345
- # DATABASE
346
- ###########################################################################
347
- DB_ECHO: bool = False
348
- """
349
- If `True`, make database operations verbose.
350
- """
351
- POSTGRES_USER: str | None = None
352
- """
353
- User to use when connecting to the PostgreSQL database.
354
- """
355
- POSTGRES_PASSWORD: SecretStr | None = None
356
- """
357
- Password to use when connecting to the PostgreSQL database.
358
- """
359
- POSTGRES_HOST: str | None = "localhost"
360
- """
361
- URL to the PostgreSQL server or path to a UNIX domain socket.
362
- """
363
- POSTGRES_PORT: str | None = "5432"
364
- """
365
- Port number to use when connecting to the PostgreSQL server.
366
- """
367
- POSTGRES_DB: str | None = None
368
- """
369
- Name of the PostgreSQL database to connect to.
370
- """
371
-
372
- @property
373
- def DATABASE_ASYNC_URL(self) -> URL:
374
- if self.POSTGRES_PASSWORD is None:
375
- password = None
376
- else:
377
- password = self.POSTGRES_PASSWORD.get_secret_value()
378
-
379
- url = URL.create(
380
- drivername="postgresql+psycopg",
381
- username=self.POSTGRES_USER,
382
- password=password,
383
- host=self.POSTGRES_HOST,
384
- port=self.POSTGRES_PORT,
385
- database=self.POSTGRES_DB,
386
- )
387
- return url
388
-
389
- @property
390
- def DATABASE_SYNC_URL(self):
391
- return self.DATABASE_ASYNC_URL.set(drivername="postgresql+psycopg")
392
-
393
- ###########################################################################
394
- # FRACTAL SPECIFIC
395
- ###########################################################################
396
-
397
- FRACTAL_DEFAULT_ADMIN_EMAIL: str = "admin@fractal.xy"
398
- """
399
- Admin default email, used upon creation of the first superuser during
400
- server startup.
401
-
402
- ⚠️ **IMPORTANT**: After the server startup, you should always edit the
403
- default admin credentials.
404
- """
405
-
406
- FRACTAL_DEFAULT_ADMIN_PASSWORD: SecretStr = "1234"
407
- """
408
- Admin default password, used upon creation of the first superuser during
409
- server startup.
410
-
411
- ⚠️ **IMPORTANT**: After the server startup, you should always edit the
412
- default admin credentials.
413
- """
414
-
415
- FRACTAL_DEFAULT_ADMIN_USERNAME: str = "admin"
416
- """
417
- Admin default username, used upon creation of the first superuser during
418
- server startup.
419
-
420
- ⚠️ **IMPORTANT**: After the server startup, you should always edit the
421
- default admin credentials.
422
- """
423
-
424
- FRACTAL_TASKS_DIR: Path | None = None
425
- """
426
- Directory under which all the tasks will be saved (either an absolute path
427
- or a path relative to current working directory).
428
- """
429
-
430
- FRACTAL_RUNNER_WORKING_BASE_DIR: Path | None = None
431
- """
432
- Base directory for job files (either an absolute path or a path relative to
433
- current working directory).
434
- """
435
-
436
- @field_validator(
437
- "FRACTAL_TASKS_DIR",
438
- "FRACTAL_RUNNER_WORKING_BASE_DIR",
439
- mode="after",
440
- )
441
- @classmethod
442
- def make_paths_absolute(cls, path: Path | None) -> Path | None:
443
- if path is None or path.is_absolute():
444
- return path
445
- else:
446
- logging.warning(
447
- f"'{path}' is not an absolute path; "
448
- f"converting it to '{path.resolve()}'"
449
- )
450
- return path.resolve()
451
-
452
- FRACTAL_RUNNER_BACKEND: Literal[
453
- "local",
454
- "slurm",
455
- "slurm_ssh",
456
- ] = "local"
457
- """
458
- Select which runner backend to use.
459
- """
460
-
461
- FRACTAL_LOGGING_LEVEL: int = logging.INFO
462
- """
463
- Logging-level threshold for logging
464
-
465
- Only logs of with this level (or higher) will appear in the console logs.
466
- """
467
-
468
- FRACTAL_LOCAL_CONFIG_FILE: Path | None = None
469
- """
470
- Path of JSON file with configuration for the local backend.
471
- """
472
-
473
- FRACTAL_API_MAX_JOB_LIST_LENGTH: int = 50
474
- """
475
- Number of ids that can be stored in the `jobsV2` attribute of
476
- `app.state`.
477
- """
478
-
479
- FRACTAL_GRACEFUL_SHUTDOWN_TIME: int = 30
480
- """
481
- Waiting time for the shutdown phase of executors
482
- """
483
-
484
- FRACTAL_SLURM_CONFIG_FILE: Path | None = None
485
- """
486
- Path of JSON file with configuration for the SLURM backend.
487
- """
488
-
489
- FRACTAL_SLURM_WORKER_PYTHON: AbsolutePathStr | None = None
490
- """
491
- Absolute path to Python interpreter that will run the jobs on the SLURM
492
- nodes. If not specified, the same interpreter that runs the server is used.
493
- """
494
-
495
- FRACTAL_TASKS_PYTHON_DEFAULT_VERSION: None | (
496
- Literal["3.9", "3.10", "3.11", "3.12", "3.13"]
497
- ) = None
498
- """
499
- Default Python version to be used for task collection. Defaults to the
500
- current version. Requires the corresponding variable (e.g
501
- `FRACTAL_TASKS_PYTHON_3_10`) to be set.
502
- """
503
-
504
- FRACTAL_TASKS_PYTHON_3_9: str | None = None
505
- """
506
- Absolute path to the Python 3.9 interpreter that serves as base for virtual
507
- environments tasks. Note that this interpreter must have the `venv` module
508
- installed. If set, this must be an absolute path. If the version specified
509
- in `FRACTAL_TASKS_PYTHON_DEFAULT_VERSION` is `"3.9"` and this attribute is
510
- unset, `sys.executable` is used as a default.
511
- """
512
-
513
- FRACTAL_TASKS_PYTHON_3_10: str | None = None
514
- """
515
- Same as `FRACTAL_TASKS_PYTHON_3_9`, for Python 3.10.
516
- """
517
-
518
- FRACTAL_TASKS_PYTHON_3_11: str | None = None
519
- """
520
- Same as `FRACTAL_TASKS_PYTHON_3_9`, for Python 3.11.
521
- """
522
-
523
- FRACTAL_TASKS_PYTHON_3_12: str | None = None
524
- """
525
- Same as `FRACTAL_TASKS_PYTHON_3_9`, for Python 3.12.
526
- """
527
-
528
- FRACTAL_TASKS_PYTHON_3_13: str | None = None
529
- """
530
- Same as `FRACTAL_TASKS_PYTHON_3_9`, for Python 3.13.
531
- """
532
-
533
- @model_validator(mode="before")
534
- @classmethod
535
- def check_tasks_python(cls, values):
536
- """
537
- Perform multiple checks of the Python-interpreter variables.
538
-
539
- 1. Each `FRACTAL_TASKS_PYTHON_X_Y` variable must be an absolute path,
540
- if set.
541
- 2. If `FRACTAL_TASKS_PYTHON_DEFAULT_VERSION` is unset, use
542
- `sys.executable` and set the corresponding
543
- `FRACTAL_TASKS_PYTHON_X_Y` (and unset all others).
544
- """
545
- # `FRACTAL_TASKS_PYTHON_X_Y` variables can only be absolute paths
546
- for version in ["3_9", "3_10", "3_11", "3_12", "3_13"]:
547
- key = f"FRACTAL_TASKS_PYTHON_{version}"
548
- value = values.get(key)
549
- if value is not None and not Path(value).is_absolute():
550
- raise FractalConfigurationError(
551
- f"Non-absolute value {key}={value}"
552
- )
553
-
554
- default_version = values.get("FRACTAL_TASKS_PYTHON_DEFAULT_VERSION")
555
-
556
- if default_version is not None:
557
- # "production/slurm" branch
558
- # If a default version is set, then the corresponding interpreter
559
- # must also be set
560
- default_version_undescore = default_version.replace(".", "_")
561
- key = f"FRACTAL_TASKS_PYTHON_{default_version_undescore}"
562
- value = values.get(key)
563
- if value is None:
564
- msg = (
565
- f"FRACTAL_TASKS_PYTHON_DEFAULT_VERSION={default_version} "
566
- f"but {key}={value}."
567
- )
568
- logging.error(msg)
569
- raise FractalConfigurationError(msg)
570
-
571
- else:
572
- # If no default version is set, then only `sys.executable` is made
573
- # available
574
- _info = sys.version_info
575
- current_version = f"{_info.major}_{_info.minor}"
576
- current_version_dot = f"{_info.major}.{_info.minor}"
577
- values[
578
- "FRACTAL_TASKS_PYTHON_DEFAULT_VERSION"
579
- ] = current_version_dot
580
- logging.info(
581
- "Setting FRACTAL_TASKS_PYTHON_DEFAULT_VERSION to "
582
- f"{current_version_dot}"
583
- )
584
-
585
- # Unset all existing interpreters variable
586
- for _version in ["3_9", "3_10", "3_11", "3_12", "3_13"]:
587
- key = f"FRACTAL_TASKS_PYTHON_{_version}"
588
- if _version == current_version:
589
- values[key] = sys.executable
590
- logging.info(f"Setting {key} to {sys.executable}.")
591
- else:
592
- value = values.get(key)
593
- if value is not None:
594
- logging.info(
595
- f"Setting {key} to None (given: {value}), "
596
- "because FRACTAL_TASKS_PYTHON_DEFAULT_VERSION was "
597
- "not set."
598
- )
599
- values[key] = None
600
- return values
601
-
602
- FRACTAL_SLURM_POLL_INTERVAL: int = 5
603
- """
604
- Interval to wait (in seconds) before checking whether unfinished job are
605
- still running on SLURM.
606
- """
607
-
608
- FRACTAL_PIP_CACHE_DIR: AbsolutePathStr | None = None
609
- """
610
- Absolute path to the cache directory for `pip`; if unset,
611
- `--no-cache-dir` is used.
612
- """
613
-
614
- @property
615
- def PIP_CACHE_DIR_ARG(self) -> str:
616
- """
617
- Option for `pip install`, based on `FRACTAL_PIP_CACHE_DIR` value.
618
-
619
- If `FRACTAL_PIP_CACHE_DIR` is set, then return
620
- `--cache-dir /somewhere`; else return `--no-cache-dir`.
621
- """
622
- if self.FRACTAL_PIP_CACHE_DIR is not None:
623
- return f"--cache-dir {self.FRACTAL_PIP_CACHE_DIR}"
624
- else:
625
- return "--no-cache-dir"
626
-
627
- FRACTAL_VIEWER_AUTHORIZATION_SCHEME: Literal[
628
- "viewer-paths", "users-folders", "none"
629
- ] = "none"
630
- """
631
- Defines how the list of allowed viewer paths is built.
632
-
633
- This variable affects the `GET /auth/current-user/allowed-viewer-paths/`
634
- response, which is then consumed by
635
- [fractal-vizarr-viewer](https://github.com/fractal-analytics-platform/fractal-vizarr-viewer).
636
-
637
- Options:
638
-
639
- - "viewer-paths": The list of allowed viewer paths will include the user's
640
- `project_dir` along with any path defined in user groups' `viewer_paths`
641
- attributes.
642
- - "users-folders": The list will consist of the user's `project_dir` and a
643
- user-specific folder. The user folder is constructed by concatenating
644
- the base folder `FRACTAL_VIEWER_BASE_FOLDER` with the user's
645
- `slurm_user`.
646
- - "none": An empty list will be returned, indicating no access to
647
- viewer paths. Useful when vizarr viewer is not used.
648
- """
649
-
650
- FRACTAL_VIEWER_BASE_FOLDER: str | None = None
651
- """
652
- Base path to Zarr files that will be served by fractal-vizarr-viewer;
653
- This variable is required and used only when
654
- FRACTAL_VIEWER_AUTHORIZATION_SCHEME is set to "users-folders".
655
- """
656
-
657
- FRACTAL_PIXI_CONFIG_FILE: Path | None = None
658
- """
659
- Path to the Pixi configuration JSON file that will populate `PixiSettings`.
660
- """
661
-
662
- pixi: PixiSettings | None = None
663
-
664
- @model_validator(mode="after")
665
- def populate_pixi_settings(self):
666
- if self.FRACTAL_PIXI_CONFIG_FILE is not None:
667
- with self.FRACTAL_PIXI_CONFIG_FILE.open("r") as f:
668
- self.pixi = PixiSettings(**json.load(f))
669
- return self
670
-
671
- ###########################################################################
672
- # SMTP SERVICE
673
- ###########################################################################
674
-
675
- FRACTAL_EMAIL_SENDER: EmailStr | None = None
676
- """
677
- Address of the OAuth-signup email sender.
678
- """
679
- FRACTAL_EMAIL_PASSWORD: SecretStr | None = None
680
- """
681
- Password for the OAuth-signup email sender.
682
- """
683
- FRACTAL_EMAIL_PASSWORD_KEY: SecretStr | None = None
684
- """
685
- Key value for `cryptography.fernet` decrypt
686
- """
687
- FRACTAL_EMAIL_SMTP_SERVER: str | None = None
688
- """
689
- SMTP server for the OAuth-signup emails.
690
- """
691
- FRACTAL_EMAIL_SMTP_PORT: int | None = None
692
- """
693
- SMTP server port for the OAuth-signup emails.
694
- """
695
- FRACTAL_EMAIL_INSTANCE_NAME: str | None = None
696
- """
697
- Fractal instance name, to be included in the OAuth-signup emails.
698
- """
699
- FRACTAL_EMAIL_RECIPIENTS: str | None = None
700
- """
701
- Comma-separated list of recipients of the OAuth-signup emails.
702
- """
703
- FRACTAL_EMAIL_USE_STARTTLS: Literal["true", "false"] = "true"
704
- """
705
- Whether to use StartTLS when using the SMTP server.
706
- Accepted values: 'true', 'false'.
707
- """
708
- FRACTAL_EMAIL_USE_LOGIN: Literal["true", "false"] = "true"
709
- """
710
- Whether to use login when using the SMTP server.
711
- If 'true', FRACTAL_EMAIL_PASSWORD and FRACTAL_EMAIL_PASSWORD_KEY must be
712
- provided.
713
- Accepted values: 'true', 'false'.
714
- """
715
- email_settings: MailSettings | None = None
716
-
717
- @model_validator(mode="after")
718
- def validate_email_settings(self):
719
- email_values = [
720
- self.FRACTAL_EMAIL_SENDER,
721
- self.FRACTAL_EMAIL_SMTP_SERVER,
722
- self.FRACTAL_EMAIL_SMTP_PORT,
723
- self.FRACTAL_EMAIL_INSTANCE_NAME,
724
- self.FRACTAL_EMAIL_RECIPIENTS,
725
- ]
726
- if len(set(email_values)) == 1:
727
- # All required EMAIL attributes are None
728
- pass
729
- elif None in email_values:
730
- # Not all required EMAIL attributes are set
731
- error_msg = (
732
- "Invalid FRACTAL_EMAIL configuration. "
733
- f"Given values: {email_values}."
734
- )
735
- raise ValueError(error_msg)
736
- else:
737
- use_starttls = self.FRACTAL_EMAIL_USE_STARTTLS == "true"
738
- use_login = self.FRACTAL_EMAIL_USE_LOGIN == "true"
739
-
740
- if use_login:
741
- if self.FRACTAL_EMAIL_PASSWORD is None:
742
- raise ValueError(
743
- "'FRACTAL_EMAIL_USE_LOGIN' is 'true' but "
744
- "'FRACTAL_EMAIL_PASSWORD' is not provided."
745
- )
746
- if self.FRACTAL_EMAIL_PASSWORD_KEY is None:
747
- raise ValueError(
748
- "'FRACTAL_EMAIL_USE_LOGIN' is 'true' but "
749
- "'FRACTAL_EMAIL_PASSWORD_KEY' is not provided."
750
- )
751
- try:
752
- (
753
- Fernet(
754
- self.FRACTAL_EMAIL_PASSWORD_KEY.get_secret_value()
755
- )
756
- .decrypt(
757
- self.FRACTAL_EMAIL_PASSWORD.get_secret_value()
758
- )
759
- .decode("utf-8")
760
- )
761
- except Exception as e:
762
- raise ValueError(
763
- "Invalid pair (FRACTAL_EMAIL_PASSWORD, "
764
- "FRACTAL_EMAIL_PASSWORD_KEY). "
765
- f"Original error: {str(e)}."
766
- )
767
- password = self.FRACTAL_EMAIL_PASSWORD.get_secret_value()
768
- else:
769
- password = None
770
-
771
- if self.FRACTAL_EMAIL_PASSWORD_KEY is not None:
772
- key = self.FRACTAL_EMAIL_PASSWORD_KEY.get_secret_value()
773
- else:
774
- key = None
775
-
776
- self.email_settings = MailSettings(
777
- sender=self.FRACTAL_EMAIL_SENDER,
778
- recipients=self.FRACTAL_EMAIL_RECIPIENTS.split(","),
779
- smtp_server=self.FRACTAL_EMAIL_SMTP_SERVER,
780
- port=self.FRACTAL_EMAIL_SMTP_PORT,
781
- encrypted_password=password,
782
- encryption_key=key,
783
- instance_name=self.FRACTAL_EMAIL_INSTANCE_NAME,
784
- use_starttls=use_starttls,
785
- use_login=use_login,
786
- )
787
-
788
- return self
789
-
790
- ###########################################################################
791
- # BUSINESS LOGIC
792
- ###########################################################################
793
-
794
- def check_db(self) -> None:
795
- """
796
- Checks that db environment variables are properly set.
797
- """
798
- if not self.POSTGRES_DB:
799
- raise FractalConfigurationError("POSTGRES_DB cannot be None.")
800
-
801
- def check_runner(self) -> None:
802
- if not self.FRACTAL_RUNNER_WORKING_BASE_DIR:
803
- raise FractalConfigurationError(
804
- "FRACTAL_RUNNER_WORKING_BASE_DIR cannot be None."
805
- )
806
-
807
- info = f"FRACTAL_RUNNER_BACKEND={self.FRACTAL_RUNNER_BACKEND}"
808
- if self.FRACTAL_RUNNER_BACKEND == "slurm":
809
- from fractal_server.runner.executors.slurm_common._slurm_config import ( # noqa: E501
810
- load_slurm_config_file,
811
- )
812
-
813
- if not self.FRACTAL_SLURM_CONFIG_FILE:
814
- raise FractalConfigurationError(
815
- f"Must set FRACTAL_SLURM_CONFIG_FILE when {info}"
816
- )
817
- else:
818
- if not self.FRACTAL_SLURM_CONFIG_FILE.exists():
819
- raise FractalConfigurationError(
820
- f"{info} but FRACTAL_SLURM_CONFIG_FILE="
821
- f"{self.FRACTAL_SLURM_CONFIG_FILE} not found."
822
- )
823
-
824
- load_slurm_config_file(self.FRACTAL_SLURM_CONFIG_FILE)
825
- if not shutil.which("sbatch"):
826
- raise FractalConfigurationError(
827
- f"{info} but `sbatch` command not found."
828
- )
829
- if not shutil.which("squeue"):
830
- raise FractalConfigurationError(
831
- f"{info} but `squeue` command not found."
832
- )
833
- elif self.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
834
- if self.FRACTAL_SLURM_WORKER_PYTHON is None:
835
- raise FractalConfigurationError(
836
- f"Must set FRACTAL_SLURM_WORKER_PYTHON when {info}"
837
- )
838
- if self.pixi and self.pixi.SLURM_CONFIG is None:
839
- raise FractalConfigurationError(
840
- "Pixi config must include SLURM_CONFIG."
841
- )
842
-
843
- from fractal_server.runner.executors.slurm_common._slurm_config import ( # noqa: E501
844
- load_slurm_config_file,
845
- )
846
-
847
- if not self.FRACTAL_SLURM_CONFIG_FILE:
848
- raise FractalConfigurationError(
849
- f"Must set FRACTAL_SLURM_CONFIG_FILE when {info}"
850
- )
851
- else:
852
- if not self.FRACTAL_SLURM_CONFIG_FILE.exists():
853
- raise FractalConfigurationError(
854
- f"{info} but FRACTAL_SLURM_CONFIG_FILE="
855
- f"{self.FRACTAL_SLURM_CONFIG_FILE} not found."
856
- )
857
-
858
- load_slurm_config_file(self.FRACTAL_SLURM_CONFIG_FILE)
859
- if not shutil.which("ssh"):
860
- raise FractalConfigurationError(
861
- f"{info} but `ssh` command not found."
862
- )
863
- else: # i.e. self.FRACTAL_RUNNER_BACKEND == "local"
864
- if self.FRACTAL_LOCAL_CONFIG_FILE:
865
- if not self.FRACTAL_LOCAL_CONFIG_FILE.exists():
866
- raise FractalConfigurationError(
867
- f"{info} but FRACTAL_LOCAL_CONFIG_FILE="
868
- f"{self.FRACTAL_LOCAL_CONFIG_FILE} not found."
869
- )
870
-
871
- def check(self):
872
- """
873
- Make sure that required variables are set
874
-
875
- This method must be called before the server starts
876
- """
877
-
878
- if not self.JWT_SECRET_KEY:
879
- raise FractalConfigurationError("JWT_SECRET_KEY cannot be None")
880
-
881
- if not self.FRACTAL_TASKS_DIR:
882
- raise FractalConfigurationError("FRACTAL_TASKS_DIR cannot be None")
883
-
884
- # FRACTAL_VIEWER_BASE_FOLDER is required when
885
- # FRACTAL_VIEWER_AUTHORIZATION_SCHEME is set to "users-folders"
886
- # and it must be an absolute path
887
- if self.FRACTAL_VIEWER_AUTHORIZATION_SCHEME == "users-folders":
888
- viewer_base_folder = self.FRACTAL_VIEWER_BASE_FOLDER
889
- if viewer_base_folder is None:
890
- raise FractalConfigurationError(
891
- "FRACTAL_VIEWER_BASE_FOLDER is required when "
892
- "FRACTAL_VIEWER_AUTHORIZATION_SCHEME is set to "
893
- "users-folders"
894
- )
895
- if not Path(viewer_base_folder).is_absolute():
896
- raise FractalConfigurationError(
897
- f"Non-absolute value for "
898
- f"FRACTAL_VIEWER_BASE_FOLDER={viewer_base_folder}"
899
- )
900
-
901
- self.check_db()
902
- self.check_runner()
903
-
904
-
905
- def get_settings(settings=Settings()) -> Settings:
906
- return settings