fractal-server 2.2.0a1__py3-none-any.whl → 2.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. fractal_server/__init__.py +1 -1
  2. fractal_server/app/models/v1/state.py +1 -2
  3. fractal_server/app/routes/admin/v1.py +2 -2
  4. fractal_server/app/routes/admin/v2.py +2 -2
  5. fractal_server/app/routes/api/v1/job.py +2 -2
  6. fractal_server/app/routes/api/v1/task_collection.py +4 -4
  7. fractal_server/app/routes/api/v2/__init__.py +23 -3
  8. fractal_server/app/routes/api/v2/job.py +2 -2
  9. fractal_server/app/routes/api/v2/submit.py +6 -0
  10. fractal_server/app/routes/api/v2/task_collection.py +74 -34
  11. fractal_server/app/routes/api/v2/task_collection_custom.py +170 -0
  12. fractal_server/app/routes/api/v2/task_collection_ssh.py +125 -0
  13. fractal_server/app/routes/aux/_runner.py +10 -2
  14. fractal_server/app/runner/compress_folder.py +120 -0
  15. fractal_server/app/runner/executors/slurm/__init__.py +0 -3
  16. fractal_server/app/runner/executors/slurm/_batching.py +0 -1
  17. fractal_server/app/runner/executors/slurm/_slurm_config.py +9 -9
  18. fractal_server/app/runner/executors/slurm/ssh/__init__.py +3 -0
  19. fractal_server/app/runner/executors/slurm/ssh/_executor_wait_thread.py +112 -0
  20. fractal_server/app/runner/executors/slurm/ssh/_slurm_job.py +120 -0
  21. fractal_server/app/runner/executors/slurm/ssh/executor.py +1488 -0
  22. fractal_server/app/runner/executors/slurm/sudo/__init__.py +3 -0
  23. fractal_server/app/runner/executors/slurm/{_check_jobs_status.py → sudo/_check_jobs_status.py} +1 -1
  24. fractal_server/app/runner/executors/slurm/{_executor_wait_thread.py → sudo/_executor_wait_thread.py} +1 -1
  25. fractal_server/app/runner/executors/slurm/{_subprocess_run_as_user.py → sudo/_subprocess_run_as_user.py} +1 -1
  26. fractal_server/app/runner/executors/slurm/{executor.py → sudo/executor.py} +12 -12
  27. fractal_server/app/runner/extract_archive.py +38 -0
  28. fractal_server/app/runner/v1/__init__.py +78 -40
  29. fractal_server/app/runner/v1/_slurm/__init__.py +1 -1
  30. fractal_server/app/runner/v2/__init__.py +147 -62
  31. fractal_server/app/runner/v2/_local_experimental/__init__.py +22 -12
  32. fractal_server/app/runner/v2/_local_experimental/executor.py +12 -8
  33. fractal_server/app/runner/v2/_slurm/__init__.py +1 -6
  34. fractal_server/app/runner/v2/_slurm_ssh/__init__.py +125 -0
  35. fractal_server/app/runner/v2/_slurm_ssh/_submit_setup.py +83 -0
  36. fractal_server/app/runner/v2/_slurm_ssh/get_slurm_config.py +182 -0
  37. fractal_server/app/runner/v2/runner_functions_low_level.py +9 -11
  38. fractal_server/app/runner/versions.py +30 -0
  39. fractal_server/app/schemas/v1/__init__.py +1 -0
  40. fractal_server/app/schemas/{state.py → v1/state.py} +4 -21
  41. fractal_server/app/schemas/v2/__init__.py +4 -1
  42. fractal_server/app/schemas/v2/task_collection.py +101 -30
  43. fractal_server/config.py +184 -3
  44. fractal_server/main.py +27 -1
  45. fractal_server/ssh/__init__.py +4 -0
  46. fractal_server/ssh/_fabric.py +245 -0
  47. fractal_server/tasks/utils.py +12 -64
  48. fractal_server/tasks/v1/background_operations.py +2 -2
  49. fractal_server/tasks/{endpoint_operations.py → v1/endpoint_operations.py} +7 -12
  50. fractal_server/tasks/v1/utils.py +67 -0
  51. fractal_server/tasks/v2/_TaskCollectPip.py +61 -32
  52. fractal_server/tasks/v2/_venv_pip.py +195 -0
  53. fractal_server/tasks/v2/background_operations.py +257 -295
  54. fractal_server/tasks/v2/background_operations_ssh.py +317 -0
  55. fractal_server/tasks/v2/endpoint_operations.py +136 -0
  56. fractal_server/tasks/v2/templates/_1_create_venv.sh +46 -0
  57. fractal_server/tasks/v2/templates/_2_upgrade_pip.sh +30 -0
  58. fractal_server/tasks/v2/templates/_3_pip_install.sh +32 -0
  59. fractal_server/tasks/v2/templates/_4_pip_freeze.sh +21 -0
  60. fractal_server/tasks/v2/templates/_5_pip_show.sh +59 -0
  61. fractal_server/tasks/v2/utils.py +54 -0
  62. {fractal_server-2.2.0a1.dist-info → fractal_server-2.3.0.dist-info}/METADATA +4 -2
  63. {fractal_server-2.2.0a1.dist-info → fractal_server-2.3.0.dist-info}/RECORD +66 -42
  64. fractal_server/tasks/v2/get_collection_data.py +0 -14
  65. {fractal_server-2.2.0a1.dist-info → fractal_server-2.3.0.dist-info}/LICENSE +0 -0
  66. {fractal_server-2.2.0a1.dist-info → fractal_server-2.3.0.dist-info}/WHEEL +0 -0
  67. {fractal_server-2.2.0a1.dist-info → fractal_server-2.3.0.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,317 @@
1
+ import json
2
+ import os
3
+ from pathlib import Path
4
+ from tempfile import TemporaryDirectory
5
+
6
+ from sqlalchemy.orm.attributes import flag_modified
7
+
8
+ from ...app.models.v2 import CollectionStateV2
9
+ from ._TaskCollectPip import _TaskCollectPip
10
+ from .background_operations import _handle_failure
11
+ from .background_operations import _insert_tasks
12
+ from .background_operations import _prepare_tasks_metadata
13
+ from .background_operations import _set_collection_state_data_status
14
+ from fractal_server.app.db import get_sync_db
15
+ from fractal_server.app.schemas.v2 import CollectionStatusV2
16
+ from fractal_server.app.schemas.v2.manifest import ManifestV2
17
+ from fractal_server.config import get_settings
18
+ from fractal_server.logger import get_logger
19
+ from fractal_server.logger import set_logger
20
+ from fractal_server.ssh._fabric import FractalSSH
21
+ from fractal_server.ssh._fabric import put_over_ssh
22
+ from fractal_server.ssh._fabric import run_command_over_ssh
23
+ from fractal_server.syringe import Inject
24
+ from fractal_server.tasks.v2.utils import get_python_interpreter_v2
25
+
26
+ TEMPLATES_DIR = Path(__file__).parent / "templates"
27
+
28
+
29
+ def _parse_script_5_stdout(stdout: str) -> dict[str, str]:
30
+ searches = [
31
+ ("Python interpreter:", "python_bin"),
32
+ ("Package name:", "package_name"),
33
+ ("Package version:", "package_version"),
34
+ ("Package parent folder:", "package_root_parent_remote"),
35
+ ("Manifest absolute path:", "manifest_path_remote"),
36
+ ]
37
+ stdout_lines = stdout.splitlines()
38
+ attributes = dict()
39
+ for search, attribute_name in searches:
40
+ matching_lines = [_line for _line in stdout_lines if search in _line]
41
+ if len(matching_lines) == 0:
42
+ raise ValueError(f"String '{search}' not found in stdout.")
43
+ elif len(matching_lines) > 1:
44
+ raise ValueError(
45
+ f"String '{search}' found too many times "
46
+ f"({len(matching_lines)})."
47
+ )
48
+ else:
49
+ actual_line = matching_lines[0]
50
+ attribute_value = actual_line.split(search)[-1].strip(" ")
51
+ attributes[attribute_name] = attribute_value
52
+ return attributes
53
+
54
+
55
+ def _customize_and_run_template(
56
+ script_filename: str,
57
+ templates_folder: Path,
58
+ replacements: list[tuple[str, str]],
59
+ tmpdir: str,
60
+ logger_name: str,
61
+ fractal_ssh: FractalSSH,
62
+ ) -> str:
63
+ """
64
+ Customize one of the template bash scripts, transfer it to the remote host
65
+ via SFTP and then run it via SSH.
66
+
67
+ Args:
68
+ script_filename:
69
+ templates_folder:
70
+ replacements:
71
+ tmpdir:
72
+ logger_name:
73
+ fractal_ssh:
74
+ """
75
+ logger = get_logger(logger_name)
76
+ logger.debug(f"_customize_and_run_template {script_filename} - START")
77
+ settings = Inject(get_settings)
78
+
79
+ # Read template
80
+ template_path = templates_folder / script_filename
81
+ with template_path.open("r") as f:
82
+ script_contents = f.read()
83
+ # Customize template
84
+ for old_new in replacements:
85
+ script_contents = script_contents.replace(old_new[0], old_new[1])
86
+ # Write script locally
87
+ script_path_local = (Path(tmpdir) / script_filename).as_posix()
88
+ with open(script_path_local, "w") as f:
89
+ f.write(script_contents)
90
+
91
+ # Transfer script to remote host
92
+ script_path_remote = os.path.join(
93
+ settings.FRACTAL_SLURM_SSH_WORKING_BASE_DIR,
94
+ f"script_{abs(hash(tmpdir))}{script_filename}",
95
+ )
96
+ logger.debug(f"Now transfer {script_path_local=} over SSH.")
97
+ put_over_ssh(
98
+ local=script_path_local,
99
+ remote=script_path_remote,
100
+ fractal_ssh=fractal_ssh,
101
+ logger_name=logger_name,
102
+ )
103
+
104
+ # Execute script remotely
105
+ cmd = f"bash {script_path_remote}"
106
+ logger.debug(f"Now run '{cmd}' over SSH.")
107
+ stdout = run_command_over_ssh(cmd=cmd, fractal_ssh=fractal_ssh)
108
+ logger.debug(f"Standard output of '{cmd}':\n{stdout}")
109
+
110
+ logger.debug(f"_customize_and_run_template {script_filename} - END")
111
+ return stdout
112
+
113
+
114
+ def background_collect_pip_ssh(
115
+ state_id: int,
116
+ task_pkg: _TaskCollectPip,
117
+ fractal_ssh: FractalSSH,
118
+ ) -> None:
119
+ """
120
+ Collect a task package over SSH
121
+
122
+ This function is run as a background task, therefore exceptions must be
123
+ handled.
124
+
125
+ NOTE: by making this function sync, it will run within a thread - due to
126
+ starlette/fastapi handling of background tasks (see
127
+ https://github.com/encode/starlette/blob/master/starlette/background.py).
128
+ """
129
+ # Work within a temporary folder, where also logs will be placed
130
+ with TemporaryDirectory() as tmpdir:
131
+ LOGGER_NAME = "task_collection_ssh"
132
+ log_file_path = Path(tmpdir) / "log"
133
+ logger = set_logger(
134
+ logger_name=LOGGER_NAME,
135
+ log_file_path=log_file_path,
136
+ )
137
+
138
+ logger.debug("START")
139
+ for key, value in task_pkg.dict(exclude={"package_manifest"}).items():
140
+ logger.debug(f"task_pkg.{key}: {value}")
141
+
142
+ # Open a DB session soon, since it is needed for updating `state`
143
+ with next(get_sync_db()) as db:
144
+ try:
145
+ # Prepare replacements for task-collection scripts
146
+ settings = Inject(get_settings)
147
+ python_bin = get_python_interpreter_v2(
148
+ python_version=task_pkg.python_version
149
+ )
150
+ package_version = (
151
+ ""
152
+ if task_pkg.package_version is None
153
+ else task_pkg.package_version
154
+ )
155
+
156
+ install_string = task_pkg.package
157
+ if task_pkg.package_extras is not None:
158
+ install_string = (
159
+ f"{install_string}[{task_pkg.package_extras}]"
160
+ )
161
+ if (
162
+ task_pkg.package_version is not None
163
+ and not task_pkg.is_local_package
164
+ ):
165
+ install_string = (
166
+ f"{install_string}=={task_pkg.package_version}"
167
+ )
168
+ package_env_dir = (
169
+ Path(settings.FRACTAL_SLURM_SSH_WORKING_BASE_DIR)
170
+ / ".fractal"
171
+ / f"{task_pkg.package_name}{package_version}"
172
+ ).as_posix()
173
+
174
+ replacements = [
175
+ ("__PACKAGE_NAME__", task_pkg.package_name),
176
+ ("__PACKAGE_ENV_DIR__", package_env_dir),
177
+ ("__PACKAGE__", task_pkg.package),
178
+ ("__PYTHON__", python_bin),
179
+ ("__INSTALL_STRING__", install_string),
180
+ ]
181
+
182
+ common_args = dict(
183
+ templates_folder=TEMPLATES_DIR,
184
+ replacements=replacements,
185
+ tmpdir=tmpdir,
186
+ logger_name=LOGGER_NAME,
187
+ fractal_ssh=fractal_ssh,
188
+ )
189
+
190
+ fractal_ssh.check_connection()
191
+
192
+ logger.debug("installing - START")
193
+ _set_collection_state_data_status(
194
+ state_id=state_id,
195
+ new_status=CollectionStatusV2.INSTALLING,
196
+ logger_name=LOGGER_NAME,
197
+ db=db,
198
+ )
199
+ # Avoid keeping the db session open as we start some possibly
200
+ # long operations that do not use the db
201
+ db.close()
202
+
203
+ stdout = _customize_and_run_template(
204
+ script_filename="_1_create_venv.sh",
205
+ **common_args,
206
+ )
207
+ stdout = _customize_and_run_template(
208
+ script_filename="_2_upgrade_pip.sh",
209
+ **common_args,
210
+ )
211
+ stdout = _customize_and_run_template(
212
+ script_filename="_3_pip_install.sh",
213
+ **common_args,
214
+ )
215
+ stdout_pip_freeze = _customize_and_run_template(
216
+ script_filename="_4_pip_freeze.sh",
217
+ **common_args,
218
+ )
219
+ logger.debug("installing - END")
220
+
221
+ logger.debug("collecting - START")
222
+ _set_collection_state_data_status(
223
+ state_id=state_id,
224
+ new_status=CollectionStatusV2.COLLECTING,
225
+ logger_name=LOGGER_NAME,
226
+ db=db,
227
+ )
228
+ # Avoid keeping the db session open as we start some possibly
229
+ # long operations that do not use the db
230
+ db.close()
231
+
232
+ stdout = _customize_and_run_template(
233
+ script_filename="_5_pip_show.sh",
234
+ **common_args,
235
+ )
236
+
237
+ pkg_attrs = _parse_script_5_stdout(stdout)
238
+ for key, value in pkg_attrs.items():
239
+ logger.debug(
240
+ f"collecting - parsed from pip-show: {key}={value}"
241
+ )
242
+ # Check package_name match
243
+ # FIXME SSH: Does this work for non-canonical `package_name`?
244
+ package_name_pip_show = pkg_attrs.get("package_name")
245
+ package_name_task_pkg = task_pkg.package_name
246
+ if package_name_pip_show != package_name_task_pkg:
247
+ error_msg = (
248
+ f"`package_name` mismatch: "
249
+ f"{package_name_task_pkg=} but "
250
+ f"{package_name_pip_show=}"
251
+ )
252
+ logger.error(error_msg)
253
+ raise ValueError(error_msg)
254
+ # Extract/drop parsed attributes
255
+ package_name = pkg_attrs.pop("package_name")
256
+ python_bin = pkg_attrs.pop("python_bin")
257
+ package_root_parent_remote = pkg_attrs.pop(
258
+ "package_root_parent_remote"
259
+ )
260
+ manifest_path_remote = pkg_attrs.pop("manifest_path_remote")
261
+
262
+ # FIXME SSH: Use more robust logic to determine `package_root`,
263
+ # e.g. as in the custom task-collection endpoint (where we use
264
+ # `importlib.util.find_spec`)
265
+ package_name_underscore = package_name.replace("-", "_")
266
+ package_root_remote = (
267
+ Path(package_root_parent_remote) / package_name_underscore
268
+ ).as_posix()
269
+
270
+ # Read and validate remote manifest file
271
+ with fractal_ssh.sftp().open(manifest_path_remote, "r") as f:
272
+ manifest = json.load(f)
273
+ logger.info(f"collecting - loaded {manifest_path_remote=}")
274
+ ManifestV2(**manifest)
275
+ logger.info("collecting - manifest is a valid ManifestV2")
276
+
277
+ # Create new _TaskCollectPip object
278
+ new_pkg = _TaskCollectPip(
279
+ **task_pkg.dict(
280
+ exclude={"package_version", "package_name"},
281
+ exclude_unset=True,
282
+ exclude_none=True,
283
+ ),
284
+ package_manifest=manifest,
285
+ **pkg_attrs,
286
+ )
287
+
288
+ task_list = _prepare_tasks_metadata(
289
+ package_manifest=new_pkg.package_manifest,
290
+ package_version=new_pkg.package_version,
291
+ package_source=new_pkg.package_source,
292
+ package_root=Path(package_root_remote),
293
+ python_bin=Path(python_bin),
294
+ )
295
+ _insert_tasks(task_list=task_list, db=db)
296
+ logger.debug("collecting - END")
297
+
298
+ # Finalize (write metadata to DB)
299
+ logger.debug("finalising - START")
300
+ collection_state = db.get(CollectionStateV2, state_id)
301
+ collection_state.data["log"] = log_file_path.open("r").read()
302
+ collection_state.data["freeze"] = stdout_pip_freeze
303
+ collection_state.data["status"] = CollectionStatusV2.OK
304
+ flag_modified(collection_state, "data")
305
+ db.commit()
306
+ logger.debug("finalising - END")
307
+ logger.debug("END")
308
+
309
+ except Exception as e:
310
+ _handle_failure(
311
+ state_id=state_id,
312
+ log_file_path=log_file_path,
313
+ logger_name=LOGGER_NAME,
314
+ exception=e,
315
+ db=db,
316
+ )
317
+ return
@@ -0,0 +1,136 @@
1
+ import json
2
+ from pathlib import Path
3
+ from typing import Literal
4
+ from typing import Optional
5
+ from typing import Union
6
+ from zipfile import ZipFile
7
+
8
+ from ._TaskCollectPip import _TaskCollectPip
9
+ from .utils import _parse_wheel_filename
10
+ from .utils import get_python_interpreter_v2
11
+ from fractal_server.app.schemas.v2 import ManifestV2
12
+ from fractal_server.config import get_settings
13
+ from fractal_server.logger import get_logger
14
+ from fractal_server.syringe import Inject
15
+ from fractal_server.utils import execute_command
16
+
17
+
18
+ FRACTAL_PUBLIC_TASK_SUBDIR = ".fractal"
19
+
20
+
21
+ async def download_package(
22
+ *,
23
+ task_pkg: _TaskCollectPip,
24
+ dest: Union[str, Path],
25
+ ) -> Path:
26
+ """
27
+ Download package to destination and return wheel-file path.
28
+ """
29
+ interpreter = get_python_interpreter_v2(
30
+ python_version=task_pkg.python_version
31
+ )
32
+ pip = f"{interpreter} -m pip"
33
+ if task_pkg.package_version is None:
34
+ package_and_version = f"{task_pkg.package_name}"
35
+ else:
36
+ package_and_version = (
37
+ f"{task_pkg.package_name}=={task_pkg.package_version}"
38
+ )
39
+ cmd = f"{pip} download --no-deps {package_and_version} -d {dest}"
40
+ stdout = await execute_command(command=cmd, cwd=Path("."))
41
+ pkg_file = next(
42
+ line.split()[-1] for line in stdout.split("\n") if "Saved" in line
43
+ )
44
+ return Path(pkg_file)
45
+
46
+
47
+ def _load_manifest_from_wheel(
48
+ path: Path, wheel: ZipFile, logger_name: Optional[str] = None
49
+ ) -> ManifestV2:
50
+ logger = get_logger(logger_name)
51
+ namelist = wheel.namelist()
52
+ try:
53
+ manifest = next(
54
+ name for name in namelist if "__FRACTAL_MANIFEST__.json" in name
55
+ )
56
+ except StopIteration:
57
+ msg = f"{path.as_posix()} does not include __FRACTAL_MANIFEST__.json"
58
+ logger.error(msg)
59
+ raise ValueError(msg)
60
+ with wheel.open(manifest) as manifest_fd:
61
+ manifest_dict = json.load(manifest_fd)
62
+ manifest_version = str(manifest_dict["manifest_version"])
63
+ if manifest_version == "2":
64
+ pkg_manifest = ManifestV2(**manifest_dict)
65
+ return pkg_manifest
66
+ else:
67
+ msg = f"Manifest version {manifest_version=} not supported"
68
+ logger.error(msg)
69
+ raise ValueError(msg)
70
+
71
+
72
+ def inspect_package(
73
+ path: Path, logger_name: Optional[str] = None
74
+ ) -> dict[Literal["pkg_version", "pkg_manifest"], str]:
75
+ """
76
+ Inspect task package to extract version and manifest
77
+
78
+ Note that this only works with wheel files, which have a well-defined
79
+ dist-info section. If we need to generalize to to tar.gz archives, we would
80
+ need to go and look for `PKG-INFO`.
81
+
82
+ Args:
83
+ path: Path of the package wheel file.
84
+ logger_name:
85
+
86
+ Returns:
87
+ A dictionary with keys `pkg_version` and `pkg_manifest`.
88
+ """
89
+
90
+ logger = get_logger(logger_name)
91
+
92
+ if not path.as_posix().endswith(".whl"):
93
+ raise ValueError(
94
+ "Only wheel packages are supported in Fractal "
95
+ f"(given {path.name})."
96
+ )
97
+
98
+ # Extract package name and version from wheel filename
99
+ _info = _parse_wheel_filename(wheel_filename=path.name)
100
+ pkg_version = _info["version"]
101
+
102
+ # Read and validate task manifest
103
+ logger.debug(f"Now reading manifest for {path.as_posix()}")
104
+ with ZipFile(path) as wheel:
105
+ pkg_manifest = _load_manifest_from_wheel(
106
+ path, wheel, logger_name=logger_name
107
+ )
108
+ logger.debug("Manifest read correctly.")
109
+
110
+ info = dict(
111
+ pkg_version=pkg_version,
112
+ pkg_manifest=pkg_manifest,
113
+ )
114
+ return info
115
+
116
+
117
+ def create_package_dir_pip(
118
+ *,
119
+ task_pkg: _TaskCollectPip,
120
+ create: bool = True,
121
+ ) -> Path:
122
+ """
123
+ Create venv folder for a task package and return corresponding Path object
124
+ """
125
+ settings = Inject(get_settings)
126
+ user = FRACTAL_PUBLIC_TASK_SUBDIR
127
+ if task_pkg.package_version is None:
128
+ raise ValueError(
129
+ f"Cannot create venv folder for package `{task_pkg.package}` "
130
+ "with `version=None`."
131
+ )
132
+ package_dir = f"{task_pkg.package_name}{task_pkg.package_version}"
133
+ venv_path = settings.FRACTAL_TASKS_DIR / user / package_dir
134
+ if create:
135
+ venv_path.mkdir(exist_ok=False, parents=True)
136
+ return venv_path
@@ -0,0 +1,46 @@
1
+ set -e
2
+
3
+ write_log(){
4
+ TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
5
+ echo "[collect-task, $TIMESTAMP] $1"
6
+ }
7
+
8
+
9
+ # Variables to be filled within fractal-server
10
+ PACKAGE_ENV_DIR=__PACKAGE_ENV_DIR__
11
+ PYTHON=__PYTHON__
12
+
13
+ TIME_START=$(date +%s)
14
+
15
+
16
+ # Create main folder
17
+ if [ -d "$PACKAGE_ENV_DIR" ]; then
18
+ write_log "ERROR: Folder $PACKAGE_ENV_DIR already exists. Exit."
19
+ exit 1
20
+ fi
21
+ write_log "START mkdir -p $PACKAGE_ENV_DIR"
22
+ mkdir -p $PACKAGE_ENV_DIR
23
+ write_log "END mkdir -p $PACKAGE_ENV_DIR"
24
+ echo
25
+
26
+
27
+ # Create venv
28
+ write_log "START create venv in ${PACKAGE_ENV_DIR}"
29
+ "$PYTHON" -m venv "$PACKAGE_ENV_DIR" --copies
30
+ write_log "END create venv in ${PACKAGE_ENV_DIR}"
31
+ echo
32
+ VENVPYTHON=${PACKAGE_ENV_DIR}/bin/python
33
+ if [ -f "$VENVPYTHON" ]; then
34
+ write_log "OK: $VENVPYTHON exists."
35
+ echo
36
+ else
37
+ write_log "ERROR: $VENVPYTHON not found"
38
+ exit 2
39
+ fi
40
+
41
+ # End
42
+ TIME_END=$(date +%s)
43
+ write_log "All good up to here."
44
+ write_log "Elapsed: $((TIME_END - TIME_START)) seconds"
45
+ write_log "Exit."
46
+ echo
@@ -0,0 +1,30 @@
1
+ set -e
2
+
3
+ write_log(){
4
+ TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
5
+ echo "[collect-task, $TIMESTAMP] $1"
6
+ }
7
+
8
+ # Variables to be filled within fractal-server
9
+ PACKAGE_ENV_DIR=__PACKAGE_ENV_DIR__
10
+ PACKAGE_NAME=__PACKAGE_NAME__
11
+ PACKAGE=__PACKAGE__
12
+ PYTHON=__PYTHON__
13
+ INSTALL_STRING=__INSTALL_STRING__
14
+
15
+ TIME_START=$(date +%s)
16
+
17
+ VENVPYTHON=${PACKAGE_ENV_DIR}/bin/python
18
+
19
+ # Upgrade pip
20
+ write_log "START upgrade pip"
21
+ "$VENVPYTHON" -m pip install pip --upgrade
22
+ write_log "END upgrade pip"
23
+ echo
24
+
25
+ # End
26
+ TIME_END=$(date +%s)
27
+ write_log "All good up to here."
28
+ write_log "Elapsed: $((TIME_END - TIME_START)) seconds"
29
+ write_log "Exit."
30
+ echo
@@ -0,0 +1,32 @@
1
+ set -e
2
+
3
+ write_log(){
4
+ TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
5
+ echo "[collect-task, $TIMESTAMP] $1"
6
+ }
7
+
8
+
9
+ # Variables to be filled within fractal-server
10
+ PACKAGE_ENV_DIR=__PACKAGE_ENV_DIR__
11
+ PACKAGE_NAME=__PACKAGE_NAME__
12
+ PACKAGE=__PACKAGE__
13
+ PYTHON=__PYTHON__
14
+ INSTALL_STRING=__INSTALL_STRING__
15
+
16
+
17
+ TIME_START=$(date +%s)
18
+
19
+ VENVPYTHON=${PACKAGE_ENV_DIR}/bin/python
20
+
21
+ # Install package
22
+ write_log "START install ${INSTALL_STRING}"
23
+ "$VENVPYTHON" -m pip install "$INSTALL_STRING"
24
+ write_log "END install ${INSTALL_STRING}"
25
+ echo
26
+
27
+ # End
28
+ TIME_END=$(date +%s)
29
+ write_log "All good up to here."
30
+ write_log "Elapsed: $((TIME_END - TIME_START)) seconds"
31
+ write_log "Exit."
32
+ echo
@@ -0,0 +1,21 @@
1
+ set -e
2
+
3
+ write_log(){
4
+ TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
5
+ echo "[collect-task, $TIMESTAMP] $1"
6
+ }
7
+
8
+
9
+
10
+ # Variables to be filled within fractal-server
11
+ PACKAGE_ENV_DIR=__PACKAGE_ENV_DIR__
12
+ PACKAGE_NAME=__PACKAGE_NAME__
13
+ PACKAGE=__PACKAGE__
14
+ PYTHON=__PYTHON__
15
+ INSTALL_STRING=__INSTALL_STRING__
16
+
17
+
18
+
19
+ VENVPYTHON=${PACKAGE_ENV_DIR}/bin/python
20
+
21
+ "$VENVPYTHON" -m pip freeze
@@ -0,0 +1,59 @@
1
+ set -e
2
+
3
+ write_log(){
4
+ TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
5
+ echo "[collect-task, $TIMESTAMP] $1"
6
+ }
7
+
8
+
9
+ # Variables to be filled within fractal-server
10
+ PACKAGE_ENV_DIR=__PACKAGE_ENV_DIR__
11
+ PACKAGE_NAME=__PACKAGE_NAME__
12
+ PACKAGE=__PACKAGE__
13
+ PYTHON=__PYTHON__
14
+ INSTALL_STRING=__INSTALL_STRING__
15
+
16
+
17
+ TIME_START=$(date +%s)
18
+
19
+ VENVPYTHON=${PACKAGE_ENV_DIR}/bin/python
20
+ write_log "Python interpreter: $VENVPYTHON"
21
+ echo
22
+
23
+ # FIXME: only run pip-show once!
24
+
25
+ # Extract information about paths
26
+ # WARNING: this block will fail for paths which inlcude whitespace characters
27
+ write_log "START pip show"
28
+ $VENVPYTHON -m pip show ${PACKAGE_NAME}
29
+ write_log "END pip show"
30
+ echo
31
+ PACKAGE_NAME=$($VENVPYTHON -m pip show ${PACKAGE_NAME} | grep "Name:" | cut -d ":" -f 2 | tr -d "[:space:]")
32
+ write_log "Package name: $PACKAGE_NAME"
33
+ echo
34
+ PACKAGE_VERSION=$($VENVPYTHON -m pip show ${PACKAGE_NAME} | grep "Version:" | cut -d ":" -f 2 | tr -d "[:space:]")
35
+ write_log "Package version: $PACKAGE_VERSION"
36
+ echo
37
+ PACKAGE_PARENT_FOLDER=$($VENVPYTHON -m pip show ${PACKAGE_NAME} | grep "Location:" | cut -d ":" -f 2 | tr -d "[:space:]")
38
+ write_log "Package parent folder: $PACKAGE_PARENT_FOLDER"
39
+ echo
40
+ MANIFEST_RELATIVE_PATH=$($VENVPYTHON -m pip show ${PACKAGE_NAME} --files | grep "__FRACTAL_MANIFEST__.json" | tr -d "[:space:]")
41
+ write_log "Manifest relative path: $MANIFEST_RELATIVE_PATH"
42
+ echo
43
+ MANIFEST_ABSOLUTE_PATH="${PACKAGE_PARENT_FOLDER}/${MANIFEST_RELATIVE_PATH}"
44
+ write_log "Manifest absolute path: $MANIFEST_ABSOLUTE_PATH"
45
+ echo
46
+ if [ -f "$MANIFEST_ABSOLUTE_PATH" ]; then
47
+ write_log "OK: manifest path exists"
48
+ echo
49
+ else
50
+ write_log "ERROR: manifest path not found at $MANIFEST_ABSOLUTE_PATH"
51
+ exit 3
52
+ fi
53
+
54
+ # End
55
+ TIME_END=$(date +%s)
56
+ write_log "All good up to here."
57
+ write_log "Elapsed: $((TIME_END - TIME_START)) seconds"
58
+ write_log "Exit."
59
+ echo
@@ -0,0 +1,54 @@
1
+ from typing import Literal
2
+
3
+ from fractal_server.config import get_settings
4
+ from fractal_server.syringe import Inject
5
+
6
+
7
+ def get_python_interpreter_v2(
8
+ python_version: Literal["3.9", "3.10", "3.11", "3.12"]
9
+ ) -> str:
10
+ """
11
+ Return the path to the python interpreter
12
+
13
+ Args:
14
+ version: Python version
15
+
16
+ Raises:
17
+ ValueError: If the python version requested is not available on the
18
+ host.
19
+
20
+ Returns:
21
+ interpreter: string representing the python executable or its path
22
+ """
23
+
24
+ if python_version not in ["3.9", "3.10", "3.11", "3.12"]:
25
+ raise ValueError(f"Invalid {python_version=}.")
26
+
27
+ settings = Inject(get_settings)
28
+ version_underscore = python_version.replace(".", "_")
29
+ key = f"FRACTAL_TASKS_PYTHON_{version_underscore}"
30
+ value = getattr(settings, key)
31
+ if value is None:
32
+ raise ValueError(f"Requested {python_version=}, but {key}={value}.")
33
+ return value
34
+
35
+
36
+ def _parse_wheel_filename(wheel_filename: str) -> dict[str, str]:
37
+ """
38
+ Extract distribution and version from a wheel filename.
39
+
40
+ The structure of a wheel filename is fixed, and it must start with
41
+ `{distribution}-{version}` (see
42
+ https://packaging.python.org/en/latest/specifications/binary-distribution-format
43
+ ).
44
+
45
+ Note that we transform exceptions in `ValueError`s, since this function is
46
+ also used within Pydantic validators.
47
+ """
48
+ try:
49
+ parts = wheel_filename.split("-")
50
+ return dict(distribution=parts[0], version=parts[1])
51
+ except Exception as e:
52
+ raise ValueError(
53
+ f"Invalid {wheel_filename=}. Original error: {str(e)}."
54
+ )