fractal-server 2.9.0a2__py3-none-any.whl → 2.9.0a4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. fractal_server/__init__.py +1 -1
  2. fractal_server/app/routes/admin/v2/__init__.py +4 -0
  3. fractal_server/app/routes/admin/v2/task_group_lifecycle.py +260 -0
  4. fractal_server/app/routes/api/v2/task_collection.py +3 -10
  5. fractal_server/app/routes/api/v2/task_group_lifecycle.py +50 -6
  6. fractal_server/app/schemas/_validators.py +0 -14
  7. fractal_server/app/schemas/v1/applyworkflow.py +0 -8
  8. fractal_server/app/schemas/v1/dataset.py +0 -5
  9. fractal_server/app/schemas/v1/project.py +0 -5
  10. fractal_server/app/schemas/v1/state.py +0 -5
  11. fractal_server/app/schemas/v1/workflow.py +0 -5
  12. fractal_server/app/schemas/v2/dataset.py +0 -6
  13. fractal_server/app/schemas/v2/job.py +0 -8
  14. fractal_server/app/schemas/v2/project.py +0 -5
  15. fractal_server/app/schemas/v2/workflow.py +0 -5
  16. fractal_server/ssh/_fabric.py +4 -2
  17. fractal_server/tasks/v2/local/{utils_local.py → _utils.py} +25 -0
  18. fractal_server/tasks/v2/local/collect.py +2 -2
  19. fractal_server/tasks/v2/local/deactivate.py +1 -1
  20. fractal_server/tasks/v2/local/reactivate.py +1 -1
  21. fractal_server/tasks/v2/ssh/__init__.py +3 -0
  22. fractal_server/tasks/v2/ssh/_utils.py +87 -0
  23. fractal_server/tasks/v2/ssh/collect.py +4 -79
  24. fractal_server/tasks/v2/ssh/deactivate.py +243 -2
  25. fractal_server/tasks/v2/ssh/reactivate.py +199 -2
  26. fractal_server/tasks/v2/utils_background.py +0 -24
  27. fractal_server/zip_tools.py +21 -4
  28. {fractal_server-2.9.0a2.dist-info → fractal_server-2.9.0a4.dist-info}/METADATA +1 -1
  29. {fractal_server-2.9.0a2.dist-info → fractal_server-2.9.0a4.dist-info}/RECORD +32 -30
  30. {fractal_server-2.9.0a2.dist-info → fractal_server-2.9.0a4.dist-info}/LICENSE +0 -0
  31. {fractal_server-2.9.0a2.dist-info → fractal_server-2.9.0a4.dist-info}/WHEEL +0 -0
  32. {fractal_server-2.9.0a2.dist-info → fractal_server-2.9.0a4.dist-info}/entry_points.txt +0 -0
@@ -7,7 +7,7 @@ from tempfile import TemporaryDirectory
7
7
  from ..utils_background import add_commit_refresh
8
8
  from ..utils_background import fail_and_cleanup
9
9
  from ..utils_templates import get_collection_replacements
10
- from .utils_local import _customize_and_run_template
10
+ from ._utils import _customize_and_run_template
11
11
  from fractal_server.app.db import get_sync_db
12
12
  from fractal_server.app.models.v2 import TaskGroupActivityV2
13
13
  from fractal_server.app.models.v2 import TaskGroupV2
@@ -7,7 +7,7 @@ from tempfile import TemporaryDirectory
7
7
  from ..utils_background import add_commit_refresh
8
8
  from ..utils_background import fail_and_cleanup
9
9
  from ..utils_templates import get_collection_replacements
10
- from .utils_local import _customize_and_run_template
10
+ from ._utils import _customize_and_run_template
11
11
  from fractal_server.app.db import get_sync_db
12
12
  from fractal_server.app.models.v2 import TaskGroupActivityV2
13
13
  from fractal_server.app.models.v2 import TaskGroupV2
@@ -0,0 +1,3 @@
1
+ from .collect import collect_ssh # noqa
2
+ from .deactivate import deactivate_ssh # noqa
3
+ from .reactivate import reactivate_ssh # noqa
@@ -0,0 +1,87 @@
1
+ import os
2
+ from pathlib import Path
3
+
4
+ from fractal_server.app.models.v2 import TaskGroupV2
5
+ from fractal_server.logger import get_logger
6
+ from fractal_server.ssh._fabric import FractalSSH
7
+ from fractal_server.tasks.v2.utils_templates import customize_template
8
+
9
+
10
+ def _customize_and_run_template(
11
+ *,
12
+ template_filename: str,
13
+ replacements: list[tuple[str, str]],
14
+ script_dir_local: str,
15
+ prefix: str,
16
+ fractal_ssh: FractalSSH,
17
+ script_dir_remote: str,
18
+ logger_name: str,
19
+ ) -> str:
20
+ """
21
+ Customize one of the template bash scripts, transfer it to the remote host
22
+ via SFTP and then run it via SSH.
23
+
24
+ Args:
25
+ template_filename: Filename of the template file (ends with ".sh").
26
+ replacements: Dictionary of replacements.
27
+ script_dir: Local folder where the script will be placed.
28
+ prefix: Prefix for the script filename.
29
+ fractal_ssh: FractalSSH object
30
+ script_dir_remote: Remote scripts directory
31
+ """
32
+ logger = get_logger(logger_name=logger_name)
33
+ logger.debug(f"_customize_and_run_template {template_filename} - START")
34
+ # Prepare name and path of script
35
+ if not template_filename.endswith(".sh"):
36
+ raise ValueError(
37
+ f"Invalid {template_filename=} (it must end with '.sh')."
38
+ )
39
+ script_filename = f"{prefix}_{template_filename}"
40
+ script_path_local = (Path(script_dir_local) / script_filename).as_posix()
41
+
42
+ customize_template(
43
+ template_name=template_filename,
44
+ replacements=replacements,
45
+ script_path=script_path_local,
46
+ )
47
+
48
+ # Transfer script to remote host
49
+ script_path_remote = os.path.join(
50
+ script_dir_remote,
51
+ script_filename,
52
+ )
53
+ logger.debug(f"Now transfer {script_path_local=} over SSH.")
54
+ fractal_ssh.send_file(
55
+ local=script_path_local,
56
+ remote=script_path_remote,
57
+ )
58
+
59
+ # Execute script remotely
60
+ cmd = f"bash {script_path_remote}"
61
+ logger.debug(f"Now run '{cmd}' over SSH.")
62
+ stdout = fractal_ssh.run_command(cmd=cmd)
63
+
64
+ logger.debug(f"_customize_and_run_template {template_filename} - END")
65
+ return stdout
66
+
67
+
68
+ def _copy_wheel_file_ssh(
69
+ *, task_group: TaskGroupV2, fractal_ssh: FractalSSH, logger_name: str
70
+ ) -> str:
71
+ """
72
+ Handle the situation where `task_group.wheel_path` is not part of
73
+ `task_group.path`, by copying `wheel_path` into `path`.
74
+
75
+ Returns:
76
+ The new `wheel_path`.
77
+ """
78
+ logger = get_logger(logger_name=logger_name)
79
+ source = task_group.wheel_path
80
+ dest = (
81
+ Path(task_group.path) / Path(task_group.wheel_path).name
82
+ ).as_posix()
83
+ cmd = f"cp {source} {dest}"
84
+ logger.debug(f"[_copy_wheel_file] START {source=} {dest=}")
85
+ fractal_ssh.run_command(cmd=cmd)
86
+ logger.debug(f"[_copy_wheel_file] END {source=} {dest=}")
87
+ return dest
@@ -1,5 +1,4 @@
1
1
  import logging
2
- import os
3
2
  import time
4
3
  from pathlib import Path
5
4
  from tempfile import TemporaryDirectory
@@ -13,16 +12,16 @@ from fractal_server.app.models.v2 import TaskGroupV2
13
12
  from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
14
13
  from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
15
14
  from fractal_server.app.schemas.v2.manifest import ManifestV2
16
- from fractal_server.logger import get_logger
17
15
  from fractal_server.logger import set_logger
18
16
  from fractal_server.ssh._fabric import FractalSSH
17
+ from fractal_server.tasks.v2.ssh._utils import _copy_wheel_file_ssh
18
+ from fractal_server.tasks.v2.ssh._utils import _customize_and_run_template
19
19
  from fractal_server.tasks.v2.utils_background import add_commit_refresh
20
20
  from fractal_server.tasks.v2.utils_background import get_current_log
21
21
  from fractal_server.tasks.v2.utils_package_names import compare_package_names
22
22
  from fractal_server.tasks.v2.utils_python_interpreter import (
23
23
  get_python_interpreter_v2,
24
24
  )
25
- from fractal_server.tasks.v2.utils_templates import customize_template
26
25
  from fractal_server.tasks.v2.utils_templates import get_collection_replacements
27
26
  from fractal_server.tasks.v2.utils_templates import (
28
27
  parse_script_pip_show_stdout,
@@ -33,81 +32,6 @@ from fractal_server.utils import get_timestamp
33
32
  LOGGER_NAME = __name__
34
33
 
35
34
 
36
- def _customize_and_run_template(
37
- *,
38
- template_filename: str,
39
- replacements: list[tuple[str, str]],
40
- script_dir_local: str,
41
- prefix: str,
42
- fractal_ssh: FractalSSH,
43
- script_dir_remote: str,
44
- logger_name: str,
45
- ) -> str:
46
- """
47
- Customize one of the template bash scripts, transfer it to the remote host
48
- via SFTP and then run it via SSH.
49
-
50
- Args:
51
- template_filename: Filename of the template file (ends with ".sh").
52
- replacements: Dictionary of replacements.
53
- script_dir: Local folder where the script will be placed.
54
- prefix: Prefix for the script filename.
55
- fractal_ssh: FractalSSH object
56
- script_dir_remote: Remote scripts directory
57
- """
58
- logger = get_logger(logger_name=logger_name)
59
- logger.debug(f"_customize_and_run_template {template_filename} - START")
60
-
61
- # Prepare name and path of script
62
- if not template_filename.endswith(".sh"):
63
- raise ValueError(
64
- f"Invalid {template_filename=} (it must end with '.sh')."
65
- )
66
- script_filename = f"{prefix}_{template_filename}"
67
- script_path_local = Path(script_dir_local) / script_filename
68
-
69
- customize_template(
70
- template_name=template_filename,
71
- replacements=replacements,
72
- script_path=script_path_local,
73
- )
74
-
75
- # Transfer script to remote host
76
- script_path_remote = os.path.join(
77
- script_dir_remote,
78
- script_filename,
79
- )
80
- logger.debug(f"Now transfer {script_path_local=} over SSH.")
81
- fractal_ssh.send_file(
82
- local=script_path_local,
83
- remote=script_path_remote,
84
- )
85
-
86
- # Execute script remotely
87
- cmd = f"bash {script_path_remote}"
88
- logger.debug(f"Now run '{cmd}' over SSH.")
89
- stdout = fractal_ssh.run_command(cmd=cmd)
90
- logger.debug(f"Standard output of '{cmd}':\n{stdout}")
91
-
92
- logger.debug(f"_customize_and_run_template {template_filename} - END")
93
- return stdout
94
-
95
-
96
- def _copy_wheel_file_ssh(
97
- task_group: TaskGroupV2, fractal_ssh: FractalSSH
98
- ) -> str:
99
- logger = get_logger(LOGGER_NAME)
100
- source = task_group.wheel_path
101
- dest = (
102
- Path(task_group.path) / Path(task_group.wheel_path).name
103
- ).as_posix()
104
- cmd = f"cp {source} {dest}"
105
- logger.debug(f"[_copy_wheel_file] START {source=} {dest=}")
106
- fractal_ssh.run_command(cmd=cmd)
107
- logger.debug(f"[_copy_wheel_file] END {source=} {dest=}")
108
- return dest
109
-
110
-
111
35
  def collect_ssh(
112
36
  *,
113
37
  task_group_id: int,
@@ -214,7 +138,7 @@ def collect_ssh(
214
138
  script_dir_remote=script_dir_remote,
215
139
  prefix=(
216
140
  f"{int(time.time())}_"
217
- f"{TaskGroupActivityActionV2.COLLECT}_"
141
+ f"{TaskGroupActivityActionV2.COLLECT}"
218
142
  ),
219
143
  fractal_ssh=fractal_ssh,
220
144
  logger_name=LOGGER_NAME,
@@ -232,6 +156,7 @@ def collect_ssh(
232
156
  new_wheel_path = _copy_wheel_file_ssh(
233
157
  task_group=task_group,
234
158
  fractal_ssh=fractal_ssh,
159
+ logger_name=LOGGER_NAME,
235
160
  )
236
161
  task_group.wheel_path = new_wheel_path
237
162
  task_group = add_commit_refresh(obj=task_group, db=db)
@@ -1,2 +1,243 @@
1
- def deactivate_ssh():
2
- pass
1
+ import logging
2
+ import time
3
+ from pathlib import Path
4
+ from tempfile import TemporaryDirectory
5
+
6
+ from ..utils_background import add_commit_refresh
7
+ from ..utils_background import fail_and_cleanup
8
+ from ..utils_templates import get_collection_replacements
9
+ from ._utils import _copy_wheel_file_ssh
10
+ from ._utils import _customize_and_run_template
11
+ from fractal_server.app.db import get_sync_db
12
+ from fractal_server.app.models.v2 import TaskGroupActivityV2
13
+ from fractal_server.app.models.v2 import TaskGroupV2
14
+ from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
15
+ from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
16
+ from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
17
+ from fractal_server.logger import set_logger
18
+ from fractal_server.ssh._fabric import FractalSSH
19
+ from fractal_server.tasks.utils import get_log_path
20
+ from fractal_server.tasks.v2.utils_background import get_current_log
21
+ from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
22
+ from fractal_server.utils import get_timestamp
23
+
24
+ LOGGER_NAME = __name__
25
+
26
+
27
+ def deactivate_ssh(
28
+ *,
29
+ task_group_activity_id: int,
30
+ task_group_id: int,
31
+ fractal_ssh: FractalSSH,
32
+ tasks_base_dir: str,
33
+ ) -> None:
34
+ """
35
+ Deactivate a task group venv.
36
+
37
+ This function is run as a background task, therefore exceptions must be
38
+ handled.
39
+
40
+ Arguments:
41
+ task_group_id:
42
+ task_group_activity_id:
43
+ fractal_ssh:
44
+ tasks_base_dir:
45
+ Only used as a `safe_root` in `remove_dir`, and typically set to
46
+ `user_settings.ssh_tasks_dir`.
47
+ """
48
+
49
+ with TemporaryDirectory() as tmpdir:
50
+ log_file_path = get_log_path(Path(tmpdir))
51
+ logger = set_logger(
52
+ logger_name=LOGGER_NAME,
53
+ log_file_path=log_file_path,
54
+ )
55
+
56
+ with next(get_sync_db()) as db:
57
+
58
+ # Get main objects from db
59
+ activity = db.get(TaskGroupActivityV2, task_group_activity_id)
60
+ task_group = db.get(TaskGroupV2, task_group_id)
61
+ if activity is None or task_group is None:
62
+ # Use `logging` directly
63
+ logging.error(
64
+ "Cannot find database rows with "
65
+ f"{task_group_id=} and {task_group_activity_id=}:\n"
66
+ f"{task_group=}\n{activity=}. Exit."
67
+ )
68
+ return
69
+
70
+ # Log some info
71
+ logger.debug("START")
72
+ for key, value in task_group.model_dump().items():
73
+ logger.debug(f"task_group.{key}: {value}")
74
+
75
+ # Check that SSH connection works
76
+ try:
77
+ fractal_ssh.check_connection()
78
+ except Exception as e:
79
+ logger.error("Cannot establish SSH connection.")
80
+ fail_and_cleanup(
81
+ task_group=task_group,
82
+ task_group_activity=activity,
83
+ logger_name=LOGGER_NAME,
84
+ log_file_path=log_file_path,
85
+ exception=e,
86
+ db=db,
87
+ )
88
+ return
89
+
90
+ # Check that the (local) task_group venv_path does exist
91
+ if not fractal_ssh.remote_exists(task_group.venv_path):
92
+ error_msg = f"{task_group.venv_path} does not exist."
93
+ logger.error(error_msg)
94
+ fail_and_cleanup(
95
+ task_group=task_group,
96
+ task_group_activity=activity,
97
+ logger_name=LOGGER_NAME,
98
+ log_file_path=log_file_path,
99
+ exception=FileNotFoundError(error_msg),
100
+ db=db,
101
+ )
102
+ return
103
+
104
+ try:
105
+
106
+ activity.status = TaskGroupActivityStatusV2.ONGOING
107
+ activity = add_commit_refresh(obj=activity, db=db)
108
+
109
+ if task_group.pip_freeze is None:
110
+ logger.warning(
111
+ "Recreate pip-freeze information, since "
112
+ f"{task_group.pip_freeze=}. NOTE: this should only "
113
+ "happen for task groups created before 2.9.0."
114
+ )
115
+
116
+ # Prepare replacements for templates
117
+ replacements = get_collection_replacements(
118
+ task_group=task_group,
119
+ python_bin="/not/applicable",
120
+ )
121
+
122
+ # Prepare arguments for `_customize_and_run_template`
123
+ script_dir_remote = (
124
+ Path(task_group.path) / SCRIPTS_SUBFOLDER
125
+ ).as_posix()
126
+ common_args = dict(
127
+ replacements=replacements,
128
+ script_dir_local=(
129
+ Path(tmpdir) / SCRIPTS_SUBFOLDER
130
+ ).as_posix(),
131
+ script_dir_remote=script_dir_remote,
132
+ prefix=(
133
+ f"{int(time.time())}_"
134
+ f"{TaskGroupActivityActionV2.DEACTIVATE}"
135
+ ),
136
+ fractal_ssh=fractal_ssh,
137
+ logger_name=LOGGER_NAME,
138
+ )
139
+
140
+ # Run `pip freeze`
141
+ pip_freeze_stdout = _customize_and_run_template(
142
+ template_filename="3_pip_freeze.sh",
143
+ **common_args,
144
+ )
145
+
146
+ # Update pip-freeze data
147
+ logger.info("Add pip freeze stdout to TaskGroupV2 - start")
148
+ activity.log = get_current_log(log_file_path)
149
+ activity = add_commit_refresh(obj=activity, db=db)
150
+ task_group.pip_freeze = pip_freeze_stdout
151
+ task_group = add_commit_refresh(obj=task_group, db=db)
152
+ logger.info("Add pip freeze stdout to TaskGroupV2 - end")
153
+
154
+ # Handle some specific cases for wheel-file case
155
+ if task_group.origin == TaskGroupV2OriginEnum.WHEELFILE:
156
+
157
+ logger.info(
158
+ f"Handle specific cases for {task_group.origin=}."
159
+ )
160
+
161
+ # Blocking situation: `wheel_path` is not set or points
162
+ # to a missing path
163
+ if (
164
+ task_group.wheel_path is None
165
+ or not fractal_ssh.remote_exists(task_group.wheel_path)
166
+ ):
167
+ error_msg = (
168
+ "Invalid wheel path for task group with "
169
+ f"{task_group_id=}. {task_group.wheel_path=} is "
170
+ "unset or does not exist."
171
+ )
172
+ logger.error(error_msg)
173
+ fail_and_cleanup(
174
+ task_group=task_group,
175
+ task_group_activity=activity,
176
+ logger_name=LOGGER_NAME,
177
+ log_file_path=log_file_path,
178
+ exception=FileNotFoundError(error_msg),
179
+ db=db,
180
+ )
181
+ return
182
+
183
+ # Recoverable situation: `wheel_path` was not yet copied
184
+ # over to the correct server-side folder
185
+ wheel_path_parent_dir = Path(task_group.wheel_path).parent
186
+ if wheel_path_parent_dir != Path(task_group.path):
187
+ logger.warning(
188
+ f"{wheel_path_parent_dir.as_posix()} differs from "
189
+ f"{task_group.path}. NOTE: this should only "
190
+ "happen for task groups created before 2.9.0."
191
+ )
192
+
193
+ if task_group.wheel_path not in task_group.pip_freeze:
194
+ raise ValueError(
195
+ f"Cannot find {task_group.wheel_path=} in "
196
+ "pip-freeze data. Exit."
197
+ )
198
+
199
+ logger.info(
200
+ f"Now copy wheel file into {task_group.path}."
201
+ )
202
+ new_wheel_path = _copy_wheel_file_ssh(
203
+ task_group=task_group,
204
+ fractal_ssh=fractal_ssh,
205
+ logger_name=LOGGER_NAME,
206
+ )
207
+ logger.info(f"Copied wheel file to {new_wheel_path}.")
208
+
209
+ task_group.wheel_path = new_wheel_path
210
+ new_pip_freeze = task_group.pip_freeze.replace(
211
+ task_group.wheel_path,
212
+ new_wheel_path,
213
+ )
214
+ task_group.pip_freeze = new_pip_freeze
215
+ task_group = add_commit_refresh(obj=task_group, db=db)
216
+ logger.info(
217
+ "Updated `wheel_path` and `pip_freeze` "
218
+ "task-group attributes."
219
+ )
220
+
221
+ # We now have all required information for reactivating the
222
+ # virtual environment at a later point
223
+ logger.info(f"Now removing {task_group.venv_path}.")
224
+ fractal_ssh.remove_folder(
225
+ folder=task_group.venv_path,
226
+ safe_root=tasks_base_dir,
227
+ )
228
+ logger.info(f"All good, {task_group.venv_path} removed.")
229
+ activity.status = TaskGroupActivityStatusV2.OK
230
+ activity.log = get_current_log(log_file_path)
231
+ activity.timestamp_ended = get_timestamp()
232
+ activity = add_commit_refresh(obj=activity, db=db)
233
+
234
+ except Exception as e:
235
+ fail_and_cleanup(
236
+ task_group=task_group,
237
+ task_group_activity=activity,
238
+ logger_name=LOGGER_NAME,
239
+ log_file_path=log_file_path,
240
+ exception=e,
241
+ db=db,
242
+ )
243
+ return
@@ -1,2 +1,199 @@
1
- def reactivate_ssh():
2
- pass
1
+ import logging
2
+ import time
3
+ from pathlib import Path
4
+ from tempfile import TemporaryDirectory
5
+
6
+ from ..utils_background import add_commit_refresh
7
+ from ..utils_background import fail_and_cleanup
8
+ from ..utils_templates import get_collection_replacements
9
+ from ._utils import _customize_and_run_template
10
+ from fractal_server.app.db import get_sync_db
11
+ from fractal_server.app.models.v2 import TaskGroupActivityV2
12
+ from fractal_server.app.models.v2 import TaskGroupV2
13
+ from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
14
+ from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
15
+ from fractal_server.logger import set_logger
16
+ from fractal_server.ssh._fabric import FractalSSH
17
+ from fractal_server.tasks.utils import get_log_path
18
+ from fractal_server.tasks.v2.utils_background import get_current_log
19
+ from fractal_server.tasks.v2.utils_python_interpreter import (
20
+ get_python_interpreter_v2,
21
+ )
22
+ from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
23
+ from fractal_server.utils import get_timestamp
24
+
25
+ LOGGER_NAME = __name__
26
+
27
+
28
+ def reactivate_ssh(
29
+ *,
30
+ task_group_activity_id: int,
31
+ task_group_id: int,
32
+ fractal_ssh: FractalSSH,
33
+ tasks_base_dir: str,
34
+ ) -> None:
35
+ """
36
+ Reactivate a task group venv.
37
+
38
+ This function is run as a background task, therefore exceptions must be
39
+ handled.
40
+
41
+ Arguments:
42
+ task_group_id:
43
+ task_group_activity_id:
44
+ fractal_ssh:
45
+ tasks_base_dir:
46
+ Only used as a `safe_root` in `remove_dir`, and typically set to
47
+ `user_settings.ssh_tasks_dir`.
48
+ """
49
+
50
+ with TemporaryDirectory() as tmpdir:
51
+ log_file_path = get_log_path(Path(tmpdir))
52
+ logger = set_logger(
53
+ logger_name=LOGGER_NAME,
54
+ log_file_path=log_file_path,
55
+ )
56
+
57
+ with next(get_sync_db()) as db:
58
+
59
+ # Get main objects from db
60
+ activity = db.get(TaskGroupActivityV2, task_group_activity_id)
61
+ task_group = db.get(TaskGroupV2, task_group_id)
62
+ if activity is None or task_group is None:
63
+ # Use `logging` directly
64
+ logging.error(
65
+ "Cannot find database rows with "
66
+ f"{task_group_id=} and {task_group_activity_id=}:\n"
67
+ f"{task_group=}\n{activity=}. Exit."
68
+ )
69
+ return
70
+
71
+ # Log some info
72
+ logger.debug("START")
73
+ for key, value in task_group.model_dump().items():
74
+ logger.debug(f"task_group.{key}: {value}")
75
+
76
+ # Check that SSH connection works
77
+ try:
78
+ fractal_ssh.check_connection()
79
+ except Exception as e:
80
+ logger.error("Cannot establish SSH connection.")
81
+ fail_and_cleanup(
82
+ task_group=task_group,
83
+ task_group_activity=activity,
84
+ logger_name=LOGGER_NAME,
85
+ log_file_path=log_file_path,
86
+ exception=e,
87
+ db=db,
88
+ )
89
+ return
90
+
91
+ # Check that the (remote) task_group venv_path does not exist
92
+ if fractal_ssh.remote_exists(task_group.venv_path):
93
+ error_msg = f"{task_group.venv_path} already exists."
94
+ logger.error(error_msg)
95
+ fail_and_cleanup(
96
+ task_group=task_group,
97
+ task_group_activity=activity,
98
+ logger_name=LOGGER_NAME,
99
+ log_file_path=log_file_path,
100
+ exception=FileExistsError(error_msg),
101
+ db=db,
102
+ )
103
+ return
104
+
105
+ try:
106
+ activity.status = TaskGroupActivityStatusV2.ONGOING
107
+ activity = add_commit_refresh(obj=activity, db=db)
108
+
109
+ # Prepare replacements for templates
110
+ replacements = get_collection_replacements(
111
+ task_group=task_group,
112
+ python_bin=get_python_interpreter_v2(
113
+ python_version=task_group.python_version
114
+ ),
115
+ )
116
+
117
+ # Prepare replacements for templates
118
+ pip_freeze_file_local = f"{tmpdir}/pip_freeze.txt"
119
+ pip_freeze_file_remote = (
120
+ Path(task_group.path) / "_tmp_pip_freeze.txt"
121
+ ).as_posix()
122
+ with open(pip_freeze_file_local, "w") as f:
123
+ f.write(task_group.pip_freeze)
124
+ fractal_ssh.send_file(
125
+ local=pip_freeze_file_local, remote=pip_freeze_file_remote
126
+ )
127
+ replacements.append(
128
+ ("__PIP_FREEZE_FILE__", pip_freeze_file_remote)
129
+ )
130
+
131
+ # Prepare common arguments for `_customize_and_run_template``
132
+ script_dir_remote = (
133
+ Path(task_group.path) / SCRIPTS_SUBFOLDER
134
+ ).as_posix()
135
+ common_args = dict(
136
+ replacements=replacements,
137
+ script_dir_local=(
138
+ Path(tmpdir) / SCRIPTS_SUBFOLDER
139
+ ).as_posix(),
140
+ script_dir_remote=script_dir_remote,
141
+ prefix=(
142
+ f"{int(time.time())}_"
143
+ f"{TaskGroupActivityActionV2.REACTIVATE}"
144
+ ),
145
+ fractal_ssh=fractal_ssh,
146
+ logger_name=LOGGER_NAME,
147
+ )
148
+
149
+ # Create remote directory for scripts
150
+ fractal_ssh.mkdir(folder=script_dir_remote)
151
+
152
+ logger.debug("start - create venv")
153
+ _customize_and_run_template(
154
+ template_filename="1_create_venv.sh",
155
+ **common_args,
156
+ )
157
+ logger.debug("end - create venv")
158
+ activity.log = get_current_log(log_file_path)
159
+ activity.timestamp_ended = get_timestamp()
160
+ activity = add_commit_refresh(obj=activity, db=db)
161
+
162
+ logger.debug("start - install from pip freeze")
163
+ _customize_and_run_template(
164
+ template_filename="6_pip_install_from_freeze.sh",
165
+ **common_args,
166
+ )
167
+ logger.debug("end - install from pip freeze")
168
+ activity.log = get_current_log(log_file_path)
169
+ activity.status = TaskGroupActivityStatusV2.OK
170
+ activity.timestamp_ended = get_timestamp()
171
+ activity = add_commit_refresh(obj=activity, db=db)
172
+ task_group.active = True
173
+ task_group = add_commit_refresh(obj=task_group, db=db)
174
+ logger.debug("END")
175
+
176
+ except Exception as reactivate_e:
177
+ # Delete corrupted venv_path
178
+ try:
179
+ logger.info(f"Now delete folder {task_group.venv_path}")
180
+ fractal_ssh.remove_folder(
181
+ folder=task_group.venv_path,
182
+ safe_root=tasks_base_dir,
183
+ )
184
+ logger.info(f"Deleted folder {task_group.venv_path}")
185
+ except Exception as rm_e:
186
+ logger.error(
187
+ "Removing folder failed.\n"
188
+ f"Original error:\n{str(rm_e)}"
189
+ )
190
+
191
+ fail_and_cleanup(
192
+ task_group=task_group,
193
+ task_group_activity=activity,
194
+ logger_name=LOGGER_NAME,
195
+ log_file_path=log_file_path,
196
+ exception=reactivate_e,
197
+ db=db,
198
+ )
199
+ return