fractal-server 2.0.0a6__py3-none-any.whl → 2.0.0a9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1 +1 @@
1
- __VERSION__ = "2.0.0a6"
1
+ __VERSION__ = "2.0.0a9"
@@ -33,11 +33,3 @@ class WorkflowV2(SQLModel, table=True):
33
33
  default_factory=get_timestamp,
34
34
  sa_column=Column(DateTime(timezone=True), nullable=False),
35
35
  )
36
-
37
- @property
38
- def input_types(self):
39
- return self.task_list[0].task.input_types
40
-
41
- @property
42
- def output_types(self):
43
- return self.task_list[-1].task.output_types
@@ -51,14 +51,13 @@ class WorkflowTaskV2(SQLModel, table=True):
51
51
 
52
52
  @validator("args_non_parallel")
53
53
  def validate_args_non_parallel(cls, value):
54
- """
55
- FIXME V2 this requires an update
56
- """
57
54
  if value is None:
58
55
  return
59
56
  forbidden_args_keys = {
60
- "metadata",
61
- "component",
57
+ "zarr_dir",
58
+ "zarr_url",
59
+ "zarr_urls",
60
+ "init_args",
62
61
  }
63
62
  args_keys = set(value.keys())
64
63
  intersect_keys = forbidden_args_keys.intersection(args_keys)
@@ -71,14 +70,13 @@ class WorkflowTaskV2(SQLModel, table=True):
71
70
 
72
71
  @validator("args_parallel")
73
72
  def validate_args_parallel(cls, value):
74
- """
75
- FIXME V2 this requires an update
76
- """
77
73
  if value is None:
78
74
  return
79
75
  forbidden_args_keys = {
80
- "metadata",
81
- "component",
76
+ "zarr_dir",
77
+ "zarr_url",
78
+ "zarr_urls",
79
+ "init_args",
82
80
  }
83
81
  args_keys = set(value.keys())
84
82
  intersect_keys = forbidden_args_keys.intersection(args_keys)
@@ -17,6 +17,8 @@ from ....models.v2 import ProjectV2
17
17
  from ....schemas.v2 import DatasetCreateV2
18
18
  from ....schemas.v2 import DatasetReadV2
19
19
  from ....schemas.v2 import DatasetUpdateV2
20
+ from ....schemas.v2.dataset import DatasetExportV2
21
+ from ....schemas.v2.dataset import DatasetImportV2
20
22
  from ....schemas.v2.dataset import DatasetStatusReadV2
21
23
  from ....schemas.v2.dataset import WorkflowTaskStatusTypeV2
22
24
  from ....security import current_active_user
@@ -134,6 +136,15 @@ async def update_dataset(
134
136
  )
135
137
  db_dataset = output["dataset"]
136
138
 
139
+ if (dataset_update.zarr_dir is not None) and (len(db_dataset.images) != 0):
140
+ raise HTTPException(
141
+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
142
+ detail=(
143
+ "Cannot modify `zarr_dir` because the dataset has a non-empty "
144
+ "image list."
145
+ ),
146
+ )
147
+
137
148
  for key, value in dataset_update.dict(exclude_unset=True).items():
138
149
  setattr(db_dataset, key, value)
139
150
 
@@ -306,3 +317,67 @@ async def get_workflowtask_status(
306
317
 
307
318
  response_body = DatasetStatusReadV2(status=workflow_tasks_status_dict)
308
319
  return response_body
320
+
321
+
322
+ # /api/v2/project/{project_id}/dataset/{dataset_id}/export/
323
+
324
+
325
+ @router.get(
326
+ "/project/{project_id}/dataset/{dataset_id}/export/",
327
+ response_model=DatasetExportV2,
328
+ )
329
+ async def export_dataset(
330
+ project_id: int,
331
+ dataset_id: int,
332
+ user: User = Depends(current_active_user),
333
+ db: AsyncSession = Depends(get_async_db),
334
+ ) -> Optional[DatasetExportV2]:
335
+ """
336
+ Export an existing dataset
337
+ """
338
+ dict_dataset_project = await _get_dataset_check_owner(
339
+ project_id=project_id,
340
+ dataset_id=dataset_id,
341
+ user_id=user.id,
342
+ db=db,
343
+ )
344
+ await db.close()
345
+
346
+ dataset = dict_dataset_project["dataset"]
347
+
348
+ return dataset
349
+
350
+
351
+ @router.post(
352
+ "/project/{project_id}/dataset/import/",
353
+ response_model=DatasetReadV2,
354
+ status_code=status.HTTP_201_CREATED,
355
+ )
356
+ async def import_dataset(
357
+ project_id: int,
358
+ dataset: DatasetImportV2,
359
+ user: User = Depends(current_active_user),
360
+ db: AsyncSession = Depends(get_async_db),
361
+ ) -> Optional[DatasetReadV2]:
362
+ """
363
+ Import an existing dataset into a project
364
+ """
365
+
366
+ # Preliminary checks
367
+ await _get_project_check_owner(
368
+ project_id=project_id,
369
+ user_id=user.id,
370
+ db=db,
371
+ )
372
+
373
+ # Create new Dataset
374
+ db_dataset = DatasetV2(
375
+ project_id=project_id,
376
+ **dataset.dict(exclude_none=True),
377
+ )
378
+ db.add(db_dataset)
379
+ await db.commit()
380
+ await db.refresh(db_dataset)
381
+ await db.close()
382
+
383
+ return db_dataset
@@ -17,6 +17,8 @@ from fractal_server.app.security import current_active_user
17
17
  from fractal_server.app.security import User
18
18
  from fractal_server.images import Filters
19
19
  from fractal_server.images import SingleImage
20
+ from fractal_server.images import SingleImageUpdate
21
+ from fractal_server.images.tools import find_image_by_zarr_url
20
22
  from fractal_server.images.tools import match_filter
21
23
 
22
24
  router = APIRouter()
@@ -56,6 +58,23 @@ async def post_new_image(
56
58
  )
57
59
  dataset = output["dataset"]
58
60
 
61
+ if not new_image.zarr_url.startswith(dataset.zarr_dir):
62
+ raise HTTPException(
63
+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
64
+ detail=(
65
+ "Cannot create image with zarr_url which is not relative to "
66
+ f"{dataset.zarr_dir}."
67
+ ),
68
+ )
69
+ elif new_image.zarr_url == dataset.zarr_dir:
70
+ raise HTTPException(
71
+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
72
+ detail=(
73
+ "`SingleImage.zarr_url` cannot be equal to `Dataset.zarr_dir`:"
74
+ f" {dataset.zarr_dir}"
75
+ ),
76
+ )
77
+
59
78
  if new_image.zarr_url in dataset.image_zarr_urls:
60
79
  raise HTTPException(
61
80
  status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
@@ -213,3 +232,48 @@ async def delete_dataset_images(
213
232
  await db.commit()
214
233
 
215
234
  return Response(status_code=status.HTTP_204_NO_CONTENT)
235
+
236
+
237
+ @router.patch(
238
+ "/project/{project_id}/dataset/{dataset_id}/images/",
239
+ response_model=SingleImage,
240
+ status_code=status.HTTP_200_OK,
241
+ )
242
+ async def patch_dataset_image(
243
+ project_id: int,
244
+ dataset_id: int,
245
+ image_update: SingleImageUpdate,
246
+ user: User = Depends(current_active_user),
247
+ db: AsyncSession = Depends(get_async_db),
248
+ ):
249
+ output = await _get_dataset_check_owner(
250
+ project_id=project_id,
251
+ dataset_id=dataset_id,
252
+ user_id=user.id,
253
+ db=db,
254
+ )
255
+ db_dataset = output["dataset"]
256
+
257
+ ret = find_image_by_zarr_url(
258
+ images=db_dataset.images, zarr_url=image_update.zarr_url
259
+ )
260
+ if ret is None:
261
+ raise HTTPException(
262
+ status_code=status.HTTP_404_NOT_FOUND,
263
+ detail=(
264
+ f"No image with zarr_url '{image_update.zarr_url}' in "
265
+ f"DatasetV2 {dataset_id}."
266
+ ),
267
+ )
268
+ index = ret["index"]
269
+
270
+ for key, value in image_update.dict(
271
+ exclude_none=True, exclude={"zarr_url"}
272
+ ).items():
273
+ db_dataset.images[index][key] = value
274
+
275
+ flag_modified(db_dataset, "images")
276
+
277
+ await db.commit()
278
+ await db.close()
279
+ return db_dataset.images[index]
@@ -200,9 +200,15 @@ async def update_workflowtask(
200
200
  setattr(db_wf_task, key, actual_args)
201
201
  elif key == "args_non_parallel":
202
202
  # Get default arguments via a Task property method
203
- default_args = deepcopy(
204
- db_wf_task.task.default_args_non_parallel_from_args_schema
205
- )
203
+ if db_wf_task.is_legacy_task:
204
+ # This is only needed so that we don't have to modify the rest
205
+ # of this block, but legacy task cannot take any non-parallel
206
+ # args (see checks above).
207
+ default_args = {}
208
+ else:
209
+ default_args = deepcopy(
210
+ db_wf_task.task.default_args_non_parallel_from_args_schema
211
+ )
206
212
  # Override default_args with args value items
207
213
  actual_args = default_args.copy()
208
214
  if value is not None:
@@ -125,7 +125,10 @@ def get_slurm_config(
125
125
  slurm_dict["mem_per_task_MB"] = mem_per_task_MB
126
126
 
127
127
  # Job name
128
- job_name = wftask.task.name.replace(" ", "_")
128
+ if wftask.is_legacy_task:
129
+ job_name = wftask.task_legacy.name.replace(" ", "_")
130
+ else:
131
+ job_name = wftask.task.name.replace(" ", "_")
129
132
  slurm_dict["job_name"] = job_name
130
133
 
131
134
  # Optional SLURM arguments and extra lines
@@ -96,8 +96,15 @@ def assemble_history_failed_job(
96
96
 
97
97
  # Part 3/B: Append failed task to history
98
98
  if failed_wftask is not None:
99
- failed_wftask_dump = failed_wftask.model_dump(exclude={"task"})
100
- failed_wftask_dump["task"] = failed_wftask.task.model_dump()
99
+ failed_wftask_dump = failed_wftask.model_dump(
100
+ exclude={"task", "task_legacy"}
101
+ )
102
+ if failed_wftask.is_legacy_task:
103
+ failed_wftask_dump[
104
+ "task_legacy"
105
+ ] = failed_wftask.task_legacy.model_dump()
106
+ else:
107
+ failed_wftask_dump["task"] = failed_wftask.task.model_dump()
101
108
  new_history_item = dict(
102
109
  workflowtask=failed_wftask_dump,
103
110
  status=WorkflowTaskStatusTypeV2.FAILED,
@@ -1,4 +1,5 @@
1
1
  import json
2
+ import logging
2
3
  from concurrent.futures import ThreadPoolExecutor
3
4
  from copy import copy
4
5
  from copy import deepcopy
@@ -37,6 +38,8 @@ def execute_tasks_v2(
37
38
  submit_setup_call: Callable = no_op_submit_setup_call,
38
39
  ) -> DatasetV2:
39
40
 
41
+ logger = logging.getLogger(logger_name)
42
+
40
43
  if not workflow_dir.exists(): # FIXME: this should have already happened
41
44
  workflow_dir.mkdir()
42
45
 
@@ -48,6 +51,9 @@ def execute_tasks_v2(
48
51
 
49
52
  for wftask in wf_task_list:
50
53
  task = wftask.task
54
+ task_legacy = wftask.task_legacy
55
+ task_name = task_legacy.name if wftask.is_legacy_task else task.name
56
+ logger.debug(f'SUBMIT {wftask.order}-th task (name="{task_name}")')
51
57
 
52
58
  # PRE TASK EXECUTION
53
59
 
@@ -63,12 +69,13 @@ def execute_tasks_v2(
63
69
  filters=Filters(**pre_filters),
64
70
  )
65
71
  # Verify that filtered images comply with task input_types
66
- for image in filtered_images:
67
- if not match_filter(image, Filters(types=task.input_types)):
68
- raise ValueError(
69
- f"Filtered images include {image}, which does "
70
- f"not comply with {task.input_types=}."
71
- )
72
+ if not wftask.is_legacy_task:
73
+ for image in filtered_images:
74
+ if not match_filter(image, Filters(types=task.input_types)):
75
+ raise ValueError(
76
+ f"Filtered images include {image}, which does "
77
+ f"not comply with {task.input_types=}."
78
+ )
72
79
 
73
80
  # TASK EXECUTION (V2)
74
81
  if not wftask.is_legacy_task:
@@ -77,7 +84,7 @@ def execute_tasks_v2(
77
84
  images=filtered_images,
78
85
  zarr_dir=zarr_dir,
79
86
  wftask=wftask,
80
- task=wftask.task,
87
+ task=task,
81
88
  workflow_dir=workflow_dir,
82
89
  workflow_dir_user=workflow_dir_user,
83
90
  executor=executor,
@@ -88,7 +95,7 @@ def execute_tasks_v2(
88
95
  current_task_output = run_v2_task_parallel(
89
96
  images=filtered_images,
90
97
  wftask=wftask,
91
- task=wftask.task,
98
+ task=task,
92
99
  workflow_dir=workflow_dir,
93
100
  workflow_dir_user=workflow_dir_user,
94
101
  executor=executor,
@@ -100,7 +107,7 @@ def execute_tasks_v2(
100
107
  images=filtered_images,
101
108
  zarr_dir=zarr_dir,
102
109
  wftask=wftask,
103
- task=wftask.task,
110
+ task=task,
104
111
  workflow_dir=workflow_dir,
105
112
  workflow_dir_user=workflow_dir_user,
106
113
  executor=executor,
@@ -114,9 +121,11 @@ def execute_tasks_v2(
114
121
  current_task_output = run_v1_task_parallel(
115
122
  images=filtered_images,
116
123
  wftask=wftask,
117
- task_legacy=wftask.task_legacy,
124
+ task_legacy=task_legacy,
118
125
  executor=executor,
119
126
  logger_name=logger_name,
127
+ workflow_dir=workflow_dir,
128
+ workflow_dir_user=workflow_dir_user,
120
129
  submit_setup_call=submit_setup_call,
121
130
  )
122
131
 
@@ -155,7 +164,8 @@ def execute_tasks_v2(
155
164
  # Update image attributes/types with task output and manifest
156
165
  updated_attributes.update(image["attributes"])
157
166
  updated_types.update(image["types"])
158
- updated_types.update(task.output_types)
167
+ if not wftask.is_legacy_task:
168
+ updated_types.update(task.output_types)
159
169
 
160
170
  # Unset attributes with None value
161
171
  updated_attributes = {
@@ -182,6 +192,11 @@ def execute_tasks_v2(
182
192
  f"{zarr_dir} is not a parent directory of "
183
193
  f"{image['zarr_url']}"
184
194
  )
195
+ # Check that image['zarr_url'] is not equal to zarr_dir
196
+ if image["zarr_url"] == zarr_dir:
197
+ raise ValueError(
198
+ "image['zarr_url'] cannot be equal to zarr_dir"
199
+ )
185
200
  # Propagate attributes and types from `origin` (if any)
186
201
  updated_attributes = {}
187
202
  updated_types = {}
@@ -202,7 +217,8 @@ def execute_tasks_v2(
202
217
  if value is not None
203
218
  }
204
219
  updated_types.update(image["types"])
205
- updated_types.update(task.output_types)
220
+ if not wftask.is_legacy_task:
221
+ updated_types.update(task.output_types)
206
222
  new_image = dict(
207
223
  zarr_url=image["zarr_url"],
208
224
  origin=image["origin"],
@@ -277,6 +293,8 @@ def execute_tasks_v2(
277
293
  with open(workflow_dir / IMAGES_FILENAME, "w") as f:
278
294
  json.dump(tmp_images, f, indent=2)
279
295
 
296
+ logger.debug(f'END {wftask.order}-th task (name="{task_name}")')
297
+
280
298
  # NOTE: tmp_history only contains the newly-added history items (to be
281
299
  # appended to the original history), while tmp_filters and tmp_images
282
300
  # represent the new attributes (to replace the original ones)
@@ -313,10 +313,11 @@ def run_v1_task_parallel(
313
313
  for ind, image in enumerate(images):
314
314
  list_function_kwargs.append(
315
315
  convert_v2_args_into_v1(
316
- dict(
316
+ kwargs_v2=dict(
317
317
  zarr_url=image["zarr_url"],
318
318
  **(wftask.args_parallel or {}),
319
- )
319
+ ),
320
+ parallelization_level=task_legacy.parallelization_level,
320
321
  ),
321
322
  )
322
323
  list_function_kwargs[-1][_COMPONENT_KEY_] = _index_to_component(ind)
@@ -116,7 +116,10 @@ def run_single_task(
116
116
  except TaskExecutionError as e:
117
117
  e.workflow_task_order = wftask.order
118
118
  e.workflow_task_id = wftask.id
119
- e.task_name = wftask.task.name
119
+ if wftask.is_legacy_task:
120
+ e.task_name = wftask.task_legacy.name
121
+ else:
122
+ e.task_name = wftask.task.name
120
123
  raise e
121
124
 
122
125
  try:
@@ -2,9 +2,11 @@ from typing import Any
2
2
 
3
3
  from pydantic import BaseModel
4
4
  from pydantic import Field
5
+ from pydantic import validator
5
6
 
6
7
  from ....images import SingleImageTaskOutput
7
8
  from fractal_server.images import Filters
9
+ from fractal_server.urls import normalize_url
8
10
 
9
11
 
10
12
  class TaskOutput(BaseModel):
@@ -34,6 +36,10 @@ class TaskOutput(BaseModel):
34
36
  msg = f"{msg}\n{duplicate}"
35
37
  raise ValueError(msg)
36
38
 
39
+ @validator("image_list_removals")
40
+ def normalize_paths(cls, v: list[str]) -> list[str]:
41
+ return [normalize_url(zarr_url) for zarr_url in v]
42
+
37
43
 
38
44
  class InitArgsModel(BaseModel):
39
45
  class Config:
@@ -42,6 +48,10 @@ class InitArgsModel(BaseModel):
42
48
  zarr_url: str
43
49
  init_args: dict[str, Any] = Field(default_factory=dict)
44
50
 
51
+ @validator("zarr_url")
52
+ def normalize_path(cls, v: str) -> str:
53
+ return normalize_url(v)
54
+
45
55
 
46
56
  class InitTaskOutput(BaseModel):
47
57
  class Config:
@@ -3,13 +3,23 @@ from pathlib import Path
3
3
  from typing import Any
4
4
 
5
5
 
6
- def convert_v2_args_into_v1(kwargs_v2: dict[str, Any]) -> dict[str, Any]:
7
-
6
+ def convert_v2_args_into_v1(
7
+ kwargs_v2: dict[str, Any],
8
+ parallelization_level: str = "image",
9
+ ) -> dict[str, Any]:
8
10
  kwargs_v1 = deepcopy(kwargs_v2)
9
11
 
10
12
  zarr_url = kwargs_v1.pop("zarr_url")
11
13
  input_path = Path(zarr_url).parents[3].as_posix()
12
- component = zarr_url.replace(input_path, "").lstrip("/")
14
+ image_component = zarr_url.replace(input_path, "").lstrip("/")
15
+ if parallelization_level == "image":
16
+ component = image_component
17
+ elif parallelization_level == "well":
18
+ component = str(Path(image_component).parent)
19
+ elif parallelization_level == "plate":
20
+ component = str(Path(image_component).parents[2])
21
+ else:
22
+ raise ValueError(f"Invalid {parallelization_level=}.")
13
23
 
14
24
  kwargs_v1.update(
15
25
  input_paths=[input_path],
@@ -2,6 +2,7 @@ import os
2
2
  from datetime import datetime
3
3
  from datetime import timezone
4
4
  from typing import Any
5
+ from typing import Optional
5
6
 
6
7
 
7
8
  def valstr(attribute: str, accept_none: bool = False):
@@ -12,7 +13,7 @@ def valstr(attribute: str, accept_none: bool = False):
12
13
  If `accept_none`, the validator also accepts `None`.
13
14
  """
14
15
 
15
- def val(string: str):
16
+ def val(string: Optional[str]) -> Optional[str]:
16
17
  if string is None:
17
18
  if accept_none:
18
19
  return string
@@ -29,7 +30,7 @@ def valstr(attribute: str, accept_none: bool = False):
29
30
 
30
31
 
31
32
  def valdictkeys(attribute: str):
32
- def val(d: dict[str, Any]):
33
+ def val(d: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]:
33
34
  """
34
35
  Apply valstr to every key of the dictionary, and fail if there are
35
36
  identical keys.
@@ -55,7 +56,7 @@ def valint(attribute: str, min_val: int = 1):
55
56
  database entry) is greater or equal to min_val.
56
57
  """
57
58
 
58
- def val(integer: int):
59
+ def val(integer: Optional[int]) -> Optional[int]:
59
60
  if integer is None:
60
61
  raise ValueError(f"Integer attribute '{attribute}' cannot be None")
61
62
  if integer < min_val:
@@ -73,7 +74,7 @@ def val_absolute_path(attribute: str):
73
74
  Check that a string attribute is an absolute path
74
75
  """
75
76
 
76
- def val(string: str):
77
+ def val(string: Optional[str]) -> str:
77
78
  if string is None:
78
79
  raise ValueError(f"String attribute '{attribute}' cannot be None")
79
80
  s = string.strip()
@@ -90,7 +91,7 @@ def val_absolute_path(attribute: str):
90
91
 
91
92
 
92
93
  def val_unique_list(attribute: str):
93
- def val(must_be_unique: list):
94
+ def val(must_be_unique: Optional[list]) -> Optional[list]:
94
95
  if must_be_unique is not None:
95
96
  if len(set(must_be_unique)) != len(must_be_unique):
96
97
  raise ValueError(f"`{attribute}` list has repetitions")
@@ -100,7 +101,7 @@ def val_unique_list(attribute: str):
100
101
 
101
102
 
102
103
  def valutc(attribute: str):
103
- def val(timestamp: datetime):
104
+ def val(timestamp: Optional[datetime]) -> Optional[datetime]:
104
105
  """
105
106
  Replacing `tzinfo` with `timezone.utc` is just required by SQLite data.
106
107
  If using Postgres, this function leaves the datetime exactly as it is.
@@ -12,6 +12,8 @@ from .dumps import WorkflowTaskDumpV2
12
12
  from .project import ProjectReadV2
13
13
  from .workflowtask import WorkflowTaskStatusTypeV2
14
14
  from fractal_server.images import Filters
15
+ from fractal_server.images import SingleImage
16
+ from fractal_server.urls import normalize_url
15
17
 
16
18
 
17
19
  class _DatasetHistoryItemV2(BaseModel):
@@ -50,6 +52,10 @@ class DatasetCreateV2(BaseModel, extra=Extra.forbid):
50
52
  filters: Filters = Field(default_factory=Filters)
51
53
 
52
54
  # Validators
55
+ @validator("zarr_dir")
56
+ def normalize_zarr_dir(cls, v: str) -> str:
57
+ return normalize_url(v)
58
+
53
59
  _name = validator("name", allow_reuse=True)(valstr("name"))
54
60
 
55
61
 
@@ -83,4 +89,47 @@ class DatasetUpdateV2(BaseModel):
83
89
  filters: Optional[Filters]
84
90
 
85
91
  # Validators
92
+ @validator("zarr_dir")
93
+ def normalize_zarr_dir(cls, v: Optional[str]) -> Optional[str]:
94
+ if v is not None:
95
+ return normalize_url(v)
96
+ return v
97
+
86
98
  _name = validator("name", allow_reuse=True)(valstr("name"))
99
+
100
+
101
+ class DatasetImportV2(BaseModel):
102
+ """
103
+ Class for `Dataset` import.
104
+
105
+ Attributes:
106
+ name:
107
+ zarr_dir:
108
+ images:
109
+ filters:
110
+ """
111
+
112
+ class Config:
113
+ extra = "forbid"
114
+
115
+ name: str
116
+ zarr_dir: str
117
+ images: list[SingleImage] = Field(default_factory=[])
118
+ filters: Filters = Field(default_factory=Filters)
119
+
120
+
121
+ class DatasetExportV2(BaseModel):
122
+ """
123
+ Class for `Dataset` export.
124
+
125
+ Attributes:
126
+ name:
127
+ zarr_dir:
128
+ images:
129
+ filters:
130
+ """
131
+
132
+ name: str
133
+ zarr_dir: str
134
+ images: list[SingleImage]
135
+ filters: Filters
@@ -45,6 +45,8 @@ class WorkflowTaskDumpV2(BaseModel):
45
45
  workflow_id: int
46
46
  order: Optional[int]
47
47
 
48
+ is_legacy_task: bool
49
+
48
50
  input_filters: Filters
49
51
 
50
52
  task_id: Optional[int]
@@ -1,3 +1,4 @@
1
1
  from .models import Filters # noqa: F401
2
2
  from .models import SingleImage # noqa: F401
3
3
  from .models import SingleImageTaskOutput # noqa: F401
4
+ from .models import SingleImageUpdate # noqa: F401
@@ -7,6 +7,7 @@ from pydantic import Field
7
7
  from pydantic import validator
8
8
 
9
9
  from fractal_server.app.schemas._validators import valdictkeys
10
+ from fractal_server.urls import normalize_url
10
11
 
11
12
 
12
13
  class SingleImageBase(BaseModel):
@@ -32,6 +33,15 @@ class SingleImageBase(BaseModel):
32
33
  )
33
34
  _types = validator("types", allow_reuse=True)(valdictkeys("types"))
34
35
 
36
+ @validator("zarr_url")
37
+ def normalize_zarr_url(cls, v: str) -> str:
38
+ return normalize_url(v)
39
+
40
+ @validator("origin")
41
+ def normalize_orig(cls, v: Optional[str]) -> Optional[str]:
42
+ if v is not None:
43
+ return normalize_url(v)
44
+
35
45
 
36
46
  class SingleImageTaskOutput(SingleImageBase):
37
47
  """
@@ -70,6 +80,35 @@ class SingleImage(SingleImageBase):
70
80
  return v
71
81
 
72
82
 
83
+ class SingleImageUpdate(BaseModel):
84
+ zarr_url: str
85
+ attributes: Optional[dict[str, Any]]
86
+ types: Optional[dict[str, bool]]
87
+
88
+ @validator("zarr_url")
89
+ def normalize_zarr_url(cls, v: str) -> str:
90
+ return normalize_url(v)
91
+
92
+ @validator("attributes")
93
+ def validate_attributes(
94
+ cls, v: dict[str, Any]
95
+ ) -> dict[str, Union[int, float, str, bool]]:
96
+ if v is not None:
97
+ # validate keys
98
+ valdictkeys("attributes")(v)
99
+ # validate values
100
+ for key, value in v.items():
101
+ if not isinstance(value, (int, float, str, bool)):
102
+ raise ValueError(
103
+ f"SingleImageUpdate.attributes[{key}] must be a scalar"
104
+ " (int, float, str or bool). "
105
+ f"Given {value} ({type(value)})"
106
+ )
107
+ return v
108
+
109
+ _types = validator("types", allow_reuse=True)(valdictkeys("types"))
110
+
111
+
73
112
  class Filters(BaseModel):
74
113
  attributes: dict[str, Any] = Field(default_factory=dict)
75
114
  types: dict[str, bool] = Field(default_factory=dict)
fractal_server/urls.py ADDED
@@ -0,0 +1,13 @@
1
+ from os.path import normpath
2
+
3
+
4
+ def normalize_url(url: str) -> str:
5
+ if url.startswith("/"):
6
+ return normpath(url)
7
+ elif url.startswith("s3"):
8
+ # It would be better to have a NotImplementedError
9
+ # but Pydantic Validation + FastAPI require
10
+ # ValueError, TypeError or AssertionError
11
+ raise ValueError("S3 handling not implemented yet")
12
+ else:
13
+ raise ValueError("URLs must begin with '/' or 's3'.")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: fractal-server
3
- Version: 2.0.0a6
3
+ Version: 2.0.0a9
4
4
  Summary: Server component of the Fractal analytics platform
5
5
  Home-page: https://github.com/fractal-analytics-platform/fractal-server
6
6
  License: BSD-3-Clause
@@ -23,7 +23,7 @@ Requires-Dist: cloudpickle (>=3.0.0,<3.1.0)
23
23
  Requires-Dist: clusterfutures (>=0.5,<0.6)
24
24
  Requires-Dist: fastapi (>=0.110.0,<0.111.0)
25
25
  Requires-Dist: fastapi-users[oauth] (>=12.1.0,<13.0.0)
26
- Requires-Dist: gunicorn (>=21.2.0,<22.0.0) ; extra == "gunicorn"
26
+ Requires-Dist: gunicorn (>=21.2,<23.0) ; extra == "gunicorn"
27
27
  Requires-Dist: packaging (>=23.2,<24.0)
28
28
  Requires-Dist: psycopg2 (>=2.9.5,<3.0.0) ; extra == "postgres"
29
29
  Requires-Dist: pydantic (>=1.10.8,<2)
@@ -1,4 +1,4 @@
1
- fractal_server/__init__.py,sha256=5qWDQ_nyhHlAAd6zm7X6HCZpyfGaVJKhBA0qSF2KZfA,24
1
+ fractal_server/__init__.py,sha256=igx3UAg7e-LuCe083CY3r_FQooF1tJV_FTJy9JRvzEo,24
2
2
  fractal_server/__main__.py,sha256=CocbzZooX1UtGqPi55GcHGNxnrJXFg5tUU5b3wyFCyo,4958
3
3
  fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
4
4
  fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -18,8 +18,8 @@ fractal_server/app/models/v2/dataset.py,sha256=-7sxHEw4IIAvF_uSan7tA3o8hvoakBkQ0
18
18
  fractal_server/app/models/v2/job.py,sha256=PCJf0_NYIc5boXL6e6P72BvYJGydCZOGKnW2DT4Sw9g,1535
19
19
  fractal_server/app/models/v2/project.py,sha256=CqDEKzdVxmFDMee6DnVOyX7WGmdn-dQSLSekzw_OLUc,817
20
20
  fractal_server/app/models/v2/task.py,sha256=9ZPhug3VWyeqgT8wQ9_8ZXQ2crSiiicRipxrxTslOso,3257
21
- fractal_server/app/models/v2/workflow.py,sha256=4pSTeZC78OQbgHHC5S0ge6pK1AP6ak7Qew_0ZNM9xuw,1256
22
- fractal_server/app/models/v2/workflowtask.py,sha256=f2a85MSAyBAdC7oG6SR8mViMNqlomQWaIB08n3ZhT-0,2727
21
+ fractal_server/app/models/v2/workflow.py,sha256=YBgFGCziUgU0aJ5EM3Svu9W2c46AewZO9VBlFCHiSps,1069
22
+ fractal_server/app/models/v2/workflowtask.py,sha256=kEm2k1LI0KK9vlTH7DL1NddaEUpIvMkFi42vahwDpd8,2695
23
23
  fractal_server/app/routes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
24
24
  fractal_server/app/routes/admin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
25
25
  fractal_server/app/routes/admin/v1.py,sha256=uY6H1znlAlrM9e1MG2EThTqwciCl87Twew34JM5W6IU,13981
@@ -36,8 +36,8 @@ fractal_server/app/routes/api/v1/workflow.py,sha256=ZObifWTPi100oRQ1wEER8Sgsr3Ne
36
36
  fractal_server/app/routes/api/v1/workflowtask.py,sha256=ox-DIIqYV4K35hCu86eGa2SHnR5IQml-I00UHEwnmHQ,5579
37
37
  fractal_server/app/routes/api/v2/__init__.py,sha256=x56HcY1uBNCgq4BRVj-0j6bAj6OsTN97RNDqY8NefJ8,1373
38
38
  fractal_server/app/routes/api/v2/_aux_functions.py,sha256=TCHf3aM-KQxaNJen10CGX1Da5IIra00xRF39FUTU698,14301
39
- fractal_server/app/routes/api/v2/dataset.py,sha256=qQi9jfT9YLu6DrRCPh280J3MoFWs9yMiejkCNaauCyQ,9680
40
- fractal_server/app/routes/api/v2/images.py,sha256=b1NM9Y0ocuRYRec-3UcVAizB0vFkmzPEHfObaoCnIMY,5956
39
+ fractal_server/app/routes/api/v2/dataset.py,sha256=mgz8746jOhXDdKkNY7dDN3bM0QgXFBMk1VUFqnxU-B0,11573
40
+ fractal_server/app/routes/api/v2/images.py,sha256=4r_HblPWyuKSZSJZfn8mbDaLv1ncwZU0gWdKneZcNG4,7894
41
41
  fractal_server/app/routes/api/v2/job.py,sha256=9mXaKCX_N3FXM0GIxdE49nWl_hJZ8CBLBIaMMhaCKOM,5334
42
42
  fractal_server/app/routes/api/v2/project.py,sha256=i9a19HAqE36N92G60ZYgObIP9nv-hR7Jt5nd9Dkhz1g,6024
43
43
  fractal_server/app/routes/api/v2/submit.py,sha256=iszII5CvWDEjGPTphBgH9FVS1pNb5m11Xc8xozGgjgI,6901
@@ -45,7 +45,7 @@ fractal_server/app/routes/api/v2/task.py,sha256=gJ0LruSk-Q1iMw8ZOX8C0wrZ4S4DGlQT
45
45
  fractal_server/app/routes/api/v2/task_collection.py,sha256=iw74UF8qdQa9pJf0DvSjihng6ri2k2HtW2UhMS_a8Zc,8904
46
46
  fractal_server/app/routes/api/v2/task_legacy.py,sha256=P_VJv9v0yzFUBuS-DQHhMVSOe20ecGJJcFBqiiFciOM,1628
47
47
  fractal_server/app/routes/api/v2/workflow.py,sha256=sw-1phO_rrmDAcWX9Zqb9M8SfrWF78-02AuLB1-D1PU,11845
48
- fractal_server/app/routes/api/v2/workflowtask.py,sha256=L4hYpb-ihKNfPxM5AnZqPhCdiojI9Eq5TR0wf-0vP_s,8414
48
+ fractal_server/app/routes/api/v2/workflowtask.py,sha256=I1nrIV5J_DW1IeBq0q9VmUeBDo7P6x7qYO_Ocls2Pno,8720
49
49
  fractal_server/app/routes/auth.py,sha256=Xv80iqdyfY3lyicYs2Y8B6zEDEnyUu_H6_6psYtv3R4,4885
50
50
  fractal_server/app/routes/aux/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
51
51
  fractal_server/app/routes/aux/_job.py,sha256=5gKgvArAruSkMQuPN34Vvzi89WJbwWPsx0oDAa_iXu4,1248
@@ -85,17 +85,17 @@ fractal_server/app/runner/v2/_local/_submit_setup.py,sha256=deagsLSy6A3ZHKaSDcQq
85
85
  fractal_server/app/runner/v2/_local/executor.py,sha256=QrJlD77G6q4WohoJQO7XXbvi2RlCUsNvMnPDEZIoAqA,3620
86
86
  fractal_server/app/runner/v2/_slurm/__init__.py,sha256=srxn5-KdQxqD8cWJmOJlSoctbXYlyCMM249xWGY9bhI,4409
87
87
  fractal_server/app/runner/v2/_slurm/_submit_setup.py,sha256=tsZHQdVy3VxENMdsBzHltrVWzugBppq0cFrHtaVzoUA,2793
88
- fractal_server/app/runner/v2/_slurm/get_slurm_config.py,sha256=sqP-hs58TPt849rx10VRFKWX_DgLDPQcKZJcE0zKBXs,6621
88
+ fractal_server/app/runner/v2/_slurm/get_slurm_config.py,sha256=I_lOS75iGYyJ74-gNwwcPadvZ9vI9HYe04WMln5GJ5Q,6726
89
89
  fractal_server/app/runner/v2/deduplicate_list.py,sha256=-imwO7OB7ATADEnqVbTElUwoY0YIJCTf_SbWJNN9OZg,639
90
- fractal_server/app/runner/v2/handle_failed_job.py,sha256=fipRJT5Y8UY0US4bXUX-4ORTAQ1AetZcCAOVCjDO3_c,5202
90
+ fractal_server/app/runner/v2/handle_failed_job.py,sha256=M1r3dnrbUMo_AI2qjaVuGhieMAyLh5gcvB10YOBpjvI,5415
91
91
  fractal_server/app/runner/v2/merge_outputs.py,sha256=IHuHqbKmk97K35BFvTrKVBs60z3e_--OzXTnsvmA02c,1281
92
- fractal_server/app/runner/v2/runner.py,sha256=rBRehRDduGU0TUOkgQN6WbIGhDWZ6GOat4bv7IVB8cA,11784
93
- fractal_server/app/runner/v2/runner_functions.py,sha256=LfO1-FJF70_Qh78NQTCHJWyzyr011wvvtnzB6nTj5ZM,10087
94
- fractal_server/app/runner/v2/runner_functions_low_level.py,sha256=Pp3hsj1i1t4ExDMcUBkQ27yEi7kjlvymY6q6eDiC8DM,3845
95
- fractal_server/app/runner/v2/task_interface.py,sha256=NJZUMHtEs5C3bFdXX42Kv1GMzQ7xPW2v5ZRxGNANOec,1410
96
- fractal_server/app/runner/v2/v1_compat.py,sha256=6UijuRYbB2ry2mM073u1fW4CSTeelB11lmoj_TOGtm4,511
92
+ fractal_server/app/runner/v2/runner.py,sha256=K6bmWbQRSZwbO6ZI2Bp7wNxYdkHcXxhWwBObMxJ0iSU,12599
93
+ fractal_server/app/runner/v2/runner_functions.py,sha256=kN_xuaAg4qeRNIXijo30F1WSOo3zbjz5JCuS87EQf4g,10171
94
+ fractal_server/app/runner/v2/runner_functions_low_level.py,sha256=djNKD1y_EE0Q9Jkzh1QdKpjM66JVsLQgX2_zJT0xQlA,3947
95
+ fractal_server/app/runner/v2/task_interface.py,sha256=TZLVJs6CNFo2lFhr-lsDxe585cEhRv48eA490LS9aqc,1746
96
+ fractal_server/app/runner/v2/v1_compat.py,sha256=t0ficzAHUFaaeI56nqTb4YEKxfARF7L9Y6ijtJCwjP8,912
97
97
  fractal_server/app/schemas/__init__.py,sha256=VL55f3CTFngXHYkOsFaLBEEkEEewEWI5ODlcGTI7cqA,157
98
- fractal_server/app/schemas/_validators.py,sha256=Pdff5plJJmoUTf_nZpMA24tZlFJb84EdRSnLwRZDxfE,3264
98
+ fractal_server/app/schemas/_validators.py,sha256=1dTOYr1IZykrxuQSV2-zuEMZbKe_nGwrfS7iUrsh-sE,3461
99
99
  fractal_server/app/schemas/state.py,sha256=t4XM04aqxeluh8MfvD7LfEc-8-dOmUVluZHhLsfxxkc,692
100
100
  fractal_server/app/schemas/user.py,sha256=rE8WgBz-ceVUs0Sz2ZwcjUrSTZTnS0ys5SBtD2XD9r8,3113
101
101
  fractal_server/app/schemas/v1/__init__.py,sha256=gZLfkANl4YtZ7aV3PFoUj5w0m1-riQv9iRomJhZRLZo,2078
@@ -108,8 +108,8 @@ fractal_server/app/schemas/v1/task.py,sha256=7BxOZ_qoRQ8n3YbQpDvB7VMcxB5fSYQmR5R
108
108
  fractal_server/app/schemas/v1/task_collection.py,sha256=uvq9bcMaGD_qHsh7YtcpoSAkVAbw12eY4DocIO3MKOg,3057
109
109
  fractal_server/app/schemas/v1/workflow.py,sha256=tuOs5E5Q_ozA8if7YPZ07cQjzqB_QMkBS4u92qo4Ro0,4618
110
110
  fractal_server/app/schemas/v2/__init__.py,sha256=zlCYrplCWwnCL9-BYsExRMfVzhBy21IMBfdHPMgJZYk,1752
111
- fractal_server/app/schemas/v2/dataset.py,sha256=VSAB8np2TdZf_a2NJKkpBVJFDSs0IYpLan5RDzu6g3E,1757
112
- fractal_server/app/schemas/v2/dumps.py,sha256=Xen0OPf1Ax9i_7ItrAPvCk1OCNcUsnhlLRiyny89aLM,1997
111
+ fractal_server/app/schemas/v2/dataset.py,sha256=_nnpGqaD7HJNC125jAyPn05iavy5uy4jxEMDB40TXCA,2737
112
+ fractal_server/app/schemas/v2/dumps.py,sha256=IpIT_2KxJd7qTgW2NllDknGeP7vBAJDfyz1I5p3TytU,2023
113
113
  fractal_server/app/schemas/v2/job.py,sha256=zfF9K3v4jWUJ7M482ta2CkqUJ4tVT4XfVt60p9IRhP0,3250
114
114
  fractal_server/app/schemas/v2/manifest.py,sha256=N37IWohcfO3_y2l8rVM0h_1nZq7m4Izxk9iL1vtwBJw,6243
115
115
  fractal_server/app/schemas/v2/project.py,sha256=u7S4B-bote1oGjzAGiZ-DuQIyeRAGqJsI71Tc1EtYE0,736
@@ -120,8 +120,8 @@ fractal_server/app/schemas/v2/workflowtask.py,sha256=vRyPca8smu6fzwd9gO1eOd3qdPL
120
120
  fractal_server/app/security/__init__.py,sha256=wxosoHc3mJYPCdPMyWnRD8w_2OgnKYp2aDkdmwrZh5k,11203
121
121
  fractal_server/config.py,sha256=CA8ASObADaME5chDiBXawAJZ3MvjTRpCKP0jvdYtSh8,15080
122
122
  fractal_server/data_migrations/README.md,sha256=_3AEFvDg9YkybDqCLlFPdDmGJvr6Tw7HRI14aZ3LOIw,398
123
- fractal_server/images/__init__.py,sha256=KWLVMlWqTY85qq1VUpzaJi5Sl2VOYWEn0vIEiD-QZ5k,144
124
- fractal_server/images/models.py,sha256=hgDQf1-SsMJw504GFUufVETedPPEweCQxUhA2uDfdao,2904
123
+ fractal_server/images/__init__.py,sha256=xO6jTLE4EZKO6cTDdJsBmK9cdeh9hFTaSbSuWgQg7y4,196
124
+ fractal_server/images/models.py,sha256=9ipU5h4N6ogBChoB-2vHoqtL0TXOHCv6kRR-fER3mkM,4167
125
125
  fractal_server/images/tools.py,sha256=Q7jM60r_jq5bttrt1b4bU29n717RSUMMPbAbAkzWjgw,2234
126
126
  fractal_server/logger.py,sha256=95duXY8eSxf1HWg0CVn8SUGNzgJw9ZR0FlapDDF6WAY,3924
127
127
  fractal_server/main.py,sha256=7CpwPfCsHxBAo5fWuXPCsYOFCpbBI0F7Z0jsgCQdou8,3001
@@ -157,9 +157,10 @@ fractal_server/tasks/v2/_TaskCollectPip.py,sha256=QeCqXDgOnMjk3diVlC5bgGEywyQjYF
157
157
  fractal_server/tasks/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
158
158
  fractal_server/tasks/v2/background_operations.py,sha256=zr6j3uoWmCeW2EA9auxWNZ0sG3SHgSxUVTC1OpQXE3Y,12803
159
159
  fractal_server/tasks/v2/get_collection_data.py,sha256=Qhf2T_aaqAfqu9_KpUSlXsS7EJoZQbEPEreHHa2jco8,502
160
+ fractal_server/urls.py,sha256=5o_qq7PzKKbwq12NHSQZDmDitn5RAOeQ4xufu-2v9Zk,448
160
161
  fractal_server/utils.py,sha256=b7WwFdcFZ8unyT65mloFToYuEDXpQoHRcmRNqrhd_dQ,2115
161
- fractal_server-2.0.0a6.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
162
- fractal_server-2.0.0a6.dist-info/METADATA,sha256=VzuxOIkkB0Uv-MzjnyEf9Xu3xhQzJQUNCYROj_tVrwM,4204
163
- fractal_server-2.0.0a6.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
164
- fractal_server-2.0.0a6.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
165
- fractal_server-2.0.0a6.dist-info/RECORD,,
162
+ fractal_server-2.0.0a9.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
163
+ fractal_server-2.0.0a9.dist-info/METADATA,sha256=P8NzTlZ9SHoftxPyngm08c824qK_b9sdIXvfrxt_e5Y,4200
164
+ fractal_server-2.0.0a9.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
165
+ fractal_server-2.0.0a9.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
166
+ fractal_server-2.0.0a9.dist-info/RECORD,,