fractal-server 2.12.1__py3-none-any.whl → 2.13.1__py3-none-any.whl

fractal_server/app/runner/v2/runner.py

@@ -18,6 +18,7 @@ from .runner_functions import run_v2_task_non_parallel
 from .runner_functions import run_v2_task_parallel
 from .task_interface import TaskOutput
 from fractal_server.app.db import get_sync_db
+from fractal_server.app.models.v2 import AccountingRecord
 from fractal_server.app.models.v2 import DatasetV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.schemas.v2.dataset import _DatasetHistoryItemV2
@@ -31,6 +32,7 @@ def execute_tasks_v2(
     wf_task_list: list[WorkflowTaskV2],
     dataset: DatasetV2,
     executor: ThreadPoolExecutor,
+    user_id: int,
     workflow_dir_local: Path,
     workflow_dir_remote: Optional[Path] = None,
     logger_name: Optional[str] = None,
@@ -75,17 +77,20 @@ def execute_tasks_v2(
         with next(get_sync_db()) as db:
             db_dataset = db.get(DatasetV2, dataset.id)
             new_history_item = _DatasetHistoryItemV2(
-                workflowtask=wftask,
+                workflowtask=dict(
+                    **wftask.model_dump(exclude={"task"}),
+                    task=wftask.task.model_dump(),
+                ),
                 status=WorkflowTaskStatusTypeV2.SUBMITTED,
                 parallelization=dict(),  # FIXME: re-include parallelization
-            ).dict()
+            ).model_dump()
             db_dataset.history.append(new_history_item)
             flag_modified(db_dataset, "history")
             db.merge(db_dataset)
             db.commit()
         # TASK EXECUTION (V2)
         if task.type == "non_parallel":
-            current_task_output = run_v2_task_non_parallel(
+            current_task_output, num_tasks = run_v2_task_non_parallel(
                 images=filtered_images,
                 zarr_dir=zarr_dir,
                 wftask=wftask,
@@ -93,22 +98,20 @@ def execute_tasks_v2(
                 workflow_dir_local=workflow_dir_local,
                 workflow_dir_remote=workflow_dir_remote,
                 executor=executor,
-                logger_name=logger_name,
                 submit_setup_call=submit_setup_call,
             )
         elif task.type == "parallel":
-            current_task_output = run_v2_task_parallel(
+            current_task_output, num_tasks = run_v2_task_parallel(
                 images=filtered_images,
                 wftask=wftask,
                 task=task,
                 workflow_dir_local=workflow_dir_local,
                 workflow_dir_remote=workflow_dir_remote,
                 executor=executor,
-                logger_name=logger_name,
                 submit_setup_call=submit_setup_call,
             )
         elif task.type == "compound":
-            current_task_output = run_v2_task_compound(
+            current_task_output, num_tasks = run_v2_task_compound(
                 images=filtered_images,
                 zarr_dir=zarr_dir,
                 wftask=wftask,
@@ -116,7 +119,6 @@ def execute_tasks_v2(
                 workflow_dir_local=workflow_dir_local,
                 workflow_dir_remote=workflow_dir_remote,
                 executor=executor,
-                logger_name=logger_name,
                 submit_setup_call=submit_setup_call,
             )
         else:
@@ -132,16 +134,19 @@ def execute_tasks_v2(
             and current_task_output.image_list_removals == []
         ):
             current_task_output = TaskOutput(
-                **current_task_output.dict(exclude={"image_list_updates"}),
+                **current_task_output.model_dump(
+                    exclude={"image_list_updates"}
+                ),
                 image_list_updates=[
                     dict(zarr_url=img["zarr_url"]) for img in filtered_images
                 ],
             )
 
         # Update image list
+        num_new_images = 0
         current_task_output.check_zarr_urls_are_unique()
         for image_obj in current_task_output.image_list_updates:
-            image = image_obj.dict()
+            image = image_obj.model_dump()
             # Edit existing image
             tmp_image_paths = [img["zarr_url"] for img in tmp_images]
             if image["zarr_url"] in tmp_image_paths:
@@ -241,6 +246,7 @@ def execute_tasks_v2(
                 SingleImage(**new_image)
                 # Add image into the dataset image list
                 tmp_images.append(new_image)
+                num_new_images += 1
 
         # Remove images from tmp_images
         for img_zarr_url in current_task_output.image_list_removals:
@@ -276,4 +282,13 @@ def execute_tasks_v2(
             db.merge(db_dataset)
             db.commit()
 
+            # Create accounting record
+            record = AccountingRecord(
+                user_id=user_id,
+                num_tasks=num_tasks,
+                num_new_images=num_new_images,
+            )
+            db.add(record)
+            db.commit()
+
         logger.debug(f'END    {wftask.order}-th task (name="{task_name}")')

fractal_server/app/runner/v2/runner_functions.py

@@ -115,9 +115,8 @@ def run_v2_task_non_parallel(
     workflow_dir_local: Path,
     workflow_dir_remote: Optional[Path] = None,
     executor: Executor,
-    logger_name: Optional[str] = None,
     submit_setup_call: Callable = no_op_submit_setup_call,
-) -> TaskOutput:
+) -> tuple[TaskOutput, int]:
     """
     This runs server-side (see `executor` argument)
     """
@@ -154,10 +153,11 @@ def run_v2_task_non_parallel(
         **executor_options,
     )
     output = future.result()
+    num_tasks = 1
     if output is None:
-        return TaskOutput()
+        return (TaskOutput(), num_tasks)
     else:
-        return _cast_and_validate_TaskOutput(output)
+        return (_cast_and_validate_TaskOutput(output), num_tasks)
 
 
 def run_v2_task_parallel(
@@ -168,12 +168,11 @@ def run_v2_task_parallel(
     executor: Executor,
     workflow_dir_local: Path,
     workflow_dir_remote: Optional[Path] = None,
-    logger_name: Optional[str] = None,
     submit_setup_call: Callable = no_op_submit_setup_call,
-) -> TaskOutput:
+) -> tuple[TaskOutput, int]:
 
     if len(images) == 0:
-        return TaskOutput()
+        return (TaskOutput(), 0)
 
     _check_parallelization_list_size(images)
 
@@ -216,8 +215,9 @@ def run_v2_task_parallel(
         else:
             outputs[ind] = _cast_and_validate_TaskOutput(output)
 
+    num_tasks = len(images)
     merged_output = merge_outputs(outputs)
-    return merged_output
+    return (merged_output, num_tasks)
 
 
 def run_v2_task_compound(
@@ -229,7 +229,6 @@ def run_v2_task_compound(
     executor: Executor,
     workflow_dir_local: Path,
     workflow_dir_remote: Optional[Path] = None,
-    logger_name: Optional[str] = None,
     submit_setup_call: Callable = no_op_submit_setup_call,
 ) -> TaskOutput:
 
@@ -273,11 +272,13 @@ def run_v2_task_compound(
     parallelization_list = init_task_output.parallelization_list
     parallelization_list = deduplicate_list(parallelization_list)
 
+    num_task = 1 + len(parallelization_list)
+
     # 3/B: parallel part of a compound task
     _check_parallelization_list_size(parallelization_list)
 
     if len(parallelization_list) == 0:
-        return TaskOutput()
+        return (TaskOutput(), 0)
 
     list_function_kwargs = []
     for ind, parallelization_item in enumerate(parallelization_list):
@@ -313,4 +314,4 @@ def run_v2_task_compound(
             outputs[ind] = validated_output
 
     merged_output = merge_outputs(outputs)
-    return merged_output
+    return (merged_output, num_task)

fractal_server/app/runner/v2/task_interface.py

@@ -1,15 +1,17 @@
 from typing import Any
 
 from pydantic import BaseModel
-from pydantic import Extra
+from pydantic import ConfigDict
 from pydantic import Field
-from pydantic import validator
+from pydantic import field_validator
 
 from ....images import SingleImageTaskOutput
 from fractal_server.urls import normalize_url
 
 
-class TaskOutput(BaseModel, extra=Extra.forbid):
+class TaskOutput(BaseModel):
+
+    model_config = ConfigDict(extra="forbid")
 
     image_list_updates: list[SingleImageTaskOutput] = Field(
         default_factory=list
@@ -35,21 +37,27 @@ class TaskOutput(BaseModel, extra=Extra.forbid):
                 msg = f"{msg}\n{duplicate}"
             raise ValueError(msg)
 
-    @validator("image_list_removals")
+    @field_validator("image_list_removals")
+    @classmethod
     def normalize_paths(cls, v: list[str]) -> list[str]:
         return [normalize_url(zarr_url) for zarr_url in v]
 
 
-class InitArgsModel(BaseModel, extra=Extra.forbid):
+class InitArgsModel(BaseModel):
+
+    model_config = ConfigDict(extra="forbid")
 
     zarr_url: str
     init_args: dict[str, Any] = Field(default_factory=dict)
 
-    @validator("zarr_url")
+    @field_validator("zarr_url")
+    @classmethod
     def normalize_path(cls, v: str) -> str:
         return normalize_url(v)
 
 
-class InitTaskOutput(BaseModel, extra=Extra.forbid):
+class InitTaskOutput(BaseModel):
+
+    model_config = ConfigDict(extra="forbid")
 
     parallelization_list: list[InitArgsModel] = Field(default_factory=list)

fractal_server/app/schemas/_filter_validators.py

@@ -5,22 +5,25 @@ from fractal_server.images.models import AttributeFiltersType
 
 
 def validate_type_filters(
-    type_filters: Optional[dict[str, bool]]
+    cls, type_filters: Optional[dict[str, bool]]
 ) -> dict[str, bool]:
     if type_filters is None:
         raise ValueError("'type_filters' cannot be 'None'.")
 
-    type_filters = valdict_keys("type_filters")(type_filters)
+    type_filters = valdict_keys("type_filters")(cls, type_filters)
     return type_filters
 
 
 def validate_attribute_filters(
+    cls,
     attribute_filters: Optional[AttributeFiltersType],
 ) -> AttributeFiltersType:
     if attribute_filters is None:
         raise ValueError("'attribute_filters' cannot be 'None'.")
 
-    attribute_filters = valdict_keys("attribute_filters")(attribute_filters)
+    attribute_filters = valdict_keys("attribute_filters")(
+        cls, attribute_filters
+    )
     for key, values in attribute_filters.items():
         if values == []:
             raise ValueError(

fractal_server/app/schemas/_validators.py

@@ -11,7 +11,7 @@ def valstr(attribute: str, accept_none: bool = False):
     If `accept_none`, the validator also accepts `None`.
     """
 
-    def val(string: Optional[str]) -> Optional[str]:
+    def val(cls, string: Optional[str]) -> Optional[str]:
         if string is None:
             if accept_none:
                 return string
@@ -28,14 +28,16 @@ def valstr(attribute: str, accept_none: bool = False):
 
 
 def valdict_keys(attribute: str):
-    def val(d: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]:
+    def val(cls, d: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]:
         """
         Apply valstr to every key of the dictionary, and fail if there are
         identical keys.
         """
         if d is not None:
             old_keys = list(d.keys())
-            new_keys = [valstr(f"{attribute}[{key}]")(key) for key in old_keys]
+            new_keys = [
+                valstr(f"{attribute}[{key}]")(cls, key) for key in old_keys
+            ]
             if len(new_keys) != len(set(new_keys)):
                 raise ValueError(
                     f"Dictionary contains multiple identical keys: '{d}'."
@@ -53,7 +55,7 @@ def val_absolute_path(attribute: str, accept_none: bool = False):
     Check that a string attribute is an absolute path
     """
 
-    def val(string: Optional[str]) -> Optional[str]:
+    def val(cls, string: Optional[str]) -> Optional[str]:
         if string is None:
             if accept_none:
                 return string
@@ -75,7 +77,7 @@ def val_absolute_path(attribute: str, accept_none: bool = False):
 
 
 def val_unique_list(attribute: str):
-    def val(must_be_unique: Optional[list]) -> Optional[list]:
+    def val(cls, must_be_unique: Optional[list]) -> Optional[list]:
         if must_be_unique is not None:
             if len(set(must_be_unique)) != len(must_be_unique):
                 raise ValueError(f"`{attribute}` list has repetitions")

fractal_server/app/schemas/user.py

@@ -2,9 +2,10 @@ from typing import Optional
 
 from fastapi_users import schemas
 from pydantic import BaseModel
-from pydantic import Extra
+from pydantic import ConfigDict
 from pydantic import Field
-from pydantic import validator
+from pydantic import field_validator
+from pydantic import ValidationInfo
 
 from ._validators import val_unique_list
 from ._validators import valstr
@@ -41,12 +42,12 @@ class UserRead(schemas.BaseUser[int]):
         username:
     """
 
-    username: Optional[str]
+    username: Optional[str] = None
     group_ids_names: Optional[list[tuple[int, str]]] = None
     oauth_accounts: list[OAuthAccountRead]
 
 
-class UserUpdate(schemas.BaseUserUpdate, extra=Extra.forbid):
+class UserUpdate(schemas.BaseUserUpdate):
     """
     Schema for `User` update.
 
@@ -54,33 +55,35 @@ class UserUpdate(schemas.BaseUserUpdate, extra=Extra.forbid):
         username:
     """
 
-    username: Optional[str]
+    model_config = ConfigDict(extra="forbid")
+
+    username: Optional[str] = None
 
     # Validators
-    _username = validator("username", allow_reuse=True)(valstr("username"))
+    _username = field_validator("username")(classmethod(valstr("username")))
 
-    @validator(
+    @field_validator(
         "is_active",
         "is_verified",
         "is_superuser",
         "email",
         "password",
-        always=False,
     )
-    def cant_set_none(cls, v, field):
+    @classmethod
+    def cant_set_none(cls, v, info: ValidationInfo):
         if v is None:
-            raise ValueError(f"Cannot set {field.name}=None")
+            raise ValueError(f"Cannot set {info.field_name}=None")
         return v
 
 
-class UserUpdateStrict(BaseModel, extra=Extra.forbid):
+class UserUpdateStrict(BaseModel):
     """
     Schema for `User` self-editing.
 
     Attributes:
     """
 
-    pass
+    model_config = ConfigDict(extra="forbid")
 
 
 class UserCreate(schemas.BaseUserCreate):
@@ -91,21 +94,23 @@ class UserCreate(schemas.BaseUserCreate):
         username:
     """
 
-    username: Optional[str]
+    username: Optional[str] = None
 
     # Validators
 
-    _username = validator("username", allow_reuse=True)(valstr("username"))
+    _username = field_validator("username")(classmethod(valstr("username")))
 
 
-class UserUpdateGroups(BaseModel, extra=Extra.forbid):
+class UserUpdateGroups(BaseModel):
     """
     Schema for `POST /auth/users/{user_id}/set-groups/`
 
     """
 
-    group_ids: list[int] = Field(min_items=1)
+    model_config = ConfigDict(extra="forbid")
+
+    group_ids: list[int] = Field(min_length=1)
 
-    _group_ids = validator("group_ids", allow_reuse=True)(
-        val_unique_list("group_ids")
+    _group_ids = field_validator("group_ids")(
+        classmethod(val_unique_list("group_ids"))
     )

fractal_server/app/schemas/user_group.py

@@ -2,9 +2,11 @@ from datetime import datetime
 from typing import Optional
 
 from pydantic import BaseModel
-from pydantic import Extra
+from pydantic import ConfigDict
 from pydantic import Field
-from pydantic import validator
+from pydantic import field_serializer
+from pydantic import field_validator
+from pydantic.types import AwareDatetime
 
 from ._validators import val_absolute_path
 from ._validators import val_unique_list
@@ -32,12 +34,16 @@ class UserGroupRead(BaseModel):
 
     id: int
     name: str
-    timestamp_created: datetime
+    timestamp_created: AwareDatetime
     user_ids: Optional[list[int]] = None
     viewer_paths: list[str]
 
+    @field_serializer("timestamp_created")
+    def serialize_datetime(v: datetime) -> str:
+        return v.isoformat()
 
-class UserGroupCreate(BaseModel, extra=Extra.forbid):
+
+class UserGroupCreate(BaseModel):
     """
     Schema for `UserGroup` creation
 
@@ -45,27 +51,35 @@ class UserGroupCreate(BaseModel, extra=Extra.forbid):
         name: Group name
     """
 
+    model_config = ConfigDict(extra="forbid")
+
     name: str
     viewer_paths: list[str] = Field(default_factory=list)
 
-    @validator("viewer_paths")
+    @field_validator("viewer_paths")
+    @classmethod
     def viewer_paths_validator(cls, value):
         for i, path in enumerate(value):
-            value[i] = val_absolute_path(f"viewer_paths[{i}]")(path)
-        value = val_unique_list("viewer_paths")(value)
+            value[i] = val_absolute_path(f"viewer_paths[{i}]")(cls, path)
+        value = val_unique_list("viewer_paths")(cls, value)
         return value
 
 
-class UserGroupUpdate(BaseModel, extra=Extra.forbid):
+class UserGroupUpdate(BaseModel):
     """
     Schema for `UserGroup` update
     """
 
+    model_config = ConfigDict(extra="forbid")
+
     viewer_paths: Optional[list[str]] = None
 
-    @validator("viewer_paths")
+    @field_validator("viewer_paths")
+    @classmethod
     def viewer_paths_validator(cls, value):
+        if value is None:
+            raise ValueError("Cannot set `viewer_paths=None`.")
         for i, path in enumerate(value):
-            value[i] = val_absolute_path(f"viewer_paths[{i}]")(path)
-        value = val_unique_list("viewer_paths")(value)
+            value[i] = val_absolute_path(f"viewer_paths[{i}]")(cls, path)
+        value = val_unique_list("viewer_paths")(cls, value)
         return value

fractal_server/app/schemas/user_settings.py

@@ -1,8 +1,8 @@
 from typing import Optional
 
 from pydantic import BaseModel
-from pydantic import Extra
-from pydantic import validator
+from pydantic import ConfigDict
+from pydantic import field_validator
 from pydantic.types import StrictStr
 
 from ._validators import val_absolute_path
@@ -41,11 +41,13 @@ class UserSettingsReadStrict(BaseModel):
     project_dir: Optional[str] = None
 
 
-class UserSettingsUpdate(BaseModel, extra=Extra.forbid):
+class UserSettingsUpdate(BaseModel):
     """
     Schema reserved for superusers
     """
 
+    model_config = ConfigDict(extra="forbid")
+
     ssh_host: Optional[str] = None
     ssh_username: Optional[str] = None
     ssh_private_key_path: Optional[str] = None
@@ -55,46 +57,51 @@ class UserSettingsUpdate(BaseModel, extra=Extra.forbid):
     slurm_accounts: Optional[list[StrictStr]] = None
     project_dir: Optional[str] = None
 
-    _ssh_host = validator("ssh_host", allow_reuse=True)(
-        valstr("ssh_host", accept_none=True)
+    _ssh_host = field_validator("ssh_host")(
+        classmethod(valstr("ssh_host", accept_none=True))
+    )
+    _ssh_username = field_validator("ssh_username")(
+        classmethod(valstr("ssh_username", accept_none=True))
     )
-    _ssh_username = validator("ssh_username", allow_reuse=True)(
-        valstr("ssh_username", accept_none=True)
+    _ssh_private_key_path = field_validator("ssh_private_key_path")(
+        classmethod(
+            val_absolute_path("ssh_private_key_path", accept_none=True)
+        )
     )
-    _ssh_private_key_path = validator(
-        "ssh_private_key_path", allow_reuse=True
-    )(val_absolute_path("ssh_private_key_path", accept_none=True))
 
-    _ssh_tasks_dir = validator("ssh_tasks_dir", allow_reuse=True)(
-        val_absolute_path("ssh_tasks_dir", accept_none=True)
+    _ssh_tasks_dir = field_validator("ssh_tasks_dir")(
+        classmethod(val_absolute_path("ssh_tasks_dir", accept_none=True))
     )
-    _ssh_jobs_dir = validator("ssh_jobs_dir", allow_reuse=True)(
-        val_absolute_path("ssh_jobs_dir", accept_none=True)
+    _ssh_jobs_dir = field_validator("ssh_jobs_dir")(
+        classmethod(val_absolute_path("ssh_jobs_dir", accept_none=True))
     )
 
-    _slurm_user = validator("slurm_user", allow_reuse=True)(
-        valstr("slurm_user", accept_none=True)
+    _slurm_user = field_validator("slurm_user")(
+        classmethod(valstr("slurm_user", accept_none=True))
     )
 
-    @validator("slurm_accounts")
+    @field_validator("slurm_accounts")
+    @classmethod
     def slurm_accounts_validator(cls, value):
         if value is None:
             return value
         for i, item in enumerate(value):
-            value[i] = valstr(f"slurm_accounts[{i}]")(item)
-        return val_unique_list("slurm_accounts")(value)
+            value[i] = valstr(f"slurm_accounts[{i}]")(cls, item)
+        return val_unique_list("slurm_accounts")(cls, value)
 
-    @validator("project_dir")
+    @field_validator("project_dir")
+    @classmethod
     def project_dir_validator(cls, value):
         if value is None:
             return None
         validate_cmd(value)
-        return val_absolute_path("project_dir")(value)
+        return val_absolute_path("project_dir")(cls, value)
 
 
-class UserSettingsUpdateStrict(BaseModel, extra=Extra.forbid):
+class UserSettingsUpdateStrict(BaseModel):
+    model_config = ConfigDict(extra="forbid")
     slurm_accounts: Optional[list[StrictStr]] = None
 
-    _slurm_accounts = validator("slurm_accounts", allow_reuse=True)(
-        val_unique_list("slurm_accounts")
+    _slurm_accounts = field_validator("slurm_accounts")(
+        classmethod(val_unique_list("slurm_accounts"))
     )

fractal_server/app/schemas/v2/__init__.py

@@ -1,3 +1,4 @@
+from .accounting import AccountingRecordRead  # noqa F401
 from .dataset import DatasetCreateV2  # noqa F401
 from .dataset import DatasetExportV2  # noqa F401
 from .dataset import DatasetImportV2  # noqa F401

fractal_server/app/schemas/v2/accounting.py

@@ -0,0 +1,18 @@
+from datetime import datetime
+
+from pydantic import BaseModel
+from pydantic import field_serializer
+from pydantic.types import AwareDatetime
+
+
+class AccountingRecordRead(BaseModel):
+
+    id: int
+    user_id: int
+    timestamp: AwareDatetime
+    num_tasks: int
+    num_new_images: int
+
+    @field_serializer("timestamp")
+    def serialize_datetime(v: datetime) -> str:
+        return v.isoformat()