fractal-server 2.10.5__py3-none-any.whl → 2.11.0a2__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.10.5"
+__VERSION__ = "2.11.0a2"

fractal_server/app/models/v2/dataset.py

@@ -11,6 +11,7 @@ from sqlmodel import Relationship
 from sqlmodel import SQLModel
 
 from ....utils import get_timestamp
+from fractal_server.images.models import AttributeFiltersType
 
 
 class DatasetV2(SQLModel, table=True):
@@ -41,12 +42,14 @@ class DatasetV2(SQLModel, table=True):
         sa_column=Column(JSON, server_default="[]", nullable=False)
     )
 
-    filters: dict[Literal["attributes", "types"], dict[str, Any]] = Field(
-        sa_column=Column(
-            JSON,
-            nullable=False,
-            server_default='{"attributes": {}, "types": {}}',
-        )
+    filters: Optional[
+        dict[Literal["attributes", "types"], dict[str, Any]]
+    ] = Field(sa_column=Column(JSON, nullable=True, server_default="null"))
+    type_filters: dict[str, bool] = Field(
+        sa_column=Column(JSON, nullable=False, server_default="{}")
+    )
+    attribute_filters: AttributeFiltersType = Field(
+        sa_column=Column(JSON, nullable=False, server_default="{}")
     )
 
     @property

fractal_server/app/models/v2/job.py

@@ -10,6 +10,7 @@ from sqlmodel import SQLModel
 
 from ....utils import get_timestamp
 from ...schemas.v2 import JobStatusTypeV2
+from fractal_server.images.models import AttributeFiltersType
 
 
 class JobV2(SQLModel, table=True):
@@ -49,3 +50,7 @@ class JobV2(SQLModel, table=True):
     )
     status: str = JobStatusTypeV2.SUBMITTED
     log: Optional[str] = None
+
+    attribute_filters: AttributeFiltersType = Field(
+        sa_column=Column(JSON, nullable=False, server_default="{}")
+    )

fractal_server/app/models/v2/workflowtask.py

@@ -25,14 +25,11 @@ class WorkflowTaskV2(SQLModel, table=True):
     args_parallel: Optional[dict[str, Any]] = Field(sa_column=Column(JSON))
     args_non_parallel: Optional[dict[str, Any]] = Field(sa_column=Column(JSON))
 
-    input_filters: dict[
-        Literal["attributes", "types"], dict[str, Any]
-    ] = Field(
-        sa_column=Column(
-            JSON,
-            nullable=False,
-            server_default='{"attributes": {}, "types": {}}',
-        )
+    input_filters: Optional[
+        dict[Literal["attributes", "types"], dict[str, Any]]
+    ] = Field(sa_column=Column(JSON, nullable=True, server_default="null"))
+    type_filters: dict[str, bool] = Field(
+        sa_column=Column(JSON, nullable=False, server_default="{}")
     )
 
     # Task

fractal_server/app/routes/api/v1/dataset.py

@@ -17,7 +17,7 @@ from ....models.v1 import ApplyWorkflow
 from ....models.v1 import Dataset
 from ....models.v1 import Project
 from ....models.v1 import Resource
-from ....runner.filenames import HISTORY_FILENAME
+from ....runner.filenames import HISTORY_FILENAME_V1
 from ....schemas.v1 import DatasetCreateV1
 from ....schemas.v1 import DatasetReadV1
 from ....schemas.v1 import DatasetStatusReadV1
@@ -511,7 +511,7 @@ async def get_workflowtask_status(
         # Highest priority: Read status updates coming from the running-job
         # temporary file. Note: this file only contains information on
         # WorkflowTask's that ran through successfully
-        tmp_file = Path(running_job.working_dir) / HISTORY_FILENAME
+        tmp_file = Path(running_job.working_dir) / HISTORY_FILENAME_V1
         try:
             with tmp_file.open("r") as f:
                 history = json.load(f)

fractal_server/app/routes/api/v2/_aux_functions.py

@@ -21,7 +21,6 @@ from ....models.v2 import TaskV2
 from ....models.v2 import WorkflowTaskV2
 from ....models.v2 import WorkflowV2
 from ....schemas.v2 import JobStatusTypeV2
-from fractal_server.images import Filters
 
 
 async def _get_project_check_owner(
@@ -336,7 +335,7 @@ async def _workflow_insert_task(
     meta_non_parallel: Optional[dict[str, Any]] = None,
     args_non_parallel: Optional[dict[str, Any]] = None,
     args_parallel: Optional[dict[str, Any]] = None,
-    input_filters: Optional[Filters] = None,
+    type_filters: Optional[dict[str, bool]] = None,
     db: AsyncSession,
 ) -> WorkflowTaskV2:
     """
@@ -350,7 +349,7 @@ async def _workflow_insert_task(
         meta_non_parallel:
         args_non_parallel:
         args_parallel:
-        input_filters:
+        type_filters:
         db:
     """
     db_workflow = await db.get(WorkflowV2, workflow_id)
@@ -376,12 +375,6 @@ async def _workflow_insert_task(
     if final_meta_non_parallel == {}:
         final_meta_non_parallel = None
 
-    # Prepare input_filters attribute
-    if input_filters is None:
-        input_filters_kwarg = {}
-    else:
-        input_filters_kwarg = dict(input_filters=input_filters)
-
     # Create DB entry
     wf_task = WorkflowTaskV2(
         task_type=task_type,
@@ -390,7 +383,7 @@ async def _workflow_insert_task(
         args_parallel=args_parallel,
         meta_parallel=final_meta_parallel,
         meta_non_parallel=final_meta_non_parallel,
-        **input_filters_kwarg,
+        type_filters=(type_filters or dict()),
     )
     db_workflow.task_list.append(wf_task)
     flag_modified(db_workflow, "task_list")

fractal_server/app/routes/api/v2/images.py

@@ -8,6 +8,8 @@ from fastapi import Response
 from fastapi import status
 from pydantic import BaseModel
 from pydantic import Field
+from pydantic import root_validator
+from pydantic import validator
 from sqlalchemy.orm.attributes import flag_modified
 
 from ._aux_functions import _get_dataset_check_owner
@@ -15,9 +17,14 @@ from fractal_server.app.db import AsyncSession
 from fractal_server.app.db import get_async_db
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_user
-from fractal_server.images import Filters
+from fractal_server.app.schemas._filter_validators import (
+    validate_attribute_filters,
+)
+from fractal_server.app.schemas._filter_validators import validate_type_filters
+from fractal_server.app.schemas._validators import root_validate_dict_keys
 from fractal_server.images import SingleImage
 from fractal_server.images import SingleImageUpdate
+from fractal_server.images.models import AttributeFiltersType
 from fractal_server.images.tools import find_image_by_zarr_url
 from fractal_server.images.tools import match_filter
 
@@ -38,7 +45,18 @@ class ImagePage(BaseModel):
 
 class ImageQuery(BaseModel):
     zarr_url: Optional[str]
-    filters: Filters = Field(default_factory=Filters)
+    type_filters: dict[str, bool] = Field(default_factory=dict)
+    attribute_filters: AttributeFiltersType = Field(default_factory=dict)
+
+    _dict_keys = root_validator(pre=True, allow_reuse=True)(
+        root_validate_dict_keys
+    )
+    _type_filters = validator("type_filters", allow_reuse=True)(
+        validate_type_filters
+    )
+    _attribute_filters = validator("attribute_filters", allow_reuse=True)(
+        validate_attribute_filters
+    )
 
 
 @router.post(
@@ -124,7 +142,11 @@ async def query_dataset_images(
         images = [
             image
             for image in images
-            if match_filter(image, Filters(**dataset.filters))
+            if match_filter(
+                image=image,
+                type_filters=dataset.type_filters,
+                attribute_filters=dataset.attribute_filters,
+            )
         ]
 
     attributes = {}
@@ -154,13 +176,14 @@ async def query_dataset_images(
             else:
                 images = [image]
 
-        if query.filters.attributes or query.filters.types:
+        if query.attribute_filters or query.type_filters:
             images = [
                 image
                 for image in images
                 if match_filter(
-                    image,
-                    Filters(**query.filters.dict()),
+                    image=image,
+                    type_filters=query.type_filters,
+                    attribute_filters=query.attribute_filters,
                 )
             ]
 

fractal_server/app/routes/api/v2/status.py

@@ -1,5 +1,3 @@
-import json
-from pathlib import Path
 from typing import Optional
 
 from fastapi import APIRouter
@@ -18,7 +16,6 @@ from ._aux_functions import _get_submitted_jobs_statement
 from ._aux_functions import _get_workflow_check_owner
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_user
-from fractal_server.app.runner.filenames import HISTORY_FILENAME
 
 router = APIRouter()
 
@@ -98,8 +95,8 @@ async def get_workflowtask_status(
     if running_job is None:
         # If no job is running, the chronological-last history item is also the
         # positional-last workflow task to be included in the response.
-        if len(dataset.history) > 0:
-            last_valid_wftask_id = dataset.history[-1]["workflowtask"]["id"]
+        if len(history) > 0:
+            last_valid_wftask_id = history[-1]["workflowtask"]["id"]
         else:
             last_valid_wftask_id = None
     else:
@@ -109,7 +106,24 @@ async def get_workflowtask_status(
         # as "submitted"
         start = running_job.first_task_index
         end = running_job.last_task_index + 1
-        for wftask in workflow.task_list[start:end]:
+
+        running_job_wftasks = workflow.task_list[start:end]
+        running_job_statuses = [
+            workflow_tasks_status_dict.get(wft.id, None)
+            for wft in running_job_wftasks
+        ]
+        try:
+            first_submitted_index = running_job_statuses.index(
+                WorkflowTaskStatusTypeV2.SUBMITTED
+            )
+        except ValueError:
+            logger.warning(
+                f"Job {running_job.id} is submitted but its task list does "
+                f"not contain a {WorkflowTaskStatusTypeV2.SUBMITTED} task."
+            )
+            first_submitted_index = 0
+
+        for wftask in running_job_wftasks[first_submitted_index:]:
             workflow_tasks_status_dict[
                 wftask.id
             ] = WorkflowTaskStatusTypeV2.SUBMITTED
@@ -133,20 +147,6 @@ async def get_workflowtask_status(
             last_valid_wftask_id = None
             logger.warning(f"Now setting {last_valid_wftask_id=}.")
 
-        # Highest priority: Read status updates coming from the running-job
-        # temporary file. Note: this file only contains information on
-        # WorkflowTask's that ran through successfully.
-        tmp_file = Path(running_job.working_dir) / HISTORY_FILENAME
-        try:
-            with tmp_file.open("r") as f:
-                history = json.load(f)
-        except FileNotFoundError:
-            history = []
-        for history_item in history:
-            wftask_id = history_item["workflowtask"]["id"]
-            wftask_status = history_item["status"]
-            workflow_tasks_status_dict[wftask_id] = wftask_status
-
     # Based on previously-gathered information, clean up the response body
     clean_workflow_tasks_status_dict = {}
     for wf_task in workflow.task_list:

fractal_server/app/routes/api/v2/submit.py

@@ -159,7 +159,11 @@ async def apply_workflow(
         dataset_id=dataset_id,
         workflow_id=workflow_id,
         user_email=user.email,
-        dataset_dump=json.loads(dataset.json(exclude={"images", "history"})),
+        # The 'filters' field is not supported any more but still exists as a
+        # database column, therefore we manually exclude it from dumps.
+        dataset_dump=json.loads(
+            dataset.json(exclude={"images", "history", "filters"})
+        ),
         workflow_dump=json.loads(workflow.json(exclude={"task_list"})),
         project_dump=json.loads(project.json(exclude={"user_list"})),
         **job_create.dict(),

fractal_server/app/routes/api/v2/workflowtask.py

@@ -109,7 +109,7 @@ async def replace_workflowtask(
         task_type=task.type,
         task=task,
         # old-task values
-        input_filters=old_workflow_task.input_filters,
+        type_filters=old_workflow_task.type_filters,
         # possibly new values
         args_non_parallel=_args_non_parallel,
         args_parallel=_args_parallel,
@@ -183,7 +183,7 @@ async def create_workflowtask(
         meta_parallel=new_task.meta_parallel,
         args_non_parallel=new_task.args_non_parallel,
         args_parallel=new_task.args_parallel,
-        input_filters=new_task.input_filters,
+        type_filters=new_task.type_filters,
         db=db,
     )
 
@@ -274,7 +274,7 @@ async def update_workflowtask(
             if not actual_args:
                 actual_args = None
             setattr(db_wf_task, key, actual_args)
-        elif key in ["meta_parallel", "meta_non_parallel", "input_filters"]:
+        elif key in ["meta_parallel", "meta_non_parallel", "type_filters"]:
             setattr(db_wf_task, key, value)
         else:
             raise HTTPException(

fractal_server/app/runner/filenames.py

@@ -1,6 +1,4 @@
-HISTORY_FILENAME = "history.json"
-FILTERS_FILENAME = "filters.json"
-IMAGES_FILENAME = "images.json"
-METADATA_FILENAME = "metadata.json"
+HISTORY_FILENAME_V1 = "history.json"
+METADATA_FILENAME_V1 = "metadata.json"
 SHUTDOWN_FILENAME = "shutdown"
 WORKFLOW_LOG_FILENAME = "workflow.log"

fractal_server/app/runner/v1/_common.py

@@ -28,8 +28,8 @@ from ..exceptions import JobExecutionError
 from ..exceptions import TaskExecutionError
 from .common import TaskParameters
 from .common import write_args_file
-from fractal_server.app.runner.filenames import HISTORY_FILENAME
-from fractal_server.app.runner.filenames import METADATA_FILENAME
+from fractal_server.app.runner.filenames import HISTORY_FILENAME_V1
+from fractal_server.app.runner.filenames import METADATA_FILENAME_V1
 from fractal_server.app.runner.task_files import get_task_file_paths
 from fractal_server.string_tools import validate_cmd
 
@@ -610,11 +610,11 @@ def execute_tasks(
         )
 
         # Write most recent metadata to METADATA_FILENAME
-        with open(workflow_dir_local / METADATA_FILENAME, "w") as f:
+        with open(workflow_dir_local / METADATA_FILENAME_V1, "w") as f:
             json.dump(current_task_pars.metadata, f, indent=2)
 
         # Write most recent metadata to HISTORY_FILENAME
-        with open(workflow_dir_local / HISTORY_FILENAME, "w") as f:
+        with open(workflow_dir_local / HISTORY_FILENAME_V1, "w") as f:
             json.dump(current_task_pars.history, f, indent=2)
 
     return current_task_pars

fractal_server/app/runner/v1/handle_failed_job.py

@@ -24,8 +24,8 @@ from ...models.v1 import Dataset
 from ...models.v1 import Workflow
 from ...models.v1 import WorkflowTask
 from ...schemas.v1 import WorkflowTaskStatusTypeV1
-from ..filenames import HISTORY_FILENAME
-from ..filenames import METADATA_FILENAME
+from ..filenames import HISTORY_FILENAME_V1
+from ..filenames import METADATA_FILENAME_V1
 
 
 def assemble_history_failed_job(
@@ -64,7 +64,7 @@ def assemble_history_failed_job(
     new_history = output_dataset.history
 
     # Part 2: Extend history based on tmp_metadata_file
-    tmp_history_file = Path(job.working_dir) / HISTORY_FILENAME
+    tmp_history_file = Path(job.working_dir) / HISTORY_FILENAME_V1
     try:
         with tmp_history_file.open("r") as f:
             tmp_file_history = json.load(f)
@@ -129,7 +129,7 @@ def assemble_meta_failed_job(
     """
 
     new_meta = deepcopy(output_dataset.meta)
-    metadata_file = Path(job.working_dir) / METADATA_FILENAME
+    metadata_file = Path(job.working_dir) / METADATA_FILENAME_V1
     try:
         with metadata_file.open("r") as f:
             metadata_update = json.load(f)

fractal_server/app/runner/v2/__init__.py

@@ -11,7 +11,6 @@ from pathlib import Path
 from typing import Optional
 
 from sqlalchemy.orm import Session as DBSyncSession
-from sqlalchemy.orm.attributes import flag_modified
 
 from ....config import get_settings
 from ....logger import get_logger
@@ -24,7 +23,6 @@ from ....zip_tools import _zip_folder_to_file_and_remove
 from ...db import DB
 from ...models.v2 import DatasetV2
 from ...models.v2 import JobV2
-from ...models.v2 import WorkflowTaskV2
 from ...models.v2 import WorkflowV2
 from ...schemas.v2 import JobStatusTypeV2
 from ..exceptions import JobExecutionError
@@ -38,12 +36,11 @@ from ._local_experimental import (
 )
 from ._slurm_ssh import process_workflow as slurm_ssh_process_workflow
 from ._slurm_sudo import process_workflow as slurm_sudo_process_workflow
-from .handle_failed_job import assemble_filters_failed_job
-from .handle_failed_job import assemble_history_failed_job
-from .handle_failed_job import assemble_images_failed_job
+from .handle_failed_job import mark_last_wftask_as_failed
 from fractal_server import __VERSION__
 from fractal_server.app.models import UserSettings
 
+
 _backends = {}
 _backends["local"] = local_process_workflow
 _backends["slurm"] = slurm_sudo_process_workflow
@@ -115,7 +112,6 @@ async def submit_workflow(
     logger = set_logger(logger_name=logger_name)
 
     with next(DB.get_sync_db()) as db_sync:
-
         try:
             job: Optional[JobV2] = db_sync.get(JobV2, job_id)
             dataset: Optional[DatasetV2] = db_sync.get(DatasetV2, dataset_id)
@@ -322,7 +318,7 @@ async def submit_workflow(
         db_sync = next(DB.get_sync_db())
         db_sync.close()
 
-        new_dataset_attributes = await process_workflow(
+        await process_workflow(
             workflow=workflow,
             dataset=dataset,
             workflow_dir_local=WORKFLOW_DIR_LOCAL,
@@ -331,6 +327,7 @@ async def submit_workflow(
             worker_init=worker_init,
             first_task_index=job.first_task_index,
             last_task_index=job.last_task_index,
+            job_attribute_filters=job.attribute_filters,
             **backend_specific_kwargs,
         )
 
@@ -340,14 +337,6 @@ async def submit_workflow(
         )
         logger.debug(f'END workflow "{workflow.name}"')
 
-        # Update dataset attributes, in case of successful execution
-        dataset.history.extend(new_dataset_attributes["history"])
-        dataset.filters = new_dataset_attributes["filters"]
-        dataset.images = new_dataset_attributes["images"]
-        for attribute_name in ["filters", "history", "images"]:
-            flag_modified(dataset, attribute_name)
-        db_sync.merge(dataset)
-
         # Update job DB entry
         job.status = JobStatusTypeV2.DONE
         job.end_timestamp = get_timestamp()
@@ -358,28 +347,13 @@ async def submit_workflow(
         db_sync.commit()
 
     except TaskExecutionError as e:
-
         logger.debug(f'FAILED workflow "{workflow.name}", TaskExecutionError.')
         logger.info(f'Workflow "{workflow.name}" failed (TaskExecutionError).')
 
-        # Read dataset attributes produced by the last successful task, and
-        # update the DB dataset accordingly
-        failed_wftask = db_sync.get(WorkflowTaskV2, e.workflow_task_id)
-        dataset.history = assemble_history_failed_job(
-            job,
-            dataset,
-            workflow,
+        mark_last_wftask_as_failed(
+            dataset_id=dataset_id,
             logger_name=logger_name,
-            failed_wftask=failed_wftask,
         )
-        latest_filters = assemble_filters_failed_job(job)
-        if latest_filters is not None:
-            dataset.filters = latest_filters
-        latest_images = assemble_images_failed_job(job)
-        if latest_images is not None:
-            dataset.images = latest_images
-        db_sync.merge(dataset)
-
         exception_args_string = "\n".join(e.args)
         log_msg = (
             f"TASK ERROR: "
@@ -390,26 +364,12 @@ async def submit_workflow(
         fail_job(db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name)
 
     except JobExecutionError as e:
-
         logger.debug(f'FAILED workflow "{workflow.name}", JobExecutionError.')
         logger.info(f'Workflow "{workflow.name}" failed (JobExecutionError).')
-
-        # Read dataset attributes produced by the last successful task, and
-        # update the DB dataset accordingly
-        dataset.history = assemble_history_failed_job(
-            job,
-            dataset,
-            workflow,
+        mark_last_wftask_as_failed(
+            dataset_id=dataset_id,
             logger_name=logger_name,
         )
-        latest_filters = assemble_filters_failed_job(job)
-        if latest_filters is not None:
-            dataset.filters = latest_filters
-        latest_images = assemble_images_failed_job(job)
-        if latest_images is not None:
-            dataset.images = latest_images
-        db_sync.merge(dataset)
-
         fail_job(
             db=db_sync,
             job=job,
@@ -421,27 +381,13 @@ async def submit_workflow(
         )
 
     except Exception:
-
         logger.debug(f'FAILED workflow "{workflow.name}", unknown error.')
         logger.info(f'Workflow "{workflow.name}" failed (unkwnon error).')
-
-        current_traceback = traceback.format_exc()
-
-        # Read dataset attributes produced by the last successful task, and
-        # update the DB dataset accordingly
-        dataset.history = assemble_history_failed_job(
-            job,
-            dataset,
-            workflow,
+        mark_last_wftask_as_failed(
+            dataset_id=dataset_id,
             logger_name=logger_name,
         )
-        latest_filters = assemble_filters_failed_job(job)
-        if latest_filters is not None:
-            dataset.filters = latest_filters
-        latest_images = assemble_images_failed_job(job)
-        if latest_images is not None:
-            dataset.images = latest_images
-        db_sync.merge(dataset)
+        current_traceback = traceback.format_exc()
         fail_job(
             db=db_sync,
             job=job,

fractal_server/app/runner/v2/_local/__init__.py

@@ -29,6 +29,7 @@ from ...set_start_and_last_task_index import set_start_and_last_task_index
 from ..runner import execute_tasks_v2
 from ._submit_setup import _local_submit_setup
 from .executor import FractalThreadPoolExecutor
+from fractal_server.images.models import AttributeFiltersType
 
 
 def _process_workflow(
@@ -39,26 +40,24 @@ def _process_workflow(
     workflow_dir_local: Path,
     first_task_index: int,
     last_task_index: int,
-) -> dict:
+    job_attribute_filters: AttributeFiltersType,
+) -> None:
     """
-    Internal processing routine
-
-    Schedules the workflow using a `FractalThreadPoolExecutor`.
+    Run the workflow using a `FractalThreadPoolExecutor`.
     """
-
     with FractalThreadPoolExecutor() as executor:
-        new_dataset_attributes = execute_tasks_v2(
+        execute_tasks_v2(
             wf_task_list=workflow.task_list[
-                first_task_index : (last_task_index + 1)  # noqa
-            ],  # noqa
+                first_task_index : (last_task_index + 1)
+            ],
             dataset=dataset,
             executor=executor,
             workflow_dir_local=workflow_dir_local,
             workflow_dir_remote=workflow_dir_local,
             logger_name=logger_name,
             submit_setup_call=_local_submit_setup,
+            job_attribute_filters=job_attribute_filters,
         )
-    return new_dataset_attributes
 
 
 async def process_workflow(
@@ -70,12 +69,13 @@ async def process_workflow(
     first_task_index: Optional[int] = None,
     last_task_index: Optional[int] = None,
     logger_name: str,
+    job_attribute_filters: AttributeFiltersType,
     # Slurm-specific
     user_cache_dir: Optional[str] = None,
     slurm_user: Optional[str] = None,
     slurm_account: Optional[str] = None,
     worker_init: Optional[str] = None,
-) -> dict:
+) -> None:
     """
     Run a workflow
 
@@ -127,11 +127,6 @@ async def process_workflow(
                             (positive exit codes).
         JobExecutionError: wrapper for errors raised by the tasks' executors
                            (negative exit codes).
-
-    Returns:
-        output_dataset_metadata:
-            The updated metadata for the dataset, as returned by the last task
-            of the workflow
     """
 
     if workflow_dir_remote and (workflow_dir_remote != workflow_dir_local):
@@ -148,12 +143,12 @@ async def process_workflow(
         last_task_index=last_task_index,
     )
 
-    new_dataset_attributes = await async_wrap(_process_workflow)(
+    await async_wrap(_process_workflow)(
         workflow=workflow,
         dataset=dataset,
         logger_name=logger_name,
         workflow_dir_local=workflow_dir_local,
         first_task_index=first_task_index,
         last_task_index=last_task_index,
+        job_attribute_filters=job_attribute_filters,
     )
-    return new_dataset_attributes