fractal-server 1.4.6__py3-none-any.whl → 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. fractal_server/__init__.py +1 -1
  2. fractal_server/app/db/__init__.py +0 -1
  3. fractal_server/app/models/__init__.py +6 -8
  4. fractal_server/app/models/linkuserproject.py +9 -0
  5. fractal_server/app/models/security.py +6 -0
  6. fractal_server/app/models/v1/__init__.py +12 -0
  7. fractal_server/app/models/{dataset.py → v1/dataset.py} +5 -5
  8. fractal_server/app/models/{job.py → v1/job.py} +5 -5
  9. fractal_server/app/models/{project.py → v1/project.py} +5 -5
  10. fractal_server/app/models/{state.py → v1/state.py} +2 -2
  11. fractal_server/app/models/{task.py → v1/task.py} +7 -2
  12. fractal_server/app/models/{workflow.py → v1/workflow.py} +5 -5
  13. fractal_server/app/models/v2/__init__.py +22 -0
  14. fractal_server/app/models/v2/collection_state.py +21 -0
  15. fractal_server/app/models/v2/dataset.py +54 -0
  16. fractal_server/app/models/v2/job.py +51 -0
  17. fractal_server/app/models/v2/project.py +30 -0
  18. fractal_server/app/models/v2/task.py +93 -0
  19. fractal_server/app/models/v2/workflow.py +35 -0
  20. fractal_server/app/models/v2/workflowtask.py +49 -0
  21. fractal_server/app/routes/admin/__init__.py +0 -0
  22. fractal_server/app/routes/{admin.py → admin/v1.py} +42 -42
  23. fractal_server/app/routes/admin/v2.py +309 -0
  24. fractal_server/app/routes/api/v1/__init__.py +7 -7
  25. fractal_server/app/routes/api/v1/_aux_functions.py +8 -8
  26. fractal_server/app/routes/api/v1/dataset.py +48 -41
  27. fractal_server/app/routes/api/v1/job.py +14 -14
  28. fractal_server/app/routes/api/v1/project.py +30 -27
  29. fractal_server/app/routes/api/v1/task.py +26 -16
  30. fractal_server/app/routes/api/v1/task_collection.py +28 -16
  31. fractal_server/app/routes/api/v1/workflow.py +28 -28
  32. fractal_server/app/routes/api/v1/workflowtask.py +11 -11
  33. fractal_server/app/routes/api/v2/__init__.py +34 -0
  34. fractal_server/app/routes/api/v2/_aux_functions.py +502 -0
  35. fractal_server/app/routes/api/v2/dataset.py +293 -0
  36. fractal_server/app/routes/api/v2/images.py +279 -0
  37. fractal_server/app/routes/api/v2/job.py +200 -0
  38. fractal_server/app/routes/api/v2/project.py +186 -0
  39. fractal_server/app/routes/api/v2/status.py +150 -0
  40. fractal_server/app/routes/api/v2/submit.py +210 -0
  41. fractal_server/app/routes/api/v2/task.py +222 -0
  42. fractal_server/app/routes/api/v2/task_collection.py +239 -0
  43. fractal_server/app/routes/api/v2/task_legacy.py +59 -0
  44. fractal_server/app/routes/api/v2/workflow.py +380 -0
  45. fractal_server/app/routes/api/v2/workflowtask.py +265 -0
  46. fractal_server/app/routes/aux/_job.py +2 -2
  47. fractal_server/app/runner/__init__.py +0 -379
  48. fractal_server/app/runner/async_wrap.py +27 -0
  49. fractal_server/app/runner/components.py +5 -0
  50. fractal_server/app/runner/exceptions.py +129 -0
  51. fractal_server/app/runner/executors/__init__.py +0 -0
  52. fractal_server/app/runner/executors/slurm/__init__.py +3 -0
  53. fractal_server/app/runner/{_slurm → executors/slurm}/_batching.py +1 -1
  54. fractal_server/app/runner/executors/slurm/_check_jobs_status.py +72 -0
  55. fractal_server/app/runner/{_slurm → executors/slurm}/_executor_wait_thread.py +3 -4
  56. fractal_server/app/runner/{_slurm → executors/slurm}/_slurm_config.py +3 -152
  57. fractal_server/app/runner/{_slurm → executors/slurm}/_subprocess_run_as_user.py +42 -1
  58. fractal_server/app/runner/{_slurm → executors/slurm}/executor.py +46 -27
  59. fractal_server/app/runner/filenames.py +6 -0
  60. fractal_server/app/runner/set_start_and_last_task_index.py +39 -0
  61. fractal_server/app/runner/task_files.py +103 -0
  62. fractal_server/app/runner/v1/__init__.py +366 -0
  63. fractal_server/app/runner/{_common.py → v1/_common.py} +56 -111
  64. fractal_server/app/runner/{_local → v1/_local}/__init__.py +5 -4
  65. fractal_server/app/runner/{_local → v1/_local}/_local_config.py +6 -7
  66. fractal_server/app/runner/{_local → v1/_local}/_submit_setup.py +1 -5
  67. fractal_server/app/runner/v1/_slurm/__init__.py +312 -0
  68. fractal_server/app/runner/{_slurm → v1/_slurm}/_submit_setup.py +5 -11
  69. fractal_server/app/runner/v1/_slurm/get_slurm_config.py +163 -0
  70. fractal_server/app/runner/v1/common.py +117 -0
  71. fractal_server/app/runner/{handle_failed_job.py → v1/handle_failed_job.py} +8 -8
  72. fractal_server/app/runner/v2/__init__.py +336 -0
  73. fractal_server/app/runner/v2/_local/__init__.py +162 -0
  74. fractal_server/app/runner/v2/_local/_local_config.py +118 -0
  75. fractal_server/app/runner/v2/_local/_submit_setup.py +52 -0
  76. fractal_server/app/runner/v2/_local/executor.py +100 -0
  77. fractal_server/app/runner/{_slurm → v2/_slurm}/__init__.py +38 -47
  78. fractal_server/app/runner/v2/_slurm/_submit_setup.py +82 -0
  79. fractal_server/app/runner/v2/_slurm/get_slurm_config.py +182 -0
  80. fractal_server/app/runner/v2/deduplicate_list.py +23 -0
  81. fractal_server/app/runner/v2/handle_failed_job.py +165 -0
  82. fractal_server/app/runner/v2/merge_outputs.py +38 -0
  83. fractal_server/app/runner/v2/runner.py +343 -0
  84. fractal_server/app/runner/v2/runner_functions.py +374 -0
  85. fractal_server/app/runner/v2/runner_functions_low_level.py +130 -0
  86. fractal_server/app/runner/v2/task_interface.py +62 -0
  87. fractal_server/app/runner/v2/v1_compat.py +31 -0
  88. fractal_server/app/schemas/__init__.py +1 -42
  89. fractal_server/app/schemas/_validators.py +28 -5
  90. fractal_server/app/schemas/v1/__init__.py +36 -0
  91. fractal_server/app/schemas/{applyworkflow.py → v1/applyworkflow.py} +18 -18
  92. fractal_server/app/schemas/{dataset.py → v1/dataset.py} +30 -30
  93. fractal_server/app/schemas/{dumps.py → v1/dumps.py} +8 -8
  94. fractal_server/app/schemas/{manifest.py → v1/manifest.py} +5 -5
  95. fractal_server/app/schemas/{project.py → v1/project.py} +9 -9
  96. fractal_server/app/schemas/{task.py → v1/task.py} +12 -12
  97. fractal_server/app/schemas/{task_collection.py → v1/task_collection.py} +7 -7
  98. fractal_server/app/schemas/{workflow.py → v1/workflow.py} +38 -38
  99. fractal_server/app/schemas/v2/__init__.py +37 -0
  100. fractal_server/app/schemas/v2/dataset.py +126 -0
  101. fractal_server/app/schemas/v2/dumps.py +87 -0
  102. fractal_server/app/schemas/v2/job.py +114 -0
  103. fractal_server/app/schemas/v2/manifest.py +159 -0
  104. fractal_server/app/schemas/v2/project.py +34 -0
  105. fractal_server/app/schemas/v2/status.py +16 -0
  106. fractal_server/app/schemas/v2/task.py +151 -0
  107. fractal_server/app/schemas/v2/task_collection.py +109 -0
  108. fractal_server/app/schemas/v2/workflow.py +79 -0
  109. fractal_server/app/schemas/v2/workflowtask.py +208 -0
  110. fractal_server/config.py +13 -10
  111. fractal_server/images/__init__.py +4 -0
  112. fractal_server/images/models.py +136 -0
  113. fractal_server/images/tools.py +84 -0
  114. fractal_server/main.py +11 -3
  115. fractal_server/migrations/env.py +0 -2
  116. fractal_server/migrations/versions/5bf02391cfef_v2.py +245 -0
  117. fractal_server/tasks/__init__.py +0 -5
  118. fractal_server/tasks/endpoint_operations.py +13 -19
  119. fractal_server/tasks/utils.py +35 -0
  120. fractal_server/tasks/{_TaskCollectPip.py → v1/_TaskCollectPip.py} +3 -3
  121. fractal_server/tasks/v1/__init__.py +0 -0
  122. fractal_server/tasks/{background_operations.py → v1/background_operations.py} +20 -52
  123. fractal_server/tasks/v1/get_collection_data.py +14 -0
  124. fractal_server/tasks/v2/_TaskCollectPip.py +103 -0
  125. fractal_server/tasks/v2/__init__.py +0 -0
  126. fractal_server/tasks/v2/background_operations.py +381 -0
  127. fractal_server/tasks/v2/get_collection_data.py +14 -0
  128. fractal_server/urls.py +13 -0
  129. {fractal_server-1.4.6.dist-info → fractal_server-2.0.0.dist-info}/METADATA +11 -12
  130. fractal_server-2.0.0.dist-info/RECORD +169 -0
  131. fractal_server/app/runner/_slurm/.gitignore +0 -2
  132. fractal_server/app/runner/common.py +0 -307
  133. fractal_server/app/schemas/json_schemas/manifest.json +0 -81
  134. fractal_server-1.4.6.dist-info/RECORD +0 -97
  135. /fractal_server/app/runner/{_slurm → executors/slurm}/remote.py +0 -0
  136. /fractal_server/app/runner/{_local → v1/_local}/executor.py +0 -0
  137. {fractal_server-1.4.6.dist-info → fractal_server-2.0.0.dist-info}/LICENSE +0 -0
  138. {fractal_server-1.4.6.dist-info → fractal_server-2.0.0.dist-info}/WHEEL +0 -0
  139. {fractal_server-1.4.6.dist-info → fractal_server-2.0.0.dist-info}/entry_points.txt +0 -0
@@ -1 +1 @@
1
- __VERSION__ = "1.4.6"
1
+ __VERSION__ = "2.0.0"
@@ -18,7 +18,6 @@ from ...logger import set_logger
18
18
  from ...syringe import Inject
19
19
 
20
20
 
21
- print(__name__)
22
21
  logger = set_logger(__name__)
23
22
 
24
23
  SQLITE_WARNING_MESSAGE = (
@@ -1,11 +1,9 @@
1
1
  """
2
2
  `models` module
3
3
  """
4
- from ..schemas import * # noqa F401
5
- from .dataset import * # noqa: F403, F401
6
- from .job import * # noqa: F403, F401
7
- from .project import * # noqa: F403, F401
8
- from .security import * # noqa: F403, F401
9
- from .state import State # noqa: F401
10
- from .task import * # noqa: F403, F401
11
- from .workflow import * # noqa: F401, F403
4
+ from .security import * # noqa: F401, F403
5
+ from .v1 import Project # noqa: F401
6
+ from .v2 import ProjectV2 # noqa: F401
7
+
8
+ # We include the project models to avoid issues with LinkUserProject
9
+ # (sometimes taking place in alembic autogenerate)
@@ -9,3 +9,12 @@ class LinkUserProject(SQLModel, table=True):
9
9
 
10
10
  project_id: int = Field(foreign_key="project.id", primary_key=True)
11
11
  user_id: int = Field(foreign_key="user_oauth.id", primary_key=True)
12
+
13
+
14
+ class LinkUserProjectV2(SQLModel, table=True):
15
+ """
16
+ Crossing table between User and ProjectV2
17
+ """
18
+
19
+ project_id: int = Field(foreign_key="projectv2.id", primary_key=True)
20
+ user_id: int = Field(foreign_key="user_oauth.id", primary_key=True)
@@ -19,6 +19,7 @@ from sqlmodel import Relationship
19
19
  from sqlmodel import SQLModel
20
20
 
21
21
  from .linkuserproject import LinkUserProject
22
+ from .linkuserproject import LinkUserProjectV2
22
23
 
23
24
 
24
25
  class OAuthAccount(SQLModel, table=True):
@@ -107,6 +108,11 @@ class UserOAuth(SQLModel, table=True):
107
108
  link_model=LinkUserProject,
108
109
  sa_relationship_kwargs={"lazy": "selectin"},
109
110
  )
111
+ project_list_v2: list["ProjectV2"] = Relationship( # noqa
112
+ back_populates="user_list",
113
+ link_model=LinkUserProjectV2,
114
+ sa_relationship_kwargs={"lazy": "selectin"},
115
+ )
110
116
 
111
117
  class Config:
112
118
  orm_mode = True
@@ -0,0 +1,12 @@
1
+ """
2
+ `models` module
3
+ """
4
+ from .dataset import Dataset # noqa: F401
5
+ from .dataset import Resource # noqa: F401
6
+ from .job import ApplyWorkflow # noqa: F403, F401
7
+ from .job import JobStatusTypeV1 # noqa: F401, F403
8
+ from .project import Project # noqa: F403, F401
9
+ from .state import State # noqa: F403, F401
10
+ from .task import Task # noqa: F403, F401
11
+ from .workflow import Workflow # noqa: F401, F403
12
+ from .workflow import WorkflowTask # noqa: F401, F403
@@ -10,17 +10,17 @@ from sqlmodel import Field
10
10
  from sqlmodel import Relationship
11
11
  from sqlmodel import SQLModel
12
12
 
13
- from ...utils import get_timestamp
14
- from ..schemas.dataset import _DatasetBase
15
- from ..schemas.dataset import _ResourceBase
13
+ from ....utils import get_timestamp
14
+ from ...schemas.v1.dataset import _DatasetBaseV1
15
+ from ...schemas.v1.dataset import _ResourceBaseV1
16
16
 
17
17
 
18
- class Resource(_ResourceBase, SQLModel, table=True):
18
+ class Resource(_ResourceBaseV1, SQLModel, table=True):
19
19
  id: Optional[int] = Field(default=None, primary_key=True)
20
20
  dataset_id: int = Field(foreign_key="dataset.id")
21
21
 
22
22
 
23
- class Dataset(_DatasetBase, SQLModel, table=True):
23
+ class Dataset(_DatasetBaseV1, SQLModel, table=True):
24
24
  """
25
25
  Represent a dataset
26
26
 
@@ -8,12 +8,12 @@ from sqlalchemy.types import JSON
8
8
  from sqlmodel import Field
9
9
  from sqlmodel import SQLModel
10
10
 
11
- from ...utils import get_timestamp
12
- from ..schemas import JobStatusType
13
- from ..schemas.applyworkflow import _ApplyWorkflowBase
11
+ from ....utils import get_timestamp
12
+ from ...schemas.v1 import JobStatusTypeV1
13
+ from ...schemas.v1.applyworkflow import _ApplyWorkflowBaseV1
14
14
 
15
15
 
16
- class ApplyWorkflow(_ApplyWorkflowBase, SQLModel, table=True):
16
+ class ApplyWorkflow(_ApplyWorkflowBaseV1, SQLModel, table=True):
17
17
  """
18
18
  Represent a workflow run
19
19
 
@@ -97,5 +97,5 @@ class ApplyWorkflow(_ApplyWorkflowBase, SQLModel, table=True):
97
97
  end_timestamp: Optional[datetime] = Field(
98
98
  default=None, sa_column=Column(DateTime(timezone=True))
99
99
  )
100
- status: str = JobStatusType.SUBMITTED
100
+ status: str = JobStatusTypeV1.SUBMITTED
101
101
  log: Optional[str] = None
@@ -7,13 +7,13 @@ from sqlmodel import Field
7
7
  from sqlmodel import Relationship
8
8
  from sqlmodel import SQLModel
9
9
 
10
- from ...utils import get_timestamp
11
- from ..schemas.project import _ProjectBase
12
- from .linkuserproject import LinkUserProject
13
- from .security import UserOAuth
10
+ from ....utils import get_timestamp
11
+ from ...schemas.v1.project import _ProjectBaseV1
12
+ from ..linkuserproject import LinkUserProject
13
+ from ..security import UserOAuth
14
14
 
15
15
 
16
- class Project(_ProjectBase, SQLModel, table=True):
16
+ class Project(_ProjectBaseV1, SQLModel, table=True):
17
17
 
18
18
  id: Optional[int] = Field(default=None, primary_key=True)
19
19
  timestamp_created: datetime = Field(
@@ -8,8 +8,8 @@ from sqlalchemy.types import JSON
8
8
  from sqlmodel import Field
9
9
  from sqlmodel import SQLModel
10
10
 
11
- from ...utils import get_timestamp
12
- from ..schemas import _StateBase
11
+ from ....utils import get_timestamp
12
+ from ...schemas.state import _StateBase
13
13
 
14
14
 
15
15
  class State(_StateBase, SQLModel, table=True):
@@ -5,14 +5,15 @@ from typing import Optional
5
5
 
6
6
  from pydantic import HttpUrl
7
7
  from sqlalchemy import Column
8
+ from sqlalchemy import sql
8
9
  from sqlalchemy.types import JSON
9
10
  from sqlmodel import Field
10
11
  from sqlmodel import SQLModel
11
12
 
12
- from ..schemas.task import _TaskBase
13
+ from ...schemas.v1.task import _TaskBaseV1
13
14
 
14
15
 
15
- class Task(_TaskBase, SQLModel, table=True):
16
+ class Task(_TaskBaseV1, SQLModel, table=True):
16
17
  """
17
18
  Task model
18
19
 
@@ -48,6 +49,10 @@ class Task(_TaskBase, SQLModel, table=True):
48
49
  docs_info: Optional[str] = None
49
50
  docs_link: Optional[HttpUrl] = None
50
51
 
52
+ is_v2_compatible: bool = Field(
53
+ default=False, sa_column_kwargs={"server_default": sql.false()}
54
+ )
55
+
51
56
  @property
52
57
  def parallelization_level(self) -> Optional[str]:
53
58
  try:
@@ -12,13 +12,13 @@ from sqlmodel import Field
12
12
  from sqlmodel import Relationship
13
13
  from sqlmodel import SQLModel
14
14
 
15
- from ...utils import get_timestamp
16
- from ..schemas.workflow import _WorkflowBase
17
- from ..schemas.workflow import _WorkflowTaskBase
15
+ from ....utils import get_timestamp
16
+ from ...schemas.v1.workflow import _WorkflowBaseV1
17
+ from ...schemas.v1.workflow import _WorkflowTaskBaseV1
18
18
  from .task import Task
19
19
 
20
20
 
21
- class WorkflowTask(_WorkflowTaskBase, SQLModel, table=True):
21
+ class WorkflowTask(_WorkflowTaskBaseV1, SQLModel, table=True):
22
22
  """
23
23
  A Task as part of a Workflow
24
24
 
@@ -92,7 +92,7 @@ class WorkflowTask(_WorkflowTaskBase, SQLModel, table=True):
92
92
  return self.task.parallelization_level
93
93
 
94
94
 
95
- class Workflow(_WorkflowBase, SQLModel, table=True):
95
+ class Workflow(_WorkflowBaseV1, SQLModel, table=True):
96
96
  """
97
97
  Workflow
98
98
 
@@ -0,0 +1,22 @@
1
+ """
2
+ v2 `models` module
3
+ """
4
+ from ..linkuserproject import LinkUserProjectV2
5
+ from .collection_state import CollectionStateV2
6
+ from .dataset import DatasetV2
7
+ from .job import JobV2
8
+ from .project import ProjectV2
9
+ from .task import TaskV2
10
+ from .workflow import WorkflowV2
11
+ from .workflowtask import WorkflowTaskV2
12
+
13
+ __all__ = [
14
+ "LinkUserProjectV2",
15
+ "DatasetV2",
16
+ "JobV2",
17
+ "ProjectV2",
18
+ "CollectionStateV2",
19
+ "TaskV2",
20
+ "WorkflowTaskV2",
21
+ "WorkflowV2",
22
+ ]
@@ -0,0 +1,21 @@
1
+ from datetime import datetime
2
+ from typing import Any
3
+ from typing import Optional
4
+
5
+ from sqlalchemy import Column
6
+ from sqlalchemy.types import DateTime
7
+ from sqlalchemy.types import JSON
8
+ from sqlmodel import Field
9
+ from sqlmodel import SQLModel
10
+
11
+ from ....utils import get_timestamp
12
+
13
+
14
+ class CollectionStateV2(SQLModel, table=True):
15
+
16
+ id: Optional[int] = Field(default=None, primary_key=True)
17
+ data: dict[str, Any] = Field(sa_column=Column(JSON), default={})
18
+ timestamp: datetime = Field(
19
+ default_factory=get_timestamp,
20
+ sa_column=Column(DateTime(timezone=True)),
21
+ )
@@ -0,0 +1,54 @@
1
+ from datetime import datetime
2
+ from typing import Any
3
+ from typing import Literal
4
+ from typing import Optional
5
+
6
+ from sqlalchemy import Column
7
+ from sqlalchemy.types import DateTime
8
+ from sqlalchemy.types import JSON
9
+ from sqlmodel import Field
10
+ from sqlmodel import Relationship
11
+ from sqlmodel import SQLModel
12
+
13
+ from ....utils import get_timestamp
14
+
15
+
16
+ class DatasetV2(SQLModel, table=True):
17
+ class Config:
18
+ arbitrary_types_allowed = True
19
+
20
+ id: Optional[int] = Field(default=None, primary_key=True)
21
+ name: str
22
+
23
+ project_id: int = Field(foreign_key="projectv2.id")
24
+ project: "ProjectV2" = Relationship( # noqa: F821
25
+ sa_relationship_kwargs=dict(lazy="selectin"),
26
+ )
27
+
28
+ history: list[dict[str, Any]] = Field(
29
+ sa_column=Column(JSON, server_default="[]", nullable=False)
30
+ )
31
+
32
+ timestamp_created: datetime = Field(
33
+ default_factory=get_timestamp,
34
+ sa_column=Column(DateTime(timezone=True), nullable=False),
35
+ )
36
+
37
+ # New in V2
38
+
39
+ zarr_dir: str
40
+ images: list[dict[str, Any]] = Field(
41
+ sa_column=Column(JSON, server_default="[]", nullable=False)
42
+ )
43
+
44
+ filters: dict[Literal["attributes", "types"], dict[str, Any]] = Field(
45
+ sa_column=Column(
46
+ JSON,
47
+ nullable=False,
48
+ server_default='{"attributes": {}, "types": {}}',
49
+ )
50
+ )
51
+
52
+ @property
53
+ def image_zarr_urls(self) -> list[str]:
54
+ return [image["zarr_url"] for image in self.images]
@@ -0,0 +1,51 @@
1
+ from datetime import datetime
2
+ from typing import Any
3
+ from typing import Optional
4
+
5
+ from sqlalchemy import Column
6
+ from sqlalchemy.types import DateTime
7
+ from sqlalchemy.types import JSON
8
+ from sqlmodel import Field
9
+ from sqlmodel import SQLModel
10
+
11
+ from ....utils import get_timestamp
12
+ from ...schemas.v2 import JobStatusTypeV2
13
+
14
+
15
+ class JobV2(SQLModel, table=True):
16
+ class Config:
17
+ arbitrary_types_allowed = True
18
+
19
+ id: Optional[int] = Field(default=None, primary_key=True)
20
+ project_id: Optional[int] = Field(foreign_key="projectv2.id")
21
+ workflow_id: Optional[int] = Field(foreign_key="workflowv2.id")
22
+ dataset_id: Optional[int] = Field(foreign_key="datasetv2.id")
23
+
24
+ user_email: str = Field(nullable=False)
25
+ slurm_account: Optional[str]
26
+
27
+ dataset_dump: dict[str, Any] = Field(
28
+ sa_column=Column(JSON, nullable=False)
29
+ )
30
+ workflow_dump: dict[str, Any] = Field(
31
+ sa_column=Column(JSON, nullable=False)
32
+ )
33
+ project_dump: dict[str, Any] = Field(
34
+ sa_column=Column(JSON, nullable=False)
35
+ )
36
+
37
+ worker_init: Optional[str]
38
+ working_dir: Optional[str]
39
+ working_dir_user: Optional[str]
40
+ first_task_index: int
41
+ last_task_index: int
42
+
43
+ start_timestamp: datetime = Field(
44
+ default_factory=get_timestamp,
45
+ sa_column=Column(DateTime(timezone=True), nullable=False),
46
+ )
47
+ end_timestamp: Optional[datetime] = Field(
48
+ default=None, sa_column=Column(DateTime(timezone=True))
49
+ )
50
+ status: str = JobStatusTypeV2.SUBMITTED
51
+ log: Optional[str] = None
@@ -0,0 +1,30 @@
1
+ from datetime import datetime
2
+ from typing import Optional
3
+
4
+ from sqlalchemy import Column
5
+ from sqlalchemy.types import DateTime
6
+ from sqlmodel import Field
7
+ from sqlmodel import Relationship
8
+ from sqlmodel import SQLModel
9
+
10
+ from ....utils import get_timestamp
11
+ from ..linkuserproject import LinkUserProjectV2
12
+ from ..security import UserOAuth
13
+
14
+
15
+ class ProjectV2(SQLModel, table=True):
16
+
17
+ id: Optional[int] = Field(default=None, primary_key=True)
18
+ name: str
19
+ timestamp_created: datetime = Field(
20
+ default_factory=get_timestamp,
21
+ sa_column=Column(DateTime(timezone=True), nullable=False),
22
+ )
23
+
24
+ user_list: list[UserOAuth] = Relationship(
25
+ link_model=LinkUserProjectV2,
26
+ back_populates="project_list_v2",
27
+ sa_relationship_kwargs={
28
+ "lazy": "selectin",
29
+ },
30
+ )
@@ -0,0 +1,93 @@
1
+ import json
2
+ import logging
3
+ from typing import Any
4
+ from typing import Optional
5
+
6
+ from pydantic import HttpUrl
7
+ from sqlalchemy import Column
8
+ from sqlalchemy.types import JSON
9
+ from sqlmodel import Field
10
+ from sqlmodel import SQLModel
11
+
12
+
13
+ class TaskV2(SQLModel, table=True):
14
+
15
+ id: Optional[int] = Field(default=None, primary_key=True)
16
+ name: str
17
+
18
+ type: str
19
+ command_non_parallel: Optional[str] = None
20
+ command_parallel: Optional[str] = None
21
+ source: str = Field(unique=True)
22
+
23
+ meta_non_parallel: dict[str, Any] = Field(
24
+ sa_column=Column(JSON, server_default="{}", default={}, nullable=False)
25
+ )
26
+ meta_parallel: dict[str, Any] = Field(
27
+ sa_column=Column(JSON, server_default="{}", default={}, nullable=False)
28
+ )
29
+
30
+ owner: Optional[str] = None
31
+ version: Optional[str] = None
32
+ args_schema_non_parallel: Optional[dict[str, Any]] = Field(
33
+ sa_column=Column(JSON), default=None
34
+ )
35
+ args_schema_parallel: Optional[dict[str, Any]] = Field(
36
+ sa_column=Column(JSON), default=None
37
+ )
38
+ args_schema_version: Optional[str]
39
+ docs_info: Optional[str] = None
40
+ docs_link: Optional[HttpUrl] = None
41
+
42
+ input_types: dict[str, bool] = Field(sa_column=Column(JSON), default={})
43
+ output_types: dict[str, bool] = Field(sa_column=Column(JSON), default={})
44
+
45
+ @property
46
+ def default_args_non_parallel_from_args_schema(self) -> dict[str, Any]:
47
+ """
48
+ Extract default arguments from args_schema
49
+ """
50
+ # Return {} if there is no args_schema
51
+ if self.args_schema_non_parallel is None:
52
+ return {}
53
+ # Try to construct default_args
54
+ try:
55
+ default_args = {}
56
+ properties = self.args_schema_non_parallel["properties"]
57
+ for prop_name, prop_schema in properties.items():
58
+ default_value = prop_schema.get("default", None)
59
+ if default_value is not None:
60
+ default_args[prop_name] = default_value
61
+ return default_args
62
+ except KeyError as e:
63
+ logging.warning(
64
+ "Cannot set default_args from args_schema_non_parallel="
65
+ f"{json.dumps(self.args_schema_non_parallel)}\n"
66
+ f"Original KeyError: {str(e)}"
67
+ )
68
+ return {}
69
+
70
+ @property
71
+ def default_args_parallel_from_args_schema(self) -> dict[str, Any]:
72
+ """
73
+ Extract default arguments from args_schema
74
+ """
75
+ # Return {} if there is no args_schema
76
+ if self.args_schema_parallel is None:
77
+ return {}
78
+ # Try to construct default_args
79
+ try:
80
+ default_args = {}
81
+ properties = self.args_schema_parallel["properties"]
82
+ for prop_name, prop_schema in properties.items():
83
+ default_value = prop_schema.get("default", None)
84
+ if default_value is not None:
85
+ default_args[prop_name] = default_value
86
+ return default_args
87
+ except KeyError as e:
88
+ logging.warning(
89
+ "Cannot set default_args from args_schema_parallel="
90
+ f"{json.dumps(self.args_schema_parallel)}\n"
91
+ f"Original KeyError: {str(e)}"
92
+ )
93
+ return {}
@@ -0,0 +1,35 @@
1
+ from datetime import datetime
2
+ from typing import Optional
3
+
4
+ from sqlalchemy import Column
5
+ from sqlalchemy.ext.orderinglist import ordering_list
6
+ from sqlalchemy.types import DateTime
7
+ from sqlmodel import Field
8
+ from sqlmodel import Relationship
9
+ from sqlmodel import SQLModel
10
+
11
+ from ....utils import get_timestamp
12
+ from .workflowtask import WorkflowTaskV2
13
+
14
+
15
+ class WorkflowV2(SQLModel, table=True):
16
+
17
+ id: Optional[int] = Field(default=None, primary_key=True)
18
+ name: str
19
+ project_id: int = Field(foreign_key="projectv2.id")
20
+ project: "ProjectV2" = Relationship( # noqa: F821
21
+ sa_relationship_kwargs=dict(lazy="selectin"),
22
+ )
23
+
24
+ task_list: list[WorkflowTaskV2] = Relationship(
25
+ sa_relationship_kwargs=dict(
26
+ lazy="selectin",
27
+ order_by="WorkflowTaskV2.order",
28
+ collection_class=ordering_list("order"),
29
+ cascade="all, delete-orphan",
30
+ ),
31
+ )
32
+ timestamp_created: datetime = Field(
33
+ default_factory=get_timestamp,
34
+ sa_column=Column(DateTime(timezone=True), nullable=False),
35
+ )
@@ -0,0 +1,49 @@
1
+ from typing import Any
2
+ from typing import Literal
3
+ from typing import Optional
4
+
5
+ from sqlalchemy import Column
6
+ from sqlalchemy.types import JSON
7
+ from sqlmodel import Field
8
+ from sqlmodel import Relationship
9
+ from sqlmodel import SQLModel
10
+
11
+ from ..v1.task import Task
12
+ from .task import TaskV2
13
+
14
+
15
+ class WorkflowTaskV2(SQLModel, table=True):
16
+ class Config:
17
+ arbitrary_types_allowed = True
18
+ fields = {"parent": {"exclude": True}}
19
+
20
+ id: Optional[int] = Field(default=None, primary_key=True)
21
+
22
+ workflow_id: int = Field(foreign_key="workflowv2.id")
23
+ order: Optional[int]
24
+ meta_parallel: Optional[dict[str, Any]] = Field(sa_column=Column(JSON))
25
+ meta_non_parallel: Optional[dict[str, Any]] = Field(sa_column=Column(JSON))
26
+ args_parallel: Optional[dict[str, Any]] = Field(sa_column=Column(JSON))
27
+ args_non_parallel: Optional[dict[str, Any]] = Field(sa_column=Column(JSON))
28
+
29
+ input_filters: dict[
30
+ Literal["attributes", "types"], dict[str, Any]
31
+ ] = Field(
32
+ sa_column=Column(
33
+ JSON,
34
+ nullable=False,
35
+ server_default='{"attributes": {}, "types": {}}',
36
+ )
37
+ )
38
+
39
+ # Task
40
+ is_legacy_task: bool
41
+ task_type: str
42
+ task_id: Optional[int] = Field(foreign_key="taskv2.id")
43
+ task: Optional[TaskV2] = Relationship(
44
+ sa_relationship_kwargs=dict(lazy="selectin")
45
+ )
46
+ task_legacy_id: Optional[int] = Field(foreign_key="task.id")
47
+ task_legacy: Optional[Task] = Relationship(
48
+ sa_relationship_kwargs=dict(lazy="selectin")
49
+ )
File without changes