zenml-nightly 0.75.0.dev20250313__py3-none-any.whl → 0.75.0.dev20250314__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zenml/VERSION +1 -1
- zenml/analytics/context.py +4 -4
- zenml/analytics/enums.py +2 -2
- zenml/artifacts/utils.py +2 -2
- zenml/cli/__init__.py +8 -9
- zenml/cli/base.py +2 -2
- zenml/cli/code_repository.py +1 -1
- zenml/cli/pipeline.py +3 -3
- zenml/cli/project.py +172 -0
- zenml/cli/service_accounts.py +0 -1
- zenml/cli/service_connectors.py +15 -16
- zenml/cli/stack.py +0 -2
- zenml/cli/stack_components.py +2 -2
- zenml/cli/utils.py +3 -3
- zenml/client.py +347 -340
- zenml/config/global_config.py +41 -43
- zenml/constants.py +5 -3
- zenml/event_hub/event_hub.py +1 -1
- zenml/integrations/gcp/service_connectors/gcp_service_connector.py +7 -6
- zenml/integrations/mlflow/steps/mlflow_registry.py +3 -3
- zenml/integrations/wandb/__init__.py +1 -1
- zenml/integrations/wandb/experiment_trackers/wandb_experiment_tracker.py +29 -9
- zenml/integrations/wandb/flavors/wandb_experiment_tracker_flavor.py +2 -0
- zenml/model/model.py +2 -2
- zenml/model_registries/base_model_registry.py +1 -1
- zenml/models/__init__.py +29 -29
- zenml/models/v2/base/filter.py +1 -1
- zenml/models/v2/base/scoped.py +49 -53
- zenml/models/v2/core/action.py +12 -12
- zenml/models/v2/core/artifact.py +15 -15
- zenml/models/v2/core/artifact_version.py +15 -15
- zenml/models/v2/core/code_repository.py +12 -12
- zenml/models/v2/core/event_source.py +12 -12
- zenml/models/v2/core/model.py +17 -17
- zenml/models/v2/core/model_version.py +15 -15
- zenml/models/v2/core/pipeline.py +15 -15
- zenml/models/v2/core/pipeline_build.py +14 -14
- zenml/models/v2/core/pipeline_deployment.py +12 -14
- zenml/models/v2/core/pipeline_run.py +16 -16
- zenml/models/v2/core/project.py +203 -0
- zenml/models/v2/core/run_metadata.py +2 -2
- zenml/models/v2/core/run_template.py +15 -15
- zenml/models/v2/core/schedule.py +12 -12
- zenml/models/v2/core/secret.py +1 -1
- zenml/models/v2/core/service.py +14 -14
- zenml/models/v2/core/step_run.py +13 -13
- zenml/models/v2/core/trigger.py +13 -13
- zenml/models/v2/core/trigger_execution.py +2 -2
- zenml/models/v2/core/user.py +0 -17
- zenml/models/v2/misc/statistics.py +4 -4
- zenml/orchestrators/cache_utils.py +7 -7
- zenml/orchestrators/input_utils.py +1 -1
- zenml/orchestrators/step_launcher.py +1 -1
- zenml/orchestrators/step_run_utils.py +2 -2
- zenml/orchestrators/utils.py +4 -4
- zenml/pipelines/build_utils.py +2 -2
- zenml/pipelines/pipeline_definition.py +5 -5
- zenml/pipelines/run_utils.py +1 -1
- zenml/service_connectors/service_connector.py +0 -3
- zenml/service_connectors/service_connector_utils.py +0 -1
- zenml/stack/stack.py +0 -1
- zenml/steps/base_step.py +10 -2
- zenml/zen_server/rbac/endpoint_utils.py +17 -17
- zenml/zen_server/rbac/models.py +20 -20
- zenml/zen_server/rbac/rbac_sql_zen_store.py +3 -3
- zenml/zen_server/rbac/utils.py +23 -25
- zenml/zen_server/rbac/zenml_cloud_rbac.py +12 -16
- zenml/zen_server/routers/artifact_version_endpoints.py +10 -10
- zenml/zen_server/routers/auth_endpoints.py +6 -6
- zenml/zen_server/routers/code_repositories_endpoints.py +12 -14
- zenml/zen_server/routers/model_versions_endpoints.py +13 -15
- zenml/zen_server/routers/models_endpoints.py +7 -9
- zenml/zen_server/routers/pipeline_builds_endpoints.py +14 -16
- zenml/zen_server/routers/pipeline_deployments_endpoints.py +13 -15
- zenml/zen_server/routers/pipelines_endpoints.py +16 -18
- zenml/zen_server/routers/{workspaces_endpoints.py → projects_endpoints.py} +111 -68
- zenml/zen_server/routers/run_metadata_endpoints.py +7 -9
- zenml/zen_server/routers/run_templates_endpoints.py +15 -17
- zenml/zen_server/routers/runs_endpoints.py +12 -14
- zenml/zen_server/routers/schedule_endpoints.py +12 -14
- zenml/zen_server/routers/secrets_endpoints.py +1 -3
- zenml/zen_server/routers/server_endpoints.py +5 -5
- zenml/zen_server/routers/service_connectors_endpoints.py +11 -13
- zenml/zen_server/routers/service_endpoints.py +7 -9
- zenml/zen_server/routers/stack_components_endpoints.py +9 -11
- zenml/zen_server/routers/stacks_endpoints.py +9 -11
- zenml/zen_server/routers/steps_endpoints.py +6 -6
- zenml/zen_server/routers/users_endpoints.py +5 -43
- zenml/zen_server/template_execution/utils.py +4 -4
- zenml/zen_server/utils.py +10 -10
- zenml/zen_server/zen_server_api.py +3 -2
- zenml/zen_stores/base_zen_store.py +35 -39
- zenml/zen_stores/migrations/versions/12eff0206201_rename_workspace_to_project.py +768 -0
- zenml/zen_stores/migrations/versions/41b28cae31ce_make_artifacts_workspace_scoped.py +3 -3
- zenml/zen_stores/migrations/versions/cbc6acd71f92_add_workspace_display_name.py +58 -0
- zenml/zen_stores/rest_zen_store.py +54 -62
- zenml/zen_stores/schemas/__init__.py +2 -2
- zenml/zen_stores/schemas/action_schemas.py +9 -9
- zenml/zen_stores/schemas/artifact_schemas.py +15 -17
- zenml/zen_stores/schemas/code_repository_schemas.py +16 -18
- zenml/zen_stores/schemas/event_source_schemas.py +9 -9
- zenml/zen_stores/schemas/model_schemas.py +15 -17
- zenml/zen_stores/schemas/pipeline_build_schemas.py +7 -7
- zenml/zen_stores/schemas/pipeline_deployment_schemas.py +7 -7
- zenml/zen_stores/schemas/pipeline_run_schemas.py +9 -9
- zenml/zen_stores/schemas/pipeline_schemas.py +9 -9
- zenml/zen_stores/schemas/{workspace_schemas.py → project_schemas.py} +47 -41
- zenml/zen_stores/schemas/run_metadata_schemas.py +5 -5
- zenml/zen_stores/schemas/run_template_schemas.py +9 -9
- zenml/zen_stores/schemas/schedule_schema.py +9 -9
- zenml/zen_stores/schemas/service_schemas.py +7 -7
- zenml/zen_stores/schemas/step_run_schemas.py +7 -7
- zenml/zen_stores/schemas/trigger_schemas.py +9 -9
- zenml/zen_stores/schemas/user_schemas.py +0 -12
- zenml/zen_stores/sql_zen_store.py +258 -268
- zenml/zen_stores/zen_store_interface.py +56 -70
- {zenml_nightly-0.75.0.dev20250313.dist-info → zenml_nightly-0.75.0.dev20250314.dist-info}/METADATA +1 -1
- {zenml_nightly-0.75.0.dev20250313.dist-info → zenml_nightly-0.75.0.dev20250314.dist-info}/RECORD +121 -119
- zenml/cli/workspace.py +0 -160
- zenml/models/v2/core/workspace.py +0 -131
- {zenml_nightly-0.75.0.dev20250313.dist-info → zenml_nightly-0.75.0.dev20250314.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.75.0.dev20250313.dist-info → zenml_nightly-0.75.0.dev20250314.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.75.0.dev20250313.dist-info → zenml_nightly-0.75.0.dev20250314.dist-info}/entry_points.txt +0 -0
@@ -225,6 +225,12 @@ from zenml.models import (
|
|
225
225
|
PipelineRunResponse,
|
226
226
|
PipelineRunUpdate,
|
227
227
|
PipelineUpdate,
|
228
|
+
ProjectFilter,
|
229
|
+
ProjectRequest,
|
230
|
+
ProjectResponse,
|
231
|
+
ProjectScopedFilter,
|
232
|
+
ProjectScopedRequest,
|
233
|
+
ProjectUpdate,
|
228
234
|
RunMetadataRequest,
|
229
235
|
RunMetadataResource,
|
230
236
|
RunTemplateFilter,
|
@@ -288,12 +294,6 @@ from zenml.models import (
|
|
288
294
|
UserResponse,
|
289
295
|
UserScopedRequest,
|
290
296
|
UserUpdate,
|
291
|
-
WorkspaceFilter,
|
292
|
-
WorkspaceRequest,
|
293
|
-
WorkspaceResponse,
|
294
|
-
WorkspaceScopedFilter,
|
295
|
-
WorkspaceScopedRequest,
|
296
|
-
WorkspaceUpdate,
|
297
297
|
)
|
298
298
|
from zenml.service_connectors.service_connector_registry import (
|
299
299
|
service_connector_registry,
|
@@ -340,6 +340,7 @@ from zenml.zen_stores.schemas import (
|
|
340
340
|
PipelineDeploymentSchema,
|
341
341
|
PipelineRunSchema,
|
342
342
|
PipelineSchema,
|
343
|
+
ProjectSchema,
|
343
344
|
RunMetadataResourceSchema,
|
344
345
|
RunMetadataSchema,
|
345
346
|
RunTemplateSchema,
|
@@ -357,7 +358,6 @@ from zenml.zen_stores.schemas import (
|
|
357
358
|
TagSchema,
|
358
359
|
TriggerExecutionSchema,
|
359
360
|
UserSchema,
|
360
|
-
WorkspaceSchema,
|
361
361
|
)
|
362
362
|
from zenml.zen_stores.schemas.artifact_visualization_schemas import (
|
363
363
|
ArtifactVisualizationSchema,
|
@@ -1155,9 +1155,9 @@ class SqlZenStore(BaseZenStore):
|
|
1155
1155
|
|
1156
1156
|
def _initialize_database(self) -> None:
|
1157
1157
|
"""Initialize the database if not already initialized."""
|
1158
|
-
# When running in a Pro ZenML server, the default
|
1158
|
+
# When running in a Pro ZenML server, the default project is not
|
1159
1159
|
# created on database initialization but on server onboarding.
|
1160
|
-
|
1160
|
+
create_default_project = True
|
1161
1161
|
if ENV_ZENML_SERVER in os.environ:
|
1162
1162
|
from zenml.config.server_config import ServerConfiguration
|
1163
1163
|
|
@@ -1165,11 +1165,11 @@ class SqlZenStore(BaseZenStore):
|
|
1165
1165
|
ServerConfiguration.get_server_config().deployment_type
|
1166
1166
|
== ServerDeploymentType.CLOUD
|
1167
1167
|
):
|
1168
|
-
|
1168
|
+
create_default_project = False
|
1169
1169
|
|
1170
|
-
if
|
1171
|
-
# Make sure the default
|
1172
|
-
self.
|
1170
|
+
if create_default_project:
|
1171
|
+
# Make sure the default project exists
|
1172
|
+
self._get_or_create_default_project()
|
1173
1173
|
# Make sure the default stack exists
|
1174
1174
|
self._get_or_create_default_stack()
|
1175
1175
|
# Make sure the server is activated and the default user exists, if
|
@@ -1944,7 +1944,7 @@ class SqlZenStore(BaseZenStore):
|
|
1944
1944
|
A page of actions matching the filter criteria.
|
1945
1945
|
"""
|
1946
1946
|
with Session(self.engine) as session:
|
1947
|
-
self.
|
1947
|
+
self._set_filter_project_id(
|
1948
1948
|
filter_model=action_filter_model,
|
1949
1949
|
session=session,
|
1950
1950
|
)
|
@@ -2388,7 +2388,7 @@ class SqlZenStore(BaseZenStore):
|
|
2388
2388
|
EntityExistsError: If a service with the given name and
|
2389
2389
|
type already exists.
|
2390
2390
|
"""
|
2391
|
-
# Check if service with the same domain key (name, config,
|
2391
|
+
# Check if service with the same domain key (name, config, project)
|
2392
2392
|
# already exists
|
2393
2393
|
existing_domain_service = session.exec(
|
2394
2394
|
select(ServiceSchema).where(
|
@@ -2487,7 +2487,7 @@ class SqlZenStore(BaseZenStore):
|
|
2487
2487
|
A list of all services matching the filter criteria.
|
2488
2488
|
"""
|
2489
2489
|
with Session(self.engine) as session:
|
2490
|
-
self.
|
2490
|
+
self._set_filter_project_id(
|
2491
2491
|
filter_model=filter_model,
|
2492
2492
|
session=session,
|
2493
2493
|
)
|
@@ -2631,7 +2631,7 @@ class SqlZenStore(BaseZenStore):
|
|
2631
2631
|
A list of all artifacts matching the filter criteria.
|
2632
2632
|
"""
|
2633
2633
|
with Session(self.engine) as session:
|
2634
|
-
self.
|
2634
|
+
self._set_filter_project_id(
|
2635
2635
|
filter_model=filter_model,
|
2636
2636
|
session=session,
|
2637
2637
|
)
|
@@ -2711,7 +2711,7 @@ class SqlZenStore(BaseZenStore):
|
|
2711
2711
|
def _get_or_create_artifact_for_name(
|
2712
2712
|
self,
|
2713
2713
|
name: str,
|
2714
|
-
|
2714
|
+
project_id: UUID,
|
2715
2715
|
has_custom_name: bool,
|
2716
2716
|
session: Session,
|
2717
2717
|
) -> ArtifactSchema:
|
@@ -2719,7 +2719,7 @@ class SqlZenStore(BaseZenStore):
|
|
2719
2719
|
|
2720
2720
|
Args:
|
2721
2721
|
name: The artifact name.
|
2722
|
-
|
2722
|
+
project_id: The project ID.
|
2723
2723
|
has_custom_name: Whether the artifact has a custom name.
|
2724
2724
|
session: DB session.
|
2725
2725
|
|
@@ -2729,7 +2729,7 @@ class SqlZenStore(BaseZenStore):
|
|
2729
2729
|
artifact_query = (
|
2730
2730
|
select(ArtifactSchema)
|
2731
2731
|
.where(ArtifactSchema.name == name)
|
2732
|
-
.where(ArtifactSchema.
|
2732
|
+
.where(ArtifactSchema.project_id == project_id)
|
2733
2733
|
)
|
2734
2734
|
artifact = session.exec(artifact_query).first()
|
2735
2735
|
|
@@ -2738,7 +2738,7 @@ class SqlZenStore(BaseZenStore):
|
|
2738
2738
|
with session.begin_nested():
|
2739
2739
|
artifact_request = ArtifactRequest(
|
2740
2740
|
name=name,
|
2741
|
-
|
2741
|
+
project=project_id,
|
2742
2742
|
has_custom_name=has_custom_name,
|
2743
2743
|
)
|
2744
2744
|
self._set_request_user_id(
|
@@ -2822,7 +2822,7 @@ class SqlZenStore(BaseZenStore):
|
|
2822
2822
|
if artifact_name := artifact_version.artifact_name:
|
2823
2823
|
artifact_schema = self._get_or_create_artifact_for_name(
|
2824
2824
|
name=artifact_name,
|
2825
|
-
|
2825
|
+
project_id=artifact_version.project,
|
2826
2826
|
has_custom_name=artifact_version.has_custom_name,
|
2827
2827
|
session=session,
|
2828
2828
|
)
|
@@ -2949,7 +2949,7 @@ class SqlZenStore(BaseZenStore):
|
|
2949
2949
|
types[key] = metadata_type
|
2950
2950
|
self.create_run_metadata(
|
2951
2951
|
RunMetadataRequest(
|
2952
|
-
|
2952
|
+
project=artifact_version.project,
|
2953
2953
|
resources=[
|
2954
2954
|
RunMetadataResource(
|
2955
2955
|
id=artifact_version_schema.id,
|
@@ -3024,7 +3024,7 @@ class SqlZenStore(BaseZenStore):
|
|
3024
3024
|
A list of all artifact versions matching the filter criteria.
|
3025
3025
|
"""
|
3026
3026
|
with Session(self.engine) as session:
|
3027
|
-
self.
|
3027
|
+
self._set_filter_project_id(
|
3028
3028
|
filter_model=artifact_version_filter_model,
|
3029
3029
|
session=session,
|
3030
3030
|
)
|
@@ -3101,20 +3101,20 @@ class SqlZenStore(BaseZenStore):
|
|
3101
3101
|
|
3102
3102
|
def prune_artifact_versions(
|
3103
3103
|
self,
|
3104
|
-
|
3104
|
+
project_name_or_id: Union[str, UUID],
|
3105
3105
|
only_versions: bool = True,
|
3106
3106
|
) -> None:
|
3107
3107
|
"""Prunes unused artifact versions and their artifacts.
|
3108
3108
|
|
3109
3109
|
Args:
|
3110
|
-
|
3110
|
+
project_name_or_id: The project name or ID to prune artifact
|
3111
3111
|
versions for.
|
3112
3112
|
only_versions: Only delete artifact versions, keeping artifacts
|
3113
3113
|
"""
|
3114
3114
|
with Session(self.engine) as session:
|
3115
|
-
|
3116
|
-
object_name_or_id=
|
3117
|
-
schema_class=
|
3115
|
+
project_id = self._get_schema_by_name_or_id(
|
3116
|
+
object_name_or_id=project_name_or_id,
|
3117
|
+
schema_class=ProjectSchema,
|
3118
3118
|
session=session,
|
3119
3119
|
).id
|
3120
3120
|
|
@@ -3129,8 +3129,8 @@ class SqlZenStore(BaseZenStore):
|
|
3129
3129
|
col(ArtifactVersionSchema.id).notin_(
|
3130
3130
|
select(StepRunInputArtifactSchema.artifact_id)
|
3131
3131
|
),
|
3132
|
-
col(ArtifactVersionSchema.
|
3133
|
-
==
|
3132
|
+
col(ArtifactVersionSchema.project_id)
|
3133
|
+
== project_id,
|
3134
3134
|
)
|
3135
3135
|
)
|
3136
3136
|
).fetchall()
|
@@ -3286,7 +3286,7 @@ class SqlZenStore(BaseZenStore):
|
|
3286
3286
|
A page of all code repositories.
|
3287
3287
|
"""
|
3288
3288
|
with Session(self.engine) as session:
|
3289
|
-
self.
|
3289
|
+
self._set_filter_project_id(
|
3290
3290
|
filter_model=filter_model,
|
3291
3291
|
session=session,
|
3292
3292
|
)
|
@@ -4172,7 +4172,7 @@ class SqlZenStore(BaseZenStore):
|
|
4172
4172
|
self,
|
4173
4173
|
pipeline: PipelineRequest,
|
4174
4174
|
) -> PipelineResponse:
|
4175
|
-
"""Creates a new pipeline
|
4175
|
+
"""Creates a new pipeline.
|
4176
4176
|
|
4177
4177
|
Args:
|
4178
4178
|
pipeline: The pipeline to create.
|
@@ -4196,8 +4196,8 @@ class SqlZenStore(BaseZenStore):
|
|
4196
4196
|
# to continue using it
|
4197
4197
|
session.rollback()
|
4198
4198
|
raise EntityExistsError(
|
4199
|
-
f"Unable to create pipeline in
|
4200
|
-
f"'{pipeline.
|
4199
|
+
f"Unable to create pipeline in project "
|
4200
|
+
f"'{pipeline.project}': A pipeline with the name "
|
4201
4201
|
f"{pipeline.name} already exists."
|
4202
4202
|
)
|
4203
4203
|
session.refresh(new_pipeline)
|
@@ -4255,7 +4255,7 @@ class SqlZenStore(BaseZenStore):
|
|
4255
4255
|
A list of all pipelines matching the filter criteria.
|
4256
4256
|
"""
|
4257
4257
|
with Session(self.engine) as session:
|
4258
|
-
self.
|
4258
|
+
self._set_filter_project_id(
|
4259
4259
|
filter_model=pipeline_filter_model,
|
4260
4260
|
session=session,
|
4261
4261
|
)
|
@@ -4347,7 +4347,7 @@ class SqlZenStore(BaseZenStore):
|
|
4347
4347
|
self,
|
4348
4348
|
build: PipelineBuildRequest,
|
4349
4349
|
) -> PipelineBuildResponse:
|
4350
|
-
"""Creates a new build
|
4350
|
+
"""Creates a new build.
|
4351
4351
|
|
4352
4352
|
Args:
|
4353
4353
|
build: The build to create.
|
@@ -4421,7 +4421,7 @@ class SqlZenStore(BaseZenStore):
|
|
4421
4421
|
A page of all builds matching the filter criteria.
|
4422
4422
|
"""
|
4423
4423
|
with Session(self.engine) as session:
|
4424
|
-
self.
|
4424
|
+
self._set_filter_project_id(
|
4425
4425
|
filter_model=build_filter_model,
|
4426
4426
|
session=session,
|
4427
4427
|
)
|
@@ -4456,14 +4456,14 @@ class SqlZenStore(BaseZenStore):
|
|
4456
4456
|
@staticmethod
|
4457
4457
|
def _create_or_reuse_code_reference(
|
4458
4458
|
session: Session,
|
4459
|
-
|
4459
|
+
project_id: UUID,
|
4460
4460
|
code_reference: Optional["CodeReferenceRequest"],
|
4461
4461
|
) -> Optional[UUID]:
|
4462
4462
|
"""Creates or reuses a code reference.
|
4463
4463
|
|
4464
4464
|
Args:
|
4465
4465
|
session: The database session to use.
|
4466
|
-
|
4466
|
+
project_id: ID of the project in which the code reference
|
4467
4467
|
should be.
|
4468
4468
|
code_reference: Request of the reference to create.
|
4469
4469
|
|
@@ -4475,7 +4475,7 @@ class SqlZenStore(BaseZenStore):
|
|
4475
4475
|
|
4476
4476
|
existing_reference = session.exec(
|
4477
4477
|
select(CodeReferenceSchema)
|
4478
|
-
.where(CodeReferenceSchema.
|
4478
|
+
.where(CodeReferenceSchema.project_id == project_id)
|
4479
4479
|
.where(
|
4480
4480
|
CodeReferenceSchema.code_repository_id
|
4481
4481
|
== code_reference.code_repository
|
@@ -4489,7 +4489,7 @@ class SqlZenStore(BaseZenStore):
|
|
4489
4489
|
return existing_reference.id
|
4490
4490
|
|
4491
4491
|
new_reference = CodeReferenceSchema.from_request(
|
4492
|
-
code_reference,
|
4492
|
+
code_reference, project_id=project_id
|
4493
4493
|
)
|
4494
4494
|
|
4495
4495
|
session.add(new_reference)
|
@@ -4499,7 +4499,7 @@ class SqlZenStore(BaseZenStore):
|
|
4499
4499
|
self,
|
4500
4500
|
deployment: PipelineDeploymentRequest,
|
4501
4501
|
) -> PipelineDeploymentResponse:
|
4502
|
-
"""Creates a new deployment
|
4502
|
+
"""Creates a new deployment.
|
4503
4503
|
|
4504
4504
|
Args:
|
4505
4505
|
deployment: The deployment to create.
|
@@ -4556,7 +4556,7 @@ class SqlZenStore(BaseZenStore):
|
|
4556
4556
|
|
4557
4557
|
code_reference_id = self._create_or_reuse_code_reference(
|
4558
4558
|
session=session,
|
4559
|
-
|
4559
|
+
project_id=deployment.project,
|
4560
4560
|
code_reference=deployment.code_reference,
|
4561
4561
|
)
|
4562
4562
|
|
@@ -4613,7 +4613,7 @@ class SqlZenStore(BaseZenStore):
|
|
4613
4613
|
A page of all deployments matching the filter criteria.
|
4614
4614
|
"""
|
4615
4615
|
with Session(self.engine) as session:
|
4616
|
-
self.
|
4616
|
+
self._set_filter_project_id(
|
4617
4617
|
filter_model=deployment_filter_model,
|
4618
4618
|
session=session,
|
4619
4619
|
)
|
@@ -4734,7 +4734,7 @@ class SqlZenStore(BaseZenStore):
|
|
4734
4734
|
A list of all templates matching the filter criteria.
|
4735
4735
|
"""
|
4736
4736
|
with Session(self.engine) as session:
|
4737
|
-
self.
|
4737
|
+
self._set_filter_project_id(
|
4738
4738
|
filter_model=template_filter_model,
|
4739
4739
|
session=session,
|
4740
4740
|
)
|
@@ -4910,7 +4910,7 @@ class SqlZenStore(BaseZenStore):
|
|
4910
4910
|
A list of all event_sources matching the filter criteria.
|
4911
4911
|
"""
|
4912
4912
|
with Session(self.engine) as session:
|
4913
|
-
self.
|
4913
|
+
self._set_filter_project_id(
|
4914
4914
|
filter_model=event_source_filter_model,
|
4915
4915
|
session=session,
|
4916
4916
|
)
|
@@ -5128,7 +5128,7 @@ class SqlZenStore(BaseZenStore):
|
|
5128
5128
|
.where(
|
5129
5129
|
PipelineRunSchema.orchestrator_run_id.is_(None) # type: ignore[union-attr]
|
5130
5130
|
)
|
5131
|
-
.where(PipelineRunSchema.
|
5131
|
+
.where(PipelineRunSchema.project_id == pipeline_run.project)
|
5132
5132
|
).first()
|
5133
5133
|
|
5134
5134
|
if not run_schema:
|
@@ -5312,7 +5312,7 @@ class SqlZenStore(BaseZenStore):
|
|
5312
5312
|
A list of all pipeline runs matching the filter criteria.
|
5313
5313
|
"""
|
5314
5314
|
with Session(self.engine) as session:
|
5315
|
-
self.
|
5315
|
+
self._set_filter_project_id(
|
5316
5316
|
filter_model=runs_filter_model,
|
5317
5317
|
session=session,
|
5318
5318
|
)
|
@@ -5451,7 +5451,7 @@ class SqlZenStore(BaseZenStore):
|
|
5451
5451
|
type_ = run_metadata.types[key]
|
5452
5452
|
|
5453
5453
|
run_metadata_schema = RunMetadataSchema(
|
5454
|
-
|
5454
|
+
project_id=run_metadata.project,
|
5455
5455
|
user_id=run_metadata.user,
|
5456
5456
|
stack_component_id=run_metadata.stack_component_id,
|
5457
5457
|
key=key,
|
@@ -5544,7 +5544,7 @@ class SqlZenStore(BaseZenStore):
|
|
5544
5544
|
schedule_filter_model: ScheduleFilter,
|
5545
5545
|
hydrate: bool = False,
|
5546
5546
|
) -> Page[ScheduleResponse]:
|
5547
|
-
"""List all schedules
|
5547
|
+
"""List all schedules.
|
5548
5548
|
|
5549
5549
|
Args:
|
5550
5550
|
schedule_filter_model: All filter parameters including pagination
|
@@ -5556,7 +5556,7 @@ class SqlZenStore(BaseZenStore):
|
|
5556
5556
|
A list of schedules.
|
5557
5557
|
"""
|
5558
5558
|
with Session(self.engine) as session:
|
5559
|
-
self.
|
5559
|
+
self._set_filter_project_id(
|
5560
5560
|
filter_model=schedule_filter_model,
|
5561
5561
|
session=session,
|
5562
5562
|
)
|
@@ -8140,7 +8140,7 @@ class SqlZenStore(BaseZenStore):
|
|
8140
8140
|
A list of all step runs matching the filter criteria.
|
8141
8141
|
"""
|
8142
8142
|
with Session(self.engine) as session:
|
8143
|
-
self.
|
8143
|
+
self._set_filter_project_id(
|
8144
8144
|
filter_model=step_run_filter_model,
|
8145
8145
|
session=session,
|
8146
8146
|
)
|
@@ -8442,7 +8442,7 @@ class SqlZenStore(BaseZenStore):
|
|
8442
8442
|
AnalyticsEvent.RUN_PIPELINE_ENDED
|
8443
8443
|
) as analytics_handler:
|
8444
8444
|
analytics_handler.metadata = {
|
8445
|
-
"
|
8445
|
+
"project_id": pipeline_run.project_id,
|
8446
8446
|
"pipeline_run_id": pipeline_run_id,
|
8447
8447
|
"template_id": pipeline_run.deployment.template_id,
|
8448
8448
|
"status": new_status,
|
@@ -8562,7 +8562,7 @@ class SqlZenStore(BaseZenStore):
|
|
8562
8562
|
A list of all triggers matching the filter criteria.
|
8563
8563
|
"""
|
8564
8564
|
with Session(self.engine) as session:
|
8565
|
-
self.
|
8565
|
+
self._set_filter_project_id(
|
8566
8566
|
filter_model=trigger_filter_model,
|
8567
8567
|
session=session,
|
8568
8568
|
)
|
@@ -8592,7 +8592,7 @@ class SqlZenStore(BaseZenStore):
|
|
8592
8592
|
ValueError: If both a schedule and an event source are provided.
|
8593
8593
|
"""
|
8594
8594
|
with Session(self.engine) as session:
|
8595
|
-
# Check if trigger with the domain key (name,
|
8595
|
+
# Check if trigger with the domain key (name, project, owner)
|
8596
8596
|
# already exists
|
8597
8597
|
existing_trigger = self._get_schema_by_id(
|
8598
8598
|
resource_id=trigger_id,
|
@@ -8719,7 +8719,7 @@ class SqlZenStore(BaseZenStore):
|
|
8719
8719
|
A list of all trigger executions matching the filter criteria.
|
8720
8720
|
"""
|
8721
8721
|
with Session(self.engine) as session:
|
8722
|
-
self.
|
8722
|
+
self._set_filter_project_id(
|
8723
8723
|
filter_model=trigger_execution_filter_model,
|
8724
8724
|
session=session,
|
8725
8725
|
)
|
@@ -9298,214 +9298,212 @@ class SqlZenStore(BaseZenStore):
|
|
9298
9298
|
|
9299
9299
|
return False
|
9300
9300
|
|
9301
|
-
# -----------------------------
|
9301
|
+
# ----------------------------- Projects -----------------------------
|
9302
9302
|
|
9303
|
-
@track_decorator(AnalyticsEvent.
|
9304
|
-
def
|
9305
|
-
|
9306
|
-
) -> WorkspaceResponse:
|
9307
|
-
"""Creates a new workspace.
|
9303
|
+
@track_decorator(AnalyticsEvent.CREATED_PROJECT)
|
9304
|
+
def create_project(self, project: ProjectRequest) -> ProjectResponse:
|
9305
|
+
"""Creates a new project.
|
9308
9306
|
|
9309
9307
|
Args:
|
9310
|
-
|
9308
|
+
project: The project to create.
|
9311
9309
|
|
9312
9310
|
Returns:
|
9313
|
-
The newly created
|
9311
|
+
The newly created project.
|
9314
9312
|
"""
|
9315
9313
|
with Session(self.engine) as session:
|
9316
|
-
# Check if
|
9314
|
+
# Check if project with the given name already exists
|
9317
9315
|
self._verify_name_uniqueness(
|
9318
|
-
resource=
|
9319
|
-
schema=
|
9316
|
+
resource=project,
|
9317
|
+
schema=ProjectSchema,
|
9320
9318
|
session=session,
|
9321
9319
|
)
|
9322
9320
|
|
9323
|
-
# Create the
|
9324
|
-
|
9325
|
-
session.add(
|
9321
|
+
# Create the project
|
9322
|
+
new_project = ProjectSchema.from_request(project)
|
9323
|
+
session.add(new_project)
|
9326
9324
|
session.commit()
|
9327
9325
|
|
9328
|
-
# Explicitly refresh the
|
9329
|
-
session.refresh(
|
9326
|
+
# Explicitly refresh the new_project schema
|
9327
|
+
session.refresh(new_project)
|
9330
9328
|
|
9331
|
-
|
9329
|
+
project_model = new_project.to_model(
|
9332
9330
|
include_metadata=True, include_resources=True
|
9333
9331
|
)
|
9334
9332
|
|
9335
|
-
return
|
9333
|
+
return project_model
|
9336
9334
|
|
9337
|
-
def
|
9338
|
-
self,
|
9339
|
-
) ->
|
9340
|
-
"""Get an existing
|
9335
|
+
def get_project(
|
9336
|
+
self, project_name_or_id: Union[str, UUID], hydrate: bool = True
|
9337
|
+
) -> ProjectResponse:
|
9338
|
+
"""Get an existing project by name or ID.
|
9341
9339
|
|
9342
9340
|
Args:
|
9343
|
-
|
9341
|
+
project_name_or_id: Name or ID of the project to get.
|
9344
9342
|
hydrate: Flag deciding whether to hydrate the output model(s)
|
9345
9343
|
by including metadata fields in the response.
|
9346
9344
|
|
9347
9345
|
Returns:
|
9348
|
-
The requested
|
9346
|
+
The requested project if one was found.
|
9349
9347
|
"""
|
9350
9348
|
with Session(self.engine) as session:
|
9351
|
-
|
9352
|
-
object_name_or_id=
|
9353
|
-
schema_class=
|
9349
|
+
project = self._get_schema_by_name_or_id(
|
9350
|
+
object_name_or_id=project_name_or_id,
|
9351
|
+
schema_class=ProjectSchema,
|
9354
9352
|
session=session,
|
9355
9353
|
)
|
9356
|
-
return
|
9354
|
+
return project.to_model(
|
9357
9355
|
include_metadata=hydrate, include_resources=True
|
9358
9356
|
)
|
9359
9357
|
|
9360
|
-
def
|
9358
|
+
def list_projects(
|
9361
9359
|
self,
|
9362
|
-
|
9360
|
+
project_filter_model: ProjectFilter,
|
9363
9361
|
hydrate: bool = False,
|
9364
|
-
) -> Page[
|
9365
|
-
"""List all
|
9362
|
+
) -> Page[ProjectResponse]:
|
9363
|
+
"""List all projects matching the given filter criteria.
|
9366
9364
|
|
9367
9365
|
Args:
|
9368
|
-
|
9366
|
+
project_filter_model: All filter parameters including pagination
|
9369
9367
|
params.
|
9370
9368
|
hydrate: Flag deciding whether to hydrate the output model(s)
|
9371
9369
|
by including metadata fields in the response.
|
9372
9370
|
|
9373
9371
|
Returns:
|
9374
|
-
A list of all
|
9372
|
+
A list of all projects matching the filter criteria.
|
9375
9373
|
"""
|
9376
9374
|
with Session(self.engine) as session:
|
9377
|
-
query = select(
|
9375
|
+
query = select(ProjectSchema)
|
9378
9376
|
return self.filter_and_paginate(
|
9379
9377
|
session=session,
|
9380
9378
|
query=query,
|
9381
|
-
table=
|
9382
|
-
filter_model=
|
9379
|
+
table=ProjectSchema,
|
9380
|
+
filter_model=project_filter_model,
|
9383
9381
|
hydrate=hydrate,
|
9384
9382
|
)
|
9385
9383
|
|
9386
|
-
def
|
9387
|
-
self,
|
9388
|
-
) ->
|
9389
|
-
"""Update an existing
|
9384
|
+
def update_project(
|
9385
|
+
self, project_id: UUID, project_update: ProjectUpdate
|
9386
|
+
) -> ProjectResponse:
|
9387
|
+
"""Update an existing project.
|
9390
9388
|
|
9391
9389
|
Args:
|
9392
|
-
|
9393
|
-
|
9390
|
+
project_id: The ID of the project to be updated.
|
9391
|
+
project_update: The update to be applied to the project.
|
9394
9392
|
|
9395
9393
|
Returns:
|
9396
|
-
The updated
|
9394
|
+
The updated project.
|
9397
9395
|
|
9398
9396
|
Raises:
|
9399
9397
|
IllegalOperationError: If the request tries to update the name of
|
9400
|
-
the default
|
9398
|
+
the default project.
|
9401
9399
|
"""
|
9402
9400
|
with Session(self.engine) as session:
|
9403
|
-
|
9404
|
-
resource_id=
|
9405
|
-
schema_class=
|
9401
|
+
existing_project = self._get_schema_by_id(
|
9402
|
+
resource_id=project_id,
|
9403
|
+
schema_class=ProjectSchema,
|
9406
9404
|
session=session,
|
9407
9405
|
)
|
9408
9406
|
if (
|
9409
|
-
|
9410
|
-
and "name" in
|
9411
|
-
and
|
9407
|
+
existing_project.name == self._default_project_name
|
9408
|
+
and "name" in project_update.model_fields_set
|
9409
|
+
and project_update.name != existing_project.name
|
9412
9410
|
):
|
9413
9411
|
raise IllegalOperationError(
|
9414
|
-
"The name of the default
|
9412
|
+
"The name of the default project cannot be changed."
|
9415
9413
|
)
|
9416
9414
|
|
9417
9415
|
self._verify_name_uniqueness(
|
9418
|
-
resource=
|
9419
|
-
schema=
|
9416
|
+
resource=project_update,
|
9417
|
+
schema=existing_project,
|
9420
9418
|
session=session,
|
9421
9419
|
)
|
9422
9420
|
|
9423
|
-
# Update the
|
9424
|
-
|
9425
|
-
session.add(
|
9421
|
+
# Update the project
|
9422
|
+
existing_project.update(project_update=project_update)
|
9423
|
+
session.add(existing_project)
|
9426
9424
|
session.commit()
|
9427
9425
|
|
9428
9426
|
# Refresh the Model that was just created
|
9429
|
-
session.refresh(
|
9430
|
-
return
|
9427
|
+
session.refresh(existing_project)
|
9428
|
+
return existing_project.to_model(
|
9431
9429
|
include_metadata=True, include_resources=True
|
9432
9430
|
)
|
9433
9431
|
|
9434
|
-
def
|
9435
|
-
"""Deletes a
|
9432
|
+
def delete_project(self, project_name_or_id: Union[str, UUID]) -> None:
|
9433
|
+
"""Deletes a project.
|
9436
9434
|
|
9437
9435
|
Args:
|
9438
|
-
|
9436
|
+
project_name_or_id: Name or ID of the project to delete.
|
9439
9437
|
|
9440
9438
|
Raises:
|
9441
|
-
IllegalOperationError: If the
|
9439
|
+
IllegalOperationError: If the project is the default project.
|
9442
9440
|
"""
|
9443
9441
|
with Session(self.engine) as session:
|
9444
|
-
# Check if
|
9445
|
-
|
9446
|
-
object_name_or_id=
|
9447
|
-
schema_class=
|
9442
|
+
# Check if project with the given name exists
|
9443
|
+
project = self._get_schema_by_name_or_id(
|
9444
|
+
object_name_or_id=project_name_or_id,
|
9445
|
+
schema_class=ProjectSchema,
|
9448
9446
|
session=session,
|
9449
9447
|
)
|
9450
|
-
if
|
9448
|
+
if project.name == self._default_project_name:
|
9451
9449
|
raise IllegalOperationError(
|
9452
|
-
"The default
|
9450
|
+
"The default project cannot be deleted."
|
9453
9451
|
)
|
9454
9452
|
|
9455
|
-
session.delete(
|
9453
|
+
session.delete(project)
|
9456
9454
|
session.commit()
|
9457
9455
|
|
9458
|
-
def
|
9459
|
-
self, filter_model: Optional[
|
9456
|
+
def count_projects(
|
9457
|
+
self, filter_model: Optional[ProjectFilter] = None
|
9460
9458
|
) -> int:
|
9461
|
-
"""Count all
|
9459
|
+
"""Count all projects.
|
9462
9460
|
|
9463
9461
|
Args:
|
9464
|
-
filter_model: The filter model to use for counting
|
9462
|
+
filter_model: The filter model to use for counting projects.
|
9465
9463
|
|
9466
9464
|
Returns:
|
9467
|
-
The number of
|
9465
|
+
The number of projects.
|
9468
9466
|
"""
|
9469
9467
|
return self._count_entity(
|
9470
|
-
schema=
|
9468
|
+
schema=ProjectSchema, filter_model=filter_model
|
9471
9469
|
)
|
9472
9470
|
|
9473
|
-
def
|
9471
|
+
def set_filter_project_id(
|
9474
9472
|
self,
|
9475
|
-
filter_model:
|
9476
|
-
|
9473
|
+
filter_model: ProjectScopedFilter,
|
9474
|
+
project_name_or_id: Optional[Union[UUID, str]] = None,
|
9477
9475
|
) -> None:
|
9478
|
-
"""Set the
|
9476
|
+
"""Set the project ID on a filter model.
|
9479
9477
|
|
9480
9478
|
Args:
|
9481
|
-
filter_model: The filter model to set the
|
9482
|
-
|
9483
|
-
provided, the
|
9484
|
-
|
9479
|
+
filter_model: The filter model to set the project ID on.
|
9480
|
+
project_name_or_id: The project to set the scope for. If not
|
9481
|
+
provided, the project scope is determined from the request
|
9482
|
+
project filter or the default project, in that order.
|
9485
9483
|
"""
|
9486
9484
|
with Session(self.engine) as session:
|
9487
|
-
self.
|
9485
|
+
self._set_filter_project_id(
|
9488
9486
|
filter_model=filter_model,
|
9489
9487
|
session=session,
|
9490
|
-
|
9488
|
+
project_name_or_id=project_name_or_id,
|
9491
9489
|
)
|
9492
9490
|
|
9493
|
-
def
|
9494
|
-
"""Get or create the default
|
9491
|
+
def _get_or_create_default_project(self) -> ProjectResponse:
|
9492
|
+
"""Get or create the default project if it doesn't exist.
|
9495
9493
|
|
9496
9494
|
Returns:
|
9497
|
-
The default
|
9495
|
+
The default project.
|
9498
9496
|
"""
|
9499
|
-
|
9497
|
+
default_project_name = self._default_project_name
|
9500
9498
|
|
9501
9499
|
try:
|
9502
|
-
return self.
|
9500
|
+
return self.get_project(default_project_name)
|
9503
9501
|
except KeyError:
|
9504
9502
|
logger.info(
|
9505
|
-
f"Creating default
|
9503
|
+
f"Creating default project '{default_project_name}' ..."
|
9506
9504
|
)
|
9507
|
-
return self.
|
9508
|
-
|
9505
|
+
return self.create_project(
|
9506
|
+
ProjectRequest(name=default_project_name)
|
9509
9507
|
)
|
9510
9508
|
|
9511
9509
|
# =======================
|
@@ -9530,11 +9528,11 @@ class SqlZenStore(BaseZenStore):
|
|
9530
9528
|
query = select(func.count(schema.id)) # type: ignore[arg-type]
|
9531
9529
|
|
9532
9530
|
if filter_model:
|
9533
|
-
if isinstance(filter_model,
|
9534
|
-
self.
|
9531
|
+
if isinstance(filter_model, ProjectScopedFilter):
|
9532
|
+
self._set_filter_project_id(
|
9535
9533
|
filter_model=filter_model,
|
9536
9534
|
session=session,
|
9537
|
-
|
9535
|
+
project_name_or_id=filter_model.project,
|
9538
9536
|
)
|
9539
9537
|
query = filter_model.apply_filter(query=query, table=schema)
|
9540
9538
|
|
@@ -9599,26 +9597,26 @@ class SqlZenStore(BaseZenStore):
|
|
9599
9597
|
schema_class: Type[AnySchema],
|
9600
9598
|
session: Session,
|
9601
9599
|
resource_type: Optional[str] = None,
|
9602
|
-
|
9600
|
+
project_id: Optional[UUID] = None,
|
9603
9601
|
) -> AnySchema:
|
9604
9602
|
"""Query a schema by its 'id' field.
|
9605
9603
|
|
9606
9604
|
Args:
|
9607
9605
|
resource_id: The ID of the resource to query.
|
9608
|
-
schema_class: The schema class to query. E.g., `
|
9606
|
+
schema_class: The schema class to query. E.g., `StackSchema`.
|
9609
9607
|
session: The database session to use.
|
9610
9608
|
resource_type: Optional name of the resource type to use in error
|
9611
9609
|
messages. If not provided, the type name will be inferred
|
9612
9610
|
from the schema class.
|
9613
|
-
|
9611
|
+
project_id: Optional ID of a project to filter by.
|
9614
9612
|
|
9615
9613
|
Returns:
|
9616
9614
|
The schema object.
|
9617
9615
|
|
9618
9616
|
Raises:
|
9619
9617
|
KeyError: if the object couldn't be found.
|
9620
|
-
RuntimeError: if the schema is not
|
9621
|
-
|
9618
|
+
RuntimeError: if the schema is not project-scoped but the
|
9619
|
+
project ID is provided.
|
9622
9620
|
"""
|
9623
9621
|
resource_type = resource_type or get_resource_type_name(schema_class)
|
9624
9622
|
error_msg = (
|
@@ -9626,14 +9624,14 @@ class SqlZenStore(BaseZenStore):
|
|
9626
9624
|
f"'{resource_id}': No {resource_type} with this ID found"
|
9627
9625
|
)
|
9628
9626
|
query = select(schema_class).where(schema_class.id == resource_id)
|
9629
|
-
if
|
9630
|
-
error_msg += f" in
|
9631
|
-
if not hasattr(schema_class, "
|
9627
|
+
if project_id:
|
9628
|
+
error_msg += f" in project `{str(project_id)}`"
|
9629
|
+
if not hasattr(schema_class, "project_id"):
|
9632
9630
|
raise RuntimeError(
|
9633
|
-
f"Schema {schema_class.__name__} is not
|
9631
|
+
f"Schema {schema_class.__name__} is not project-scoped."
|
9634
9632
|
)
|
9635
9633
|
|
9636
|
-
query = query.where(schema_class.
|
9634
|
+
query = query.where(schema_class.project_id == project_id) # type: ignore[attr-defined]
|
9637
9635
|
|
9638
9636
|
schema = session.exec(query).first()
|
9639
9637
|
|
@@ -9646,16 +9644,16 @@ class SqlZenStore(BaseZenStore):
|
|
9646
9644
|
object_name_or_id: Union[str, UUID],
|
9647
9645
|
schema_class: Type[AnyNamedSchema],
|
9648
9646
|
session: Session,
|
9649
|
-
|
9647
|
+
project_name_or_id: Optional[Union[UUID, str]] = None,
|
9650
9648
|
) -> AnyNamedSchema:
|
9651
9649
|
"""Query a schema by its 'name' or 'id' field.
|
9652
9650
|
|
9653
9651
|
Args:
|
9654
9652
|
object_name_or_id: The name or ID of the object to query.
|
9655
|
-
schema_class: The schema class to query. E.g., `
|
9653
|
+
schema_class: The schema class to query. E.g., `ProjectSchema`.
|
9656
9654
|
session: The database session to use.
|
9657
|
-
|
9658
|
-
Required if the resource is
|
9655
|
+
project_name_or_id: The name or ID of the project to filter by.
|
9656
|
+
Required if the resource is project-scoped and the
|
9659
9657
|
object_name_or_id is not a UUID.
|
9660
9658
|
|
9661
9659
|
Returns:
|
@@ -9680,17 +9678,17 @@ class SqlZenStore(BaseZenStore):
|
|
9680
9678
|
)
|
9681
9679
|
|
9682
9680
|
query = select(schema_class).where(filter_params)
|
9683
|
-
if
|
9684
|
-
if uuid_utils.is_valid_uuid(
|
9681
|
+
if project_name_or_id and hasattr(schema_class, "project_id"):
|
9682
|
+
if uuid_utils.is_valid_uuid(project_name_or_id):
|
9685
9683
|
query = query.where(
|
9686
|
-
schema_class.
|
9684
|
+
schema_class.project_id == project_name_or_id # type: ignore[attr-defined]
|
9687
9685
|
)
|
9688
9686
|
else:
|
9689
|
-
# Join the
|
9690
|
-
query = query.join(
|
9691
|
-
|
9687
|
+
# Join the project table to get the project name
|
9688
|
+
query = query.join(ProjectSchema).where(
|
9689
|
+
ProjectSchema.name == project_name_or_id
|
9692
9690
|
)
|
9693
|
-
error_msg += f" in
|
9691
|
+
error_msg += f" in project `{project_name_or_id}`."
|
9694
9692
|
else:
|
9695
9693
|
error_msg += "."
|
9696
9694
|
|
@@ -9734,13 +9732,13 @@ class SqlZenStore(BaseZenStore):
|
|
9734
9732
|
1. Fetch a referenced resource from the database.
|
9735
9733
|
2. Enforce the scope relationship rules established between any main
|
9736
9734
|
resource and its references:
|
9737
|
-
a) a
|
9738
|
-
another
|
9739
|
-
the same
|
9740
|
-
b) a
|
9735
|
+
a) a project-scoped resource (e.g. pipeline run) may reference
|
9736
|
+
another project-scoped resource (e.g. pipeline) if it is within
|
9737
|
+
the same project.
|
9738
|
+
b) a project-scoped resource (e.g. pipeline run) may reference a
|
9741
9739
|
global-scoped resource (e.g. stack).
|
9742
9740
|
c) a global-scoped resource (e.g. stack) may never reference a
|
9743
|
-
|
9741
|
+
project-scoped resource (e.g. pipeline).
|
9744
9742
|
d) a global-scoped resource (e.g. stack) may reference another
|
9745
9743
|
global-scoped resource (e.g. component).
|
9746
9744
|
|
@@ -9760,7 +9758,7 @@ class SqlZenStore(BaseZenStore):
|
|
9760
9758
|
The referenced resource.
|
9761
9759
|
|
9762
9760
|
Raises:
|
9763
|
-
RuntimeError: If the schema has no
|
9761
|
+
RuntimeError: If the schema has no project_id attribute.
|
9764
9762
|
KeyError: If the referenced resource is not found.
|
9765
9763
|
"""
|
9766
9764
|
if reference_id is None:
|
@@ -9783,53 +9781,51 @@ class SqlZenStore(BaseZenStore):
|
|
9783
9781
|
if isinstance(resource, BaseSchema):
|
9784
9782
|
operation = "updated"
|
9785
9783
|
|
9786
|
-
|
9787
|
-
|
9788
|
-
if isinstance(resource,
|
9789
|
-
|
9790
|
-
|
9784
|
+
resource_project_id: Optional[UUID] = None
|
9785
|
+
resource_project_name: Optional[str] = None
|
9786
|
+
if isinstance(resource, ProjectScopedRequest):
|
9787
|
+
resource_project_id = resource.project
|
9788
|
+
resource_project_name = str(resource.project)
|
9791
9789
|
elif isinstance(resource, BaseSchema):
|
9792
|
-
|
9793
|
-
|
9794
|
-
if
|
9795
|
-
assert isinstance(
|
9796
|
-
|
9790
|
+
resource_project_id = getattr(resource, "project_id", None)
|
9791
|
+
resource_project = getattr(resource, "project", None)
|
9792
|
+
if resource_project:
|
9793
|
+
assert isinstance(resource_project, ProjectSchema)
|
9794
|
+
resource_project_name = resource_project.name
|
9797
9795
|
|
9798
9796
|
error_msg = (
|
9799
9797
|
f"The {reference_type} with ID {str(reference_id)} referenced by "
|
9800
9798
|
f"the {resource_type} being {operation} was not found"
|
9801
9799
|
)
|
9802
9800
|
|
9803
|
-
|
9804
|
-
reference_schema, "workspace_id"
|
9805
|
-
)
|
9801
|
+
reference_is_project_scoped = hasattr(reference_schema, "project_id")
|
9806
9802
|
|
9807
9803
|
# There's one particular case that should never happen: if a global
|
9808
|
-
# resource (e.g. a stack) references a
|
9804
|
+
# resource (e.g. a stack) references a project-scoped resource
|
9809
9805
|
# (e.g. a pipeline), this is a design error.
|
9810
|
-
if
|
9806
|
+
if resource_project_id is None and reference_is_project_scoped:
|
9811
9807
|
raise RuntimeError(
|
9812
9808
|
f"A global resource {resource_type} cannot reference a "
|
9813
|
-
f"
|
9809
|
+
f"project-scoped resource {reference_type}. This is a "
|
9814
9810
|
"design error."
|
9815
9811
|
)
|
9816
9812
|
|
9817
|
-
# Filter the reference by
|
9818
|
-
#
|
9819
|
-
|
9820
|
-
|
9813
|
+
# Filter the reference by project if the resource itself is
|
9814
|
+
# project-scoped and the reference is project-scoped.
|
9815
|
+
reference_project_filter = (
|
9816
|
+
resource_project_id if reference_is_project_scoped else None
|
9821
9817
|
)
|
9822
9818
|
try:
|
9823
9819
|
return self._get_schema_by_id(
|
9824
9820
|
resource_id=reference_id,
|
9825
9821
|
schema_class=reference_schema,
|
9826
9822
|
session=session,
|
9827
|
-
|
9823
|
+
project_id=reference_project_filter,
|
9828
9824
|
resource_type=reference_type,
|
9829
9825
|
)
|
9830
9826
|
except KeyError:
|
9831
|
-
if
|
9832
|
-
error_msg += f" in the '{
|
9827
|
+
if reference_project_filter:
|
9828
|
+
error_msg += f" in the '{resource_project_name}' project"
|
9833
9829
|
|
9834
9830
|
raise KeyError(error_msg)
|
9835
9831
|
|
@@ -9851,57 +9847,51 @@ class SqlZenStore(BaseZenStore):
|
|
9851
9847
|
|
9852
9848
|
request_model.user = self._get_active_user(session).id
|
9853
9849
|
|
9854
|
-
def
|
9850
|
+
def _set_filter_project_id(
|
9855
9851
|
self,
|
9856
|
-
filter_model:
|
9852
|
+
filter_model: ProjectScopedFilter,
|
9857
9853
|
session: Session,
|
9858
|
-
|
9854
|
+
project_name_or_id: Optional[Union[UUID, str]] = None,
|
9859
9855
|
) -> None:
|
9860
|
-
"""Set the
|
9856
|
+
"""Set the project ID on a filter model.
|
9861
9857
|
|
9862
9858
|
Args:
|
9863
|
-
filter_model: The filter model to set the
|
9859
|
+
filter_model: The filter model to set the project ID on.
|
9864
9860
|
session: The DB session to use for queries.
|
9865
|
-
|
9866
|
-
provided, the
|
9867
|
-
|
9861
|
+
project_name_or_id: The project to set the scope for. If not
|
9862
|
+
provided, the project scope is determined from the request
|
9863
|
+
project filter or the default project, in that order.
|
9868
9864
|
|
9869
9865
|
Raises:
|
9870
|
-
ValueError: If the
|
9866
|
+
ValueError: If the project scope is missing from the filter.
|
9871
9867
|
"""
|
9872
|
-
if
|
9873
|
-
|
9874
|
-
object_name_or_id=
|
9875
|
-
schema_class=
|
9868
|
+
if project_name_or_id:
|
9869
|
+
project = self._get_schema_by_name_or_id(
|
9870
|
+
object_name_or_id=project_name_or_id,
|
9871
|
+
schema_class=ProjectSchema,
|
9876
9872
|
session=session,
|
9877
9873
|
)
|
9878
|
-
|
9879
|
-
elif filter_model.
|
9880
|
-
|
9881
|
-
object_name_or_id=filter_model.
|
9882
|
-
schema_class=
|
9874
|
+
project_id = project.id
|
9875
|
+
elif filter_model.project:
|
9876
|
+
project = self._get_schema_by_name_or_id(
|
9877
|
+
object_name_or_id=filter_model.project,
|
9878
|
+
schema_class=ProjectSchema,
|
9883
9879
|
session=session,
|
9884
9880
|
)
|
9885
|
-
|
9881
|
+
project_id = project.id
|
9886
9882
|
else:
|
9887
|
-
# Use the default
|
9888
|
-
|
9889
|
-
|
9890
|
-
|
9891
|
-
|
9892
|
-
|
9893
|
-
|
9894
|
-
|
9895
|
-
|
9896
|
-
|
9897
|
-
schema_class=WorkspaceSchema,
|
9898
|
-
session=session,
|
9899
|
-
)
|
9900
|
-
workspace_id = workspace.id
|
9901
|
-
except KeyError:
|
9902
|
-
raise ValueError("Workspace scope missing from the filter")
|
9883
|
+
# Use the default project as a last resort.
|
9884
|
+
try:
|
9885
|
+
project = self._get_schema_by_name_or_id(
|
9886
|
+
object_name_or_id=self._default_project_name,
|
9887
|
+
schema_class=ProjectSchema,
|
9888
|
+
session=session,
|
9889
|
+
)
|
9890
|
+
project_id = project.id
|
9891
|
+
except KeyError:
|
9892
|
+
raise ValueError("Project scope missing from the filter")
|
9903
9893
|
|
9904
|
-
filter_model.
|
9894
|
+
filter_model.project = project_id
|
9905
9895
|
|
9906
9896
|
def _verify_name_uniqueness(
|
9907
9897
|
self,
|
@@ -9920,9 +9910,9 @@ class SqlZenStore(BaseZenStore):
|
|
9920
9910
|
|
9921
9911
|
The scope in which the name uniqueness is verified depends on the
|
9922
9912
|
entity type: for global resources (e.g. stack, service-connector), the
|
9923
|
-
name must be globally unique, while for
|
9913
|
+
name must be globally unique, while for project-scoped resources (e.g.
|
9924
9914
|
pipeline, artifact, etc.), the name must be unique within the
|
9925
|
-
|
9915
|
+
project.
|
9926
9916
|
|
9927
9917
|
Args:
|
9928
9918
|
resource: The resource to verify the name uniqueness for. This can
|
@@ -9954,13 +9944,13 @@ class SqlZenStore(BaseZenStore):
|
|
9954
9944
|
raise RuntimeError(f"Schema {schema_class.__name__} has no name.")
|
9955
9945
|
|
9956
9946
|
operation: Literal["create", "update"] = "create"
|
9957
|
-
|
9947
|
+
project_id: Optional[UUID] = None
|
9958
9948
|
if isinstance(resource, BaseRequest):
|
9959
9949
|
# Create operation
|
9960
|
-
if isinstance(resource,
|
9961
|
-
|
9950
|
+
if isinstance(resource, ProjectScopedRequest):
|
9951
|
+
project_id = resource.project
|
9962
9952
|
else:
|
9963
|
-
|
9953
|
+
project_id = None
|
9964
9954
|
else:
|
9965
9955
|
# Update operation
|
9966
9956
|
if name is None:
|
@@ -9978,29 +9968,29 @@ class SqlZenStore(BaseZenStore):
|
|
9978
9968
|
# If the name is not being changed during an update, we don't
|
9979
9969
|
# need to verify the name uniqueness.
|
9980
9970
|
return
|
9981
|
-
|
9971
|
+
project_id = getattr(schema, "project_id", None)
|
9982
9972
|
operation = "update"
|
9983
9973
|
|
9984
9974
|
query = select(schema_class).where(schema_class.name == name)
|
9985
9975
|
|
9986
|
-
# We "detect" if the entity is
|
9987
|
-
#
|
9988
|
-
if
|
9989
|
-
if not hasattr(schema_class, "
|
9990
|
-
schema_class, "
|
9976
|
+
# We "detect" if the entity is project-scoped by looking at the
|
9977
|
+
# project_id attribute.
|
9978
|
+
if project_id:
|
9979
|
+
if not hasattr(schema_class, "project_id") or not hasattr(
|
9980
|
+
schema_class, "project"
|
9991
9981
|
):
|
9992
9982
|
raise RuntimeError(
|
9993
|
-
f"Model {type(resource)} is
|
9983
|
+
f"Model {type(resource)} is project-scoped, but "
|
9994
9984
|
f"schema {schema_class.__name__} has no "
|
9995
|
-
"
|
9985
|
+
"project_id and project attributes."
|
9996
9986
|
)
|
9997
|
-
query = query.where(schema_class.
|
9987
|
+
query = query.where(schema_class.project_id == project_id) # type: ignore[attr-defined]
|
9998
9988
|
|
9999
9989
|
existing_entry = session.exec(query).first()
|
10000
9990
|
if existing_entry is not None:
|
10001
9991
|
resource_name = get_resource_type_name(schema_class)
|
10002
|
-
if
|
10003
|
-
scope = f" in the '{existing_entry.
|
9992
|
+
if project_id:
|
9993
|
+
scope = f" in the '{existing_entry.project.name}' project" # type: ignore[attr-defined]
|
10004
9994
|
else:
|
10005
9995
|
scope = ""
|
10006
9996
|
raise EntityExistsError(
|
@@ -10149,14 +10139,14 @@ class SqlZenStore(BaseZenStore):
|
|
10149
10139
|
def get_model_by_name_or_id(
|
10150
10140
|
self,
|
10151
10141
|
model_name_or_id: Union[str, UUID],
|
10152
|
-
|
10142
|
+
project: UUID,
|
10153
10143
|
hydrate: bool = True,
|
10154
10144
|
) -> ModelResponse:
|
10155
10145
|
"""Get a model by name or ID.
|
10156
10146
|
|
10157
10147
|
Args:
|
10158
10148
|
model_name_or_id: The name or ID of the model to get.
|
10159
|
-
|
10149
|
+
project: The project ID of the model to get.
|
10160
10150
|
hydrate: Flag deciding whether to hydrate the output model(s)
|
10161
10151
|
by including metadata fields in the response.
|
10162
10152
|
|
@@ -10168,7 +10158,7 @@ class SqlZenStore(BaseZenStore):
|
|
10168
10158
|
object_name_or_id=model_name_or_id,
|
10169
10159
|
schema_class=ModelSchema,
|
10170
10160
|
session=session,
|
10171
|
-
|
10161
|
+
project_name_or_id=project,
|
10172
10162
|
)
|
10173
10163
|
|
10174
10164
|
return model.to_model(
|
@@ -10192,7 +10182,7 @@ class SqlZenStore(BaseZenStore):
|
|
10192
10182
|
A page of all models.
|
10193
10183
|
"""
|
10194
10184
|
with Session(self.engine) as session:
|
10195
|
-
self.
|
10185
|
+
self._set_filter_project_id(
|
10196
10186
|
filter_model=model_filter_model,
|
10197
10187
|
session=session,
|
10198
10188
|
)
|
@@ -10295,7 +10285,7 @@ class SqlZenStore(BaseZenStore):
|
|
10295
10285
|
except EntityExistsError:
|
10296
10286
|
return False, self.get_model_by_name_or_id(
|
10297
10287
|
model_name_or_id=model_request.name,
|
10298
|
-
|
10288
|
+
project=model_request.project,
|
10299
10289
|
)
|
10300
10290
|
|
10301
10291
|
def _get_next_numeric_version_for_model(
|
@@ -10460,7 +10450,7 @@ class SqlZenStore(BaseZenStore):
|
|
10460
10450
|
)
|
10461
10451
|
track(
|
10462
10452
|
event=AnalyticsEvent.CREATED_MODEL_VERSION,
|
10463
|
-
metadata={"
|
10453
|
+
metadata={"project_id": model_version.project.id},
|
10464
10454
|
)
|
10465
10455
|
return True, model_version
|
10466
10456
|
except EntityCreationError:
|
@@ -10517,7 +10507,7 @@ class SqlZenStore(BaseZenStore):
|
|
10517
10507
|
ethics=configured_model.ethics,
|
10518
10508
|
save_models_to_registry=configured_model.save_models_to_registry,
|
10519
10509
|
user=pipeline_or_step_run.user_id,
|
10520
|
-
|
10510
|
+
project=pipeline_or_step_run.project_id,
|
10521
10511
|
)
|
10522
10512
|
|
10523
10513
|
_, model_response = self._get_or_create_model(
|
@@ -10547,7 +10537,7 @@ class SqlZenStore(BaseZenStore):
|
|
10547
10537
|
description=configured_model.description,
|
10548
10538
|
tags=configured_model.tags,
|
10549
10539
|
user=pipeline_or_step_run.user_id,
|
10550
|
-
|
10540
|
+
project=pipeline_or_step_run.project_id,
|
10551
10541
|
)
|
10552
10542
|
|
10553
10543
|
_, model_version_response = self._get_or_create_model_version(
|
@@ -10750,7 +10740,7 @@ class SqlZenStore(BaseZenStore):
|
|
10750
10740
|
model = self._get_schema_by_name_or_id(
|
10751
10741
|
object_name_or_id=filter_model.model,
|
10752
10742
|
schema_class=ModelSchema,
|
10753
|
-
|
10743
|
+
project_name_or_id=filter_model.project,
|
10754
10744
|
session=session,
|
10755
10745
|
)
|
10756
10746
|
else:
|
@@ -10775,7 +10765,7 @@ class SqlZenStore(BaseZenStore):
|
|
10775
10765
|
A page of all model versions.
|
10776
10766
|
"""
|
10777
10767
|
with Session(self.engine) as session:
|
10778
|
-
self.
|
10768
|
+
self._set_filter_project_id(
|
10779
10769
|
filter_model=model_version_filter_model,
|
10780
10770
|
session=session,
|
10781
10771
|
)
|
@@ -11269,7 +11259,7 @@ class SqlZenStore(BaseZenStore):
|
|
11269
11259
|
"""Gets a tag schema by name or ID.
|
11270
11260
|
|
11271
11261
|
This is a helper method that is used in various places to find a tag
|
11272
|
-
by its name
|
11262
|
+
by its name or ID.
|
11273
11263
|
|
11274
11264
|
Args:
|
11275
11265
|
tag_name_or_id: The name or ID of the tag to get.
|