fdai-deployment-cli 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fdai_deployment_cli/__about__.py +3 -0
- fdai_deployment_cli/__init__.py +5 -0
- fdai_deployment_cli/__main__.py +5 -0
- fdai_deployment_cli/acr_publish.py +585 -0
- fdai_deployment_cli/aks_catalog_review.py +137 -0
- fdai_deployment_cli/aks_historical_reconciliation.py +732 -0
- fdai_deployment_cli/aks_job_environment.py +59 -0
- fdai_deployment_cli/aks_job_execution.py +794 -0
- fdai_deployment_cli/aks_kubernetes_defaults.py +66 -0
- fdai_deployment_cli/aks_preflight.py +422 -0
- fdai_deployment_cli/aks_readiness.py +220 -0
- fdai_deployment_cli/aks_service_update.py +180 -0
- fdai_deployment_cli/aks_workload_jobs.py +281 -0
- fdai_deployment_cli/application_state_adoption.py +347 -0
- fdai_deployment_cli/azure_actor.py +61 -0
- fdai_deployment_cli/azure_naming.py +252 -0
- fdai_deployment_cli/azure_retail_prices.py +173 -0
- fdai_deployment_cli/bootstrap_approval.py +99 -0
- fdai_deployment_cli/bootstrap_channel.py +321 -0
- fdai_deployment_cli/bootstrap_executor.py +311 -0
- fdai_deployment_cli/bootstrap_plan.py +237 -0
- fdai_deployment_cli/bootstrap_reconcile.py +402 -0
- fdai_deployment_cli/bootstrap_support.py +138 -0
- fdai_deployment_cli/bootstrap_worker.py +448 -0
- fdai_deployment_cli/bundle.py +352 -0
- fdai_deployment_cli/capacity.py +143 -0
- fdai_deployment_cli/catalog_review_profile.py +306 -0
- fdai_deployment_cli/cli.py +798 -0
- fdai_deployment_cli/cli_help.py +214 -0
- fdai_deployment_cli/cli_parser.py +791 -0
- fdai_deployment_cli/cli_plan.py +450 -0
- fdai_deployment_cli/compiler.py +99 -0
- fdai_deployment_cli/console_artifact.py +218 -0
- fdai_deployment_cli/console_config.py +214 -0
- fdai_deployment_cli/console_update.py +832 -0
- fdai_deployment_cli/contracts.py +307 -0
- fdai_deployment_cli/control_package.py +140 -0
- fdai_deployment_cli/deadline_transport.py +70 -0
- fdai_deployment_cli/deployment_cost.py +131 -0
- fdai_deployment_cli/deployment_deadline.py +28 -0
- fdai_deployment_cli/deployment_kit.py +790 -0
- fdai_deployment_cli/deployment_kit_cache.py +191 -0
- fdai_deployment_cli/deployment_progress.py +648 -0
- fdai_deployment_cli/development_authority_cli.py +56 -0
- fdai_deployment_cli/development_authority_profile.py +319 -0
- fdai_deployment_cli/doctor.py +136 -0
- fdai_deployment_cli/entitlement_preflight.py +68 -0
- fdai_deployment_cli/entitlement_transport.py +49 -0
- fdai_deployment_cli/entra_bootstrap.py +579 -0
- fdai_deployment_cli/entra_control_profile.py +214 -0
- fdai_deployment_cli/entra_graph.py +349 -0
- fdai_deployment_cli/entra_profiles.py +215 -0
- fdai_deployment_cli/entra_source.py +244 -0
- fdai_deployment_cli/execution_bundle.py +478 -0
- fdai_deployment_cli/execution_copies.py +63 -0
- fdai_deployment_cli/foundation_adoption.py +438 -0
- fdai_deployment_cli/foundation_adoption_evidence.py +245 -0
- fdai_deployment_cli/foundation_adoption_host.py +157 -0
- fdai_deployment_cli/foundation_adoption_transport.py +51 -0
- fdai_deployment_cli/foundation_failure.py +72 -0
- fdai_deployment_cli/foundation_image.py +168 -0
- fdai_deployment_cli/foundation_input.py +235 -0
- fdai_deployment_cli/foundation_output.py +154 -0
- fdai_deployment_cli/foundation_plan.py +381 -0
- fdai_deployment_cli/foundation_process.py +64 -0
- fdai_deployment_cli/foundation_progress.py +231 -0
- fdai_deployment_cli/github_actions.py +171 -0
- fdai_deployment_cli/github_apply_artifact.py +259 -0
- fdai_deployment_cli/github_artifact_io.py +48 -0
- fdai_deployment_cli/github_dispatch.py +248 -0
- fdai_deployment_cli/github_plan_artifact.py +181 -0
- fdai_deployment_cli/github_provider_artifact.py +170 -0
- fdai_deployment_cli/github_transport.py +46 -0
- fdai_deployment_cli/github_workflow_values.py +312 -0
- fdai_deployment_cli/hub_image_deploy.py +354 -0
- fdai_deployment_cli/hub_image_stage.py +162 -0
- fdai_deployment_cli/hub_image_trust.py +281 -0
- fdai_deployment_cli/hub_ownership.py +161 -0
- fdai_deployment_cli/hub_workload_handoff.py +264 -0
- fdai_deployment_cli/hub_workload_handoff_inputs.py +227 -0
- fdai_deployment_cli/identity_profile.py +227 -0
- fdai_deployment_cli/image_targets.py +197 -0
- fdai_deployment_cli/installation_scope.py +280 -0
- fdai_deployment_cli/license.py +311 -0
- fdai_deployment_cli/license_issue.py +239 -0
- fdai_deployment_cli/lifecycle_authorization.py +252 -0
- fdai_deployment_cli/lifecycle_commands.py +173 -0
- fdai_deployment_cli/lifecycle_configuration.py +722 -0
- fdai_deployment_cli/lifecycle_configuration_artifact.py +45 -0
- fdai_deployment_cli/lifecycle_configuration_cli.py +324 -0
- fdai_deployment_cli/lifecycle_configuration_document.py +74 -0
- fdai_deployment_cli/lifecycle_configuration_entities.py +146 -0
- fdai_deployment_cli/lifecycle_configuration_package.py +492 -0
- fdai_deployment_cli/lifecycle_configuration_publication.py +277 -0
- fdai_deployment_cli/lifecycle_configuration_publication_cli.py +191 -0
- fdai_deployment_cli/lifecycle_configuration_sealing.py +127 -0
- fdai_deployment_cli/lifecycle_enrollment.py +45 -0
- fdai_deployment_cli/lifecycle_infrastructure_agent.py +174 -0
- fdai_deployment_cli/lifecycle_installation_binding.py +65 -0
- fdai_deployment_cli/lifecycle_native.py +247 -0
- fdai_deployment_cli/lifecycle_native_reconciliation.py +568 -0
- fdai_deployment_cli/lifecycle_native_transition.py +339 -0
- fdai_deployment_cli/lifecycle_ownership_exclusion.py +78 -0
- fdai_deployment_cli/lifecycle_plan.py +800 -0
- fdai_deployment_cli/lifecycle_promotion.py +185 -0
- fdai_deployment_cli/lifecycle_release.py +308 -0
- fdai_deployment_cli/lifecycle_self_upgrade.py +202 -0
- fdai_deployment_cli/lifecycle_upgrade_bundle.py +176 -0
- fdai_deployment_cli/notification_recipients_input.py +222 -0
- fdai_deployment_cli/observer_artifact.py +106 -0
- fdai_deployment_cli/oci_archive.py +353 -0
- fdai_deployment_cli/offline_kit.py +797 -0
- fdai_deployment_cli/offline_prepare.py +135 -0
- fdai_deployment_cli/operational_evidence_verifier_input.py +575 -0
- fdai_deployment_cli/plan_input.py +101 -0
- fdai_deployment_cli/private_output.py +192 -0
- fdai_deployment_cli/profile.py +70 -0
- fdai_deployment_cli/progress.py +183 -0
- fdai_deployment_cli/readiness.py +286 -0
- fdai_deployment_cli/run_command_receiver.py +418 -0
- fdai_deployment_cli/runtime_build.py +386 -0
- fdai_deployment_cli/runtime_profile.py +287 -0
- fdai_deployment_cli/runtime_release.py +745 -0
- fdai_deployment_cli/runtime_stage.py +82 -0
- fdai_deployment_cli/runtime_support_installation.py +215 -0
- fdai_deployment_cli/simulation.py +80 -0
- fdai_deployment_cli/source_application_inputs.py +397 -0
- fdai_deployment_cli/source_azure.py +586 -0
- fdai_deployment_cli/source_change_cli.py +107 -0
- fdai_deployment_cli/source_change_parser.py +75 -0
- fdai_deployment_cli/source_change_plan.py +181 -0
- fdai_deployment_cli/source_changed_update.py +196 -0
- fdai_deployment_cli/source_deploy.py +170 -0
- fdai_deployment_cli/source_foundation.py +174 -0
- fdai_deployment_cli/source_host_tools.py +163 -0
- fdai_deployment_cli/source_image_stage.py +400 -0
- fdai_deployment_cli/source_input.py +173 -0
- fdai_deployment_cli/source_projection.py +164 -0
- fdai_deployment_cli/source_receiver.py +76 -0
- fdai_deployment_cli/source_recovery.py +264 -0
- fdai_deployment_cli/source_runtime.py +81 -0
- fdai_deployment_cli/source_runtime_support.py +368 -0
- fdai_deployment_cli/source_service_update.py +726 -0
- fdai_deployment_cli/source_service_update_cli.py +47 -0
- fdai_deployment_cli/source_snapshot.py +178 -0
- fdai_deployment_cli/source_teardown.py +574 -0
- fdai_deployment_cli/source_terraform.py +85 -0
- fdai_deployment_cli/source_transport.py +399 -0
- fdai_deployment_cli/standalone_aks_inventory.py +558 -0
- fdai_deployment_cli/standalone_aks_job_execution.py +769 -0
- fdai_deployment_cli/standalone_aks_nodepool_guard.py +141 -0
- fdai_deployment_cli/standalone_application.py +977 -0
- fdai_deployment_cli/standalone_application_completion.py +151 -0
- fdai_deployment_cli/standalone_application_input_recovery.py +107 -0
- fdai_deployment_cli/standalone_application_recovery.py +116 -0
- fdai_deployment_cli/standalone_approval_input.py +24 -0
- fdai_deployment_cli/standalone_catalog_checkpoint.py +122 -0
- fdai_deployment_cli/standalone_catalog_review.py +251 -0
- fdai_deployment_cli/standalone_checkpoint_failure.py +127 -0
- fdai_deployment_cli/standalone_console_publish.py +174 -0
- fdai_deployment_cli/standalone_database_principals.py +41 -0
- fdai_deployment_cli/standalone_deploy.py +638 -0
- fdai_deployment_cli/standalone_development_authority.py +166 -0
- fdai_deployment_cli/standalone_foundation_adoption.py +87 -0
- fdai_deployment_cli/standalone_foundation_transition.py +767 -0
- fdai_deployment_cli/standalone_host.py +4608 -0
- fdai_deployment_cli/standalone_host_rebind.py +226 -0
- fdai_deployment_cli/standalone_host_residual.py +104 -0
- fdai_deployment_cli/standalone_host_state.py +145 -0
- fdai_deployment_cli/standalone_host_values.py +563 -0
- fdai_deployment_cli/standalone_kubernetes_tools.py +100 -0
- fdai_deployment_cli/standalone_license_installation.py +177 -0
- fdai_deployment_cli/standalone_management_egress.py +83 -0
- fdai_deployment_cli/standalone_migration_evidence.py +38 -0
- fdai_deployment_cli/standalone_operational_evidence.py +356 -0
- fdai_deployment_cli/standalone_planned_outputs.py +147 -0
- fdai_deployment_cli/standalone_private_access_probe.py +100 -0
- fdai_deployment_cli/standalone_product_profile.py +110 -0
- fdai_deployment_cli/standalone_remote_prepare.py +442 -0
- fdai_deployment_cli/standalone_residual_apply.py +540 -0
- fdai_deployment_cli/standalone_residual_successor.py +431 -0
- fdai_deployment_cli/standalone_review.py +266 -0
- fdai_deployment_cli/standalone_service_update_finalization.py +116 -0
- fdai_deployment_cli/standalone_service_update_recovery.py +277 -0
- fdai_deployment_cli/standalone_stage_targets.py +325 -0
- fdai_deployment_cli/standalone_status.py +67 -0
- fdai_deployment_cli/standalone_terraform_environment.py +237 -0
- fdai_deployment_cli/standalone_transfer_cleanup.py +237 -0
- fdai_deployment_cli/standalone_trial_activation.py +193 -0
- fdai_deployment_cli/state.py +434 -0
- fdai_deployment_cli/state_handoff.py +253 -0
- fdai_deployment_cli/status_projection.py +97 -0
- fdai_deployment_cli/support_install.py +257 -0
- fdai_deployment_cli/target.py +46 -0
- fdai_deployment_cli/trust/capabilities.json +62 -0
- fdai_deployment_cli/trust/deployment-bundle-root.pub +3 -0
- fdai_deployment_cli/trust/deployment-release-root.pub +3 -0
- fdai_deployment_cli/trust/upstream-signing-key.pub +3 -0
- fdai_deployment_cli/trust_roots.py +50 -0
- fdai_deployment_cli-0.1.2.dist-info/METADATA +112 -0
- fdai_deployment_cli-0.1.2.dist-info/RECORD +203 -0
- fdai_deployment_cli-0.1.2.dist-info/WHEEL +4 -0
- fdai_deployment_cli-0.1.2.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,4608 @@
|
|
|
1
|
+
"""Execute standalone application checkpoints on the Bastion-reachable managed host."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import base64
|
|
7
|
+
import hashlib
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import re
|
|
11
|
+
import stat
|
|
12
|
+
import subprocess
|
|
13
|
+
import sys
|
|
14
|
+
import uuid
|
|
15
|
+
from datetime import UTC, datetime, timedelta
|
|
16
|
+
from functools import partial
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any, Final
|
|
19
|
+
|
|
20
|
+
from fdai_service_contracts.product_profile import ObservationDataSource, ProductAddOn
|
|
21
|
+
|
|
22
|
+
from fdai_deployment_cli import (
|
|
23
|
+
aks_readiness,
|
|
24
|
+
catalog_review_profile,
|
|
25
|
+
hub_image_stage,
|
|
26
|
+
hub_ownership,
|
|
27
|
+
source_application_inputs,
|
|
28
|
+
standalone_host_values,
|
|
29
|
+
standalone_kubernetes_tools,
|
|
30
|
+
standalone_planned_outputs,
|
|
31
|
+
standalone_terraform_environment,
|
|
32
|
+
)
|
|
33
|
+
from fdai_deployment_cli import foundation_adoption_host as foundation_host
|
|
34
|
+
from fdai_deployment_cli import notification_recipients_input as notification_recipients
|
|
35
|
+
from fdai_deployment_cli import operational_evidence_verifier_input as oev
|
|
36
|
+
from fdai_deployment_cli import standalone_service_update_recovery as service_recovery
|
|
37
|
+
from fdai_deployment_cli.aks_historical_reconciliation import (
|
|
38
|
+
reconciled_variables,
|
|
39
|
+
secret_binding_reordered,
|
|
40
|
+
validate_reconciliation_plan,
|
|
41
|
+
)
|
|
42
|
+
from fdai_deployment_cli.aks_historical_reconciliation import (
|
|
43
|
+
single_block as _single,
|
|
44
|
+
)
|
|
45
|
+
from fdai_deployment_cli.aks_service_update import (
|
|
46
|
+
SERVICES as AKS_SERVICES,
|
|
47
|
+
)
|
|
48
|
+
from fdai_deployment_cli.aks_service_update import (
|
|
49
|
+
deployment_snapshot,
|
|
50
|
+
peers_unchanged,
|
|
51
|
+
validate_plan_scope,
|
|
52
|
+
validate_update_request,
|
|
53
|
+
)
|
|
54
|
+
from fdai_deployment_cli.aks_service_update import (
|
|
55
|
+
service_update_review as _service_update_review,
|
|
56
|
+
)
|
|
57
|
+
from fdai_deployment_cli.aks_service_update import (
|
|
58
|
+
service_update_target as _service_update_target,
|
|
59
|
+
)
|
|
60
|
+
from fdai_deployment_cli.aks_service_update import (
|
|
61
|
+
validated_secret_environment as _validated_secret_env,
|
|
62
|
+
)
|
|
63
|
+
from fdai_deployment_cli.aks_workload_jobs import (
|
|
64
|
+
aks_kubernetes_direct_api_environment as _aks_kubernetes_direct_api_environment,
|
|
65
|
+
)
|
|
66
|
+
from fdai_deployment_cli.aks_workload_jobs import (
|
|
67
|
+
prepare_aks_scheduled_jobs as _prepare_aks_scheduled_jobs,
|
|
68
|
+
)
|
|
69
|
+
from fdai_deployment_cli.contracts import canonical_digest, load_json_object
|
|
70
|
+
from fdai_deployment_cli.deployment_kit import acquire_deployment_kit
|
|
71
|
+
from fdai_deployment_cli.oci_archive import validate_oci_archive
|
|
72
|
+
from fdai_deployment_cli.private_output import read_private_bytes, write_private_output
|
|
73
|
+
from fdai_deployment_cli.runtime_profile import (
|
|
74
|
+
RuntimeDeploymentProfile,
|
|
75
|
+
legacy_full_product_matches,
|
|
76
|
+
legacy_runtime_profile_digest,
|
|
77
|
+
)
|
|
78
|
+
from fdai_deployment_cli.runtime_support_installation import (
|
|
79
|
+
install_runtime_support as _install_runtime_support,
|
|
80
|
+
)
|
|
81
|
+
from fdai_deployment_cli.standalone_aks_inventory import (
|
|
82
|
+
initial_inventory_binding as _initial_inventory_binding,
|
|
83
|
+
)
|
|
84
|
+
from fdai_deployment_cli.standalone_aks_inventory import (
|
|
85
|
+
run_initial_aks_inventory as _initial_aks_inventory,
|
|
86
|
+
)
|
|
87
|
+
from fdai_deployment_cli.standalone_aks_nodepool_guard import (
|
|
88
|
+
guard_existing_runtime_node_pools as _guard_existing_runtime_node_pools,
|
|
89
|
+
)
|
|
90
|
+
from fdai_deployment_cli.standalone_application_input_recovery import (
|
|
91
|
+
bind_authoritative_backend_nsg,
|
|
92
|
+
)
|
|
93
|
+
from fdai_deployment_cli.standalone_application_input_recovery import (
|
|
94
|
+
successor_prerequisite_verified as _application_successor_prerequisite,
|
|
95
|
+
)
|
|
96
|
+
from fdai_deployment_cli.standalone_catalog_review import run_catalog_review
|
|
97
|
+
from fdai_deployment_cli.standalone_checkpoint_failure import (
|
|
98
|
+
failure_record as _failure_record,
|
|
99
|
+
)
|
|
100
|
+
from fdai_deployment_cli.standalone_checkpoint_failure import (
|
|
101
|
+
terraform_failure as _terraform_failure,
|
|
102
|
+
)
|
|
103
|
+
from fdai_deployment_cli.standalone_database_principals import database_reader_principals
|
|
104
|
+
from fdai_deployment_cli.standalone_development_authority import (
|
|
105
|
+
apply_initial_development_authority,
|
|
106
|
+
apply_service_update_development_authority,
|
|
107
|
+
development_authority_from_args,
|
|
108
|
+
reconcile_retained_development_authority,
|
|
109
|
+
)
|
|
110
|
+
from fdai_deployment_cli.standalone_host_rebind import (
|
|
111
|
+
bind_installed_license as _bind_installed_license,
|
|
112
|
+
)
|
|
113
|
+
from fdai_deployment_cli.standalone_host_rebind import (
|
|
114
|
+
pending_application_operation as _pending_application_operation,
|
|
115
|
+
)
|
|
116
|
+
from fdai_deployment_cli.standalone_host_rebind import (
|
|
117
|
+
prepare_managed_identity_login,
|
|
118
|
+
refresh_private_access,
|
|
119
|
+
)
|
|
120
|
+
from fdai_deployment_cli.standalone_host_rebind import (
|
|
121
|
+
reconcile_retained_aks_license_binding as _reconcile_retained_aks_license_binding,
|
|
122
|
+
)
|
|
123
|
+
from fdai_deployment_cli.standalone_host_residual import apply_residual_checkpoint
|
|
124
|
+
from fdai_deployment_cli.standalone_host_state import (
|
|
125
|
+
absolute as _absolute,
|
|
126
|
+
)
|
|
127
|
+
from fdai_deployment_cli.standalone_host_state import (
|
|
128
|
+
acquire_checkpoint_lock as _acquire_checkpoint_lock,
|
|
129
|
+
)
|
|
130
|
+
from fdai_deployment_cli.standalone_host_state import (
|
|
131
|
+
executable_digest as _executable_digest,
|
|
132
|
+
)
|
|
133
|
+
from fdai_deployment_cli.standalone_host_state import (
|
|
134
|
+
file_digest as _file_digest,
|
|
135
|
+
)
|
|
136
|
+
from fdai_deployment_cli.standalone_host_state import (
|
|
137
|
+
moment as _moment,
|
|
138
|
+
)
|
|
139
|
+
from fdai_deployment_cli.standalone_host_state import (
|
|
140
|
+
parse_moment as _parse_moment,
|
|
141
|
+
)
|
|
142
|
+
from fdai_deployment_cli.standalone_host_state import (
|
|
143
|
+
private_directory as _private_directory,
|
|
144
|
+
)
|
|
145
|
+
from fdai_deployment_cli.standalone_host_state import (
|
|
146
|
+
private_json as _private_json,
|
|
147
|
+
)
|
|
148
|
+
from fdai_deployment_cli.standalone_host_state import (
|
|
149
|
+
replace_or_verify_private_json as _replace_or_verify_private_json,
|
|
150
|
+
)
|
|
151
|
+
from fdai_deployment_cli.standalone_host_state import (
|
|
152
|
+
replace_private_json as _replace_private_json,
|
|
153
|
+
)
|
|
154
|
+
from fdai_deployment_cli.standalone_host_values import (
|
|
155
|
+
AKS_CORE_STARTUP_READINESS,
|
|
156
|
+
aks_operator_environment,
|
|
157
|
+
aks_operator_request_receipts,
|
|
158
|
+
)
|
|
159
|
+
from fdai_deployment_cli.standalone_host_values import (
|
|
160
|
+
console_origin as _console_origin,
|
|
161
|
+
)
|
|
162
|
+
from fdai_deployment_cli.standalone_host_values import (
|
|
163
|
+
foundation_application_workload as _foundation_application_workload,
|
|
164
|
+
)
|
|
165
|
+
from fdai_deployment_cli.standalone_host_values import (
|
|
166
|
+
foundation_binding_digest as _foundation_binding_digest,
|
|
167
|
+
)
|
|
168
|
+
from fdai_deployment_cli.standalone_host_values import (
|
|
169
|
+
plan_summary as _plan_summary,
|
|
170
|
+
)
|
|
171
|
+
from fdai_deployment_cli.standalone_host_values import (
|
|
172
|
+
required_image_digest as _required_image_digest,
|
|
173
|
+
)
|
|
174
|
+
from fdai_deployment_cli.standalone_host_values import (
|
|
175
|
+
vault_name as _vault_name,
|
|
176
|
+
)
|
|
177
|
+
from fdai_deployment_cli.standalone_license_installation import (
|
|
178
|
+
aks_license_environment,
|
|
179
|
+
install_license,
|
|
180
|
+
)
|
|
181
|
+
from fdai_deployment_cli.standalone_management_egress import management_egress_cidrs
|
|
182
|
+
from fdai_deployment_cli.standalone_migration_evidence import verify_service_evidence
|
|
183
|
+
from fdai_deployment_cli.standalone_operational_evidence import (
|
|
184
|
+
add_aks_operational_evidence_verifier_workload as _add_aks_operational_evidence_verifier_workload,
|
|
185
|
+
)
|
|
186
|
+
from fdai_deployment_cli.standalone_operational_evidence import aks_workload as _aks_workload
|
|
187
|
+
from fdai_deployment_cli.standalone_private_access_probe import wait_for_private_data_planes
|
|
188
|
+
from fdai_deployment_cli.standalone_product_profile import (
|
|
189
|
+
context_selects_add_on as _selects_add_on,
|
|
190
|
+
)
|
|
191
|
+
from fdai_deployment_cli.standalone_product_profile import product_terraform_values
|
|
192
|
+
from fdai_deployment_cli.standalone_residual_apply import (
|
|
193
|
+
prepare_residual_review as _prepare_residual_review,
|
|
194
|
+
)
|
|
195
|
+
from fdai_deployment_cli.standalone_residual_apply import (
|
|
196
|
+
recover_residual_apply as _recover_residual_apply,
|
|
197
|
+
)
|
|
198
|
+
from fdai_deployment_cli.standalone_residual_apply import (
|
|
199
|
+
residual_claim_exists as _residual_claim_exists,
|
|
200
|
+
)
|
|
201
|
+
from fdai_deployment_cli.standalone_residual_apply import (
|
|
202
|
+
seal_terraform_plan as _seal_terraform_plan,
|
|
203
|
+
)
|
|
204
|
+
from fdai_deployment_cli.standalone_residual_successor import (
|
|
205
|
+
successor_prerequisite_verified as _residual_successor_prerequisite,
|
|
206
|
+
)
|
|
207
|
+
from fdai_deployment_cli.standalone_service_update_finalization import (
|
|
208
|
+
add_finalization_parser,
|
|
209
|
+
)
|
|
210
|
+
from fdai_deployment_cli.standalone_stage_targets import (
|
|
211
|
+
database_placement as _database_placement,
|
|
212
|
+
)
|
|
213
|
+
from fdai_deployment_cli.standalone_stage_targets import (
|
|
214
|
+
focused_private_access as _focused_private_access,
|
|
215
|
+
)
|
|
216
|
+
from fdai_deployment_cli.standalone_stage_targets import (
|
|
217
|
+
not_required_recovery as _not_required_recovery,
|
|
218
|
+
)
|
|
219
|
+
from fdai_deployment_cli.standalone_stage_targets import (
|
|
220
|
+
operation_targets as _operation_targets,
|
|
221
|
+
)
|
|
222
|
+
from fdai_deployment_cli.standalone_stage_targets import (
|
|
223
|
+
reconcile_retained_private_access as _reconcile_retained_private_access,
|
|
224
|
+
)
|
|
225
|
+
from fdai_deployment_cli.standalone_stage_targets import (
|
|
226
|
+
runtime_operation as _runtime_operation,
|
|
227
|
+
)
|
|
228
|
+
from fdai_deployment_cli.standalone_stage_targets import (
|
|
229
|
+
stage_targets as _stage_targets,
|
|
230
|
+
)
|
|
231
|
+
from fdai_deployment_cli.standalone_terraform_environment import (
|
|
232
|
+
configure_terraform as _configure_terraform,
|
|
233
|
+
)
|
|
234
|
+
from fdai_deployment_cli.standalone_terraform_environment import (
|
|
235
|
+
terraform_configuration as _terraform_configuration,
|
|
236
|
+
)
|
|
237
|
+
from fdai_deployment_cli.standalone_trial_activation import (
|
|
238
|
+
activate_trial,
|
|
239
|
+
deployment_binding_digest,
|
|
240
|
+
)
|
|
241
|
+
from fdai_deployment_cli.target import compute_target_binding
|
|
242
|
+
|
|
243
|
+
_DIGEST = re.compile(r"[0-9a-f]{64}")
|
|
244
|
+
_GUID = re.compile(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}")
|
|
245
|
+
_mapping = standalone_host_values.mapping
|
|
246
|
+
_required_guid = standalone_host_values.required_guid
|
|
247
|
+
_selected_identity = standalone_host_values.selected_identity
|
|
248
|
+
_aks_core_conversation_environment = standalone_host_values.aks_core_conversation_environment
|
|
249
|
+
_SOURCE_COMMIT = re.compile(r"[0-9a-f]{40}")
|
|
250
|
+
_AKS_RUNTIME_NAMESPACE = "fdai-runtime"
|
|
251
|
+
_STAGES: Final = ("access", "substrate", "runtime", "database", "application")
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def main(argv: list[str] | None = None) -> int:
|
|
255
|
+
"""Run one remote checkpoint with an owner-only umask and emit only sanitized JSON."""
|
|
256
|
+
|
|
257
|
+
os.umask(0o077)
|
|
258
|
+
parser = argparse.ArgumentParser(prog="python -m fdai_deployment_cli.standalone_host")
|
|
259
|
+
parser.add_argument("--work-dir", type=Path, required=True)
|
|
260
|
+
subcommands = parser.add_subparsers(required=True)
|
|
261
|
+
source_runtime = subcommands.add_parser("verify-source-runtime")
|
|
262
|
+
source_runtime.add_argument("--source-snapshot", type=Path, required=True)
|
|
263
|
+
source_runtime.add_argument("--snapshot-digest", required=True)
|
|
264
|
+
source_runtime.add_argument("--runtime-root", type=Path, required=True)
|
|
265
|
+
source_runtime.add_argument("--runtime-digest", required=True)
|
|
266
|
+
source_runtime.add_argument("--deployment-bundle", type=Path, required=True)
|
|
267
|
+
source_runtime.add_argument("--bundle-digest", required=True)
|
|
268
|
+
source_runtime.add_argument("--platform-tag", required=True)
|
|
269
|
+
source_runtime.set_defaults(handler=_verify_source_runtime, read_only=True)
|
|
270
|
+
|
|
271
|
+
prepare = subcommands.add_parser("prepare")
|
|
272
|
+
prepare.add_argument("--kit", type=Path, required=True)
|
|
273
|
+
prepare.add_argument("--handoff", type=Path, required=True)
|
|
274
|
+
prepare.add_argument("--entra", type=Path)
|
|
275
|
+
prepare.add_argument("--foundation-adoption", type=Path)
|
|
276
|
+
prepare.add_argument("--adoption-state", type=Path)
|
|
277
|
+
prepare.add_argument("--adoption-models", type=Path)
|
|
278
|
+
prepare.add_argument("--adoption-descriptor", type=Path)
|
|
279
|
+
prepare.add_argument("--catalog-review-profile", type=Path)
|
|
280
|
+
prepare.add_argument("--catalog-review-private-key", type=Path)
|
|
281
|
+
prepare.add_argument("--operational-evidence-verifier-input", type=Path)
|
|
282
|
+
prepare.add_argument("--notification-recipients", type=Path)
|
|
283
|
+
prepare.add_argument("--runtime-platform", default="aks")
|
|
284
|
+
prepare.add_argument("--database-placement", default="postgres-flex")
|
|
285
|
+
prepare.add_argument("--system-node-count", type=int, default=3)
|
|
286
|
+
prepare.add_argument("--system-node-sku", default=None)
|
|
287
|
+
prepare.add_argument("--user-node-min-count", type=int, default=3)
|
|
288
|
+
prepare.add_argument("--user-node-max-count", type=int, default=5)
|
|
289
|
+
prepare.add_argument("--user-node-sku", default="Standard_D4ds_v4")
|
|
290
|
+
prepare.add_argument("--database-sku")
|
|
291
|
+
prepare.add_argument(
|
|
292
|
+
"--product-add-on",
|
|
293
|
+
action="append",
|
|
294
|
+
choices=tuple(item.value for item in ProductAddOn),
|
|
295
|
+
default=[],
|
|
296
|
+
)
|
|
297
|
+
prepare.add_argument(
|
|
298
|
+
"--observation-source",
|
|
299
|
+
action="append",
|
|
300
|
+
choices=tuple(item.value for item in ObservationDataSource),
|
|
301
|
+
default=[],
|
|
302
|
+
)
|
|
303
|
+
prepare.set_defaults(handler=_prepare)
|
|
304
|
+
|
|
305
|
+
subcommands.add_parser("refresh-private-access").set_defaults(
|
|
306
|
+
handler=partial(refresh_private_access, login=_managed_identity_login_from_context)
|
|
307
|
+
)
|
|
308
|
+
|
|
309
|
+
source_application_inputs.add_prepare_source_parser(subcommands, _prepare)
|
|
310
|
+
|
|
311
|
+
prepare_runtime = subcommands.add_parser("prepare-runtime")
|
|
312
|
+
prepare_runtime.set_defaults(handler=_prepare_runtime)
|
|
313
|
+
|
|
314
|
+
prepare_database = subcommands.add_parser("prepare-database")
|
|
315
|
+
prepare_database.set_defaults(handler=_prepare_database)
|
|
316
|
+
|
|
317
|
+
prepare_application = subcommands.add_parser("prepare-application")
|
|
318
|
+
prepare_application.set_defaults(handler=_prepare_aks_application)
|
|
319
|
+
|
|
320
|
+
prepare_service_update = subcommands.add_parser("prepare-service-update")
|
|
321
|
+
prepare_service_update.add_argument("--service", choices=sorted(AKS_SERVICES), required=True)
|
|
322
|
+
prepare_service_update.add_argument("--image", required=True)
|
|
323
|
+
prepare_service_update.add_argument("--source-commit", required=True)
|
|
324
|
+
prepare_service_update.add_argument("--development-authority-profile", type=Path)
|
|
325
|
+
prepare_service_update.add_argument(
|
|
326
|
+
"--clear-development-profile",
|
|
327
|
+
action="store_true",
|
|
328
|
+
dest="clear_development_authority_profile",
|
|
329
|
+
)
|
|
330
|
+
prepare_service_update.set_defaults(handler=_prepare_aks_service_update)
|
|
331
|
+
|
|
332
|
+
add_finalization_parser(subcommands, services=AKS_SERVICES)
|
|
333
|
+
|
|
334
|
+
adopt_historical = subcommands.add_parser("adopt-historical-aks-application")
|
|
335
|
+
adopt_historical.add_argument("--binding", type=Path, required=True)
|
|
336
|
+
adopt_historical.add_argument("--state", type=Path, required=True)
|
|
337
|
+
adopt_historical.add_argument("--variables", type=Path, required=True)
|
|
338
|
+
adopt_historical.add_argument("--live", type=Path, required=True)
|
|
339
|
+
adopt_historical.add_argument("--plan", type=Path, required=True)
|
|
340
|
+
adopt_historical.set_defaults(handler=_adopt_historical_aks_application)
|
|
341
|
+
|
|
342
|
+
apply_historical = subcommands.add_parser("apply-historical-aks-reconciliation")
|
|
343
|
+
apply_historical.add_argument("--approval", type=Path, required=True)
|
|
344
|
+
apply_historical.set_defaults(handler=_apply_historical_aks_reconciliation)
|
|
345
|
+
|
|
346
|
+
recover_historical = subcommands.add_parser("recover-historical-aks-reconciliation")
|
|
347
|
+
recover_historical.set_defaults(handler=_recover_historical_aks_reconciliation)
|
|
348
|
+
|
|
349
|
+
plan = subcommands.add_parser("plan")
|
|
350
|
+
plan.add_argument("--stage", choices=_STAGES, required=True)
|
|
351
|
+
plan.add_argument("--service", choices=sorted(AKS_SERVICES))
|
|
352
|
+
plan.set_defaults(handler=_plan)
|
|
353
|
+
|
|
354
|
+
apply = subcommands.add_parser("apply")
|
|
355
|
+
apply.add_argument("--stage", choices=_STAGES, required=True)
|
|
356
|
+
apply.add_argument("--service", choices=sorted(AKS_SERVICES))
|
|
357
|
+
apply.add_argument("--approval", type=Path, required=True)
|
|
358
|
+
apply.set_defaults(handler=_apply)
|
|
359
|
+
|
|
360
|
+
apply_residual = subcommands.add_parser("apply-residual")
|
|
361
|
+
apply_residual.add_argument("--stage", choices=_STAGES, required=True)
|
|
362
|
+
apply_residual.add_argument("--approval", type=Path, required=True)
|
|
363
|
+
apply_residual.set_defaults(handler=_apply_residual)
|
|
364
|
+
|
|
365
|
+
recover = subcommands.add_parser("recover-apply")
|
|
366
|
+
recover.add_argument("--stage", choices=_STAGES, required=True)
|
|
367
|
+
recover.add_argument("--service", choices=sorted(AKS_SERVICES))
|
|
368
|
+
recover.set_defaults(handler=_recover_apply)
|
|
369
|
+
|
|
370
|
+
images = subcommands.add_parser("import-images")
|
|
371
|
+
images.set_defaults(handler=_import_images)
|
|
372
|
+
|
|
373
|
+
source_image = subcommands.add_parser("import-source-image")
|
|
374
|
+
source_image.add_argument("--service", choices=sorted(AKS_SERVICES), required=True)
|
|
375
|
+
source_image.add_argument("--archive", type=Path, required=True)
|
|
376
|
+
source_image.add_argument("--archive-sha256", required=True)
|
|
377
|
+
source_image.add_argument("--image-digest", required=True)
|
|
378
|
+
source_image.add_argument("--source-commit", required=True)
|
|
379
|
+
source_image.add_argument("--approval", type=Path, required=True)
|
|
380
|
+
source_image.set_defaults(handler=_import_source_service_image)
|
|
381
|
+
|
|
382
|
+
update_context = subcommands.add_parser("service-update-context")
|
|
383
|
+
update_context.add_argument("--service", choices=sorted(AKS_SERVICES), required=True)
|
|
384
|
+
update_context.set_defaults(handler=_service_update_context, read_only=True)
|
|
385
|
+
|
|
386
|
+
subcommands.add_parser("deployment-binding").set_defaults(handler=_deployment_binding)
|
|
387
|
+
hub_ownership.add_handoff_parser(subcommands)
|
|
388
|
+
|
|
389
|
+
migrate = subcommands.add_parser("migrate")
|
|
390
|
+
migrate.set_defaults(handler=_migrate)
|
|
391
|
+
|
|
392
|
+
subcommands.add_parser("activate-trial").set_defaults(handler=_activate_trial)
|
|
393
|
+
|
|
394
|
+
initial_inventory = subcommands.add_parser("initial-inventory")
|
|
395
|
+
initial_inventory.set_defaults(handler=_initial_inventory)
|
|
396
|
+
|
|
397
|
+
catalog_review = subcommands.add_parser("catalog-review")
|
|
398
|
+
catalog_review.set_defaults(handler=_catalog_review)
|
|
399
|
+
|
|
400
|
+
license_command = subcommands.add_parser("install-license")
|
|
401
|
+
license_command.add_argument("--image-digest", required=True)
|
|
402
|
+
license_command.add_argument("--deployment-binding", required=True)
|
|
403
|
+
license_command.add_argument("--installation-binding", default=None)
|
|
404
|
+
license_command.set_defaults(handler=_install_license)
|
|
405
|
+
|
|
406
|
+
subcommands.add_parser("bind-installed-license").set_defaults(handler=_bind_installed_license)
|
|
407
|
+
|
|
408
|
+
verify = subcommands.add_parser("verify")
|
|
409
|
+
verify.set_defaults(handler=_verify)
|
|
410
|
+
|
|
411
|
+
args = parser.parse_args(argv)
|
|
412
|
+
lock_descriptor: int | None = None
|
|
413
|
+
try:
|
|
414
|
+
work_dir = _absolute(args.work_dir)
|
|
415
|
+
if not getattr(args, "read_only", False):
|
|
416
|
+
_private_directory(work_dir)
|
|
417
|
+
lock_descriptor = _acquire_checkpoint_lock(work_dir)
|
|
418
|
+
result = args.handler(args, work_dir)
|
|
419
|
+
except (OSError, ValueError, subprocess.SubprocessError) as exc:
|
|
420
|
+
print(
|
|
421
|
+
json.dumps(
|
|
422
|
+
_failure_record(exc),
|
|
423
|
+
sort_keys=True,
|
|
424
|
+
separators=(",", ":"),
|
|
425
|
+
),
|
|
426
|
+
file=sys.stderr,
|
|
427
|
+
)
|
|
428
|
+
return 3
|
|
429
|
+
finally:
|
|
430
|
+
if lock_descriptor is not None:
|
|
431
|
+
os.close(lock_descriptor)
|
|
432
|
+
print(json.dumps(result, sort_keys=True, separators=(",", ":")))
|
|
433
|
+
return 0
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def _verify_source_runtime(args: argparse.Namespace, _work_dir: Path) -> dict[str, object]:
|
|
437
|
+
from fdai_deployment_cli.source_runtime import verify_source_runtime
|
|
438
|
+
|
|
439
|
+
return verify_source_runtime(
|
|
440
|
+
snapshot=_absolute(args.source_snapshot),
|
|
441
|
+
snapshot_digest=args.snapshot_digest,
|
|
442
|
+
runtime_root=_absolute(args.runtime_root),
|
|
443
|
+
runtime_digest=args.runtime_digest,
|
|
444
|
+
deployment_bundle=_absolute(args.deployment_bundle),
|
|
445
|
+
bundle_digest=args.bundle_digest,
|
|
446
|
+
platform_tag=args.platform_tag,
|
|
447
|
+
)
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def _prepare(args: argparse.Namespace, work_dir: Path) -> dict[str, object]:
|
|
451
|
+
_private_directory(work_dir)
|
|
452
|
+
hub_selection = hub_image_stage.prepare_host_inputs(args, work_dir)
|
|
453
|
+
source_mode = hasattr(args, "source_snapshot")
|
|
454
|
+
handoff = _private_json(_absolute(args.handoff), "Foundation handoff")
|
|
455
|
+
runtime_profile = RuntimeDeploymentProfile.from_prepare_arguments(args)
|
|
456
|
+
enterprise_identity_selected = runtime_profile.product_profile.selects(
|
|
457
|
+
ProductAddOn.ENTERPRISE_IDENTITY_GOVERNANCE
|
|
458
|
+
)
|
|
459
|
+
if enterprise_identity_selected and args.entra is None:
|
|
460
|
+
raise ValueError("enterprise identity governance requires Entra bindings")
|
|
461
|
+
if not enterprise_identity_selected and args.entra is not None:
|
|
462
|
+
raise ValueError("Entra bindings require enterprise identity governance")
|
|
463
|
+
entra = _private_json(_absolute(args.entra), "Entra bindings") if args.entra is not None else {}
|
|
464
|
+
runner = _mapping(handoff.get("runner"), "Foundation runner")
|
|
465
|
+
state = _mapping(handoff.get("state"), "Foundation state")
|
|
466
|
+
ops = _mapping(handoff.get("ops"), "Foundation operations")
|
|
467
|
+
app = _mapping(handoff.get("app_resource_group"), "Foundation application group")
|
|
468
|
+
foundation = foundation_host.load_foundation_host_context(
|
|
469
|
+
_absolute(args.foundation_adoption) if args.foundation_adoption is not None else None,
|
|
470
|
+
handoff=handoff,
|
|
471
|
+
runner=runner,
|
|
472
|
+
)
|
|
473
|
+
hub_image_stage.require_foundation_source(hub_selection, foundation.adoption.source_commit)
|
|
474
|
+
initial_inventory_binding = _initial_inventory_binding(foundation.subscription_id)
|
|
475
|
+
foundation_binding_digest = _foundation_binding_digest(
|
|
476
|
+
handoff,
|
|
477
|
+
runner=runner,
|
|
478
|
+
state=state,
|
|
479
|
+
ops=ops,
|
|
480
|
+
app=app,
|
|
481
|
+
)
|
|
482
|
+
entra_binding_digest = canonical_digest(entra) if enterprise_identity_selected else None
|
|
483
|
+
adoption = _application_state_adoption(args)
|
|
484
|
+
adoption_digest = canonical_digest(adoption[0]) if adoption is not None else ""
|
|
485
|
+
catalog_profile_path = getattr(args, "catalog_review_profile", None)
|
|
486
|
+
catalog_key_path = getattr(args, "catalog_review_private_key", None)
|
|
487
|
+
catalog_profile = catalog_review_profile.staged_catalog_review_profile_from_paths(
|
|
488
|
+
_absolute(catalog_profile_path) if catalog_profile_path else None,
|
|
489
|
+
_absolute(catalog_key_path) if catalog_key_path else None,
|
|
490
|
+
)
|
|
491
|
+
verifier_input_path = getattr(args, "operational_evidence_verifier_input", None)
|
|
492
|
+
verifier_input = oev.staged_operational_evidence_verifier_input_from_path(
|
|
493
|
+
_absolute(verifier_input_path) if verifier_input_path else None
|
|
494
|
+
)
|
|
495
|
+
verifier_input_digest = verifier_input.input_digest if verifier_input is not None else ""
|
|
496
|
+
recipients_path = getattr(args, "notification_recipients", None)
|
|
497
|
+
recipients_input = notification_recipients.staged_notification_recipients_from_path(
|
|
498
|
+
_absolute(recipients_path) if recipients_path else None
|
|
499
|
+
)
|
|
500
|
+
development_authority = development_authority_from_args(args, absolute=_absolute)
|
|
501
|
+
foundation.login(_managed_identity_login, work_dir)
|
|
502
|
+
suffix = (
|
|
503
|
+
str(adoption[0]["resource_name_suffix"])
|
|
504
|
+
if adoption is not None
|
|
505
|
+
else hashlib.sha256(str(handoff["run_digest"]).encode()).hexdigest()[:6]
|
|
506
|
+
)
|
|
507
|
+
region = str(handoff["region"])
|
|
508
|
+
region_short = str(handoff["region_short"])
|
|
509
|
+
workload = _foundation_application_workload(
|
|
510
|
+
app,
|
|
511
|
+
environment="dev",
|
|
512
|
+
region_short=region_short,
|
|
513
|
+
)
|
|
514
|
+
key_vault_name = standalone_host_values.planned_key_vault_name(
|
|
515
|
+
workload=workload,
|
|
516
|
+
environment="dev",
|
|
517
|
+
region_short=region_short,
|
|
518
|
+
resource_suffix=suffix,
|
|
519
|
+
)
|
|
520
|
+
document_storage_account_name = standalone_host_values.planned_document_storage_name(
|
|
521
|
+
subscription_id=foundation.subscription_id,
|
|
522
|
+
workload=workload,
|
|
523
|
+
environment="dev",
|
|
524
|
+
region_short=region_short,
|
|
525
|
+
)
|
|
526
|
+
key_vault_private_access = (
|
|
527
|
+
runtime_profile.runtime_platform.value == "aks"
|
|
528
|
+
and standalone_terraform_environment.requires_aks_key_vault_private_access(
|
|
529
|
+
subscription_id=foundation.subscription_id,
|
|
530
|
+
resource_group_name=str(app["name"]),
|
|
531
|
+
vault_name=key_vault_name,
|
|
532
|
+
work_dir=work_dir,
|
|
533
|
+
)
|
|
534
|
+
)
|
|
535
|
+
document_storage_private_access = (
|
|
536
|
+
runtime_profile.runtime_platform.value == "aks"
|
|
537
|
+
and standalone_terraform_environment.requires_aks_document_storage_private_access(
|
|
538
|
+
subscription_id=foundation.subscription_id,
|
|
539
|
+
resource_group_name=str(app["name"]),
|
|
540
|
+
account_name=document_storage_account_name,
|
|
541
|
+
work_dir=work_dir,
|
|
542
|
+
)
|
|
543
|
+
)
|
|
544
|
+
retained_context = work_dir / "context.json"
|
|
545
|
+
retained_variables = work_dir / "application.auto.tfvars.json"
|
|
546
|
+
if retained_context.exists() or retained_variables.exists():
|
|
547
|
+
if not retained_context.exists() or not retained_variables.exists():
|
|
548
|
+
raise ValueError("standalone host preparation is incomplete")
|
|
549
|
+
retained = _private_json(retained_context, "standalone host context")
|
|
550
|
+
hub_image_stage.require_retained_artifact_mode(retained, source_mode, hub_selection)
|
|
551
|
+
retained_values = _private_json(retained_variables, "standalone host Terraform variables")
|
|
552
|
+
posture_changed = _reconcile_retained_private_access(
|
|
553
|
+
retained, retained_values, key_vault_private_access, document_storage_private_access
|
|
554
|
+
)
|
|
555
|
+
if (
|
|
556
|
+
retained.get("subscription_id") != foundation.subscription_id
|
|
557
|
+
or retained.get("tenant_id") != foundation.tenant_id
|
|
558
|
+
or retained.get("client_id") != foundation.client_id
|
|
559
|
+
or retained.get("principal_id") != foundation.principal_id
|
|
560
|
+
or retained.get("foundation_binding_digest") != foundation_binding_digest
|
|
561
|
+
or retained.get("entra_binding_digest") != entra_binding_digest
|
|
562
|
+
or retained.get("application_state_adoption_digest", "") != adoption_digest
|
|
563
|
+
or retained.get("catalog_review_profile_digest") != catalog_profile.profile_digest
|
|
564
|
+
or retained.get("initial_inventory_binding") != initial_inventory_binding
|
|
565
|
+
or retained.get("key_vault_name") != key_vault_name
|
|
566
|
+
or retained.get("document_storage_account_name") != document_storage_account_name
|
|
567
|
+
or not _runtime_profile_matches(retained, runtime_profile)
|
|
568
|
+
):
|
|
569
|
+
raise ValueError("standalone host retained context differs")
|
|
570
|
+
oev_changed = oev.reconcile_retained_operational_evidence_verifier_input(
|
|
571
|
+
verifier_input,
|
|
572
|
+
retained=retained,
|
|
573
|
+
values=retained_values,
|
|
574
|
+
)
|
|
575
|
+
recipients_changed = notification_recipients.reconcile_retained_notification_recipients(
|
|
576
|
+
recipients_input,
|
|
577
|
+
retained=retained,
|
|
578
|
+
values=retained_values,
|
|
579
|
+
)
|
|
580
|
+
foundation.adoption.require_context(retained)
|
|
581
|
+
source_changed = source_application_inputs.rebind_source_tree(retained, work_dir)
|
|
582
|
+
authority_changed = reconcile_retained_development_authority(
|
|
583
|
+
development_authority,
|
|
584
|
+
retained=retained,
|
|
585
|
+
values=retained_values,
|
|
586
|
+
tenant_id=foundation.tenant_id,
|
|
587
|
+
subscription_id=foundation.subscription_id,
|
|
588
|
+
resource_groups=(str(app["name"]), str(ops["resource_group_name"])),
|
|
589
|
+
)
|
|
590
|
+
if (
|
|
591
|
+
posture_changed
|
|
592
|
+
or oev_changed
|
|
593
|
+
or recipients_changed
|
|
594
|
+
or source_changed
|
|
595
|
+
or authority_changed
|
|
596
|
+
):
|
|
597
|
+
_replace_private_json(retained_context, retained)
|
|
598
|
+
_replace_private_json(retained_variables, retained_values)
|
|
599
|
+
_terraform_init(work_dir, retained)
|
|
600
|
+
if adoption is not None:
|
|
601
|
+
_adopt_application_state(work_dir, retained, *adoption)
|
|
602
|
+
return {
|
|
603
|
+
"schema_version": "fdai.standalone-host-prepare.v1",
|
|
604
|
+
"state": "prepared",
|
|
605
|
+
"source_commit": retained["source_commit"],
|
|
606
|
+
"kit_manifest_digest": retained["kit_manifest_digest"],
|
|
607
|
+
"runtime_release_digest": retained["runtime_release_digest"],
|
|
608
|
+
"runtime_profile_digest": runtime_profile.digest,
|
|
609
|
+
"focused_private_access": _focused_private_access(retained),
|
|
610
|
+
"mutation_performed": False,
|
|
611
|
+
"subscription_ready": False,
|
|
612
|
+
**(hub_selection.metadata() if hub_selection is not None else {}),
|
|
613
|
+
}
|
|
614
|
+
if source_mode:
|
|
615
|
+
source_artifacts = source_application_inputs.prepare_host_artifacts(
|
|
616
|
+
args, work_dir, foundation.adoption.source_commit
|
|
617
|
+
)
|
|
618
|
+
source_commit = source_artifacts.source_commit
|
|
619
|
+
infra = source_artifacts.infra
|
|
620
|
+
terraform = source_artifacts.terraform
|
|
621
|
+
provider_mirror: Path | None = None
|
|
622
|
+
runtime_release_digest = source_artifacts.digest
|
|
623
|
+
kit_manifest_digest = source_artifacts.digest
|
|
624
|
+
kit_bin = source_artifacts.kit_bin
|
|
625
|
+
provider_installation = "direct"
|
|
626
|
+
else:
|
|
627
|
+
kit_work = work_dir / "kit-work"
|
|
628
|
+
_private_directory(kit_work)
|
|
629
|
+
kit = acquire_deployment_kit(
|
|
630
|
+
work_dir=kit_work,
|
|
631
|
+
offline_kit=_absolute(args.kit),
|
|
632
|
+
)
|
|
633
|
+
foundation.adoption.require_kit(kit)
|
|
634
|
+
_install_runtime_support(
|
|
635
|
+
work_dir,
|
|
636
|
+
artifact_root=kit.materialized_root,
|
|
637
|
+
kit_manifest_digest=kit.verification.manifest_digest,
|
|
638
|
+
)
|
|
639
|
+
infra = kit.bundle_root / "infra"
|
|
640
|
+
terraform = kit.materialized_root / kit.verification.terraform_binary
|
|
641
|
+
provider_mirror = kit.materialized_root / kit.verification.provider_mirror_prefix
|
|
642
|
+
runtime = kit.runtime.to_mapping()
|
|
643
|
+
runtime_release_digest = kit.runtime.digest
|
|
644
|
+
kit_manifest_digest = kit.verification.manifest_digest
|
|
645
|
+
source_commit = kit.source_commit
|
|
646
|
+
kit_bin = kit.materialized_root / "bin"
|
|
647
|
+
provider_installation = "filesystem_mirror"
|
|
648
|
+
terraform_config = work_dir / "terraform.rc"
|
|
649
|
+
expected_terraform_config = source_application_inputs.terraform_configuration(
|
|
650
|
+
source_mode=source_mode, provider_mirror=provider_mirror
|
|
651
|
+
)
|
|
652
|
+
if not terraform_config.exists():
|
|
653
|
+
write_private_output(terraform_config, expected_terraform_config)
|
|
654
|
+
elif (
|
|
655
|
+
read_private_bytes(terraform_config, max_bytes=16_384).decode("utf-8")
|
|
656
|
+
!= expected_terraform_config
|
|
657
|
+
):
|
|
658
|
+
raise ValueError("retained Terraform provider configuration differs")
|
|
659
|
+
standalone_terraform_environment.ensure_backend(infra)
|
|
660
|
+
registry = standalone_host_values.planned_container_registry_name(
|
|
661
|
+
workload=workload, environment="dev", region_short=region_short, resource_suffix=suffix
|
|
662
|
+
)
|
|
663
|
+
login_server = f"{registry}.azurecr.io"
|
|
664
|
+
if source_mode:
|
|
665
|
+
refs = source_application_inputs.placeholder_image_refs(login_server, include_pgvector=True)
|
|
666
|
+
else:
|
|
667
|
+
services = _mapping(runtime.get("services"), "runtime services")
|
|
668
|
+
sidecars = _mapping(runtime.get("sidecars"), "runtime sidecars")
|
|
669
|
+
refs = {
|
|
670
|
+
name: f"{login_server}/{name}@{_required_image_digest(services, name)}"
|
|
671
|
+
for name in (
|
|
672
|
+
"core-control-plane",
|
|
673
|
+
"operator-service",
|
|
674
|
+
"document-ingestion-api",
|
|
675
|
+
"document-processing-worker",
|
|
676
|
+
"isolated-executor",
|
|
677
|
+
)
|
|
678
|
+
}
|
|
679
|
+
refs["clamav"] = f"{login_server}/clamav@{_required_image_digest(sidecars, 'clamav')}"
|
|
680
|
+
if runtime_profile.database_placement.value == "postgres-aks":
|
|
681
|
+
refs["pgvector"] = (
|
|
682
|
+
f"{login_server}/pgvector@{_required_image_digest(sidecars, 'pgvector')}"
|
|
683
|
+
)
|
|
684
|
+
aks_baseline = runtime_profile.runtime_platform.value == "aks"
|
|
685
|
+
values: dict[str, object] = {
|
|
686
|
+
"workload": workload,
|
|
687
|
+
"env": "dev",
|
|
688
|
+
"region": region,
|
|
689
|
+
"region_short": region_short,
|
|
690
|
+
"tenant_id": foundation.tenant_id,
|
|
691
|
+
"deploy_runner_principal_id": foundation.principal_id,
|
|
692
|
+
"postgres_admin_login": "fdaiadmin",
|
|
693
|
+
"generate_initial_postgres_password": True,
|
|
694
|
+
"resource_name_suffix": suffix,
|
|
695
|
+
"foundation_resource_group_context_digest": str(app["foundation_context_digest"]),
|
|
696
|
+
"enable_private_networking": not aks_baseline,
|
|
697
|
+
"enable_operational_history": not aks_baseline,
|
|
698
|
+
"enable_aks_key_vault_private_access": key_vault_private_access,
|
|
699
|
+
"enable_aks_document_storage_private_access": document_storage_private_access,
|
|
700
|
+
"compute_kind": (
|
|
701
|
+
"aks" if runtime_profile.runtime_platform.value == "aks" else "container_apps"
|
|
702
|
+
),
|
|
703
|
+
"state_store_kind": (
|
|
704
|
+
"postgres_aks"
|
|
705
|
+
if runtime_profile.database_placement.value == "postgres-aks"
|
|
706
|
+
else "postgres_flex"
|
|
707
|
+
),
|
|
708
|
+
"enable_private_postgres": False,
|
|
709
|
+
"runner_vnet_id": str(ops["vnet_id"]),
|
|
710
|
+
"runner_vnet_name": str(ops["vnet_name"]),
|
|
711
|
+
"ops_resource_group_name": str(ops["resource_group_name"]),
|
|
712
|
+
"acr_sku": "Basic" if aks_baseline else "Premium",
|
|
713
|
+
"core_image": refs["core-control-plane"],
|
|
714
|
+
"operator_api_image": refs["operator-service"],
|
|
715
|
+
"operator_api_migration_image": refs["operator-service"],
|
|
716
|
+
"ingestion_image": refs["document-ingestion-api"],
|
|
717
|
+
"ingestion_migration_image": refs["document-ingestion-api"],
|
|
718
|
+
"clamav_image": refs["clamav"],
|
|
719
|
+
"ingestion_cohost_worker": False,
|
|
720
|
+
"ingestion_cors_allow_origins": "https://localhost",
|
|
721
|
+
"enable_llm": adoption is not None,
|
|
722
|
+
}
|
|
723
|
+
development_profile_digest = apply_initial_development_authority(
|
|
724
|
+
development_authority,
|
|
725
|
+
values=values,
|
|
726
|
+
tenant_id=foundation.tenant_id,
|
|
727
|
+
subscription_id=foundation.subscription_id,
|
|
728
|
+
resource_groups=(str(app["name"]), str(ops["resource_group_name"])),
|
|
729
|
+
source_revision=source_commit,
|
|
730
|
+
)
|
|
731
|
+
values.update(
|
|
732
|
+
product_terraform_values(
|
|
733
|
+
runtime_profile,
|
|
734
|
+
entra,
|
|
735
|
+
require_guid=_required_guid,
|
|
736
|
+
)
|
|
737
|
+
)
|
|
738
|
+
values.update(catalog_review_profile.catalog_review_terraform_values(catalog_profile))
|
|
739
|
+
oev.apply_operational_evidence_verifier_input(values, verifier_input)
|
|
740
|
+
notification_recipients.apply_notification_recipients(values, recipients_input)
|
|
741
|
+
if adoption is not None:
|
|
742
|
+
values.update(
|
|
743
|
+
resolved_capabilities=adoption[0]["resolved_capabilities"],
|
|
744
|
+
resolved_models_json=read_private_bytes(adoption[2], max_bytes=8 * 1024 * 1024).decode(
|
|
745
|
+
"utf-8"
|
|
746
|
+
),
|
|
747
|
+
resolved_models_sha256=adoption[0]["resolved_models_sha256"],
|
|
748
|
+
)
|
|
749
|
+
context: dict[str, object] = {
|
|
750
|
+
"source_commit": source_commit,
|
|
751
|
+
"artifact_source": "operator-selected-source" if source_mode else "signed-kit",
|
|
752
|
+
"target_binding": foundation.target_binding,
|
|
753
|
+
"foundation_adoption_digest": foundation.adoption.digest,
|
|
754
|
+
"subscription_id": foundation.subscription_id,
|
|
755
|
+
"tenant_id": foundation.tenant_id,
|
|
756
|
+
"client_id": foundation.client_id,
|
|
757
|
+
"principal_id": foundation.principal_id,
|
|
758
|
+
"foundation_binding_digest": foundation_binding_digest,
|
|
759
|
+
"key_vault_private_access": key_vault_private_access,
|
|
760
|
+
"key_vault_name": key_vault_name,
|
|
761
|
+
"document_storage_private_access": document_storage_private_access,
|
|
762
|
+
"document_storage_account_name": document_storage_account_name,
|
|
763
|
+
"resource_group_name": str(app["name"]),
|
|
764
|
+
"entra_binding_digest": entra_binding_digest,
|
|
765
|
+
"application_state_adoption_digest": adoption_digest,
|
|
766
|
+
"catalog_review_profile_digest": catalog_profile.profile_digest,
|
|
767
|
+
"catalog_review_selected": catalog_profile.selected,
|
|
768
|
+
"oev_input_digest": verifier_input_digest,
|
|
769
|
+
"notification_recipients_digest": notification_recipients.context_digest(recipients_input),
|
|
770
|
+
"development_authority_profile_digest": development_profile_digest,
|
|
771
|
+
"state_resource_group": str(ops["resource_group_name"]),
|
|
772
|
+
"state_account": str(state["account_name"]),
|
|
773
|
+
"state_container": str(state["container_name"]),
|
|
774
|
+
"inventory_progress_container_url": str(state["progress_container_url"]),
|
|
775
|
+
"initial_inventory_binding": initial_inventory_binding,
|
|
776
|
+
"state_key": f"fdai-{values['env']}.tfstate",
|
|
777
|
+
"kit_manifest_digest": kit_manifest_digest,
|
|
778
|
+
"runtime_release_digest": runtime_release_digest,
|
|
779
|
+
"runtime_profile": runtime_profile.to_mapping(),
|
|
780
|
+
"runtime_profile_digest": runtime_profile.digest,
|
|
781
|
+
"registry_name": registry,
|
|
782
|
+
"registry_login_server": login_server,
|
|
783
|
+
"image_refs": refs,
|
|
784
|
+
"infra": str(infra),
|
|
785
|
+
"terraform": str(terraform),
|
|
786
|
+
"provider_mirror": "" if provider_mirror is None else str(provider_mirror),
|
|
787
|
+
"provider_installation": provider_installation,
|
|
788
|
+
"kit_bin": str(kit_bin),
|
|
789
|
+
"terraform_config": str(terraform_config),
|
|
790
|
+
"terraform_data": str(work_dir / "terraform-data"),
|
|
791
|
+
}
|
|
792
|
+
if source_mode:
|
|
793
|
+
context.update(source_artifacts.snapshot_metadata())
|
|
794
|
+
hub_image_stage.bind_host_selection(context, hub_selection)
|
|
795
|
+
_replace_private_json(work_dir / "application.auto.tfvars.json", values)
|
|
796
|
+
_replace_private_json(work_dir / "context.json", context)
|
|
797
|
+
_terraform_init(work_dir, context)
|
|
798
|
+
if adoption is not None:
|
|
799
|
+
_adopt_application_state(work_dir, context, *adoption)
|
|
800
|
+
return {
|
|
801
|
+
"schema_version": "fdai.standalone-host-prepare.v1",
|
|
802
|
+
"state": "prepared",
|
|
803
|
+
"source_commit": source_commit,
|
|
804
|
+
"kit_manifest_digest": kit_manifest_digest,
|
|
805
|
+
"runtime_release_digest": runtime_release_digest,
|
|
806
|
+
"runtime_profile_digest": runtime_profile.digest,
|
|
807
|
+
"application_state_adopted": adoption is not None,
|
|
808
|
+
"focused_private_access": _focused_private_access(context),
|
|
809
|
+
"mutation_performed": False,
|
|
810
|
+
"subscription_ready": False,
|
|
811
|
+
**(hub_selection.metadata() if hub_selection is not None else {}),
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
def _prepare_runtime(_args: argparse.Namespace, work_dir: Path) -> dict[str, object]:
|
|
816
|
+
context = _private_json(work_dir / "context.json", "standalone host context")
|
|
817
|
+
if _runtime_platform(context) != "aks":
|
|
818
|
+
raise ValueError("runtime preparation is only valid for AKS")
|
|
819
|
+
if not (work_dir / "substrate-receipt.json").is_file():
|
|
820
|
+
raise ValueError("runtime preparation requires the applied substrate plan")
|
|
821
|
+
_managed_identity_login_from_context(context, work_dir)
|
|
822
|
+
substrate = Path(str(context["infra"]))
|
|
823
|
+
profile = _mapping(context.get("runtime_profile"), "runtime deployment profile")
|
|
824
|
+
application_values = _private_json(
|
|
825
|
+
work_dir / "application.auto.tfvars.json", "application variables"
|
|
826
|
+
)
|
|
827
|
+
runtime_infra = substrate / "runtimes/aks/cluster"
|
|
828
|
+
values = {
|
|
829
|
+
"location": application_values["region"],
|
|
830
|
+
"resource_group_name": _terraform_output(substrate, "resource_group_name"),
|
|
831
|
+
"aks_subnet_id": _terraform_output(substrate, "aks_subnet_id"),
|
|
832
|
+
"aks_api_server_subnet_id": _terraform_output(substrate, "aks_api_server_subnet_id"),
|
|
833
|
+
"api_server_authorized_ip_ranges": management_egress_cidrs(context, cwd=work_dir),
|
|
834
|
+
"container_registry_id": _terraform_output(substrate, "container_registry_id"),
|
|
835
|
+
"log_analytics_workspace_id": _terraform_output(substrate, "log_workspace_id"),
|
|
836
|
+
"managed_host_principal_id": context["principal_id"],
|
|
837
|
+
"environment": application_values["env"],
|
|
838
|
+
"region_short": application_values["region_short"],
|
|
839
|
+
"database_placement": profile["database_placement"],
|
|
840
|
+
"system_node_count": profile["system_node_count"],
|
|
841
|
+
"system_node_sku": profile["system_node_sku"],
|
|
842
|
+
"user_node_min_count": profile["user_node_min_count"],
|
|
843
|
+
"user_node_max_count": profile["user_node_max_count"],
|
|
844
|
+
"user_node_sku": profile["user_node_sku"],
|
|
845
|
+
"tags": {"fdai:runtime": "aks"},
|
|
846
|
+
}
|
|
847
|
+
context.update(
|
|
848
|
+
runtime_infra=str(runtime_infra),
|
|
849
|
+
runtime_state_key=f"fdai-{application_values['env']}-aks-cluster.tfstate",
|
|
850
|
+
runtime_terraform_data=str(work_dir / "terraform-data-runtime"),
|
|
851
|
+
resource_group_name=values["resource_group_name"],
|
|
852
|
+
)
|
|
853
|
+
_replace_or_verify_private_json(work_dir / "runtime.auto.tfvars.json", values)
|
|
854
|
+
_replace_private_json(work_dir / "context.json", context)
|
|
855
|
+
_initialize_terraform_stage("runtime", context, work_dir)
|
|
856
|
+
return {
|
|
857
|
+
"schema_version": "fdai.standalone-runtime-prepare.v1",
|
|
858
|
+
"state": "prepared",
|
|
859
|
+
"runtime_profile_digest": _runtime_profile_digest(context),
|
|
860
|
+
"state_key": context["runtime_state_key"],
|
|
861
|
+
"mutation_performed": False,
|
|
862
|
+
"subscription_ready": False,
|
|
863
|
+
}
|
|
864
|
+
|
|
865
|
+
|
|
866
|
+
def _prepare_database(_args: argparse.Namespace, work_dir: Path) -> dict[str, object]:
|
|
867
|
+
context = _private_json(work_dir / "context.json", "standalone host context")
|
|
868
|
+
profile = _mapping(context.get("runtime_profile"), "runtime deployment profile")
|
|
869
|
+
if _runtime_platform(context) != "aks" or profile.get("database_placement") != "postgres-aks":
|
|
870
|
+
raise ValueError("database preparation is only valid for postgres-aks")
|
|
871
|
+
for prerequisite in ("runtime-receipt.json", "image-import-receipt.json"):
|
|
872
|
+
if not (work_dir / prerequisite).is_file():
|
|
873
|
+
raise ValueError("database preparation prerequisites are incomplete")
|
|
874
|
+
_managed_identity_login_from_context(context, work_dir)
|
|
875
|
+
substrate = Path(str(context["infra"]))
|
|
876
|
+
identities = _terraform_json_output(substrate, "runtime_identity_bindings")
|
|
877
|
+
if not isinstance(identities, dict):
|
|
878
|
+
raise TypeError("AKS runtime identity output contract is invalid")
|
|
879
|
+
principals, ingestion_principal, ingestion_worker_principal = database_reader_principals(
|
|
880
|
+
identities
|
|
881
|
+
)
|
|
882
|
+
key_vault_id = _terraform_output(substrate, "key_vault_id")
|
|
883
|
+
_activate_terraform_stage("runtime", context, work_dir)
|
|
884
|
+
runtime_infra = Path(str(context["runtime_infra"]))
|
|
885
|
+
cluster_name = _terraform_output(runtime_infra, "cluster_name")
|
|
886
|
+
kubeconfig = _prepare_aks_kubeconfig(
|
|
887
|
+
context,
|
|
888
|
+
work_dir,
|
|
889
|
+
resource_group=str(context["resource_group_name"]),
|
|
890
|
+
cluster_name=cluster_name,
|
|
891
|
+
)
|
|
892
|
+
application_values = _private_json(
|
|
893
|
+
work_dir / "application.auto.tfvars.json", "application variables"
|
|
894
|
+
)
|
|
895
|
+
refs = _mapping(context.get("image_refs"), "runtime image references")
|
|
896
|
+
database_infra = substrate / "runtimes/aks/database"
|
|
897
|
+
values = {
|
|
898
|
+
"kubeconfig_path": str(kubeconfig),
|
|
899
|
+
"image": refs["pgvector"],
|
|
900
|
+
"key_vault_id": key_vault_id,
|
|
901
|
+
"runtime_principal_ids": sorted(principals),
|
|
902
|
+
"ingestion_api_principal_id": ingestion_principal,
|
|
903
|
+
"ingestion_worker_principal_id": ingestion_worker_principal,
|
|
904
|
+
"tags": {"fdai:runtime": "aks", "fdai:database-placement": "postgres-aks"},
|
|
905
|
+
}
|
|
906
|
+
context.update(
|
|
907
|
+
database_infra=str(database_infra),
|
|
908
|
+
database_state_key=f"fdai-{application_values['env']}-aks-database.tfstate",
|
|
909
|
+
database_terraform_data=str(work_dir / "terraform-data-database"),
|
|
910
|
+
kubeconfig=str(kubeconfig),
|
|
911
|
+
)
|
|
912
|
+
_replace_or_verify_private_json(work_dir / "database.auto.tfvars.json", values)
|
|
913
|
+
_replace_private_json(work_dir / "context.json", context)
|
|
914
|
+
_initialize_terraform_stage("database", context, work_dir)
|
|
915
|
+
return {
|
|
916
|
+
"schema_version": "fdai.standalone-database-prepare.v1",
|
|
917
|
+
"state": "prepared",
|
|
918
|
+
"runtime_profile_digest": _runtime_profile_digest(context),
|
|
919
|
+
"state_key": context["database_state_key"],
|
|
920
|
+
"mutation_performed": False,
|
|
921
|
+
"subscription_ready": False,
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
|
|
925
|
+
def _prepare_aks_application(_args: argparse.Namespace, work_dir: Path) -> dict[str, object]:
|
|
926
|
+
context = _private_json(work_dir / "context.json", "standalone host context")
|
|
927
|
+
if _runtime_platform(context) != "aks":
|
|
928
|
+
raise ValueError("AKS application preparation is only valid for AKS")
|
|
929
|
+
for prerequisite in (
|
|
930
|
+
"runtime-receipt.json",
|
|
931
|
+
"image-import-receipt.json",
|
|
932
|
+
"migration-receipt.json",
|
|
933
|
+
):
|
|
934
|
+
if not (work_dir / prerequisite).is_file():
|
|
935
|
+
raise ValueError("AKS application preparation prerequisites are incomplete")
|
|
936
|
+
_managed_identity_login_from_context(context, work_dir)
|
|
937
|
+
substrate = Path(str(context["infra"]))
|
|
938
|
+
runtime_infra = Path(str(context["runtime_infra"]))
|
|
939
|
+
substrate_outputs = {
|
|
940
|
+
"resource_group": _terraform_output(substrate, "resource_group_name"),
|
|
941
|
+
"identities": _terraform_json_output(substrate, "runtime_identity_bindings"),
|
|
942
|
+
"aks_subnet_id": _terraform_output(substrate, "aks_subnet_id"),
|
|
943
|
+
"console_hostname": _terraform_output(substrate, "console_default_hostname"),
|
|
944
|
+
"topics": _terraform_json_output(substrate, "event_bus_topics"),
|
|
945
|
+
"semantic_topics": _terraform_json_output(substrate, "event_bus_semantic_topics"),
|
|
946
|
+
"operating_model_topic": _terraform_output(substrate, "event_bus_operating_model_topic"),
|
|
947
|
+
"kafka": _terraform_output(substrate, "event_bus_kafka_bootstrap"),
|
|
948
|
+
"operational_kafka": _terraform_output(substrate, "event_bus_operational_kafka_bootstrap"),
|
|
949
|
+
"workspace": _terraform_output(substrate, "log_workspace_customer_id"),
|
|
950
|
+
"semantic_physical": _terraform_output(substrate, "event_bus_semantic_physical_topic"),
|
|
951
|
+
"llm_endpoint": _terraform_output(substrate, "llm_endpoint"),
|
|
952
|
+
"llm_model_endpoints": _terraform_json_output(substrate, "llm_model_endpoints"),
|
|
953
|
+
"resolved_models_sha256": _terraform_output(substrate, "resolved_models_sha256"),
|
|
954
|
+
"key_vault_uri": _terraform_output(substrate, "key_vault_uri"),
|
|
955
|
+
"application_insights_secret_name": _terraform_output(
|
|
956
|
+
substrate, "application_insights_connection_string_secret_name"
|
|
957
|
+
),
|
|
958
|
+
"cost_pseudonym_key_secret_name": _terraform_output(
|
|
959
|
+
substrate, "cost_pseudonym_key_secret_name"
|
|
960
|
+
),
|
|
961
|
+
"operational_history_container_url": _terraform_output(
|
|
962
|
+
substrate, "operational_history_container_url"
|
|
963
|
+
),
|
|
964
|
+
"document_store": _terraform_json_output(substrate, "document_storage_binding"),
|
|
965
|
+
"document_topics": _terraform_json_output(substrate, "document_event_topics"),
|
|
966
|
+
"catalog_review_gitops_binding": _terraform_json_output(
|
|
967
|
+
substrate, "catalog_review_gitops_binding"
|
|
968
|
+
),
|
|
969
|
+
"installation_binding": _terraform_output(substrate, "installation_binding"),
|
|
970
|
+
"operator_request_receipts": _terraform_json_output(
|
|
971
|
+
substrate, "operator_request_receipt_binding"
|
|
972
|
+
),
|
|
973
|
+
"operational_evidence_verifier_identity": _terraform_json_output(
|
|
974
|
+
substrate, "operational_evidence_verifier_identity"
|
|
975
|
+
),
|
|
976
|
+
"notification_email": _terraform_json_output(substrate, "notification_email_binding"),
|
|
977
|
+
}
|
|
978
|
+
if _database_placement(context) == "postgres-flex":
|
|
979
|
+
substrate_outputs.update(
|
|
980
|
+
postgres_fqdn=_terraform_output(substrate, "postgres_fqdn"),
|
|
981
|
+
postgres_database=_terraform_output(substrate, "postgres_database"),
|
|
982
|
+
)
|
|
983
|
+
else:
|
|
984
|
+
_activate_terraform_stage("database", context, work_dir)
|
|
985
|
+
database_infra = Path(str(context["database_infra"]))
|
|
986
|
+
substrate_outputs.update(
|
|
987
|
+
postgres_fqdn=_terraform_output(database_infra, "private_host"),
|
|
988
|
+
postgres_database=_terraform_output(database_infra, "database_name"),
|
|
989
|
+
)
|
|
990
|
+
resource_group = str(substrate_outputs["resource_group"])
|
|
991
|
+
_activate_terraform_stage("runtime", context, work_dir)
|
|
992
|
+
cluster_id = _terraform_output(runtime_infra, "cluster_id")
|
|
993
|
+
cluster_name = _terraform_output(runtime_infra, "cluster_name")
|
|
994
|
+
deployment_binding = deployment_binding_digest(context, cluster_name)
|
|
995
|
+
oidc_issuer_url = _terraform_output(runtime_infra, "oidc_issuer_url")
|
|
996
|
+
kubeconfig = _prepare_aks_kubeconfig(
|
|
997
|
+
context, work_dir, resource_group=resource_group, cluster_name=cluster_name
|
|
998
|
+
)
|
|
999
|
+
application_values = _private_json(
|
|
1000
|
+
work_dir / "application.auto.tfvars.json", "application variables"
|
|
1001
|
+
)
|
|
1002
|
+
console_selected = _selects_add_on(context, ProductAddOn.READ_ONLY_CONSOLE)
|
|
1003
|
+
enterprise_identity_selected = _selects_add_on(
|
|
1004
|
+
context, ProductAddOn.ENTERPRISE_IDENTITY_GOVERNANCE
|
|
1005
|
+
)
|
|
1006
|
+
operator_api_selected = console_selected or enterprise_identity_selected
|
|
1007
|
+
governed_execution_selected = _selects_add_on(context, ProductAddOn.GOVERNED_EXECUTION)
|
|
1008
|
+
notifications_selected = _selects_add_on(context, ProductAddOn.NOTIFICATIONS)
|
|
1009
|
+
console_origin = (
|
|
1010
|
+
_console_origin(str(substrate_outputs["console_hostname"])) if console_selected else ""
|
|
1011
|
+
)
|
|
1012
|
+
backend_nsg_id = standalone_host_values.require_browser_gateway_backend_nsg(
|
|
1013
|
+
enabled=console_selected,
|
|
1014
|
+
backend_nsg_id=_subnet_network_security_group(
|
|
1015
|
+
str(substrate_outputs["aks_subnet_id"]),
|
|
1016
|
+
context=context,
|
|
1017
|
+
cwd=work_dir,
|
|
1018
|
+
),
|
|
1019
|
+
)
|
|
1020
|
+
identities = substrate_outputs["identities"]
|
|
1021
|
+
topics = substrate_outputs["topics"]
|
|
1022
|
+
semantic_topics = substrate_outputs["semantic_topics"]
|
|
1023
|
+
if (
|
|
1024
|
+
not isinstance(identities, dict)
|
|
1025
|
+
or not isinstance(topics, list)
|
|
1026
|
+
or not isinstance(semantic_topics, list)
|
|
1027
|
+
or len(semantic_topics) < 3
|
|
1028
|
+
):
|
|
1029
|
+
raise ValueError("AKS substrate output contract is invalid")
|
|
1030
|
+
core_identity = _mapping(identities.get("core"), "core runtime identity")
|
|
1031
|
+
operator_identity = _selected_identity(identities, "operator", operator_api_selected)
|
|
1032
|
+
command_identity = _selected_identity(identities, "command", operator_api_selected)
|
|
1033
|
+
executor_identity = _selected_identity(identities, "executor", governed_execution_selected)
|
|
1034
|
+
notification_identity = _selected_identity(identities, "notification", notifications_selected)
|
|
1035
|
+
inventory_identity = _mapping(identities.get("inventory"), "inventory runtime identity")
|
|
1036
|
+
canary_identity = _mapping(identities.get("canary"), "canary runtime identity")
|
|
1037
|
+
ingestion_identity = _selected_identity(identities, "ingestion", console_selected)
|
|
1038
|
+
ingestion_worker_identity = _selected_identity(identities, "ingestion_worker", console_selected)
|
|
1039
|
+
core_environment = standalone_host_values.aks_core_environment(
|
|
1040
|
+
context=context,
|
|
1041
|
+
application_values=application_values,
|
|
1042
|
+
substrate_outputs=substrate_outputs,
|
|
1043
|
+
resource_group=resource_group,
|
|
1044
|
+
core_identity=core_identity,
|
|
1045
|
+
topics=topics,
|
|
1046
|
+
deployment_binding=deployment_binding,
|
|
1047
|
+
)
|
|
1048
|
+
core_environment.update(
|
|
1049
|
+
_aks_core_conversation_environment(
|
|
1050
|
+
application_values=application_values,
|
|
1051
|
+
substrate_outputs=substrate_outputs,
|
|
1052
|
+
semantic_topics=semantic_topics,
|
|
1053
|
+
)
|
|
1054
|
+
)
|
|
1055
|
+
receipt_core_env, receipt_core_secrets, receipt_operator_env, receipt_operator_secrets = (
|
|
1056
|
+
aks_operator_request_receipts(substrate_outputs["operator_request_receipts"])
|
|
1057
|
+
)
|
|
1058
|
+
core_environment.update({**AKS_CORE_STARTUP_READINESS, **receipt_core_env})
|
|
1059
|
+
license_environment, license_secrets = aks_license_environment(application_values)
|
|
1060
|
+
core_environment.update(license_environment)
|
|
1061
|
+
if notifications_selected:
|
|
1062
|
+
core_environment.update(
|
|
1063
|
+
standalone_host_values.aks_notification_environment(
|
|
1064
|
+
application_values=application_values,
|
|
1065
|
+
email_binding=substrate_outputs["notification_email"],
|
|
1066
|
+
identity=notification_identity,
|
|
1067
|
+
)
|
|
1068
|
+
)
|
|
1069
|
+
operator_environment = (
|
|
1070
|
+
aks_operator_environment(
|
|
1071
|
+
application_values=application_values,
|
|
1072
|
+
core_environment=core_environment,
|
|
1073
|
+
substrate_outputs=substrate_outputs,
|
|
1074
|
+
semantic_topics=semantic_topics,
|
|
1075
|
+
operator_identity=operator_identity,
|
|
1076
|
+
command_identity=command_identity,
|
|
1077
|
+
tenant_id=str(context["tenant_id"]),
|
|
1078
|
+
console_origin=console_origin,
|
|
1079
|
+
)
|
|
1080
|
+
if operator_api_selected
|
|
1081
|
+
else {}
|
|
1082
|
+
)
|
|
1083
|
+
dsn_secret = {
|
|
1084
|
+
"APPLICATIONINSIGHTS_CONNECTION_STRING": str(
|
|
1085
|
+
substrate_outputs["application_insights_secret_name"]
|
|
1086
|
+
),
|
|
1087
|
+
"FDAI_STATE_STORE_DSN": "fdai-state-store-dsn",
|
|
1088
|
+
**receipt_core_secrets,
|
|
1089
|
+
**license_secrets,
|
|
1090
|
+
}
|
|
1091
|
+
refs = _mapping(context.get("image_refs"), "runtime image references")
|
|
1092
|
+
workloads = {
|
|
1093
|
+
"core-control-plane": _aks_workload(
|
|
1094
|
+
"core",
|
|
1095
|
+
refs,
|
|
1096
|
+
core_identity,
|
|
1097
|
+
core_environment,
|
|
1098
|
+
dsn_secret,
|
|
1099
|
+
"/ready",
|
|
1100
|
+
"/live",
|
|
1101
|
+
additional_identities=(
|
|
1102
|
+
{"notification": notification_identity} if notifications_selected else None
|
|
1103
|
+
),
|
|
1104
|
+
),
|
|
1105
|
+
}
|
|
1106
|
+
if operator_api_selected:
|
|
1107
|
+
workloads["operator-service"] = _aks_workload(
|
|
1108
|
+
"operator",
|
|
1109
|
+
refs,
|
|
1110
|
+
operator_identity,
|
|
1111
|
+
{**operator_environment, **receipt_operator_env},
|
|
1112
|
+
{
|
|
1113
|
+
"FDAI_DATABASE_URL": "fdai-state-store-dsn",
|
|
1114
|
+
"FDAI_COST_PSEUDONYM_KEY": str(substrate_outputs["cost_pseudonym_key_secret_name"]),
|
|
1115
|
+
**receipt_operator_secrets,
|
|
1116
|
+
},
|
|
1117
|
+
"/healthz",
|
|
1118
|
+
"/healthz",
|
|
1119
|
+
external=True,
|
|
1120
|
+
service_port=80,
|
|
1121
|
+
additional_identities={"command": command_identity},
|
|
1122
|
+
)
|
|
1123
|
+
if governed_execution_selected:
|
|
1124
|
+
workloads["isolated-executor"] = _aks_workload(
|
|
1125
|
+
"executor",
|
|
1126
|
+
refs,
|
|
1127
|
+
executor_identity,
|
|
1128
|
+
{
|
|
1129
|
+
"AZURE_CLIENT_ID": executor_identity["client_id"],
|
|
1130
|
+
"RUNTIME_ENV": application_values["env"],
|
|
1131
|
+
"FDAI_ISOLATED_EXECUTOR_DEPLOYED": "1",
|
|
1132
|
+
"FDAI_ISOLATED_EXECUTOR_AUTHORITY_CUTOVER": "1",
|
|
1133
|
+
"FDAI_ISOLATED_EXECUTOR_MI_CLIENT_ID": executor_identity["client_id"],
|
|
1134
|
+
"KAFKA_BOOTSTRAP_SERVERS": substrate_outputs["operational_kafka"],
|
|
1135
|
+
"FDAI_ISOLATED_EXECUTOR_HEALTH_PORT": "8000",
|
|
1136
|
+
**_aks_kubernetes_direct_api_environment(
|
|
1137
|
+
cluster_id,
|
|
1138
|
+
namespace=_AKS_RUNTIME_NAMESPACE,
|
|
1139
|
+
),
|
|
1140
|
+
},
|
|
1141
|
+
{
|
|
1142
|
+
"FDAI_STATE_STORE_DSN": "fdai-state-store-dsn",
|
|
1143
|
+
"FDAI_RESOURCE_LOCK_DSN": "fdai-state-store-dsn",
|
|
1144
|
+
},
|
|
1145
|
+
"/ready",
|
|
1146
|
+
"/live",
|
|
1147
|
+
)
|
|
1148
|
+
if console_selected:
|
|
1149
|
+
workloads.update(
|
|
1150
|
+
_aks_document_workloads(
|
|
1151
|
+
refs=refs,
|
|
1152
|
+
ingestion_identity=ingestion_identity,
|
|
1153
|
+
worker_identity=ingestion_worker_identity,
|
|
1154
|
+
application_values=application_values,
|
|
1155
|
+
kafka=str(substrate_outputs["kafka"]),
|
|
1156
|
+
postgres_fqdn=str(substrate_outputs["postgres_fqdn"]),
|
|
1157
|
+
document_store=_mapping(
|
|
1158
|
+
substrate_outputs["document_store"], "document storage binding"
|
|
1159
|
+
),
|
|
1160
|
+
document_topics=_mapping(
|
|
1161
|
+
substrate_outputs["document_topics"], "document event topics"
|
|
1162
|
+
),
|
|
1163
|
+
console_origin=console_origin,
|
|
1164
|
+
)
|
|
1165
|
+
)
|
|
1166
|
+
_add_aks_operational_evidence_verifier_workload(
|
|
1167
|
+
workloads,
|
|
1168
|
+
refs=refs,
|
|
1169
|
+
identities=identities,
|
|
1170
|
+
substrate_outputs=substrate_outputs,
|
|
1171
|
+
context=context,
|
|
1172
|
+
application_values=application_values,
|
|
1173
|
+
)
|
|
1174
|
+
for workload in workloads.values():
|
|
1175
|
+
workload["source_commit"] = context["source_commit"]
|
|
1176
|
+
job_preparation = _prepare_aks_scheduled_jobs(
|
|
1177
|
+
refs=refs,
|
|
1178
|
+
core_identity=core_identity,
|
|
1179
|
+
inventory_identity=inventory_identity,
|
|
1180
|
+
canary_identity=canary_identity,
|
|
1181
|
+
core_environment=core_environment,
|
|
1182
|
+
substrate_outputs=substrate_outputs,
|
|
1183
|
+
source_revision=str(context["source_commit"]),
|
|
1184
|
+
tenant_id=str(context["tenant_id"]),
|
|
1185
|
+
subscription_id=str(context["subscription_id"]),
|
|
1186
|
+
catalog_review_selected=context.get("catalog_review_selected") is True,
|
|
1187
|
+
cluster_id=cluster_id,
|
|
1188
|
+
namespace=_AKS_RUNTIME_NAMESPACE,
|
|
1189
|
+
)
|
|
1190
|
+
scheduled_jobs = job_preparation.jobs
|
|
1191
|
+
workloads_infra = substrate / "runtimes/aks/workloads"
|
|
1192
|
+
values = {
|
|
1193
|
+
"kubeconfig_path": str(kubeconfig),
|
|
1194
|
+
"namespace": _AKS_RUNTIME_NAMESPACE,
|
|
1195
|
+
"tenant_id": context["tenant_id"],
|
|
1196
|
+
"oidc_issuer_url": oidc_issuer_url,
|
|
1197
|
+
"key_vault_name": _vault_name(str(substrate_outputs["key_vault_uri"])),
|
|
1198
|
+
"workloads": workloads,
|
|
1199
|
+
"scheduled_jobs": scheduled_jobs,
|
|
1200
|
+
"browser_gateway": {
|
|
1201
|
+
"enabled": console_selected,
|
|
1202
|
+
"resource_group_name": resource_group,
|
|
1203
|
+
"location": application_values["region"],
|
|
1204
|
+
"workload": application_values["workload"],
|
|
1205
|
+
"environment": application_values["env"],
|
|
1206
|
+
"region_short": application_values["region_short"],
|
|
1207
|
+
"resource_name_suffix": application_values["resource_name_suffix"],
|
|
1208
|
+
"backend_nsg_id": backend_nsg_id,
|
|
1209
|
+
},
|
|
1210
|
+
"tags": {"fdai.io/source-commit": context["source_commit"]},
|
|
1211
|
+
}
|
|
1212
|
+
context.update(
|
|
1213
|
+
workloads_infra=str(workloads_infra),
|
|
1214
|
+
workloads_state_key=f"fdai-{application_values['env']}-aks-workloads.tfstate",
|
|
1215
|
+
workloads_terraform_data=str(work_dir / "terraform-data-workloads"),
|
|
1216
|
+
kubeconfig=str(kubeconfig),
|
|
1217
|
+
deployment_binding=deployment_binding,
|
|
1218
|
+
expected_workloads={
|
|
1219
|
+
name: aks_readiness.workload_contract(workload) for name, workload in workloads.items()
|
|
1220
|
+
},
|
|
1221
|
+
protected_aks_job_template_digests=job_preparation.protected_template_digests,
|
|
1222
|
+
protected_aks_job_identity_bindings=job_preparation.protected_identity_bindings,
|
|
1223
|
+
catalog_review_available=job_preparation.catalog_review_available,
|
|
1224
|
+
)
|
|
1225
|
+
workloads_path = work_dir / "workloads.auto.tfvars.json"
|
|
1226
|
+
if workloads_path.exists():
|
|
1227
|
+
retained_values = _private_json(workloads_path, "workloads.auto.tfvars.json")
|
|
1228
|
+
license_binding_changed = _reconcile_retained_aks_license_binding(
|
|
1229
|
+
retained_values,
|
|
1230
|
+
values,
|
|
1231
|
+
)
|
|
1232
|
+
input_rebind_changed = False
|
|
1233
|
+
if retained_values != values:
|
|
1234
|
+
input_rebind_changed = bind_authoritative_backend_nsg(
|
|
1235
|
+
work_dir=work_dir,
|
|
1236
|
+
retained=retained_values,
|
|
1237
|
+
current=values,
|
|
1238
|
+
)
|
|
1239
|
+
if license_binding_changed and not input_rebind_changed:
|
|
1240
|
+
_replace_private_json(workloads_path, retained_values)
|
|
1241
|
+
else:
|
|
1242
|
+
_replace_or_verify_private_json(workloads_path, values)
|
|
1243
|
+
_replace_private_json(work_dir / "context.json", context)
|
|
1244
|
+
_initialize_terraform_stage("application", context, work_dir)
|
|
1245
|
+
return {
|
|
1246
|
+
"schema_version": "fdai.standalone-aks-application-prepare.v1",
|
|
1247
|
+
"state": "prepared",
|
|
1248
|
+
"runtime_profile_digest": _runtime_profile_digest(context),
|
|
1249
|
+
"workload_count": len(workloads),
|
|
1250
|
+
"scheduled_job_count": len(scheduled_jobs),
|
|
1251
|
+
"state_key": context["workloads_state_key"],
|
|
1252
|
+
"mutation_performed": False,
|
|
1253
|
+
"subscription_ready": False,
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1256
|
+
|
|
1257
|
+
def _prepare_aks_service_update(args: argparse.Namespace, work_dir: Path) -> dict[str, object]:
|
|
1258
|
+
"""Seal one service image update against the current complete AKS workload baseline."""
|
|
1259
|
+
|
|
1260
|
+
context = _private_json(work_dir / "context.json", "standalone host context")
|
|
1261
|
+
if _runtime_platform(context) != "aks":
|
|
1262
|
+
raise ValueError("AKS service update requires an AKS installation")
|
|
1263
|
+
service = str(args.service)
|
|
1264
|
+
_require_aks_application_baseline(work_dir, context, service=service)
|
|
1265
|
+
_managed_identity_login_from_context(context, work_dir)
|
|
1266
|
+
values = _private_json(work_dir / "workloads.auto.tfvars.json", "AKS workload variables")
|
|
1267
|
+
workloads = _mapping(values.get("workloads"), "AKS workloads")
|
|
1268
|
+
expected = _mapping(context.get("expected_workloads"), "expected AKS workloads")
|
|
1269
|
+
if set(workloads) != set(expected) or service not in workloads:
|
|
1270
|
+
raise ValueError("AKS service update workload inventory is incomplete")
|
|
1271
|
+
superseded_update_digest = service_recovery.superseded_update_digest(
|
|
1272
|
+
work_dir, context, service=service
|
|
1273
|
+
)
|
|
1274
|
+
default_source = str(context.get("source_commit", ""))
|
|
1275
|
+
for name, workload_value in workloads.items():
|
|
1276
|
+
workload = _mapping(workload_value, f"AKS workload {name}")
|
|
1277
|
+
contract = _mapping(expected.get(name), f"expected AKS workload {name}")
|
|
1278
|
+
source_commit = workload.get("source_commit", contract.get("source_commit", default_source))
|
|
1279
|
+
if not isinstance(source_commit, str) or _SOURCE_COMMIT.fullmatch(source_commit) is None:
|
|
1280
|
+
raise ValueError("AKS workload source revision is invalid")
|
|
1281
|
+
workload["source_commit"] = source_commit
|
|
1282
|
+
contract["source_commit"] = source_commit
|
|
1283
|
+
secrets = workload.get("secret_environment", {})
|
|
1284
|
+
contract["secret_environment"] = _validated_secret_env(secrets)
|
|
1285
|
+
workloads[name] = workload
|
|
1286
|
+
expected[name] = contract
|
|
1287
|
+
selected = _mapping(workloads[service], f"AKS workload {service}")
|
|
1288
|
+
current_image = selected.get("image")
|
|
1289
|
+
if not isinstance(current_image, str):
|
|
1290
|
+
raise TypeError("AKS service update current image is invalid")
|
|
1291
|
+
validate_update_request(
|
|
1292
|
+
service=service,
|
|
1293
|
+
image=str(args.image),
|
|
1294
|
+
source_commit=str(args.source_commit),
|
|
1295
|
+
current_image=current_image,
|
|
1296
|
+
)
|
|
1297
|
+
baseline = deployment_snapshot(
|
|
1298
|
+
_capture_aks_deployments(context), expected_services=set(workloads)
|
|
1299
|
+
)
|
|
1300
|
+
for name, contract_value in expected.items():
|
|
1301
|
+
contract = _mapping(contract_value, f"expected AKS workload {name}")
|
|
1302
|
+
if baseline[name]["image"] != contract.get("image") or baseline[name][
|
|
1303
|
+
"source_commit"
|
|
1304
|
+
] != contract.get("source_commit"):
|
|
1305
|
+
raise ValueError("AKS service update baseline differs from retained desired state")
|
|
1306
|
+
selected["image"] = str(args.image)
|
|
1307
|
+
selected["source_commit"] = str(args.source_commit)
|
|
1308
|
+
authority_operation, authority_digest = apply_service_update_development_authority(
|
|
1309
|
+
args,
|
|
1310
|
+
context=context,
|
|
1311
|
+
workload=selected,
|
|
1312
|
+
source_revision=str(args.source_commit),
|
|
1313
|
+
)
|
|
1314
|
+
workloads[service] = selected
|
|
1315
|
+
selected_expected = _mapping(expected[service], f"expected AKS workload {service}")
|
|
1316
|
+
selected_expected["image"] = str(args.image)
|
|
1317
|
+
selected_expected["source_commit"] = str(args.source_commit)
|
|
1318
|
+
expected[service] = selected_expected
|
|
1319
|
+
values["workloads"] = workloads
|
|
1320
|
+
update_identity = {
|
|
1321
|
+
"service": service,
|
|
1322
|
+
"image": str(args.image),
|
|
1323
|
+
"source_commit": str(args.source_commit),
|
|
1324
|
+
"target_binding": context["target_binding"],
|
|
1325
|
+
"runtime_profile_digest": _runtime_profile_digest(context),
|
|
1326
|
+
"development_authority_operation": authority_operation,
|
|
1327
|
+
"development_authority_profile_digest": authority_digest,
|
|
1328
|
+
}
|
|
1329
|
+
operation = f"service-update-{service}-{canonical_digest(update_identity)[:12]}"
|
|
1330
|
+
record: dict[str, object] = {
|
|
1331
|
+
"schema_version": "fdai.aks-service-update.v1",
|
|
1332
|
+
"operation": operation,
|
|
1333
|
+
**update_identity,
|
|
1334
|
+
"previous_image": current_image,
|
|
1335
|
+
"baseline": baseline,
|
|
1336
|
+
"prepared_at": _moment(datetime.now(UTC)),
|
|
1337
|
+
"mutation_performed": False,
|
|
1338
|
+
"subscription_ready": False,
|
|
1339
|
+
}
|
|
1340
|
+
if superseded_update_digest is not None:
|
|
1341
|
+
record["superseded_update_digest"] = superseded_update_digest
|
|
1342
|
+
record["update_digest"] = canonical_digest(record)
|
|
1343
|
+
_replace_private_json(work_dir / f"{operation}.json", record)
|
|
1344
|
+
context["expected_workloads"] = expected
|
|
1345
|
+
context["active_service_update"] = {
|
|
1346
|
+
"operation": operation,
|
|
1347
|
+
"service": service,
|
|
1348
|
+
"update_digest": record["update_digest"],
|
|
1349
|
+
}
|
|
1350
|
+
_replace_private_json(work_dir / "workloads.auto.tfvars.json", values)
|
|
1351
|
+
_replace_private_json(work_dir / "context.json", context)
|
|
1352
|
+
return {
|
|
1353
|
+
"schema_version": "fdai.aks-service-update-prepare.v1",
|
|
1354
|
+
"state": "prepared",
|
|
1355
|
+
"operation": operation,
|
|
1356
|
+
"service": service,
|
|
1357
|
+
"image": str(args.image),
|
|
1358
|
+
"source_commit": str(args.source_commit),
|
|
1359
|
+
"development_authority_operation": authority_operation,
|
|
1360
|
+
"development_authority_profile_digest": authority_digest,
|
|
1361
|
+
"update_digest": record["update_digest"],
|
|
1362
|
+
"mutation_performed": False,
|
|
1363
|
+
"subscription_ready": False,
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1366
|
+
|
|
1367
|
+
def _validate_historical_aks_baseline(
|
|
1368
|
+
*,
|
|
1369
|
+
state: dict[str, Any],
|
|
1370
|
+
variables: dict[str, Any],
|
|
1371
|
+
live: dict[str, Any],
|
|
1372
|
+
plan: dict[str, Any],
|
|
1373
|
+
) -> dict[str, object]:
|
|
1374
|
+
"""Cross-check retained Terraform state, desired inputs, live AKS, and convergence."""
|
|
1375
|
+
|
|
1376
|
+
lineage = state.get("lineage")
|
|
1377
|
+
serial = state.get("serial")
|
|
1378
|
+
resources = state.get("resources")
|
|
1379
|
+
if (
|
|
1380
|
+
state.get("version") != 4
|
|
1381
|
+
or not isinstance(lineage, str)
|
|
1382
|
+
or not lineage
|
|
1383
|
+
or type(serial) is not int
|
|
1384
|
+
or serial < 0
|
|
1385
|
+
or not isinstance(resources, list)
|
|
1386
|
+
):
|
|
1387
|
+
raise ValueError("historical AKS Terraform state is invalid")
|
|
1388
|
+
if variables.get("namespace") != _AKS_RUNTIME_NAMESPACE:
|
|
1389
|
+
raise ValueError("historical AKS workload namespace is invalid")
|
|
1390
|
+
workloads = _mapping(variables.get("workloads"), "historical AKS workloads")
|
|
1391
|
+
if set(workloads) != set(AKS_SERVICES):
|
|
1392
|
+
raise ValueError("historical AKS workload inventory is incomplete")
|
|
1393
|
+
|
|
1394
|
+
expected: dict[str, dict[str, object]] = {}
|
|
1395
|
+
for name in sorted(AKS_SERVICES):
|
|
1396
|
+
workload = _mapping(workloads.get(name), f"historical AKS workload {name}")
|
|
1397
|
+
image = workload.get("image")
|
|
1398
|
+
source_commit = workload.get("source_commit")
|
|
1399
|
+
replicas = workload.get("replicas")
|
|
1400
|
+
max_replicas = workload.get("max_replicas")
|
|
1401
|
+
if (
|
|
1402
|
+
not isinstance(image, str)
|
|
1403
|
+
or re.fullmatch(r"[a-z0-9.-]+(?::[0-9]+)?/[a-z0-9._/-]+@sha256:[0-9a-f]{64}", image)
|
|
1404
|
+
is None
|
|
1405
|
+
or not isinstance(source_commit, str)
|
|
1406
|
+
or _SOURCE_COMMIT.fullmatch(source_commit) is None
|
|
1407
|
+
or type(replicas) is not int
|
|
1408
|
+
or type(max_replicas) is not int
|
|
1409
|
+
or replicas < 1
|
|
1410
|
+
or max_replicas < replicas
|
|
1411
|
+
or not aks_readiness.valid_secret_environment(workload.get("secret_environment", {}))
|
|
1412
|
+
):
|
|
1413
|
+
raise ValueError("historical AKS workload contract is invalid")
|
|
1414
|
+
expected[name] = {
|
|
1415
|
+
"image": image,
|
|
1416
|
+
"replicas": replicas,
|
|
1417
|
+
"max_replicas": max_replicas,
|
|
1418
|
+
"source_commit": source_commit,
|
|
1419
|
+
"secret_environment": dict(workload.get("secret_environment", {})),
|
|
1420
|
+
}
|
|
1421
|
+
|
|
1422
|
+
state_workloads = _historical_state_workloads(resources)
|
|
1423
|
+
desired_identity = {
|
|
1424
|
+
name: {
|
|
1425
|
+
"image": workload["image"],
|
|
1426
|
+
"source_commit": workload["source_commit"],
|
|
1427
|
+
}
|
|
1428
|
+
for name, workload in expected.items()
|
|
1429
|
+
}
|
|
1430
|
+
if state_workloads != desired_identity:
|
|
1431
|
+
raise ValueError("historical AKS Terraform state differs from desired workloads")
|
|
1432
|
+
observed = deployment_snapshot(json.dumps(live), expected_services=set(AKS_SERVICES))
|
|
1433
|
+
live_identity = {
|
|
1434
|
+
name: {
|
|
1435
|
+
"image": workload["image"],
|
|
1436
|
+
"source_commit": workload["source_commit"],
|
|
1437
|
+
}
|
|
1438
|
+
for name, workload in observed.items()
|
|
1439
|
+
}
|
|
1440
|
+
if live_identity != desired_identity:
|
|
1441
|
+
raise ValueError("historical AKS live baseline differs from desired workloads")
|
|
1442
|
+
|
|
1443
|
+
changes = plan.get("resource_changes")
|
|
1444
|
+
if plan.get("errored") is not False:
|
|
1445
|
+
raise ValueError("historical AKS deployment plan is invalid")
|
|
1446
|
+
if not isinstance(changes, list) or not changes:
|
|
1447
|
+
raise ValueError("historical AKS deployment plan inventory is invalid")
|
|
1448
|
+
deployment_seen = False
|
|
1449
|
+
projected: list[dict[str, object]] = []
|
|
1450
|
+
mutating_services: list[str] = []
|
|
1451
|
+
out_of_scope_mutation = False
|
|
1452
|
+
for value in changes:
|
|
1453
|
+
change = _mapping(value, "historical AKS deployment change")
|
|
1454
|
+
actions = _mapping(change.get("change"), "historical AKS deployment actions").get("actions")
|
|
1455
|
+
address = change.get("address")
|
|
1456
|
+
if (
|
|
1457
|
+
not isinstance(address, str)
|
|
1458
|
+
or not isinstance(actions, list)
|
|
1459
|
+
or not all(isinstance(action, str) for action in actions)
|
|
1460
|
+
):
|
|
1461
|
+
raise ValueError("historical AKS deployment plan change is invalid")
|
|
1462
|
+
reordered = secret_binding_reordered(address, change.get("change"))
|
|
1463
|
+
projected.append({"address": address, "actions": ["no-op"] if reordered else actions})
|
|
1464
|
+
if not reordered and not set(actions).issubset({"no-op", "read"}):
|
|
1465
|
+
out_of_scope_mutation = True
|
|
1466
|
+
if isinstance(address, str) and address.startswith("kubernetes_deployment_v1.workload["):
|
|
1467
|
+
deployment_seen = True
|
|
1468
|
+
if set(actions) == {"update"}:
|
|
1469
|
+
match = re.fullmatch(r'kubernetes_deployment_v1[.]workload\["([^"\\]+)"\]', address)
|
|
1470
|
+
if match is not None:
|
|
1471
|
+
mutating_services.append(match.group(1))
|
|
1472
|
+
if not deployment_seen:
|
|
1473
|
+
raise ValueError("historical AKS deployment plan omits workload state")
|
|
1474
|
+
if mutating_services:
|
|
1475
|
+
if (
|
|
1476
|
+
plan.get("applyable") is not True
|
|
1477
|
+
or len(mutating_services) != 1
|
|
1478
|
+
or mutating_services[0] not in AKS_SERVICES
|
|
1479
|
+
):
|
|
1480
|
+
raise ValueError("historical AKS deployment plan mutation is invalid")
|
|
1481
|
+
validate_plan_scope(
|
|
1482
|
+
{"resource_changes": projected},
|
|
1483
|
+
service=mutating_services[0],
|
|
1484
|
+
)
|
|
1485
|
+
else:
|
|
1486
|
+
if out_of_scope_mutation:
|
|
1487
|
+
raise ValueError("historical AKS deployment plan contains an out-of-scope mutation")
|
|
1488
|
+
if plan.get("complete") is not True:
|
|
1489
|
+
raise ValueError("historical AKS zero-change plan is incomplete")
|
|
1490
|
+
return {
|
|
1491
|
+
"expected_workloads": expected,
|
|
1492
|
+
"state_lineage": lineage,
|
|
1493
|
+
"state_serial": serial,
|
|
1494
|
+
"historical_plan_service": mutating_services[0] if mutating_services else None,
|
|
1495
|
+
}
|
|
1496
|
+
|
|
1497
|
+
|
|
1498
|
+
def _adopt_historical_aks_application(
|
|
1499
|
+
args: argparse.Namespace, work_dir: Path
|
|
1500
|
+
) -> dict[str, object]:
|
|
1501
|
+
"""Adopt one verified historical AKS workload baseline without changing Azure state."""
|
|
1502
|
+
|
|
1503
|
+
binding_path = _absolute(args.binding)
|
|
1504
|
+
state_path = _absolute(args.state)
|
|
1505
|
+
variables_path = _absolute(args.variables)
|
|
1506
|
+
live_path = _absolute(args.live)
|
|
1507
|
+
plan_path = _absolute(args.plan)
|
|
1508
|
+
binding = _private_json(binding_path, "historical AKS adoption binding")
|
|
1509
|
+
state = _private_json(state_path, "historical AKS Terraform state")
|
|
1510
|
+
variables = _private_json(variables_path, "historical AKS workload variables")
|
|
1511
|
+
live = _private_json(live_path, "historical AKS live baseline")
|
|
1512
|
+
plan = _private_json(plan_path, "historical AKS deployment plan")
|
|
1513
|
+
baseline = _validate_historical_aks_baseline(
|
|
1514
|
+
state=state,
|
|
1515
|
+
variables=variables,
|
|
1516
|
+
live=live,
|
|
1517
|
+
plan=plan,
|
|
1518
|
+
)
|
|
1519
|
+
expected_binding_fields = {
|
|
1520
|
+
"schema_version",
|
|
1521
|
+
"source_commit",
|
|
1522
|
+
"subscription_id",
|
|
1523
|
+
"tenant_id",
|
|
1524
|
+
"client_id",
|
|
1525
|
+
"principal_id",
|
|
1526
|
+
"runtime_profile",
|
|
1527
|
+
"source_root",
|
|
1528
|
+
"terraform",
|
|
1529
|
+
"terraform_sha256",
|
|
1530
|
+
"provider_mirror",
|
|
1531
|
+
"workloads_infra",
|
|
1532
|
+
"terraform_data",
|
|
1533
|
+
"kubeconfig",
|
|
1534
|
+
"kit_bin",
|
|
1535
|
+
}
|
|
1536
|
+
if set(binding) != expected_binding_fields:
|
|
1537
|
+
raise ValueError("historical AKS adoption binding fields are invalid")
|
|
1538
|
+
if binding.get("schema_version") != "fdai.historical-aks-application-binding.v1":
|
|
1539
|
+
raise ValueError("historical AKS adoption binding schema is invalid")
|
|
1540
|
+
subscription_id = _required_guid(binding, "subscription_id")
|
|
1541
|
+
tenant_id = _required_guid(binding, "tenant_id")
|
|
1542
|
+
client_id = _required_guid(binding, "client_id")
|
|
1543
|
+
principal_id = _required_guid(binding, "principal_id")
|
|
1544
|
+
source_commit = str(binding.get("source_commit", ""))
|
|
1545
|
+
if _SOURCE_COMMIT.fullmatch(source_commit) is None:
|
|
1546
|
+
raise ValueError("historical AKS adoption source revision is invalid")
|
|
1547
|
+
profile_value = _mapping(binding.get("runtime_profile"), "historical AKS runtime profile")
|
|
1548
|
+
profile = RuntimeDeploymentProfile.from_mapping(profile_value)
|
|
1549
|
+
if profile.runtime_platform.value != "aks" or not profile.matches_mapping(profile_value):
|
|
1550
|
+
raise ValueError("historical AKS adoption runtime profile differs")
|
|
1551
|
+
|
|
1552
|
+
evidence_root = binding_path.parent.resolve()
|
|
1553
|
+
source_root = _adoption_work_path(evidence_root, binding, "source_root", directory=True)
|
|
1554
|
+
terraform = _adoption_work_path(evidence_root, binding, "terraform", executable=True)
|
|
1555
|
+
provider_mirror = _adoption_work_path(evidence_root, binding, "provider_mirror", directory=True)
|
|
1556
|
+
workloads_infra = _adoption_work_path(evidence_root, binding, "workloads_infra", directory=True)
|
|
1557
|
+
terraform_data = _adoption_work_path(evidence_root, binding, "terraform_data", directory=True)
|
|
1558
|
+
kubeconfig = _adoption_work_path(evidence_root, binding, "kubeconfig")
|
|
1559
|
+
kit_bin = _adoption_work_path(evidence_root, binding, "kit_bin", directory=True)
|
|
1560
|
+
if _executable_digest(terraform) != binding.get("terraform_sha256"):
|
|
1561
|
+
raise ValueError("historical AKS Terraform binary digest differs")
|
|
1562
|
+
observed_source = _capture(
|
|
1563
|
+
("git", "rev-parse", "HEAD"),
|
|
1564
|
+
cwd=source_root,
|
|
1565
|
+
timeout=30,
|
|
1566
|
+
reason="historical AKS source revision readback failed",
|
|
1567
|
+
).strip()
|
|
1568
|
+
if observed_source != source_commit:
|
|
1569
|
+
raise ValueError("historical AKS source revision differs")
|
|
1570
|
+
|
|
1571
|
+
baseline_workloads = _mapping(
|
|
1572
|
+
baseline.get("expected_workloads"), "historical AKS expected workloads"
|
|
1573
|
+
)
|
|
1574
|
+
registry_hosts = {
|
|
1575
|
+
str(workload["image"]).partition("/")[0]
|
|
1576
|
+
for workload in baseline_workloads.values()
|
|
1577
|
+
if isinstance(workload, dict)
|
|
1578
|
+
}
|
|
1579
|
+
if len(registry_hosts) != 1:
|
|
1580
|
+
raise ValueError("historical AKS workload registries differ")
|
|
1581
|
+
registry_login_server = registry_hosts.pop()
|
|
1582
|
+
match = re.fullmatch(r"([a-z0-9]{5,50})[.]azurecr[.]io", registry_login_server)
|
|
1583
|
+
if match is None:
|
|
1584
|
+
raise ValueError("historical AKS registry is invalid")
|
|
1585
|
+
target_binding = compute_target_binding(
|
|
1586
|
+
tenant_id=tenant_id,
|
|
1587
|
+
subscription_id=subscription_id,
|
|
1588
|
+
)
|
|
1589
|
+
terraform_config = work_dir / "historical-adoption-terraform.rc"
|
|
1590
|
+
expected_config = _terraform_configuration(provider_mirror)
|
|
1591
|
+
if terraform_config.exists():
|
|
1592
|
+
if read_private_bytes(terraform_config, max_bytes=16_384).decode() != expected_config:
|
|
1593
|
+
raise ValueError("historical AKS Terraform configuration differs")
|
|
1594
|
+
else:
|
|
1595
|
+
write_private_output(terraform_config, expected_config)
|
|
1596
|
+
context: dict[str, object] = {
|
|
1597
|
+
"source_commit": source_commit,
|
|
1598
|
+
"target_binding": target_binding,
|
|
1599
|
+
"subscription_id": subscription_id,
|
|
1600
|
+
"tenant_id": tenant_id,
|
|
1601
|
+
"client_id": client_id,
|
|
1602
|
+
"principal_id": principal_id,
|
|
1603
|
+
"runtime_profile": profile.to_mapping(),
|
|
1604
|
+
"runtime_profile_digest": profile.digest,
|
|
1605
|
+
"registry_name": match.group(1),
|
|
1606
|
+
"registry_login_server": registry_login_server,
|
|
1607
|
+
"infra": str(workloads_infra),
|
|
1608
|
+
"terraform": str(terraform),
|
|
1609
|
+
"provider_mirror": str(provider_mirror),
|
|
1610
|
+
"terraform_config": str(terraform_config),
|
|
1611
|
+
"terraform_data": str(terraform_data),
|
|
1612
|
+
"kit_bin": str(kit_bin),
|
|
1613
|
+
"workloads_infra": str(workloads_infra),
|
|
1614
|
+
"workloads_terraform_data": str(terraform_data),
|
|
1615
|
+
"kubeconfig": str(kubeconfig),
|
|
1616
|
+
"expected_workloads": baseline["expected_workloads"],
|
|
1617
|
+
}
|
|
1618
|
+
reconciliation_applied = (work_dir / "historical-reconciliation-receipt.json").is_file()
|
|
1619
|
+
if reconciliation_applied:
|
|
1620
|
+
retained_context = _private_json(
|
|
1621
|
+
work_dir / "context.json",
|
|
1622
|
+
"historical AKS reconciliation context",
|
|
1623
|
+
)
|
|
1624
|
+
reconciliation = _historical_reconciliation_record(
|
|
1625
|
+
retained_context,
|
|
1626
|
+
work_dir,
|
|
1627
|
+
stage="application",
|
|
1628
|
+
service=None,
|
|
1629
|
+
)
|
|
1630
|
+
if reconciliation is None:
|
|
1631
|
+
raise ValueError("historical AKS reconciliation context is unavailable")
|
|
1632
|
+
_historical_reconciliation_receipt(work_dir, context, reconciliation)
|
|
1633
|
+
state_path = work_dir / "historical-reconciliation-state.json"
|
|
1634
|
+
variables_path = work_dir / "workloads.auto.tfvars.json"
|
|
1635
|
+
live_path = work_dir / "historical-reconciliation-live.json"
|
|
1636
|
+
plan_path = work_dir / "historical-reconciliation-zero-plan.json"
|
|
1637
|
+
state = _private_json(state_path, "reconciled historical AKS Terraform state")
|
|
1638
|
+
variables = _private_json(variables_path, "reconciled historical AKS variables")
|
|
1639
|
+
live = _private_json(live_path, "reconciled historical AKS live baseline")
|
|
1640
|
+
plan = _private_json(plan_path, "reconciled historical AKS zero-change plan")
|
|
1641
|
+
baseline = _validate_historical_aks_baseline(
|
|
1642
|
+
state=state,
|
|
1643
|
+
variables=variables,
|
|
1644
|
+
live=live,
|
|
1645
|
+
plan=plan,
|
|
1646
|
+
)
|
|
1647
|
+
_managed_identity_login_from_context(context, work_dir)
|
|
1648
|
+
remote_state = json.loads(
|
|
1649
|
+
_capture(
|
|
1650
|
+
("terraform", "state", "pull"),
|
|
1651
|
+
cwd=workloads_infra,
|
|
1652
|
+
timeout=300,
|
|
1653
|
+
reason="historical AKS remote state readback failed",
|
|
1654
|
+
)
|
|
1655
|
+
)
|
|
1656
|
+
if not isinstance(remote_state, dict):
|
|
1657
|
+
raise TypeError("historical AKS remote state readback is invalid")
|
|
1658
|
+
if (
|
|
1659
|
+
remote_state.get("lineage") != baseline["state_lineage"]
|
|
1660
|
+
or type(remote_state.get("serial")) is not int
|
|
1661
|
+
or remote_state["serial"] < baseline["state_serial"]
|
|
1662
|
+
or canonical_digest(remote_state.get("resources"))
|
|
1663
|
+
!= canonical_digest(state.get("resources"))
|
|
1664
|
+
):
|
|
1665
|
+
raise ValueError("historical AKS remote state differs from retained evidence")
|
|
1666
|
+
observed_live = json.loads(_capture_aks_deployments(context))
|
|
1667
|
+
if not isinstance(observed_live, dict):
|
|
1668
|
+
raise TypeError("historical AKS live readback is invalid")
|
|
1669
|
+
current_variables = reconciled_variables(
|
|
1670
|
+
state=state,
|
|
1671
|
+
variables=variables,
|
|
1672
|
+
live=observed_live,
|
|
1673
|
+
)
|
|
1674
|
+
current_variables_path = work_dir / "historical-reconciliation.auto.tfvars.json"
|
|
1675
|
+
_replace_private_json(current_variables_path, current_variables)
|
|
1676
|
+
_activate_terraform_stage("application", context, work_dir)
|
|
1677
|
+
current_plan_path = work_dir / (
|
|
1678
|
+
"historical-adoption-current.tfplan"
|
|
1679
|
+
if reconciliation_applied
|
|
1680
|
+
else "historical-reconciliation.tfplan"
|
|
1681
|
+
)
|
|
1682
|
+
current_plan_path.unlink(missing_ok=True)
|
|
1683
|
+
current_plan = subprocess.run(
|
|
1684
|
+
(
|
|
1685
|
+
"terraform",
|
|
1686
|
+
"plan",
|
|
1687
|
+
"-detailed-exitcode",
|
|
1688
|
+
"-input=false",
|
|
1689
|
+
"-no-color",
|
|
1690
|
+
f"-var-file={current_variables_path}",
|
|
1691
|
+
f"-out={current_plan_path}",
|
|
1692
|
+
),
|
|
1693
|
+
cwd=workloads_infra,
|
|
1694
|
+
check=False,
|
|
1695
|
+
capture_output=True,
|
|
1696
|
+
timeout=1800,
|
|
1697
|
+
)
|
|
1698
|
+
if current_plan.returncode not in {0, 2}:
|
|
1699
|
+
raise ValueError("historical AKS current plan failed")
|
|
1700
|
+
current_plan_document = json.loads(
|
|
1701
|
+
_capture(
|
|
1702
|
+
("terraform", "show", "-json", str(current_plan_path)),
|
|
1703
|
+
cwd=workloads_infra,
|
|
1704
|
+
timeout=300,
|
|
1705
|
+
reason="historical AKS current plan projection failed",
|
|
1706
|
+
)
|
|
1707
|
+
)
|
|
1708
|
+
if not isinstance(current_plan_document, dict):
|
|
1709
|
+
raise TypeError("historical AKS current plan projection is invalid")
|
|
1710
|
+
if current_plan.returncode == 2:
|
|
1711
|
+
mutations = validate_reconciliation_plan(
|
|
1712
|
+
current_plan_document,
|
|
1713
|
+
variables=current_variables,
|
|
1714
|
+
)
|
|
1715
|
+
summary = _plan_summary(current_plan_document)
|
|
1716
|
+
review: dict[str, object] = {
|
|
1717
|
+
"schema_version": "fdai.standalone-application-plan.v1",
|
|
1718
|
+
"stage": "application",
|
|
1719
|
+
"plan_digest": _file_digest(current_plan_path),
|
|
1720
|
+
"target_binding": target_binding,
|
|
1721
|
+
"source_commit": source_commit,
|
|
1722
|
+
"runtime_profile_digest": profile.digest,
|
|
1723
|
+
"runtime_platform": "aks",
|
|
1724
|
+
"summary": summary,
|
|
1725
|
+
"expires_at": _moment(datetime.now(UTC) + timedelta(hours=1)),
|
|
1726
|
+
"mutation_performed": False,
|
|
1727
|
+
"subscription_ready": False,
|
|
1728
|
+
"historical_reconciliation": {
|
|
1729
|
+
"operation": "historical-reconciliation",
|
|
1730
|
+
"variables_digest": canonical_digest(current_variables),
|
|
1731
|
+
"mutations": list(mutations),
|
|
1732
|
+
},
|
|
1733
|
+
}
|
|
1734
|
+
review["review_digest"] = canonical_digest(review)
|
|
1735
|
+
context["active_historical_reconciliation"] = {
|
|
1736
|
+
"operation": "historical-reconciliation",
|
|
1737
|
+
"variables_digest": canonical_digest(current_variables),
|
|
1738
|
+
"mutations": list(mutations),
|
|
1739
|
+
"review_digest": review["review_digest"],
|
|
1740
|
+
}
|
|
1741
|
+
_replace_private_json(work_dir / "context.json", context)
|
|
1742
|
+
_replace_private_json(work_dir / "workloads.auto.tfvars.json", current_variables)
|
|
1743
|
+
_replace_private_json(work_dir / "application.auto.tfvars.json", {"env": "dev"})
|
|
1744
|
+
_replace_private_json(work_dir / "historical-reconciliation-review.json", review)
|
|
1745
|
+
return {
|
|
1746
|
+
"schema_version": "fdai.historical-aks-application-adoption.v1",
|
|
1747
|
+
"state": "reconciliation-required",
|
|
1748
|
+
"reconciliation_review": review,
|
|
1749
|
+
"mutation_performed": False,
|
|
1750
|
+
"subscription_ready": False,
|
|
1751
|
+
}
|
|
1752
|
+
_validate_historical_aks_baseline(
|
|
1753
|
+
state=remote_state,
|
|
1754
|
+
variables=current_variables,
|
|
1755
|
+
live=observed_live,
|
|
1756
|
+
plan=current_plan_document,
|
|
1757
|
+
)
|
|
1758
|
+
current_plan_summary = _plan_summary(current_plan_document)
|
|
1759
|
+
|
|
1760
|
+
context.pop("active_historical_reconciliation", None)
|
|
1761
|
+
context_digest = canonical_digest(context)
|
|
1762
|
+
receipt: dict[str, object] = {
|
|
1763
|
+
"schema_version": "fdai.historical-aks-application-adoption.v1",
|
|
1764
|
+
"state": "adopted",
|
|
1765
|
+
"target_binding": target_binding,
|
|
1766
|
+
"source_commit": source_commit,
|
|
1767
|
+
"runtime_profile_digest": profile.digest,
|
|
1768
|
+
"context_digest": context_digest,
|
|
1769
|
+
"binding_sha256": _file_digest(binding_path),
|
|
1770
|
+
"retained_state_sha256": _file_digest(state_path),
|
|
1771
|
+
"retained_variables_sha256": _file_digest(variables_path),
|
|
1772
|
+
"retained_live_sha256": _file_digest(live_path),
|
|
1773
|
+
"retained_plan_sha256": _file_digest(plan_path),
|
|
1774
|
+
"remote_state_lineage": remote_state["lineage"],
|
|
1775
|
+
"remote_state_serial": remote_state["serial"],
|
|
1776
|
+
"historical_plan_service": baseline["historical_plan_service"],
|
|
1777
|
+
"current_plan_summary": current_plan_summary,
|
|
1778
|
+
"managed_identity_verified": True,
|
|
1779
|
+
"remote_state_verified": True,
|
|
1780
|
+
"live_baseline_verified": True,
|
|
1781
|
+
"terraform_zero_change_verified": True,
|
|
1782
|
+
"azure_resource_mutation_performed": False,
|
|
1783
|
+
"mutation_performed": True,
|
|
1784
|
+
"subscription_ready": False,
|
|
1785
|
+
}
|
|
1786
|
+
receipt["receipt_digest"] = canonical_digest(receipt)
|
|
1787
|
+
retained_receipt = work_dir / "historical-aks-application-adoption-receipt.json"
|
|
1788
|
+
if retained_receipt.exists():
|
|
1789
|
+
if _private_json(retained_receipt, "historical AKS adoption receipt") != receipt:
|
|
1790
|
+
raise ValueError("retained historical AKS adoption receipt differs")
|
|
1791
|
+
else:
|
|
1792
|
+
_replace_private_json(work_dir / "context.json", context)
|
|
1793
|
+
_replace_private_json(work_dir / "workloads.auto.tfvars.json", current_variables)
|
|
1794
|
+
_replace_private_json(work_dir / "application.auto.tfvars.json", {"env": "dev"})
|
|
1795
|
+
_replace_private_json(retained_receipt, receipt)
|
|
1796
|
+
return receipt
|
|
1797
|
+
|
|
1798
|
+
|
|
1799
|
+
def _adoption_work_path(
|
|
1800
|
+
evidence_root: Path,
|
|
1801
|
+
binding: dict[str, Any],
|
|
1802
|
+
name: str,
|
|
1803
|
+
*,
|
|
1804
|
+
directory: bool = False,
|
|
1805
|
+
executable: bool = False,
|
|
1806
|
+
) -> Path:
|
|
1807
|
+
value = binding.get(name)
|
|
1808
|
+
if not isinstance(value, str) or not value or Path(value).is_absolute():
|
|
1809
|
+
raise ValueError(f"historical AKS adoption {name} path is invalid")
|
|
1810
|
+
selected = Path(value)
|
|
1811
|
+
candidate = evidence_root
|
|
1812
|
+
for part in selected.parts:
|
|
1813
|
+
candidate /= part
|
|
1814
|
+
if candidate.is_symlink():
|
|
1815
|
+
raise ValueError(
|
|
1816
|
+
f"historical AKS adoption {name} path is outside the evidence directory"
|
|
1817
|
+
)
|
|
1818
|
+
path = candidate.resolve()
|
|
1819
|
+
if not path.is_relative_to(evidence_root):
|
|
1820
|
+
raise ValueError(f"historical AKS adoption {name} path is outside the evidence directory")
|
|
1821
|
+
details = path.stat()
|
|
1822
|
+
if directory and not stat.S_ISDIR(details.st_mode):
|
|
1823
|
+
raise ValueError(f"historical AKS adoption {name} directory is unavailable")
|
|
1824
|
+
if not directory and not stat.S_ISREG(details.st_mode):
|
|
1825
|
+
raise ValueError(f"historical AKS adoption {name} file is unavailable")
|
|
1826
|
+
if executable and not details.st_mode & stat.S_IXUSR:
|
|
1827
|
+
raise ValueError(f"historical AKS adoption {name} file is not executable")
|
|
1828
|
+
return path
|
|
1829
|
+
|
|
1830
|
+
|
|
1831
|
+
def _require_aks_application_baseline(
|
|
1832
|
+
work_dir: Path,
|
|
1833
|
+
context: dict[str, object],
|
|
1834
|
+
*,
|
|
1835
|
+
service: str | None = None,
|
|
1836
|
+
) -> None:
|
|
1837
|
+
if (work_dir / "application-receipt.json").is_file():
|
|
1838
|
+
return
|
|
1839
|
+
if service_recovery.partial_core_update_admissible(work_dir, context, service=service):
|
|
1840
|
+
return
|
|
1841
|
+
path = work_dir / "historical-aks-application-adoption-receipt.json"
|
|
1842
|
+
if not path.is_file():
|
|
1843
|
+
raise ValueError("AKS service update requires an applied or adopted application baseline")
|
|
1844
|
+
receipt = _private_json(path, "historical AKS adoption receipt")
|
|
1845
|
+
digest = receipt.get("receipt_digest")
|
|
1846
|
+
document = {key: value for key, value in receipt.items() if key != "receipt_digest"}
|
|
1847
|
+
summary = _mapping(receipt.get("current_plan_summary"), "historical AKS plan summary")
|
|
1848
|
+
action_counts = _mapping(summary.get("action_counts"), "historical AKS plan action counts")
|
|
1849
|
+
resource_types = _mapping(
|
|
1850
|
+
summary.get("resource_type_counts"), "historical AKS plan resource types"
|
|
1851
|
+
)
|
|
1852
|
+
expected_fields = {
|
|
1853
|
+
"schema_version",
|
|
1854
|
+
"state",
|
|
1855
|
+
"target_binding",
|
|
1856
|
+
"source_commit",
|
|
1857
|
+
"runtime_profile_digest",
|
|
1858
|
+
"context_digest",
|
|
1859
|
+
"binding_sha256",
|
|
1860
|
+
"retained_state_sha256",
|
|
1861
|
+
"retained_variables_sha256",
|
|
1862
|
+
"retained_live_sha256",
|
|
1863
|
+
"retained_plan_sha256",
|
|
1864
|
+
"remote_state_lineage",
|
|
1865
|
+
"remote_state_serial",
|
|
1866
|
+
"historical_plan_service",
|
|
1867
|
+
"current_plan_summary",
|
|
1868
|
+
"managed_identity_verified",
|
|
1869
|
+
"remote_state_verified",
|
|
1870
|
+
"live_baseline_verified",
|
|
1871
|
+
"terraform_zero_change_verified",
|
|
1872
|
+
"azure_resource_mutation_performed",
|
|
1873
|
+
"mutation_performed",
|
|
1874
|
+
"subscription_ready",
|
|
1875
|
+
"receipt_digest",
|
|
1876
|
+
}
|
|
1877
|
+
if (
|
|
1878
|
+
set(receipt) != expected_fields
|
|
1879
|
+
or receipt.get("schema_version") != "fdai.historical-aks-application-adoption.v1"
|
|
1880
|
+
or receipt.get("state") != "adopted"
|
|
1881
|
+
or receipt.get("target_binding") != context.get("target_binding")
|
|
1882
|
+
or receipt.get("source_commit") != context.get("source_commit")
|
|
1883
|
+
or receipt.get("runtime_profile_digest") != _runtime_profile_digest(context)
|
|
1884
|
+
or receipt.get("context_digest") != canonical_digest(context)
|
|
1885
|
+
or receipt.get("managed_identity_verified") is not True
|
|
1886
|
+
or receipt.get("remote_state_verified") is not True
|
|
1887
|
+
or receipt.get("live_baseline_verified") is not True
|
|
1888
|
+
or receipt.get("terraform_zero_change_verified") is not True
|
|
1889
|
+
or receipt.get("azure_resource_mutation_performed") is not False
|
|
1890
|
+
or receipt.get("mutation_performed") is not True
|
|
1891
|
+
or receipt.get("subscription_ready") is not False
|
|
1892
|
+
or receipt.get("historical_plan_service") not in {*AKS_SERVICES, None}
|
|
1893
|
+
or not isinstance(receipt.get("remote_state_lineage"), str)
|
|
1894
|
+
or type(receipt.get("remote_state_serial")) is not int
|
|
1895
|
+
or any(
|
|
1896
|
+
not isinstance(receipt.get(field), str)
|
|
1897
|
+
or _DIGEST.fullmatch(str(receipt[field])) is None
|
|
1898
|
+
for field in (
|
|
1899
|
+
"binding_sha256",
|
|
1900
|
+
"retained_state_sha256",
|
|
1901
|
+
"retained_variables_sha256",
|
|
1902
|
+
"retained_live_sha256",
|
|
1903
|
+
"retained_plan_sha256",
|
|
1904
|
+
)
|
|
1905
|
+
)
|
|
1906
|
+
or set(action_counts) != {"create", "update", "delete", "replace", "read", "no-op"}
|
|
1907
|
+
or any(type(value) is not int or value < 0 for value in action_counts.values())
|
|
1908
|
+
or any(action_counts[action] != 0 for action in ("create", "update", "delete", "replace"))
|
|
1909
|
+
or resource_types.get("kubernetes_deployment_v1") != len(AKS_SERVICES)
|
|
1910
|
+
or not isinstance(summary.get("resource_changes"), list)
|
|
1911
|
+
or digest != canonical_digest(document)
|
|
1912
|
+
):
|
|
1913
|
+
raise ValueError("historical AKS adoption receipt is invalid")
|
|
1914
|
+
|
|
1915
|
+
|
|
1916
|
+
def _historical_state_workloads(resources: list[object]) -> dict[str, dict[str, str]]:
|
|
1917
|
+
deployments = [
|
|
1918
|
+
_mapping(value, "historical AKS Terraform resource")
|
|
1919
|
+
for value in resources
|
|
1920
|
+
if isinstance(value, dict)
|
|
1921
|
+
and value.get("mode") == "managed"
|
|
1922
|
+
and value.get("type") == "kubernetes_deployment_v1"
|
|
1923
|
+
and value.get("name") == "workload"
|
|
1924
|
+
]
|
|
1925
|
+
if len(deployments) != 1 or not isinstance(deployments[0].get("instances"), list):
|
|
1926
|
+
raise ValueError("historical AKS Terraform workload resource is invalid")
|
|
1927
|
+
result: dict[str, dict[str, str]] = {}
|
|
1928
|
+
for value in deployments[0]["instances"]:
|
|
1929
|
+
instance = _mapping(value, "historical AKS Terraform workload instance")
|
|
1930
|
+
name = instance.get("index_key")
|
|
1931
|
+
attributes = _mapping(
|
|
1932
|
+
instance.get("attributes"), "historical AKS Terraform workload attributes"
|
|
1933
|
+
)
|
|
1934
|
+
try:
|
|
1935
|
+
metadata = _mapping(
|
|
1936
|
+
_single(attributes.get("metadata")), "historical AKS Terraform metadata"
|
|
1937
|
+
)
|
|
1938
|
+
template = _mapping(
|
|
1939
|
+
_single(_single(attributes.get("spec")).get("template")),
|
|
1940
|
+
"historical AKS Terraform Pod template",
|
|
1941
|
+
)
|
|
1942
|
+
labels = _mapping(
|
|
1943
|
+
_single(template.get("metadata")).get("labels"),
|
|
1944
|
+
"historical AKS Terraform Pod labels",
|
|
1945
|
+
)
|
|
1946
|
+
containers = _single(template.get("spec")).get("container")
|
|
1947
|
+
except (AttributeError, TypeError) as exc:
|
|
1948
|
+
raise ValueError("historical AKS Terraform workload structure is invalid") from exc
|
|
1949
|
+
if (
|
|
1950
|
+
not isinstance(name, str)
|
|
1951
|
+
or name not in AKS_SERVICES
|
|
1952
|
+
or name in result
|
|
1953
|
+
or metadata.get("name") != name
|
|
1954
|
+
or not isinstance(containers, list)
|
|
1955
|
+
):
|
|
1956
|
+
raise ValueError("historical AKS Terraform workload identity is invalid")
|
|
1957
|
+
selected = [
|
|
1958
|
+
_mapping(container, "historical AKS Terraform container")
|
|
1959
|
+
for container in containers
|
|
1960
|
+
if isinstance(container, dict) and container.get("name") == name
|
|
1961
|
+
]
|
|
1962
|
+
if len(selected) != 1:
|
|
1963
|
+
raise ValueError("historical AKS Terraform workload container is invalid")
|
|
1964
|
+
image = selected[0].get("image")
|
|
1965
|
+
source_commit = labels.get("fdai.io/source-commit")
|
|
1966
|
+
if not isinstance(image, str) or not isinstance(source_commit, str):
|
|
1967
|
+
raise TypeError("historical AKS Terraform rollout identity is invalid")
|
|
1968
|
+
result[name] = {"image": image, "source_commit": source_commit}
|
|
1969
|
+
if set(result) != set(AKS_SERVICES):
|
|
1970
|
+
raise ValueError("historical AKS Terraform workload inventory is incomplete")
|
|
1971
|
+
return dict(sorted(result.items()))
|
|
1972
|
+
|
|
1973
|
+
|
|
1974
|
+
def _service_update_record(
|
|
1975
|
+
context: dict[str, object],
|
|
1976
|
+
work_dir: Path,
|
|
1977
|
+
*,
|
|
1978
|
+
stage: str,
|
|
1979
|
+
service: str | None,
|
|
1980
|
+
) -> dict[str, Any] | None:
|
|
1981
|
+
if service is None:
|
|
1982
|
+
return None
|
|
1983
|
+
if stage != "application" or _runtime_platform(context) != "aks":
|
|
1984
|
+
raise ValueError("--service is supported only for an AKS application stage")
|
|
1985
|
+
active = _mapping(context.get("active_service_update"), "active AKS service update")
|
|
1986
|
+
operation = active.get("operation")
|
|
1987
|
+
if (
|
|
1988
|
+
active.get("service") != service
|
|
1989
|
+
or not isinstance(operation, str)
|
|
1990
|
+
or re.fullmatch(rf"service-update-{re.escape(service)}-[0-9a-f]{{12}}", operation) is None
|
|
1991
|
+
):
|
|
1992
|
+
raise ValueError("active AKS service update does not match the selected service")
|
|
1993
|
+
record = _private_json(work_dir / f"{operation}.json", "AKS service update")
|
|
1994
|
+
update_digest = record.get("update_digest")
|
|
1995
|
+
document = {key: value for key, value in record.items() if key != "update_digest"}
|
|
1996
|
+
if (
|
|
1997
|
+
record.get("schema_version") != "fdai.aks-service-update.v1"
|
|
1998
|
+
or record.get("operation") != operation
|
|
1999
|
+
or record.get("service") != service
|
|
2000
|
+
or active.get("update_digest") != update_digest
|
|
2001
|
+
or update_digest != canonical_digest(document)
|
|
2002
|
+
or record.get("target_binding") != context.get("target_binding")
|
|
2003
|
+
or record.get("runtime_profile_digest") != _runtime_profile_digest(context)
|
|
2004
|
+
or not isinstance(record.get("baseline"), dict)
|
|
2005
|
+
):
|
|
2006
|
+
raise ValueError("active AKS service update record is invalid")
|
|
2007
|
+
service_recovery.validate_repair_binding(record)
|
|
2008
|
+
image = record.get("image")
|
|
2009
|
+
source_commit = record.get("source_commit")
|
|
2010
|
+
previous_image = record.get("previous_image")
|
|
2011
|
+
if not all(isinstance(value, str) for value in (image, source_commit, previous_image)):
|
|
2012
|
+
raise ValueError("active AKS service update binding is invalid")
|
|
2013
|
+
validate_update_request(
|
|
2014
|
+
service=service,
|
|
2015
|
+
image=str(image),
|
|
2016
|
+
source_commit=str(source_commit),
|
|
2017
|
+
current_image=str(previous_image),
|
|
2018
|
+
)
|
|
2019
|
+
return record
|
|
2020
|
+
|
|
2021
|
+
|
|
2022
|
+
def _historical_reconciliation_record(
|
|
2023
|
+
context: dict[str, object],
|
|
2024
|
+
work_dir: Path,
|
|
2025
|
+
*,
|
|
2026
|
+
stage: str,
|
|
2027
|
+
service: str | None,
|
|
2028
|
+
) -> dict[str, Any] | None:
|
|
2029
|
+
value = context.get("active_historical_reconciliation")
|
|
2030
|
+
if value is None:
|
|
2031
|
+
return None
|
|
2032
|
+
if stage != "application" or service is not None:
|
|
2033
|
+
raise ValueError("historical AKS reconciliation blocks another application operation")
|
|
2034
|
+
record = _mapping(value, "active historical AKS reconciliation")
|
|
2035
|
+
mutations = record.get("mutations")
|
|
2036
|
+
variables = _private_json(
|
|
2037
|
+
work_dir / "workloads.auto.tfvars.json",
|
|
2038
|
+
"historical AKS reconciliation variables",
|
|
2039
|
+
)
|
|
2040
|
+
review = _private_json(
|
|
2041
|
+
work_dir / "historical-reconciliation-review.json",
|
|
2042
|
+
"historical AKS reconciliation review",
|
|
2043
|
+
)
|
|
2044
|
+
review_binding = review.get("historical_reconciliation")
|
|
2045
|
+
if (
|
|
2046
|
+
set(record) != {"operation", "variables_digest", "mutations", "review_digest"}
|
|
2047
|
+
or record.get("operation") != "historical-reconciliation"
|
|
2048
|
+
or record.get("variables_digest") != canonical_digest(variables)
|
|
2049
|
+
or not isinstance(mutations, list)
|
|
2050
|
+
or not mutations
|
|
2051
|
+
or len(mutations) > 64
|
|
2052
|
+
or mutations != sorted(set(mutations))
|
|
2053
|
+
or any(not isinstance(item, str) or not item for item in mutations)
|
|
2054
|
+
or record.get("review_digest") != review.get("review_digest")
|
|
2055
|
+
or review_binding
|
|
2056
|
+
!= {
|
|
2057
|
+
"operation": record.get("operation"),
|
|
2058
|
+
"variables_digest": record.get("variables_digest"),
|
|
2059
|
+
"mutations": mutations,
|
|
2060
|
+
}
|
|
2061
|
+
):
|
|
2062
|
+
raise ValueError("active historical AKS reconciliation is invalid")
|
|
2063
|
+
return record
|
|
2064
|
+
|
|
2065
|
+
|
|
2066
|
+
def _application_operation(
|
|
2067
|
+
context: dict[str, object],
|
|
2068
|
+
work_dir: Path,
|
|
2069
|
+
*,
|
|
2070
|
+
stage: str,
|
|
2071
|
+
service: str | None,
|
|
2072
|
+
update: dict[str, Any] | None,
|
|
2073
|
+
) -> tuple[str, dict[str, Any] | None]:
|
|
2074
|
+
reconciliation = _historical_reconciliation_record(
|
|
2075
|
+
context,
|
|
2076
|
+
work_dir,
|
|
2077
|
+
stage=stage,
|
|
2078
|
+
service=service,
|
|
2079
|
+
)
|
|
2080
|
+
if reconciliation is not None:
|
|
2081
|
+
return str(reconciliation["operation"]), reconciliation
|
|
2082
|
+
if stage == "application" and service is None and update is None:
|
|
2083
|
+
pending = _pending_application_operation(
|
|
2084
|
+
context,
|
|
2085
|
+
work_dir,
|
|
2086
|
+
input_rebind_pending=_application_successor_prerequisite(work_dir, stage),
|
|
2087
|
+
)
|
|
2088
|
+
if pending is not None:
|
|
2089
|
+
return pending, None
|
|
2090
|
+
return (str(update["operation"]) if update is not None else stage), None
|
|
2091
|
+
|
|
2092
|
+
|
|
2093
|
+
def _capture_aks_deployments(context: dict[str, object], *, service: str | None = None) -> str:
|
|
2094
|
+
return standalone_kubernetes_tools.capture_deployments(
|
|
2095
|
+
context,
|
|
2096
|
+
namespace=_AKS_RUNTIME_NAMESPACE,
|
|
2097
|
+
service=service,
|
|
2098
|
+
capture=_capture_env,
|
|
2099
|
+
)
|
|
2100
|
+
|
|
2101
|
+
|
|
2102
|
+
def _capture_aks_pods(context: dict[str, object], *, service: str) -> str:
|
|
2103
|
+
return standalone_kubernetes_tools.capture_pods(
|
|
2104
|
+
context,
|
|
2105
|
+
namespace=_AKS_RUNTIME_NAMESPACE,
|
|
2106
|
+
service=service,
|
|
2107
|
+
capture=_capture_env,
|
|
2108
|
+
)
|
|
2109
|
+
|
|
2110
|
+
|
|
2111
|
+
def _readback_aks_service_update(context: dict[str, object], update: dict[str, Any]) -> bool:
|
|
2112
|
+
service = str(update["service"])
|
|
2113
|
+
expected = _mapping(context.get("expected_workloads"), "expected AKS workloads")
|
|
2114
|
+
contract = _mapping(expected.get(service), f"expected AKS workload {service}")
|
|
2115
|
+
if contract.get("image") != update.get("image") or contract.get("source_commit") != update.get(
|
|
2116
|
+
"source_commit"
|
|
2117
|
+
):
|
|
2118
|
+
return False
|
|
2119
|
+
try:
|
|
2120
|
+
before = _mapping(update.get("baseline"), "AKS service update baseline")
|
|
2121
|
+
after = deployment_snapshot(
|
|
2122
|
+
_capture_aks_deployments(context), expected_services=set(expected)
|
|
2123
|
+
)
|
|
2124
|
+
except ValueError:
|
|
2125
|
+
return False
|
|
2126
|
+
if not peers_unchanged(before=before, after=after, service=service):
|
|
2127
|
+
return False
|
|
2128
|
+
return aks_readiness.verify_workload_health(
|
|
2129
|
+
deployments=_capture_aks_deployments(context, service=service),
|
|
2130
|
+
pods=_capture_aks_pods(context, service=service),
|
|
2131
|
+
expected={service: contract},
|
|
2132
|
+
)
|
|
2133
|
+
|
|
2134
|
+
|
|
2135
|
+
def _service_update_zero_change(
|
|
2136
|
+
context: dict[str, object], work_dir: Path, update: dict[str, Any]
|
|
2137
|
+
) -> bool:
|
|
2138
|
+
infra, variables = _stage_paths("application", context, work_dir)
|
|
2139
|
+
_activate_terraform_stage("application", context, work_dir)
|
|
2140
|
+
completed = subprocess.run(
|
|
2141
|
+
(
|
|
2142
|
+
"terraform",
|
|
2143
|
+
"plan",
|
|
2144
|
+
"-detailed-exitcode",
|
|
2145
|
+
"-input=false",
|
|
2146
|
+
"-no-color",
|
|
2147
|
+
f"-var-file={variables}",
|
|
2148
|
+
f"-target={_service_update_target(str(update['service']))}",
|
|
2149
|
+
),
|
|
2150
|
+
cwd=infra,
|
|
2151
|
+
check=False,
|
|
2152
|
+
capture_output=True,
|
|
2153
|
+
timeout=1800,
|
|
2154
|
+
)
|
|
2155
|
+
return completed.returncode == 0
|
|
2156
|
+
|
|
2157
|
+
|
|
2158
|
+
def _historical_reconciliation_postconditions(
|
|
2159
|
+
context: dict[str, object],
|
|
2160
|
+
work_dir: Path,
|
|
2161
|
+
reconciliation: dict[str, Any],
|
|
2162
|
+
) -> dict[str, str]:
|
|
2163
|
+
"""Retain authoritative state, typed live readback, and a full zero-change plan."""
|
|
2164
|
+
|
|
2165
|
+
infra, variables_path = _stage_paths("application", context, work_dir)
|
|
2166
|
+
variables = _private_json(variables_path, "historical AKS reconciliation variables")
|
|
2167
|
+
if reconciliation.get("variables_digest") != canonical_digest(variables):
|
|
2168
|
+
raise ValueError("historical AKS reconciliation variables changed")
|
|
2169
|
+
_activate_terraform_stage("application", context, work_dir)
|
|
2170
|
+
zero_plan_path = work_dir / "historical-reconciliation-zero.tfplan"
|
|
2171
|
+
zero_plan_path.unlink(missing_ok=True)
|
|
2172
|
+
completed = subprocess.run(
|
|
2173
|
+
(
|
|
2174
|
+
"terraform",
|
|
2175
|
+
"plan",
|
|
2176
|
+
"-detailed-exitcode",
|
|
2177
|
+
"-input=false",
|
|
2178
|
+
"-no-color",
|
|
2179
|
+
f"-var-file={variables_path}",
|
|
2180
|
+
f"-out={zero_plan_path}",
|
|
2181
|
+
),
|
|
2182
|
+
cwd=infra,
|
|
2183
|
+
check=False,
|
|
2184
|
+
capture_output=True,
|
|
2185
|
+
timeout=1800,
|
|
2186
|
+
)
|
|
2187
|
+
if completed.returncode != 0:
|
|
2188
|
+
raise ValueError("historical AKS reconciliation is not zero-change")
|
|
2189
|
+
zero_plan = json.loads(
|
|
2190
|
+
_capture(
|
|
2191
|
+
("terraform", "show", "-json", str(zero_plan_path)),
|
|
2192
|
+
cwd=infra,
|
|
2193
|
+
timeout=300,
|
|
2194
|
+
reason="historical AKS reconciliation plan projection failed",
|
|
2195
|
+
)
|
|
2196
|
+
)
|
|
2197
|
+
state = json.loads(
|
|
2198
|
+
_capture(
|
|
2199
|
+
("terraform", "state", "pull"),
|
|
2200
|
+
cwd=infra,
|
|
2201
|
+
timeout=300,
|
|
2202
|
+
reason="historical AKS reconciliation state readback failed",
|
|
2203
|
+
)
|
|
2204
|
+
)
|
|
2205
|
+
live = json.loads(_capture_aks_deployments(context))
|
|
2206
|
+
if not all(isinstance(value, dict) for value in (zero_plan, state, live)):
|
|
2207
|
+
raise ValueError("historical AKS reconciliation readback is invalid")
|
|
2208
|
+
_validate_historical_aks_baseline(
|
|
2209
|
+
state=state,
|
|
2210
|
+
variables=variables,
|
|
2211
|
+
live=live,
|
|
2212
|
+
plan=zero_plan,
|
|
2213
|
+
)
|
|
2214
|
+
outputs = {
|
|
2215
|
+
"state": work_dir / "historical-reconciliation-state.json",
|
|
2216
|
+
"live": work_dir / "historical-reconciliation-live.json",
|
|
2217
|
+
"plan": work_dir / "historical-reconciliation-zero-plan.json",
|
|
2218
|
+
}
|
|
2219
|
+
for name, path in outputs.items():
|
|
2220
|
+
value = {"state": state, "live": live, "plan": zero_plan}[name]
|
|
2221
|
+
_replace_private_json(path, value)
|
|
2222
|
+
return {
|
|
2223
|
+
"post_state_sha256": _file_digest(outputs["state"]),
|
|
2224
|
+
"post_variables_sha256": _file_digest(variables_path),
|
|
2225
|
+
"post_live_sha256": _file_digest(outputs["live"]),
|
|
2226
|
+
"zero_plan_sha256": _file_digest(outputs["plan"]),
|
|
2227
|
+
}
|
|
2228
|
+
|
|
2229
|
+
|
|
2230
|
+
def _validate_historical_reconciliation_plan(
|
|
2231
|
+
context: dict[str, object],
|
|
2232
|
+
work_dir: Path,
|
|
2233
|
+
reconciliation: dict[str, Any],
|
|
2234
|
+
plan_path: Path,
|
|
2235
|
+
) -> None:
|
|
2236
|
+
infra, variables_path = _stage_paths("application", context, work_dir)
|
|
2237
|
+
document = json.loads(
|
|
2238
|
+
_capture(
|
|
2239
|
+
("terraform", "show", "-json", str(plan_path)),
|
|
2240
|
+
cwd=infra,
|
|
2241
|
+
timeout=300,
|
|
2242
|
+
reason="historical AKS reconciliation plan projection failed",
|
|
2243
|
+
)
|
|
2244
|
+
)
|
|
2245
|
+
if not isinstance(document, dict):
|
|
2246
|
+
raise TypeError("historical AKS reconciliation plan projection is invalid")
|
|
2247
|
+
mutations = validate_reconciliation_plan(
|
|
2248
|
+
document,
|
|
2249
|
+
variables=_private_json(variables_path, "historical AKS reconciliation variables"),
|
|
2250
|
+
)
|
|
2251
|
+
if list(mutations) != reconciliation.get("mutations"):
|
|
2252
|
+
raise ValueError("historical AKS reconciliation plan scope changed")
|
|
2253
|
+
|
|
2254
|
+
|
|
2255
|
+
def _historical_reconciliation_receipt(
|
|
2256
|
+
work_dir: Path,
|
|
2257
|
+
context: dict[str, object],
|
|
2258
|
+
reconciliation: dict[str, Any],
|
|
2259
|
+
) -> dict[str, object]:
|
|
2260
|
+
path = work_dir / "historical-reconciliation-receipt.json"
|
|
2261
|
+
receipt = _private_json(path, "historical AKS reconciliation receipt")
|
|
2262
|
+
required = {
|
|
2263
|
+
"schema_version",
|
|
2264
|
+
"state",
|
|
2265
|
+
"stage",
|
|
2266
|
+
"operation",
|
|
2267
|
+
"plan_digest",
|
|
2268
|
+
"runtime_profile_digest",
|
|
2269
|
+
"claim_digest",
|
|
2270
|
+
"control_plane_readback_verified",
|
|
2271
|
+
"effect_verified",
|
|
2272
|
+
"terraform_zero_change_verified",
|
|
2273
|
+
"mutations",
|
|
2274
|
+
"post_state_sha256",
|
|
2275
|
+
"post_variables_sha256",
|
|
2276
|
+
"post_live_sha256",
|
|
2277
|
+
"zero_plan_sha256",
|
|
2278
|
+
"mutation_performed",
|
|
2279
|
+
"subscription_ready",
|
|
2280
|
+
"receipt_digest",
|
|
2281
|
+
}
|
|
2282
|
+
optional = {"verification_only_recovery"}
|
|
2283
|
+
document = {key: value for key, value in receipt.items() if key != "receipt_digest"}
|
|
2284
|
+
expected_files = {
|
|
2285
|
+
"post_state_sha256": work_dir / "historical-reconciliation-state.json",
|
|
2286
|
+
"post_variables_sha256": work_dir / "workloads.auto.tfvars.json",
|
|
2287
|
+
"post_live_sha256": work_dir / "historical-reconciliation-live.json",
|
|
2288
|
+
"zero_plan_sha256": work_dir / "historical-reconciliation-zero-plan.json",
|
|
2289
|
+
}
|
|
2290
|
+
review = _private_json(
|
|
2291
|
+
work_dir / "historical-reconciliation-review.json",
|
|
2292
|
+
"historical AKS reconciliation review",
|
|
2293
|
+
)
|
|
2294
|
+
claim = _private_json(
|
|
2295
|
+
work_dir / "historical-reconciliation-claim.json",
|
|
2296
|
+
"historical AKS reconciliation claim",
|
|
2297
|
+
)
|
|
2298
|
+
if (
|
|
2299
|
+
not required.issubset(receipt)
|
|
2300
|
+
or set(receipt) - required != set(receipt).intersection(optional)
|
|
2301
|
+
or receipt.get("schema_version") != "fdai.historical-aks-reconciliation-receipt.v1"
|
|
2302
|
+
or receipt.get("state") != "applied"
|
|
2303
|
+
or receipt.get("stage") != "application"
|
|
2304
|
+
or receipt.get("operation") != reconciliation.get("operation")
|
|
2305
|
+
or receipt.get("plan_digest") != review.get("plan_digest")
|
|
2306
|
+
or receipt.get("claim_digest") != canonical_digest(claim)
|
|
2307
|
+
or receipt.get("runtime_profile_digest") != _runtime_profile_digest(context)
|
|
2308
|
+
or receipt.get("control_plane_readback_verified") is not True
|
|
2309
|
+
or receipt.get("effect_verified") is not True
|
|
2310
|
+
or receipt.get("terraform_zero_change_verified") is not True
|
|
2311
|
+
or receipt.get("mutations") != reconciliation.get("mutations")
|
|
2312
|
+
or receipt.get("mutation_performed") is not True
|
|
2313
|
+
or receipt.get("subscription_ready") is not False
|
|
2314
|
+
or receipt.get("receipt_digest") != canonical_digest(document)
|
|
2315
|
+
or any(
|
|
2316
|
+
not isinstance(receipt.get(field), str) or receipt[field] != _file_digest(file_path)
|
|
2317
|
+
for field, file_path in expected_files.items()
|
|
2318
|
+
)
|
|
2319
|
+
):
|
|
2320
|
+
raise ValueError("historical AKS reconciliation receipt is invalid")
|
|
2321
|
+
return receipt
|
|
2322
|
+
|
|
2323
|
+
|
|
2324
|
+
def _plan(args: argparse.Namespace, work_dir: Path) -> dict[str, object]:
|
|
2325
|
+
stage = str(args.stage)
|
|
2326
|
+
service = getattr(args, "service", None)
|
|
2327
|
+
historical_baseline = (work_dir / "historical-aks-application-adoption-receipt.json").is_file()
|
|
2328
|
+
if stage == "runtime" and not (work_dir / "substrate-receipt.json").is_file():
|
|
2329
|
+
raise ValueError("runtime plan prerequisites are incomplete")
|
|
2330
|
+
if stage == "database":
|
|
2331
|
+
for prerequisite in ("runtime-receipt.json", "image-import-receipt.json"):
|
|
2332
|
+
if not (work_dir / prerequisite).is_file():
|
|
2333
|
+
raise ValueError("database plan prerequisites are incomplete")
|
|
2334
|
+
if stage == "application":
|
|
2335
|
+
context = _private_json(work_dir / "context.json", "standalone host context")
|
|
2336
|
+
if historical_baseline:
|
|
2337
|
+
_require_aks_application_baseline(work_dir, context)
|
|
2338
|
+
else:
|
|
2339
|
+
for prerequisite in (
|
|
2340
|
+
"substrate-receipt.json",
|
|
2341
|
+
"image-import-receipt.json",
|
|
2342
|
+
"migration-receipt.json",
|
|
2343
|
+
):
|
|
2344
|
+
if not (work_dir / prerequisite).is_file():
|
|
2345
|
+
raise ValueError("application plan prerequisites are incomplete")
|
|
2346
|
+
if (
|
|
2347
|
+
_runtime_platform(context) == "aks"
|
|
2348
|
+
and not (work_dir / "runtime-receipt.json").is_file()
|
|
2349
|
+
):
|
|
2350
|
+
raise ValueError("AKS application plan requires the applied runtime plan")
|
|
2351
|
+
if (
|
|
2352
|
+
_database_placement(context) == "postgres-aks"
|
|
2353
|
+
and not (work_dir / "database-receipt.json").is_file()
|
|
2354
|
+
):
|
|
2355
|
+
raise ValueError("AKS application plan requires the applied database plan")
|
|
2356
|
+
context = _private_json(work_dir / "context.json", "standalone host context")
|
|
2357
|
+
update = _service_update_record(context, work_dir, stage=stage, service=service)
|
|
2358
|
+
operation, reconciliation = _application_operation(
|
|
2359
|
+
context,
|
|
2360
|
+
work_dir,
|
|
2361
|
+
stage=stage,
|
|
2362
|
+
service=service,
|
|
2363
|
+
update=update,
|
|
2364
|
+
)
|
|
2365
|
+
if reconciliation is not None:
|
|
2366
|
+
raise ValueError("historical AKS reconciliation requires its retained exact review")
|
|
2367
|
+
if not (stage == "runtime" and update is None):
|
|
2368
|
+
claim_path = work_dir / f"{operation}-claim.json"
|
|
2369
|
+
receipt_path = work_dir / f"{operation}-receipt.json"
|
|
2370
|
+
if (claim_path.exists() or claim_path.is_symlink()) and not (
|
|
2371
|
+
receipt_path.exists() or receipt_path.is_symlink()
|
|
2372
|
+
):
|
|
2373
|
+
raise ValueError("claimed standalone apply requires verification-only recovery")
|
|
2374
|
+
_managed_identity_login_from_context(context, work_dir)
|
|
2375
|
+
infra, variables = _stage_paths(stage, context, work_dir)
|
|
2376
|
+
_activate_terraform_stage(stage, context, work_dir)
|
|
2377
|
+
if stage == "runtime" and update is None:
|
|
2378
|
+
operation = _runtime_operation(work_dir, infra)
|
|
2379
|
+
claim_path = work_dir / f"{operation}-claim.json"
|
|
2380
|
+
receipt_path = work_dir / f"{operation}-receipt.json"
|
|
2381
|
+
if (claim_path.exists() or claim_path.is_symlink()) and not (
|
|
2382
|
+
receipt_path.exists() or receipt_path.is_symlink()
|
|
2383
|
+
):
|
|
2384
|
+
raise ValueError("claimed standalone apply requires verification-only recovery")
|
|
2385
|
+
plan_path = work_dir / f"{operation}.tfplan"
|
|
2386
|
+
plan_path.unlink(missing_ok=True)
|
|
2387
|
+
command = [
|
|
2388
|
+
"terraform",
|
|
2389
|
+
"plan",
|
|
2390
|
+
"-input=false",
|
|
2391
|
+
"-no-color",
|
|
2392
|
+
f"-var-file={variables}",
|
|
2393
|
+
f"-out={plan_path}",
|
|
2394
|
+
]
|
|
2395
|
+
command.extend(f"-target={target}" for target in _operation_targets(stage, context, operation))
|
|
2396
|
+
if update is not None:
|
|
2397
|
+
command.append(f"-target={_service_update_target(str(update['service']))}")
|
|
2398
|
+
if stage == "runtime":
|
|
2399
|
+
_guard_existing_runtime_node_pools(context, work_dir)
|
|
2400
|
+
_run(command, cwd=infra, timeout=3600, reason=f"{stage} Terraform plan failed")
|
|
2401
|
+
_seal_terraform_plan(plan_path)
|
|
2402
|
+
show = _capture(
|
|
2403
|
+
("terraform", "show", "-json", str(plan_path)),
|
|
2404
|
+
cwd=infra,
|
|
2405
|
+
timeout=300,
|
|
2406
|
+
reason="Terraform plan projection failed",
|
|
2407
|
+
)
|
|
2408
|
+
value = json.loads(show)
|
|
2409
|
+
summary = _plan_summary(value)
|
|
2410
|
+
if update is not None:
|
|
2411
|
+
service_recovery.validate_service_update_plan(summary, work_dir, context, update)
|
|
2412
|
+
digest = _file_digest(plan_path)
|
|
2413
|
+
review: dict[str, object] = {
|
|
2414
|
+
"schema_version": "fdai.standalone-application-plan.v1",
|
|
2415
|
+
"stage": stage,
|
|
2416
|
+
"plan_digest": digest,
|
|
2417
|
+
"target_binding": context["target_binding"],
|
|
2418
|
+
"source_commit": context["source_commit"],
|
|
2419
|
+
"runtime_profile_digest": _runtime_profile_digest(context),
|
|
2420
|
+
"runtime_platform": _mapping(context.get("runtime_profile"), "runtime deployment profile")[
|
|
2421
|
+
"runtime_platform"
|
|
2422
|
+
],
|
|
2423
|
+
"operation": operation,
|
|
2424
|
+
"summary": summary,
|
|
2425
|
+
"expires_at": _moment(datetime.now(UTC) + timedelta(hours=1)),
|
|
2426
|
+
"mutation_performed": False,
|
|
2427
|
+
"subscription_ready": False,
|
|
2428
|
+
}
|
|
2429
|
+
if update is not None:
|
|
2430
|
+
review["service_update"] = _service_update_review(update)
|
|
2431
|
+
review["review_digest"] = canonical_digest(review)
|
|
2432
|
+
_replace_private_json(work_dir / f"{operation}-review.json", review)
|
|
2433
|
+
return review
|
|
2434
|
+
|
|
2435
|
+
|
|
2436
|
+
def _apply(args: argparse.Namespace, work_dir: Path) -> dict[str, object]:
|
|
2437
|
+
stage = str(args.stage)
|
|
2438
|
+
context = _private_json(work_dir / "context.json", "standalone host context")
|
|
2439
|
+
service = getattr(args, "service", None)
|
|
2440
|
+
update = _service_update_record(context, work_dir, stage=stage, service=service)
|
|
2441
|
+
operation, reconciliation = _application_operation(
|
|
2442
|
+
context,
|
|
2443
|
+
work_dir,
|
|
2444
|
+
stage=stage,
|
|
2445
|
+
service=service,
|
|
2446
|
+
update=update,
|
|
2447
|
+
)
|
|
2448
|
+
if stage == "runtime" and update is None:
|
|
2449
|
+
operation = (
|
|
2450
|
+
"runtime-cluster"
|
|
2451
|
+
if (work_dir / "runtime-cluster-review.json").exists()
|
|
2452
|
+
and not (work_dir / "runtime-cluster-receipt.json").exists()
|
|
2453
|
+
else "runtime"
|
|
2454
|
+
)
|
|
2455
|
+
review = _private_json(work_dir / f"{operation}-review.json", "standalone plan review")
|
|
2456
|
+
if update is not None and review.get("service_update") != _service_update_review(update):
|
|
2457
|
+
raise ValueError("AKS service update review differs from the prepared operation")
|
|
2458
|
+
approval = _private_json(_absolute(args.approval), "standalone plan approval")
|
|
2459
|
+
_validate_approval(review, approval, context=context)
|
|
2460
|
+
_managed_identity_login_from_context(context, work_dir)
|
|
2461
|
+
infra, _variables = _stage_paths(stage, context, work_dir)
|
|
2462
|
+
_activate_terraform_stage(stage, context, work_dir)
|
|
2463
|
+
plan_path = work_dir / f"{operation}.tfplan"
|
|
2464
|
+
if _file_digest(plan_path) != review["plan_digest"]:
|
|
2465
|
+
raise ValueError("standalone application plan changed before apply")
|
|
2466
|
+
if stage == "runtime":
|
|
2467
|
+
_guard_existing_runtime_node_pools(context, work_dir)
|
|
2468
|
+
if reconciliation is not None:
|
|
2469
|
+
_validate_historical_reconciliation_plan(
|
|
2470
|
+
context,
|
|
2471
|
+
work_dir,
|
|
2472
|
+
reconciliation,
|
|
2473
|
+
plan_path,
|
|
2474
|
+
)
|
|
2475
|
+
claim_path = work_dir / f"{operation}-claim.json"
|
|
2476
|
+
receipt_path = work_dir / f"{operation}-receipt.json"
|
|
2477
|
+
if receipt_path.exists():
|
|
2478
|
+
if reconciliation is not None:
|
|
2479
|
+
return _historical_reconciliation_receipt(work_dir, context, reconciliation)
|
|
2480
|
+
return _private_json(receipt_path, "standalone apply receipt")
|
|
2481
|
+
claim = {
|
|
2482
|
+
"schema_version": "fdai.standalone-application-claim.v1",
|
|
2483
|
+
"stage": stage,
|
|
2484
|
+
"operation": operation,
|
|
2485
|
+
"plan_digest": review["plan_digest"],
|
|
2486
|
+
"approval_digest": canonical_digest(approval),
|
|
2487
|
+
"idempotency_key": canonical_digest(
|
|
2488
|
+
{"target_binding": context["target_binding"], "plan_digest": review["plan_digest"]}
|
|
2489
|
+
),
|
|
2490
|
+
"claimed_at": _moment(datetime.now(UTC)),
|
|
2491
|
+
"mutation_performed": False,
|
|
2492
|
+
}
|
|
2493
|
+
try:
|
|
2494
|
+
write_private_output(
|
|
2495
|
+
claim_path,
|
|
2496
|
+
json.dumps(claim, sort_keys=True, separators=(",", ":")) + "\n",
|
|
2497
|
+
)
|
|
2498
|
+
except FileExistsError as exc:
|
|
2499
|
+
raise ValueError(
|
|
2500
|
+
"standalone apply claim already exists; automatic retry is blocked"
|
|
2501
|
+
) from exc
|
|
2502
|
+
_run(
|
|
2503
|
+
("terraform", "apply", "-input=false", "-no-color", str(plan_path)),
|
|
2504
|
+
cwd=infra,
|
|
2505
|
+
timeout=7200,
|
|
2506
|
+
reason=f"{stage} exact apply failed; verification-only recovery is required",
|
|
2507
|
+
)
|
|
2508
|
+
effect_verified = (
|
|
2509
|
+
_readback_aks_service_update(service_recovery.readback_context(work_dir, context), update)
|
|
2510
|
+
if update is not None
|
|
2511
|
+
else _readback_stage(stage, context)
|
|
2512
|
+
)
|
|
2513
|
+
if not effect_verified:
|
|
2514
|
+
raise ValueError(f"{stage} apply effect readback is incomplete")
|
|
2515
|
+
if update is not None and not _service_update_zero_change(context, work_dir, update):
|
|
2516
|
+
raise ValueError("AKS service update did not converge to a zero-change targeted plan")
|
|
2517
|
+
reconciliation_evidence = (
|
|
2518
|
+
_historical_reconciliation_postconditions(context, work_dir, reconciliation)
|
|
2519
|
+
if reconciliation is not None
|
|
2520
|
+
else None
|
|
2521
|
+
)
|
|
2522
|
+
recovery_receipt: dict[str, object] = {
|
|
2523
|
+
"schema_version": "fdai.standalone-application-apply-receipt.v1",
|
|
2524
|
+
"state": "applied",
|
|
2525
|
+
"stage": stage,
|
|
2526
|
+
"operation": operation,
|
|
2527
|
+
"plan_digest": review["plan_digest"],
|
|
2528
|
+
"runtime_profile_digest": _runtime_profile_digest(context),
|
|
2529
|
+
"claim_digest": canonical_digest(claim),
|
|
2530
|
+
"control_plane_readback_verified": True,
|
|
2531
|
+
"mutation_performed": True,
|
|
2532
|
+
"subscription_ready": False,
|
|
2533
|
+
}
|
|
2534
|
+
if update is not None:
|
|
2535
|
+
recovery_receipt["service_update"] = _service_update_review(update)
|
|
2536
|
+
recovery_receipt["peer_state_unchanged_verified"] = True
|
|
2537
|
+
recovery_receipt["terraform_zero_change_verified"] = True
|
|
2538
|
+
if reconciliation is not None and reconciliation_evidence is not None:
|
|
2539
|
+
recovery_receipt.update(
|
|
2540
|
+
{
|
|
2541
|
+
"schema_version": "fdai.historical-aks-reconciliation-receipt.v1",
|
|
2542
|
+
"operation": operation,
|
|
2543
|
+
"effect_verified": True,
|
|
2544
|
+
"terraform_zero_change_verified": True,
|
|
2545
|
+
"mutations": reconciliation["mutations"],
|
|
2546
|
+
**reconciliation_evidence,
|
|
2547
|
+
}
|
|
2548
|
+
)
|
|
2549
|
+
recovery_receipt["receipt_digest"] = canonical_digest(recovery_receipt)
|
|
2550
|
+
_replace_private_json(receipt_path, recovery_receipt)
|
|
2551
|
+
return recovery_receipt
|
|
2552
|
+
|
|
2553
|
+
|
|
2554
|
+
def _apply_residual(args: argparse.Namespace, work_dir: Path) -> dict[str, object]:
|
|
2555
|
+
return apply_residual_checkpoint(
|
|
2556
|
+
args,
|
|
2557
|
+
work_dir,
|
|
2558
|
+
private_json=_private_json,
|
|
2559
|
+
validate_approval=_validate_approval,
|
|
2560
|
+
managed_identity_login=_managed_identity_login_from_context,
|
|
2561
|
+
stage_paths=_stage_paths,
|
|
2562
|
+
activate_stage=_activate_terraform_stage,
|
|
2563
|
+
stage_targets=_stage_targets,
|
|
2564
|
+
guard_runtime=_guard_existing_runtime_node_pools,
|
|
2565
|
+
readback_stage=_readback_stage,
|
|
2566
|
+
absolute_path=_absolute,
|
|
2567
|
+
)
|
|
2568
|
+
|
|
2569
|
+
|
|
2570
|
+
def _apply_historical_aks_reconciliation(
|
|
2571
|
+
args: argparse.Namespace, work_dir: Path
|
|
2572
|
+
) -> dict[str, object]:
|
|
2573
|
+
context = _private_json(work_dir / "context.json", "standalone host context")
|
|
2574
|
+
if context.get("active_historical_reconciliation") is None:
|
|
2575
|
+
raise ValueError("no active historical AKS reconciliation is pending")
|
|
2576
|
+
return _apply(
|
|
2577
|
+
argparse.Namespace(stage="application", service=None, approval=args.approval),
|
|
2578
|
+
work_dir,
|
|
2579
|
+
)
|
|
2580
|
+
|
|
2581
|
+
|
|
2582
|
+
def _recover_apply(args: argparse.Namespace, work_dir: Path) -> dict[str, object]:
|
|
2583
|
+
"""Verify an ambiguous prior apply without repeating its mutation."""
|
|
2584
|
+
stage = str(args.stage)
|
|
2585
|
+
context = _private_json(work_dir / "context.json", "standalone host context")
|
|
2586
|
+
service = getattr(args, "service", None)
|
|
2587
|
+
update = _service_update_record(context, work_dir, stage=stage, service=service)
|
|
2588
|
+
operation, reconciliation = _application_operation(
|
|
2589
|
+
context,
|
|
2590
|
+
work_dir,
|
|
2591
|
+
stage=stage,
|
|
2592
|
+
service=service,
|
|
2593
|
+
update=update,
|
|
2594
|
+
)
|
|
2595
|
+
if (
|
|
2596
|
+
stage == "runtime"
|
|
2597
|
+
and update is None
|
|
2598
|
+
and (work_dir / "runtime-cluster-claim.json").exists()
|
|
2599
|
+
and not (work_dir / "runtime-cluster-receipt.json").exists()
|
|
2600
|
+
):
|
|
2601
|
+
operation = "runtime-cluster"
|
|
2602
|
+
receipt_path = work_dir / f"{operation}-receipt.json"
|
|
2603
|
+
successor_pending = (
|
|
2604
|
+
stage == "application"
|
|
2605
|
+
and update is None
|
|
2606
|
+
and _application_successor_prerequisite(work_dir, stage)
|
|
2607
|
+
)
|
|
2608
|
+
if receipt_path.exists() and not successor_pending:
|
|
2609
|
+
if reconciliation is not None:
|
|
2610
|
+
return _historical_reconciliation_receipt(work_dir, context, reconciliation)
|
|
2611
|
+
receipt = _private_json(receipt_path, "standalone apply receipt")
|
|
2612
|
+
if receipt.get("operation") == "runtime-cluster":
|
|
2613
|
+
return _not_required_recovery(stage)
|
|
2614
|
+
return receipt
|
|
2615
|
+
claim_path = work_dir / f"{operation}-claim.json"
|
|
2616
|
+
if not claim_path.exists():
|
|
2617
|
+
return _not_required_recovery(stage)
|
|
2618
|
+
review = _private_json(work_dir / f"{operation}-review.json", "standalone plan review")
|
|
2619
|
+
claim = _private_json(claim_path, "standalone apply claim")
|
|
2620
|
+
if (
|
|
2621
|
+
claim.get("schema_version") != "fdai.standalone-application-claim.v1"
|
|
2622
|
+
or claim.get("stage") != stage
|
|
2623
|
+
or claim.get("plan_digest") != review.get("plan_digest")
|
|
2624
|
+
or claim.get("idempotency_key")
|
|
2625
|
+
!= canonical_digest(
|
|
2626
|
+
{
|
|
2627
|
+
"target_binding": context["target_binding"],
|
|
2628
|
+
"plan_digest": review["plan_digest"],
|
|
2629
|
+
}
|
|
2630
|
+
)
|
|
2631
|
+
):
|
|
2632
|
+
raise ValueError("standalone apply recovery claim is invalid")
|
|
2633
|
+
_managed_identity_login_from_context(context, work_dir)
|
|
2634
|
+
infra, variables = _stage_paths(stage, context, work_dir)
|
|
2635
|
+
_activate_terraform_stage(stage, context, work_dir)
|
|
2636
|
+
if stage == "runtime" and update is None:
|
|
2637
|
+
operation = _runtime_operation(work_dir, infra)
|
|
2638
|
+
receipt_path = work_dir / f"{operation}-receipt.json"
|
|
2639
|
+
if receipt_path.exists():
|
|
2640
|
+
retained_receipt = _private_json(receipt_path, "standalone apply receipt")
|
|
2641
|
+
if retained_receipt.get("operation") == "runtime-cluster":
|
|
2642
|
+
return _not_required_recovery(stage)
|
|
2643
|
+
return retained_receipt
|
|
2644
|
+
targets = _operation_targets(stage, context, operation)
|
|
2645
|
+
if stage == "runtime":
|
|
2646
|
+
_guard_existing_runtime_node_pools(context, work_dir)
|
|
2647
|
+
if reconciliation is not None:
|
|
2648
|
+
plan_path = work_dir / f"{operation}.tfplan"
|
|
2649
|
+
if _file_digest(plan_path) != review.get("plan_digest"):
|
|
2650
|
+
raise ValueError("historical AKS reconciliation plan changed before recovery")
|
|
2651
|
+
_validate_historical_reconciliation_plan(
|
|
2652
|
+
context,
|
|
2653
|
+
work_dir,
|
|
2654
|
+
reconciliation,
|
|
2655
|
+
plan_path,
|
|
2656
|
+
)
|
|
2657
|
+
elif update is None:
|
|
2658
|
+
if (
|
|
2659
|
+
stage == "substrate"
|
|
2660
|
+
and context.get("key_vault_private_access") is True
|
|
2661
|
+
and context.get("document_storage_private_access") is True
|
|
2662
|
+
):
|
|
2663
|
+
wait_for_private_data_planes(context=context, work_dir=work_dir)
|
|
2664
|
+
if _residual_claim_exists(work_dir, stage):
|
|
2665
|
+
return _recover_residual_apply(
|
|
2666
|
+
work_dir=work_dir,
|
|
2667
|
+
stage=stage,
|
|
2668
|
+
context=context,
|
|
2669
|
+
infra=infra,
|
|
2670
|
+
variables=variables,
|
|
2671
|
+
targets=targets,
|
|
2672
|
+
original_review=review,
|
|
2673
|
+
original_claim=claim,
|
|
2674
|
+
effect_readback=lambda: _readback_stage(stage, context),
|
|
2675
|
+
successor_prerequisite_verified=(
|
|
2676
|
+
_residual_successor_prerequisite(work_dir, stage)
|
|
2677
|
+
or _application_successor_prerequisite(work_dir, stage)
|
|
2678
|
+
),
|
|
2679
|
+
)
|
|
2680
|
+
residual_review = _prepare_residual_review(
|
|
2681
|
+
work_dir=work_dir,
|
|
2682
|
+
stage=stage,
|
|
2683
|
+
context=context,
|
|
2684
|
+
infra=infra,
|
|
2685
|
+
variables=variables,
|
|
2686
|
+
targets=targets,
|
|
2687
|
+
original_review=review,
|
|
2688
|
+
original_claim=claim,
|
|
2689
|
+
)
|
|
2690
|
+
if residual_review is not None:
|
|
2691
|
+
return residual_review
|
|
2692
|
+
else:
|
|
2693
|
+
command = [
|
|
2694
|
+
"terraform",
|
|
2695
|
+
"plan",
|
|
2696
|
+
"-detailed-exitcode",
|
|
2697
|
+
"-input=false",
|
|
2698
|
+
"-no-color",
|
|
2699
|
+
f"-var-file={variables}",
|
|
2700
|
+
]
|
|
2701
|
+
if update is not None:
|
|
2702
|
+
command.append(f"-target={_service_update_target(str(update['service']))}")
|
|
2703
|
+
completed = subprocess.run(
|
|
2704
|
+
command,
|
|
2705
|
+
cwd=infra,
|
|
2706
|
+
check=False,
|
|
2707
|
+
capture_output=True,
|
|
2708
|
+
timeout=1800,
|
|
2709
|
+
)
|
|
2710
|
+
if completed.returncode != 0:
|
|
2711
|
+
raise ValueError("standalone apply effect is not recoverably converged")
|
|
2712
|
+
effect_verified = (
|
|
2713
|
+
_readback_aks_service_update(service_recovery.readback_context(work_dir, context), update)
|
|
2714
|
+
if update is not None
|
|
2715
|
+
else _readback_stage(stage, context)
|
|
2716
|
+
)
|
|
2717
|
+
if not effect_verified:
|
|
2718
|
+
raise ValueError("standalone apply effect readback is incomplete")
|
|
2719
|
+
reconciliation_evidence = (
|
|
2720
|
+
_historical_reconciliation_postconditions(context, work_dir, reconciliation)
|
|
2721
|
+
if reconciliation is not None
|
|
2722
|
+
else None
|
|
2723
|
+
)
|
|
2724
|
+
recovery_receipt: dict[str, object] = {
|
|
2725
|
+
"schema_version": "fdai.standalone-application-apply-receipt.v1",
|
|
2726
|
+
"state": "applied",
|
|
2727
|
+
"stage": stage,
|
|
2728
|
+
"operation": operation,
|
|
2729
|
+
"plan_digest": review["plan_digest"],
|
|
2730
|
+
"runtime_profile_digest": _runtime_profile_digest(context),
|
|
2731
|
+
"claim_digest": canonical_digest(claim),
|
|
2732
|
+
"control_plane_readback_verified": True,
|
|
2733
|
+
"verification_only_recovery": True,
|
|
2734
|
+
"mutation_performed": True,
|
|
2735
|
+
"subscription_ready": False,
|
|
2736
|
+
}
|
|
2737
|
+
if update is not None:
|
|
2738
|
+
recovery_receipt["service_update"] = _service_update_review(update)
|
|
2739
|
+
recovery_receipt["peer_state_unchanged_verified"] = True
|
|
2740
|
+
recovery_receipt["terraform_zero_change_verified"] = True
|
|
2741
|
+
if reconciliation is not None and reconciliation_evidence is not None:
|
|
2742
|
+
recovery_receipt.update(
|
|
2743
|
+
{
|
|
2744
|
+
"schema_version": "fdai.historical-aks-reconciliation-receipt.v1",
|
|
2745
|
+
"operation": operation,
|
|
2746
|
+
"effect_verified": True,
|
|
2747
|
+
"terraform_zero_change_verified": True,
|
|
2748
|
+
"mutations": reconciliation["mutations"],
|
|
2749
|
+
**reconciliation_evidence,
|
|
2750
|
+
}
|
|
2751
|
+
)
|
|
2752
|
+
recovery_receipt["receipt_digest"] = canonical_digest(recovery_receipt)
|
|
2753
|
+
_replace_private_json(receipt_path, recovery_receipt)
|
|
2754
|
+
return recovery_receipt
|
|
2755
|
+
|
|
2756
|
+
|
|
2757
|
+
def _recover_historical_aks_reconciliation(
|
|
2758
|
+
_args: argparse.Namespace, work_dir: Path
|
|
2759
|
+
) -> dict[str, object]:
|
|
2760
|
+
context = _private_json(work_dir / "context.json", "standalone host context")
|
|
2761
|
+
if context.get("active_historical_reconciliation") is None:
|
|
2762
|
+
raise ValueError("no active historical AKS reconciliation is pending")
|
|
2763
|
+
return _recover_apply(
|
|
2764
|
+
argparse.Namespace(stage="application", service=None),
|
|
2765
|
+
work_dir,
|
|
2766
|
+
)
|
|
2767
|
+
|
|
2768
|
+
|
|
2769
|
+
def _import_images(_args: argparse.Namespace, work_dir: Path) -> dict[str, object]:
|
|
2770
|
+
context = _private_json(work_dir / "context.json", "standalone host context")
|
|
2771
|
+
_managed_identity_login_from_context(context, work_dir)
|
|
2772
|
+
if not (work_dir / "substrate-receipt.json").exists():
|
|
2773
|
+
raise ValueError("image import requires the applied substrate plan")
|
|
2774
|
+
if context.get("artifact_source") in {"operator-selected-source", "hub-signed-images"}:
|
|
2775
|
+
return source_application_inputs.write_source_image_import_receipt(
|
|
2776
|
+
context=context, work_dir=work_dir, write_json=_replace_private_json
|
|
2777
|
+
)
|
|
2778
|
+
token = _capture(
|
|
2779
|
+
(
|
|
2780
|
+
"az",
|
|
2781
|
+
"acr",
|
|
2782
|
+
"login",
|
|
2783
|
+
"--name",
|
|
2784
|
+
str(context["registry_name"]),
|
|
2785
|
+
"--subscription",
|
|
2786
|
+
str(context["subscription_id"]),
|
|
2787
|
+
"--expose-token",
|
|
2788
|
+
"--query",
|
|
2789
|
+
"accessToken",
|
|
2790
|
+
"--output",
|
|
2791
|
+
"tsv",
|
|
2792
|
+
"--only-show-errors",
|
|
2793
|
+
),
|
|
2794
|
+
cwd=work_dir,
|
|
2795
|
+
timeout=120,
|
|
2796
|
+
reason="ACR token acquisition failed",
|
|
2797
|
+
).strip()
|
|
2798
|
+
if not token or any(character in token for character in "\r\n"):
|
|
2799
|
+
raise ValueError("ACR token response is invalid")
|
|
2800
|
+
registry_config = work_dir / "oras-auth.json"
|
|
2801
|
+
registry_config.unlink(missing_ok=True)
|
|
2802
|
+
login = subprocess.run(
|
|
2803
|
+
(
|
|
2804
|
+
"oras",
|
|
2805
|
+
"login",
|
|
2806
|
+
"--registry-config",
|
|
2807
|
+
str(registry_config),
|
|
2808
|
+
str(context["registry_login_server"]),
|
|
2809
|
+
"--username",
|
|
2810
|
+
"00000000-0000-0000-0000-000000000000",
|
|
2811
|
+
"--password-stdin",
|
|
2812
|
+
),
|
|
2813
|
+
input=token,
|
|
2814
|
+
check=False,
|
|
2815
|
+
capture_output=True,
|
|
2816
|
+
text=True,
|
|
2817
|
+
timeout=60,
|
|
2818
|
+
)
|
|
2819
|
+
token = ""
|
|
2820
|
+
if login.returncode != 0:
|
|
2821
|
+
registry_config.unlink(missing_ok=True)
|
|
2822
|
+
raise ValueError("ACR login failed")
|
|
2823
|
+
kit_root = work_dir / "kit-work/verified"
|
|
2824
|
+
runtime = _private_json(kit_root / "runtime/release.json", "runtime release")
|
|
2825
|
+
records = {
|
|
2826
|
+
**_mapping(runtime.get("services"), "runtime services"),
|
|
2827
|
+
**_mapping(runtime.get("sidecars"), "runtime sidecars"),
|
|
2828
|
+
}
|
|
2829
|
+
imported: dict[str, str] = {}
|
|
2830
|
+
image_root = work_dir / "oci"
|
|
2831
|
+
image_root.mkdir(mode=0o700, exist_ok=True)
|
|
2832
|
+
try:
|
|
2833
|
+
for name, record_raw in sorted(records.items()):
|
|
2834
|
+
record = _mapping(record_raw, f"runtime image {name}")
|
|
2835
|
+
archive = kit_root / str(record["archive"])
|
|
2836
|
+
layout = image_root / name
|
|
2837
|
+
if not layout.exists():
|
|
2838
|
+
layout.mkdir(mode=0o700)
|
|
2839
|
+
_run(
|
|
2840
|
+
("tar", "-xf", str(archive), "-C", str(layout), "--no-same-owner"),
|
|
2841
|
+
cwd=work_dir,
|
|
2842
|
+
timeout=300,
|
|
2843
|
+
reason="OCI archive extraction failed",
|
|
2844
|
+
)
|
|
2845
|
+
digest = str(record["image_digest"])
|
|
2846
|
+
destination = (
|
|
2847
|
+
f"{context['registry_login_server']}/{name}:sha-{context['source_commit']}"
|
|
2848
|
+
)
|
|
2849
|
+
_run(
|
|
2850
|
+
(
|
|
2851
|
+
"oras",
|
|
2852
|
+
"cp",
|
|
2853
|
+
"--to-registry-config",
|
|
2854
|
+
str(registry_config),
|
|
2855
|
+
"--from-oci-layout",
|
|
2856
|
+
f"{layout}@{digest}",
|
|
2857
|
+
destination,
|
|
2858
|
+
),
|
|
2859
|
+
cwd=work_dir,
|
|
2860
|
+
timeout=1800,
|
|
2861
|
+
reason="verified OCI image import failed",
|
|
2862
|
+
)
|
|
2863
|
+
observed = _capture(
|
|
2864
|
+
(
|
|
2865
|
+
"az",
|
|
2866
|
+
"acr",
|
|
2867
|
+
"manifest",
|
|
2868
|
+
"show-metadata",
|
|
2869
|
+
"--registry",
|
|
2870
|
+
str(context["registry_name"]),
|
|
2871
|
+
"--name",
|
|
2872
|
+
f"{name}@{digest}",
|
|
2873
|
+
"--query",
|
|
2874
|
+
"digest",
|
|
2875
|
+
"--output",
|
|
2876
|
+
"tsv",
|
|
2877
|
+
"--only-show-errors",
|
|
2878
|
+
),
|
|
2879
|
+
cwd=work_dir,
|
|
2880
|
+
timeout=120,
|
|
2881
|
+
reason="imported OCI image readback failed",
|
|
2882
|
+
).strip()
|
|
2883
|
+
if observed != digest:
|
|
2884
|
+
raise ValueError("imported OCI image digest differs from the signed release")
|
|
2885
|
+
imported[name] = f"{context['registry_login_server']}/{name}@{digest}"
|
|
2886
|
+
finally:
|
|
2887
|
+
registry_config.unlink(missing_ok=True)
|
|
2888
|
+
result: dict[str, object] = {
|
|
2889
|
+
"schema_version": "fdai.standalone-image-import-receipt.v1",
|
|
2890
|
+
"state": "imported",
|
|
2891
|
+
"image_digests": {name: value.rsplit("@", 1)[1] for name, value in imported.items()},
|
|
2892
|
+
"effect_verified": True,
|
|
2893
|
+
"mutation_performed": True,
|
|
2894
|
+
"subscription_ready": False,
|
|
2895
|
+
}
|
|
2896
|
+
result["receipt_digest"] = canonical_digest(result)
|
|
2897
|
+
_replace_private_json(work_dir / "image-import-receipt.json", result)
|
|
2898
|
+
return result
|
|
2899
|
+
|
|
2900
|
+
|
|
2901
|
+
def _import_source_service_image(args: argparse.Namespace, work_dir: Path) -> dict[str, object]:
|
|
2902
|
+
"""Import one source-built OCI archive through the retained deployment identity."""
|
|
2903
|
+
|
|
2904
|
+
context = _private_json(work_dir / "context.json", "standalone host context")
|
|
2905
|
+
if _runtime_platform(context) != "aks":
|
|
2906
|
+
raise ValueError("source service image import requires an AKS installation")
|
|
2907
|
+
_require_source_service_update_dev_environment(work_dir)
|
|
2908
|
+
if not (work_dir / "substrate-receipt.json").is_file():
|
|
2909
|
+
_require_aks_application_baseline(work_dir, context)
|
|
2910
|
+
service = str(args.service)
|
|
2911
|
+
source_commit = str(args.source_commit)
|
|
2912
|
+
archive_digest = str(args.archive_sha256)
|
|
2913
|
+
image_digest = str(args.image_digest)
|
|
2914
|
+
if (
|
|
2915
|
+
service not in AKS_SERVICES
|
|
2916
|
+
or _SOURCE_COMMIT.fullmatch(source_commit) is None
|
|
2917
|
+
or _DIGEST.fullmatch(archive_digest) is None
|
|
2918
|
+
or re.fullmatch(r"sha256:[0-9a-f]{64}", image_digest) is None
|
|
2919
|
+
):
|
|
2920
|
+
raise ValueError("source service image identity is invalid")
|
|
2921
|
+
archive = _absolute(args.archive)
|
|
2922
|
+
verified = validate_oci_archive(
|
|
2923
|
+
archive,
|
|
2924
|
+
expected_archive_sha256=archive_digest,
|
|
2925
|
+
expected_manifest_digest=image_digest,
|
|
2926
|
+
expected_source_commit=source_commit,
|
|
2927
|
+
expected_platform_tag="linux-x86_64",
|
|
2928
|
+
)
|
|
2929
|
+
target_binding = context.get("target_binding")
|
|
2930
|
+
if not isinstance(target_binding, str) or _DIGEST.fullmatch(target_binding) is None:
|
|
2931
|
+
raise ValueError("source service image target binding is invalid")
|
|
2932
|
+
claim_path, receipt_path = service_recovery.source_image_import_paths(
|
|
2933
|
+
work_dir, service, source_commit, image_digest
|
|
2934
|
+
)
|
|
2935
|
+
verification_only = claim_path.exists() or claim_path.is_symlink()
|
|
2936
|
+
approval = _private_json(_absolute(args.approval), "source service image import approval")
|
|
2937
|
+
approval_digest = _validate_source_image_import_approval(
|
|
2938
|
+
approval,
|
|
2939
|
+
service=service,
|
|
2940
|
+
source_commit=source_commit,
|
|
2941
|
+
archive_digest=verified.archive_sha256,
|
|
2942
|
+
image_digest=verified.manifest.digest,
|
|
2943
|
+
target_binding=target_binding,
|
|
2944
|
+
allow_expired=verification_only,
|
|
2945
|
+
)
|
|
2946
|
+
claim: dict[str, object] = {
|
|
2947
|
+
"schema_version": "fdai.source-service-image-import-claim.v1",
|
|
2948
|
+
"service": service,
|
|
2949
|
+
"source_commit": source_commit,
|
|
2950
|
+
"archive_sha256": verified.archive_sha256,
|
|
2951
|
+
"image_digest": verified.manifest.digest,
|
|
2952
|
+
"target_binding": target_binding,
|
|
2953
|
+
"approval_digest": approval_digest,
|
|
2954
|
+
"mutation_performed": False,
|
|
2955
|
+
}
|
|
2956
|
+
if verification_only:
|
|
2957
|
+
retained_claim = _private_json(claim_path, "source service image import claim")
|
|
2958
|
+
if retained_claim != claim:
|
|
2959
|
+
raise ValueError("source service image import claim differs")
|
|
2960
|
+
elif receipt_path.exists() or receipt_path.is_symlink():
|
|
2961
|
+
raise ValueError("source service image import receipt exists without its claim")
|
|
2962
|
+
|
|
2963
|
+
_managed_identity_login_from_context(context, work_dir)
|
|
2964
|
+
token = _capture(
|
|
2965
|
+
(
|
|
2966
|
+
"az",
|
|
2967
|
+
"acr",
|
|
2968
|
+
"login",
|
|
2969
|
+
"--name",
|
|
2970
|
+
str(context["registry_name"]),
|
|
2971
|
+
"--subscription",
|
|
2972
|
+
str(context["subscription_id"]),
|
|
2973
|
+
"--expose-token",
|
|
2974
|
+
"--query",
|
|
2975
|
+
"accessToken",
|
|
2976
|
+
"--output",
|
|
2977
|
+
"tsv",
|
|
2978
|
+
"--only-show-errors",
|
|
2979
|
+
),
|
|
2980
|
+
cwd=work_dir,
|
|
2981
|
+
timeout=120,
|
|
2982
|
+
reason="ACR token acquisition failed",
|
|
2983
|
+
).strip()
|
|
2984
|
+
if not token or any(character in token for character in "\r\n"):
|
|
2985
|
+
raise ValueError("ACR token response is invalid")
|
|
2986
|
+
registry_config = work_dir / f"oras-auth-{service}.json"
|
|
2987
|
+
registry_config.unlink(missing_ok=True)
|
|
2988
|
+
login = subprocess.run(
|
|
2989
|
+
(
|
|
2990
|
+
"oras",
|
|
2991
|
+
"login",
|
|
2992
|
+
"--registry-config",
|
|
2993
|
+
str(registry_config),
|
|
2994
|
+
str(context["registry_login_server"]),
|
|
2995
|
+
"--username",
|
|
2996
|
+
"00000000-0000-0000-0000-000000000000",
|
|
2997
|
+
"--password-stdin",
|
|
2998
|
+
),
|
|
2999
|
+
input=token,
|
|
3000
|
+
check=False,
|
|
3001
|
+
capture_output=True,
|
|
3002
|
+
text=True,
|
|
3003
|
+
timeout=60,
|
|
3004
|
+
)
|
|
3005
|
+
token = ""
|
|
3006
|
+
if login.returncode != 0:
|
|
3007
|
+
registry_config.unlink(missing_ok=True)
|
|
3008
|
+
raise ValueError("ACR login failed")
|
|
3009
|
+
try:
|
|
3010
|
+
if not verification_only:
|
|
3011
|
+
write_private_output(
|
|
3012
|
+
claim_path,
|
|
3013
|
+
json.dumps(claim, sort_keys=True, separators=(",", ":")) + "\n",
|
|
3014
|
+
)
|
|
3015
|
+
layout = work_dir / "source-oci" / f"{service}-{image_digest[7:19]}"
|
|
3016
|
+
layout.mkdir(mode=0o700, parents=True)
|
|
3017
|
+
_run(
|
|
3018
|
+
("tar", "-xf", str(archive), "-C", str(layout), "--no-same-owner"),
|
|
3019
|
+
cwd=work_dir,
|
|
3020
|
+
timeout=300,
|
|
3021
|
+
reason="source OCI archive extraction failed",
|
|
3022
|
+
)
|
|
3023
|
+
_run(
|
|
3024
|
+
(
|
|
3025
|
+
"oras",
|
|
3026
|
+
"cp",
|
|
3027
|
+
"--to-registry-config",
|
|
3028
|
+
str(registry_config),
|
|
3029
|
+
"--from-oci-layout",
|
|
3030
|
+
f"{layout}@{image_digest}",
|
|
3031
|
+
f"{context['registry_login_server']}/{service}:sha-{source_commit}",
|
|
3032
|
+
),
|
|
3033
|
+
cwd=work_dir,
|
|
3034
|
+
timeout=1800,
|
|
3035
|
+
reason=("source OCI image import failed; verification-only recovery is required"),
|
|
3036
|
+
)
|
|
3037
|
+
observed = _capture(
|
|
3038
|
+
(
|
|
3039
|
+
"az",
|
|
3040
|
+
"acr",
|
|
3041
|
+
"manifest",
|
|
3042
|
+
"show-metadata",
|
|
3043
|
+
"--registry",
|
|
3044
|
+
str(context["registry_name"]),
|
|
3045
|
+
"--name",
|
|
3046
|
+
f"{service}@{image_digest}",
|
|
3047
|
+
"--query",
|
|
3048
|
+
"digest",
|
|
3049
|
+
"--output",
|
|
3050
|
+
"tsv",
|
|
3051
|
+
"--only-show-errors",
|
|
3052
|
+
),
|
|
3053
|
+
cwd=work_dir,
|
|
3054
|
+
timeout=120,
|
|
3055
|
+
reason="source OCI image readback failed",
|
|
3056
|
+
).strip()
|
|
3057
|
+
if observed != image_digest:
|
|
3058
|
+
raise ValueError("source OCI image digest readback differs")
|
|
3059
|
+
finally:
|
|
3060
|
+
registry_config.unlink(missing_ok=True)
|
|
3061
|
+
receipt: dict[str, object] = {
|
|
3062
|
+
"schema_version": "fdai.source-service-image-import-receipt.v1",
|
|
3063
|
+
"state": "imported",
|
|
3064
|
+
"service": service,
|
|
3065
|
+
"source_commit": source_commit,
|
|
3066
|
+
"archive_sha256": archive_digest,
|
|
3067
|
+
"image_digest": image_digest,
|
|
3068
|
+
"image": f"{context['registry_login_server']}/{service}@{image_digest}",
|
|
3069
|
+
"claim_digest": canonical_digest(claim),
|
|
3070
|
+
"effect_verified": True,
|
|
3071
|
+
"mutation_performed": True,
|
|
3072
|
+
"subscription_ready": False,
|
|
3073
|
+
}
|
|
3074
|
+
receipt["receipt_digest"] = canonical_digest(receipt)
|
|
3075
|
+
if receipt_path.exists() or receipt_path.is_symlink():
|
|
3076
|
+
retained_receipt = _private_json(receipt_path, "source service image import receipt")
|
|
3077
|
+
if retained_receipt != receipt:
|
|
3078
|
+
raise ValueError("source service image import receipt differs")
|
|
3079
|
+
return retained_receipt
|
|
3080
|
+
else:
|
|
3081
|
+
_replace_private_json(receipt_path, receipt)
|
|
3082
|
+
return receipt
|
|
3083
|
+
|
|
3084
|
+
|
|
3085
|
+
def _validate_source_image_import_approval(
|
|
3086
|
+
approval: dict[str, object],
|
|
3087
|
+
*,
|
|
3088
|
+
service: str,
|
|
3089
|
+
source_commit: str,
|
|
3090
|
+
archive_digest: str,
|
|
3091
|
+
image_digest: str,
|
|
3092
|
+
target_binding: str,
|
|
3093
|
+
allow_expired: bool = False,
|
|
3094
|
+
) -> str:
|
|
3095
|
+
digest = approval.get("approval_digest")
|
|
3096
|
+
document = {key: value for key, value in approval.items() if key != "approval_digest"}
|
|
3097
|
+
if (
|
|
3098
|
+
set(approval)
|
|
3099
|
+
!= {
|
|
3100
|
+
"schema_version",
|
|
3101
|
+
"service",
|
|
3102
|
+
"source_commit",
|
|
3103
|
+
"archive_sha256",
|
|
3104
|
+
"image_digest",
|
|
3105
|
+
"target_binding",
|
|
3106
|
+
"actor_digest",
|
|
3107
|
+
"approved_at",
|
|
3108
|
+
"expires_at",
|
|
3109
|
+
"approval_digest",
|
|
3110
|
+
}
|
|
3111
|
+
or approval.get("schema_version") != "fdai.source-service-image-import-approval.v1"
|
|
3112
|
+
or approval.get("service") != service
|
|
3113
|
+
or approval.get("source_commit") != source_commit
|
|
3114
|
+
or approval.get("archive_sha256") != archive_digest
|
|
3115
|
+
or approval.get("image_digest") != image_digest
|
|
3116
|
+
or approval.get("target_binding") != target_binding
|
|
3117
|
+
or not isinstance(approval.get("actor_digest"), str)
|
|
3118
|
+
or _DIGEST.fullmatch(str(approval["actor_digest"])) is None
|
|
3119
|
+
or not isinstance(digest, str)
|
|
3120
|
+
or digest != canonical_digest(document)
|
|
3121
|
+
):
|
|
3122
|
+
raise ValueError("source service image import approval is invalid")
|
|
3123
|
+
try:
|
|
3124
|
+
approved_at = datetime.fromisoformat(str(approval["approved_at"]))
|
|
3125
|
+
expires_at = datetime.fromisoformat(str(approval["expires_at"]))
|
|
3126
|
+
except ValueError as exc:
|
|
3127
|
+
raise ValueError("source service image import approval is invalid") from exc
|
|
3128
|
+
now = datetime.now(UTC)
|
|
3129
|
+
if (
|
|
3130
|
+
approved_at.tzinfo is None
|
|
3131
|
+
or expires_at.tzinfo is None
|
|
3132
|
+
or approved_at > now
|
|
3133
|
+
or (expires_at <= now and not allow_expired)
|
|
3134
|
+
or expires_at > approved_at + timedelta(hours=1)
|
|
3135
|
+
):
|
|
3136
|
+
raise ValueError("source service image import approval is invalid or expired")
|
|
3137
|
+
return digest
|
|
3138
|
+
|
|
3139
|
+
|
|
3140
|
+
def _service_update_context(args: argparse.Namespace, work_dir: Path) -> dict[str, object]:
|
|
3141
|
+
"""Return sanitized retained state needed to bind one source service update."""
|
|
3142
|
+
|
|
3143
|
+
context = _private_json(work_dir / "context.json", "standalone host context")
|
|
3144
|
+
if _runtime_platform(context) != "aks":
|
|
3145
|
+
raise ValueError("source service update requires an applied AKS application baseline")
|
|
3146
|
+
service = str(args.service)
|
|
3147
|
+
_require_aks_application_baseline(work_dir, context, service=service)
|
|
3148
|
+
_require_source_service_update_dev_environment(work_dir)
|
|
3149
|
+
expected = _mapping(context.get("expected_workloads"), "expected AKS workloads")
|
|
3150
|
+
selected = _mapping(expected.get(service), f"expected AKS workload {service}")
|
|
3151
|
+
active = context.get("active_service_update")
|
|
3152
|
+
active_review = None
|
|
3153
|
+
if isinstance(active, dict):
|
|
3154
|
+
record = _service_update_record(context, work_dir, stage="application", service=service)
|
|
3155
|
+
if record is not None:
|
|
3156
|
+
active_review = _service_update_review(record)
|
|
3157
|
+
return {
|
|
3158
|
+
"schema_version": "fdai.source-service-update-context.v1",
|
|
3159
|
+
"state": "verified",
|
|
3160
|
+
"service": service,
|
|
3161
|
+
"target_binding": context["target_binding"],
|
|
3162
|
+
"current_image": selected["image"],
|
|
3163
|
+
"current_source_commit": selected["source_commit"],
|
|
3164
|
+
"active_service_update": active_review,
|
|
3165
|
+
"mutation_performed": False,
|
|
3166
|
+
"subscription_ready": False,
|
|
3167
|
+
}
|
|
3168
|
+
|
|
3169
|
+
|
|
3170
|
+
def _require_source_service_update_dev_environment(work_dir: Path) -> None:
|
|
3171
|
+
values = _private_json(work_dir / "application.auto.tfvars.json", "application variables")
|
|
3172
|
+
if values.get("env") != "dev":
|
|
3173
|
+
raise ValueError("source service updates are supported only for retained dev installations")
|
|
3174
|
+
|
|
3175
|
+
|
|
3176
|
+
def _migrate(_args: argparse.Namespace, work_dir: Path) -> dict[str, object]:
|
|
3177
|
+
receipt_path = work_dir / "migration-receipt.json"
|
|
3178
|
+
if receipt_path.exists():
|
|
3179
|
+
return _private_json(receipt_path, "standalone migration receipt")
|
|
3180
|
+
context = _private_json(work_dir / "context.json", "standalone host context")
|
|
3181
|
+
_managed_identity_login_from_context(context, work_dir)
|
|
3182
|
+
if not (work_dir / "substrate-receipt.json").exists():
|
|
3183
|
+
raise ValueError("database migration requires the applied substrate plan")
|
|
3184
|
+
if (
|
|
3185
|
+
_database_placement(context) == "postgres-aks"
|
|
3186
|
+
and not (work_dir / "database-receipt.json").exists()
|
|
3187
|
+
):
|
|
3188
|
+
raise ValueError("database migration requires the applied postgres-aks plan")
|
|
3189
|
+
infra = Path(str(context["infra"]))
|
|
3190
|
+
bundle = infra.parent
|
|
3191
|
+
vault_name = _vault_name(_terraform_output(infra, "key_vault_uri"))
|
|
3192
|
+
dsn = _capture(
|
|
3193
|
+
(
|
|
3194
|
+
"az",
|
|
3195
|
+
"keyvault",
|
|
3196
|
+
"secret",
|
|
3197
|
+
"show",
|
|
3198
|
+
"--vault-name",
|
|
3199
|
+
vault_name,
|
|
3200
|
+
"--name",
|
|
3201
|
+
"fdai-state-store-dsn",
|
|
3202
|
+
"--query",
|
|
3203
|
+
"value",
|
|
3204
|
+
"--output",
|
|
3205
|
+
"tsv",
|
|
3206
|
+
"--only-show-errors",
|
|
3207
|
+
),
|
|
3208
|
+
cwd=work_dir,
|
|
3209
|
+
timeout=120,
|
|
3210
|
+
reason="database migration secret is unavailable",
|
|
3211
|
+
).strip()
|
|
3212
|
+
if not dsn or any(character in dsn for character in "\r\n"):
|
|
3213
|
+
raise ValueError("database migration secret is invalid")
|
|
3214
|
+
runtime_python = work_dir / "runtime-venv/bin/python"
|
|
3215
|
+
environment = {
|
|
3216
|
+
**os.environ,
|
|
3217
|
+
"FDAI_DATABASE_URL": dsn,
|
|
3218
|
+
"FDAI_STATE_STORE_DSN": dsn,
|
|
3219
|
+
"FDAI_MIGRATION_PYTHON": str(runtime_python),
|
|
3220
|
+
"PYTHONPATH": os.pathsep.join(
|
|
3221
|
+
(
|
|
3222
|
+
str(bundle / "services/core-control-plane/src"),
|
|
3223
|
+
str(bundle / "packages/service-contracts/src"),
|
|
3224
|
+
)
|
|
3225
|
+
),
|
|
3226
|
+
}
|
|
3227
|
+
_run_env(
|
|
3228
|
+
(
|
|
3229
|
+
str(runtime_python),
|
|
3230
|
+
"-m",
|
|
3231
|
+
"alembic",
|
|
3232
|
+
"-c",
|
|
3233
|
+
str(bundle / "alembic.ini"),
|
|
3234
|
+
"upgrade",
|
|
3235
|
+
"head",
|
|
3236
|
+
),
|
|
3237
|
+
cwd=bundle,
|
|
3238
|
+
env=environment,
|
|
3239
|
+
timeout=1200,
|
|
3240
|
+
reason="legacy database migration failed",
|
|
3241
|
+
)
|
|
3242
|
+
order = _capture_env(
|
|
3243
|
+
(str(runtime_python), str(bundle / "service-migrations/migrate.py"), "all", "order"),
|
|
3244
|
+
cwd=bundle,
|
|
3245
|
+
env=environment,
|
|
3246
|
+
timeout=120,
|
|
3247
|
+
reason="service migration order is unavailable",
|
|
3248
|
+
).splitlines()
|
|
3249
|
+
evidence = work_dir / "migration-evidence"
|
|
3250
|
+
evidence.mkdir(mode=0o700, exist_ok=True)
|
|
3251
|
+
for service in order:
|
|
3252
|
+
if not re.fullmatch(r"[a-z][a-z0-9-]{1,63}", service):
|
|
3253
|
+
raise ValueError("service migration order is invalid")
|
|
3254
|
+
_run_env(
|
|
3255
|
+
(
|
|
3256
|
+
"/bin/sh",
|
|
3257
|
+
str(bundle / f"service-migrations/bin/{service}"),
|
|
3258
|
+
"bootstrap",
|
|
3259
|
+
"--evidence-output",
|
|
3260
|
+
str(evidence / f"{service}.json"),
|
|
3261
|
+
"--schema-output",
|
|
3262
|
+
str(evidence / f"{service}-schema.json"),
|
|
3263
|
+
"--rollback-reference",
|
|
3264
|
+
f"bundle:{context['source_commit']}:service-migrations/branches/{service}/adoption.json#rollback",
|
|
3265
|
+
),
|
|
3266
|
+
cwd=bundle,
|
|
3267
|
+
env=environment,
|
|
3268
|
+
timeout=1200,
|
|
3269
|
+
reason="service database migration failed",
|
|
3270
|
+
)
|
|
3271
|
+
verify_service_evidence(evidence, service)
|
|
3272
|
+
_run_env(
|
|
3273
|
+
(
|
|
3274
|
+
str(runtime_python),
|
|
3275
|
+
str(bundle / "scripts/deployment/local/materialize-authoritative-catalogs.py"),
|
|
3276
|
+
),
|
|
3277
|
+
cwd=bundle,
|
|
3278
|
+
env=environment,
|
|
3279
|
+
timeout=600,
|
|
3280
|
+
reason="authoritative catalog materialization failed",
|
|
3281
|
+
)
|
|
3282
|
+
dsn = ""
|
|
3283
|
+
receipt: dict[str, object] = {
|
|
3284
|
+
"schema_version": "fdai.standalone-migration-receipt.v1",
|
|
3285
|
+
"state": "migrated",
|
|
3286
|
+
"service_count": len(order),
|
|
3287
|
+
"catalogs_materialized": True,
|
|
3288
|
+
"effect_verified": True,
|
|
3289
|
+
"mutation_performed": True,
|
|
3290
|
+
"subscription_ready": False,
|
|
3291
|
+
}
|
|
3292
|
+
receipt["receipt_digest"] = canonical_digest(receipt)
|
|
3293
|
+
_replace_private_json(receipt_path, receipt)
|
|
3294
|
+
return receipt
|
|
3295
|
+
|
|
3296
|
+
|
|
3297
|
+
def _initial_inventory(_args: argparse.Namespace, work_dir: Path) -> dict[str, object]:
|
|
3298
|
+
"""Run and independently process-check one explicit full-subscription inventory."""
|
|
3299
|
+
|
|
3300
|
+
receipt_path = work_dir / "initial-inventory-receipt.json"
|
|
3301
|
+
if receipt_path.exists():
|
|
3302
|
+
return _private_json(receipt_path, "standalone initial inventory receipt")
|
|
3303
|
+
context = _private_json(work_dir / "context.json", "standalone host context")
|
|
3304
|
+
_managed_identity_login_from_context(context, work_dir)
|
|
3305
|
+
for prerequisite in ("migration-receipt.json", "application-receipt.json"):
|
|
3306
|
+
if not (work_dir / prerequisite).exists():
|
|
3307
|
+
raise ValueError("initial inventory prerequisites are incomplete")
|
|
3308
|
+
if _runtime_platform(context) == "aks":
|
|
3309
|
+
return _initial_aks_inventory(context, work_dir, receipt_path=receipt_path)
|
|
3310
|
+
infra = Path(str(context["infra"]))
|
|
3311
|
+
bundle = infra.parent
|
|
3312
|
+
vault_name = _vault_name(_terraform_output(infra, "key_vault_uri"))
|
|
3313
|
+
dsn = _capture(
|
|
3314
|
+
(
|
|
3315
|
+
"az",
|
|
3316
|
+
"keyvault",
|
|
3317
|
+
"secret",
|
|
3318
|
+
"show",
|
|
3319
|
+
"--vault-name",
|
|
3320
|
+
vault_name,
|
|
3321
|
+
"--name",
|
|
3322
|
+
"fdai-state-store-dsn",
|
|
3323
|
+
"--query",
|
|
3324
|
+
"value",
|
|
3325
|
+
"--output",
|
|
3326
|
+
"tsv",
|
|
3327
|
+
"--only-show-errors",
|
|
3328
|
+
),
|
|
3329
|
+
cwd=work_dir,
|
|
3330
|
+
timeout=120,
|
|
3331
|
+
reason="initial inventory database reference is unavailable",
|
|
3332
|
+
).strip()
|
|
3333
|
+
if not dsn or any(character in dsn for character in "\r\n"):
|
|
3334
|
+
raise ValueError("initial inventory database reference is invalid")
|
|
3335
|
+
progress_url = str(context["inventory_progress_container_url"])
|
|
3336
|
+
if not progress_url.startswith("https://"):
|
|
3337
|
+
raise ValueError("initial inventory progress endpoint is invalid")
|
|
3338
|
+
runtime_python = work_dir / "runtime-venv/bin/python"
|
|
3339
|
+
attempt_id = f"attempt.{uuid.uuid4().hex}"
|
|
3340
|
+
environment = {
|
|
3341
|
+
**os.environ,
|
|
3342
|
+
"AZURE_CLIENT_ID": str(context["client_id"]),
|
|
3343
|
+
"FDAI_MI_CLIENT_ID": str(context["client_id"]),
|
|
3344
|
+
"FDAI_EXECUTION_VENUE": "deployed",
|
|
3345
|
+
"FDAI_INVENTORY_DSN": dsn,
|
|
3346
|
+
"FDAI_INVENTORY_SCOPES": str(context["subscription_id"]),
|
|
3347
|
+
"FDAI_INVENTORY_SOURCES": "arg,arm",
|
|
3348
|
+
"FDAI_INVENTORY_PROGRESS_CONTAINER_URL": progress_url,
|
|
3349
|
+
"FDAI_INVENTORY_PROGRESS_RUN_ID": f"genesis.{context['source_commit']}",
|
|
3350
|
+
"FDAI_INVENTORY_PROGRESS_ATTEMPT_ID": attempt_id,
|
|
3351
|
+
"KAFKA_BOOTSTRAP_SERVERS": _terraform_output(
|
|
3352
|
+
infra, "event_bus_operational_kafka_bootstrap"
|
|
3353
|
+
),
|
|
3354
|
+
"PYTHONPATH": os.pathsep.join(
|
|
3355
|
+
(
|
|
3356
|
+
str(bundle / "services/core-control-plane/src"),
|
|
3357
|
+
str(bundle / "packages/service-contracts/src"),
|
|
3358
|
+
)
|
|
3359
|
+
),
|
|
3360
|
+
}
|
|
3361
|
+
_run_env(
|
|
3362
|
+
(str(runtime_python), "-m", "fdai.delivery.inventory_sync_cli", "--initial"),
|
|
3363
|
+
cwd=bundle,
|
|
3364
|
+
env=environment,
|
|
3365
|
+
timeout=3600,
|
|
3366
|
+
reason="initial inventory reconciliation failed",
|
|
3367
|
+
)
|
|
3368
|
+
closure_raw = _capture_env(
|
|
3369
|
+
(str(runtime_python), "-m", "fdai.delivery.inventory_closure_cli"),
|
|
3370
|
+
cwd=bundle,
|
|
3371
|
+
env=environment,
|
|
3372
|
+
timeout=300,
|
|
3373
|
+
reason="initial inventory independent closure failed",
|
|
3374
|
+
)
|
|
3375
|
+
try:
|
|
3376
|
+
closure = json.loads(closure_raw)
|
|
3377
|
+
except json.JSONDecodeError as exc:
|
|
3378
|
+
raise ValueError("initial inventory closure receipt is invalid") from exc
|
|
3379
|
+
if (
|
|
3380
|
+
not isinstance(closure, dict)
|
|
3381
|
+
or closure.get("observer_distinct") is not True
|
|
3382
|
+
or closure.get("active_generation_matches") is not True
|
|
3383
|
+
or closure.get("provider_coverage_complete") is not True
|
|
3384
|
+
or closure.get("receipt_digest") is None
|
|
3385
|
+
):
|
|
3386
|
+
raise ValueError("initial inventory closure receipt is incomplete")
|
|
3387
|
+
dsn = ""
|
|
3388
|
+
receipt: dict[str, object] = {
|
|
3389
|
+
"schema_version": "fdai.standalone-initial-inventory-receipt.v1",
|
|
3390
|
+
"state": "inventory-verified",
|
|
3391
|
+
"run_id": f"genesis.{context['source_commit']}",
|
|
3392
|
+
"attempt_id": attempt_id,
|
|
3393
|
+
"full_subscription": True,
|
|
3394
|
+
"resource_type_filter": False,
|
|
3395
|
+
"progress_persisted": True,
|
|
3396
|
+
"active_generation_readback_verified": True,
|
|
3397
|
+
"closure_receipt_digest": closure["receipt_digest"],
|
|
3398
|
+
"effect_verified": True,
|
|
3399
|
+
"mutation_performed": True,
|
|
3400
|
+
"subscription_ready": False,
|
|
3401
|
+
}
|
|
3402
|
+
receipt["receipt_digest"] = canonical_digest(receipt)
|
|
3403
|
+
_replace_private_json(receipt_path, receipt)
|
|
3404
|
+
return receipt
|
|
3405
|
+
|
|
3406
|
+
|
|
3407
|
+
def _catalog_review(_args: argparse.Namespace, work_dir: Path) -> dict[str, object]:
|
|
3408
|
+
context = _private_json(work_dir / "context.json", "standalone host context")
|
|
3409
|
+
return run_catalog_review(context, work_dir, login=_managed_identity_login_from_context)
|
|
3410
|
+
|
|
3411
|
+
|
|
3412
|
+
def _activate_trial(_args: argparse.Namespace, work_dir: Path) -> dict[str, object]:
|
|
3413
|
+
context = _private_json(work_dir / "context.json", "standalone host context")
|
|
3414
|
+
return activate_trial(context, work_dir, login=_managed_identity_login_from_context)
|
|
3415
|
+
|
|
3416
|
+
|
|
3417
|
+
def _install_license(args: argparse.Namespace, work_dir: Path) -> dict[str, object]:
|
|
3418
|
+
return install_license(args, work_dir, login=_managed_identity_login_from_context)
|
|
3419
|
+
|
|
3420
|
+
|
|
3421
|
+
def _verify(_args: argparse.Namespace, work_dir: Path) -> dict[str, object]:
|
|
3422
|
+
context = _private_json(work_dir / "context.json", "standalone host context")
|
|
3423
|
+
_managed_identity_login_from_context(context, work_dir)
|
|
3424
|
+
stages: tuple[str, ...]
|
|
3425
|
+
if _runtime_platform(context) == "aks":
|
|
3426
|
+
stages = (
|
|
3427
|
+
("substrate", "runtime", "database", "application")
|
|
3428
|
+
if _database_placement(context) == "postgres-aks"
|
|
3429
|
+
else ("substrate", "runtime", "application")
|
|
3430
|
+
)
|
|
3431
|
+
if _focused_private_access(context):
|
|
3432
|
+
stages = ("access", *stages)
|
|
3433
|
+
else:
|
|
3434
|
+
stages = ("application",)
|
|
3435
|
+
for stage in stages:
|
|
3436
|
+
infra, variables = _stage_paths(stage, context, work_dir)
|
|
3437
|
+
_activate_terraform_stage(stage, context, work_dir)
|
|
3438
|
+
command: tuple[str, ...] = (
|
|
3439
|
+
"terraform",
|
|
3440
|
+
"plan",
|
|
3441
|
+
"-detailed-exitcode",
|
|
3442
|
+
"-input=false",
|
|
3443
|
+
"-no-color",
|
|
3444
|
+
f"-var-file={variables}",
|
|
3445
|
+
)
|
|
3446
|
+
command += tuple(f"-target={target}" for target in _stage_targets(stage, context))
|
|
3447
|
+
completed = subprocess.run(
|
|
3448
|
+
command, cwd=infra, check=False, capture_output=True, timeout=1800
|
|
3449
|
+
)
|
|
3450
|
+
if completed.returncode != 0:
|
|
3451
|
+
raise ValueError(f"standalone {stage} second plan is not zero-change")
|
|
3452
|
+
health = _readback_stage("application", context)
|
|
3453
|
+
if not health:
|
|
3454
|
+
raise ValueError("standalone application runtime health is incomplete")
|
|
3455
|
+
browser_console = (
|
|
3456
|
+
_browser_console_binding(context, work_dir)
|
|
3457
|
+
if _runtime_platform(context) == "aks"
|
|
3458
|
+
and _selects_add_on(context, ProductAddOn.READ_ONLY_CONSOLE)
|
|
3459
|
+
else None
|
|
3460
|
+
)
|
|
3461
|
+
receipt: dict[str, object] = {
|
|
3462
|
+
"schema_version": "fdai.standalone-application-verification.v1",
|
|
3463
|
+
"state": "verified",
|
|
3464
|
+
"terraform_zero_change_verified": True,
|
|
3465
|
+
"runtime_health_verified": True,
|
|
3466
|
+
"runtime_profile_digest": _runtime_profile_digest(context),
|
|
3467
|
+
"effect_verified": True,
|
|
3468
|
+
"mutation_performed": False,
|
|
3469
|
+
"subscription_ready": False,
|
|
3470
|
+
}
|
|
3471
|
+
if browser_console is not None:
|
|
3472
|
+
receipt["browser_console"] = browser_console
|
|
3473
|
+
receipt["receipt_digest"] = canonical_digest(receipt)
|
|
3474
|
+
_replace_private_json(work_dir / "application-verification-receipt.json", receipt)
|
|
3475
|
+
return receipt
|
|
3476
|
+
|
|
3477
|
+
|
|
3478
|
+
def _browser_console_binding(context: dict[str, object], work_dir: Path) -> dict[str, str]:
|
|
3479
|
+
"""Read and validate the public SWA and APIM bindings across their owning states."""
|
|
3480
|
+
|
|
3481
|
+
_activate_terraform_stage("substrate", context, work_dir)
|
|
3482
|
+
substrate = Path(str(context["infra"]))
|
|
3483
|
+
hostname = _terraform_output(substrate, "console_default_hostname")
|
|
3484
|
+
resource_id = _terraform_output(substrate, "console_static_web_app_id")
|
|
3485
|
+
workloads_path = work_dir / "workloads.auto.tfvars.json"
|
|
3486
|
+
if workloads_path.is_file():
|
|
3487
|
+
standalone_host_values.verify_browser_gateway_backend_nsg(
|
|
3488
|
+
_private_json(workloads_path, "workloads.auto.tfvars.json"),
|
|
3489
|
+
observed_nsg_id=_subnet_network_security_group(
|
|
3490
|
+
_terraform_output(substrate, "aks_subnet_id"),
|
|
3491
|
+
context=context,
|
|
3492
|
+
cwd=work_dir,
|
|
3493
|
+
),
|
|
3494
|
+
)
|
|
3495
|
+
_activate_terraform_stage("application", context, work_dir)
|
|
3496
|
+
workloads = Path(str(context["workloads_infra"]))
|
|
3497
|
+
return standalone_host_values.validate_browser_console_binding(
|
|
3498
|
+
subscription_id=str(context["subscription_id"]),
|
|
3499
|
+
hostname=hostname,
|
|
3500
|
+
console_origin=_console_origin(hostname),
|
|
3501
|
+
resource_id=resource_id,
|
|
3502
|
+
operator_url=_terraform_output(workloads, "browser_gateway_operator_url").rstrip("/"),
|
|
3503
|
+
ingestion_url=_terraform_output(workloads, "browser_gateway_ingestion_url").rstrip("/"),
|
|
3504
|
+
)
|
|
3505
|
+
|
|
3506
|
+
|
|
3507
|
+
def _terraform_init(work_dir: Path, context: dict[str, object]) -> None:
|
|
3508
|
+
infra = Path(str(context["infra"]))
|
|
3509
|
+
standalone_terraform_environment.ensure_backend(infra)
|
|
3510
|
+
_configure_terraform(context)
|
|
3511
|
+
_run(
|
|
3512
|
+
(
|
|
3513
|
+
"terraform",
|
|
3514
|
+
"init",
|
|
3515
|
+
"-input=false",
|
|
3516
|
+
f"-backend-config=resource_group_name={context['state_resource_group']}",
|
|
3517
|
+
f"-backend-config=storage_account_name={context['state_account']}",
|
|
3518
|
+
f"-backend-config=container_name={context['state_container']}",
|
|
3519
|
+
f"-backend-config=key={context['state_key']}",
|
|
3520
|
+
),
|
|
3521
|
+
cwd=infra,
|
|
3522
|
+
timeout=600,
|
|
3523
|
+
reason="standalone application remote state initialization failed",
|
|
3524
|
+
)
|
|
3525
|
+
|
|
3526
|
+
|
|
3527
|
+
def _stage_paths(stage: str, context: dict[str, object], work_dir: Path) -> tuple[Path, Path]:
|
|
3528
|
+
platform = _runtime_platform(context)
|
|
3529
|
+
if stage == "runtime":
|
|
3530
|
+
if platform != "aks":
|
|
3531
|
+
raise ValueError("runtime stage is only valid for AKS")
|
|
3532
|
+
return Path(str(context["runtime_infra"])), work_dir / "runtime.auto.tfvars.json"
|
|
3533
|
+
if stage == "database":
|
|
3534
|
+
if platform != "aks":
|
|
3535
|
+
raise ValueError("database stage is only valid for AKS")
|
|
3536
|
+
return (
|
|
3537
|
+
Path(str(context["database_infra"])),
|
|
3538
|
+
work_dir / "database.auto.tfvars.json",
|
|
3539
|
+
)
|
|
3540
|
+
if stage == "application" and platform == "aks":
|
|
3541
|
+
return (
|
|
3542
|
+
Path(str(context["workloads_infra"])),
|
|
3543
|
+
work_dir / "workloads.auto.tfvars.json",
|
|
3544
|
+
)
|
|
3545
|
+
return Path(str(context["infra"])), work_dir / "application.auto.tfvars.json"
|
|
3546
|
+
|
|
3547
|
+
|
|
3548
|
+
def _activate_terraform_stage(stage: str, context: dict[str, object], work_dir: Path) -> None:
|
|
3549
|
+
platform = _runtime_platform(context)
|
|
3550
|
+
if stage == "runtime":
|
|
3551
|
+
data = context.get("runtime_terraform_data")
|
|
3552
|
+
elif stage == "database":
|
|
3553
|
+
data = context.get("database_terraform_data")
|
|
3554
|
+
elif stage == "application" and platform == "aks":
|
|
3555
|
+
data = hub_ownership.workloads_terraform_data(context, work_dir)
|
|
3556
|
+
else:
|
|
3557
|
+
data = context.get("terraform_data", str(work_dir / "terraform-data"))
|
|
3558
|
+
if not isinstance(data, str) or not data:
|
|
3559
|
+
raise ValueError(f"{stage} Terraform data path is unavailable")
|
|
3560
|
+
path = Path(data)
|
|
3561
|
+
if not path.is_absolute() or work_dir not in path.parents:
|
|
3562
|
+
raise ValueError(f"{stage} Terraform data path is outside the managed work directory")
|
|
3563
|
+
path.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
3564
|
+
os.environ["TF_DATA_DIR"] = str(path)
|
|
3565
|
+
|
|
3566
|
+
|
|
3567
|
+
def _initialize_terraform_stage(stage: str, context: dict[str, object], work_dir: Path) -> None:
|
|
3568
|
+
infra, _variables = _stage_paths(stage, context, work_dir)
|
|
3569
|
+
if stage == "runtime":
|
|
3570
|
+
state_key = context.get("runtime_state_key")
|
|
3571
|
+
elif stage == "database":
|
|
3572
|
+
state_key = context.get("database_state_key")
|
|
3573
|
+
elif stage == "application" and _runtime_platform(context) == "aks":
|
|
3574
|
+
state_key = context.get("workloads_state_key")
|
|
3575
|
+
else:
|
|
3576
|
+
state_key = context.get("state_key")
|
|
3577
|
+
if not isinstance(state_key, str) or not state_key:
|
|
3578
|
+
raise ValueError(f"{stage} Terraform state key is unavailable")
|
|
3579
|
+
standalone_terraform_environment.ensure_backend(infra)
|
|
3580
|
+
_configure_terraform(context)
|
|
3581
|
+
_activate_terraform_stage(stage, context, work_dir)
|
|
3582
|
+
_run(
|
|
3583
|
+
(
|
|
3584
|
+
"terraform",
|
|
3585
|
+
"init",
|
|
3586
|
+
"-input=false",
|
|
3587
|
+
f"-backend-config=resource_group_name={context['state_resource_group']}",
|
|
3588
|
+
f"-backend-config=storage_account_name={context['state_account']}",
|
|
3589
|
+
f"-backend-config=container_name={context['state_container']}",
|
|
3590
|
+
f"-backend-config=key={state_key}",
|
|
3591
|
+
),
|
|
3592
|
+
cwd=infra,
|
|
3593
|
+
timeout=600,
|
|
3594
|
+
reason=f"{stage} Terraform remote state initialization failed",
|
|
3595
|
+
)
|
|
3596
|
+
|
|
3597
|
+
|
|
3598
|
+
def _runtime_platform(context: dict[str, object]) -> str:
|
|
3599
|
+
value = context.get("runtime_profile")
|
|
3600
|
+
if value is None:
|
|
3601
|
+
return "container-apps"
|
|
3602
|
+
profile = _mapping(value, "runtime deployment profile")
|
|
3603
|
+
platform = profile.get("runtime_platform")
|
|
3604
|
+
if platform not in {"container-apps", "aks"}:
|
|
3605
|
+
raise ValueError("runtime deployment platform is invalid")
|
|
3606
|
+
return str(platform)
|
|
3607
|
+
|
|
3608
|
+
|
|
3609
|
+
def _prepare_aks_kubeconfig(
|
|
3610
|
+
context: dict[str, object], work_dir: Path, *, resource_group: str, cluster_name: str
|
|
3611
|
+
) -> Path:
|
|
3612
|
+
"""Select explicit managed-host MSI authentication before any Kubernetes operation."""
|
|
3613
|
+
client_id = _required_guid(context, "client_id")
|
|
3614
|
+
subscription_id = _required_guid(context, "subscription_id")
|
|
3615
|
+
kubectl, kubelogin, environment = standalone_kubernetes_tools.tools(context)
|
|
3616
|
+
kubeconfig = work_dir / "aks.kubeconfig"
|
|
3617
|
+
if kubeconfig.exists() or kubeconfig.is_symlink():
|
|
3618
|
+
read_private_bytes(kubeconfig, max_bytes=1024 * 1024)
|
|
3619
|
+
else:
|
|
3620
|
+
write_private_output(kubeconfig, "")
|
|
3621
|
+
_run(
|
|
3622
|
+
(
|
|
3623
|
+
"az",
|
|
3624
|
+
"aks",
|
|
3625
|
+
"get-credentials",
|
|
3626
|
+
"--subscription",
|
|
3627
|
+
subscription_id,
|
|
3628
|
+
"--resource-group",
|
|
3629
|
+
resource_group,
|
|
3630
|
+
"--name",
|
|
3631
|
+
cluster_name,
|
|
3632
|
+
"--file",
|
|
3633
|
+
str(kubeconfig),
|
|
3634
|
+
"--overwrite-existing",
|
|
3635
|
+
"--format",
|
|
3636
|
+
"exec",
|
|
3637
|
+
"--only-show-errors",
|
|
3638
|
+
),
|
|
3639
|
+
cwd=work_dir,
|
|
3640
|
+
timeout=180,
|
|
3641
|
+
reason="private AKS kubeconfig acquisition failed",
|
|
3642
|
+
)
|
|
3643
|
+
read_private_bytes(kubeconfig, max_bytes=1024 * 1024)
|
|
3644
|
+
_run_env(
|
|
3645
|
+
(
|
|
3646
|
+
kubelogin,
|
|
3647
|
+
"convert-kubeconfig",
|
|
3648
|
+
"--kubeconfig",
|
|
3649
|
+
str(kubeconfig),
|
|
3650
|
+
"--login",
|
|
3651
|
+
"msi",
|
|
3652
|
+
"--client-id",
|
|
3653
|
+
client_id,
|
|
3654
|
+
),
|
|
3655
|
+
cwd=work_dir,
|
|
3656
|
+
timeout=60,
|
|
3657
|
+
reason="private AKS managed-host authentication setup failed",
|
|
3658
|
+
env=environment,
|
|
3659
|
+
)
|
|
3660
|
+
read_private_bytes(kubeconfig, max_bytes=1024 * 1024)
|
|
3661
|
+
rendered = _capture_env(
|
|
3662
|
+
(
|
|
3663
|
+
kubectl,
|
|
3664
|
+
"config",
|
|
3665
|
+
"view",
|
|
3666
|
+
"--minify",
|
|
3667
|
+
"--output",
|
|
3668
|
+
"json",
|
|
3669
|
+
"--kubeconfig",
|
|
3670
|
+
str(kubeconfig),
|
|
3671
|
+
),
|
|
3672
|
+
cwd=work_dir,
|
|
3673
|
+
timeout=30,
|
|
3674
|
+
reason="private AKS authentication configuration readback failed",
|
|
3675
|
+
env=environment,
|
|
3676
|
+
)
|
|
3677
|
+
configuration = load_json_object(
|
|
3678
|
+
rendered.encode(), label="AKS authentication configuration", max_bytes=1024 * 1024
|
|
3679
|
+
)
|
|
3680
|
+
users = configuration.get("users")
|
|
3681
|
+
if not isinstance(users, list) or len(users) != 1 or not isinstance(users[0], dict):
|
|
3682
|
+
raise ValueError("AKS authentication configuration has ambiguous users")
|
|
3683
|
+
user = _mapping(users[0].get("user"), "AKS authentication user")
|
|
3684
|
+
execution = _mapping(user.get("exec"), "AKS authentication command")
|
|
3685
|
+
arguments = execution.get("args")
|
|
3686
|
+
if (
|
|
3687
|
+
set(user) != {"exec"}
|
|
3688
|
+
or execution.get("command") != "kubelogin"
|
|
3689
|
+
or not isinstance(arguments, list)
|
|
3690
|
+
or not all(isinstance(value, str) for value in arguments)
|
|
3691
|
+
or arguments[:1] != ["get-token"]
|
|
3692
|
+
or execution.get("env") not in (None, [])
|
|
3693
|
+
or "-l" in arguments
|
|
3694
|
+
or any(value.startswith(("--login=", "--client-id=")) for value in arguments)
|
|
3695
|
+
):
|
|
3696
|
+
raise ValueError("AKS authentication must use only the selected managed identity")
|
|
3697
|
+
for flag, expected in (("--login", "msi"), ("--client-id", client_id)):
|
|
3698
|
+
if arguments.count(flag) != 1 or arguments[
|
|
3699
|
+
arguments.index(flag) + 1 : arguments.index(flag) + 2
|
|
3700
|
+
] != [expected]:
|
|
3701
|
+
raise ValueError("AKS authentication does not match the managed host identity")
|
|
3702
|
+
return kubeconfig
|
|
3703
|
+
|
|
3704
|
+
|
|
3705
|
+
def _aks_document_workloads(
|
|
3706
|
+
*,
|
|
3707
|
+
refs: dict[str, Any],
|
|
3708
|
+
ingestion_identity: dict[str, Any],
|
|
3709
|
+
worker_identity: dict[str, Any],
|
|
3710
|
+
application_values: dict[str, Any],
|
|
3711
|
+
kafka: str,
|
|
3712
|
+
postgres_fqdn: str,
|
|
3713
|
+
document_store: dict[str, Any],
|
|
3714
|
+
document_topics: dict[str, Any],
|
|
3715
|
+
console_origin: str,
|
|
3716
|
+
) -> dict[str, dict[str, object]]:
|
|
3717
|
+
common = {
|
|
3718
|
+
"RUNTIME_ENV": application_values["env"],
|
|
3719
|
+
"FDAI_KAFKA_BOOTSTRAP_SERVERS": kafka,
|
|
3720
|
+
"FDAI_DOCUMENT_EVENT_TOPIC": document_topics["pipeline_stages"],
|
|
3721
|
+
"FDAI_DOCUMENT_RETRIEVAL_MODE": "lexical",
|
|
3722
|
+
"FDAI_ADLS_ACCOUNT_NAME": document_store["account_name"],
|
|
3723
|
+
"FDAI_ADLS_ACCOUNT_URL": document_store["account_url"],
|
|
3724
|
+
"FDAI_ADLS_SOURCE_FILE_SYSTEM": document_store["source_file_system"],
|
|
3725
|
+
"POSTGRES_HOST": postgres_fqdn,
|
|
3726
|
+
}
|
|
3727
|
+
api_environment = {
|
|
3728
|
+
**common,
|
|
3729
|
+
"FDAI_MI_CLIENT_ID": ingestion_identity["client_id"],
|
|
3730
|
+
"FDAI_INGESTION_DEPLOYMENT_ROLE": "api",
|
|
3731
|
+
"FDAI_INGESTION_COHOST_WORKER": "0",
|
|
3732
|
+
"FDAI_ENTRA_TENANT_ID": application_values["tenant_id"],
|
|
3733
|
+
"FDAI_API_AUDIENCE": application_values["operator_api_audience"],
|
|
3734
|
+
"FDAI_RBAC_READERS_GROUP_ID": application_values["rbac_readers_group_id"],
|
|
3735
|
+
"FDAI_RBAC_CONTRIBUTORS_GROUP_ID": application_values["rbac_contributors_group_id"],
|
|
3736
|
+
"FDAI_RBAC_APPROVERS_GROUP_ID": application_values["rbac_approvers_group_id"],
|
|
3737
|
+
"FDAI_RBAC_OWNERS_GROUP_ID": application_values["rbac_owners_group_id"],
|
|
3738
|
+
"FDAI_RBAC_BREAK_GLASS_GROUP_ID": application_values["rbac_break_glass_group_id"],
|
|
3739
|
+
"FDAI_INGESTION_CORS_ALLOW_ORIGINS": console_origin,
|
|
3740
|
+
}
|
|
3741
|
+
worker_environment = {
|
|
3742
|
+
**common,
|
|
3743
|
+
"FDAI_MI_CLIENT_ID": worker_identity["client_id"],
|
|
3744
|
+
"FDAI_INGESTION_DEPLOYMENT_ROLE": "worker",
|
|
3745
|
+
"FDAI_PANTHEON_OBJECT_TOPIC": document_topics["pantheon_objects"],
|
|
3746
|
+
"FDAI_ADLS_DERIVED_FILE_SYSTEM": document_store["derived_file_system"],
|
|
3747
|
+
"FDAI_INGESTION_WORKER_HEALTH_PORT": "8000",
|
|
3748
|
+
"FDAI_CLAMAV_HOST": "127.0.0.1",
|
|
3749
|
+
"FDAI_CLAMAV_PORT": "3310",
|
|
3750
|
+
}
|
|
3751
|
+
return {
|
|
3752
|
+
"document-ingestion-api": _aks_workload(
|
|
3753
|
+
"ingestion",
|
|
3754
|
+
refs,
|
|
3755
|
+
ingestion_identity,
|
|
3756
|
+
api_environment,
|
|
3757
|
+
{"FDAI_DATABASE_URL": "fdai-ingestion-api-dsn"},
|
|
3758
|
+
"/healthz",
|
|
3759
|
+
"/healthz",
|
|
3760
|
+
external=True,
|
|
3761
|
+
service_port=80,
|
|
3762
|
+
),
|
|
3763
|
+
"document-processing-worker": _aks_workload(
|
|
3764
|
+
"worker",
|
|
3765
|
+
refs,
|
|
3766
|
+
worker_identity,
|
|
3767
|
+
worker_environment,
|
|
3768
|
+
{"FDAI_DATABASE_URL": "fdai-ingestion-worker-dsn"},
|
|
3769
|
+
"/ready",
|
|
3770
|
+
"/live",
|
|
3771
|
+
fs_group=101,
|
|
3772
|
+
sidecars={
|
|
3773
|
+
"clamav": {
|
|
3774
|
+
"image": refs["clamav"],
|
|
3775
|
+
"command": ["clamd"],
|
|
3776
|
+
"args": ["--foreground=true"],
|
|
3777
|
+
"cpu": "500m",
|
|
3778
|
+
"memory": "2Gi",
|
|
3779
|
+
"port": 3310,
|
|
3780
|
+
"run_as_user": 100,
|
|
3781
|
+
"run_as_group": 101,
|
|
3782
|
+
"init": {
|
|
3783
|
+
"name": "clamav-database",
|
|
3784
|
+
"command": ["/bin/sh", "-c"],
|
|
3785
|
+
"args": ["cp -a /var/lib/clamav/. /target/"],
|
|
3786
|
+
"image_pull_policy": "IfNotPresent",
|
|
3787
|
+
"run_as_user": 100,
|
|
3788
|
+
"run_as_group": 101,
|
|
3789
|
+
"writable_path": "database",
|
|
3790
|
+
"mount_path": "/target",
|
|
3791
|
+
},
|
|
3792
|
+
"writable_paths": {
|
|
3793
|
+
"database": {"mount_path": "/var/lib/clamav", "size_limit": "1Gi"},
|
|
3794
|
+
"log": {"mount_path": "/var/log/clamav", "size_limit": "64Mi"},
|
|
3795
|
+
"run": {"mount_path": "/run/clamav", "size_limit": "64Mi"},
|
|
3796
|
+
"tmp": {"mount_path": "/tmp", "size_limit": "256Mi"},
|
|
3797
|
+
},
|
|
3798
|
+
}
|
|
3799
|
+
},
|
|
3800
|
+
),
|
|
3801
|
+
}
|
|
3802
|
+
|
|
3803
|
+
|
|
3804
|
+
def _application_state_adoption(
|
|
3805
|
+
args: argparse.Namespace,
|
|
3806
|
+
) -> tuple[dict[str, Any], Path, Path, Path] | None:
|
|
3807
|
+
paths = tuple(
|
|
3808
|
+
getattr(args, name) for name in ("adoption_state", "adoption_models", "adoption_descriptor")
|
|
3809
|
+
)
|
|
3810
|
+
if not any(path is not None for path in paths):
|
|
3811
|
+
return None
|
|
3812
|
+
if not all(isinstance(path, Path) for path in paths):
|
|
3813
|
+
raise ValueError("standalone application state adoption inputs are incomplete")
|
|
3814
|
+
state_path, models_path, descriptor_path = (_absolute(path) for path in paths)
|
|
3815
|
+
descriptor = _private_json(descriptor_path, "application state adoption descriptor")
|
|
3816
|
+
expected_fields = {
|
|
3817
|
+
"schema_version",
|
|
3818
|
+
"state",
|
|
3819
|
+
"source_state_sha256",
|
|
3820
|
+
"staged_state_sha256",
|
|
3821
|
+
"resolved_models_sha256",
|
|
3822
|
+
"recovery_receipt_sha256",
|
|
3823
|
+
"source_commit",
|
|
3824
|
+
"verified_source_commit",
|
|
3825
|
+
"resource_name_suffix",
|
|
3826
|
+
"managed_resource_count",
|
|
3827
|
+
"removed_addresses",
|
|
3828
|
+
"resolved_capabilities",
|
|
3829
|
+
"original_state_retained",
|
|
3830
|
+
"remote_backend_authority_verified",
|
|
3831
|
+
"mutation_performed",
|
|
3832
|
+
"subscription_ready",
|
|
3833
|
+
}
|
|
3834
|
+
if (
|
|
3835
|
+
set(descriptor) != expected_fields
|
|
3836
|
+
or descriptor.get("schema_version") != "fdai.application-state-adoption.v1"
|
|
3837
|
+
or descriptor.get("state") != "staged"
|
|
3838
|
+
or descriptor.get("staged_state_sha256") != _file_digest(state_path)
|
|
3839
|
+
or descriptor.get("resolved_models_sha256") != _file_digest(models_path)
|
|
3840
|
+
or _DIGEST.fullmatch(str(descriptor.get("source_state_sha256", ""))) is None
|
|
3841
|
+
or _DIGEST.fullmatch(str(descriptor.get("recovery_receipt_sha256", ""))) is None
|
|
3842
|
+
or _SOURCE_COMMIT.fullmatch(str(descriptor.get("source_commit", ""))) is None
|
|
3843
|
+
or _SOURCE_COMMIT.fullmatch(str(descriptor.get("verified_source_commit", ""))) is None
|
|
3844
|
+
or re.fullmatch(r"[a-z0-9]{6}", str(descriptor.get("resource_name_suffix", ""))) is None
|
|
3845
|
+
or type(descriptor.get("managed_resource_count")) is not int
|
|
3846
|
+
or descriptor["managed_resource_count"] <= 0
|
|
3847
|
+
or descriptor.get("removed_addresses")
|
|
3848
|
+
!= [
|
|
3849
|
+
"module.resource_group.terraform_data.ownership",
|
|
3850
|
+
"module.resource_group.azurerm_resource_group.primary[0]",
|
|
3851
|
+
]
|
|
3852
|
+
or not isinstance(descriptor.get("resolved_capabilities"), list)
|
|
3853
|
+
or not descriptor["resolved_capabilities"]
|
|
3854
|
+
or descriptor.get("original_state_retained") is not True
|
|
3855
|
+
or descriptor.get("remote_backend_authority_verified") is not False
|
|
3856
|
+
or descriptor.get("mutation_performed") is not False
|
|
3857
|
+
or descriptor.get("subscription_ready") is not False
|
|
3858
|
+
):
|
|
3859
|
+
raise ValueError("standalone application state adoption descriptor is invalid")
|
|
3860
|
+
_validate_staged_application_state(state_path, descriptor)
|
|
3861
|
+
return descriptor, state_path, models_path, descriptor_path
|
|
3862
|
+
|
|
3863
|
+
|
|
3864
|
+
def _validate_staged_application_state(state_path: Path, descriptor: dict[str, Any]) -> None:
|
|
3865
|
+
state = json.loads(read_private_bytes(state_path, max_bytes=64 * 1024 * 1024))
|
|
3866
|
+
if (
|
|
3867
|
+
not isinstance(state, dict)
|
|
3868
|
+
or state.get("version") != 4
|
|
3869
|
+
or type(state.get("serial")) is not int
|
|
3870
|
+
or not isinstance(state.get("lineage"), str)
|
|
3871
|
+
or not state["lineage"]
|
|
3872
|
+
or not isinstance(state.get("resources"), list)
|
|
3873
|
+
):
|
|
3874
|
+
raise ValueError("standalone application staged state is invalid")
|
|
3875
|
+
managed_count = 0
|
|
3876
|
+
owner_addresses = {
|
|
3877
|
+
("module.resource_group", "terraform_data", "ownership"),
|
|
3878
|
+
("module.resource_group", "azurerm_resource_group", "primary"),
|
|
3879
|
+
}
|
|
3880
|
+
for value in state["resources"]:
|
|
3881
|
+
if not isinstance(value, dict) or not isinstance(value.get("instances"), list):
|
|
3882
|
+
raise ValueError( # noqa: TRY004 - normalize state JSON into a stable CLI error
|
|
3883
|
+
"standalone application staged state resource is invalid"
|
|
3884
|
+
)
|
|
3885
|
+
instances = value["instances"]
|
|
3886
|
+
if (
|
|
3887
|
+
instances
|
|
3888
|
+
and (
|
|
3889
|
+
value.get("module"),
|
|
3890
|
+
value.get("type"),
|
|
3891
|
+
value.get("name"),
|
|
3892
|
+
)
|
|
3893
|
+
in owner_addresses
|
|
3894
|
+
):
|
|
3895
|
+
raise ValueError("standalone application staged state retains a resource-group owner")
|
|
3896
|
+
if value.get("mode") == "managed":
|
|
3897
|
+
managed_count += len(instances)
|
|
3898
|
+
if managed_count != descriptor["managed_resource_count"]:
|
|
3899
|
+
raise ValueError("standalone application staged state count differs")
|
|
3900
|
+
|
|
3901
|
+
|
|
3902
|
+
def _adopt_application_state(
|
|
3903
|
+
work_dir: Path,
|
|
3904
|
+
context: dict[str, object],
|
|
3905
|
+
descriptor: dict[str, Any],
|
|
3906
|
+
state_path: Path,
|
|
3907
|
+
models_path: Path,
|
|
3908
|
+
descriptor_path: Path,
|
|
3909
|
+
) -> None:
|
|
3910
|
+
infra = Path(str(context["infra"]))
|
|
3911
|
+
claim_path = work_dir / "application-state-adoption-claim.json"
|
|
3912
|
+
receipt_path = work_dir / "application-state-adoption-receipt.json"
|
|
3913
|
+
claim: dict[str, object] = {
|
|
3914
|
+
"schema_version": "fdai.application-state-adoption-claim.v1",
|
|
3915
|
+
"target_binding": context["target_binding"],
|
|
3916
|
+
"foundation_binding_digest": context["foundation_binding_digest"],
|
|
3917
|
+
"adoption_descriptor_digest": canonical_digest(descriptor),
|
|
3918
|
+
"staged_state_sha256": descriptor["staged_state_sha256"],
|
|
3919
|
+
"managed_resource_count": descriptor["managed_resource_count"],
|
|
3920
|
+
"mutation_performed": False,
|
|
3921
|
+
}
|
|
3922
|
+
if receipt_path.exists():
|
|
3923
|
+
retained_receipt = _private_json(receipt_path, "application state adoption receipt")
|
|
3924
|
+
if (
|
|
3925
|
+
retained_receipt.get("schema_version") != "fdai.application-state-adoption-receipt.v1"
|
|
3926
|
+
or retained_receipt.get("claim_digest") != canonical_digest(claim)
|
|
3927
|
+
or retained_receipt.get("state") != "adopted"
|
|
3928
|
+
or retained_receipt.get("remote_backend_authority_verified") is not True
|
|
3929
|
+
or retained_receipt.get("original_state_retained") is not True
|
|
3930
|
+
):
|
|
3931
|
+
raise ValueError("retained application state adoption receipt differs")
|
|
3932
|
+
_verify_remote_application_state_continuity(infra, retained_receipt)
|
|
3933
|
+
_remove_adoption_inputs(state_path, models_path, descriptor_path)
|
|
3934
|
+
return
|
|
3935
|
+
if claim_path.exists():
|
|
3936
|
+
retained_claim = _private_json(claim_path, "application state adoption claim")
|
|
3937
|
+
if retained_claim != claim:
|
|
3938
|
+
raise ValueError("retained application state adoption claim differs")
|
|
3939
|
+
else:
|
|
3940
|
+
exists = _capture(
|
|
3941
|
+
(
|
|
3942
|
+
"az",
|
|
3943
|
+
"storage",
|
|
3944
|
+
"blob",
|
|
3945
|
+
"exists",
|
|
3946
|
+
"--auth-mode",
|
|
3947
|
+
"login",
|
|
3948
|
+
"--account-name",
|
|
3949
|
+
str(context["state_account"]),
|
|
3950
|
+
"--container-name",
|
|
3951
|
+
str(context["state_container"]),
|
|
3952
|
+
"--name",
|
|
3953
|
+
str(context["state_key"]),
|
|
3954
|
+
"--query",
|
|
3955
|
+
"exists",
|
|
3956
|
+
"--output",
|
|
3957
|
+
"tsv",
|
|
3958
|
+
"--only-show-errors",
|
|
3959
|
+
),
|
|
3960
|
+
cwd=infra,
|
|
3961
|
+
timeout=120,
|
|
3962
|
+
reason="standalone application remote state existence read failed",
|
|
3963
|
+
).strip()
|
|
3964
|
+
if exists not in {"true", "false"}:
|
|
3965
|
+
raise ValueError("standalone application remote state existence is invalid")
|
|
3966
|
+
if exists == "true":
|
|
3967
|
+
raise ValueError("standalone application remote state is not empty")
|
|
3968
|
+
write_private_output(
|
|
3969
|
+
claim_path,
|
|
3970
|
+
json.dumps(claim, sort_keys=True, separators=(",", ":")) + "\n",
|
|
3971
|
+
)
|
|
3972
|
+
_run(
|
|
3973
|
+
("terraform", "state", "push", str(state_path)),
|
|
3974
|
+
cwd=infra,
|
|
3975
|
+
timeout=600,
|
|
3976
|
+
reason="standalone application state push failed; verification-only recovery is required",
|
|
3977
|
+
)
|
|
3978
|
+
remote_digest, remote_lineage, remote_serial = _verify_remote_application_state(
|
|
3979
|
+
infra, state_path, descriptor
|
|
3980
|
+
)
|
|
3981
|
+
adoption_receipt: dict[str, object] = {
|
|
3982
|
+
"schema_version": "fdai.application-state-adoption-receipt.v1",
|
|
3983
|
+
"state": "adopted",
|
|
3984
|
+
"claim_digest": canonical_digest(claim),
|
|
3985
|
+
"remote_state_sha256": remote_digest,
|
|
3986
|
+
"remote_state_lineage": remote_lineage,
|
|
3987
|
+
"remote_state_serial": remote_serial,
|
|
3988
|
+
"managed_resource_count": descriptor["managed_resource_count"],
|
|
3989
|
+
"remote_backend_authority_verified": True,
|
|
3990
|
+
"original_state_retained": True,
|
|
3991
|
+
"azure_resource_mutation_performed": False,
|
|
3992
|
+
"mutation_performed": True,
|
|
3993
|
+
"subscription_ready": False,
|
|
3994
|
+
}
|
|
3995
|
+
write_private_output(
|
|
3996
|
+
receipt_path,
|
|
3997
|
+
json.dumps(adoption_receipt, sort_keys=True, separators=(",", ":")) + "\n",
|
|
3998
|
+
)
|
|
3999
|
+
_remove_adoption_inputs(state_path, models_path, descriptor_path)
|
|
4000
|
+
|
|
4001
|
+
|
|
4002
|
+
def _verify_remote_application_state(
|
|
4003
|
+
infra: Path, state_path: Path, descriptor: dict[str, Any]
|
|
4004
|
+
) -> tuple[str, str, int]:
|
|
4005
|
+
expected = json.loads(read_private_bytes(state_path, max_bytes=64 * 1024 * 1024))
|
|
4006
|
+
remote = json.loads(
|
|
4007
|
+
_capture(
|
|
4008
|
+
("terraform", "state", "pull"),
|
|
4009
|
+
cwd=infra,
|
|
4010
|
+
timeout=300,
|
|
4011
|
+
reason="standalone application remote state readback failed",
|
|
4012
|
+
)
|
|
4013
|
+
)
|
|
4014
|
+
if not isinstance(expected, dict) or not isinstance(remote, dict):
|
|
4015
|
+
raise ValueError( # noqa: TRY004 - normalize untrusted state into a stable CLI error
|
|
4016
|
+
"standalone application state readback is invalid"
|
|
4017
|
+
)
|
|
4018
|
+
expected_serial = expected.get("serial")
|
|
4019
|
+
if type(expected_serial) is not int or remote.get("serial") != expected_serial + 1:
|
|
4020
|
+
raise ValueError("standalone application remote state serial differs after adoption")
|
|
4021
|
+
remote_serial = remote["serial"]
|
|
4022
|
+
remote_lineage = remote.get("lineage")
|
|
4023
|
+
if not isinstance(remote_lineage, str) or remote_lineage != expected.get("lineage"):
|
|
4024
|
+
raise ValueError("standalone application remote state lineage differs after adoption")
|
|
4025
|
+
expected.pop("serial", None)
|
|
4026
|
+
remote.pop("serial", None)
|
|
4027
|
+
expected.pop("check_results", None)
|
|
4028
|
+
remote.pop("check_results", None)
|
|
4029
|
+
if canonical_digest(expected) != canonical_digest(remote):
|
|
4030
|
+
raise ValueError("standalone application remote state differs after adoption")
|
|
4031
|
+
count = sum(
|
|
4032
|
+
len(resource.get("instances", []))
|
|
4033
|
+
for resource in remote.get("resources", [])
|
|
4034
|
+
if isinstance(resource, dict) and resource.get("mode") == "managed"
|
|
4035
|
+
)
|
|
4036
|
+
if count != descriptor["managed_resource_count"]:
|
|
4037
|
+
raise ValueError("standalone application remote state count differs")
|
|
4038
|
+
return canonical_digest(remote), remote_lineage, remote_serial
|
|
4039
|
+
|
|
4040
|
+
|
|
4041
|
+
def _verify_remote_application_state_continuity(infra: Path, receipt: dict[str, Any]) -> None:
|
|
4042
|
+
remote = json.loads(
|
|
4043
|
+
_capture(
|
|
4044
|
+
("terraform", "state", "pull"),
|
|
4045
|
+
cwd=infra,
|
|
4046
|
+
timeout=300,
|
|
4047
|
+
reason="standalone application remote state continuity readback failed",
|
|
4048
|
+
)
|
|
4049
|
+
)
|
|
4050
|
+
if (
|
|
4051
|
+
not isinstance(remote, dict)
|
|
4052
|
+
or remote.get("lineage") != receipt.get("remote_state_lineage")
|
|
4053
|
+
or type(remote.get("serial")) is not int
|
|
4054
|
+
or type(receipt.get("remote_state_serial")) is not int
|
|
4055
|
+
or remote["serial"] < receipt["remote_state_serial"]
|
|
4056
|
+
):
|
|
4057
|
+
raise ValueError("standalone application remote state continuity differs")
|
|
4058
|
+
|
|
4059
|
+
|
|
4060
|
+
def _remove_adoption_inputs(*paths: Path) -> None:
|
|
4061
|
+
for path in paths:
|
|
4062
|
+
path.unlink(missing_ok=True)
|
|
4063
|
+
|
|
4064
|
+
|
|
4065
|
+
def _deployment_binding(_args: argparse.Namespace, work_dir: Path) -> dict[str, object]:
|
|
4066
|
+
if not (work_dir / "substrate-receipt.json").is_file():
|
|
4067
|
+
raise ValueError("deployment binding requires verified substrate")
|
|
4068
|
+
context = _private_json(work_dir / "context.json", "standalone host context")
|
|
4069
|
+
_managed_identity_login_from_context(context, work_dir)
|
|
4070
|
+
installation_binding = _terraform_output(Path(str(context["infra"])), "installation_binding")
|
|
4071
|
+
if _runtime_platform(context) == "aks":
|
|
4072
|
+
_activate_terraform_stage("runtime", context, work_dir)
|
|
4073
|
+
runtime_name = _terraform_output(Path(str(context["runtime_infra"])), "cluster_name")
|
|
4074
|
+
else:
|
|
4075
|
+
runtime_name = _terraform_output(Path(str(context["infra"])), "core_app_name")
|
|
4076
|
+
deployment_binding = deployment_binding_digest(context, runtime_name)
|
|
4077
|
+
return {
|
|
4078
|
+
"schema_version": "fdai.standalone-deployment-binding.v1",
|
|
4079
|
+
"state": "verified",
|
|
4080
|
+
"deployment_binding": deployment_binding,
|
|
4081
|
+
"installation_binding": installation_binding,
|
|
4082
|
+
"terraform_name_verified": True,
|
|
4083
|
+
"mutation_performed": False,
|
|
4084
|
+
"subscription_ready": False,
|
|
4085
|
+
}
|
|
4086
|
+
|
|
4087
|
+
|
|
4088
|
+
def _managed_identity_login_from_context(context: dict[str, object], work_dir: Path) -> None:
|
|
4089
|
+
prepare_managed_identity_login(context, work_dir, login=_managed_identity_login)
|
|
4090
|
+
|
|
4091
|
+
|
|
4092
|
+
def _managed_identity_login(
|
|
4093
|
+
subscription: str, tenant: str, client_id: str, principal_id: str, work_dir: Path
|
|
4094
|
+
) -> None:
|
|
4095
|
+
config = work_dir / "azure"
|
|
4096
|
+
config.mkdir(mode=0o700, exist_ok=True)
|
|
4097
|
+
os.environ["AZURE_CONFIG_DIR"] = str(config)
|
|
4098
|
+
_run(
|
|
4099
|
+
(
|
|
4100
|
+
"az",
|
|
4101
|
+
"login",
|
|
4102
|
+
"--identity",
|
|
4103
|
+
"--client-id",
|
|
4104
|
+
client_id,
|
|
4105
|
+
"--allow-no-subscriptions",
|
|
4106
|
+
"--output",
|
|
4107
|
+
"none",
|
|
4108
|
+
"--only-show-errors",
|
|
4109
|
+
),
|
|
4110
|
+
cwd=work_dir,
|
|
4111
|
+
timeout=120,
|
|
4112
|
+
reason="managed identity login failed",
|
|
4113
|
+
)
|
|
4114
|
+
_run(
|
|
4115
|
+
("az", "account", "set", "--subscription", subscription),
|
|
4116
|
+
cwd=work_dir,
|
|
4117
|
+
timeout=60,
|
|
4118
|
+
reason="managed identity subscription selection failed",
|
|
4119
|
+
)
|
|
4120
|
+
value = json.loads(
|
|
4121
|
+
_capture(
|
|
4122
|
+
(
|
|
4123
|
+
"az",
|
|
4124
|
+
"account",
|
|
4125
|
+
"show",
|
|
4126
|
+
"--subscription",
|
|
4127
|
+
subscription,
|
|
4128
|
+
"--query",
|
|
4129
|
+
"{id:id,tenantId:tenantId}",
|
|
4130
|
+
"--output",
|
|
4131
|
+
"json",
|
|
4132
|
+
"--only-show-errors",
|
|
4133
|
+
),
|
|
4134
|
+
cwd=work_dir,
|
|
4135
|
+
timeout=60,
|
|
4136
|
+
reason="managed identity target readback failed",
|
|
4137
|
+
)
|
|
4138
|
+
)
|
|
4139
|
+
if (
|
|
4140
|
+
value.get("id", "").casefold() != subscription.casefold()
|
|
4141
|
+
or value.get("tenantId", "").casefold() != tenant.casefold()
|
|
4142
|
+
):
|
|
4143
|
+
raise ValueError("managed identity target differs from the Foundation handoff")
|
|
4144
|
+
token = _capture(
|
|
4145
|
+
(
|
|
4146
|
+
"az",
|
|
4147
|
+
"account",
|
|
4148
|
+
"get-access-token",
|
|
4149
|
+
"--subscription",
|
|
4150
|
+
subscription,
|
|
4151
|
+
"--resource",
|
|
4152
|
+
"https://management.azure.com/",
|
|
4153
|
+
"--query",
|
|
4154
|
+
"accessToken",
|
|
4155
|
+
"--output",
|
|
4156
|
+
"tsv",
|
|
4157
|
+
"--only-show-errors",
|
|
4158
|
+
),
|
|
4159
|
+
cwd=work_dir,
|
|
4160
|
+
timeout=60,
|
|
4161
|
+
reason="managed identity token readback failed",
|
|
4162
|
+
).strip()
|
|
4163
|
+
parts = token.split(".")
|
|
4164
|
+
try:
|
|
4165
|
+
claims = json.loads(base64.urlsafe_b64decode(parts[1] + "=" * (-len(parts[1]) % 4)))
|
|
4166
|
+
except (IndexError, ValueError, json.JSONDecodeError) as exc:
|
|
4167
|
+
raise ValueError("managed identity token readback is invalid") from exc
|
|
4168
|
+
if (
|
|
4169
|
+
not isinstance(claims, dict)
|
|
4170
|
+
or str(claims.get("oid", "")).casefold() != principal_id.casefold()
|
|
4171
|
+
):
|
|
4172
|
+
raise ValueError("managed identity principal differs from the Foundation handoff")
|
|
4173
|
+
|
|
4174
|
+
|
|
4175
|
+
def _validate_approval(
|
|
4176
|
+
review: dict[str, object],
|
|
4177
|
+
approval: dict[str, object],
|
|
4178
|
+
*,
|
|
4179
|
+
context: dict[str, object],
|
|
4180
|
+
) -> None:
|
|
4181
|
+
expected = {
|
|
4182
|
+
"schema_version",
|
|
4183
|
+
"stage",
|
|
4184
|
+
"plan_digest",
|
|
4185
|
+
"review_digest",
|
|
4186
|
+
"target_binding",
|
|
4187
|
+
"source_commit",
|
|
4188
|
+
"actor_digest",
|
|
4189
|
+
"approved_at",
|
|
4190
|
+
"expires_at",
|
|
4191
|
+
}
|
|
4192
|
+
review_document = {key: value for key, value in review.items() if key != "review_digest"}
|
|
4193
|
+
now = datetime.now(UTC)
|
|
4194
|
+
try:
|
|
4195
|
+
review_expiry = _parse_moment(str(review["expires_at"]))
|
|
4196
|
+
approved_at = _parse_moment(str(approval["approved_at"]))
|
|
4197
|
+
approval_expiry = _parse_moment(str(approval["expires_at"]))
|
|
4198
|
+
except KeyError as exc:
|
|
4199
|
+
raise ValueError("standalone application approval is invalid or expired") from exc
|
|
4200
|
+
if (
|
|
4201
|
+
review.get("schema_version") != "fdai.standalone-application-plan.v1"
|
|
4202
|
+
or review.get("review_digest") != canonical_digest(review_document)
|
|
4203
|
+
or review.get("target_binding") != context.get("target_binding")
|
|
4204
|
+
or review.get("source_commit") != context.get("source_commit")
|
|
4205
|
+
or _runtime_profile_digest(review) != _runtime_profile_digest(context)
|
|
4206
|
+
or review_expiry <= now
|
|
4207
|
+
or set(approval) != expected
|
|
4208
|
+
or approval.get("schema_version") != "fdai.standalone-plan-approval.v1"
|
|
4209
|
+
or approval.get("stage") != review.get("stage")
|
|
4210
|
+
or approval.get("plan_digest") != review.get("plan_digest")
|
|
4211
|
+
or approval.get("review_digest") != review.get("review_digest")
|
|
4212
|
+
or approval.get("target_binding") != review.get("target_binding")
|
|
4213
|
+
or approval.get("source_commit") != review.get("source_commit")
|
|
4214
|
+
or not isinstance(approval.get("actor_digest"), str)
|
|
4215
|
+
or _DIGEST.fullmatch(str(approval["actor_digest"])) is None
|
|
4216
|
+
or approved_at > now + timedelta(minutes=5)
|
|
4217
|
+
or approval_expiry <= now
|
|
4218
|
+
or approval_expiry > review_expiry
|
|
4219
|
+
or approval_expiry > approved_at + timedelta(hours=1)
|
|
4220
|
+
):
|
|
4221
|
+
raise ValueError("standalone application approval is invalid or expired")
|
|
4222
|
+
|
|
4223
|
+
|
|
4224
|
+
def _readback_stage(stage: str, context: dict[str, object]) -> bool:
|
|
4225
|
+
if stage == "access":
|
|
4226
|
+
if not _focused_private_access(context):
|
|
4227
|
+
raise ValueError("focused private-access readback was not selected")
|
|
4228
|
+
cwd = Path(str(context["infra"]))
|
|
4229
|
+
if context["key_vault_private_access"] is True:
|
|
4230
|
+
_capture(
|
|
4231
|
+
(
|
|
4232
|
+
"az",
|
|
4233
|
+
"keyvault",
|
|
4234
|
+
"secret",
|
|
4235
|
+
"list",
|
|
4236
|
+
"--vault-name",
|
|
4237
|
+
str(context["key_vault_name"]),
|
|
4238
|
+
"--output",
|
|
4239
|
+
"none",
|
|
4240
|
+
"--only-show-errors",
|
|
4241
|
+
),
|
|
4242
|
+
cwd=cwd,
|
|
4243
|
+
timeout=120,
|
|
4244
|
+
reason="focused Key Vault private-access readback failed",
|
|
4245
|
+
)
|
|
4246
|
+
if context["document_storage_private_access"] is True:
|
|
4247
|
+
_capture(
|
|
4248
|
+
(
|
|
4249
|
+
"az",
|
|
4250
|
+
"storage",
|
|
4251
|
+
"fs",
|
|
4252
|
+
"list",
|
|
4253
|
+
"--account-name",
|
|
4254
|
+
str(context["document_storage_account_name"]),
|
|
4255
|
+
"--auth-mode",
|
|
4256
|
+
"login",
|
|
4257
|
+
"--output",
|
|
4258
|
+
"none",
|
|
4259
|
+
"--only-show-errors",
|
|
4260
|
+
),
|
|
4261
|
+
cwd=cwd,
|
|
4262
|
+
timeout=120,
|
|
4263
|
+
reason="focused document storage private-access readback failed",
|
|
4264
|
+
)
|
|
4265
|
+
return True
|
|
4266
|
+
if stage == "substrate":
|
|
4267
|
+
infra = Path(str(context["infra"]))
|
|
4268
|
+
value = _capture(
|
|
4269
|
+
(
|
|
4270
|
+
"az",
|
|
4271
|
+
"acr",
|
|
4272
|
+
"show",
|
|
4273
|
+
*_terraform_registry_selector(infra, str(context["registry_name"])),
|
|
4274
|
+
"--query",
|
|
4275
|
+
"provisioningState",
|
|
4276
|
+
"--output",
|
|
4277
|
+
"tsv",
|
|
4278
|
+
"--only-show-errors",
|
|
4279
|
+
),
|
|
4280
|
+
cwd=infra,
|
|
4281
|
+
timeout=60,
|
|
4282
|
+
reason="standalone substrate ACR readback failed",
|
|
4283
|
+
)
|
|
4284
|
+
return value.strip() == "Succeeded"
|
|
4285
|
+
if stage == "database":
|
|
4286
|
+
kubeconfig = Path(str(context.get("kubeconfig", "")))
|
|
4287
|
+
if not kubeconfig.is_file():
|
|
4288
|
+
raise ValueError("AKS kubeconfig is unavailable for database readback")
|
|
4289
|
+
kubectl, _kubelogin, environment = standalone_kubernetes_tools.tools(context)
|
|
4290
|
+
result = subprocess.run(
|
|
4291
|
+
(
|
|
4292
|
+
kubectl,
|
|
4293
|
+
"rollout",
|
|
4294
|
+
"status",
|
|
4295
|
+
"statefulset/postgres",
|
|
4296
|
+
"--namespace",
|
|
4297
|
+
"fdai-data",
|
|
4298
|
+
"--timeout=5m",
|
|
4299
|
+
f"--kubeconfig={kubeconfig}",
|
|
4300
|
+
),
|
|
4301
|
+
check=False,
|
|
4302
|
+
capture_output=True,
|
|
4303
|
+
timeout=360,
|
|
4304
|
+
env=environment,
|
|
4305
|
+
)
|
|
4306
|
+
return result.returncode == 0
|
|
4307
|
+
if stage == "runtime":
|
|
4308
|
+
value = _capture(
|
|
4309
|
+
(
|
|
4310
|
+
"az",
|
|
4311
|
+
"aks",
|
|
4312
|
+
"show",
|
|
4313
|
+
"--resource-group",
|
|
4314
|
+
str(context["resource_group_name"]),
|
|
4315
|
+
"--name",
|
|
4316
|
+
_terraform_output(Path(str(context["runtime_infra"])), "cluster_name"),
|
|
4317
|
+
"--subscription",
|
|
4318
|
+
str(context["subscription_id"]),
|
|
4319
|
+
"--query",
|
|
4320
|
+
"provisioningState",
|
|
4321
|
+
"--output",
|
|
4322
|
+
"tsv",
|
|
4323
|
+
"--only-show-errors",
|
|
4324
|
+
),
|
|
4325
|
+
cwd=Path(str(context["runtime_infra"])),
|
|
4326
|
+
timeout=60,
|
|
4327
|
+
reason="standalone AKS cluster readback failed",
|
|
4328
|
+
)
|
|
4329
|
+
return value.strip() == "Succeeded"
|
|
4330
|
+
if _runtime_platform(context) == "aks":
|
|
4331
|
+
kubeconfig = Path(str(context.get("kubeconfig", "")))
|
|
4332
|
+
if not kubeconfig.is_file():
|
|
4333
|
+
raise ValueError("AKS kubeconfig is unavailable for workload readback")
|
|
4334
|
+
kubectl, _kubelogin, environment = standalone_kubernetes_tools.tools(context)
|
|
4335
|
+
expected = _mapping(context.get("expected_workloads"), "expected AKS workloads")
|
|
4336
|
+
if "core-control-plane" not in expected:
|
|
4337
|
+
return False
|
|
4338
|
+
observed = []
|
|
4339
|
+
for resource in ("deployments", "pods"):
|
|
4340
|
+
observed.append(
|
|
4341
|
+
_capture_env(
|
|
4342
|
+
(
|
|
4343
|
+
kubectl,
|
|
4344
|
+
"get",
|
|
4345
|
+
resource,
|
|
4346
|
+
"--namespace",
|
|
4347
|
+
"fdai-runtime",
|
|
4348
|
+
"--output",
|
|
4349
|
+
"json",
|
|
4350
|
+
"--request-timeout=60s",
|
|
4351
|
+
f"--kubeconfig={kubeconfig}",
|
|
4352
|
+
),
|
|
4353
|
+
cwd=kubeconfig.parent,
|
|
4354
|
+
timeout=90,
|
|
4355
|
+
reason="AKS workload observation failed",
|
|
4356
|
+
env=environment,
|
|
4357
|
+
)
|
|
4358
|
+
)
|
|
4359
|
+
return aks_readiness.verify_workload_health(
|
|
4360
|
+
deployments=observed[0],
|
|
4361
|
+
pods=observed[1],
|
|
4362
|
+
expected=expected,
|
|
4363
|
+
source_commit=str(context["source_commit"]),
|
|
4364
|
+
)
|
|
4365
|
+
return _container_app_health(context, Path(str(context["infra"])))
|
|
4366
|
+
|
|
4367
|
+
|
|
4368
|
+
def _runtime_profile_digest(context: dict[str, object]) -> str:
|
|
4369
|
+
"""Return the sealed profile digest, treating legacy context as the ACA default."""
|
|
4370
|
+
|
|
4371
|
+
value = context.get("runtime_profile_digest")
|
|
4372
|
+
if value is None:
|
|
4373
|
+
return legacy_runtime_profile_digest(
|
|
4374
|
+
RuntimeDeploymentProfile.create(
|
|
4375
|
+
runtime_platform="container-apps",
|
|
4376
|
+
database_placement="postgres-flex",
|
|
4377
|
+
)
|
|
4378
|
+
)
|
|
4379
|
+
if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
|
|
4380
|
+
raise ValueError("standalone runtime profile digest is invalid")
|
|
4381
|
+
return value
|
|
4382
|
+
|
|
4383
|
+
|
|
4384
|
+
def _runtime_profile_matches(
|
|
4385
|
+
context: dict[str, object],
|
|
4386
|
+
profile: RuntimeDeploymentProfile,
|
|
4387
|
+
) -> bool:
|
|
4388
|
+
retained = _runtime_profile_digest(context)
|
|
4389
|
+
if retained == profile.digest:
|
|
4390
|
+
return True
|
|
4391
|
+
return legacy_full_product_matches(profile, retained)
|
|
4392
|
+
|
|
4393
|
+
|
|
4394
|
+
def _container_app_health(context: dict[str, object], infra: Path) -> bool:
|
|
4395
|
+
resource_group = _terraform_output(infra, "resource_group_name")
|
|
4396
|
+
names = (_terraform_output(infra, "core_app_name"),)
|
|
4397
|
+
for name in names:
|
|
4398
|
+
app = json.loads(
|
|
4399
|
+
_capture(
|
|
4400
|
+
(
|
|
4401
|
+
"az",
|
|
4402
|
+
"containerapp",
|
|
4403
|
+
"show",
|
|
4404
|
+
"--resource-group",
|
|
4405
|
+
resource_group,
|
|
4406
|
+
"--name",
|
|
4407
|
+
name,
|
|
4408
|
+
"--output",
|
|
4409
|
+
"json",
|
|
4410
|
+
"--only-show-errors",
|
|
4411
|
+
),
|
|
4412
|
+
cwd=infra,
|
|
4413
|
+
timeout=60,
|
|
4414
|
+
reason="standalone runtime health readback failed",
|
|
4415
|
+
)
|
|
4416
|
+
)
|
|
4417
|
+
properties = app.get("properties") if isinstance(app, dict) else None
|
|
4418
|
+
revision = (
|
|
4419
|
+
properties.get("latestReadyRevisionName") if isinstance(properties, dict) else None
|
|
4420
|
+
)
|
|
4421
|
+
if (
|
|
4422
|
+
not isinstance(properties, dict)
|
|
4423
|
+
or properties.get("provisioningState") != "Succeeded"
|
|
4424
|
+
or not isinstance(revision, str)
|
|
4425
|
+
or not revision
|
|
4426
|
+
):
|
|
4427
|
+
return False
|
|
4428
|
+
observed = json.loads(
|
|
4429
|
+
_capture(
|
|
4430
|
+
(
|
|
4431
|
+
"az",
|
|
4432
|
+
"containerapp",
|
|
4433
|
+
"revision",
|
|
4434
|
+
"show",
|
|
4435
|
+
"--resource-group",
|
|
4436
|
+
resource_group,
|
|
4437
|
+
"--name",
|
|
4438
|
+
name,
|
|
4439
|
+
"--revision",
|
|
4440
|
+
revision,
|
|
4441
|
+
"--output",
|
|
4442
|
+
"json",
|
|
4443
|
+
"--only-show-errors",
|
|
4444
|
+
),
|
|
4445
|
+
cwd=infra,
|
|
4446
|
+
timeout=60,
|
|
4447
|
+
reason="standalone runtime revision readback failed",
|
|
4448
|
+
)
|
|
4449
|
+
)
|
|
4450
|
+
revision_properties = observed.get("properties") if isinstance(observed, dict) else None
|
|
4451
|
+
if (
|
|
4452
|
+
not isinstance(revision_properties, dict)
|
|
4453
|
+
or revision_properties.get("active") is not True
|
|
4454
|
+
or revision_properties.get("provisioningState") != "Provisioned"
|
|
4455
|
+
or revision_properties.get("healthState") != "Healthy"
|
|
4456
|
+
):
|
|
4457
|
+
return False
|
|
4458
|
+
return True
|
|
4459
|
+
|
|
4460
|
+
|
|
4461
|
+
def _subnet_network_security_group(
|
|
4462
|
+
subnet_id: str,
|
|
4463
|
+
*,
|
|
4464
|
+
context: dict[str, object],
|
|
4465
|
+
cwd: Path,
|
|
4466
|
+
) -> str:
|
|
4467
|
+
"""Read the optional policy-attached NSG ID for one exact AKS subnet."""
|
|
4468
|
+
|
|
4469
|
+
subnet_match = re.fullmatch(
|
|
4470
|
+
r"/subscriptions/([^/]+)/resourceGroups/[^/]+/providers/"
|
|
4471
|
+
r"Microsoft[.]Network/virtualNetworks/[^/]+/subnets/[^/]+",
|
|
4472
|
+
subnet_id,
|
|
4473
|
+
flags=re.IGNORECASE,
|
|
4474
|
+
)
|
|
4475
|
+
if subnet_match is None:
|
|
4476
|
+
raise ValueError("AKS subnet resource ID is invalid")
|
|
4477
|
+
subscription_id = str(context["subscription_id"])
|
|
4478
|
+
if subnet_match.group(1).casefold() != subscription_id.casefold():
|
|
4479
|
+
raise ValueError("AKS subnet belongs to a different subscription")
|
|
4480
|
+
value = _capture(
|
|
4481
|
+
(
|
|
4482
|
+
"az",
|
|
4483
|
+
"network",
|
|
4484
|
+
"vnet",
|
|
4485
|
+
"subnet",
|
|
4486
|
+
"show",
|
|
4487
|
+
"--ids",
|
|
4488
|
+
subnet_id,
|
|
4489
|
+
"--subscription",
|
|
4490
|
+
subscription_id,
|
|
4491
|
+
"--query",
|
|
4492
|
+
"networkSecurityGroup.id",
|
|
4493
|
+
"--output",
|
|
4494
|
+
"tsv",
|
|
4495
|
+
"--only-show-errors",
|
|
4496
|
+
),
|
|
4497
|
+
cwd=cwd,
|
|
4498
|
+
timeout=60,
|
|
4499
|
+
reason="AKS subnet network security group readback failed",
|
|
4500
|
+
).strip()
|
|
4501
|
+
if value:
|
|
4502
|
+
nsg_match = re.fullmatch(
|
|
4503
|
+
r"/subscriptions/([^/]+)/resourceGroups/[^/]+/providers/"
|
|
4504
|
+
r"Microsoft[.]Network/networkSecurityGroups/[^/]+",
|
|
4505
|
+
value,
|
|
4506
|
+
flags=re.IGNORECASE,
|
|
4507
|
+
)
|
|
4508
|
+
if nsg_match is None:
|
|
4509
|
+
raise ValueError("AKS subnet network security group ID is invalid")
|
|
4510
|
+
if nsg_match.group(1).casefold() != subscription_id.casefold():
|
|
4511
|
+
raise ValueError("AKS subnet network security group belongs to another subscription")
|
|
4512
|
+
return value
|
|
4513
|
+
|
|
4514
|
+
|
|
4515
|
+
def _terraform_output(infra: Path, name: str) -> str:
|
|
4516
|
+
value = standalone_planned_outputs.read_output(
|
|
4517
|
+
infra, name, raw=True, reason="Terraform output readback failed"
|
|
4518
|
+
)
|
|
4519
|
+
return str(value)
|
|
4520
|
+
|
|
4521
|
+
|
|
4522
|
+
def _terraform_registry_selector(infra: Path, expected_name: str) -> tuple[str, ...]:
|
|
4523
|
+
registry_id = _terraform_output(infra, "container_registry_id")
|
|
4524
|
+
return standalone_host_values.registry_selector(registry_id, expected_name)
|
|
4525
|
+
|
|
4526
|
+
|
|
4527
|
+
def _terraform_json_output(infra: Path, name: str) -> object:
|
|
4528
|
+
return standalone_planned_outputs.read_output(
|
|
4529
|
+
infra, name, raw=False, reason="Terraform JSON output readback failed"
|
|
4530
|
+
)
|
|
4531
|
+
|
|
4532
|
+
|
|
4533
|
+
def _run(
|
|
4534
|
+
command: tuple[str, ...] | list[str],
|
|
4535
|
+
*,
|
|
4536
|
+
cwd: Path,
|
|
4537
|
+
timeout: int,
|
|
4538
|
+
reason: str,
|
|
4539
|
+
) -> None:
|
|
4540
|
+
result = subprocess.run(command, cwd=cwd, check=False, capture_output=True, timeout=timeout)
|
|
4541
|
+
if result.returncode != 0:
|
|
4542
|
+
raise (
|
|
4543
|
+
_terraform_failure(reason, result) if "terraform" in command[0] else ValueError(reason)
|
|
4544
|
+
)
|
|
4545
|
+
|
|
4546
|
+
|
|
4547
|
+
def _capture(
|
|
4548
|
+
command: tuple[str, ...],
|
|
4549
|
+
*,
|
|
4550
|
+
cwd: Path,
|
|
4551
|
+
timeout: int,
|
|
4552
|
+
reason: str,
|
|
4553
|
+
) -> str:
|
|
4554
|
+
result = subprocess.run(
|
|
4555
|
+
command, cwd=cwd, check=False, capture_output=True, text=True, timeout=timeout
|
|
4556
|
+
)
|
|
4557
|
+
if result.returncode != 0:
|
|
4558
|
+
raise (
|
|
4559
|
+
_terraform_failure(reason, result) if "terraform" in command[0] else ValueError(reason)
|
|
4560
|
+
)
|
|
4561
|
+
return result.stdout
|
|
4562
|
+
|
|
4563
|
+
|
|
4564
|
+
def _run_env(
|
|
4565
|
+
command: tuple[str, ...],
|
|
4566
|
+
*,
|
|
4567
|
+
cwd: Path,
|
|
4568
|
+
env: dict[str, str],
|
|
4569
|
+
timeout: int,
|
|
4570
|
+
reason: str,
|
|
4571
|
+
) -> None:
|
|
4572
|
+
result = subprocess.run(
|
|
4573
|
+
command,
|
|
4574
|
+
cwd=cwd,
|
|
4575
|
+
env=env,
|
|
4576
|
+
check=False,
|
|
4577
|
+
stdout=subprocess.DEVNULL,
|
|
4578
|
+
stderr=subprocess.DEVNULL,
|
|
4579
|
+
timeout=timeout,
|
|
4580
|
+
)
|
|
4581
|
+
if result.returncode != 0:
|
|
4582
|
+
raise ValueError(reason)
|
|
4583
|
+
|
|
4584
|
+
|
|
4585
|
+
def _capture_env(
|
|
4586
|
+
command: tuple[str, ...],
|
|
4587
|
+
*,
|
|
4588
|
+
cwd: Path,
|
|
4589
|
+
env: dict[str, str],
|
|
4590
|
+
timeout: int,
|
|
4591
|
+
reason: str,
|
|
4592
|
+
) -> str:
|
|
4593
|
+
result = subprocess.run(
|
|
4594
|
+
command,
|
|
4595
|
+
cwd=cwd,
|
|
4596
|
+
env=env,
|
|
4597
|
+
check=False,
|
|
4598
|
+
capture_output=True,
|
|
4599
|
+
text=True,
|
|
4600
|
+
timeout=timeout,
|
|
4601
|
+
)
|
|
4602
|
+
if result.returncode != 0:
|
|
4603
|
+
raise ValueError(reason)
|
|
4604
|
+
return result.stdout
|
|
4605
|
+
|
|
4606
|
+
|
|
4607
|
+
if __name__ == "__main__":
|
|
4608
|
+
raise SystemExit(main())
|