thinkwork-cli 0.13.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -14
- package/dist/{api-client-4DZ266J5.js → api-client-4P4JZOBX.js} +3 -1
- package/dist/bowser-6Z6JFAIR.js +2822 -0
- package/dist/{chunk-Y2WHJECR.js → chunk-4OMDWYZQ.js} +5 -44
- package/dist/chunk-AD2NIWB5.js +6516 -0
- package/dist/chunk-DV3MRBR5.js +43 -0
- package/dist/chunk-EI5ZX7CN.js +522 -0
- package/dist/chunk-EUJ7X6YD.js +126 -0
- package/dist/chunk-GG5WKBOX.js +436 -0
- package/dist/chunk-I64EZSU5.js +358 -0
- package/dist/chunk-KMIJ5KGT.js +128 -0
- package/dist/chunk-MWQ7RO76.js +137 -0
- package/dist/chunk-OCPHU34C.js +996 -0
- package/dist/chunk-OLUQ4U2Z.js +1728 -0
- package/dist/chunk-XYM7PE2A.js +2264 -0
- package/dist/chunk-YQCYD6NE.js +3299 -0
- package/dist/cli.js +49156 -31257
- package/dist/commands/enterprise/templates/deploy-repo/scripts/apply-release.mjs +0 -1
- package/dist/commands/enterprise/templates/deploy-repo/scripts/smoke.mjs +0 -1
- package/dist/commands/enterprise/templates/deploy-repo/terraform/main.tf +0 -14
- package/dist/dist-es-2OTNTL4N.js +315 -0
- package/dist/dist-es-2XVAH4BQ.js +14726 -0
- package/dist/dist-es-3DPDUJAH.js +2000 -0
- package/dist/dist-es-BNZTBBVC.js +176 -0
- package/dist/dist-es-CD2RBDAC.js +88 -0
- package/dist/dist-es-I4FW57PS.js +88 -0
- package/dist/dist-es-KL7Y5NB2.js +377 -0
- package/dist/dist-es-KVFWJNDY.js +318 -0
- package/dist/dist-es-PS73DWNL.js +70 -0
- package/dist/dist-es-S3NZQFGT.js +170 -0
- package/dist/dist-es-SGCA4P32.js +48 -0
- package/dist/dist-es-WBM6J4RA.js +70 -0
- package/dist/dist-es-ZGVADYIF.js +485 -0
- package/dist/dist-es-ZKC2QKD2.js +377 -0
- package/dist/drizzle/0076_scheduled_jobs_marco_backfill.sql +2 -1
- package/dist/drizzle/0079_seed_tenant_customize_catalog.sql +1 -0
- package/dist/drizzle/0090_brain_schema_extraction.sql +10 -0
- package/dist/drizzle/0132_drop_computer_tables.sql +3 -0
- package/dist/drizzle/0160_compliance_event_types_plugins.sql +2 -0
- package/dist/drizzle/0214_think137_drop_judge_evidence_roi.sql +1 -0
- package/dist/drizzle/0216_drop_agent_loop_versions_goal_worker_policy.sql +1 -0
- package/dist/drizzle/0227_analyst_reader_role.sql +17 -20
- package/dist/drizzle/0230_analyst_rls.sql +28 -36
- package/dist/drizzle/0250_kg_schema_extraction.sql +27 -0
- package/dist/drizzle/0251_brain_stragglers.sql +21 -0
- package/dist/drizzle/0253_drop_brain_substrate.sql +2 -0
- package/dist/drizzle/0257_agents_runtime_allow_harness.sql +24 -0
- package/dist/drizzle/0261_native_auth_control_plane.sql +332 -0
- package/dist/drizzle/0262_auth_subscription_tickets.sql +59 -0
- package/dist/drizzle/0263_agentcore_runtime_identifier.sql +69 -0
- package/dist/drizzle/0263_drop_workos_auth_runtime.sql +236 -0
- package/dist/drizzle/0264_auth_enrollment_attempt_limits.sql +15 -0
- package/dist/drizzle/0265_eval_runs_requester_identity.sql +28 -0
- package/dist/drizzle/0265_require_fully_valid_auth_publication.sql +27 -0
- package/dist/drizzle/0266_auth_identity_recovery_grants.sql +9 -0
- package/dist/drizzle/0266_eval_profiles_runtime_type.sql +12 -0
- package/dist/drizzle/0267_ontology_candidate_rejections.sql +53 -0
- package/dist/drizzle/0268_identity_crosswalk_routing.sql +182 -0
- package/dist/drizzle/0269_ontology_system_map.sql +31 -0
- package/dist/drizzle/0270_tool_execution_ledger.sql +115 -0
- package/dist/drizzle/0271_ontology_twin_declarations.sql +47 -0
- package/dist/drizzle/0272_identity_graph_projection_cursor.sql +23 -0
- package/dist/drizzle/0273_twin_materialization_suggestions.sql +29 -0
- package/dist/drizzle/0274_tenant_mcp_twin_keys.sql +59 -0
- package/dist/drizzle/0275_identity_bulk_rebuild_fence.sql +30 -0
- package/dist/drizzle/0276_tool_execution_ledger_allow_cascade_delete.sql +18 -0
- package/dist/drizzle/0277_knowledge_base_sources.sql +107 -0
- package/dist/drizzle/0278_kb_page_transcription.sql +55 -0
- package/dist/drizzle/0279_mcp_servers_off_plugin_provenance.sql +93 -0
- package/dist/drizzle/0280_work_item_external_refs_drop_plane_provider.sql +46 -0
- package/dist/drizzle/0281_tenant_mcp_twin_keys_expiry_suffix.sql +9 -0
- package/dist/drizzle/0282_tenant_mcp_twin_keys_grants.sql +24 -0
- package/dist/drizzle/0283_agents_agentcore_runtime_dispatch.sql +13 -0
- package/dist/drizzle/0284_user_brain_claims.sql +69 -0
- package/dist/drizzle/0285_tenant_mcp_twin_keys_trusted_subsystem.sql +30 -0
- package/dist/drizzle/0286_brain_analytics_key.sql +26 -0
- package/dist/drizzle/0287_brain_subject.sql +25 -0
- package/dist/drizzle/0288_multi_lane_identities.sql +25 -0
- package/dist/event-streams-7JNLJEEL.js +894 -0
- package/dist/event-streams-KDF5RWR6.js +1515 -0
- package/dist/loadSso-ETCXTR4I.js +576 -0
- package/dist/loadSso-UBD7KUHE.js +583 -0
- package/dist/plugins/README.md +2 -2
- package/dist/plugins/catalog/package.json +1 -5
- package/dist/plugins/catalog/src/__tests__/build-catalog.test.ts +6 -4
- package/dist/plugins/catalog/src/__tests__/catalog.test.ts +21 -21
- package/dist/plugins/catalog/src/__tests__/contracts.test.ts +102 -94
- package/dist/plugins/catalog/src/__tests__/plugin-package.test.ts +7 -8
- package/dist/plugins/catalog/src/__tests__/plugin-registry.test.ts +0 -20
- package/dist/plugins/catalog/src/contracts.ts +17 -179
- package/dist/plugins/catalog/src/registry/generated-first-party.ts +0 -24
- package/dist/plugins/n8n/test/integrated-app-contract.test.ts +1 -1
- package/dist/plugins/twenty/package.json +2 -1
- package/dist/plugins/twenty/scripts/LAMBDA_DEPLOY.md +114 -0
- package/dist/plugins/twenty/scripts/build-lambda.sh +29 -0
- package/dist/plugins/twenty/scripts/configure-product-views.ts +195 -0
- package/dist/plugins/twenty/scripts/lambda-handler.ts +49 -0
- package/dist/plugins/twenty/scripts/lib/__tests__/load-attachments.test.ts +38 -0
- package/dist/plugins/twenty/scripts/lib/__tests__/load-records.test.ts +145 -2
- package/dist/plugins/twenty/scripts/lib/__tests__/mappers.test.ts +397 -151
- package/dist/plugins/twenty/scripts/lib/__tests__/members-ensure.test.ts +28 -129
- package/dist/plugins/twenty/scripts/lib/__tests__/provision-members-db.test.ts +10 -0
- package/dist/plugins/twenty/scripts/lib/__tests__/schema-ensure.test.ts +13 -8
- package/dist/plugins/twenty/scripts/lib/lastmile-reader.ts +222 -2
- package/dist/plugins/twenty/scripts/lib/load-attachments.ts +29 -0
- package/dist/plugins/twenty/scripts/lib/load-records.ts +160 -43
- package/dist/plugins/twenty/scripts/lib/mappers.ts +368 -152
- package/dist/plugins/twenty/scripts/lib/members-ensure.ts +43 -268
- package/dist/plugins/twenty/scripts/lib/provision-members-db.ts +57 -0
- package/dist/plugins/twenty/scripts/lib/schema-ensure.ts +136 -31
- package/dist/plugins/twenty/scripts/migrate-lastmile.ts +254 -152
- package/dist/plugins/twenty/scripts/provision-twenty-members.ts +44 -1
- package/dist/plugins/twenty/scripts/purge-lastmile-import.ts +178 -0
- package/dist/plugins/twenty/scripts/restructure-products.ts +312 -0
- package/dist/plugins/twenty/scripts/retag-stage-values.ts +362 -0
- package/dist/plugins/twenty/test/manifest.test.ts +2 -4
- package/dist/scripts/post-deploy.sh +79 -1
- package/dist/signin-5QZNRSOO.js +821 -0
- package/dist/sso-oidc-XK673P54.js +837 -0
- package/dist/sts-OXI2UALF.js +1113 -0
- package/dist/sts-SQW4NBK3.js +1117 -0
- package/dist/terraform/examples/greenfield/main.tf +391 -176
- package/dist/terraform/examples/greenfield/terraform.tfvars.example +26 -8
- package/dist/terraform/modules/app/agentcore-admin/main.tf +47 -0
- package/dist/terraform/modules/app/agentcore-code-interpreter/README.md +39 -47
- package/dist/terraform/modules/app/agentcore-code-interpreter/main.tf +13 -66
- package/dist/terraform/modules/app/agentcore-identity/main.tf +253 -0
- package/dist/terraform/modules/app/agentcore-identity/outputs.tf +51 -0
- package/dist/terraform/modules/app/agentcore-identity/scripts/bootstrap_twenty_oauth_client.sh +239 -0
- package/dist/terraform/modules/app/agentcore-identity/scripts/delete_identity.sh +22 -0
- package/dist/terraform/modules/app/agentcore-identity/scripts/delete_twenty_identity.sh +16 -0
- package/dist/terraform/modules/app/agentcore-identity/scripts/delete_workload_identity.sh +19 -0
- package/dist/terraform/modules/app/agentcore-identity/scripts/ensure_workload_identity.sh +29 -0
- package/dist/terraform/modules/app/agentcore-identity/scripts/read_identity.sh +55 -0
- package/dist/terraform/modules/app/agentcore-identity/scripts/reconcile_identity.sh +73 -0
- package/dist/terraform/modules/app/agentcore-identity/scripts/reconcile_twenty_identity.sh +45 -0
- package/dist/terraform/modules/app/agentcore-identity/scripts/reconcile_twenty_provider.mjs +123 -0
- package/dist/terraform/modules/app/agentcore-identity/tests/tainted-marker/after.tf +15 -0
- package/dist/terraform/modules/app/agentcore-identity/tests/tainted-marker/before.tf +7 -0
- package/dist/terraform/modules/app/agentcore-identity/tests/tainted-marker/identity-after.tf +8 -0
- package/dist/terraform/modules/app/agentcore-identity/tests/tainted-marker/identity-before.tf +9 -0
- package/dist/terraform/modules/app/agentcore-identity/tests/tainted-marker/thinkwork-after.tf +3 -0
- package/dist/terraform/modules/app/agentcore-identity/tests/tainted-marker/thinkwork-before.tf +3 -0
- package/dist/terraform/modules/app/agentcore-identity/variables.tf +60 -0
- package/dist/terraform/modules/app/agentcore-memory/README.md +46 -9
- package/dist/terraform/modules/app/agentcore-memory/main.tf +27 -10
- package/dist/terraform/modules/app/agentcore-memory/scripts/create_or_find_memory.sh +156 -24
- package/dist/terraform/modules/app/agentcore-pi/README.md +1 -3
- package/dist/terraform/modules/app/agentcore-pi/main.tf +42 -52
- package/dist/terraform/modules/app/agentcore-pi/outputs.tf +3 -3
- package/dist/terraform/modules/app/agentcore-pi/scripts/reconcile_pi_runtime.mjs +175 -0
- package/dist/terraform/modules/app/agentcore-pi/variables.tf +35 -28
- package/dist/terraform/modules/app/agentcore-platform/main.tf +68 -0
- package/dist/terraform/modules/app/agentcore-runtime/main.tf +4 -20
- package/dist/terraform/modules/app/appsync-subscriptions/main.tf +127 -21
- package/dist/terraform/modules/app/appsync-subscriptions/outputs.tf +0 -6
- package/dist/terraform/modules/app/appsync-subscriptions/variables.tf +5 -0
- package/dist/terraform/modules/app/capability-broker/main.tf +2 -2
- package/dist/terraform/modules/app/capability-broker/variables.tf +11 -0
- package/dist/terraform/modules/app/deployment-control-plane/README.md +264 -0
- package/dist/terraform/modules/app/deployment-control-plane/buildspec.yml +4 -1
- package/dist/terraform/modules/app/deployment-control-plane/main.tf +94 -0
- package/dist/terraform/modules/app/deployment-control-plane/outputs.tf +5 -0
- package/dist/terraform/modules/app/deployment-control-plane/runner.py +3689 -233
- package/dist/terraform/modules/app/deployment-control-plane/test_runner_bundle.py +2579 -140
- package/dist/terraform/modules/app/deployment-control-plane/variables.tf +33 -0
- package/dist/terraform/modules/app/lambda-api/auth-state.tf +72 -0
- package/dist/terraform/modules/app/lambda-api/chat-latency-observability.tf +466 -0
- package/dist/terraform/modules/app/lambda-api/dispatch-dlq.tf +55 -0
- package/dist/terraform/modules/app/lambda-api/handlers.tf +531 -619
- package/dist/terraform/modules/app/lambda-api/iam-grouped.tf +350 -241
- package/dist/terraform/modules/app/lambda-api/main.tf +75 -13
- package/dist/terraform/modules/app/lambda-api/memory-alarms.tf +3 -28
- package/dist/terraform/modules/app/lambda-api/oauth-secrets.tf +1 -2
- package/dist/terraform/modules/app/lambda-api/outputs.tf +17 -10
- package/dist/terraform/modules/app/lambda-api/runtime-config.tf +3 -21
- package/dist/terraform/modules/app/lambda-api/turn-assertion.tf +45 -0
- package/dist/terraform/modules/app/lambda-api/variables.tf +307 -121
- package/dist/{plugins/n8n/terraform → terraform/modules/app}/n8n/README.md +1 -1
- package/dist/{plugins/n8n/terraform → terraform/modules/app}/n8n/main.tf +18 -18
- package/dist/terraform/modules/app/sandbox-log-scrubber/README.md +11 -10
- package/dist/terraform/modules/app/sandbox-log-scrubber/main.tf +5 -5
- package/dist/terraform/modules/app/static-site/main.tf +1 -1
- package/dist/terraform/modules/app/www-dns/main.tf +9 -27
- package/dist/terraform/modules/app/www-dns/variables.tf +4 -10
- package/dist/terraform/modules/data/aurora-postgres/README.md +34 -0
- package/dist/terraform/modules/data/aurora-postgres/alarms.tf +82 -0
- package/dist/terraform/modules/data/aurora-postgres/main.tf +27 -57
- package/dist/terraform/modules/data/aurora-postgres/outputs.tf +0 -9
- package/dist/terraform/modules/data/aurora-postgres/variables.tf +41 -0
- package/dist/terraform/modules/data/s3-buckets/main.tf +5 -6
- package/dist/terraform/modules/foundation/cognito/main.tf +367 -72
- package/dist/terraform/modules/foundation/cognito/outputs.tf +47 -0
- package/dist/terraform/modules/foundation/cognito/variables.tf +131 -9
- package/dist/terraform/modules/thinkwork/README.md +5 -1
- package/dist/terraform/modules/thinkwork/main.tf +459 -234
- package/dist/terraform/modules/thinkwork/outputs.tf +44 -62
- package/dist/terraform/modules/thinkwork/variables.tf +353 -89
- package/dist/terraform/schema.graphql +69 -36
- package/dist/workspace-defaults/files/INSTRUCTIONS.md +9 -11
- package/dist/workspace-defaults/files/MEMORY_GUIDE.md +38 -35
- package/dist/workspace-defaults/files/ROUTER.md +1 -1
- package/dist/workspace-defaults/files/TOOLS.md +9 -39
- package/dist/workspace-defaults/files/USER.md +1 -0
- package/dist/workspace-defaults/files/agents/analyst/INSTRUCTIONS.md +6 -3
- package/dist/workspace-defaults/files/catalog-skills/brain-brief-builder/SKILL.md +165 -0
- package/dist/workspace-defaults/files/catalog-skills/n8n-workflow-operator/SKILL.md +75 -0
- package/dist/workspace-defaults/files/catalog-skills/n8n-workflow-operator/references/mcp-tooling.md +40 -0
- package/dist/workspace-defaults/files/catalog-skills/n8n-workflow-operator/references/validation-and-handoff.md +40 -0
- package/dist/workspace-defaults/files/catalog-skills/n8n-workflow-operator/references/workflow-authoring.md +46 -0
- package/dist/workspace-defaults/files/skills/artifact-builder/references/crm-dashboard.md +1 -1
- package/dist/workspace-defaults/files/skills/document-composer/SKILL.md +22 -3
- package/package.json +3 -2
- package/dist/drizzle/0261_thread_turns_origin_turn_index.sql +0 -11
- package/dist/plugins/company-data/README.md +0 -49
- package/dist/plugins/company-data/package.json +0 -21
- package/dist/plugins/company-data/src/index.ts +0 -30
- package/dist/plugins/company-data/src/manifest.ts +0 -26
- package/dist/plugins/company-data/test/manifest.test.ts +0 -116
- package/dist/plugins/company-data/tsconfig.json +0 -8
- package/dist/plugins/company-etl/README.md +0 -46
- package/dist/plugins/company-etl/package.json +0 -21
- package/dist/plugins/company-etl/src/index.ts +0 -29
- package/dist/plugins/company-etl/src/manifest.ts +0 -26
- package/dist/plugins/company-etl/test/manifest.test.ts +0 -108
- package/dist/plugins/company-etl/tsconfig.json +0 -8
- package/dist/plugins/lastmile/README.md +0 -50
- package/dist/plugins/lastmile/package.json +0 -23
- package/dist/plugins/lastmile/smoke/lastmile-plugin-smoke.mjs +0 -661
- package/dist/plugins/lastmile/src/api/tasks-adapter.ts +0 -572
- package/dist/plugins/lastmile/src/discovery.fixture.ts +0 -64
- package/dist/plugins/lastmile/src/index.ts +0 -46
- package/dist/plugins/lastmile/src/manifest.ts +0 -139
- package/dist/plugins/lastmile/test/api/tasks-adapter.test.ts +0 -266
- package/dist/plugins/lastmile/test/discovery.test.ts +0 -88
- package/dist/plugins/lastmile/tsconfig.json +0 -8
- package/dist/plugins/n8n/smoke/n8n-managed-app-smoke.mjs +0 -409
- package/dist/plugins/twenty/smoke/twenty-managed-app-smoke.mjs +0 -359
- package/dist/plugins/workos-auth/package.json +0 -22
- package/dist/plugins/workos-auth/src/index.ts +0 -30
- package/dist/plugins/workos-auth/src/manifest.ts +0 -40
- package/dist/plugins/workos-auth/src/provider-contract.ts +0 -39
- package/dist/plugins/workos-auth/test/manifest.test.ts +0 -94
- package/dist/plugins/workos-auth/tsconfig.json +0 -8
- package/dist/terraform/modules/app/agentcore-code-interpreter/Dockerfile.sandbox-base +0 -61
- package/dist/terraform/modules/app/agentcore-code-interpreter/sandbox/sitecustomize.py +0 -227
- package/dist/terraform/modules/app/agentcore-code-interpreter/sandbox/test_sitecustomize.py +0 -277
- package/dist/terraform/modules/app/agentcore-code-interpreter/scripts/build_and_push_sandbox_base.sh +0 -70
- package/dist/terraform/modules/app/agentcore-harness/main.tf +0 -101
- package/dist/terraform/modules/app/agentcore-harness/outputs.tf +0 -9
- package/dist/terraform/modules/app/agentcore-harness/variables.tf +0 -25
- package/dist/terraform/modules/app/hindsight-memory/README.md +0 -89
- package/dist/terraform/modules/app/hindsight-memory/main.tf +0 -411
- package/dist/terraform/modules/data/bedrock-knowledge-base/main.tf +0 -102
- /package/dist/{plugins/n8n/terraform → terraform/modules/app}/n8n/outputs.tf +0 -0
- /package/dist/{plugins/n8n/terraform → terraform/modules/app}/n8n/scripts/sync-database.py +0 -0
- /package/dist/{plugins/n8n/terraform → terraform/modules/app}/n8n/scripts/test_sync_database.py +0 -0
- /package/dist/{plugins/n8n/terraform → terraform/modules/app}/n8n/variables.tf +0 -0
- /package/dist/{plugins/twenty/terraform → terraform/modules/app}/twenty/README.md +0 -0
- /package/dist/{plugins/twenty/terraform → terraform/modules/app}/twenty/main.tf +0 -0
- /package/dist/{plugins/twenty/terraform → terraform/modules/app}/twenty/outputs.tf +0 -0
- /package/dist/{plugins/twenty/terraform → terraform/modules/app}/twenty/variables.tf +0 -0
|
@@ -7,8 +7,11 @@ import re
|
|
|
7
7
|
import secrets
|
|
8
8
|
import subprocess
|
|
9
9
|
import tarfile
|
|
10
|
+
import time
|
|
11
|
+
import urllib.error
|
|
10
12
|
import urllib.parse
|
|
11
13
|
import urllib.request
|
|
14
|
+
import uuid
|
|
12
15
|
from datetime import UTC, datetime
|
|
13
16
|
from pathlib import Path
|
|
14
17
|
from tempfile import TemporaryDirectory
|
|
@@ -42,11 +45,66 @@ RELEASE_MANIFEST_TRUST_POLICIES = {
|
|
|
42
45
|
POST_SEED_MIGRATIONS = [
|
|
43
46
|
"0155_tenant_model_catalog.sql",
|
|
44
47
|
]
|
|
48
|
+
AUTH_RETIREMENT_MIGRATION_MARKER = "-- deployment-phase: auth-retired"
|
|
49
|
+
PLATFORM_MIGRATION_LEDGER_CUTOFF = 260
|
|
45
50
|
MIGRATION_MARKER_KINDS = [
|
|
46
51
|
("-- creates-column:", "column"),
|
|
47
52
|
("-- creates-constraint:", "constraint"),
|
|
48
53
|
("-- creates:", "object"),
|
|
49
54
|
]
|
|
55
|
+
NATIVE_AUTH_CUSTOM_ATTRIBUTES = [
|
|
56
|
+
{
|
|
57
|
+
"AttributeDataType": "String",
|
|
58
|
+
"DeveloperOnlyAttribute": False,
|
|
59
|
+
"Mutable": True,
|
|
60
|
+
"Name": "entra_tenant_id",
|
|
61
|
+
"Required": False,
|
|
62
|
+
"StringAttributeConstraints": {"MinLength": "0", "MaxLength": "36"},
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"AttributeDataType": "String",
|
|
66
|
+
"DeveloperOnlyAttribute": False,
|
|
67
|
+
"Mutable": True,
|
|
68
|
+
"Name": "entra_object_id",
|
|
69
|
+
"Required": False,
|
|
70
|
+
"StringAttributeConstraints": {"MinLength": "0", "MaxLength": "36"},
|
|
71
|
+
},
|
|
72
|
+
]
|
|
73
|
+
AGENTCORE_CONTROL_SDK_VERSION = "3.1089.0"
|
|
74
|
+
TERRAFORM_PLAN_APPROVAL_CONTRACT = "thinkwork.terraform.saved-plan-approval.v1"
|
|
75
|
+
TERRAFORM_PLAN_DESCRIPTOR_NAME = "terraform-plan-approval.json"
|
|
76
|
+
TERRAFORM_SAVED_PLAN_NAME = "terraform-plan.bin"
|
|
77
|
+
TEI_V380_RECOVERY_CONTRACT = "thinkwork.incident.tei-v380-proof-plane-recovery.v1"
|
|
78
|
+
TEI_V380_RECOVERY_INCIDENT_ID = "tei-v380-proof-plane-partial-apply"
|
|
79
|
+
TEI_V380_REJECTED_PLAN_SHA256 = "ba7b0ffa099f725b4f32e394446ee43693fe3dd4b7a9eb639d47eb833d40db84"
|
|
80
|
+
TEI_V380_STATE_LINEAGE = "61f4093e-41da-ac29-852a-267bc853e24f"
|
|
81
|
+
TEI_V380_STATE_SERIAL = 819
|
|
82
|
+
TEI_V380_STATE_SHA256 = "cc99973e7b80e1a0c6e1097d889e006e5e2610e236ed730a02ca5f3457b99d7f"
|
|
83
|
+
TEI_V380_STATE_BUCKET = "tei-thinkwork-terraform-state"
|
|
84
|
+
TEI_V380_STATE_KEY = "env:/tei-e2e/thinkwork/tei-e2e/terraform.tfstate"
|
|
85
|
+
TEI_V380_LOCK_TABLE = "tei-thinkwork-terraform-locks"
|
|
86
|
+
TEI_V380_LOCK_KEY = f"{TEI_V380_STATE_BUCKET}/{TEI_V380_STATE_KEY}"
|
|
87
|
+
TEI_V380_LOCK_ID = "04b3e9a1-3de6-c668-87e7-43d94280605e"
|
|
88
|
+
TEI_V380_STATE_DIGEST = "7fe214eeaae491f5c1d8d792bddbbb19"
|
|
89
|
+
TEI_V380_SECRET_ARN = (
|
|
90
|
+
"arn:aws:secretsmanager:us-east-1:637423202447:secret:"
|
|
91
|
+
"thinkwork/tei-e2e/agentcore-identity/twenty-crm-oauth-client-7OeAlG"
|
|
92
|
+
)
|
|
93
|
+
TEI_V380_SECRET_NAME = "thinkwork/tei-e2e/agentcore-identity/twenty-crm-oauth-client"
|
|
94
|
+
TEI_V380_SECRET_ADDRESS = (
|
|
95
|
+
"module.thinkwork.module.agentcore_proof_identity."
|
|
96
|
+
"aws_secretsmanager_secret.twenty_oauth_client[0]"
|
|
97
|
+
)
|
|
98
|
+
TEI_V380_KMS_KEY_ID = "8798ab48-cd60-477d-a96e-c2d56bae7b40"
|
|
99
|
+
TEI_V380_KMS_KEY_ARN = "arn:aws:kms:us-east-1:637423202447:key/8798ab48-cd60-477d-a96e-c2d56bae7b40"
|
|
100
|
+
TEI_V380_KMS_KEY_ADDRESS = 'module.thinkwork.module.api.aws_kms_key.agentcore_turn_assertion["v1"]'
|
|
101
|
+
TEI_V380_KMS_ALIAS = "alias/thinkwork-tei-e2e-agentcore-turn-assertion-v1"
|
|
102
|
+
TEI_V380_KMS_ALIAS_ADDRESS = (
|
|
103
|
+
'module.thinkwork.module.api.aws_kms_alias.agentcore_turn_assertion["v1"]'
|
|
104
|
+
)
|
|
105
|
+
TEI_V380_OWNER_ALLOWLIST = (
|
|
106
|
+
"0ea2f626-8b40-43ca-b7f1-3601b2a6bef8,f498f488-70a1-709b-24d7-f895f2164301"
|
|
107
|
+
)
|
|
50
108
|
|
|
51
109
|
|
|
52
110
|
def run(args, **kwargs):
|
|
@@ -65,6 +123,10 @@ def sha256_file(path):
|
|
|
65
123
|
return digest.hexdigest()
|
|
66
124
|
|
|
67
125
|
|
|
126
|
+
def sha256_bytes(value):
|
|
127
|
+
return hashlib.sha256(value).hexdigest()
|
|
128
|
+
|
|
129
|
+
|
|
68
130
|
def read_json_env(name, default):
|
|
69
131
|
value = os.environ.get(name)
|
|
70
132
|
if not value:
|
|
@@ -381,6 +443,118 @@ def safe_extract_tar_file(archive_path, destination):
|
|
|
381
443
|
tar.extractall(destination, members=members)
|
|
382
444
|
|
|
383
445
|
|
|
446
|
+
def validate_agentcore_node_runtime(version):
|
|
447
|
+
normalized = str(version or "").strip().removeprefix("v")
|
|
448
|
+
try:
|
|
449
|
+
major = int(normalized.split(".", 1)[0])
|
|
450
|
+
except (TypeError, ValueError):
|
|
451
|
+
raise RuntimeError(
|
|
452
|
+
f"Cannot determine Node.js version for AgentCore runtime: {version}"
|
|
453
|
+
) from None
|
|
454
|
+
if major < 22:
|
|
455
|
+
raise RuntimeError(
|
|
456
|
+
"AgentCore control runtime requires Node.js 22 or newer; "
|
|
457
|
+
f"deployment runner is using {version}"
|
|
458
|
+
)
|
|
459
|
+
return f"v{normalized}"
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def prepare_agentcore_control_runtime():
|
|
463
|
+
global RELEASE_EVIDENCE
|
|
464
|
+
node_version = validate_agentcore_node_runtime(output(["node", "--version"]))
|
|
465
|
+
runner_bundle = bundle_extract_dir({"name": "platform"}) / "runner"
|
|
466
|
+
manifest_path = runner_bundle / "agentcore-control-runtime.json"
|
|
467
|
+
if not manifest_path.is_file():
|
|
468
|
+
raise RuntimeError("Release bundle is missing the AgentCore control runtime manifest")
|
|
469
|
+
|
|
470
|
+
runtime_manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
471
|
+
expected_sdk = runtime_manifest.get("sdk") or {}
|
|
472
|
+
if (
|
|
473
|
+
expected_sdk.get("package") != "@aws-sdk/client-bedrock-agentcore-control"
|
|
474
|
+
or expected_sdk.get("version") != AGENTCORE_CONTROL_SDK_VERSION
|
|
475
|
+
):
|
|
476
|
+
raise RuntimeError("Release does not pin the required AgentCore control SDK version")
|
|
477
|
+
|
|
478
|
+
runtime_dir = safe_join(
|
|
479
|
+
runner_bundle,
|
|
480
|
+
str(runtime_manifest.get("directory") or "agentcore-control-runtime"),
|
|
481
|
+
)
|
|
482
|
+
if not runtime_dir.is_dir():
|
|
483
|
+
raise RuntimeError("Release bundle has no AgentCore control runtime directory")
|
|
484
|
+
runtime_files = runtime_manifest.get("files") or []
|
|
485
|
+
if not runtime_files:
|
|
486
|
+
raise RuntimeError("AgentCore control runtime manifest has no files")
|
|
487
|
+
runtime_paths = [str(item.get("path") or "") for item in runtime_files]
|
|
488
|
+
# THINK-324: the managed harness is retired — harness-lifecycle.js is
|
|
489
|
+
# absent from new releases but tolerated in legacy bundles.
|
|
490
|
+
required_entrypoints = {
|
|
491
|
+
"preflight.js",
|
|
492
|
+
"reconcile_twenty_provider.js",
|
|
493
|
+
}
|
|
494
|
+
required_files = required_entrypoints | {"package.json"}
|
|
495
|
+
if len(runtime_paths) != len(set(runtime_paths)) or not required_files.issubset(runtime_paths):
|
|
496
|
+
raise RuntimeError("AgentCore control runtime manifest has an invalid file set")
|
|
497
|
+
for runtime_file in runtime_files:
|
|
498
|
+
file_path = safe_join(runtime_dir, str(runtime_file.get("path") or ""))
|
|
499
|
+
if not file_path.is_file() or sha256_file(file_path) != runtime_file.get("sha256"):
|
|
500
|
+
raise RuntimeError(
|
|
501
|
+
f"AgentCore control runtime file digest mismatch: {runtime_file.get('path')}"
|
|
502
|
+
)
|
|
503
|
+
actual_runtime_paths = {
|
|
504
|
+
str(path.relative_to(runtime_dir)) for path in runtime_dir.rglob("*") if path.is_file()
|
|
505
|
+
}
|
|
506
|
+
if actual_runtime_paths != set(runtime_paths):
|
|
507
|
+
raise RuntimeError("AgentCore control runtime contains unmanifested or missing files")
|
|
508
|
+
|
|
509
|
+
module_boundary = json.loads((runtime_dir / "package.json").read_text(encoding="utf-8"))
|
|
510
|
+
if module_boundary != {"type": "module"}:
|
|
511
|
+
raise RuntimeError("AgentCore control runtime has an invalid ESM module boundary")
|
|
512
|
+
|
|
513
|
+
preflight = json.loads(output(["node", str(runtime_dir / "preflight.js")]))
|
|
514
|
+
if preflight != expected_sdk:
|
|
515
|
+
raise RuntimeError("AgentCore control runtime preflight differs from its manifest")
|
|
516
|
+
# reconcile_pi_runtime.js (THINK-584 U5) is preflighted only when present:
|
|
517
|
+
# a newer seeded runner.py may run against a pre-U5 release bundle (the
|
|
518
|
+
# runner-py-bootstrap-skew trap), and the Pi runtime reconcile step skips
|
|
519
|
+
# loudly in that case rather than failing bundle validation here.
|
|
520
|
+
optional_entrypoints = ("reconcile_pi_runtime.js",)
|
|
521
|
+
entrypoint_preflights = [
|
|
522
|
+
json.loads(
|
|
523
|
+
output(
|
|
524
|
+
[
|
|
525
|
+
"node",
|
|
526
|
+
str(runtime_dir / entrypoint),
|
|
527
|
+
"--runtime-preflight",
|
|
528
|
+
]
|
|
529
|
+
)
|
|
530
|
+
)
|
|
531
|
+
for entrypoint in ("reconcile_twenty_provider.js", *optional_entrypoints)
|
|
532
|
+
if entrypoint not in optional_entrypoints or (runtime_dir / entrypoint).is_file()
|
|
533
|
+
]
|
|
534
|
+
if any(not item.get("sdkImportReady") for item in entrypoint_preflights):
|
|
535
|
+
raise RuntimeError("An AgentCore control runtime entrypoint cannot import its SDK")
|
|
536
|
+
os.environ["THINKWORK_AGENTCORE_CONTROL_RUNTIME_DIR"] = str(runtime_dir)
|
|
537
|
+
evidence = {
|
|
538
|
+
"manifestSha256": sha256_file(manifest_path),
|
|
539
|
+
"sdkPackage": preflight["package"],
|
|
540
|
+
"sdkVersion": preflight["version"],
|
|
541
|
+
"nodeVersion": node_version,
|
|
542
|
+
"bundledEntrypoints": sorted(
|
|
543
|
+
item["path"]
|
|
544
|
+
for item in runtime_files
|
|
545
|
+
if item.get("path") in required_entrypoints or item.get("path") in optional_entrypoints
|
|
546
|
+
),
|
|
547
|
+
"bundledRuntimeVerified": True,
|
|
548
|
+
"entrypointPreflights": entrypoint_preflights,
|
|
549
|
+
}
|
|
550
|
+
RELEASE_EVIDENCE["agentCoreControlRuntime"] = evidence
|
|
551
|
+
print(
|
|
552
|
+
"[runner] AgentCore control runtime ready: "
|
|
553
|
+
f"sdk={evidence['sdkVersion']} manifest={evidence['manifestSha256']}"
|
|
554
|
+
)
|
|
555
|
+
return evidence
|
|
556
|
+
|
|
557
|
+
|
|
384
558
|
def release_artifacts_by_name(manifest):
|
|
385
559
|
return {
|
|
386
560
|
artifact.get("name"): artifact
|
|
@@ -500,12 +674,18 @@ def write_json_evidence_artifact(name, payload):
|
|
|
500
674
|
}
|
|
501
675
|
|
|
502
676
|
|
|
677
|
+
# vars_json keys the runner consumes itself; they are not variables of the
|
|
678
|
+
# generated Terraform root and are left out of terraform.auto.tfvars.json.
|
|
679
|
+
RUNNER_ONLY_VARS = frozenset({"db_access"})
|
|
680
|
+
|
|
681
|
+
|
|
503
682
|
def redacted_tfvars(vars_json):
|
|
504
683
|
redacted = dict(vars_json)
|
|
505
684
|
for key in [
|
|
506
685
|
"api_auth_secret",
|
|
507
686
|
"db_password",
|
|
508
687
|
"google_oauth_client_secret",
|
|
688
|
+
"microsoft_oauth_client_secret",
|
|
509
689
|
]:
|
|
510
690
|
if key in redacted:
|
|
511
691
|
redacted[key] = "[redacted]"
|
|
@@ -648,6 +828,259 @@ def terraform_plan_summary(plan_json):
|
|
|
648
828
|
}
|
|
649
829
|
|
|
650
830
|
|
|
831
|
+
def terraform_plan_delete_addresses(plan_json):
|
|
832
|
+
return [
|
|
833
|
+
change.get("address")
|
|
834
|
+
for change in plan_json.get("resource_changes", [])
|
|
835
|
+
if "delete" in (change.get("change", {}).get("actions") or [])
|
|
836
|
+
]
|
|
837
|
+
|
|
838
|
+
|
|
839
|
+
def deployment_config_for_approval(payload):
|
|
840
|
+
"""Return the immutable deployment intent shared by plan and apply phases.
|
|
841
|
+
|
|
842
|
+
Controller/session routing fields necessarily change between executions.
|
|
843
|
+
Everything that can influence the generated Terraform configuration stays
|
|
844
|
+
in this projection, including release pins and preserved configuration.
|
|
845
|
+
"""
|
|
846
|
+
reviewed = json.loads(json.dumps(payload))
|
|
847
|
+
for key in [
|
|
848
|
+
"action",
|
|
849
|
+
"phase",
|
|
850
|
+
"sessionId",
|
|
851
|
+
"session",
|
|
852
|
+
"evidence",
|
|
853
|
+
"approvedPlan",
|
|
854
|
+
]:
|
|
855
|
+
reviewed.pop(key, None)
|
|
856
|
+
operation = reviewed.get("operation")
|
|
857
|
+
if isinstance(operation, dict):
|
|
858
|
+
for key in ["action", "plan", "apply", "approvedPlan"]:
|
|
859
|
+
operation.pop(key, None)
|
|
860
|
+
return reviewed
|
|
861
|
+
|
|
862
|
+
|
|
863
|
+
def deployment_config_sha256(payload):
|
|
864
|
+
return sha256_bytes(stable_json_bytes(deployment_config_for_approval(payload)))
|
|
865
|
+
|
|
866
|
+
|
|
867
|
+
def controller_input_sha256(payload):
|
|
868
|
+
return sha256_bytes(stable_json_bytes(payload))
|
|
869
|
+
|
|
870
|
+
|
|
871
|
+
def raw_controller_input_sha256(payload):
|
|
872
|
+
raw = os.environ.get("THINKWORK_DEPLOYMENT_INPUT")
|
|
873
|
+
if raw:
|
|
874
|
+
return sha256_bytes(raw.encode("utf-8"))
|
|
875
|
+
return controller_input_sha256(payload)
|
|
876
|
+
|
|
877
|
+
|
|
878
|
+
def requested_plan_action(payload, action):
|
|
879
|
+
if action in {"deploy", "update", "destroy"}:
|
|
880
|
+
return action
|
|
881
|
+
operation = payload.get("operation")
|
|
882
|
+
if isinstance(operation, dict):
|
|
883
|
+
candidate = operation.get("targetAction") or operation.get("requestedAction")
|
|
884
|
+
if candidate in {"deploy", "update", "destroy"}:
|
|
885
|
+
return candidate
|
|
886
|
+
if action == "plan":
|
|
887
|
+
return "update"
|
|
888
|
+
return action
|
|
889
|
+
|
|
890
|
+
|
|
891
|
+
def validate_terraform_execution_phase(payload, action):
|
|
892
|
+
"""Select the explicit saved-plan protocol or the compatible legacy path.
|
|
893
|
+
|
|
894
|
+
The incident recovery always uses separate ``plan`` and ``apply``
|
|
895
|
+
executions. Existing callers still send ``deploy``/``update``/``destroy``
|
|
896
|
+
while their API and CLI approval surfaces are migrated, so those actions
|
|
897
|
+
retain their established combined behavior instead of becoming an
|
|
898
|
+
accidental platform-wide outage.
|
|
899
|
+
"""
|
|
900
|
+
operation_value = payload.get("operation")
|
|
901
|
+
operation_is_structured = isinstance(operation_value, dict)
|
|
902
|
+
operation = operation_value
|
|
903
|
+
if not operation_is_structured:
|
|
904
|
+
operation = {}
|
|
905
|
+
requested_apply = operation.get("apply")
|
|
906
|
+
requested_plan = operation.get("plan")
|
|
907
|
+
|
|
908
|
+
if action == "apply":
|
|
909
|
+
if requested_apply is not True or requested_plan is not False:
|
|
910
|
+
raise RuntimeError(
|
|
911
|
+
"Saved-plan apply requires operation.apply=true and operation.plan=false"
|
|
912
|
+
)
|
|
913
|
+
approved = payload.get("approvedPlan")
|
|
914
|
+
if not isinstance(approved, dict):
|
|
915
|
+
raise RuntimeError("Saved-plan apply requires approvedPlan")
|
|
916
|
+
return "apply"
|
|
917
|
+
|
|
918
|
+
if action in {"deploy", "update", "destroy"}:
|
|
919
|
+
return "legacy"
|
|
920
|
+
|
|
921
|
+
if action == "plan":
|
|
922
|
+
if not operation_is_structured:
|
|
923
|
+
# Managed-application jobs predate the structured foundation
|
|
924
|
+
# operation object and already use action=plan as a strict
|
|
925
|
+
# plan-only phase. Preserve that safe contract.
|
|
926
|
+
return "plan"
|
|
927
|
+
if requested_apply is not False:
|
|
928
|
+
raise RuntimeError(
|
|
929
|
+
"Terraform planning never implies approval: operation.apply must be false; "
|
|
930
|
+
"apply the accepted saved plan in a separate action=apply execution"
|
|
931
|
+
)
|
|
932
|
+
if requested_plan is not True:
|
|
933
|
+
raise RuntimeError("Terraform plan phase requires operation.plan=true")
|
|
934
|
+
return "plan"
|
|
935
|
+
|
|
936
|
+
return "none"
|
|
937
|
+
|
|
938
|
+
|
|
939
|
+
def canonical_state_identity_bytes(state):
|
|
940
|
+
"""`terraform state pull` re-renders the state document on every run and
|
|
941
|
+
serializes the render-only check_results block in nondeterministic map
|
|
942
|
+
order, so identical remote state produces different raw bytes across
|
|
943
|
+
runs. Hash a canonical JSON form without check_results; lineage/serial
|
|
944
|
+
stay the authoritative change signal for the infrastructure content."""
|
|
945
|
+
comparable = {key: value for key, value in state.items() if key != "check_results"}
|
|
946
|
+
return json.dumps(comparable, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
947
|
+
|
|
948
|
+
|
|
949
|
+
def terraform_state_identity():
|
|
950
|
+
raw = output(["terraform", "state", "pull"], cwd=TF)
|
|
951
|
+
state = json.loads(raw or "{}")
|
|
952
|
+
lineage = state.get("lineage")
|
|
953
|
+
serial = state.get("serial")
|
|
954
|
+
if not isinstance(lineage, str) or not lineage:
|
|
955
|
+
raise RuntimeError("Terraform state has no lineage; refusing saved-plan operation")
|
|
956
|
+
if not isinstance(serial, int) or serial < 0:
|
|
957
|
+
raise RuntimeError("Terraform state has no valid serial; refusing saved-plan operation")
|
|
958
|
+
return {
|
|
959
|
+
"lineage": lineage,
|
|
960
|
+
"serial": serial,
|
|
961
|
+
"sha256": sha256_bytes(canonical_state_identity_bytes(state)),
|
|
962
|
+
"terraformVersion": state.get("terraform_version"),
|
|
963
|
+
}
|
|
964
|
+
|
|
965
|
+
|
|
966
|
+
def _validated_evidence_s3_uri(uri, label):
|
|
967
|
+
parsed = urllib.parse.urlparse(require_string(uri, label))
|
|
968
|
+
expected_bucket = os.environ.get("THINKWORK_EVIDENCE_BUCKET")
|
|
969
|
+
if parsed.scheme != "s3" or not parsed.netloc or not parsed.path.lstrip("/"):
|
|
970
|
+
raise RuntimeError(f"{label} must be an s3:// evidence URI")
|
|
971
|
+
if expected_bucket and parsed.netloc != expected_bucket:
|
|
972
|
+
raise RuntimeError(f"{label} must use the configured deployment evidence bucket")
|
|
973
|
+
return uri
|
|
974
|
+
|
|
975
|
+
|
|
976
|
+
def _download_approved_artifact(uri, destination, expected_sha256, label):
|
|
977
|
+
_validated_evidence_s3_uri(uri, label)
|
|
978
|
+
expected = require_string(expected_sha256, f"{label}.sha256").lower()
|
|
979
|
+
if not re.fullmatch(r"[0-9a-f]{64}", expected):
|
|
980
|
+
raise RuntimeError(f"{label}.sha256 must be a lowercase SHA-256 digest")
|
|
981
|
+
run(["aws", "s3", "cp", uri, str(destination)])
|
|
982
|
+
actual = sha256_file(destination)
|
|
983
|
+
if actual != expected:
|
|
984
|
+
raise RuntimeError(f"{label} digest mismatch: expected {expected}, got {actual}")
|
|
985
|
+
return actual
|
|
986
|
+
|
|
987
|
+
|
|
988
|
+
def load_approved_plan_descriptor(payload):
|
|
989
|
+
approved = payload.get("approvedPlan")
|
|
990
|
+
if not isinstance(approved, dict):
|
|
991
|
+
raise RuntimeError("Saved-plan apply requires approvedPlan")
|
|
992
|
+
descriptor_ref = approved.get("descriptor")
|
|
993
|
+
if not isinstance(descriptor_ref, dict):
|
|
994
|
+
raise RuntimeError("approvedPlan.descriptor is required")
|
|
995
|
+
descriptor_path = WORK / TERRAFORM_PLAN_DESCRIPTOR_NAME
|
|
996
|
+
_download_approved_artifact(
|
|
997
|
+
descriptor_ref.get("s3Uri"),
|
|
998
|
+
descriptor_path,
|
|
999
|
+
descriptor_ref.get("sha256"),
|
|
1000
|
+
"approvedPlan.descriptor",
|
|
1001
|
+
)
|
|
1002
|
+
descriptor = json.loads(descriptor_path.read_text(encoding="utf-8"))
|
|
1003
|
+
if descriptor.get("contract") != TERRAFORM_PLAN_APPROVAL_CONTRACT:
|
|
1004
|
+
raise RuntimeError("Approved plan descriptor contract is unsupported")
|
|
1005
|
+
return descriptor, descriptor_ref
|
|
1006
|
+
|
|
1007
|
+
|
|
1008
|
+
def comparable_terraform_plan(value, *, _top_level=True, _parent_key=None):
|
|
1009
|
+
"""Normalize render-only Terraform JSON variance without changing plan decisions."""
|
|
1010
|
+
if isinstance(value, dict):
|
|
1011
|
+
return {
|
|
1012
|
+
key: comparable_terraform_plan(item, _top_level=False, _parent_key=key)
|
|
1013
|
+
for key, item in value.items()
|
|
1014
|
+
if not (_top_level and key == "timestamp")
|
|
1015
|
+
}
|
|
1016
|
+
if isinstance(value, list):
|
|
1017
|
+
normalized = [comparable_terraform_plan(item, _top_level=False) for item in value]
|
|
1018
|
+
if _parent_key == "references" and all(isinstance(item, str) for item in normalized):
|
|
1019
|
+
return sorted(normalized)
|
|
1020
|
+
return normalized
|
|
1021
|
+
return value
|
|
1022
|
+
|
|
1023
|
+
|
|
1024
|
+
def validate_and_materialize_approved_plan(payload, state_identity):
|
|
1025
|
+
descriptor, descriptor_ref = load_approved_plan_descriptor(payload)
|
|
1026
|
+
if descriptor.get("deploymentConfigSha256") != deployment_config_sha256(payload):
|
|
1027
|
+
raise RuntimeError("Approved plan deployment configuration does not match apply input")
|
|
1028
|
+
if descriptor.get("release") != release_selection(payload):
|
|
1029
|
+
raise RuntimeError("Approved plan release pin does not match apply input")
|
|
1030
|
+
if descriptor.get("state") != state_identity:
|
|
1031
|
+
raise RuntimeError("Terraform state changed after plan approval; refusing apply")
|
|
1032
|
+
|
|
1033
|
+
tfvars_sha256 = sha256_file(TF / "terraform.auto.tfvars.json")
|
|
1034
|
+
if descriptor.get("terraformVariablesSha256") != tfvars_sha256:
|
|
1035
|
+
raise RuntimeError("Approved plan Terraform variables do not match apply execution")
|
|
1036
|
+
|
|
1037
|
+
saved_plan = descriptor.get("savedPlan")
|
|
1038
|
+
plan_json = descriptor.get("jsonPlan")
|
|
1039
|
+
if not isinstance(saved_plan, dict) or not isinstance(plan_json, dict):
|
|
1040
|
+
raise RuntimeError("Approved plan descriptor is missing saved plan artifacts")
|
|
1041
|
+
destination = TF / "tfplan"
|
|
1042
|
+
_download_approved_artifact(
|
|
1043
|
+
saved_plan.get("s3Uri"),
|
|
1044
|
+
destination,
|
|
1045
|
+
saved_plan.get("sha256"),
|
|
1046
|
+
"approvedPlan.savedPlan",
|
|
1047
|
+
)
|
|
1048
|
+
accepted_json = WORK / "accepted-terraform-plan.json"
|
|
1049
|
+
_download_approved_artifact(
|
|
1050
|
+
plan_json.get("s3Uri"),
|
|
1051
|
+
accepted_json,
|
|
1052
|
+
plan_json.get("sha256"),
|
|
1053
|
+
"approvedPlan.jsonPlan",
|
|
1054
|
+
)
|
|
1055
|
+
rendered = WORK / "approved-terraform-plan.json"
|
|
1056
|
+
with rendered.open("w", encoding="utf-8") as handle:
|
|
1057
|
+
run(["terraform", "show", "-json", "tfplan"], cwd=TF, stdout=handle)
|
|
1058
|
+
accepted_plan = json.loads(accepted_json.read_text(encoding="utf-8"))
|
|
1059
|
+
rendered_plan = json.loads(rendered.read_text(encoding="utf-8"))
|
|
1060
|
+
if comparable_terraform_plan(rendered_plan) != comparable_terraform_plan(accepted_plan):
|
|
1061
|
+
raise RuntimeError("Approved saved plan no longer renders to the accepted JSON plan")
|
|
1062
|
+
if payload.get("incidentRecovery") is not None:
|
|
1063
|
+
delete_addresses = terraform_plan_delete_addresses(rendered_plan)
|
|
1064
|
+
if delete_addresses:
|
|
1065
|
+
raise RuntimeError(
|
|
1066
|
+
"TEI v380 recovery approved plan contains delete actions; refusing apply"
|
|
1067
|
+
)
|
|
1068
|
+
|
|
1069
|
+
return {
|
|
1070
|
+
"descriptor": {
|
|
1071
|
+
"fileName": TERRAFORM_PLAN_DESCRIPTOR_NAME,
|
|
1072
|
+
"sha256": descriptor_ref.get("sha256"),
|
|
1073
|
+
"s3Uri": descriptor_ref.get("s3Uri"),
|
|
1074
|
+
},
|
|
1075
|
+
"savedPlan": saved_plan,
|
|
1076
|
+
"jsonPlan": plan_json,
|
|
1077
|
+
"summary": descriptor.get("summary"),
|
|
1078
|
+
"plannedAction": descriptor.get("plannedAction"),
|
|
1079
|
+
"sourceSessionId": descriptor.get("sessionId"),
|
|
1080
|
+
"state": state_identity,
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
|
|
651
1084
|
def managed_app_terraform_target_args(payload):
|
|
652
1085
|
app_key = payload.get("appKey")
|
|
653
1086
|
if app_key == "n8n":
|
|
@@ -662,6 +1095,43 @@ def managed_app_terraform_target_args(payload):
|
|
|
662
1095
|
return []
|
|
663
1096
|
|
|
664
1097
|
|
|
1098
|
+
def execute_terraform_plan_phase(
|
|
1099
|
+
payload,
|
|
1100
|
+
state_identity,
|
|
1101
|
+
planned_action,
|
|
1102
|
+
terraform_phase,
|
|
1103
|
+
target_args,
|
|
1104
|
+
):
|
|
1105
|
+
"""Create plan evidence and apply only for the compatible legacy phase."""
|
|
1106
|
+
plan_args = ["terraform", "plan"]
|
|
1107
|
+
if planned_action == "destroy":
|
|
1108
|
+
plan_args.append("-destroy")
|
|
1109
|
+
plan_args.extend([*target_args, "-out=tfplan", "-no-color"])
|
|
1110
|
+
plan = subprocess.run(plan_args, cwd=TF, text=True)
|
|
1111
|
+
if plan.returncode != 0:
|
|
1112
|
+
return plan
|
|
1113
|
+
TERRAFORM_EVIDENCE["plan"] = write_terraform_plan_evidence(
|
|
1114
|
+
payload,
|
|
1115
|
+
state_identity,
|
|
1116
|
+
planned_action,
|
|
1117
|
+
)
|
|
1118
|
+
if terraform_phase != "legacy":
|
|
1119
|
+
return plan
|
|
1120
|
+
return subprocess.run(
|
|
1121
|
+
["terraform", "apply", "-auto-approve", "-no-color", "tfplan"],
|
|
1122
|
+
cwd=TF,
|
|
1123
|
+
text=True,
|
|
1124
|
+
)
|
|
1125
|
+
|
|
1126
|
+
|
|
1127
|
+
def should_reconcile_native_auth_schema(terraform_phase, planned_action, payload):
|
|
1128
|
+
return (
|
|
1129
|
+
terraform_phase in {"apply", "legacy"}
|
|
1130
|
+
and planned_action in {"deploy", "update"}
|
|
1131
|
+
and not is_managed_app_operation(payload)
|
|
1132
|
+
)
|
|
1133
|
+
|
|
1134
|
+
|
|
665
1135
|
def truthy(value):
|
|
666
1136
|
if isinstance(value, bool):
|
|
667
1137
|
return value
|
|
@@ -792,6 +1262,28 @@ def is_web_only_operation(payload, action=None):
|
|
|
792
1262
|
return component == "web" or truthy(payload.get("webOnly"))
|
|
793
1263
|
|
|
794
1264
|
|
|
1265
|
+
def release_sync_request(action, payload, identity_operation):
|
|
1266
|
+
if (
|
|
1267
|
+
action not in {"deploy", "update", "apply", "web"}
|
|
1268
|
+
or is_managed_app_operation(payload)
|
|
1269
|
+
or identity_operation is not None
|
|
1270
|
+
):
|
|
1271
|
+
return None
|
|
1272
|
+
web_only = is_web_only_operation(payload, action)
|
|
1273
|
+
return {
|
|
1274
|
+
"artifact_types": {"static-site"} if web_only else None,
|
|
1275
|
+
"artifact_names": {"web"} if web_only else None,
|
|
1276
|
+
}
|
|
1277
|
+
|
|
1278
|
+
|
|
1279
|
+
def requires_agentcore_control_runtime(action, payload, identity_operation):
|
|
1280
|
+
return (
|
|
1281
|
+
action in {"deploy", "update", "plan", "apply", "destroy"}
|
|
1282
|
+
and not is_managed_app_operation(payload)
|
|
1283
|
+
and identity_operation is None
|
|
1284
|
+
)
|
|
1285
|
+
|
|
1286
|
+
|
|
795
1287
|
def configure_managed_app_evidence_prefix(payload):
|
|
796
1288
|
if not is_managed_app_operation(payload):
|
|
797
1289
|
return
|
|
@@ -907,21 +1399,54 @@ def cloudfront_alias_items(value):
|
|
|
907
1399
|
return []
|
|
908
1400
|
|
|
909
1401
|
|
|
910
|
-
def write_terraform_plan_evidence(payload=None):
|
|
1402
|
+
def write_terraform_plan_evidence(payload=None, state_identity=None, planned_action="update"):
|
|
1403
|
+
payload = payload or {}
|
|
1404
|
+
state_identity = state_identity or terraform_state_identity()
|
|
911
1405
|
plan_path = Path("terraform-plan.json")
|
|
912
1406
|
with plan_path.open("w", encoding="utf-8") as handle:
|
|
913
1407
|
run(["terraform", "show", "-json", "tfplan"], cwd=TF, stdout=handle)
|
|
914
1408
|
plan_json = json.loads(plan_path.read_text(encoding="utf-8"))
|
|
915
|
-
validate_managed_app_plan_scope(payload
|
|
916
|
-
validate_environment_plan_scope(payload
|
|
1409
|
+
validate_managed_app_plan_scope(payload, plan_json)
|
|
1410
|
+
validate_environment_plan_scope(payload, plan_json)
|
|
917
1411
|
artifact = {
|
|
918
1412
|
"fileName": plan_path.name,
|
|
919
1413
|
"sha256": sha256_file(plan_path),
|
|
920
1414
|
"s3Uri": upload_evidence_artifact(plan_path),
|
|
921
1415
|
}
|
|
1416
|
+
saved_plan_path = TF / "tfplan"
|
|
1417
|
+
saved_plan = {
|
|
1418
|
+
"fileName": TERRAFORM_SAVED_PLAN_NAME,
|
|
1419
|
+
"sha256": sha256_file(saved_plan_path),
|
|
1420
|
+
"s3Uri": upload_evidence_artifact(saved_plan_path, TERRAFORM_SAVED_PLAN_NAME),
|
|
1421
|
+
}
|
|
1422
|
+
descriptor = {
|
|
1423
|
+
"schemaVersion": 1,
|
|
1424
|
+
"contract": TERRAFORM_PLAN_APPROVAL_CONTRACT,
|
|
1425
|
+
"createdAt": datetime.now(UTC).isoformat(),
|
|
1426
|
+
"status": "awaiting-approval",
|
|
1427
|
+
"sessionId": payload.get("sessionId") or os.environ.get("THINKWORK_DEPLOYMENT_SESSION_ID"),
|
|
1428
|
+
"plannedAction": planned_action,
|
|
1429
|
+
"controllerInputSha256": controller_input_sha256(payload),
|
|
1430
|
+
"rawControllerInputSha256": raw_controller_input_sha256(payload),
|
|
1431
|
+
"deploymentConfigSha256": deployment_config_sha256(payload),
|
|
1432
|
+
"terraformVariablesSha256": sha256_file(TF / "terraform.auto.tfvars.json"),
|
|
1433
|
+
"release": release_selection(payload),
|
|
1434
|
+
"state": state_identity,
|
|
1435
|
+
"jsonPlan": artifact,
|
|
1436
|
+
"savedPlan": saved_plan,
|
|
1437
|
+
"summary": terraform_plan_summary(plan_json),
|
|
1438
|
+
}
|
|
1439
|
+
descriptor_artifact = write_json_evidence_artifact(
|
|
1440
|
+
TERRAFORM_PLAN_DESCRIPTOR_NAME,
|
|
1441
|
+
descriptor,
|
|
1442
|
+
)
|
|
922
1443
|
return {
|
|
923
1444
|
"artifact": artifact,
|
|
924
|
-
"
|
|
1445
|
+
"savedPlan": saved_plan,
|
|
1446
|
+
"approvalDescriptor": descriptor_artifact,
|
|
1447
|
+
"summary": descriptor["summary"],
|
|
1448
|
+
"state": state_identity,
|
|
1449
|
+
"status": "awaiting-approval",
|
|
925
1450
|
}
|
|
926
1451
|
|
|
927
1452
|
|
|
@@ -969,6 +1494,38 @@ def safe_get_bool(runner_secrets, payload, name, default=False):
|
|
|
969
1494
|
return default
|
|
970
1495
|
|
|
971
1496
|
|
|
1497
|
+
def safe_get_list(runner_secrets, payload, name, default=None):
|
|
1498
|
+
"""List analogue of safe_get_bool's precedence. Generated-root
|
|
1499
|
+
variables typed list(string) reject bare strings, so string values
|
|
1500
|
+
(comma-separated, common in Secrets Manager JSON) are split here and
|
|
1501
|
+
the wiring point always emits a real list of non-empty strings."""
|
|
1502
|
+
for source in (runner_secrets, payload):
|
|
1503
|
+
value = source.get(name)
|
|
1504
|
+
if isinstance(value, list):
|
|
1505
|
+
return [str(item).strip() for item in value if str(item).strip()]
|
|
1506
|
+
if isinstance(value, str) and value.strip():
|
|
1507
|
+
return [part.strip() for part in value.split(",") if part.strip()]
|
|
1508
|
+
return list(default or [])
|
|
1509
|
+
|
|
1510
|
+
|
|
1511
|
+
def safe_get_ingress_cidrs(runner_secrets, payload, name, default=None):
|
|
1512
|
+
"""safe_get_list precedence, but an explicitly PRESENT empty value ([] or
|
|
1513
|
+
"") wins as "no ingress" instead of falling through to the default —
|
|
1514
|
+
closing the database port (THINK-1082) is spelled dbPublicIngressCidrs: []
|
|
1515
|
+
and must not silently re-seed the world-open default."""
|
|
1516
|
+
for source in (runner_secrets, payload):
|
|
1517
|
+
if name not in source:
|
|
1518
|
+
continue
|
|
1519
|
+
value = source.get(name)
|
|
1520
|
+
if isinstance(value, list):
|
|
1521
|
+
return [str(item).strip() for item in value if str(item).strip()]
|
|
1522
|
+
if isinstance(value, str):
|
|
1523
|
+
return [part.strip() for part in value.split(",") if part.strip()]
|
|
1524
|
+
if value is None:
|
|
1525
|
+
continue
|
|
1526
|
+
return list(default or [])
|
|
1527
|
+
|
|
1528
|
+
|
|
972
1529
|
def current_terraform_state(stage):
|
|
973
1530
|
bucket = os.environ.get("THINKWORK_TERRAFORM_STATE_BUCKET")
|
|
974
1531
|
if not bucket:
|
|
@@ -988,96 +1545,760 @@ def current_terraform_state(stage):
|
|
|
988
1545
|
return {}
|
|
989
1546
|
|
|
990
1547
|
|
|
991
|
-
def
|
|
992
|
-
|
|
993
|
-
|
|
1548
|
+
def _terraform_state_object_keys(stage):
|
|
1549
|
+
return [
|
|
1550
|
+
f"env:/{stage}/thinkwork/{stage}/terraform.tfstate",
|
|
1551
|
+
f"thinkwork/{stage}/terraform.tfstate",
|
|
1552
|
+
]
|
|
994
1553
|
|
|
995
1554
|
|
|
996
|
-
def
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1555
|
+
def terraform_state_object_snapshot(stage):
|
|
1556
|
+
bucket = require_string(
|
|
1557
|
+
os.environ.get("THINKWORK_TERRAFORM_STATE_BUCKET"),
|
|
1558
|
+
"THINKWORK_TERRAFORM_STATE_BUCKET",
|
|
1559
|
+
)
|
|
1560
|
+
for key in _terraform_state_object_keys(stage):
|
|
1561
|
+
try:
|
|
1562
|
+
head = json.loads(
|
|
1563
|
+
output(
|
|
1564
|
+
[
|
|
1565
|
+
"aws",
|
|
1566
|
+
"s3api",
|
|
1567
|
+
"head-object",
|
|
1568
|
+
"--bucket",
|
|
1569
|
+
bucket,
|
|
1570
|
+
"--key",
|
|
1571
|
+
key,
|
|
1572
|
+
"--output",
|
|
1573
|
+
"json",
|
|
1574
|
+
]
|
|
1575
|
+
)
|
|
1576
|
+
)
|
|
1577
|
+
body = subprocess.check_output(
|
|
1578
|
+
["aws", "s3", "cp", f"s3://{bucket}/{key}", "-"],
|
|
1579
|
+
)
|
|
1580
|
+
except (subprocess.CalledProcessError, json.JSONDecodeError):
|
|
1581
|
+
continue
|
|
1582
|
+
state = json.loads(body)
|
|
1583
|
+
return {
|
|
1584
|
+
"bucket": bucket,
|
|
1585
|
+
"key": key,
|
|
1586
|
+
"etag": str(head.get("ETag") or "").strip('"'),
|
|
1587
|
+
"size": int(head.get("ContentLength") or len(body)),
|
|
1588
|
+
"lastModified": head.get("LastModified"),
|
|
1589
|
+
"serverSideEncryption": head.get("ServerSideEncryption"),
|
|
1590
|
+
"versionId": head.get("VersionId"),
|
|
1591
|
+
"sha256": sha256_bytes(body),
|
|
1592
|
+
"lineage": state.get("lineage"),
|
|
1593
|
+
"serial": state.get("serial"),
|
|
1594
|
+
"terraformVersion": state.get("terraform_version"),
|
|
1595
|
+
}
|
|
1596
|
+
raise RuntimeError(f"Terraform state object for stage {stage} was not found")
|
|
1597
|
+
|
|
1598
|
+
|
|
1599
|
+
def _incident_operation(payload):
|
|
1600
|
+
operation = payload.get("operation")
|
|
1601
|
+
if not isinstance(operation, dict) or operation.get("kind") != "incident_recovery":
|
|
1602
|
+
raise RuntimeError("Incident recovery requires operation.kind=incident_recovery")
|
|
1603
|
+
return operation
|
|
1604
|
+
|
|
1605
|
+
|
|
1606
|
+
def _require_exact_incident_payload_aliases(payload, aliases, expected, label):
|
|
1607
|
+
present = {name: payload.get(name) for name in aliases if name in payload}
|
|
1608
|
+
if not present or any(value != expected for value in present.values()):
|
|
1609
|
+
raise RuntimeError(f"TEI v380 recovery {label} does not match exact incident scope")
|
|
1610
|
+
|
|
1611
|
+
|
|
1612
|
+
def tei_v380_recovery_scope():
|
|
1613
|
+
return {
|
|
1614
|
+
"contract": TEI_V380_RECOVERY_CONTRACT,
|
|
1615
|
+
"incidentId": TEI_V380_RECOVERY_INCIDENT_ID,
|
|
1616
|
+
"rejectedPlanSha256": TEI_V380_REJECTED_PLAN_SHA256,
|
|
1617
|
+
"stage": "tei-e2e",
|
|
1618
|
+
"accountId": "637423202447",
|
|
1619
|
+
"region": "us-east-1",
|
|
1620
|
+
"state": {
|
|
1621
|
+
"bucket": TEI_V380_STATE_BUCKET,
|
|
1622
|
+
"key": TEI_V380_STATE_KEY,
|
|
1623
|
+
"lineage": TEI_V380_STATE_LINEAGE,
|
|
1624
|
+
"serial": TEI_V380_STATE_SERIAL,
|
|
1625
|
+
"sha256": TEI_V380_STATE_SHA256,
|
|
1626
|
+
},
|
|
1627
|
+
"lock": {
|
|
1628
|
+
"table": TEI_V380_LOCK_TABLE,
|
|
1629
|
+
"key": TEI_V380_LOCK_KEY,
|
|
1630
|
+
"id": TEI_V380_LOCK_ID,
|
|
1631
|
+
"operation": "OperationTypeApply",
|
|
1632
|
+
"terraformVersion": "1.8.5",
|
|
1633
|
+
"digest": TEI_V380_STATE_DIGEST,
|
|
1634
|
+
},
|
|
1635
|
+
"secret": {
|
|
1636
|
+
"address": TEI_V380_SECRET_ADDRESS,
|
|
1637
|
+
"arn": TEI_V380_SECRET_ARN,
|
|
1638
|
+
"name": TEI_V380_SECRET_NAME,
|
|
1639
|
+
},
|
|
1640
|
+
"kmsKey": {
|
|
1641
|
+
"address": TEI_V380_KMS_KEY_ADDRESS,
|
|
1642
|
+
"id": TEI_V380_KMS_KEY_ID,
|
|
1643
|
+
"arn": TEI_V380_KMS_KEY_ARN,
|
|
1644
|
+
"aliasAddress": TEI_V380_KMS_ALIAS_ADDRESS,
|
|
1645
|
+
"alias": TEI_V380_KMS_ALIAS,
|
|
1646
|
+
},
|
|
1647
|
+
}
|
|
1648
|
+
|
|
1649
|
+
|
|
1650
|
+
def validate_tei_v380_recovery_contract(payload):
|
|
1651
|
+
operation = _incident_operation(payload)
|
|
1652
|
+
expected = tei_v380_recovery_scope()
|
|
1653
|
+
actual = {
|
|
1654
|
+
key: operation.get(key)
|
|
1655
|
+
for key in [
|
|
1656
|
+
"contract",
|
|
1657
|
+
"incidentId",
|
|
1658
|
+
"rejectedPlanSha256",
|
|
1659
|
+
"stage",
|
|
1660
|
+
"accountId",
|
|
1661
|
+
"region",
|
|
1662
|
+
"state",
|
|
1663
|
+
"lock",
|
|
1664
|
+
"secret",
|
|
1665
|
+
"kmsKey",
|
|
1666
|
+
]
|
|
1667
|
+
}
|
|
1668
|
+
if actual != expected:
|
|
1669
|
+
raise RuntimeError("TEI v380 recovery contract does not match the exact incident scope")
|
|
1670
|
+
expected_operation_keys = {"kind", *expected.keys()}
|
|
1671
|
+
if set(operation) != expected_operation_keys:
|
|
1672
|
+
raise RuntimeError("TEI v380 recovery operation contains unexpected fields")
|
|
1673
|
+
_require_exact_incident_payload_aliases(
|
|
1674
|
+
payload, ("environmentName", "stage"), "tei-e2e", "stage"
|
|
1675
|
+
)
|
|
1676
|
+
_require_exact_incident_payload_aliases(
|
|
1677
|
+
payload, ("awsAccountId", "accountId"), "637423202447", "account"
|
|
1678
|
+
)
|
|
1679
|
+
_require_exact_incident_payload_aliases(payload, ("awsRegion", "region"), "us-east-1", "region")
|
|
1680
|
+
return expected
|
|
1681
|
+
|
|
1682
|
+
|
|
1683
|
+
def assert_incident_recovery_operation_is_exclusive():
|
|
1684
|
+
current_execution = os.environ.get("THINKWORK_DEPLOYMENT_EXECUTION_ARN")
|
|
1685
|
+
state_machine_arn = require_string(
|
|
1686
|
+
os.environ.get("THINKWORK_DEPLOYMENT_STATE_MACHINE_ARN"),
|
|
1687
|
+
"THINKWORK_DEPLOYMENT_STATE_MACHINE_ARN",
|
|
1688
|
+
)
|
|
1689
|
+
executions = json.loads(
|
|
1690
|
+
output(
|
|
1003
1691
|
[
|
|
1004
1692
|
"aws",
|
|
1005
|
-
"
|
|
1006
|
-
"
|
|
1007
|
-
"--
|
|
1008
|
-
|
|
1009
|
-
"--
|
|
1010
|
-
"
|
|
1011
|
-
"Parameter.Value",
|
|
1693
|
+
"stepfunctions",
|
|
1694
|
+
"list-executions",
|
|
1695
|
+
"--state-machine-arn",
|
|
1696
|
+
state_machine_arn,
|
|
1697
|
+
"--status-filter",
|
|
1698
|
+
"RUNNING",
|
|
1012
1699
|
"--output",
|
|
1013
|
-
"
|
|
1700
|
+
"json",
|
|
1014
1701
|
]
|
|
1015
1702
|
)
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
if
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1703
|
+
or "{}"
|
|
1704
|
+
).get("executions", [])
|
|
1705
|
+
if current_execution:
|
|
1706
|
+
other_executions = [
|
|
1707
|
+
item.get("executionArn")
|
|
1708
|
+
for item in executions
|
|
1709
|
+
if item.get("executionArn") != current_execution
|
|
1710
|
+
]
|
|
1711
|
+
else:
|
|
1712
|
+
# Established control planes do not receive Execution.Id until this
|
|
1713
|
+
# release updates their state-machine definition. Permit exactly the
|
|
1714
|
+
# sole running controller execution during that bounded transition.
|
|
1715
|
+
# Zero executions would mean the recovery was launched outside the
|
|
1716
|
+
# managed controller; more than one is ambiguous and unsafe.
|
|
1717
|
+
if len(executions) != 1:
|
|
1718
|
+
raise RuntimeError(
|
|
1719
|
+
"Incident recovery without execution identity requires exactly one "
|
|
1720
|
+
"running deployment execution"
|
|
1721
|
+
)
|
|
1722
|
+
current_execution = executions[0].get("executionArn")
|
|
1723
|
+
other_executions = []
|
|
1724
|
+
if other_executions:
|
|
1725
|
+
raise RuntimeError("Another deployment execution is running; recovery refused")
|
|
1726
|
+
|
|
1727
|
+
project = require_string(
|
|
1728
|
+
os.environ.get("THINKWORK_DEPLOYMENT_RUNNER_PROJECT_NAME"),
|
|
1729
|
+
"THINKWORK_DEPLOYMENT_RUNNER_PROJECT_NAME",
|
|
1730
|
+
)
|
|
1731
|
+
current_build = os.environ.get("CODEBUILD_BUILD_ID")
|
|
1732
|
+
if not current_build:
|
|
1733
|
+
raise RuntimeError("Incident recovery must run inside the deployment CodeBuild")
|
|
1734
|
+
build_ids = json.loads(
|
|
1735
|
+
output(
|
|
1736
|
+
[
|
|
1737
|
+
"aws",
|
|
1738
|
+
"codebuild",
|
|
1739
|
+
"list-builds-for-project",
|
|
1740
|
+
"--project-name",
|
|
1741
|
+
project,
|
|
1742
|
+
"--sort-order",
|
|
1743
|
+
"DESCENDING",
|
|
1744
|
+
"--no-paginate",
|
|
1745
|
+
"--output",
|
|
1746
|
+
"json",
|
|
1747
|
+
]
|
|
1748
|
+
)
|
|
1749
|
+
or "{}"
|
|
1750
|
+
).get("ids", [])[:100]
|
|
1751
|
+
if build_ids:
|
|
1752
|
+
builds = json.loads(
|
|
1753
|
+
output(
|
|
1754
|
+
[
|
|
1755
|
+
"aws",
|
|
1756
|
+
"codebuild",
|
|
1757
|
+
"batch-get-builds",
|
|
1758
|
+
"--ids",
|
|
1759
|
+
*build_ids,
|
|
1760
|
+
"--output",
|
|
1761
|
+
"json",
|
|
1762
|
+
]
|
|
1763
|
+
)
|
|
1764
|
+
or "{}"
|
|
1765
|
+
).get("builds", [])
|
|
1766
|
+
other_builds = [
|
|
1767
|
+
item.get("id")
|
|
1768
|
+
for item in builds
|
|
1769
|
+
if item.get("buildStatus") == "IN_PROGRESS" and item.get("id") != current_build
|
|
1770
|
+
]
|
|
1771
|
+
if other_builds:
|
|
1772
|
+
raise RuntimeError("Another deployment CodeBuild is running; recovery refused")
|
|
1773
|
+
return {
|
|
1774
|
+
"currentExecutionArn": current_execution,
|
|
1775
|
+
"currentBuildId": current_build,
|
|
1776
|
+
"otherRunningExecutions": 0,
|
|
1777
|
+
"otherRunningBuilds": 0,
|
|
1778
|
+
}
|
|
1025
1779
|
|
|
1026
1780
|
|
|
1027
|
-
def
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
if not token:
|
|
1036
|
-
return ""
|
|
1037
|
-
request = urllib.request.Request(
|
|
1038
|
-
"https://api.cloudflare.com/client/v4/zones?per_page=50",
|
|
1039
|
-
headers={"Authorization": f"Bearer {token}"},
|
|
1040
|
-
)
|
|
1041
|
-
try:
|
|
1042
|
-
with urllib.request.urlopen(request, timeout=30) as response:
|
|
1043
|
-
body = json.loads(response.read().decode("utf-8"))
|
|
1044
|
-
except Exception:
|
|
1045
|
-
return ""
|
|
1046
|
-
zones = body.get("result")
|
|
1047
|
-
if not isinstance(zones, list):
|
|
1048
|
-
return ""
|
|
1049
|
-
matches = []
|
|
1050
|
-
for zone in zones:
|
|
1051
|
-
if not isinstance(zone, dict):
|
|
1052
|
-
continue
|
|
1053
|
-
name = str(zone.get("name") or "").strip().lower().rstrip(".")
|
|
1054
|
-
zone_id = str(zone.get("id") or "").strip()
|
|
1055
|
-
if not name or not zone_id:
|
|
1056
|
-
continue
|
|
1057
|
-
if hostname == name or hostname.endswith(f".{name}"):
|
|
1058
|
-
matches.append((len(name), zone_id))
|
|
1059
|
-
if not matches:
|
|
1060
|
-
return ""
|
|
1061
|
-
return sorted(matches, reverse=True)[0][1]
|
|
1781
|
+
def _redacted_secret_description(description):
|
|
1782
|
+
return {
|
|
1783
|
+
"arn": description.get("ARN"),
|
|
1784
|
+
"name": description.get("Name"),
|
|
1785
|
+
"deletedDate": description.get("DeletedDate"),
|
|
1786
|
+
"kmsKeyId": description.get("KmsKeyId"),
|
|
1787
|
+
"rotationEnabled": bool(description.get("RotationEnabled", False)),
|
|
1788
|
+
}
|
|
1062
1789
|
|
|
1063
1790
|
|
|
1064
|
-
def
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1791
|
+
def _redacted_kms_description(description):
|
|
1792
|
+
metadata = description.get("KeyMetadata") or {}
|
|
1793
|
+
return {
|
|
1794
|
+
"keyId": metadata.get("KeyId"),
|
|
1795
|
+
"arn": metadata.get("Arn"),
|
|
1796
|
+
"keyState": metadata.get("KeyState"),
|
|
1797
|
+
"deletionDate": metadata.get("DeletionDate"),
|
|
1798
|
+
"enabled": metadata.get("Enabled"),
|
|
1799
|
+
"keyManager": metadata.get("KeyManager"),
|
|
1800
|
+
"origin": metadata.get("Origin"),
|
|
1801
|
+
}
|
|
1069
1802
|
|
|
1070
1803
|
|
|
1071
|
-
def
|
|
1072
|
-
|
|
1073
|
-
|
|
1804
|
+
def _wait_for_json_description(fetch, accept, description, attempts=20, delay_seconds=1):
|
|
1805
|
+
"""Bound eventual-consistency waits without ever reading secret values."""
|
|
1806
|
+
latest = None
|
|
1807
|
+
for attempt in range(attempts):
|
|
1808
|
+
latest = fetch()
|
|
1809
|
+
if accept(latest):
|
|
1810
|
+
return latest
|
|
1811
|
+
if attempt + 1 < attempts:
|
|
1812
|
+
time.sleep(delay_seconds)
|
|
1813
|
+
raise RuntimeError(f"Timed out waiting for {description}")
|
|
1074
1814
|
|
|
1075
1815
|
|
|
1076
|
-
def
|
|
1077
|
-
|
|
1816
|
+
def _dynamodb_string(item, name):
|
|
1817
|
+
value = (item.get(name) or {}).get("S")
|
|
1818
|
+
return value if isinstance(value, str) else None
|
|
1078
1819
|
|
|
1079
1820
|
|
|
1080
|
-
def
|
|
1821
|
+
def tei_v380_lock_snapshot():
|
|
1822
|
+
def get_item(lock_key):
|
|
1823
|
+
raw = output(
|
|
1824
|
+
[
|
|
1825
|
+
"aws",
|
|
1826
|
+
"dynamodb",
|
|
1827
|
+
"get-item",
|
|
1828
|
+
"--table-name",
|
|
1829
|
+
TEI_V380_LOCK_TABLE,
|
|
1830
|
+
"--key",
|
|
1831
|
+
json.dumps({"LockID": {"S": lock_key}}, separators=(",", ":")),
|
|
1832
|
+
"--consistent-read",
|
|
1833
|
+
"--output",
|
|
1834
|
+
"json",
|
|
1835
|
+
]
|
|
1836
|
+
)
|
|
1837
|
+
return json.loads(raw or "{}").get("Item") or {}
|
|
1838
|
+
|
|
1839
|
+
lock_item = get_item(TEI_V380_LOCK_KEY)
|
|
1840
|
+
digest_item = get_item(f"{TEI_V380_LOCK_KEY}-md5")
|
|
1841
|
+
info_raw = _dynamodb_string(lock_item, "Info")
|
|
1842
|
+
info = json.loads(info_raw) if info_raw else None
|
|
1843
|
+
if info is not None and not isinstance(info, dict):
|
|
1844
|
+
raise RuntimeError("TEI v380 Terraform lock Info is malformed")
|
|
1845
|
+
return {
|
|
1846
|
+
"table": TEI_V380_LOCK_TABLE,
|
|
1847
|
+
"key": TEI_V380_LOCK_KEY,
|
|
1848
|
+
"present": bool(lock_item),
|
|
1849
|
+
"info": {
|
|
1850
|
+
"id": (info or {}).get("ID"),
|
|
1851
|
+
"operation": (info or {}).get("Operation"),
|
|
1852
|
+
"version": (info or {}).get("Version"),
|
|
1853
|
+
"created": (info or {}).get("Created"),
|
|
1854
|
+
"path": (info or {}).get("Path"),
|
|
1855
|
+
}
|
|
1856
|
+
if info
|
|
1857
|
+
else None,
|
|
1858
|
+
"digestKey": f"{TEI_V380_LOCK_KEY}-md5",
|
|
1859
|
+
"digest": _dynamodb_string(digest_item, "Digest"),
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
|
|
1863
|
+
def force_unlock_tei_v380_state():
|
|
1864
|
+
"""Use Terraform's backend-aware unlock path; never delete DynamoDB rows directly."""
|
|
1865
|
+
before = tei_v380_lock_snapshot()
|
|
1866
|
+
expected_info = {
|
|
1867
|
+
"id": TEI_V380_LOCK_ID,
|
|
1868
|
+
"operation": "OperationTypeApply",
|
|
1869
|
+
"version": "1.8.5",
|
|
1870
|
+
"path": TEI_V380_LOCK_KEY,
|
|
1871
|
+
}
|
|
1872
|
+
actual_info = before.get("info") or {}
|
|
1873
|
+
if before.get("digest") != TEI_V380_STATE_DIGEST:
|
|
1874
|
+
raise RuntimeError("TEI v380 Terraform state digest changed; unlock refused")
|
|
1875
|
+
if before.get("present") is not True:
|
|
1876
|
+
return {
|
|
1877
|
+
"method": "terraform-force-unlock",
|
|
1878
|
+
"lockId": TEI_V380_LOCK_ID,
|
|
1879
|
+
"before": before,
|
|
1880
|
+
"after": before,
|
|
1881
|
+
"alreadyUnlocked": True,
|
|
1882
|
+
"directDynamoDbMutation": False,
|
|
1883
|
+
}
|
|
1884
|
+
if any(actual_info.get(key) != value for key, value in expected_info.items()):
|
|
1885
|
+
raise RuntimeError("TEI v380 stale Terraform lock identity does not match")
|
|
1886
|
+
|
|
1887
|
+
unlock_dir = WORK / "tei-v380-force-unlock"
|
|
1888
|
+
unlock_dir.mkdir(parents=True, exist_ok=True)
|
|
1889
|
+
(unlock_dir / "main.tf").write_text(
|
|
1890
|
+
'terraform {\n required_version = ">= 1.8.5"\n backend "s3" {}\n}\n',
|
|
1891
|
+
encoding="utf-8",
|
|
1892
|
+
)
|
|
1893
|
+
(unlock_dir / "backend.hcl").write_text(
|
|
1894
|
+
"\n".join(
|
|
1895
|
+
[
|
|
1896
|
+
f"bucket = {hcl_string(TEI_V380_STATE_BUCKET)}",
|
|
1897
|
+
f"key = {hcl_string(TEI_V380_STATE_KEY)}",
|
|
1898
|
+
'region = "us-east-1"',
|
|
1899
|
+
f"dynamodb_table = {hcl_string(TEI_V380_LOCK_TABLE)}",
|
|
1900
|
+
"encrypt = true",
|
|
1901
|
+
"",
|
|
1902
|
+
]
|
|
1903
|
+
),
|
|
1904
|
+
encoding="utf-8",
|
|
1905
|
+
)
|
|
1906
|
+
run(
|
|
1907
|
+
[
|
|
1908
|
+
"terraform",
|
|
1909
|
+
"init",
|
|
1910
|
+
"-reconfigure",
|
|
1911
|
+
"-backend-config=backend.hcl",
|
|
1912
|
+
"-input=false",
|
|
1913
|
+
"-no-color",
|
|
1914
|
+
],
|
|
1915
|
+
cwd=unlock_dir,
|
|
1916
|
+
)
|
|
1917
|
+
run(
|
|
1918
|
+
["terraform", "force-unlock", "-force", TEI_V380_LOCK_ID],
|
|
1919
|
+
cwd=unlock_dir,
|
|
1920
|
+
)
|
|
1921
|
+
after = tei_v380_lock_snapshot()
|
|
1922
|
+
if after.get("present") is not False:
|
|
1923
|
+
raise RuntimeError("TEI v380 stale Terraform lock remains after force-unlock")
|
|
1924
|
+
if after.get("digest") != before.get("digest"):
|
|
1925
|
+
raise RuntimeError("Terraform state digest changed during force-unlock")
|
|
1926
|
+
return {
|
|
1927
|
+
"method": "terraform-force-unlock",
|
|
1928
|
+
"lockId": TEI_V380_LOCK_ID,
|
|
1929
|
+
"before": before,
|
|
1930
|
+
"after": after,
|
|
1931
|
+
"alreadyUnlocked": False,
|
|
1932
|
+
"directDynamoDbMutation": False,
|
|
1933
|
+
}
|
|
1934
|
+
|
|
1935
|
+
|
|
1936
|
+
def recover_tei_v380_incident(payload):
|
|
1937
|
+
validate_tei_v380_recovery_contract(payload)
|
|
1938
|
+
exclusivity = assert_incident_recovery_operation_is_exclusive()
|
|
1939
|
+
caller_account = output(
|
|
1940
|
+
["aws", "sts", "get-caller-identity", "--query", "Account", "--output", "text"]
|
|
1941
|
+
)
|
|
1942
|
+
if caller_account != "637423202447":
|
|
1943
|
+
raise RuntimeError("TEI v380 recovery caller is in the wrong AWS account")
|
|
1944
|
+
|
|
1945
|
+
before_state = terraform_state_object_snapshot("tei-e2e")
|
|
1946
|
+
for key, expected in [
|
|
1947
|
+
("bucket", TEI_V380_STATE_BUCKET),
|
|
1948
|
+
("key", TEI_V380_STATE_KEY),
|
|
1949
|
+
("lineage", TEI_V380_STATE_LINEAGE),
|
|
1950
|
+
("serial", TEI_V380_STATE_SERIAL),
|
|
1951
|
+
("sha256", TEI_V380_STATE_SHA256),
|
|
1952
|
+
]:
|
|
1953
|
+
if before_state.get(key) != expected:
|
|
1954
|
+
raise RuntimeError(f"TEI v380 recovery state {key} changed; recovery refused")
|
|
1955
|
+
|
|
1956
|
+
progress = {
|
|
1957
|
+
"schemaVersion": 1,
|
|
1958
|
+
"contract": TEI_V380_RECOVERY_CONTRACT,
|
|
1959
|
+
"incidentId": TEI_V380_RECOVERY_INCIDENT_ID,
|
|
1960
|
+
"rejectedPlanSha256": TEI_V380_REJECTED_PLAN_SHA256,
|
|
1961
|
+
"status": "preflight-passed",
|
|
1962
|
+
"startedAt": datetime.now(UTC).isoformat(),
|
|
1963
|
+
"exclusiveOperation": exclusivity,
|
|
1964
|
+
"stateBefore": before_state,
|
|
1965
|
+
"terraformStateMutation": False,
|
|
1966
|
+
"secretValueRead": False,
|
|
1967
|
+
}
|
|
1968
|
+
write_json_evidence_artifact("incident-recovery-progress.json", progress)
|
|
1969
|
+
|
|
1970
|
+
secret_before_raw = json.loads(
|
|
1971
|
+
output(
|
|
1972
|
+
[
|
|
1973
|
+
"aws",
|
|
1974
|
+
"secretsmanager",
|
|
1975
|
+
"describe-secret",
|
|
1976
|
+
"--secret-id",
|
|
1977
|
+
TEI_V380_SECRET_ARN,
|
|
1978
|
+
"--output",
|
|
1979
|
+
"json",
|
|
1980
|
+
]
|
|
1981
|
+
)
|
|
1982
|
+
)
|
|
1983
|
+
if (
|
|
1984
|
+
secret_before_raw.get("ARN") != TEI_V380_SECRET_ARN
|
|
1985
|
+
or secret_before_raw.get("Name") != TEI_V380_SECRET_NAME
|
|
1986
|
+
):
|
|
1987
|
+
raise RuntimeError("TEI v380 recovery secret identity does not match")
|
|
1988
|
+
if secret_before_raw.get("DeletedDate"):
|
|
1989
|
+
run(
|
|
1990
|
+
[
|
|
1991
|
+
"aws",
|
|
1992
|
+
"secretsmanager",
|
|
1993
|
+
"restore-secret",
|
|
1994
|
+
"--secret-id",
|
|
1995
|
+
TEI_V380_SECRET_ARN,
|
|
1996
|
+
],
|
|
1997
|
+
stdout=subprocess.DEVNULL,
|
|
1998
|
+
)
|
|
1999
|
+
|
|
2000
|
+
def describe_secret():
|
|
2001
|
+
return json.loads(
|
|
2002
|
+
output(
|
|
2003
|
+
[
|
|
2004
|
+
"aws",
|
|
2005
|
+
"secretsmanager",
|
|
2006
|
+
"describe-secret",
|
|
2007
|
+
"--secret-id",
|
|
2008
|
+
TEI_V380_SECRET_ARN,
|
|
2009
|
+
"--output",
|
|
2010
|
+
"json",
|
|
2011
|
+
]
|
|
2012
|
+
)
|
|
2013
|
+
)
|
|
2014
|
+
|
|
2015
|
+
secret_after_raw = _wait_for_json_description(
|
|
2016
|
+
describe_secret,
|
|
2017
|
+
lambda value: not value.get("DeletedDate"),
|
|
2018
|
+
"restored TEI Twenty OAuth client secret",
|
|
2019
|
+
)
|
|
2020
|
+
if (
|
|
2021
|
+
secret_after_raw.get("ARN") != TEI_V380_SECRET_ARN
|
|
2022
|
+
or secret_after_raw.get("Name") != TEI_V380_SECRET_NAME
|
|
2023
|
+
):
|
|
2024
|
+
raise RuntimeError("Restored TEI v380 recovery secret identity does not match")
|
|
2025
|
+
progress["secret"] = {
|
|
2026
|
+
"address": TEI_V380_SECRET_ADDRESS,
|
|
2027
|
+
"before": _redacted_secret_description(secret_before_raw),
|
|
2028
|
+
"after": _redacted_secret_description(secret_after_raw),
|
|
2029
|
+
"status": "restored",
|
|
2030
|
+
}
|
|
2031
|
+
progress["status"] = "secret-restored"
|
|
2032
|
+
write_json_evidence_artifact("incident-recovery-progress.json", progress)
|
|
2033
|
+
|
|
2034
|
+
kms_before_raw = json.loads(
|
|
2035
|
+
output(
|
|
2036
|
+
[
|
|
2037
|
+
"aws",
|
|
2038
|
+
"kms",
|
|
2039
|
+
"describe-key",
|
|
2040
|
+
"--key-id",
|
|
2041
|
+
TEI_V380_KMS_KEY_ID,
|
|
2042
|
+
"--output",
|
|
2043
|
+
"json",
|
|
2044
|
+
]
|
|
2045
|
+
)
|
|
2046
|
+
)
|
|
2047
|
+
kms_before = _redacted_kms_description(kms_before_raw)
|
|
2048
|
+
if (
|
|
2049
|
+
kms_before.get("keyId") != TEI_V380_KMS_KEY_ID
|
|
2050
|
+
or kms_before.get("arn") != TEI_V380_KMS_KEY_ARN
|
|
2051
|
+
):
|
|
2052
|
+
raise RuntimeError("TEI v380 recovery KMS key identity does not match")
|
|
2053
|
+
if kms_before.get("keyState") == "PendingDeletion":
|
|
2054
|
+
run(
|
|
2055
|
+
["aws", "kms", "cancel-key-deletion", "--key-id", TEI_V380_KMS_KEY_ID],
|
|
2056
|
+
stdout=subprocess.DEVNULL,
|
|
2057
|
+
)
|
|
2058
|
+
|
|
2059
|
+
def describe_kms_key():
|
|
2060
|
+
return json.loads(
|
|
2061
|
+
output(
|
|
2062
|
+
[
|
|
2063
|
+
"aws",
|
|
2064
|
+
"kms",
|
|
2065
|
+
"describe-key",
|
|
2066
|
+
"--key-id",
|
|
2067
|
+
TEI_V380_KMS_KEY_ID,
|
|
2068
|
+
"--output",
|
|
2069
|
+
"json",
|
|
2070
|
+
]
|
|
2071
|
+
)
|
|
2072
|
+
)
|
|
2073
|
+
|
|
2074
|
+
kms_mid_raw = _wait_for_json_description(
|
|
2075
|
+
describe_kms_key,
|
|
2076
|
+
lambda value: _redacted_kms_description(value).get("keyState") in {"Disabled", "Enabled"},
|
|
2077
|
+
"cancelled TEI turn-assertion KMS deletion",
|
|
2078
|
+
)
|
|
2079
|
+
if _redacted_kms_description(kms_mid_raw).get("keyState") == "Disabled":
|
|
2080
|
+
run(
|
|
2081
|
+
["aws", "kms", "enable-key", "--key-id", TEI_V380_KMS_KEY_ID],
|
|
2082
|
+
stdout=subprocess.DEVNULL,
|
|
2083
|
+
)
|
|
2084
|
+
kms_after_raw = _wait_for_json_description(
|
|
2085
|
+
describe_kms_key,
|
|
2086
|
+
lambda value: (
|
|
2087
|
+
_redacted_kms_description(value).get("keyState") == "Enabled"
|
|
2088
|
+
and _redacted_kms_description(value).get("enabled") is True
|
|
2089
|
+
),
|
|
2090
|
+
"enabled TEI turn-assertion KMS key",
|
|
2091
|
+
)
|
|
2092
|
+
kms_after = _redacted_kms_description(kms_after_raw)
|
|
2093
|
+
if (
|
|
2094
|
+
kms_after.get("keyId") != TEI_V380_KMS_KEY_ID
|
|
2095
|
+
or kms_after.get("arn") != TEI_V380_KMS_KEY_ARN
|
|
2096
|
+
):
|
|
2097
|
+
raise RuntimeError("Recovered TEI v380 KMS key identity does not match")
|
|
2098
|
+
progress["kmsKey"] = {
|
|
2099
|
+
"address": TEI_V380_KMS_KEY_ADDRESS,
|
|
2100
|
+
"aliasAddress": TEI_V380_KMS_ALIAS_ADDRESS,
|
|
2101
|
+
"alias": TEI_V380_KMS_ALIAS,
|
|
2102
|
+
"before": kms_before,
|
|
2103
|
+
"after": kms_after,
|
|
2104
|
+
"status": "enabled",
|
|
2105
|
+
}
|
|
2106
|
+
progress["status"] = "resources-recovered"
|
|
2107
|
+
write_json_evidence_artifact("incident-recovery-progress.json", progress)
|
|
2108
|
+
|
|
2109
|
+
progress["terraformLock"] = force_unlock_tei_v380_state()
|
|
2110
|
+
progress["status"] = "resources-recovered-and-state-unlocked"
|
|
2111
|
+
write_json_evidence_artifact("incident-recovery-progress.json", progress)
|
|
2112
|
+
|
|
2113
|
+
after_state = terraform_state_object_snapshot("tei-e2e")
|
|
2114
|
+
if after_state != before_state:
|
|
2115
|
+
raise RuntimeError("Terraform state object changed during incident recovery")
|
|
2116
|
+
progress.update(
|
|
2117
|
+
{
|
|
2118
|
+
"status": "succeeded",
|
|
2119
|
+
"completedAt": datetime.now(UTC).isoformat(),
|
|
2120
|
+
"stateAfter": after_state,
|
|
2121
|
+
"terraformStateMutation": False,
|
|
2122
|
+
"secretValueRead": False,
|
|
2123
|
+
"nextStep": "proof-enabled-plan-with-imports",
|
|
2124
|
+
}
|
|
2125
|
+
)
|
|
2126
|
+
artifact = write_json_evidence_artifact("incident-recovery.json", progress)
|
|
2127
|
+
return {"proof": progress, "artifact": artifact}
|
|
2128
|
+
|
|
2129
|
+
|
|
2130
|
+
def tei_v380_recovery_import_blocks(payload, vars_json):
|
|
2131
|
+
recovery = payload.get("incidentRecovery")
|
|
2132
|
+
if recovery is None:
|
|
2133
|
+
return ""
|
|
2134
|
+
if not isinstance(recovery, dict):
|
|
2135
|
+
raise RuntimeError("incidentRecovery must be an object")
|
|
2136
|
+
evidence_ref = recovery.get("evidence")
|
|
2137
|
+
actual_scope = {key: recovery.get(key) for key in tei_v380_recovery_scope()}
|
|
2138
|
+
if actual_scope != tei_v380_recovery_scope():
|
|
2139
|
+
raise RuntimeError("Incident recovery import scope does not match TEI v380 incident")
|
|
2140
|
+
if not isinstance(evidence_ref, dict):
|
|
2141
|
+
raise RuntimeError("incidentRecovery.evidence is required")
|
|
2142
|
+
|
|
2143
|
+
# The decoded Terraform state is used elsewhere to preserve outputs and
|
|
2144
|
+
# guardrails, but it does not carry an object digest. Recovery approval is
|
|
2145
|
+
# explicitly bound to the S3 state object's SHA/lineage/serial, so compare
|
|
2146
|
+
# against the authoritative object snapshot instead of the decoded state.
|
|
2147
|
+
current_state = terraform_state_object_snapshot(
|
|
2148
|
+
vars_json.get("stage") or recovery.get("stage") or ""
|
|
2149
|
+
)
|
|
2150
|
+
|
|
2151
|
+
evidence_path = WORK / "accepted-incident-recovery.json"
|
|
2152
|
+
_download_approved_artifact(
|
|
2153
|
+
evidence_ref.get("s3Uri"),
|
|
2154
|
+
evidence_path,
|
|
2155
|
+
evidence_ref.get("sha256"),
|
|
2156
|
+
"incidentRecovery.evidence",
|
|
2157
|
+
)
|
|
2158
|
+
evidence = json.loads(evidence_path.read_text(encoding="utf-8"))
|
|
2159
|
+
if (
|
|
2160
|
+
evidence.get("contract") != TEI_V380_RECOVERY_CONTRACT
|
|
2161
|
+
or evidence.get("incidentId") != TEI_V380_RECOVERY_INCIDENT_ID
|
|
2162
|
+
or evidence.get("rejectedPlanSha256") != TEI_V380_REJECTED_PLAN_SHA256
|
|
2163
|
+
or evidence.get("status") != "succeeded"
|
|
2164
|
+
or evidence.get("terraformStateMutation") is not False
|
|
2165
|
+
or evidence.get("secretValueRead") is not False
|
|
2166
|
+
or evidence.get("stateBefore") != evidence.get("stateAfter")
|
|
2167
|
+
):
|
|
2168
|
+
raise RuntimeError("Incident recovery evidence is incomplete or unsafe")
|
|
2169
|
+
evidence_state = evidence.get("stateAfter") or {}
|
|
2170
|
+
if (
|
|
2171
|
+
evidence_state.get("lineage") != TEI_V380_STATE_LINEAGE
|
|
2172
|
+
or evidence_state.get("serial") != TEI_V380_STATE_SERIAL
|
|
2173
|
+
or evidence_state.get("sha256") != TEI_V380_STATE_SHA256
|
|
2174
|
+
):
|
|
2175
|
+
raise RuntimeError("Incident recovery evidence state identity does not match")
|
|
2176
|
+
if (
|
|
2177
|
+
current_state.get("lineage") != TEI_V380_STATE_LINEAGE
|
|
2178
|
+
or current_state.get("serial") != TEI_V380_STATE_SERIAL
|
|
2179
|
+
or current_state.get("sha256") != TEI_V380_STATE_SHA256
|
|
2180
|
+
):
|
|
2181
|
+
raise RuntimeError("Current Terraform state moved after incident recovery")
|
|
2182
|
+
|
|
2183
|
+
if vars_json.get("enable_agentcore_multiplayer_proof") is not True:
|
|
2184
|
+
raise RuntimeError("Incident recovery import requires AgentCore proof enabled")
|
|
2185
|
+
if vars_json.get("agentcore_multiplayer_proof_tenant_slug") != "tei":
|
|
2186
|
+
raise RuntimeError("Incident recovery import requires the exact TEI tenant slug")
|
|
2187
|
+
if vars_json.get("agentcore_multiplayer_proof_owner_allowlist") != TEI_V380_OWNER_ALLOWLIST:
|
|
2188
|
+
raise RuntimeError("Incident recovery import requires the exact owner allowlist")
|
|
2189
|
+
if vars_json.get("finalize_auth_retirement") is not False:
|
|
2190
|
+
raise RuntimeError("Incident recovery import cannot finalize auth retirement")
|
|
2191
|
+
if vars_json.get("auth_retirement_phase") != "coexistence":
|
|
2192
|
+
raise RuntimeError("Incident recovery import requires auth coexistence")
|
|
2193
|
+
if vars_json.get("microsoft_oauth_tenant") != "organizations":
|
|
2194
|
+
raise RuntimeError("Incident recovery import requires Microsoft organizations tenant")
|
|
2195
|
+
|
|
2196
|
+
return f"""
|
|
2197
|
+
# TEI v380 incident recovery imports. These addresses and IDs are bound to the
|
|
2198
|
+
# independently accepted recovery evidence above. They recover ownership of
|
|
2199
|
+
# the original secret and KMS key without reading or replacing either value.
|
|
2200
|
+
import {{
|
|
2201
|
+
to = {TEI_V380_SECRET_ADDRESS}
|
|
2202
|
+
id = {hcl_string(TEI_V380_SECRET_ARN)}
|
|
2203
|
+
}}
|
|
2204
|
+
|
|
2205
|
+
import {{
|
|
2206
|
+
to = {TEI_V380_KMS_KEY_ADDRESS}
|
|
2207
|
+
id = {hcl_string(TEI_V380_KMS_KEY_ID)}
|
|
2208
|
+
}}
|
|
2209
|
+
"""
|
|
2210
|
+
|
|
2211
|
+
|
|
2212
|
+
def current_terraform_outputs(stage):
|
|
2213
|
+
outputs = current_terraform_state(stage).get("outputs")
|
|
2214
|
+
return outputs if isinstance(outputs, dict) else {}
|
|
2215
|
+
|
|
2216
|
+
|
|
2217
|
+
def cloudflare_api_token(stage):
|
|
2218
|
+
existing = os.environ.get("CLOUDFLARE_API_TOKEN")
|
|
2219
|
+
if existing:
|
|
2220
|
+
return existing
|
|
2221
|
+
parameter_name = f"/thinkwork/{stage}/cloudflare-namespace-token"
|
|
2222
|
+
try:
|
|
2223
|
+
token = output(
|
|
2224
|
+
[
|
|
2225
|
+
"aws",
|
|
2226
|
+
"ssm",
|
|
2227
|
+
"get-parameter",
|
|
2228
|
+
"--name",
|
|
2229
|
+
parameter_name,
|
|
2230
|
+
"--with-decryption",
|
|
2231
|
+
"--query",
|
|
2232
|
+
"Parameter.Value",
|
|
2233
|
+
"--output",
|
|
2234
|
+
"text",
|
|
2235
|
+
]
|
|
2236
|
+
)
|
|
2237
|
+
except Exception:
|
|
2238
|
+
return ""
|
|
2239
|
+
if token:
|
|
2240
|
+
os.environ["CLOUDFLARE_API_TOKEN"] = token
|
|
2241
|
+
return token
|
|
2242
|
+
|
|
2243
|
+
|
|
2244
|
+
def configure_cloudflare_provider_auth(stage):
|
|
2245
|
+
cloudflare_api_token(stage)
|
|
2246
|
+
|
|
2247
|
+
|
|
2248
|
+
def cloudflare_zone_id_for_hostname(stage, hostname):
|
|
2249
|
+
configured = os.environ.get("THINKWORK_CLOUDFLARE_ZONE_ID", "")
|
|
2250
|
+
if configured:
|
|
2251
|
+
return configured
|
|
2252
|
+
hostname = (hostname or "").strip().lower().rstrip(".")
|
|
2253
|
+
if not hostname:
|
|
2254
|
+
return ""
|
|
2255
|
+
token = cloudflare_api_token(stage)
|
|
2256
|
+
if not token:
|
|
2257
|
+
return ""
|
|
2258
|
+
request = urllib.request.Request(
|
|
2259
|
+
"https://api.cloudflare.com/client/v4/zones?per_page=50",
|
|
2260
|
+
headers={"Authorization": f"Bearer {token}"},
|
|
2261
|
+
)
|
|
2262
|
+
try:
|
|
2263
|
+
with urllib.request.urlopen(request, timeout=30) as response:
|
|
2264
|
+
body = json.loads(response.read().decode("utf-8"))
|
|
2265
|
+
except Exception:
|
|
2266
|
+
return ""
|
|
2267
|
+
zones = body.get("result")
|
|
2268
|
+
if not isinstance(zones, list):
|
|
2269
|
+
return ""
|
|
2270
|
+
matches = []
|
|
2271
|
+
for zone in zones:
|
|
2272
|
+
if not isinstance(zone, dict):
|
|
2273
|
+
continue
|
|
2274
|
+
name = str(zone.get("name") or "").strip().lower().rstrip(".")
|
|
2275
|
+
zone_id = str(zone.get("id") or "").strip()
|
|
2276
|
+
if not name or not zone_id:
|
|
2277
|
+
continue
|
|
2278
|
+
if hostname == name or hostname.endswith(f".{name}"):
|
|
2279
|
+
matches.append((len(name), zone_id))
|
|
2280
|
+
if not matches:
|
|
2281
|
+
return ""
|
|
2282
|
+
return sorted(matches, reverse=True)[0][1]
|
|
2283
|
+
|
|
2284
|
+
|
|
2285
|
+
def state_output(outputs, name, default=None):
|
|
2286
|
+
value = outputs.get(name)
|
|
2287
|
+
if isinstance(value, dict) and "value" in value:
|
|
2288
|
+
return value["value"]
|
|
2289
|
+
return default
|
|
2290
|
+
|
|
2291
|
+
|
|
2292
|
+
def string_state_output(outputs, name):
|
|
2293
|
+
value = state_output(outputs, name, "")
|
|
2294
|
+
return value.strip() if isinstance(value, str) else ""
|
|
2295
|
+
|
|
2296
|
+
|
|
2297
|
+
def bool_state_output(outputs, name):
|
|
2298
|
+
return bool(state_output(outputs, name, False))
|
|
2299
|
+
|
|
2300
|
+
|
|
2301
|
+
def has_state_output(outputs, name):
|
|
1081
2302
|
value = outputs.get(name)
|
|
1082
2303
|
return isinstance(value, dict) and "value" in value
|
|
1083
2304
|
|
|
@@ -1233,6 +2454,42 @@ def config_string_list(desired_config, key, env_name, default=None):
|
|
|
1233
2454
|
return list(default or [])
|
|
1234
2455
|
|
|
1235
2456
|
|
|
2457
|
+
def coerce_string_list(value):
|
|
2458
|
+
"""Normalize a runner-secrets / payload value into a list[str].
|
|
2459
|
+
|
|
2460
|
+
Accepts a native list, a JSON-array string (``["a","b"]``), or a
|
|
2461
|
+
comma-separated string. Blank entries are dropped. Anything else yields
|
|
2462
|
+
an empty list. Order and duplicates are preserved for the caller; the
|
|
2463
|
+
Terraform side applies distinct().
|
|
2464
|
+
"""
|
|
2465
|
+
if isinstance(value, list):
|
|
2466
|
+
return [str(item).strip() for item in value if str(item).strip()]
|
|
2467
|
+
if isinstance(value, str):
|
|
2468
|
+
text = value.strip()
|
|
2469
|
+
if not text:
|
|
2470
|
+
return []
|
|
2471
|
+
if text.startswith("["):
|
|
2472
|
+
try:
|
|
2473
|
+
parsed = json.loads(text)
|
|
2474
|
+
except ValueError:
|
|
2475
|
+
parsed = None
|
|
2476
|
+
if isinstance(parsed, list):
|
|
2477
|
+
return [str(item).strip() for item in parsed if str(item).strip()]
|
|
2478
|
+
return [item.strip() for item in text.split(",") if item.strip()]
|
|
2479
|
+
return []
|
|
2480
|
+
|
|
2481
|
+
|
|
2482
|
+
def runner_secret_string_list(runner_secrets, payload, *keys):
|
|
2483
|
+
"""First non-empty list found across runner_secrets then payload for keys."""
|
|
2484
|
+
for source in (runner_secrets or {}, payload or {}):
|
|
2485
|
+
for key in keys:
|
|
2486
|
+
if key in source:
|
|
2487
|
+
result = coerce_string_list(source.get(key))
|
|
2488
|
+
if result:
|
|
2489
|
+
return result
|
|
2490
|
+
return []
|
|
2491
|
+
|
|
2492
|
+
|
|
1236
2493
|
def required_config_value(
|
|
1237
2494
|
desired_config,
|
|
1238
2495
|
manifest_images,
|
|
@@ -1951,19 +3208,13 @@ def managed_app_terraform_overrides(payload, stage, account_id, current_outputs,
|
|
|
1951
3208
|
# plans a destroy of a cert still attached to the n8n ALB (observed:
|
|
1952
3209
|
# McPherson update to v0.1.0-canary.314). When state shows the root
|
|
1953
3210
|
# managed cert, reproduce the install-time inputs instead.
|
|
1954
|
-
managed_n8n_cert = state_root_resource_attributes(
|
|
1955
|
-
current_state, "aws_acm_certificate", "n8n"
|
|
1956
|
-
)
|
|
3211
|
+
managed_n8n_cert = state_root_resource_attributes(current_state, "aws_acm_certificate", "n8n")
|
|
1957
3212
|
if managed_n8n_cert:
|
|
1958
3213
|
overrides["n8n_domain"] = (
|
|
1959
|
-
managed_n8n_cert.get("domain_name")
|
|
1960
|
-
or url_hostname(overrides["n8n_public_url"])
|
|
1961
|
-
or ""
|
|
3214
|
+
managed_n8n_cert.get("domain_name") or url_hostname(overrides["n8n_public_url"]) or ""
|
|
1962
3215
|
)
|
|
1963
3216
|
overrides["n8n_certificate_arn"] = ""
|
|
1964
|
-
foundation_n8n_dns_name = overrides["n8n_domain"] or url_hostname(
|
|
1965
|
-
overrides["n8n_public_url"]
|
|
1966
|
-
)
|
|
3217
|
+
foundation_n8n_dns_name = overrides["n8n_domain"] or url_hostname(overrides["n8n_public_url"])
|
|
1967
3218
|
if overrides["n8n_provisioned"] and foundation_n8n_dns_name:
|
|
1968
3219
|
# State-derived zone lookup goes blank once the cloudflare_record
|
|
1969
3220
|
# resources are absent (e.g. a prior partial destroy) — fall back
|
|
@@ -2081,7 +3332,9 @@ def resolve_agentcore_pi_source_image_uri(payload):
|
|
|
2081
3332
|
safe_get(operation, "action", default=safe_get(payload, "action", "phase", default=""))
|
|
2082
3333
|
).lower()
|
|
2083
3334
|
kind = str(safe_get(operation, "kind", default="")).lower()
|
|
2084
|
-
is_customer_update =
|
|
3335
|
+
is_customer_update = (
|
|
3336
|
+
action == "update" or kind == "foundation"
|
|
3337
|
+
) and kind != "identity_provider"
|
|
2085
3338
|
explicit = safe_get(payload, "agentcorePiSourceImageUri", default="")
|
|
2086
3339
|
if explicit:
|
|
2087
3340
|
if is_customer_update:
|
|
@@ -2104,31 +3357,209 @@ def resolve_agentcore_pi_source_image_uri(payload):
|
|
|
2104
3357
|
return release_runtime_image("agentcore-pi-amd64")
|
|
2105
3358
|
|
|
2106
3359
|
|
|
2107
|
-
def
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
git_sha = release.get("gitSha")
|
|
2114
|
-
return git_sha if isinstance(git_sha, str) else ""
|
|
2115
|
-
return ""
|
|
3360
|
+
def escape_env_control_chars(value):
|
|
3361
|
+
# AgentCore rejects env values with raw control characters (the PEM's
|
|
3362
|
+
# newlines); mirror them as literal escapes exactly like the dev-pipeline
|
|
3363
|
+
# env mirror does (scripts/update-agentcore-runtime-image.sh). Consumers
|
|
3364
|
+
# un-escape (the Pi container's publicKeyPemFromEnv normalizes \n).
|
|
3365
|
+
return value.replace("\r", "\\r").replace("\n", "\\n").replace("\t", "\\t")
|
|
2116
3366
|
|
|
2117
3367
|
|
|
2118
|
-
def
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
3368
|
+
def reconcile_agentcore_pi_runtime(vars_json, payload):
|
|
3369
|
+
"""Create/update the Pi Bedrock AgentCore Runtime after a successful apply
|
|
3370
|
+
(THINK-584 U5). Pins the runtime to the immutable digest of the customer's
|
|
3371
|
+
mirrored Pi image, mirrors the Pi Lambda env atomically (abort on empty),
|
|
3372
|
+
and records the runtime id in SSM. Secret env values never appear in logs
|
|
3373
|
+
or argv — the request travels to the node entrypoint on stdin."""
|
|
3374
|
+
stage = vars_json["stage"]
|
|
3375
|
+
region = os.environ.get("AWS_REGION") or safe_get(payload, "awsRegion", default="us-east-1")
|
|
3376
|
+
runtime_bundle_dir = os.environ.get("THINKWORK_AGENTCORE_CONTROL_RUNTIME_DIR", "")
|
|
3377
|
+
entrypoint = (
|
|
3378
|
+
Path(runtime_bundle_dir) / "reconcile_pi_runtime.js" if runtime_bundle_dir else None
|
|
3379
|
+
)
|
|
3380
|
+
if entrypoint is None or not entrypoint.is_file():
|
|
3381
|
+
print(
|
|
3382
|
+
"[runner] AgentCore Pi runtime reconcile SKIPPED: release bundle predates "
|
|
3383
|
+
"reconcile_pi_runtime.js (THINK-584 U5). Re-seed the runner bundle to enable "
|
|
3384
|
+
"the customer runtime-update path."
|
|
3385
|
+
)
|
|
3386
|
+
return None
|
|
2124
3387
|
|
|
3388
|
+
lambda_name = f"thinkwork-{stage}-agentcore-pi"
|
|
3389
|
+
fn = json.loads(
|
|
3390
|
+
output(
|
|
3391
|
+
[
|
|
3392
|
+
"aws",
|
|
3393
|
+
"lambda",
|
|
3394
|
+
"get-function-configuration",
|
|
3395
|
+
"--function-name",
|
|
3396
|
+
lambda_name,
|
|
3397
|
+
"--region",
|
|
3398
|
+
region,
|
|
3399
|
+
]
|
|
3400
|
+
)
|
|
3401
|
+
)
|
|
3402
|
+
environment = {
|
|
3403
|
+
key: escape_env_control_chars(value) if isinstance(value, str) else value
|
|
3404
|
+
for key, value in ((fn.get("Environment") or {}).get("Variables") or {}).items()
|
|
3405
|
+
}
|
|
3406
|
+
if not environment:
|
|
3407
|
+
raise RuntimeError(
|
|
3408
|
+
f"AgentCore Pi runtime reconcile: {lambda_name} reported an EMPTY environment; "
|
|
3409
|
+
"mirroring it would strand the runtime (atomic-or-abort)."
|
|
3410
|
+
)
|
|
3411
|
+
# THINK-586 U7 (KTD6): runtime-only overlay — the warm-session cache
|
|
3412
|
+
# factory gates on this var; it must never enter the Lambda terraform env.
|
|
3413
|
+
environment["AGENTCORE_RUNTIME_SESSION_CACHE"] = "1"
|
|
3414
|
+
role_arn = fn.get("Role") or ""
|
|
3415
|
+
account_id = (fn.get("FunctionArn") or "").split(":")[4] if fn.get("FunctionArn") else ""
|
|
3416
|
+
if not role_arn or not account_id:
|
|
3417
|
+
raise RuntimeError("AgentCore Pi runtime reconcile: Lambda did not report Role/ARN")
|
|
3418
|
+
|
|
3419
|
+
# AgentCore runtimes are arm64-only ("Supported platforms: [arm64]" —
|
|
3420
|
+
# first tei-e2e canary, 2026-08-04). The Lambda's amd64 pin
|
|
3421
|
+
# (agentcorePiSourceImageUri) is the WRONG architecture for the runtime;
|
|
3422
|
+
# the runtime image is mirrored into the customer ECR under the release
|
|
3423
|
+
# version tag by the release workflow's `mirror-customer-images` job
|
|
3424
|
+
# (THINK-616; targets listed in .github/release-mirror-targets.json).
|
|
3425
|
+
release_version = os.environ.get("THINKWORK_RELEASE_VERSION", "") or str(
|
|
3426
|
+
safe_get(payload, "releaseVersion", default="")
|
|
3427
|
+
)
|
|
3428
|
+
if not release_version:
|
|
3429
|
+
raise RuntimeError(
|
|
3430
|
+
"AgentCore Pi runtime reconcile: no release version available to "
|
|
3431
|
+
"resolve the arm64 runtime image tag."
|
|
3432
|
+
)
|
|
3433
|
+
image_tag = f"{release_version}-pi-arm64"
|
|
3434
|
+
repository = f"thinkwork-{stage}-agentcore"
|
|
3435
|
+
digest = output(
|
|
3436
|
+
[
|
|
3437
|
+
"aws",
|
|
3438
|
+
"ecr",
|
|
3439
|
+
"describe-images",
|
|
3440
|
+
"--repository-name",
|
|
3441
|
+
repository,
|
|
3442
|
+
"--region",
|
|
3443
|
+
region,
|
|
3444
|
+
"--image-ids",
|
|
3445
|
+
f"imageTag={image_tag}",
|
|
3446
|
+
"--query",
|
|
3447
|
+
"imageDetails[0].imageDigest",
|
|
3448
|
+
"--output",
|
|
3449
|
+
"text",
|
|
3450
|
+
]
|
|
3451
|
+
)
|
|
3452
|
+
if not re.fullmatch(r"sha256:[0-9a-f]{64}", digest):
|
|
3453
|
+
raise RuntimeError(
|
|
3454
|
+
f"AgentCore Pi runtime reconcile: could not resolve an immutable digest for "
|
|
3455
|
+
f"{repository}:{image_tag} (got '{digest}'). Mirror the release's "
|
|
3456
|
+
"arm64 Pi runtime image into this customer's ECR under that tag "
|
|
3457
|
+
"before retrying."
|
|
3458
|
+
)
|
|
3459
|
+
image_uri = f"{account_id}.dkr.ecr.{region}.amazonaws.com/{repository}@{digest}"
|
|
2125
3460
|
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
3461
|
+
ssm_name = f"/thinkwork/{stage}/agentcore/runtime-id-pi"
|
|
3462
|
+
runtime_id = ""
|
|
3463
|
+
try:
|
|
3464
|
+
runtime_id = output(
|
|
3465
|
+
[
|
|
3466
|
+
"aws",
|
|
3467
|
+
"ssm",
|
|
3468
|
+
"get-parameter",
|
|
3469
|
+
"--name",
|
|
3470
|
+
ssm_name,
|
|
3471
|
+
"--region",
|
|
3472
|
+
region,
|
|
3473
|
+
"--query",
|
|
3474
|
+
"Parameter.Value",
|
|
3475
|
+
"--output",
|
|
3476
|
+
"text",
|
|
3477
|
+
],
|
|
3478
|
+
stderr=subprocess.DEVNULL,
|
|
3479
|
+
)
|
|
3480
|
+
except subprocess.CalledProcessError:
|
|
3481
|
+
runtime_id = ""
|
|
3482
|
+
if runtime_id == "None":
|
|
3483
|
+
runtime_id = ""
|
|
3484
|
+
|
|
3485
|
+
# AgentCore runtime names must match [a-zA-Z][a-zA-Z0-9_]{0,47} — no
|
|
3486
|
+
# hyphens. Customer stages like "tei-e2e" broke the create with a
|
|
3487
|
+
# ValidationException (first live canary of this step, 2026-08-04).
|
|
3488
|
+
runtime_stage = re.sub(r"[^a-zA-Z0-9_]", "_", stage)
|
|
3489
|
+
request = {
|
|
3490
|
+
"region": region,
|
|
3491
|
+
"runtimeName": f"thinkwork_{runtime_stage}_pi",
|
|
3492
|
+
"runtimeId": runtime_id,
|
|
3493
|
+
"roleArn": role_arn,
|
|
3494
|
+
"imageUri": image_uri,
|
|
3495
|
+
"environment": environment,
|
|
3496
|
+
"waitSeconds": 900,
|
|
3497
|
+
}
|
|
3498
|
+
print(
|
|
3499
|
+
f"[runner] Reconciling AgentCore Pi runtime to {image_uri} "
|
|
3500
|
+
f"({len(environment)} env vars mirrored; values not logged)"
|
|
3501
|
+
)
|
|
3502
|
+
reconciled = subprocess.run(
|
|
3503
|
+
["node", str(entrypoint)],
|
|
3504
|
+
input=json.dumps(request),
|
|
3505
|
+
capture_output=True,
|
|
3506
|
+
text=True,
|
|
3507
|
+
)
|
|
3508
|
+
if reconciled.returncode != 0:
|
|
3509
|
+
raise RuntimeError(
|
|
3510
|
+
"AgentCore Pi runtime reconcile failed: "
|
|
3511
|
+
f"{(reconciled.stderr or '').strip() or f'exit {reconciled.returncode}'}"
|
|
3512
|
+
)
|
|
3513
|
+
result = json.loads(reconciled.stdout)
|
|
3514
|
+
run(
|
|
3515
|
+
[
|
|
3516
|
+
"aws",
|
|
3517
|
+
"ssm",
|
|
3518
|
+
"put-parameter",
|
|
3519
|
+
"--name",
|
|
3520
|
+
ssm_name,
|
|
3521
|
+
"--value",
|
|
3522
|
+
str(result["runtimeId"]),
|
|
3523
|
+
"--type",
|
|
3524
|
+
"String",
|
|
3525
|
+
"--overwrite",
|
|
3526
|
+
"--region",
|
|
3527
|
+
region,
|
|
3528
|
+
],
|
|
3529
|
+
stdout=subprocess.DEVNULL,
|
|
3530
|
+
)
|
|
3531
|
+
print(
|
|
3532
|
+
f"[runner] AgentCore Pi runtime ready: {result['runtimeId']} "
|
|
3533
|
+
f"v{result.get('version')} ({'created' if result.get('created') else 'updated'})"
|
|
3534
|
+
)
|
|
3535
|
+
return result
|
|
3536
|
+
|
|
3537
|
+
|
|
3538
|
+
def release_git_sha():
|
|
3539
|
+
if not MANIFEST.exists():
|
|
3540
|
+
return ""
|
|
3541
|
+
manifest = json.loads(MANIFEST.read_text(encoding="utf-8"))
|
|
3542
|
+
release = manifest.get("release")
|
|
3543
|
+
if isinstance(release, dict):
|
|
3544
|
+
git_sha = release.get("gitSha")
|
|
3545
|
+
return git_sha if isinstance(git_sha, str) else ""
|
|
3546
|
+
return ""
|
|
3547
|
+
|
|
3548
|
+
|
|
3549
|
+
def ensure_release_manifest_available(manifest_url, manifest_sha256):
|
|
3550
|
+
if not manifest_url and not MANIFEST.exists():
|
|
3551
|
+
return
|
|
3552
|
+
if not MANIFEST.exists():
|
|
3553
|
+
download(manifest_url, MANIFEST)
|
|
3554
|
+
verify_release_manifest_digest(MANIFEST, manifest_sha256)
|
|
3555
|
+
|
|
3556
|
+
|
|
3557
|
+
def source_repo_and_ref(module_source, release_version):
|
|
3558
|
+
source = module_source.removeprefix("git::")
|
|
3559
|
+
source_path, _, query = source.partition("?")
|
|
3560
|
+
params = urllib.parse.parse_qs(query)
|
|
3561
|
+
ref = params.get("ref", [release_version])[0]
|
|
3562
|
+
if source_path == "thinkwork-ai/thinkwork/aws":
|
|
2132
3563
|
repo = "https://github.com/thinkwork-ai/thinkwork.git"
|
|
2133
3564
|
ref = release_git_sha() or release_version
|
|
2134
3565
|
elif source_path.startswith("github.com/"):
|
|
@@ -2203,6 +3634,10 @@ def checkout_source(module_source, release_version):
|
|
|
2203
3634
|
|
|
2204
3635
|
|
|
2205
3636
|
def database_url_from_outputs(outputs):
|
|
3637
|
+
"""The database handle every schema step uses: a `postgresql://` URL for
|
|
3638
|
+
psql, or a DataApiDatabase when this run's dbAccess is `data-api`."""
|
|
3639
|
+
if DATABASE_ACCESS_MODE == "data-api":
|
|
3640
|
+
return DataApiDatabase.from_outputs(outputs)
|
|
2206
3641
|
endpoint = outputs.get("db_cluster_endpoint", {}).get("value")
|
|
2207
3642
|
secret_arn = outputs.get("db_secret_arn", {}).get("value")
|
|
2208
3643
|
database_name = outputs.get("database_name", {}).get("value") or "thinkwork"
|
|
@@ -2240,18 +3675,698 @@ def psql_env(database_url):
|
|
|
2240
3675
|
return env
|
|
2241
3676
|
|
|
2242
3677
|
|
|
3678
|
+
DATABASE_URL_PASSWORD_RE = re.compile(r"(://[^/:@\s]*:)[^@\s]*(@)")
|
|
3679
|
+
|
|
3680
|
+
|
|
3681
|
+
def redact_database_url(value):
|
|
3682
|
+
"""Replace the password segment of any `scheme://user:password@host` URL
|
|
3683
|
+
in `value` with `***`. Applied to everything a database URL can reach in a
|
|
3684
|
+
log or exception message: a failed psql surfaces its argv (URL included)
|
|
3685
|
+
through CalledProcessError, which CodeBuild prints to CloudWatch."""
|
|
3686
|
+
if isinstance(value, (list, tuple)):
|
|
3687
|
+
return [redact_database_url(item) for item in value]
|
|
3688
|
+
if not isinstance(value, str):
|
|
3689
|
+
return value
|
|
3690
|
+
return DATABASE_URL_PASSWORD_RE.sub(r"\1***\2", value)
|
|
3691
|
+
|
|
3692
|
+
|
|
3693
|
+
def _redacted_psql_error(error):
|
|
3694
|
+
"""Re-raise a psql CalledProcessError with the connection password
|
|
3695
|
+
redacted from its command. Same exception type, same returncode, same
|
|
3696
|
+
captured output — only the secret is gone."""
|
|
3697
|
+
return subprocess.CalledProcessError(
|
|
3698
|
+
error.returncode,
|
|
3699
|
+
redact_database_url(error.cmd),
|
|
3700
|
+
output=redact_database_url(error.output),
|
|
3701
|
+
stderr=redact_database_url(error.stderr),
|
|
3702
|
+
)
|
|
3703
|
+
|
|
3704
|
+
|
|
2243
3705
|
def psql(database_url, sql=None, file=None, variables=None):
|
|
3706
|
+
if isinstance(database_url, DataApiDatabase):
|
|
3707
|
+
return database_url.execute(sql=sql, file=file, variables=variables)
|
|
2244
3708
|
args = ["psql", database_url, "-v", "ON_ERROR_STOP=1"]
|
|
2245
3709
|
for key, value in (variables or {}).items():
|
|
2246
3710
|
args.extend(["-v", f"{key}={value}"])
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
3711
|
+
try:
|
|
3712
|
+
if file:
|
|
3713
|
+
args.extend(["-f", str(file)])
|
|
3714
|
+
return run(args)
|
|
3715
|
+
return subprocess.run(args, input=sql, check=True, text=True)
|
|
3716
|
+
except subprocess.CalledProcessError as error:
|
|
3717
|
+
raise _redacted_psql_error(error) from None
|
|
2251
3718
|
|
|
2252
3719
|
|
|
2253
3720
|
def psql_output(database_url, sql):
|
|
2254
|
-
|
|
3721
|
+
if isinstance(database_url, DataApiDatabase):
|
|
3722
|
+
return database_url.scalar(sql)
|
|
3723
|
+
try:
|
|
3724
|
+
return output(["psql", database_url, "-tAc", sql])
|
|
3725
|
+
except subprocess.CalledProcessError as error:
|
|
3726
|
+
raise _redacted_psql_error(error) from None
|
|
3727
|
+
|
|
3728
|
+
|
|
3729
|
+
################################################################################
|
|
3730
|
+
# Database access over the RDS Data API (THINK-1082)
|
|
3731
|
+
#
|
|
3732
|
+
# The runner is the last client of the Aurora cluster's public endpoint. With
|
|
3733
|
+
# every API Lambda inside the VPC, `psql` from this out-of-VPC CodeBuild is
|
|
3734
|
+
# the only reason the world-open 0.0.0.0/0 -> 5432 rule still exists. Setting
|
|
3735
|
+
# `dbAccess: "data-api"` (runner-secrets document, or the controller payload)
|
|
3736
|
+
# routes every psql / psql_output call below through the RDS Data API instead:
|
|
3737
|
+
# HTTPS, IAM-authorised, no network path — the lane the Brain already uses for
|
|
3738
|
+
# its own migrations against these clusters. The call sites do not change;
|
|
3739
|
+
# `database_url_from_outputs` hands back a DataApiDatabase handle instead of a
|
|
3740
|
+
# URL and `psql` / `psql_output` dispatch on it.
|
|
3741
|
+
#
|
|
3742
|
+
# What the Data API does not give us and how it is bridged here:
|
|
3743
|
+
# - one statement per ExecuteStatement -> scripts are split into statements
|
|
3744
|
+
# (dollar-quote / quote / comment aware; drizzle `--> statement-breakpoint`
|
|
3745
|
+
# markers are honoured as boundaries too);
|
|
3746
|
+
# - no psql variables -> `:'name'`, `:"name"` and `:name` are substituted for
|
|
3747
|
+
# the variables the runner passes (`stage`, the compliance passwords) — in
|
|
3748
|
+
# code only, never inside quotes, never touching `::casts`;
|
|
3749
|
+
# - no psql meta-commands -> `\set`, `\echo`/`\qecho`, `\if :{?name}` /
|
|
3750
|
+
# `\else` / `\endif` and `\quit` (the forms the migration files use) are
|
|
3751
|
+
# interpreted; anything else is refused by name;
|
|
3752
|
+
# - 45 s per call -> DDL runs with continueAfterTimeout so a slow statement
|
|
3753
|
+
# is never aborted server-side, and every script runs inside ONE Data API
|
|
3754
|
+
# transaction (BeginTransaction ... CommitTransaction) so its statements
|
|
3755
|
+
# serialise on one connection and a statement that outlived its call can
|
|
3756
|
+
# never race the next one. A script's own top-level BEGIN/COMMIT map onto
|
|
3757
|
+
# that transaction; `CREATE INDEX CONCURRENTLY` (which refuses to run in a
|
|
3758
|
+
# transaction block) commits the open one and runs standalone.
|
|
3759
|
+
# - ON_ERROR_STOP -> the first failing statement stops the script, rolls the
|
|
3760
|
+
# transaction back and raises DatabaseCommandError, a CalledProcessError
|
|
3761
|
+
# subclass, so every existing `except subprocess.CalledProcessError`
|
|
3762
|
+
# keeps working and the migration ledger is recorded only after success.
|
|
3763
|
+
################################################################################
|
|
3764
|
+
|
|
3765
|
+
DATABASE_ACCESS_MODES = ("psql", "data-api")
|
|
3766
|
+
DATABASE_ACCESS_MODE = "psql"
|
|
3767
|
+
RDS_DATA_API_MAX_SQL_BYTES = 65536
|
|
3768
|
+
STATEMENT_BREAKPOINT = "--> statement-breakpoint"
|
|
3769
|
+
|
|
3770
|
+
|
|
3771
|
+
def set_database_access_mode(vars_json):
|
|
3772
|
+
"""Select psql (today) or the RDS Data API for every database step of
|
|
3773
|
+
this run; logged once so the build log proves which lane ran."""
|
|
3774
|
+
global DATABASE_ACCESS_MODE
|
|
3775
|
+
mode = str((vars_json or {}).get("db_access") or "psql")
|
|
3776
|
+
if mode not in DATABASE_ACCESS_MODES:
|
|
3777
|
+
raise RuntimeError(
|
|
3778
|
+
f"dbAccess must be one of {', '.join(DATABASE_ACCESS_MODES)}; got {mode!r}"
|
|
3779
|
+
)
|
|
3780
|
+
DATABASE_ACCESS_MODE = mode
|
|
3781
|
+
print(f"[runner] database access: {mode}")
|
|
3782
|
+
return mode
|
|
3783
|
+
|
|
3784
|
+
|
|
3785
|
+
def safe_get_db_access(runner_secrets, payload):
|
|
3786
|
+
"""`dbAccess` rides the same precedence as apiLambdasInVpc: the
|
|
3787
|
+
runner-secrets document is the durable home, the controller payload can
|
|
3788
|
+
override per deploy. Runner-only — it never reaches the Terraform root."""
|
|
3789
|
+
value = safe_get(runner_secrets, "dbAccess", default=safe_get(payload, "dbAccess", default=""))
|
|
3790
|
+
value = value.strip().lower() or "psql"
|
|
3791
|
+
if value not in DATABASE_ACCESS_MODES:
|
|
3792
|
+
raise RuntimeError(
|
|
3793
|
+
f"dbAccess must be one of {', '.join(DATABASE_ACCESS_MODES)}; got {value!r}"
|
|
3794
|
+
)
|
|
3795
|
+
return value
|
|
3796
|
+
|
|
3797
|
+
|
|
3798
|
+
class DatabaseCommandError(subprocess.CalledProcessError):
|
|
3799
|
+
"""A failed Data API statement. Subclasses CalledProcessError so every
|
|
3800
|
+
caller that catches psql failures handles it identically; `cmd` names the
|
|
3801
|
+
lane and the script, never a connection string or a secret."""
|
|
3802
|
+
|
|
3803
|
+
def __init__(self, message, label):
|
|
3804
|
+
super().__init__(1, ["rds-data", "execute-statement", label], output="", stderr=message)
|
|
3805
|
+
self.message = message
|
|
3806
|
+
|
|
3807
|
+
def __str__(self):
|
|
3808
|
+
return f"rds-data failed for {self.cmd[2]}: {self.message}"
|
|
3809
|
+
|
|
3810
|
+
|
|
3811
|
+
def _sql_segments(text):
|
|
3812
|
+
"""Split SQL text into (kind, chunk) segments: 'code', 'squote',
|
|
3813
|
+
'dquote', 'dollar', 'line_comment', 'block_comment' and 'meta' (a psql
|
|
3814
|
+
meta-command: an unquoted backslash starting a line, through the end of
|
|
3815
|
+
that line). Concatenating the chunks reproduces the input byte for byte."""
|
|
3816
|
+
segments = []
|
|
3817
|
+
i = 0
|
|
3818
|
+
n = len(text)
|
|
3819
|
+
code_start = 0
|
|
3820
|
+
|
|
3821
|
+
def flush_code(end):
|
|
3822
|
+
if end > code_start:
|
|
3823
|
+
segments.append(("code", text[code_start:end]))
|
|
3824
|
+
|
|
3825
|
+
while i < n:
|
|
3826
|
+
ch = text[i]
|
|
3827
|
+
if ch == "\\" and not text[text.rfind("\n", 0, i) + 1 : i].strip():
|
|
3828
|
+
flush_code(i)
|
|
3829
|
+
end = text.find("\n", i)
|
|
3830
|
+
end = n if end == -1 else end
|
|
3831
|
+
segments.append(("meta", text[i:end]))
|
|
3832
|
+
i = code_start = end
|
|
3833
|
+
continue
|
|
3834
|
+
if ch == "-" and text.startswith("--", i):
|
|
3835
|
+
flush_code(i)
|
|
3836
|
+
end = text.find("\n", i)
|
|
3837
|
+
end = n if end == -1 else end
|
|
3838
|
+
segments.append(("line_comment", text[i:end]))
|
|
3839
|
+
i = code_start = end
|
|
3840
|
+
continue
|
|
3841
|
+
if ch == "/" and text.startswith("/*", i):
|
|
3842
|
+
flush_code(i)
|
|
3843
|
+
depth = 0
|
|
3844
|
+
j = i
|
|
3845
|
+
while j < n:
|
|
3846
|
+
if text.startswith("/*", j):
|
|
3847
|
+
depth += 1
|
|
3848
|
+
j += 2
|
|
3849
|
+
elif text.startswith("*/", j):
|
|
3850
|
+
depth -= 1
|
|
3851
|
+
j += 2
|
|
3852
|
+
if depth == 0:
|
|
3853
|
+
break
|
|
3854
|
+
else:
|
|
3855
|
+
j += 1
|
|
3856
|
+
segments.append(("block_comment", text[i:j]))
|
|
3857
|
+
i = code_start = j
|
|
3858
|
+
continue
|
|
3859
|
+
if ch in "'\"" and PSQL_QUOTED_VARIABLE_RE.match(text, i) and _is_variable_colon(text, i):
|
|
3860
|
+
# psql's :'name' / :"name" — interpolation syntax, not a literal
|
|
3861
|
+
i = PSQL_QUOTED_VARIABLE_RE.match(text, i).end()
|
|
3862
|
+
continue
|
|
3863
|
+
if ch == "'":
|
|
3864
|
+
flush_code(i)
|
|
3865
|
+
escaped = (
|
|
3866
|
+
i > 0
|
|
3867
|
+
and text[i - 1] in "eE"
|
|
3868
|
+
and (i < 2 or not (text[i - 2].isalnum() or text[i - 2] == "_"))
|
|
3869
|
+
)
|
|
3870
|
+
j = i + 1
|
|
3871
|
+
while j < n:
|
|
3872
|
+
if escaped and text[j] == "\\":
|
|
3873
|
+
j += 2
|
|
3874
|
+
continue
|
|
3875
|
+
if text[j] == "'":
|
|
3876
|
+
if j + 1 < n and text[j + 1] == "'":
|
|
3877
|
+
j += 2
|
|
3878
|
+
continue
|
|
3879
|
+
j += 1
|
|
3880
|
+
break
|
|
3881
|
+
j += 1
|
|
3882
|
+
segments.append(("squote", text[i:j]))
|
|
3883
|
+
i = code_start = j
|
|
3884
|
+
continue
|
|
3885
|
+
if ch == '"':
|
|
3886
|
+
flush_code(i)
|
|
3887
|
+
j = i + 1
|
|
3888
|
+
while j < n:
|
|
3889
|
+
if text[j] == '"':
|
|
3890
|
+
if j + 1 < n and text[j + 1] == '"':
|
|
3891
|
+
j += 2
|
|
3892
|
+
continue
|
|
3893
|
+
j += 1
|
|
3894
|
+
break
|
|
3895
|
+
j += 1
|
|
3896
|
+
segments.append(("dquote", text[i:j]))
|
|
3897
|
+
i = code_start = j
|
|
3898
|
+
continue
|
|
3899
|
+
if ch == "$":
|
|
3900
|
+
match = DOLLAR_QUOTE_RE.match(text, i)
|
|
3901
|
+
if match and not (i > 0 and (text[i - 1].isalnum() or text[i - 1] == "_")):
|
|
3902
|
+
tag = match.group(0)
|
|
3903
|
+
flush_code(i)
|
|
3904
|
+
end = text.find(tag, i + len(tag))
|
|
3905
|
+
end = n if end == -1 else end + len(tag)
|
|
3906
|
+
segments.append(("dollar", text[i:end]))
|
|
3907
|
+
i = code_start = end
|
|
3908
|
+
continue
|
|
3909
|
+
i += 1
|
|
3910
|
+
flush_code(n)
|
|
3911
|
+
return segments
|
|
3912
|
+
|
|
3913
|
+
|
|
3914
|
+
DOLLAR_QUOTE_RE = re.compile(r"\$[A-Za-z_][A-Za-z0-9_]*\$|\$\$")
|
|
3915
|
+
PSQL_QUOTED_VARIABLE_RE = re.compile(r"'\w+'|\"\w+\"")
|
|
3916
|
+
|
|
3917
|
+
|
|
3918
|
+
def _is_variable_colon(text, i):
|
|
3919
|
+
"""True when the quote at `i` follows a psql variable colon: a `:` that
|
|
3920
|
+
is not part of a `::` cast or of an identifier."""
|
|
3921
|
+
return (
|
|
3922
|
+
i > 0
|
|
3923
|
+
and text[i - 1] == ":"
|
|
3924
|
+
and (i < 2 or not (text[i - 2] == ":" or text[i - 2].isalnum() or text[i - 2] == "_"))
|
|
3925
|
+
)
|
|
3926
|
+
|
|
3927
|
+
|
|
3928
|
+
PSQL_META_RE = re.compile(r"^\s*\\(\w+)\s*(.*?)\s*$")
|
|
3929
|
+
PSQL_VARIABLE_RE = re.compile(r"(?<![:\w]):(?:'(\w+)'|\"(\w+)\"|(\w+))")
|
|
3930
|
+
|
|
3931
|
+
|
|
3932
|
+
def split_sql_script(text):
|
|
3933
|
+
"""Split a psql script into ordered items: ('sql', statement) for each
|
|
3934
|
+
top-level `;`-terminated statement (drizzle `--> statement-breakpoint`
|
|
3935
|
+
markers are boundaries too) and ('meta', line) for each psql meta-command
|
|
3936
|
+
line (an unquoted backslash at the start of a line). Comment-only and
|
|
3937
|
+
blank chunks are dropped; comments inside a statement travel with it."""
|
|
3938
|
+
items = []
|
|
3939
|
+
buffer = []
|
|
3940
|
+
has_code = [False]
|
|
3941
|
+
|
|
3942
|
+
def emit():
|
|
3943
|
+
statement = "".join(buffer).strip()
|
|
3944
|
+
if has_code[0] and statement:
|
|
3945
|
+
items.append(("sql", statement))
|
|
3946
|
+
buffer.clear()
|
|
3947
|
+
has_code[0] = False
|
|
3948
|
+
|
|
3949
|
+
for kind, chunk in _sql_segments(text):
|
|
3950
|
+
if kind == "meta":
|
|
3951
|
+
items.append(("meta", chunk.strip()))
|
|
3952
|
+
continue
|
|
3953
|
+
if kind == "line_comment":
|
|
3954
|
+
if chunk.strip() == STATEMENT_BREAKPOINT:
|
|
3955
|
+
emit()
|
|
3956
|
+
else:
|
|
3957
|
+
buffer.append(chunk)
|
|
3958
|
+
continue
|
|
3959
|
+
if kind != "code":
|
|
3960
|
+
buffer.append(chunk)
|
|
3961
|
+
has_code[0] = True
|
|
3962
|
+
continue
|
|
3963
|
+
start = 0
|
|
3964
|
+
for i, ch in enumerate(chunk):
|
|
3965
|
+
if ch == ";":
|
|
3966
|
+
buffer.append(chunk[start:i])
|
|
3967
|
+
emit()
|
|
3968
|
+
start = i + 1
|
|
3969
|
+
elif not ch.isspace():
|
|
3970
|
+
has_code[0] = True
|
|
3971
|
+
buffer.append(chunk[start:])
|
|
3972
|
+
emit()
|
|
3973
|
+
return items
|
|
3974
|
+
|
|
3975
|
+
|
|
3976
|
+
def substitute_psql_variables(sql, variables):
|
|
3977
|
+
"""psql `-v` interpolation for the variables given: `:'name'` becomes a
|
|
3978
|
+
SQL string literal, `:"name"` an identifier, bare `:name` the raw value.
|
|
3979
|
+
Only code is touched — never quoted text, comments or `::type` casts —
|
|
3980
|
+
and only names that were actually passed (psql leaves the rest alone)."""
|
|
3981
|
+
if not variables:
|
|
3982
|
+
return sql
|
|
3983
|
+
|
|
3984
|
+
def replace(match):
|
|
3985
|
+
literal, identifier, raw = match.groups()
|
|
3986
|
+
name = literal or identifier or raw
|
|
3987
|
+
if name not in variables:
|
|
3988
|
+
return match.group(0)
|
|
3989
|
+
value = str(variables[name])
|
|
3990
|
+
if literal:
|
|
3991
|
+
return "'" + value.replace("'", "''") + "'"
|
|
3992
|
+
if identifier:
|
|
3993
|
+
return '"' + value.replace('"', '""') + '"'
|
|
3994
|
+
return value
|
|
3995
|
+
|
|
3996
|
+
return "".join(
|
|
3997
|
+
PSQL_VARIABLE_RE.sub(replace, chunk) if kind == "code" else chunk
|
|
3998
|
+
for kind, chunk in _sql_segments(sql)
|
|
3999
|
+
)
|
|
4000
|
+
|
|
4001
|
+
|
|
4002
|
+
def _leading_keywords(statement, count=2):
|
|
4003
|
+
words = []
|
|
4004
|
+
for kind, chunk in _sql_segments(statement):
|
|
4005
|
+
if kind in ("line_comment", "block_comment"):
|
|
4006
|
+
continue
|
|
4007
|
+
if kind != "code":
|
|
4008
|
+
break
|
|
4009
|
+
words.extend(chunk.split())
|
|
4010
|
+
if len(words) >= count:
|
|
4011
|
+
break
|
|
4012
|
+
return tuple(word.upper().rstrip(";") for word in words[:count])
|
|
4013
|
+
|
|
4014
|
+
|
|
4015
|
+
def _statement_runs_outside_transaction(statement):
|
|
4016
|
+
"""Postgres refuses these inside a transaction block; psql (autocommit)
|
|
4017
|
+
runs them standalone, so the Data API lane does too."""
|
|
4018
|
+
keywords = _leading_keywords(statement, 4)
|
|
4019
|
+
return "CONCURRENTLY" in keywords or keywords[:1] == ("VACUUM",)
|
|
4020
|
+
|
|
4021
|
+
|
|
4022
|
+
def _rds_data_cell_text(cell):
|
|
4023
|
+
"""psql -tA rendering of one Data API field: NULL is empty, booleans are
|
|
4024
|
+
t/f, numbers print plainly, everything else is its string form."""
|
|
4025
|
+
if not isinstance(cell, dict) or cell.get("isNull"):
|
|
4026
|
+
return ""
|
|
4027
|
+
if "booleanValue" in cell:
|
|
4028
|
+
return "t" if cell["booleanValue"] else "f"
|
|
4029
|
+
for key in ("stringValue", "longValue", "doubleValue"):
|
|
4030
|
+
if key in cell:
|
|
4031
|
+
return str(cell[key])
|
|
4032
|
+
if "arrayValue" in cell:
|
|
4033
|
+
values = next(iter(cell["arrayValue"].values()), [])
|
|
4034
|
+
return "{" + ",".join(str(value) for value in values) + "}"
|
|
4035
|
+
if "blobValue" in cell:
|
|
4036
|
+
value = cell["blobValue"]
|
|
4037
|
+
return value.decode("utf-8", "replace") if isinstance(value, bytes) else str(value)
|
|
4038
|
+
return ""
|
|
4039
|
+
|
|
4040
|
+
|
|
4041
|
+
class AwsCliRdsDataClient:
|
|
4042
|
+
"""boto3-shaped rds-data client backed by the AWS CLI, for a build image
|
|
4043
|
+
without boto3 (the runner already shells out to `aws` everywhere)."""
|
|
4044
|
+
|
|
4045
|
+
def __init__(self, region):
|
|
4046
|
+
self.region = region
|
|
4047
|
+
|
|
4048
|
+
def _call(self, operation, params):
|
|
4049
|
+
args = ["aws", "rds-data", operation, "--cli-input-json", json.dumps(params)]
|
|
4050
|
+
if self.region:
|
|
4051
|
+
args.extend(["--region", self.region])
|
|
4052
|
+
try:
|
|
4053
|
+
body = subprocess.check_output(args, text=True, stderr=subprocess.PIPE)
|
|
4054
|
+
except subprocess.CalledProcessError as error:
|
|
4055
|
+
message = (error.stderr or "").strip() or f"aws rds-data {operation} failed"
|
|
4056
|
+
raise RuntimeError(message) from None
|
|
4057
|
+
return json.loads(body or "{}")
|
|
4058
|
+
|
|
4059
|
+
def execute_statement(self, **params):
|
|
4060
|
+
return self._call("execute-statement", params)
|
|
4061
|
+
|
|
4062
|
+
def begin_transaction(self, **params):
|
|
4063
|
+
return self._call("begin-transaction", params)
|
|
4064
|
+
|
|
4065
|
+
def commit_transaction(self, **params):
|
|
4066
|
+
return self._call("commit-transaction", params)
|
|
4067
|
+
|
|
4068
|
+
def rollback_transaction(self, **params):
|
|
4069
|
+
return self._call("rollback-transaction", params)
|
|
4070
|
+
|
|
4071
|
+
|
|
4072
|
+
def rds_data_client(region):
|
|
4073
|
+
try:
|
|
4074
|
+
import boto3 # noqa: PLC0415 — optional on the build image
|
|
4075
|
+
except ImportError:
|
|
4076
|
+
return AwsCliRdsDataClient(region)
|
|
4077
|
+
return boto3.client("rds-data", region_name=region) if region else boto3.client("rds-data")
|
|
4078
|
+
|
|
4079
|
+
|
|
4080
|
+
def _cluster_arn_from_outputs(outputs):
|
|
4081
|
+
arn = (outputs.get("db_cluster_arn") or {}).get("value")
|
|
4082
|
+
if arn:
|
|
4083
|
+
return str(arn)
|
|
4084
|
+
endpoint = str((outputs.get("db_cluster_endpoint") or {}).get("value") or "")
|
|
4085
|
+
labels = endpoint.split(".")
|
|
4086
|
+
if len(labels) < 5 or labels[-2] != "amazonaws" or labels[-3] != "rds":
|
|
4087
|
+
raise RuntimeError(
|
|
4088
|
+
"Terraform outputs carry no db_cluster_arn and the endpoint "
|
|
4089
|
+
f"{endpoint!r} is not an RDS hostname; cannot address the RDS Data API"
|
|
4090
|
+
)
|
|
4091
|
+
region = labels[-4]
|
|
4092
|
+
account = output(
|
|
4093
|
+
["aws", "sts", "get-caller-identity", "--query", "Account", "--output", "text"]
|
|
4094
|
+
)
|
|
4095
|
+
return f"arn:aws:rds:{region}:{account}:cluster:{labels[0]}"
|
|
4096
|
+
|
|
4097
|
+
|
|
4098
|
+
class DataApiDatabase:
|
|
4099
|
+
"""The database handle in data-api mode: what `database_url_from_outputs`
|
|
4100
|
+
returns instead of a URL, and what `psql` / `psql_output` dispatch on."""
|
|
4101
|
+
|
|
4102
|
+
def __init__(self, cluster_arn, secret_arn, database_name, client=None, region=None):
|
|
4103
|
+
self.cluster_arn = cluster_arn
|
|
4104
|
+
self.secret_arn = secret_arn
|
|
4105
|
+
self.database_name = database_name
|
|
4106
|
+
self.region = region or os.environ.get("AWS_REGION") or cluster_arn.split(":")[3]
|
|
4107
|
+
self._client = client
|
|
4108
|
+
|
|
4109
|
+
@classmethod
|
|
4110
|
+
def from_outputs(cls, outputs, client=None):
|
|
4111
|
+
secret_arn = (outputs.get("db_secret_arn") or {}).get("value")
|
|
4112
|
+
if not secret_arn:
|
|
4113
|
+
raise RuntimeError("Terraform outputs missing database secret ARN")
|
|
4114
|
+
database_name = (outputs.get("database_name") or {}).get("value") or "thinkwork"
|
|
4115
|
+
return cls(_cluster_arn_from_outputs(outputs), str(secret_arn), str(database_name), client)
|
|
4116
|
+
|
|
4117
|
+
def __repr__(self):
|
|
4118
|
+
return f"DataApiDatabase({self.cluster_arn}, database={self.database_name})"
|
|
4119
|
+
|
|
4120
|
+
__str__ = __repr__
|
|
4121
|
+
|
|
4122
|
+
@property
|
|
4123
|
+
def client(self):
|
|
4124
|
+
if self._client is None:
|
|
4125
|
+
self._client = rds_data_client(self.region)
|
|
4126
|
+
return self._client
|
|
4127
|
+
|
|
4128
|
+
def _params(self, sql, transaction_id=None, continue_after_timeout=False):
|
|
4129
|
+
params = {
|
|
4130
|
+
"resourceArn": self.cluster_arn,
|
|
4131
|
+
"secretArn": self.secret_arn,
|
|
4132
|
+
"database": self.database_name,
|
|
4133
|
+
"sql": sql,
|
|
4134
|
+
}
|
|
4135
|
+
if transaction_id:
|
|
4136
|
+
params["transactionId"] = transaction_id
|
|
4137
|
+
if continue_after_timeout:
|
|
4138
|
+
params["continueAfterTimeout"] = True
|
|
4139
|
+
return params
|
|
4140
|
+
|
|
4141
|
+
def scalar(self, sql):
|
|
4142
|
+
"""`psql -tAc`: every row on its own line, columns `|`-joined, NULL
|
|
4143
|
+
empty, booleans t/f, whole result stripped."""
|
|
4144
|
+
try:
|
|
4145
|
+
response = self.client.execute_statement(**self._params(sql))
|
|
4146
|
+
except Exception as error: # noqa: BLE001 — every SDK/CLI failure lands here
|
|
4147
|
+
raise DatabaseCommandError(str(error), "query") from None
|
|
4148
|
+
rows = [
|
|
4149
|
+
"|".join(_rds_data_cell_text(cell) for cell in row)
|
|
4150
|
+
for row in response.get("records") or []
|
|
4151
|
+
]
|
|
4152
|
+
return "\n".join(rows).strip()
|
|
4153
|
+
|
|
4154
|
+
def execute(self, sql=None, file=None, variables=None):
|
|
4155
|
+
"""`psql -v ON_ERROR_STOP=1 [-v k=v] -f file | -c sql`."""
|
|
4156
|
+
if file is not None:
|
|
4157
|
+
label = Path(file).name
|
|
4158
|
+
text = Path(file).read_text(encoding="utf-8")
|
|
4159
|
+
else:
|
|
4160
|
+
label = "inline sql"
|
|
4161
|
+
text = sql or ""
|
|
4162
|
+
variables = {str(key): str(value) for key, value in (variables or {}).items()}
|
|
4163
|
+
statements = self._plan(text, label, variables)
|
|
4164
|
+
self._run(statements, label, variables)
|
|
4165
|
+
|
|
4166
|
+
def _plan(self, text, label, variables):
|
|
4167
|
+
"""Interpret meta-commands in order and resolve the statements that
|
|
4168
|
+
will run; refuses the script before anything runs when it needs a
|
|
4169
|
+
meta-command this lane does not implement or a statement the Data
|
|
4170
|
+
API cannot carry."""
|
|
4171
|
+
planned = []
|
|
4172
|
+
conditions = []
|
|
4173
|
+
for kind, item in split_sql_script(text):
|
|
4174
|
+
active = all(conditions)
|
|
4175
|
+
if kind == "meta":
|
|
4176
|
+
match = PSQL_META_RE.match(item)
|
|
4177
|
+
command, argument = (match.group(1), match.group(2)) if match else (item, "")
|
|
4178
|
+
if command == "if":
|
|
4179
|
+
defined = re.fullmatch(r":\{\?(\w+)\}", argument)
|
|
4180
|
+
if not defined:
|
|
4181
|
+
raise DatabaseCommandError(
|
|
4182
|
+
f"unsupported psql meta-command in Data API mode: {item}", label
|
|
4183
|
+
)
|
|
4184
|
+
conditions.append(defined.group(1) in variables)
|
|
4185
|
+
elif command == "else":
|
|
4186
|
+
if not conditions:
|
|
4187
|
+
raise DatabaseCommandError(f"\\else without \\if in {label}", label)
|
|
4188
|
+
conditions[-1] = not conditions[-1]
|
|
4189
|
+
elif command == "endif":
|
|
4190
|
+
if not conditions:
|
|
4191
|
+
raise DatabaseCommandError(f"\\endif without \\if in {label}", label)
|
|
4192
|
+
conditions.pop()
|
|
4193
|
+
elif not active:
|
|
4194
|
+
continue
|
|
4195
|
+
elif command == "set":
|
|
4196
|
+
parts = argument.split(None, 1)
|
|
4197
|
+
if parts:
|
|
4198
|
+
value = parts[1].strip() if len(parts) > 1 else ""
|
|
4199
|
+
if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"":
|
|
4200
|
+
value = value[1:-1]
|
|
4201
|
+
variables[parts[0]] = value
|
|
4202
|
+
elif command in ("echo", "qecho"):
|
|
4203
|
+
message = argument.strip("'")
|
|
4204
|
+
print(f"[psql:{label}] {message}")
|
|
4205
|
+
elif command in ("quit", "q"):
|
|
4206
|
+
# psql: \quit in a script ends that script, exit status 0
|
|
4207
|
+
# (it takes no argument; `\quit 1` still exits 0).
|
|
4208
|
+
print(f"[psql:{label}] \\quit — remaining statements skipped")
|
|
4209
|
+
conditions.clear()
|
|
4210
|
+
break
|
|
4211
|
+
else:
|
|
4212
|
+
raise DatabaseCommandError(
|
|
4213
|
+
f"unsupported psql meta-command in Data API mode: {item}", label
|
|
4214
|
+
)
|
|
4215
|
+
continue
|
|
4216
|
+
if not active:
|
|
4217
|
+
continue
|
|
4218
|
+
statement = substitute_psql_variables(item, variables)
|
|
4219
|
+
if len(
|
|
4220
|
+
statement.encode("utf-8")
|
|
4221
|
+
) > RDS_DATA_API_MAX_SQL_BYTES and _statement_runs_outside_transaction(statement):
|
|
4222
|
+
raise DatabaseCommandError(
|
|
4223
|
+
f"statement {len(planned) + 1} exceeds the RDS Data API's "
|
|
4224
|
+
f"{RDS_DATA_API_MAX_SQL_BYTES}-byte limit and cannot run inside a "
|
|
4225
|
+
"transaction; split it at `;` boundaries",
|
|
4226
|
+
label,
|
|
4227
|
+
)
|
|
4228
|
+
planned.append(statement)
|
|
4229
|
+
if conditions:
|
|
4230
|
+
raise DatabaseCommandError(f"\\if without \\endif in {label}", label)
|
|
4231
|
+
return planned
|
|
4232
|
+
|
|
4233
|
+
def _run(self, statements, label, variables):
|
|
4234
|
+
transaction_id = None
|
|
4235
|
+
secrets_to_redact = [value for value in variables.values() if len(value) >= 8]
|
|
4236
|
+
|
|
4237
|
+
def redact(message):
|
|
4238
|
+
for secret in secrets_to_redact:
|
|
4239
|
+
message = message.replace(secret, "***")
|
|
4240
|
+
return message
|
|
4241
|
+
|
|
4242
|
+
def begin():
|
|
4243
|
+
return self.client.begin_transaction(
|
|
4244
|
+
resourceArn=self.cluster_arn, secretArn=self.secret_arn, database=self.database_name
|
|
4245
|
+
)["transactionId"]
|
|
4246
|
+
|
|
4247
|
+
try:
|
|
4248
|
+
for index, statement in enumerate(statements, start=1):
|
|
4249
|
+
head = _leading_keywords(statement)
|
|
4250
|
+
if head[:1] in {("BEGIN",), ("START",)}:
|
|
4251
|
+
if transaction_id is None:
|
|
4252
|
+
transaction_id = begin()
|
|
4253
|
+
continue
|
|
4254
|
+
if head[:1] in {("COMMIT",), ("END",)}:
|
|
4255
|
+
if transaction_id is not None:
|
|
4256
|
+
self.client.commit_transaction(
|
|
4257
|
+
resourceArn=self.cluster_arn,
|
|
4258
|
+
secretArn=self.secret_arn,
|
|
4259
|
+
transactionId=transaction_id,
|
|
4260
|
+
)
|
|
4261
|
+
transaction_id = None
|
|
4262
|
+
continue
|
|
4263
|
+
if head[:1] == ("ROLLBACK",):
|
|
4264
|
+
if transaction_id is not None:
|
|
4265
|
+
self.client.rollback_transaction(
|
|
4266
|
+
resourceArn=self.cluster_arn,
|
|
4267
|
+
secretArn=self.secret_arn,
|
|
4268
|
+
transactionId=transaction_id,
|
|
4269
|
+
)
|
|
4270
|
+
transaction_id = None
|
|
4271
|
+
continue
|
|
4272
|
+
if _statement_runs_outside_transaction(statement):
|
|
4273
|
+
if transaction_id is not None:
|
|
4274
|
+
self.client.commit_transaction(
|
|
4275
|
+
resourceArn=self.cluster_arn,
|
|
4276
|
+
secretArn=self.secret_arn,
|
|
4277
|
+
transactionId=transaction_id,
|
|
4278
|
+
)
|
|
4279
|
+
transaction_id = None
|
|
4280
|
+
self._execute_one(statement, None, index, label, redact)
|
|
4281
|
+
continue
|
|
4282
|
+
if transaction_id is None:
|
|
4283
|
+
transaction_id = begin()
|
|
4284
|
+
self._execute_one(statement, transaction_id, index, label, redact)
|
|
4285
|
+
if transaction_id is not None:
|
|
4286
|
+
self.client.commit_transaction(
|
|
4287
|
+
resourceArn=self.cluster_arn,
|
|
4288
|
+
secretArn=self.secret_arn,
|
|
4289
|
+
transactionId=transaction_id,
|
|
4290
|
+
)
|
|
4291
|
+
except BaseException:
|
|
4292
|
+
if transaction_id is not None:
|
|
4293
|
+
try:
|
|
4294
|
+
self.client.rollback_transaction(
|
|
4295
|
+
resourceArn=self.cluster_arn,
|
|
4296
|
+
secretArn=self.secret_arn,
|
|
4297
|
+
transactionId=transaction_id,
|
|
4298
|
+
)
|
|
4299
|
+
except Exception: # noqa: BLE001 — rollback is best-effort
|
|
4300
|
+
pass
|
|
4301
|
+
raise
|
|
4302
|
+
|
|
4303
|
+
def _execute_one(self, statement, transaction_id, index, label, redact):
|
|
4304
|
+
if len(statement.encode("utf-8")) > RDS_DATA_API_MAX_SQL_BYTES:
|
|
4305
|
+
self._execute_oversized(statement, transaction_id, index, label, redact)
|
|
4306
|
+
return
|
|
4307
|
+
try:
|
|
4308
|
+
self.client.execute_statement(
|
|
4309
|
+
**self._params(statement, transaction_id, continue_after_timeout=True)
|
|
4310
|
+
)
|
|
4311
|
+
except Exception as error: # noqa: BLE001 — every SDK/CLI failure lands here
|
|
4312
|
+
first_line = statement.strip().splitlines()[0][:120]
|
|
4313
|
+
raise DatabaseCommandError(
|
|
4314
|
+
redact(f"statement {index} ({first_line}): {error}"), label
|
|
4315
|
+
) from None
|
|
4316
|
+
|
|
4317
|
+
def _execute_oversized(self, statement, transaction_id, index, label, redact):
|
|
4318
|
+
"""A statement over the Data API's per-call limit (0230_analyst_rls
|
|
4319
|
+
carries an 85 KB DO block) is shipped in pieces into a temp table on
|
|
4320
|
+
the transaction's connection, reassembled and run there with EXECUTE,
|
|
4321
|
+
then the temp table is dropped. Same statement, same transaction."""
|
|
4322
|
+
if transaction_id is None:
|
|
4323
|
+
raise DatabaseCommandError(
|
|
4324
|
+
f"statement {index} exceeds {RDS_DATA_API_MAX_SQL_BYTES} bytes and "
|
|
4325
|
+
"cannot run inside a transaction",
|
|
4326
|
+
label,
|
|
4327
|
+
)
|
|
4328
|
+
tag = "$chunk$"
|
|
4329
|
+
suffix = 0
|
|
4330
|
+
while tag in statement:
|
|
4331
|
+
suffix += 1
|
|
4332
|
+
tag = f"$chunk{suffix}$"
|
|
4333
|
+
table = "pg_temp.runner_oversized_statement"
|
|
4334
|
+
pieces = []
|
|
4335
|
+
current = []
|
|
4336
|
+
size = 0
|
|
4337
|
+
for char in statement:
|
|
4338
|
+
width = len(char.encode("utf-8"))
|
|
4339
|
+
if size + width > RDS_DATA_API_MAX_SQL_BYTES - 4096:
|
|
4340
|
+
pieces.append("".join(current))
|
|
4341
|
+
current, size = [], 0
|
|
4342
|
+
current.append(char)
|
|
4343
|
+
size += width
|
|
4344
|
+
if current:
|
|
4345
|
+
pieces.append("".join(current))
|
|
4346
|
+
steps = [f"CREATE TEMP TABLE {table.split('.')[1]} (n integer PRIMARY KEY, chunk text)"]
|
|
4347
|
+
steps.extend(
|
|
4348
|
+
f"INSERT INTO {table} (n, chunk) VALUES ({number}, {tag}{piece}{tag})"
|
|
4349
|
+
for number, piece in enumerate(pieces)
|
|
4350
|
+
)
|
|
4351
|
+
steps.append(
|
|
4352
|
+
"DO $runner$ BEGIN EXECUTE (SELECT string_agg(chunk, '' ORDER BY n) "
|
|
4353
|
+
f"FROM {table}); END $runner$"
|
|
4354
|
+
)
|
|
4355
|
+
steps.append(f"DROP TABLE {table}")
|
|
4356
|
+
print(
|
|
4357
|
+
f"[psql:{label}] statement {index} is {len(statement.encode('utf-8'))} bytes; "
|
|
4358
|
+
f"shipping it to the server in {len(pieces)} pieces"
|
|
4359
|
+
)
|
|
4360
|
+
for step in steps:
|
|
4361
|
+
try:
|
|
4362
|
+
self.client.execute_statement(
|
|
4363
|
+
**self._params(step, transaction_id, continue_after_timeout=True)
|
|
4364
|
+
)
|
|
4365
|
+
except Exception as error: # noqa: BLE001 — every SDK/CLI failure lands here
|
|
4366
|
+
first_line = statement.strip().splitlines()[0][:120]
|
|
4367
|
+
raise DatabaseCommandError(
|
|
4368
|
+
redact(f"statement {index} ({first_line}): {error}"), label
|
|
4369
|
+
) from None
|
|
2255
4370
|
|
|
2256
4371
|
|
|
2257
4372
|
def put_secret_value(secret_id, payload):
|
|
@@ -2298,9 +4413,24 @@ def ensure_compliance_roles(database_url, outputs, vars_json):
|
|
|
2298
4413
|
)
|
|
2299
4414
|
|
|
2300
4415
|
|
|
2301
|
-
def migration_files():
|
|
4416
|
+
def migration_files(vars_json=None):
|
|
2302
4417
|
migrations = SOURCE / "packages/database-pg/drizzle"
|
|
2303
|
-
|
|
4418
|
+
allow_auth_retirement = bool(
|
|
4419
|
+
isinstance(vars_json, dict)
|
|
4420
|
+
and vars_json.get("auth_retirement_phase") == "retired"
|
|
4421
|
+
and vars_json.get("finalize_auth_retirement") is True
|
|
4422
|
+
)
|
|
4423
|
+
paths = []
|
|
4424
|
+
for path in migrations.glob("*.sql"):
|
|
4425
|
+
if "rollback" in path.name:
|
|
4426
|
+
continue
|
|
4427
|
+
if (
|
|
4428
|
+
AUTH_RETIREMENT_MIGRATION_MARKER in path.read_text(encoding="utf-8")
|
|
4429
|
+
and not allow_auth_retirement
|
|
4430
|
+
):
|
|
4431
|
+
continue
|
|
4432
|
+
paths.append(path)
|
|
4433
|
+
return sorted(paths)
|
|
2304
4434
|
|
|
2305
4435
|
|
|
2306
4436
|
def apply_migration_file(database_url, outputs, vars_json, path):
|
|
@@ -2334,7 +4464,7 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto;
|
|
|
2334
4464
|
CREATE EXTENSION IF NOT EXISTS vector;
|
|
2335
4465
|
""",
|
|
2336
4466
|
)
|
|
2337
|
-
for path in migration_files():
|
|
4467
|
+
for path in migration_files(vars_json):
|
|
2338
4468
|
apply_migration_file(database_url, outputs, vars_json, path)
|
|
2339
4469
|
|
|
2340
4470
|
|
|
@@ -2454,7 +4584,7 @@ def platform_migration_object_present(database_url, kind, name):
|
|
|
2454
4584
|
return None
|
|
2455
4585
|
|
|
2456
4586
|
|
|
2457
|
-
def backfill_platform_migration_ledger(database_url):
|
|
4587
|
+
def backfill_platform_migration_ledger(database_url, vars_json):
|
|
2458
4588
|
"""One-time ledger bootstrap for environments installed before the ledger
|
|
2459
4589
|
existed. Every file shipping with this release is recorded as assumed
|
|
2460
4590
|
applied — auto-re-running old files is unsafe (markers can name objects
|
|
@@ -2463,11 +4593,29 @@ def backfill_platform_migration_ledger(database_url):
|
|
|
2463
4593
|
transition apply through the exact pending path."""
|
|
2464
4594
|
verified = []
|
|
2465
4595
|
assumed = []
|
|
2466
|
-
for path in migration_files():
|
|
4596
|
+
for path in migration_files(vars_json):
|
|
4597
|
+
match = re.match(r"^(\d{4})_", path.name)
|
|
4598
|
+
migration_number = int(match.group(1)) if match else None
|
|
2467
4599
|
verdicts = [
|
|
2468
4600
|
platform_migration_object_present(database_url, kind, name)
|
|
2469
4601
|
for kind, name in declared_migration_objects(path)
|
|
2470
4602
|
]
|
|
4603
|
+
# The ledger ships with the native-auth release. Files through 0260 may
|
|
4604
|
+
# already have been applied by the old deploy paths and cannot safely be
|
|
4605
|
+
# replayed. Files introduced with this release must never be silently
|
|
4606
|
+
# assumed: record them only when every declared object is present;
|
|
4607
|
+
# otherwise leave them pending so apply_pending_platform_migrations runs
|
|
4608
|
+
# them before the app is published.
|
|
4609
|
+
if (
|
|
4610
|
+
migration_number is not None
|
|
4611
|
+
and migration_number > PLATFORM_MIGRATION_LEDGER_CUTOFF
|
|
4612
|
+
and (not verdicts or any(verdict is not True for verdict in verdicts))
|
|
4613
|
+
):
|
|
4614
|
+
print(
|
|
4615
|
+
f"[migrations] transition: {path.name} is newer than the pre-ledger cutoff "
|
|
4616
|
+
"and remains pending"
|
|
4617
|
+
)
|
|
4618
|
+
continue
|
|
2471
4619
|
if any(verdict is False for verdict in verdicts):
|
|
2472
4620
|
print(
|
|
2473
4621
|
f"[migrations] transition WARNING: {path.name} declares objects missing from "
|
|
@@ -2484,7 +4632,7 @@ def backfill_platform_migration_ledger(database_url):
|
|
|
2484
4632
|
|
|
2485
4633
|
def apply_pending_platform_migrations(database_url, outputs, vars_json):
|
|
2486
4634
|
recorded = recorded_platform_migrations(database_url)
|
|
2487
|
-
for path in migration_files():
|
|
4635
|
+
for path in migration_files(vars_json):
|
|
2488
4636
|
if path.name in recorded:
|
|
2489
4637
|
continue
|
|
2490
4638
|
print(f"[migrations] applying {path.name}")
|
|
@@ -2923,31 +5071,6 @@ def pg_literal(value):
|
|
|
2923
5071
|
return "'" + value.replace("'", "''") + "'"
|
|
2924
5072
|
|
|
2925
5073
|
|
|
2926
|
-
def ensure_hindsight_database(database_url, vars_json):
|
|
2927
|
-
"""THINK-220: the dedicated Hindsight database is bootstrap SQL — Terraform
|
|
2928
|
-
cannot CREATE DATABASE inside the cluster. Runs post-apply; the Hindsight
|
|
2929
|
-
service may crash-loop briefly on a first enablement until this creates
|
|
2930
|
-
the database, then ECS stabilizes on its own. Vanilla boot migrations
|
|
2931
|
-
(RUN_MIGRATIONS_ON_STARTUP) build the schema, including the maintenance
|
|
2932
|
-
discovery functions the repair migration installs on base-schema runs."""
|
|
2933
|
-
db_name = (vars_json.get("hindsight_database_name") or "").strip()
|
|
2934
|
-
if not db_name:
|
|
2935
|
-
return
|
|
2936
|
-
if not re.fullmatch(r"[a-z][a-z0-9_]*", db_name):
|
|
2937
|
-
raise RuntimeError(f"invalid hindsight_database_name: {db_name!r}")
|
|
2938
|
-
admin_url = re.sub(r"/[^/?]+(\?|$)", r"/postgres\1", database_url, count=1)
|
|
2939
|
-
exists = psql_output(
|
|
2940
|
-
admin_url,
|
|
2941
|
-
f"SELECT 1 FROM pg_database WHERE datname = '{db_name}'",
|
|
2942
|
-
).strip()
|
|
2943
|
-
if not exists:
|
|
2944
|
-
print(f"[hindsight] creating database {db_name}")
|
|
2945
|
-
psql(admin_url, sql=f'CREATE DATABASE "{db_name}"')
|
|
2946
|
-
hindsight_url = re.sub(r"/[^/?]+(\?|$)", f"/{db_name}\\1", database_url, count=1)
|
|
2947
|
-
psql(hindsight_url, sql="CREATE EXTENSION IF NOT EXISTS vector;")
|
|
2948
|
-
print(f"[hindsight] database {db_name} ready (pgvector ensured)")
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
5074
|
def push_database_schema(outputs_path, vars_json):
|
|
2952
5075
|
outputs = json.loads(outputs_path.read_text(encoding="utf-8"))
|
|
2953
5076
|
checkout_source(
|
|
@@ -2956,20 +5079,21 @@ def push_database_schema(outputs_path, vars_json):
|
|
|
2956
5079
|
os.environ.get("THINKWORK_RELEASE_VERSION", "main"),
|
|
2957
5080
|
)
|
|
2958
5081
|
database_url = database_url_from_outputs(outputs)
|
|
2959
|
-
|
|
2960
|
-
fresh = not psql_output(database_url, "SELECT to_regclass('public.tenants')").strip()
|
|
5082
|
+
fresh = not psql_output(database_url, "SELECT to_regclass('public.tenants')::text").strip()
|
|
2961
5083
|
ledger_present = bool(
|
|
2962
|
-
psql_output(
|
|
5084
|
+
psql_output(
|
|
5085
|
+
database_url, "SELECT to_regclass('public.platform_schema_migrations')::text"
|
|
5086
|
+
).strip()
|
|
2963
5087
|
)
|
|
2964
5088
|
if fresh:
|
|
2965
5089
|
initialize_greenfield_database(database_url, outputs, vars_json)
|
|
2966
5090
|
ensure_migration_ledger(database_url)
|
|
2967
5091
|
if fresh:
|
|
2968
5092
|
record_platform_migrations(
|
|
2969
|
-
database_url, [path.name for path in migration_files()], "greenfield"
|
|
5093
|
+
database_url, [path.name for path in migration_files(vars_json)], "greenfield"
|
|
2970
5094
|
)
|
|
2971
5095
|
elif not ledger_present:
|
|
2972
|
-
backfill_platform_migration_ledger(database_url)
|
|
5096
|
+
backfill_platform_migration_ledger(database_url, vars_json)
|
|
2973
5097
|
apply_pending_platform_migrations(database_url, outputs, vars_json)
|
|
2974
5098
|
seed_platform_bootstrap_defaults(database_url)
|
|
2975
5099
|
migrations = SOURCE / "packages/database-pg/drizzle"
|
|
@@ -2980,6 +5104,55 @@ def push_database_schema(outputs_path, vars_json):
|
|
|
2980
5104
|
apply_migration_file(database_url, outputs, vars_json, path)
|
|
2981
5105
|
|
|
2982
5106
|
|
|
5107
|
+
def prepare_additive_schema_before_app(payload, vars_json):
|
|
5108
|
+
"""Materialize foundation/data dependencies and apply additive schema before
|
|
5109
|
+
the full app plan can publish Lambdas that consume it. Retirement stays
|
|
5110
|
+
disabled here and, when explicitly requested, runs only after the app
|
|
5111
|
+
retirement phase has deployed."""
|
|
5112
|
+
action = str(payload.get("action") or "").lower()
|
|
5113
|
+
if (
|
|
5114
|
+
action not in {"deploy", "update"}
|
|
5115
|
+
or is_managed_app_operation(payload)
|
|
5116
|
+
or identity_provider_operation(payload) is not None
|
|
5117
|
+
):
|
|
5118
|
+
return None
|
|
5119
|
+
target = "module.thinkwork.module.database"
|
|
5120
|
+
plan = subprocess.run(
|
|
5121
|
+
[
|
|
5122
|
+
"terraform",
|
|
5123
|
+
"plan",
|
|
5124
|
+
f"-target={target}",
|
|
5125
|
+
"-out=tfplan-data",
|
|
5126
|
+
"-no-color",
|
|
5127
|
+
],
|
|
5128
|
+
cwd=TF,
|
|
5129
|
+
text=True,
|
|
5130
|
+
)
|
|
5131
|
+
if plan.returncode != 0:
|
|
5132
|
+
return plan
|
|
5133
|
+
applied = subprocess.run(
|
|
5134
|
+
["terraform", "apply", "-auto-approve", "-no-color", "tfplan-data"],
|
|
5135
|
+
cwd=TF,
|
|
5136
|
+
text=True,
|
|
5137
|
+
)
|
|
5138
|
+
if applied.returncode != 0:
|
|
5139
|
+
return applied
|
|
5140
|
+
outputs_path = TF / "pre-app-outputs.json"
|
|
5141
|
+
outputs_path.write_text(
|
|
5142
|
+
output(["terraform", "output", "-json"], cwd=TF),
|
|
5143
|
+
encoding="utf-8",
|
|
5144
|
+
)
|
|
5145
|
+
additive_vars = dict(vars_json)
|
|
5146
|
+
additive_vars["finalize_auth_retirement"] = False
|
|
5147
|
+
push_database_schema(outputs_path, additive_vars)
|
|
5148
|
+
TERRAFORM_EVIDENCE["schemaOrdering"] = {
|
|
5149
|
+
"status": "succeeded",
|
|
5150
|
+
"target": target,
|
|
5151
|
+
"phase": "foundation-data-additive-before-app",
|
|
5152
|
+
}
|
|
5153
|
+
return applied
|
|
5154
|
+
|
|
5155
|
+
|
|
2983
5156
|
def write_runner_files(payload, runner_secrets):
|
|
2984
5157
|
preserved_config = payload.get("preservedConfig")
|
|
2985
5158
|
if not isinstance(preserved_config, dict):
|
|
@@ -3065,43 +5238,180 @@ def write_runner_files(payload, runner_secrets):
|
|
|
3065
5238
|
if not api_auth_secret:
|
|
3066
5239
|
api_auth_secret = secrets.token_urlsafe(48)
|
|
3067
5240
|
|
|
5241
|
+
auth_state = auth_reconciliation_state(stage)
|
|
5242
|
+
tenant_auth_metadata = identity_provider_desired_connections(
|
|
5243
|
+
payload,
|
|
5244
|
+
stage=stage,
|
|
5245
|
+
account_id=account_id,
|
|
5246
|
+
region=region,
|
|
5247
|
+
previous_state=auth_state,
|
|
5248
|
+
current_outputs=current_outputs,
|
|
5249
|
+
)
|
|
5250
|
+
prior_phase_output = current_outputs.get("auth_retirement_phase")
|
|
5251
|
+
prior_phase = prior_phase_output.get("value") if isinstance(prior_phase_output, dict) else None
|
|
5252
|
+
if prior_phase not in {"coexistence", "cutover", "retired"}:
|
|
5253
|
+
# An existing pre-native-auth stack must enter through coexistence.
|
|
5254
|
+
# A truly empty state is greenfield and has no WorkOS runtime to keep.
|
|
5255
|
+
prior_phase = "coexistence" if current_state else "retired"
|
|
5256
|
+
auth_retirement_phase = safe_get(
|
|
5257
|
+
runner_secrets,
|
|
5258
|
+
"authRetirementPhase",
|
|
5259
|
+
default=safe_get(
|
|
5260
|
+
reviewed_payload,
|
|
5261
|
+
"authRetirementPhase",
|
|
5262
|
+
default=prior_phase,
|
|
5263
|
+
),
|
|
5264
|
+
)
|
|
5265
|
+
if auth_retirement_phase not in {"coexistence", "cutover", "retired"}:
|
|
5266
|
+
raise RuntimeError("authRetirementPhase must be coexistence, cutover, or retired")
|
|
5267
|
+
auth_migration_recovery_deadline = safe_get(
|
|
5268
|
+
runner_secrets,
|
|
5269
|
+
"authMigrationRecoveryDeadline",
|
|
5270
|
+
default=safe_get(
|
|
5271
|
+
reviewed_payload,
|
|
5272
|
+
"authMigrationRecoveryDeadline",
|
|
5273
|
+
default=os.environ.get("AUTH_MIGRATION_RECOVERY_DEADLINE", ""),
|
|
5274
|
+
),
|
|
5275
|
+
).strip()
|
|
5276
|
+
if auth_migration_recovery_deadline:
|
|
5277
|
+
try:
|
|
5278
|
+
parsed_recovery_deadline = datetime.fromisoformat(
|
|
5279
|
+
auth_migration_recovery_deadline.replace("Z", "+00:00")
|
|
5280
|
+
)
|
|
5281
|
+
if parsed_recovery_deadline.tzinfo is None:
|
|
5282
|
+
raise ValueError("timezone required")
|
|
5283
|
+
except ValueError as exc:
|
|
5284
|
+
raise RuntimeError(
|
|
5285
|
+
"authMigrationRecoveryDeadline must be an RFC3339 timestamp"
|
|
5286
|
+
) from exc
|
|
5287
|
+
if auth_retirement_phase == "coexistence" and not auth_migration_recovery_deadline:
|
|
5288
|
+
raise RuntimeError(
|
|
5289
|
+
"authMigrationRecoveryDeadline is required while authRetirementPhase is coexistence"
|
|
5290
|
+
)
|
|
5291
|
+
finalize_auth_retirement = payload.get("finalizeAuthRetirement") is True
|
|
3068
5292
|
vars_json = {
|
|
3069
5293
|
"stage": stage,
|
|
3070
5294
|
"region": region,
|
|
3071
5295
|
"account_id": account_id,
|
|
3072
5296
|
"db_password": db_password,
|
|
3073
5297
|
"api_auth_secret": api_auth_secret,
|
|
5298
|
+
"auth_retirement_phase": auth_retirement_phase,
|
|
5299
|
+
"auth_migration_recovery_deadline": auth_migration_recovery_deadline,
|
|
5300
|
+
"finalize_auth_retirement": finalize_auth_retirement,
|
|
3074
5301
|
"database_engine": safe_get(
|
|
3075
5302
|
reviewed_payload,
|
|
3076
5303
|
"databaseEngine",
|
|
3077
5304
|
default="aurora-serverless",
|
|
3078
5305
|
),
|
|
3079
|
-
|
|
3080
|
-
|
|
5306
|
+
# External S3 KB source R20: customer-owned buckets granted to the
|
|
5307
|
+
# KB service role for s3-connect sources. Per-environment infra
|
|
5308
|
+
# posture — durable home is the runner-secrets document; a payload
|
|
5309
|
+
# list can override per-deploy. Accepts a JSON list or a
|
|
5310
|
+
# comma-separated string (Secrets Manager values often arrive as
|
|
5311
|
+
# strings).
|
|
5312
|
+
"external_kb_source_arns": safe_get_list(
|
|
5313
|
+
runner_secrets,
|
|
3081
5314
|
reviewed_payload,
|
|
3082
|
-
"
|
|
5315
|
+
"externalKbSourceArns",
|
|
5316
|
+
),
|
|
5317
|
+
# VPC endpoint diet (cost review 2026-08-28): which AWS services get
|
|
5318
|
+
# a private interface endpoint, and one ENI or one per AZ. The
|
|
5319
|
+
# private subnets are NAT-routed, so this is network posture, not
|
|
5320
|
+
# reachability. Environment-level — durable home is the
|
|
5321
|
+
# runner-secrets document; the payload can override per-deploy.
|
|
5322
|
+
# Absent = the module's legacy full set across every AZ; the literal
|
|
5323
|
+
# "none" = no interface endpoints at all.
|
|
5324
|
+
"vpc_interface_endpoint_services": safe_get_list(
|
|
5325
|
+
runner_secrets,
|
|
5326
|
+
reviewed_payload,
|
|
5327
|
+
"vpcInterfaceEndpointServices",
|
|
5328
|
+
),
|
|
5329
|
+
"vpc_interface_endpoint_single_az": safe_get_bool(
|
|
5330
|
+
runner_secrets,
|
|
5331
|
+
reviewed_payload,
|
|
5332
|
+
"vpcInterfaceEndpointSingleAz",
|
|
5333
|
+
default=False,
|
|
5334
|
+
),
|
|
5335
|
+
# AgentCore Harness is an environment-level deployment choice. The
|
|
5336
|
+
# controller carries these reviewed, non-secret values forward so a
|
|
5337
|
+
# release update cannot silently remove the managed runtime plane.
|
|
5338
|
+
# The registry module retains its rollout-era multiplayer-proof input
|
|
5339
|
+
# names for compatibility; all provisioning still happens in the
|
|
5340
|
+
# normal Terraform apply owned by this controller.
|
|
5341
|
+
# THINK-1082: run the deployment runner inside the VPC so the psql
|
|
5342
|
+
# schema step reaches Aurora by security group instead of the
|
|
5343
|
+
# world-open 0.0.0.0/0 -> 5432 rule (turns on the VPC NAT gateway).
|
|
5344
|
+
# Environment-level network posture — durable home is the
|
|
5345
|
+
# runner-secrets document (deploymentRunnerInVpc); the controller
|
|
5346
|
+
# payload can override per-deploy. Ship-inert default.
|
|
5347
|
+
"deployment_runner_in_vpc": safe_get_bool(
|
|
5348
|
+
runner_secrets,
|
|
5349
|
+
reviewed_payload,
|
|
5350
|
+
"deploymentRunnerInVpc",
|
|
5351
|
+
default=False,
|
|
5352
|
+
),
|
|
5353
|
+
# THINK-1082: every database-connected Lambda (api handlers,
|
|
5354
|
+
# agentcore-admin, Pi, broker) inside the VPC behind the api-lambda
|
|
5355
|
+
# security group, granted 5432 on the DB group — so the world-open
|
|
5356
|
+
# 0.0.0.0/0 -> 5432 rule can close without taking the app down.
|
|
5357
|
+
# Same homes as deploymentRunnerInVpc: runner-secrets document is
|
|
5358
|
+
# durable, payload overrides per-deploy. Ship-inert default.
|
|
5359
|
+
"api_lambdas_in_vpc": safe_get_bool(
|
|
5360
|
+
runner_secrets,
|
|
5361
|
+
reviewed_payload,
|
|
5362
|
+
"apiLambdasInVpc",
|
|
3083
5363
|
default=False,
|
|
3084
5364
|
),
|
|
3085
|
-
#
|
|
3086
|
-
#
|
|
3087
|
-
#
|
|
3088
|
-
#
|
|
3089
|
-
"
|
|
5365
|
+
# THINK-1082: database exposure. Defaults reproduce today's posture
|
|
5366
|
+
# (public endpoint + world-open 5432 seeded on a new DB security
|
|
5367
|
+
# group). Closed per stage only after both *InVpc flags are live.
|
|
5368
|
+
# dbPublicIngressCidrs accepts a JSON list or a comma-separated
|
|
5369
|
+
# string; an explicit empty string / [] means "seed no rule".
|
|
5370
|
+
"db_publicly_accessible": safe_get_bool(
|
|
5371
|
+
runner_secrets,
|
|
5372
|
+
reviewed_payload,
|
|
5373
|
+
"dbPubliclyAccessible",
|
|
5374
|
+
default=True,
|
|
5375
|
+
),
|
|
5376
|
+
"db_public_ingress_cidrs": safe_get_ingress_cidrs(
|
|
3090
5377
|
runner_secrets,
|
|
3091
5378
|
reviewed_payload,
|
|
3092
|
-
"
|
|
5379
|
+
"dbPublicIngressCidrs",
|
|
5380
|
+
default=["0.0.0.0/0"],
|
|
5381
|
+
),
|
|
5382
|
+
# THINK-1082: how this runner reaches the database for its schema
|
|
5383
|
+
# steps — `psql` (public endpoint, today) or `data-api` (RDS Data API:
|
|
5384
|
+
# HTTPS + IAM, no network path, so the runner can stay outside the
|
|
5385
|
+
# VPC while the public 5432 rule closes). Runner-only: it never
|
|
5386
|
+
# reaches the generated Terraform root (see RUNNER_ONLY_VARS).
|
|
5387
|
+
"db_access": safe_get_db_access(runner_secrets, reviewed_payload),
|
|
5388
|
+
"enable_agentcore_multiplayer_proof": safe_get_bool(
|
|
5389
|
+
{},
|
|
5390
|
+
reviewed_payload,
|
|
5391
|
+
"enableAgentCoreHarness",
|
|
3093
5392
|
default=False,
|
|
3094
5393
|
),
|
|
3095
|
-
|
|
3096
|
-
# Empty keeps the legacy hindsight-schema layout; set (e.g.
|
|
3097
|
-
# "thinkwork_hindsight") points the Hindsight service and platform
|
|
3098
|
-
# readers at that database's public schema, where upstream's
|
|
3099
|
-
# maintenance discovery actually runs.
|
|
3100
|
-
"hindsight_database_name": safe_get(
|
|
5394
|
+
"agentcore_multiplayer_proof_tenant_slug": safe_get(
|
|
3101
5395
|
reviewed_payload,
|
|
3102
|
-
"
|
|
5396
|
+
"agentCoreHarnessTenantSlug",
|
|
3103
5397
|
default="",
|
|
3104
5398
|
),
|
|
5399
|
+
"agentcore_multiplayer_proof_owner_allowlist": safe_get(
|
|
5400
|
+
reviewed_payload,
|
|
5401
|
+
"agentCoreHarnessOwnerAllowlist",
|
|
5402
|
+
default="",
|
|
5403
|
+
),
|
|
5404
|
+
# THINK-585/587 — stage kill-switch for AgentCore Runtime chat
|
|
5405
|
+
# dispatch. Environment posture: durable home is the runner-secrets
|
|
5406
|
+
# document (agentcoreRuntimeDispatchEnabled); a reviewed payload can
|
|
5407
|
+
# override per-deploy. Dark by default — flipping a customer stage is
|
|
5408
|
+
# an explicit operator action.
|
|
5409
|
+
"agentcore_runtime_dispatch_enabled": safe_get_bool(
|
|
5410
|
+
runner_secrets,
|
|
5411
|
+
reviewed_payload,
|
|
5412
|
+
"agentcoreRuntimeDispatchEnabled",
|
|
5413
|
+
default=False,
|
|
5414
|
+
),
|
|
3105
5415
|
"platform_operator_emails": safe_get(
|
|
3106
5416
|
runner_secrets,
|
|
3107
5417
|
"adminEmail",
|
|
@@ -3131,6 +5441,38 @@ def write_runner_files(payload, runner_secrets):
|
|
|
3131
5441
|
default="",
|
|
3132
5442
|
),
|
|
3133
5443
|
),
|
|
5444
|
+
"microsoft_oauth_client_id": safe_get(
|
|
5445
|
+
runner_secrets,
|
|
5446
|
+
"microsoftOauthClientId",
|
|
5447
|
+
default=safe_get(
|
|
5448
|
+
reviewed_payload,
|
|
5449
|
+
"microsoftOauthClientId",
|
|
5450
|
+
default="",
|
|
5451
|
+
),
|
|
5452
|
+
),
|
|
5453
|
+
"microsoft_oauth_client_secret": safe_get(
|
|
5454
|
+
runner_secrets,
|
|
5455
|
+
"microsoftOauthClientSecret",
|
|
5456
|
+
default=safe_get(
|
|
5457
|
+
reviewed_payload,
|
|
5458
|
+
"microsoftOauthClientSecret",
|
|
5459
|
+
default="",
|
|
5460
|
+
),
|
|
5461
|
+
),
|
|
5462
|
+
"microsoft_oauth_tenant": safe_get(
|
|
5463
|
+
runner_secrets,
|
|
5464
|
+
"microsoftOauthTenant",
|
|
5465
|
+
default=safe_get(
|
|
5466
|
+
reviewed_payload,
|
|
5467
|
+
"microsoftOauthTenant",
|
|
5468
|
+
default="",
|
|
5469
|
+
),
|
|
5470
|
+
),
|
|
5471
|
+
# Secret-free desired metadata. The compact Terraform projection only
|
|
5472
|
+
# creates provider-specific public app clients; the runner reconciles
|
|
5473
|
+
# the secret-bearing Cognito IdP before Terraform plans those clients.
|
|
5474
|
+
"tenant_entra_connections": tenant_entra_terraform_projection(tenant_auth_metadata),
|
|
5475
|
+
"auth_tenant_connection_metadata": tenant_auth_metadata,
|
|
3134
5476
|
"cognito_email_source_arn": safe_get(
|
|
3135
5477
|
runner_secrets,
|
|
3136
5478
|
"cognitoEmailSourceArn",
|
|
@@ -3185,6 +5527,20 @@ def write_runner_files(payload, runner_secrets):
|
|
|
3185
5527
|
"customerDomainLegacyRetired",
|
|
3186
5528
|
default=False,
|
|
3187
5529
|
),
|
|
5530
|
+
# Extra web-family OAuth callback/logout origins (e.g. the customer's
|
|
5531
|
+
# Company Brain console CloudFront domain, hosted in their own account).
|
|
5532
|
+
# Persisted in runner-secrets so they survive every controller apply
|
|
5533
|
+
# instead of being stripped back to the computed defaults.
|
|
5534
|
+
"additional_admin_callback_urls": runner_secret_string_list(
|
|
5535
|
+
runner_secrets,
|
|
5536
|
+
reviewed_payload,
|
|
5537
|
+
"adminCallbackUrls",
|
|
5538
|
+
),
|
|
5539
|
+
"additional_admin_logout_urls": runner_secret_string_list(
|
|
5540
|
+
runner_secrets,
|
|
5541
|
+
reviewed_payload,
|
|
5542
|
+
"adminLogoutUrls",
|
|
5543
|
+
),
|
|
3188
5544
|
"lambda_artifact_bucket": os.environ["THINKWORK_RELEASE_ARTIFACT_BUCKET"],
|
|
3189
5545
|
"lambda_artifact_prefix": f"releases/{release_version}/lambdas",
|
|
3190
5546
|
"deployment_release_version": release_version,
|
|
@@ -3201,19 +5557,89 @@ def write_runner_files(payload, runner_secrets):
|
|
|
3201
5557
|
"deployment_terraform_module_source": terraform_module_source,
|
|
3202
5558
|
"deployment_terraform_module_version": terraform_module_version,
|
|
3203
5559
|
"agentcore_pi_source_image_uri": resolve_agentcore_pi_source_image_uri(payload),
|
|
3204
|
-
|
|
3205
|
-
|
|
3206
|
-
|
|
3207
|
-
|
|
3208
|
-
|
|
3209
|
-
|
|
3210
|
-
|
|
3211
|
-
|
|
3212
|
-
|
|
3213
|
-
|
|
3214
|
-
|
|
3215
|
-
|
|
3216
|
-
|
|
5560
|
+
# Account-level Lambda memory ceiling. Legacy AWS accounts cap
|
|
5561
|
+
# function memory at 3008 MB (raised only via support case); set the
|
|
5562
|
+
# runner-secrets key lambdaMaxMemoryMb on such accounts so handlers
|
|
5563
|
+
# that request more are clamped instead of failing mid-apply.
|
|
5564
|
+
"lambda_max_memory_mb": safe_get(
|
|
5565
|
+
runner_secrets,
|
|
5566
|
+
"lambdaMaxMemoryMb",
|
|
5567
|
+
default=safe_get(reviewed_payload, "lambdaMaxMemoryMb", default=10240),
|
|
5568
|
+
),
|
|
5569
|
+
# Digital Twin Neptune wiring (THINK-334): values come from the
|
|
5570
|
+
# etl-platform neptune stack outputs, written into the runner-secrets
|
|
5571
|
+
# document by `thinkwork twin install`. Empty = twin disabled.
|
|
5572
|
+
"neptune_endpoint": safe_get(
|
|
5573
|
+
runner_secrets,
|
|
5574
|
+
"neptuneEndpoint",
|
|
5575
|
+
default=safe_get(reviewed_payload, "neptuneEndpoint", default=""),
|
|
5576
|
+
),
|
|
5577
|
+
"neptune_cluster_resource_id": safe_get(
|
|
5578
|
+
runner_secrets,
|
|
5579
|
+
"neptuneClusterResourceId",
|
|
5580
|
+
default=safe_get(reviewed_payload, "neptuneClusterResourceId", default=""),
|
|
5581
|
+
),
|
|
5582
|
+
"neptune_client_security_group_id": safe_get(
|
|
5583
|
+
runner_secrets,
|
|
5584
|
+
"neptuneClientSecurityGroupId",
|
|
5585
|
+
default=safe_get(reviewed_payload, "neptuneClientSecurityGroupId", default=""),
|
|
5586
|
+
),
|
|
5587
|
+
"neptune_load_bucket": safe_get(
|
|
5588
|
+
runner_secrets,
|
|
5589
|
+
"neptuneLoadBucket",
|
|
5590
|
+
default=safe_get(reviewed_payload, "neptuneLoadBucket", default=""),
|
|
5591
|
+
),
|
|
5592
|
+
"neptune_loader_role_arn": safe_get(
|
|
5593
|
+
runner_secrets,
|
|
5594
|
+
"neptuneLoaderRoleArn",
|
|
5595
|
+
default=safe_get(reviewed_payload, "neptuneLoaderRoleArn", default=""),
|
|
5596
|
+
),
|
|
5597
|
+
# Platform Brain MCP endpoint (consolidation U14). Without this
|
|
5598
|
+
# mapping, customer stages never set BRAIN_MCP_URL and twin-connector
|
|
5599
|
+
# provisioning falls back to the retired product /mcp/twin route.
|
|
5600
|
+
"brain_mcp_url": safe_get(
|
|
5601
|
+
runner_secrets,
|
|
5602
|
+
"brainMcpUrl",
|
|
5603
|
+
default=safe_get(reviewed_payload, "brainMcpUrl", default=""),
|
|
5604
|
+
),
|
|
5605
|
+
# Company Brain platform-agent machine lane (THINK-628/THINK-710).
|
|
5606
|
+
# Secrets Manager ARN of etl-platform/brain-mcp/m2m/platform-agent in
|
|
5607
|
+
# this account; grants the shared API Lambda role read so MCP config
|
|
5608
|
+
# build can mint Cognito bearers for the Brain connector. Environment
|
|
5609
|
+
# posture: durable home is the runner-secrets document. Empty leaves
|
|
5610
|
+
# the tkt_ lane untouched.
|
|
5611
|
+
"brain_m2m_platform_agent_secret_arn": safe_get(
|
|
5612
|
+
runner_secrets,
|
|
5613
|
+
"brainM2mPlatformAgentSecretArn",
|
|
5614
|
+
default=safe_get(reviewed_payload, "brainM2mPlatformAgentSecretArn", default=""),
|
|
5615
|
+
),
|
|
5616
|
+
# Brain ops-api flag intake (THINK-781): base URL + agent-identity
|
|
5617
|
+
# m2m secret ARN for flagThreadToBrain's POST /flags. Empty leaves
|
|
5618
|
+
# the Send-to-the-Brain action reporting "not configured".
|
|
5619
|
+
"brain_ops_api_url": safe_get(
|
|
5620
|
+
runner_secrets,
|
|
5621
|
+
"brainOpsApiUrl",
|
|
5622
|
+
default=safe_get(reviewed_payload, "brainOpsApiUrl", default=""),
|
|
5623
|
+
),
|
|
5624
|
+
"brain_ops_m2m_secret_arn": safe_get(
|
|
5625
|
+
runner_secrets,
|
|
5626
|
+
"brainOpsM2mSecretArn",
|
|
5627
|
+
default=safe_get(reviewed_payload, "brainOpsM2mSecretArn", default=""),
|
|
5628
|
+
),
|
|
5629
|
+
}
|
|
5630
|
+
enforce_customer_domain_preservation(
|
|
5631
|
+
current_outputs,
|
|
5632
|
+
vars_json,
|
|
5633
|
+
payload,
|
|
5634
|
+
runner_secrets,
|
|
5635
|
+
)
|
|
5636
|
+
vars_json.update(
|
|
5637
|
+
managed_app_terraform_overrides(payload, stage, account_id, current_outputs, current_state)
|
|
5638
|
+
)
|
|
5639
|
+
incident_import_blocks = tei_v380_recovery_import_blocks(payload, vars_json)
|
|
5640
|
+
|
|
5641
|
+
TF.mkdir(parents=True, exist_ok=True)
|
|
5642
|
+
(TF / "backend.hcl").write_text(
|
|
3217
5643
|
"\n".join(
|
|
3218
5644
|
[
|
|
3219
5645
|
f"bucket = {hcl_string(os.environ['THINKWORK_TERRAFORM_STATE_BUCKET'])}",
|
|
@@ -3227,13 +5653,20 @@ def write_runner_files(payload, runner_secrets):
|
|
|
3227
5653
|
encoding="utf-8",
|
|
3228
5654
|
)
|
|
3229
5655
|
(TF / "terraform.auto.tfvars.json").write_text(
|
|
3230
|
-
json.dumps(
|
|
5656
|
+
json.dumps(
|
|
5657
|
+
{key: value for key, value in vars_json.items() if key not in RUNNER_ONLY_VARS},
|
|
5658
|
+
indent=2,
|
|
5659
|
+
sort_keys=True,
|
|
5660
|
+
),
|
|
3231
5661
|
encoding="utf-8",
|
|
3232
5662
|
)
|
|
3233
5663
|
(TF / "main.tf").write_text(
|
|
3234
5664
|
f"""
|
|
3235
5665
|
terraform {{
|
|
3236
|
-
|
|
5666
|
+
# The managed deployment runtime is pinned to this floor. Terraform 1.8.5
|
|
5667
|
+
# adds declarative state-only removal, used below to forget the original
|
|
5668
|
+
# non-owning AgentCore reconciliation marker without a destroy action.
|
|
5669
|
+
required_version = ">= 1.8.5"
|
|
3237
5670
|
|
|
3238
5671
|
backend "s3" {{}}
|
|
3239
5672
|
|
|
@@ -3267,6 +5700,21 @@ provider "aws" {{
|
|
|
3267
5700
|
# even though they do not change those records.
|
|
3268
5701
|
provider "cloudflare" {{}}
|
|
3269
5702
|
|
|
5703
|
+
# The original AgentCore Twenty reconciliation marker never owned external
|
|
5704
|
+
# cleanup, but an interrupted local-exec may have left it tainted. Forget every
|
|
5705
|
+
# counted instance from managed-deployment state without refresh, provider RPC,
|
|
5706
|
+
# or destroy provisioners. The child module owns the replacement idempotent
|
|
5707
|
+
# reconciliation marker under a new address.
|
|
5708
|
+
removed {{
|
|
5709
|
+
from = module.thinkwork.module.agentcore_proof_identity.terraform_data.twenty_identity_lifecycle
|
|
5710
|
+
|
|
5711
|
+
lifecycle {{
|
|
5712
|
+
destroy = false
|
|
5713
|
+
}}
|
|
5714
|
+
}}
|
|
5715
|
+
|
|
5716
|
+
{incident_import_blocks}
|
|
5717
|
+
|
|
3270
5718
|
variable "stage" {{
|
|
3271
5719
|
type = string
|
|
3272
5720
|
}}
|
|
@@ -3289,24 +5737,105 @@ variable "api_auth_secret" {{
|
|
|
3289
5737
|
sensitive = true
|
|
3290
5738
|
}}
|
|
3291
5739
|
|
|
5740
|
+
variable "auth_retirement_phase" {{
|
|
5741
|
+
type = string
|
|
5742
|
+
default = "coexistence"
|
|
5743
|
+
}}
|
|
5744
|
+
|
|
5745
|
+
variable "auth_migration_recovery_deadline" {{
|
|
5746
|
+
type = string
|
|
5747
|
+
default = ""
|
|
5748
|
+
}}
|
|
5749
|
+
|
|
3292
5750
|
variable "database_engine" {{
|
|
3293
5751
|
type = string
|
|
3294
5752
|
}}
|
|
3295
5753
|
|
|
3296
|
-
variable "
|
|
3297
|
-
type
|
|
5754
|
+
variable "external_kb_source_arns" {{
|
|
5755
|
+
type = list(string)
|
|
5756
|
+
default = []
|
|
5757
|
+
}}
|
|
5758
|
+
|
|
5759
|
+
variable "vpc_interface_endpoint_services" {{
|
|
5760
|
+
type = list(string)
|
|
5761
|
+
default = []
|
|
3298
5762
|
}}
|
|
3299
5763
|
|
|
3300
|
-
variable "
|
|
5764
|
+
variable "vpc_interface_endpoint_single_az" {{
|
|
3301
5765
|
type = bool
|
|
3302
5766
|
default = false
|
|
3303
5767
|
}}
|
|
3304
5768
|
|
|
3305
|
-
variable "
|
|
5769
|
+
variable "lambda_max_memory_mb" {{
|
|
5770
|
+
type = number
|
|
5771
|
+
default = 10240
|
|
5772
|
+
}}
|
|
5773
|
+
|
|
5774
|
+
variable "neptune_endpoint" {{
|
|
5775
|
+
type = string
|
|
5776
|
+
default = ""
|
|
5777
|
+
}}
|
|
5778
|
+
|
|
5779
|
+
variable "neptune_cluster_resource_id" {{
|
|
3306
5780
|
type = string
|
|
3307
5781
|
default = ""
|
|
3308
5782
|
}}
|
|
3309
5783
|
|
|
5784
|
+
variable "neptune_client_security_group_id" {{
|
|
5785
|
+
type = string
|
|
5786
|
+
default = ""
|
|
5787
|
+
}}
|
|
5788
|
+
|
|
5789
|
+
variable "neptune_load_bucket" {{
|
|
5790
|
+
type = string
|
|
5791
|
+
default = ""
|
|
5792
|
+
}}
|
|
5793
|
+
|
|
5794
|
+
variable "neptune_loader_role_arn" {{
|
|
5795
|
+
type = string
|
|
5796
|
+
default = ""
|
|
5797
|
+
}}
|
|
5798
|
+
|
|
5799
|
+
variable "brain_mcp_url" {{
|
|
5800
|
+
type = string
|
|
5801
|
+
default = ""
|
|
5802
|
+
}}
|
|
5803
|
+
|
|
5804
|
+
variable "brain_m2m_platform_agent_secret_arn" {{
|
|
5805
|
+
type = string
|
|
5806
|
+
default = ""
|
|
5807
|
+
}}
|
|
5808
|
+
|
|
5809
|
+
variable "brain_ops_api_url" {{
|
|
5810
|
+
type = string
|
|
5811
|
+
default = ""
|
|
5812
|
+
}}
|
|
5813
|
+
|
|
5814
|
+
variable "brain_ops_m2m_secret_arn" {{
|
|
5815
|
+
type = string
|
|
5816
|
+
default = ""
|
|
5817
|
+
}}
|
|
5818
|
+
|
|
5819
|
+
variable "enable_agentcore_multiplayer_proof" {{
|
|
5820
|
+
type = bool
|
|
5821
|
+
default = false
|
|
5822
|
+
}}
|
|
5823
|
+
|
|
5824
|
+
variable "agentcore_multiplayer_proof_tenant_slug" {{
|
|
5825
|
+
type = string
|
|
5826
|
+
default = ""
|
|
5827
|
+
}}
|
|
5828
|
+
|
|
5829
|
+
variable "agentcore_multiplayer_proof_owner_allowlist" {{
|
|
5830
|
+
type = string
|
|
5831
|
+
default = ""
|
|
5832
|
+
}}
|
|
5833
|
+
|
|
5834
|
+
variable "agentcore_runtime_dispatch_enabled" {{
|
|
5835
|
+
type = bool
|
|
5836
|
+
default = false
|
|
5837
|
+
}}
|
|
5838
|
+
|
|
3310
5839
|
variable "platform_operator_emails" {{
|
|
3311
5840
|
type = string
|
|
3312
5841
|
}}
|
|
@@ -3343,6 +5872,16 @@ variable "customer_domain_legacy_retired" {{
|
|
|
3343
5872
|
type = bool
|
|
3344
5873
|
}}
|
|
3345
5874
|
|
|
5875
|
+
variable "additional_admin_callback_urls" {{
|
|
5876
|
+
type = list(string)
|
|
5877
|
+
default = []
|
|
5878
|
+
}}
|
|
5879
|
+
|
|
5880
|
+
variable "additional_admin_logout_urls" {{
|
|
5881
|
+
type = list(string)
|
|
5882
|
+
default = []
|
|
5883
|
+
}}
|
|
5884
|
+
|
|
3346
5885
|
variable "google_oauth_client_id" {{
|
|
3347
5886
|
type = string
|
|
3348
5887
|
}}
|
|
@@ -3352,6 +5891,35 @@ variable "google_oauth_client_secret" {{
|
|
|
3352
5891
|
sensitive = true
|
|
3353
5892
|
}}
|
|
3354
5893
|
|
|
5894
|
+
variable "microsoft_oauth_client_id" {{
|
|
5895
|
+
type = string
|
|
5896
|
+
}}
|
|
5897
|
+
|
|
5898
|
+
variable "microsoft_oauth_client_secret" {{
|
|
5899
|
+
type = string
|
|
5900
|
+
sensitive = true
|
|
5901
|
+
}}
|
|
5902
|
+
|
|
5903
|
+
variable "microsoft_oauth_tenant" {{
|
|
5904
|
+
type = string
|
|
5905
|
+
default = ""
|
|
5906
|
+
}}
|
|
5907
|
+
|
|
5908
|
+
variable "tenant_entra_connections" {{
|
|
5909
|
+
type = list(object({{
|
|
5910
|
+
connection_key = string
|
|
5911
|
+
tenant_id = string
|
|
5912
|
+
provider_name = string
|
|
5913
|
+
display_name = string
|
|
5914
|
+
}}))
|
|
5915
|
+
default = []
|
|
5916
|
+
}}
|
|
5917
|
+
|
|
5918
|
+
variable "auth_tenant_connection_metadata" {{
|
|
5919
|
+
type = list(any)
|
|
5920
|
+
default = []
|
|
5921
|
+
}}
|
|
5922
|
+
|
|
3355
5923
|
variable "lambda_artifact_bucket" {{
|
|
3356
5924
|
type = string
|
|
3357
5925
|
}}
|
|
@@ -3412,6 +5980,26 @@ variable "enable_deployment_control_plane" {{
|
|
|
3412
5980
|
type = bool
|
|
3413
5981
|
}}
|
|
3414
5982
|
|
|
5983
|
+
variable "deployment_runner_in_vpc" {{
|
|
5984
|
+
type = bool
|
|
5985
|
+
default = false
|
|
5986
|
+
}}
|
|
5987
|
+
|
|
5988
|
+
variable "api_lambdas_in_vpc" {{
|
|
5989
|
+
type = bool
|
|
5990
|
+
default = false
|
|
5991
|
+
}}
|
|
5992
|
+
|
|
5993
|
+
variable "db_publicly_accessible" {{
|
|
5994
|
+
type = bool
|
|
5995
|
+
default = true
|
|
5996
|
+
}}
|
|
5997
|
+
|
|
5998
|
+
variable "db_public_ingress_cidrs" {{
|
|
5999
|
+
type = list(string)
|
|
6000
|
+
default = ["0.0.0.0/0"]
|
|
6001
|
+
}}
|
|
6002
|
+
|
|
3415
6003
|
variable "twenty_provisioned" {{
|
|
3416
6004
|
type = bool
|
|
3417
6005
|
}}
|
|
@@ -3672,10 +6260,16 @@ module "thinkwork" {{
|
|
|
3672
6260
|
|
|
3673
6261
|
db_password = var.db_password
|
|
3674
6262
|
api_auth_secret = var.api_auth_secret
|
|
6263
|
+
auth_retirement_phase = var.auth_retirement_phase
|
|
6264
|
+
auth_migration_recovery_deadline = var.auth_migration_recovery_deadline
|
|
3675
6265
|
database_engine = var.database_engine
|
|
3676
6266
|
|
|
3677
6267
|
google_oauth_client_id = var.google_oauth_client_id
|
|
3678
6268
|
google_oauth_client_secret = var.google_oauth_client_secret
|
|
6269
|
+
microsoft_oauth_client_id = var.microsoft_oauth_client_id
|
|
6270
|
+
microsoft_oauth_client_secret = var.microsoft_oauth_client_secret
|
|
6271
|
+
microsoft_oauth_tenant = var.microsoft_oauth_tenant
|
|
6272
|
+
tenant_entra_connections = var.tenant_entra_connections
|
|
3679
6273
|
platform_operator_emails = var.platform_operator_emails
|
|
3680
6274
|
|
|
3681
6275
|
cognito_email_source_arn = var.cognito_email_source_arn
|
|
@@ -3689,15 +6283,36 @@ module "thinkwork" {{
|
|
|
3689
6283
|
customer_domain_delegated = var.customer_domain_delegated
|
|
3690
6284
|
customer_domain_legacy_retired = var.customer_domain_legacy_retired
|
|
3691
6285
|
|
|
6286
|
+
additional_admin_callback_urls = var.additional_admin_callback_urls
|
|
6287
|
+
additional_admin_logout_urls = var.additional_admin_logout_urls
|
|
6288
|
+
|
|
3692
6289
|
lambda_artifact_bucket = var.lambda_artifact_bucket
|
|
3693
6290
|
lambda_artifact_prefix = var.lambda_artifact_prefix
|
|
3694
6291
|
require_lambda_artifacts = true
|
|
3695
6292
|
agentcore_pi_source_image_uri = var.agentcore_pi_source_image_uri
|
|
3696
6293
|
|
|
3697
|
-
|
|
3698
|
-
|
|
3699
|
-
|
|
6294
|
+
external_kb_source_arns = var.external_kb_source_arns
|
|
6295
|
+
vpc_interface_endpoint_services = (
|
|
6296
|
+
length(var.vpc_interface_endpoint_services) == 0
|
|
6297
|
+
? null
|
|
6298
|
+
: (contains(var.vpc_interface_endpoint_services, "none") ? [] : var.vpc_interface_endpoint_services)
|
|
6299
|
+
)
|
|
6300
|
+
vpc_interface_endpoint_single_az = var.vpc_interface_endpoint_single_az
|
|
6301
|
+
lambda_max_memory_mb = var.lambda_max_memory_mb
|
|
6302
|
+
neptune_endpoint = var.neptune_endpoint
|
|
6303
|
+
neptune_cluster_resource_id = var.neptune_cluster_resource_id
|
|
6304
|
+
neptune_client_security_group_id = var.neptune_client_security_group_id
|
|
6305
|
+
neptune_load_bucket = var.neptune_load_bucket
|
|
6306
|
+
neptune_loader_role_arn = var.neptune_loader_role_arn
|
|
6307
|
+
brain_mcp_url = var.brain_mcp_url
|
|
6308
|
+
brain_m2m_platform_agent_secret_arn = var.brain_m2m_platform_agent_secret_arn
|
|
6309
|
+
brain_ops_api_url = var.brain_ops_api_url
|
|
6310
|
+
brain_ops_m2m_secret_arn = var.brain_ops_m2m_secret_arn
|
|
3700
6311
|
enable_workspace_orchestration = true
|
|
6312
|
+
enable_agentcore_multiplayer_proof = var.enable_agentcore_multiplayer_proof
|
|
6313
|
+
agentcore_multiplayer_proof_tenant_slug = var.agentcore_multiplayer_proof_tenant_slug
|
|
6314
|
+
agentcore_multiplayer_proof_owner_allowlist = var.agentcore_multiplayer_proof_owner_allowlist
|
|
6315
|
+
agentcore_runtime_dispatch_enabled = var.agentcore_runtime_dispatch_enabled
|
|
3701
6316
|
|
|
3702
6317
|
twenty_provisioned = var.twenty_provisioned
|
|
3703
6318
|
twenty_runtime_enabled = var.twenty_runtime_enabled
|
|
@@ -3747,6 +6362,10 @@ module "thinkwork" {{
|
|
|
3747
6362
|
enable_slack_workspace_app = false
|
|
3748
6363
|
|
|
3749
6364
|
enable_deployment_control_plane = var.enable_deployment_control_plane
|
|
6365
|
+
deployment_runner_in_vpc = var.deployment_runner_in_vpc
|
|
6366
|
+
api_lambdas_in_vpc = var.api_lambdas_in_vpc
|
|
6367
|
+
db_publicly_accessible = var.db_publicly_accessible
|
|
6368
|
+
db_public_ingress_cidrs = var.db_public_ingress_cidrs
|
|
3750
6369
|
deployment_control_plane_create_secret_placeholders = var.deployment_control_plane_create_secret_placeholders
|
|
3751
6370
|
deployment_state_machine_arn = var.deployment_state_machine_arn
|
|
3752
6371
|
deployment_evidence_bucket = var.deployment_evidence_bucket
|
|
@@ -3778,21 +6397,18 @@ output "app_distribution_id" {{ value = module.thinkwork.app_distribution_id }}
|
|
|
3778
6397
|
output "api_endpoint" {{ value = module.thinkwork.api_endpoint }}
|
|
3779
6398
|
output "appsync_api_url" {{ value = module.thinkwork.appsync_api_url }}
|
|
3780
6399
|
output "appsync_realtime_url" {{ value = module.thinkwork.appsync_realtime_url }}
|
|
3781
|
-
output "appsync_api_key" {{
|
|
3782
|
-
value = module.thinkwork.appsync_api_key
|
|
3783
|
-
sensitive = true
|
|
3784
|
-
}}
|
|
3785
6400
|
output "auth_domain" {{ value = module.thinkwork.auth_domain }}
|
|
3786
6401
|
output "customer_domain" {{ value = module.thinkwork.customer_domain }}
|
|
3787
6402
|
output "customer_domain_name_servers" {{ value = module.thinkwork.customer_domain_name_servers }}
|
|
6403
|
+
output "db_cluster_arn" {{ value = module.thinkwork.db_cluster_arn }}
|
|
3788
6404
|
output "db_cluster_endpoint" {{ value = module.thinkwork.db_cluster_endpoint }}
|
|
3789
6405
|
output "db_secret_arn" {{ value = module.thinkwork.db_secret_arn }}
|
|
3790
6406
|
output "database_name" {{ value = module.thinkwork.database_name }}
|
|
3791
6407
|
output "user_pool_id" {{ value = module.thinkwork.user_pool_id }}
|
|
3792
6408
|
output "admin_client_id" {{ value = module.thinkwork.admin_client_id }}
|
|
3793
|
-
|
|
3794
|
-
|
|
3795
|
-
|
|
6409
|
+
output "web_local_client_id" {{ value = module.thinkwork.web_local_client_id }}
|
|
6410
|
+
output "auth_route_clients" {{ value = module.thinkwork.auth_route_clients }}
|
|
6411
|
+
output "auth_retirement_phase" {{ value = module.thinkwork.auth_retirement_phase }}
|
|
3796
6412
|
output "twenty_provisioned" {{ value = module.thinkwork.twenty_provisioned }}
|
|
3797
6413
|
output "twenty_runtime_enabled" {{ value = module.thinkwork.twenty_runtime_enabled }}
|
|
3798
6414
|
output "twenty_url" {{ value = module.thinkwork.twenty_url }}
|
|
@@ -3833,7 +6449,7 @@ output "deployment_appconfig_configuration_profile_id" {{ value = module.thinkwo
|
|
|
3833
6449
|
|
|
3834
6450
|
def sync_release_artifacts(artifact_types=None, artifact_names=None):
|
|
3835
6451
|
global RELEASE_EVIDENCE
|
|
3836
|
-
artifact_types =
|
|
6452
|
+
artifact_types = {"lambda", "static-site"} if artifact_types is None else set(artifact_types)
|
|
3837
6453
|
artifact_names = set(artifact_names or [])
|
|
3838
6454
|
manifest_url = os.environ.get("THINKWORK_RELEASE_MANIFEST_URL")
|
|
3839
6455
|
expected = os.environ.get("THINKWORK_RELEASE_MANIFEST_SHA256", "").lower()
|
|
@@ -4004,6 +6620,69 @@ def write_current_outputs_from_state(stage, outputs_path):
|
|
|
4004
6620
|
return outputs
|
|
4005
6621
|
|
|
4006
6622
|
|
|
6623
|
+
def ensure_native_auth_custom_attributes(current_outputs):
|
|
6624
|
+
"""Add the native Entra claims to an existing pool before app clients use them."""
|
|
6625
|
+
user_pool_id = str(_current_output_value(current_outputs, "user_pool_id") or "").strip()
|
|
6626
|
+
if not user_pool_id:
|
|
6627
|
+
return {"status": "skipped", "reason": "user-pool-not-created"}
|
|
6628
|
+
|
|
6629
|
+
def describe_attribute_names():
|
|
6630
|
+
raw = output(
|
|
6631
|
+
[
|
|
6632
|
+
"aws",
|
|
6633
|
+
"cognito-idp",
|
|
6634
|
+
"describe-user-pool",
|
|
6635
|
+
"--user-pool-id",
|
|
6636
|
+
user_pool_id,
|
|
6637
|
+
"--output",
|
|
6638
|
+
"json",
|
|
6639
|
+
]
|
|
6640
|
+
)
|
|
6641
|
+
described = json.loads(raw or "{}")
|
|
6642
|
+
attributes = described.get("UserPool", {}).get("SchemaAttributes", [])
|
|
6643
|
+
return {
|
|
6644
|
+
str(attribute.get("Name") or "")
|
|
6645
|
+
for attribute in attributes
|
|
6646
|
+
if isinstance(attribute, dict)
|
|
6647
|
+
}
|
|
6648
|
+
|
|
6649
|
+
existing_names = describe_attribute_names()
|
|
6650
|
+
missing = [
|
|
6651
|
+
attribute
|
|
6652
|
+
for attribute in NATIVE_AUTH_CUSTOM_ATTRIBUTES
|
|
6653
|
+
if f"custom:{attribute['Name']}" not in existing_names
|
|
6654
|
+
]
|
|
6655
|
+
if not missing:
|
|
6656
|
+
return {"status": "current", "userPoolId": user_pool_id, "added": []}
|
|
6657
|
+
|
|
6658
|
+
request = {"UserPoolId": user_pool_id, "CustomAttributes": missing}
|
|
6659
|
+
run(
|
|
6660
|
+
[
|
|
6661
|
+
"aws",
|
|
6662
|
+
"cognito-idp",
|
|
6663
|
+
"add-custom-attributes",
|
|
6664
|
+
"--cli-input-json",
|
|
6665
|
+
json.dumps(request, separators=(",", ":"), sort_keys=True),
|
|
6666
|
+
],
|
|
6667
|
+
stdout=subprocess.DEVNULL,
|
|
6668
|
+
)
|
|
6669
|
+
reconciled_names = describe_attribute_names()
|
|
6670
|
+
unresolved = [
|
|
6671
|
+
attribute["Name"]
|
|
6672
|
+
for attribute in missing
|
|
6673
|
+
if f"custom:{attribute['Name']}" not in reconciled_names
|
|
6674
|
+
]
|
|
6675
|
+
if unresolved:
|
|
6676
|
+
raise RuntimeError(
|
|
6677
|
+
"Cognito native-auth custom attributes were not added: " + ", ".join(unresolved)
|
|
6678
|
+
)
|
|
6679
|
+
return {
|
|
6680
|
+
"status": "reconciled",
|
|
6681
|
+
"userPoolId": user_pool_id,
|
|
6682
|
+
"added": [attribute["Name"] for attribute in missing],
|
|
6683
|
+
}
|
|
6684
|
+
|
|
6685
|
+
|
|
4007
6686
|
def write_outputs_to_ssm(outputs_path, vars_json):
|
|
4008
6687
|
outputs = json.loads(outputs_path.read_text(encoding="utf-8"))
|
|
4009
6688
|
profile, web_env = runtime_profile(outputs, vars_json)
|
|
@@ -4013,7 +6692,6 @@ def write_outputs_to_ssm(outputs_path, vars_json):
|
|
|
4013
6692
|
"profile/graphql-http-url": None,
|
|
4014
6693
|
"profile/appsync-url": "appsync_api_url",
|
|
4015
6694
|
"profile/appsync-realtime-url": "appsync_realtime_url",
|
|
4016
|
-
"profile/appsync-api-key": "appsync_api_key",
|
|
4017
6695
|
"profile/cognito-domain": None,
|
|
4018
6696
|
"profile/cognito-user-pool-id": "user_pool_id",
|
|
4019
6697
|
"profile/cognito-client-id": "admin_client_id",
|
|
@@ -4154,7 +6832,6 @@ def runtime_profile(outputs, vars_json):
|
|
|
4154
6832
|
"graphqlHttpUrl": graphql_http_url,
|
|
4155
6833
|
"appsyncUrl": output_value("appsync_api_url"),
|
|
4156
6834
|
"appsyncRealtimeUrl": output_value("appsync_realtime_url"),
|
|
4157
|
-
"appsyncApiKey": output_value("appsync_api_key"),
|
|
4158
6835
|
"cognitoDomain": cognito_domain,
|
|
4159
6836
|
"cognitoUserPoolId": output_value("user_pool_id"),
|
|
4160
6837
|
"cognitoClientId": output_value("admin_client_id"),
|
|
@@ -4185,7 +6862,6 @@ def runtime_profile(outputs, vars_json):
|
|
|
4185
6862
|
"VITE_GRAPHQL_HTTP_URL": profile["graphqlHttpUrl"],
|
|
4186
6863
|
"VITE_GRAPHQL_URL": profile["appsyncUrl"],
|
|
4187
6864
|
"VITE_GRAPHQL_WS_URL": profile["appsyncRealtimeUrl"],
|
|
4188
|
-
"VITE_GRAPHQL_API_KEY": profile["appsyncApiKey"],
|
|
4189
6865
|
"VITE_COGNITO_DOMAIN": profile["cognitoDomain"],
|
|
4190
6866
|
"VITE_COGNITO_USER_POOL_ID": profile["cognitoUserPoolId"],
|
|
4191
6867
|
"VITE_COGNITO_CLIENT_ID": profile["cognitoClientId"],
|
|
@@ -4211,12 +6887,709 @@ def runtime_profile(outputs, vars_json):
|
|
|
4211
6887
|
return profile, web_env + "\n"
|
|
4212
6888
|
|
|
4213
6889
|
|
|
6890
|
+
IDENTITY_PROVIDER_ACTIONS = {"create", "validate", "rotate", "disable"}
|
|
6891
|
+
ENTRA_TENANT_GUID_RE = re.compile(
|
|
6892
|
+
r"^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$",
|
|
6893
|
+
re.IGNORECASE,
|
|
6894
|
+
)
|
|
6895
|
+
ENTRA_PROVIDER_NAME_RE = re.compile(r"^Entra_[a-f0-9]{16}_[a-f0-9]{8}$")
|
|
6896
|
+
|
|
6897
|
+
|
|
6898
|
+
def identity_provider_operation(payload):
|
|
6899
|
+
operation = payload.get("operation")
|
|
6900
|
+
if not isinstance(operation, dict) or operation.get("kind") != "identity_provider":
|
|
6901
|
+
return None
|
|
6902
|
+
return operation
|
|
6903
|
+
|
|
6904
|
+
|
|
6905
|
+
def _identity_string(value, label):
|
|
6906
|
+
if not isinstance(value, str) or not value.strip():
|
|
6907
|
+
raise RuntimeError(f"Identity-provider operation requires {label}")
|
|
6908
|
+
return value.strip()
|
|
6909
|
+
|
|
6910
|
+
|
|
6911
|
+
def _current_output_value(current_outputs, key):
|
|
6912
|
+
value = current_outputs.get(key) if isinstance(current_outputs, dict) else None
|
|
6913
|
+
if isinstance(value, dict) and "value" in value:
|
|
6914
|
+
return value.get("value")
|
|
6915
|
+
return value
|
|
6916
|
+
|
|
6917
|
+
|
|
6918
|
+
def validate_identity_provider_secret_arn(secret_arn, stage, account_id, region):
|
|
6919
|
+
expected = f"arn:aws:secretsmanager:{region}:{account_id}:secret:thinkwork/{stage}/auth/entra/"
|
|
6920
|
+
if not isinstance(secret_arn, str) or not secret_arn.startswith(expected):
|
|
6921
|
+
raise RuntimeError(
|
|
6922
|
+
"Tenant Entra secret ARN must belong to this account, region, stage, "
|
|
6923
|
+
"and thinkwork/<stage>/auth/entra namespace"
|
|
6924
|
+
)
|
|
6925
|
+
return secret_arn
|
|
6926
|
+
|
|
6927
|
+
|
|
6928
|
+
def validate_identity_provider_connection(raw, stage, account_id, region, user_pool_id):
|
|
6929
|
+
if not isinstance(raw, dict):
|
|
6930
|
+
raise RuntimeError("Identity-provider operation requires safe connection metadata")
|
|
6931
|
+
tenant_id = _identity_string(raw.get("tenantId"), "connection.tenantId").lower()
|
|
6932
|
+
if not ENTRA_TENANT_GUID_RE.fullmatch(tenant_id):
|
|
6933
|
+
raise RuntimeError("Tenant Entra directory ID must be a GUID")
|
|
6934
|
+
provider_name = _identity_string(raw.get("providerName"), "connection.providerName")
|
|
6935
|
+
if not ENTRA_PROVIDER_NAME_RE.fullmatch(provider_name):
|
|
6936
|
+
raise RuntimeError("Tenant Entra provider name is not deterministic")
|
|
6937
|
+
connection_key = f"microsoft:tenant:{tenant_id}"
|
|
6938
|
+
if raw.get("connectionKey") != connection_key:
|
|
6939
|
+
raise RuntimeError("Tenant Entra connection key does not match its directory ID")
|
|
6940
|
+
client_id = _identity_string(raw.get("clientId"), "connection.clientId")
|
|
6941
|
+
display_name = _identity_string(raw.get("displayName"), "connection.displayName")
|
|
6942
|
+
secret_arn = validate_identity_provider_secret_arn(
|
|
6943
|
+
raw.get("clientSecretRef"), stage, account_id, region
|
|
6944
|
+
)
|
|
6945
|
+
client_secret_version_stage = raw.get("clientSecretVersionStage")
|
|
6946
|
+
if client_secret_version_stage is not None and client_secret_version_stage != "AWSPENDING":
|
|
6947
|
+
raise RuntimeError("Tenant Entra clientSecretVersionStage must be AWSPENDING when supplied")
|
|
6948
|
+
bindings = raw.get("tenantBindings")
|
|
6949
|
+
if not isinstance(bindings, list) or len(bindings) != 1:
|
|
6950
|
+
raise RuntimeError("Tenant Entra connection requires exactly one tenant binding")
|
|
6951
|
+
binding = bindings[0]
|
|
6952
|
+
if not isinstance(binding, dict):
|
|
6953
|
+
raise RuntimeError("Tenant Entra binding must be an object")
|
|
6954
|
+
thinkwork_tenant_id = _identity_string(binding.get("tenantId"), "tenant binding ID")
|
|
6955
|
+
try:
|
|
6956
|
+
uuid.UUID(thinkwork_tenant_id)
|
|
6957
|
+
except (ValueError, AttributeError) as error:
|
|
6958
|
+
raise RuntimeError("ThinkWork tenant binding ID must be a UUID") from error
|
|
6959
|
+
hostnames = binding.get("hostnames")
|
|
6960
|
+
if not isinstance(hostnames, list) or not hostnames:
|
|
6961
|
+
raise RuntimeError("Tenant Entra binding requires at least one hostname")
|
|
6962
|
+
normalized_hostnames = []
|
|
6963
|
+
for value in hostnames:
|
|
6964
|
+
hostname = _identity_string(value, "tenant binding hostname").lower().rstrip(".")
|
|
6965
|
+
if not re.fullmatch(r"[a-z0-9](?:[a-z0-9.-]{0,251}[a-z0-9])?", hostname):
|
|
6966
|
+
raise RuntimeError(f"Invalid tenant auth hostname: {hostname}")
|
|
6967
|
+
normalized_hostnames.append(hostname)
|
|
6968
|
+
issuer_url = f"https://login.microsoftonline.com/{tenant_id}/v2.0"
|
|
6969
|
+
if raw.get("issuerUrl") != issuer_url:
|
|
6970
|
+
raise RuntimeError("Tenant Entra issuer must use its tenant GUID v2.0 authority")
|
|
6971
|
+
return {
|
|
6972
|
+
"connectionKey": connection_key,
|
|
6973
|
+
"providerKey": "microsoft",
|
|
6974
|
+
"providerKind": "microsoft_tenant",
|
|
6975
|
+
"displayName": display_name,
|
|
6976
|
+
"lifecycleState": "native",
|
|
6977
|
+
"cognitoUserPoolId": _identity_string(user_pool_id, "Cognito user pool ID"),
|
|
6978
|
+
"cognitoIdentityProviderName": provider_name,
|
|
6979
|
+
"issuerUrl": issuer_url,
|
|
6980
|
+
"clientId": client_id,
|
|
6981
|
+
"clientSecretRef": secret_arn,
|
|
6982
|
+
**(
|
|
6983
|
+
{"clientSecretVersionStage": client_secret_version_stage}
|
|
6984
|
+
if client_secret_version_stage is not None
|
|
6985
|
+
else {}
|
|
6986
|
+
),
|
|
6987
|
+
"authorizeScopes": "openid email profile",
|
|
6988
|
+
"tenantBindings": [
|
|
6989
|
+
{
|
|
6990
|
+
"tenantId": thinkwork_tenant_id,
|
|
6991
|
+
"label": _identity_string(binding.get("label"), "tenant binding label"),
|
|
6992
|
+
"hostnames": sorted(set(normalized_hostnames)),
|
|
6993
|
+
"status": "enabled",
|
|
6994
|
+
}
|
|
6995
|
+
],
|
|
6996
|
+
# Kept in runner/SSM safe state for the Terraform projection only.
|
|
6997
|
+
"tenantDirectoryId": tenant_id,
|
|
6998
|
+
}
|
|
6999
|
+
|
|
7000
|
+
|
|
7001
|
+
def identity_provider_desired_connections(
|
|
7002
|
+
payload, *, stage, account_id, region, previous_state, current_outputs
|
|
7003
|
+
):
|
|
7004
|
+
previous = previous_state.get("tenantConnections") or []
|
|
7005
|
+
if not isinstance(previous, list):
|
|
7006
|
+
raise RuntimeError("Auth reconciliation tenantConnections state must be an array")
|
|
7007
|
+
desired = [dict(value) for value in previous if isinstance(value, dict)]
|
|
7008
|
+
operation = identity_provider_operation(payload)
|
|
7009
|
+
if operation is None:
|
|
7010
|
+
return desired
|
|
7011
|
+
|
|
7012
|
+
action = str(operation.get("action") or "").lower()
|
|
7013
|
+
if action not in IDENTITY_PROVIDER_ACTIONS:
|
|
7014
|
+
raise RuntimeError(f"Unsupported identity-provider action: {action}")
|
|
7015
|
+
expected_previous = int(operation.get("expectedPreviousRevision", -1))
|
|
7016
|
+
revision = int(operation.get("revision", -1))
|
|
7017
|
+
current_revision = int(previous_state.get("desiredRevision") or 0)
|
|
7018
|
+
if expected_previous != current_revision or revision != current_revision + 1:
|
|
7019
|
+
raise RuntimeError(f"Identity-provider revision conflict: expected {current_revision + 1}")
|
|
7020
|
+
|
|
7021
|
+
user_pool_id = _current_output_value(current_outputs, "user_pool_id")
|
|
7022
|
+
connection = validate_identity_provider_connection(
|
|
7023
|
+
operation.get("connection"), stage, account_id, region, user_pool_id
|
|
7024
|
+
)
|
|
7025
|
+
client_secret_version_stage = connection.get("clientSecretVersionStage")
|
|
7026
|
+
if action == "rotate" and client_secret_version_stage != "AWSPENDING":
|
|
7027
|
+
raise RuntimeError("Tenant Entra rotation requires clientSecretVersionStage AWSPENDING")
|
|
7028
|
+
if action != "rotate" and client_secret_version_stage is not None:
|
|
7029
|
+
raise RuntimeError(
|
|
7030
|
+
"clientSecretVersionStage AWSPENDING is valid only for Tenant Entra rotation"
|
|
7031
|
+
)
|
|
7032
|
+
existing = next(
|
|
7033
|
+
(value for value in desired if value.get("connectionKey") == connection["connectionKey"]),
|
|
7034
|
+
None,
|
|
7035
|
+
)
|
|
7036
|
+
if action == "create" and existing and existing.get("lifecycleState") != "denied":
|
|
7037
|
+
raise RuntimeError("Tenant Entra connection already exists; use rotate or validate")
|
|
7038
|
+
if action in {"validate", "rotate", "disable"} and not existing:
|
|
7039
|
+
raise RuntimeError(f"Tenant Entra connection does not exist for {action}")
|
|
7040
|
+
if action in {"validate", "rotate", "disable"} and existing != connection:
|
|
7041
|
+
# Disable may change lifecycle; rotate may add only its runner-owned
|
|
7042
|
+
# pending-version label. Safe connection metadata must remain exact.
|
|
7043
|
+
comparable = dict(existing)
|
|
7044
|
+
comparable["lifecycleState"] = "native"
|
|
7045
|
+
submitted_comparable = dict(connection)
|
|
7046
|
+
comparable.pop("clientSecretVersionStage", None)
|
|
7047
|
+
submitted_comparable.pop("clientSecretVersionStage", None)
|
|
7048
|
+
if comparable != submitted_comparable:
|
|
7049
|
+
raise RuntimeError(
|
|
7050
|
+
"Identity-provider metadata differs from the active desired connection"
|
|
7051
|
+
)
|
|
7052
|
+
|
|
7053
|
+
desired = [
|
|
7054
|
+
value for value in desired if value.get("connectionKey") != connection["connectionKey"]
|
|
7055
|
+
]
|
|
7056
|
+
if action == "disable":
|
|
7057
|
+
connection["lifecycleState"] = "denied"
|
|
7058
|
+
connection["tenantBindings"][0]["status"] = "disabled"
|
|
7059
|
+
desired.append(connection)
|
|
7060
|
+
return sorted(desired, key=lambda value: value["connectionKey"])
|
|
7061
|
+
|
|
7062
|
+
|
|
7063
|
+
def tenant_entra_terraform_projection(connections):
|
|
7064
|
+
projected = []
|
|
7065
|
+
for connection in connections:
|
|
7066
|
+
if connection.get("lifecycleState") != "native":
|
|
7067
|
+
continue
|
|
7068
|
+
projected.append(
|
|
7069
|
+
{
|
|
7070
|
+
"connection_key": connection["connectionKey"],
|
|
7071
|
+
"tenant_id": connection["tenantDirectoryId"],
|
|
7072
|
+
"provider_name": connection["cognitoIdentityProviderName"],
|
|
7073
|
+
"display_name": connection["displayName"],
|
|
7074
|
+
}
|
|
7075
|
+
)
|
|
7076
|
+
return projected
|
|
7077
|
+
|
|
7078
|
+
|
|
7079
|
+
def native_auth_reconciliation_payload(outputs, vars_json, previous_state=None):
|
|
7080
|
+
"""Build the complete secret-free Cognito desired set from Terraform outputs."""
|
|
7081
|
+
previous_state = previous_state if isinstance(previous_state, dict) else {}
|
|
7082
|
+
route_manifest = outputs.get("auth_route_clients", {}).get("value") or {}
|
|
7083
|
+
if not isinstance(route_manifest, dict):
|
|
7084
|
+
raise RuntimeError("auth_route_clients Terraform output must be an object")
|
|
7085
|
+
user_pool_id = str(outputs.get("user_pool_id", {}).get("value") or "")
|
|
7086
|
+
if not user_pool_id:
|
|
7087
|
+
raise RuntimeError("Native auth reconciliation requires user_pool_id output")
|
|
7088
|
+
|
|
7089
|
+
routes = []
|
|
7090
|
+
provider_client_ids = {}
|
|
7091
|
+
for manifest_key, raw in sorted(route_manifest.items()):
|
|
7092
|
+
if not isinstance(raw, dict):
|
|
7093
|
+
raise RuntimeError(f"Invalid auth route manifest entry: {manifest_key}")
|
|
7094
|
+
provider_names = [str(value) for value in raw.get("provider_names") or []]
|
|
7095
|
+
client_id = str(raw.get("client_id") or "")
|
|
7096
|
+
for provider_name in provider_names:
|
|
7097
|
+
provider_client_ids.setdefault(provider_name, []).append(client_id)
|
|
7098
|
+
routes.append(
|
|
7099
|
+
{
|
|
7100
|
+
"routeKey": str(raw.get("route_key") or ""),
|
|
7101
|
+
"clientFamily": str(raw.get("client_family") or ""),
|
|
7102
|
+
"cognitoUserPoolId": user_pool_id,
|
|
7103
|
+
"cognitoAppClientId": client_id,
|
|
7104
|
+
"providerNames": provider_names,
|
|
7105
|
+
"explicitAuthFlows": [str(value) for value in raw.get("explicit_auth_flows") or []],
|
|
7106
|
+
"redirectUris": [str(value) for value in raw.get("callback_urls") or []],
|
|
7107
|
+
"logoutUris": [str(value) for value in raw.get("logout_urls") or []],
|
|
7108
|
+
"lifecycleState": str(raw.get("lifecycle_state") or "native"),
|
|
7109
|
+
}
|
|
7110
|
+
)
|
|
7111
|
+
|
|
7112
|
+
connections = [
|
|
7113
|
+
{
|
|
7114
|
+
"connectionKey": "local",
|
|
7115
|
+
"providerKey": "cognito",
|
|
7116
|
+
"providerKind": "local",
|
|
7117
|
+
"displayName": "Email and password",
|
|
7118
|
+
"lifecycleState": "native",
|
|
7119
|
+
"cognitoUserPoolId": user_pool_id,
|
|
7120
|
+
"cognitoIdentityProviderName": "COGNITO",
|
|
7121
|
+
"authorizeScopes": "openid email profile",
|
|
7122
|
+
"tenantBindings": [],
|
|
7123
|
+
}
|
|
7124
|
+
]
|
|
7125
|
+
if provider_client_ids.get("Google"):
|
|
7126
|
+
connections.append(
|
|
7127
|
+
{
|
|
7128
|
+
"connectionKey": "google",
|
|
7129
|
+
"providerKey": "google",
|
|
7130
|
+
"providerKind": "google",
|
|
7131
|
+
"displayName": "Google",
|
|
7132
|
+
"lifecycleState": "native",
|
|
7133
|
+
"cognitoUserPoolId": user_pool_id,
|
|
7134
|
+
"cognitoIdentityProviderName": "Google",
|
|
7135
|
+
"authorizeScopes": "openid email profile",
|
|
7136
|
+
"tenantBindings": [],
|
|
7137
|
+
}
|
|
7138
|
+
)
|
|
7139
|
+
if provider_client_ids.get("MicrosoftOrganizations"):
|
|
7140
|
+
microsoft_tenant = str(vars_json.get("microsoft_oauth_tenant") or "").lower()
|
|
7141
|
+
if microsoft_tenant not in {"common", "organizations", "consumers"} and not re.fullmatch(
|
|
7142
|
+
r"[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}",
|
|
7143
|
+
microsoft_tenant,
|
|
7144
|
+
):
|
|
7145
|
+
raise RuntimeError(
|
|
7146
|
+
"Microsoft native auth reconciliation requires microsoft_oauth_tenant to be"
|
|
7147
|
+
" an Entra directory GUID or one of: common, organizations, consumers"
|
|
7148
|
+
)
|
|
7149
|
+
connections.append(
|
|
7150
|
+
{
|
|
7151
|
+
"connectionKey": "microsoft:organizations",
|
|
7152
|
+
"providerKey": "microsoft",
|
|
7153
|
+
"providerKind": "microsoft_organizations",
|
|
7154
|
+
"displayName": "Microsoft",
|
|
7155
|
+
"lifecycleState": "native",
|
|
7156
|
+
"cognitoUserPoolId": user_pool_id,
|
|
7157
|
+
"cognitoIdentityProviderName": "MicrosoftOrganizations",
|
|
7158
|
+
"issuerUrl": (f"https://login.microsoftonline.com/{microsoft_tenant}/v2.0"),
|
|
7159
|
+
"authorizeScopes": "openid email profile",
|
|
7160
|
+
"tenantBindings": [],
|
|
7161
|
+
}
|
|
7162
|
+
)
|
|
7163
|
+
|
|
7164
|
+
# Identity-provider operations store the complete tenant connection set in
|
|
7165
|
+
# safe SSM state. Standard releases carry it forward; omission is not
|
|
7166
|
+
# interpreted as deletion.
|
|
7167
|
+
tenant_connections = vars_json.get("auth_tenant_connection_metadata")
|
|
7168
|
+
if tenant_connections is None:
|
|
7169
|
+
tenant_connections = previous_state.get("tenantConnections") or []
|
|
7170
|
+
for connection in tenant_connections:
|
|
7171
|
+
if isinstance(connection, dict):
|
|
7172
|
+
# tenantDirectoryId is runner-owned projection metadata, not part
|
|
7173
|
+
# of the public/API reconciliation contract.
|
|
7174
|
+
connections.append(
|
|
7175
|
+
{
|
|
7176
|
+
key: value
|
|
7177
|
+
for key, value in connection.items()
|
|
7178
|
+
if key not in {"tenantDirectoryId", "clientSecretVersionStage"}
|
|
7179
|
+
}
|
|
7180
|
+
)
|
|
7181
|
+
|
|
7182
|
+
desired = {
|
|
7183
|
+
"connections": sorted(connections, key=lambda item: item["connectionKey"]),
|
|
7184
|
+
"routeClients": sorted(
|
|
7185
|
+
routes,
|
|
7186
|
+
key=lambda item: f"{item['routeKey']}:{item['clientFamily']}",
|
|
7187
|
+
),
|
|
7188
|
+
}
|
|
7189
|
+
desired_fingerprint = hashlib.sha256(canonical_json(desired).encode("utf-8")).hexdigest()
|
|
7190
|
+
if desired_fingerprint == previous_state.get("desiredFingerprint"):
|
|
7191
|
+
return None, previous_state
|
|
7192
|
+
|
|
7193
|
+
previous_revision = int(previous_state.get("revision") or 0)
|
|
7194
|
+
payload = {
|
|
7195
|
+
"stage": vars_json["stage"],
|
|
7196
|
+
"awsAccountId": vars_json["account_id"],
|
|
7197
|
+
"awsRegion": vars_json["region"],
|
|
7198
|
+
"revision": previous_revision + 1,
|
|
7199
|
+
"expectedPreviousRevision": previous_revision,
|
|
7200
|
+
"idempotencyKey": str(
|
|
7201
|
+
uuid.uuid5(
|
|
7202
|
+
uuid.NAMESPACE_URL,
|
|
7203
|
+
f"thinkwork-auth:{vars_json['stage']}:{previous_revision + 1}:{desired_fingerprint}",
|
|
7204
|
+
)
|
|
7205
|
+
),
|
|
7206
|
+
"connections": desired["connections"],
|
|
7207
|
+
"routeClients": desired["routeClients"],
|
|
7208
|
+
}
|
|
7209
|
+
payload["manifestFingerprint"] = hashlib.sha256(
|
|
7210
|
+
json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
|
7211
|
+
).hexdigest()
|
|
7212
|
+
next_state = {
|
|
7213
|
+
"revision": payload["revision"],
|
|
7214
|
+
"desiredFingerprint": desired_fingerprint,
|
|
7215
|
+
"tenantConnections": tenant_connections,
|
|
7216
|
+
"desiredRevision": previous_state.get("desiredRevision", 0),
|
|
7217
|
+
}
|
|
7218
|
+
return payload, next_state
|
|
7219
|
+
|
|
7220
|
+
|
|
7221
|
+
def auth_reconciliation_state(stage):
|
|
7222
|
+
name = f"/thinkwork/{stage}/auth/reconciliation/state"
|
|
7223
|
+
try:
|
|
7224
|
+
encoded = output(
|
|
7225
|
+
[
|
|
7226
|
+
"aws",
|
|
7227
|
+
"ssm",
|
|
7228
|
+
"get-parameter",
|
|
7229
|
+
"--name",
|
|
7230
|
+
name,
|
|
7231
|
+
"--query",
|
|
7232
|
+
"Parameter.Value",
|
|
7233
|
+
"--output",
|
|
7234
|
+
"text",
|
|
7235
|
+
]
|
|
7236
|
+
)
|
|
7237
|
+
return json.loads(encoded)
|
|
7238
|
+
except Exception:
|
|
7239
|
+
return {}
|
|
7240
|
+
|
|
7241
|
+
|
|
7242
|
+
def _describe_cognito_identity_provider(user_pool_id, provider_name):
|
|
7243
|
+
try:
|
|
7244
|
+
raw = output(
|
|
7245
|
+
[
|
|
7246
|
+
"aws",
|
|
7247
|
+
"cognito-idp",
|
|
7248
|
+
"describe-identity-provider",
|
|
7249
|
+
"--user-pool-id",
|
|
7250
|
+
user_pool_id,
|
|
7251
|
+
"--provider-name",
|
|
7252
|
+
provider_name,
|
|
7253
|
+
"--output",
|
|
7254
|
+
"json",
|
|
7255
|
+
],
|
|
7256
|
+
stderr=subprocess.PIPE,
|
|
7257
|
+
)
|
|
7258
|
+
value = json.loads(raw or "{}")
|
|
7259
|
+
return value.get("IdentityProvider") if isinstance(value, dict) else None
|
|
7260
|
+
except subprocess.CalledProcessError as error:
|
|
7261
|
+
stderr = str(getattr(error, "stderr", "") or "")
|
|
7262
|
+
if "ResourceNotFoundException" in stderr:
|
|
7263
|
+
return None
|
|
7264
|
+
raise RuntimeError(
|
|
7265
|
+
"Tenant Entra Cognito identity provider could not be described"
|
|
7266
|
+
) from error
|
|
7267
|
+
except json.JSONDecodeError as error:
|
|
7268
|
+
raise RuntimeError(
|
|
7269
|
+
"Tenant Entra Cognito identity provider returned malformed metadata"
|
|
7270
|
+
) from error
|
|
7271
|
+
|
|
7272
|
+
|
|
7273
|
+
def _read_tenant_entra_secret(connection):
|
|
7274
|
+
try:
|
|
7275
|
+
args = [
|
|
7276
|
+
"aws",
|
|
7277
|
+
"secretsmanager",
|
|
7278
|
+
"get-secret-value",
|
|
7279
|
+
"--secret-id",
|
|
7280
|
+
connection["clientSecretRef"],
|
|
7281
|
+
]
|
|
7282
|
+
version_stage = connection.get("clientSecretVersionStage")
|
|
7283
|
+
if version_stage:
|
|
7284
|
+
args.extend(["--version-stage", version_stage])
|
|
7285
|
+
args.extend(
|
|
7286
|
+
[
|
|
7287
|
+
"--query",
|
|
7288
|
+
"SecretString",
|
|
7289
|
+
"--output",
|
|
7290
|
+
"text",
|
|
7291
|
+
]
|
|
7292
|
+
)
|
|
7293
|
+
body = output(
|
|
7294
|
+
args,
|
|
7295
|
+
stderr=subprocess.DEVNULL,
|
|
7296
|
+
)
|
|
7297
|
+
secret = json.loads(body or "{}")
|
|
7298
|
+
except (subprocess.CalledProcessError, json.JSONDecodeError) as error:
|
|
7299
|
+
raise RuntimeError("Tenant Entra secret could not be read") from error
|
|
7300
|
+
client_id = secret.get("clientId") if isinstance(secret, dict) else None
|
|
7301
|
+
client_secret = secret.get("clientSecret") if isinstance(secret, dict) else None
|
|
7302
|
+
if (
|
|
7303
|
+
client_id != connection["clientId"]
|
|
7304
|
+
or not isinstance(client_secret, str)
|
|
7305
|
+
or not client_secret
|
|
7306
|
+
):
|
|
7307
|
+
raise RuntimeError("Tenant Entra secret document is missing matching credentials")
|
|
7308
|
+
return client_secret
|
|
7309
|
+
|
|
7310
|
+
|
|
7311
|
+
def promote_tenant_entra_pending_secret(payload, vars_json):
|
|
7312
|
+
operation = identity_provider_operation(payload)
|
|
7313
|
+
if operation is None or str(operation.get("action") or "").lower() != "rotate":
|
|
7314
|
+
return None
|
|
7315
|
+
connection_key = operation["connection"]["connectionKey"]
|
|
7316
|
+
connection = next(
|
|
7317
|
+
value
|
|
7318
|
+
for value in vars_json["auth_tenant_connection_metadata"]
|
|
7319
|
+
if value["connectionKey"] == connection_key
|
|
7320
|
+
)
|
|
7321
|
+
if connection.get("clientSecretVersionStage") != "AWSPENDING":
|
|
7322
|
+
raise RuntimeError("Tenant Entra rotation is missing its pending secret version")
|
|
7323
|
+
raw = output(
|
|
7324
|
+
[
|
|
7325
|
+
"aws",
|
|
7326
|
+
"secretsmanager",
|
|
7327
|
+
"describe-secret",
|
|
7328
|
+
"--secret-id",
|
|
7329
|
+
connection["clientSecretRef"],
|
|
7330
|
+
"--query",
|
|
7331
|
+
"VersionIdsToStages",
|
|
7332
|
+
"--output",
|
|
7333
|
+
"json",
|
|
7334
|
+
],
|
|
7335
|
+
stderr=subprocess.DEVNULL,
|
|
7336
|
+
)
|
|
7337
|
+
versions = json.loads(raw or "{}")
|
|
7338
|
+
pending = next(
|
|
7339
|
+
(version for version, stages in versions.items() if "AWSPENDING" in stages),
|
|
7340
|
+
None,
|
|
7341
|
+
)
|
|
7342
|
+
current = next(
|
|
7343
|
+
(version for version, stages in versions.items() if "AWSCURRENT" in stages),
|
|
7344
|
+
None,
|
|
7345
|
+
)
|
|
7346
|
+
if not pending:
|
|
7347
|
+
raise RuntimeError("Tenant Entra pending secret version was not found")
|
|
7348
|
+
if pending != current:
|
|
7349
|
+
args = [
|
|
7350
|
+
"aws",
|
|
7351
|
+
"secretsmanager",
|
|
7352
|
+
"update-secret-version-stage",
|
|
7353
|
+
"--secret-id",
|
|
7354
|
+
connection["clientSecretRef"],
|
|
7355
|
+
"--version-stage",
|
|
7356
|
+
"AWSCURRENT",
|
|
7357
|
+
"--move-to-version-id",
|
|
7358
|
+
pending,
|
|
7359
|
+
]
|
|
7360
|
+
if current:
|
|
7361
|
+
args.extend(["--remove-from-version-id", current])
|
|
7362
|
+
output(args, stderr=subprocess.DEVNULL)
|
|
7363
|
+
return {"status": "promoted", "connectionKey": connection_key}
|
|
7364
|
+
|
|
7365
|
+
|
|
7366
|
+
def _assert_tenant_entra_provider_matches(existing, connection):
|
|
7367
|
+
details = existing.get("ProviderDetails") if isinstance(existing, dict) else None
|
|
7368
|
+
mapping = existing.get("AttributeMapping") if isinstance(existing, dict) else None
|
|
7369
|
+
expected_mapping = {
|
|
7370
|
+
"email": "preferred_username",
|
|
7371
|
+
"name": "name",
|
|
7372
|
+
"username": "sub",
|
|
7373
|
+
"custom:entra_tenant_id": "tid",
|
|
7374
|
+
"custom:entra_object_id": "oid",
|
|
7375
|
+
}
|
|
7376
|
+
if (
|
|
7377
|
+
not isinstance(existing, dict)
|
|
7378
|
+
or existing.get("ProviderName") != connection["cognitoIdentityProviderName"]
|
|
7379
|
+
or existing.get("ProviderType") != "OIDC"
|
|
7380
|
+
or not isinstance(details, dict)
|
|
7381
|
+
or details.get("client_id") != connection["clientId"]
|
|
7382
|
+
or str(details.get("oidc_issuer") or "").rstrip("/") != connection["issuerUrl"].rstrip("/")
|
|
7383
|
+
or details.get("authorize_scopes") != "openid email profile"
|
|
7384
|
+
or mapping != expected_mapping
|
|
7385
|
+
):
|
|
7386
|
+
raise RuntimeError("Tenant Entra Cognito identity provider has drifted")
|
|
7387
|
+
|
|
7388
|
+
|
|
7389
|
+
def reconcile_identity_provider_resource(payload, vars_json):
|
|
7390
|
+
operation = identity_provider_operation(payload)
|
|
7391
|
+
if operation is None:
|
|
7392
|
+
return None
|
|
7393
|
+
action = str(operation["action"]).lower()
|
|
7394
|
+
connection_key = operation["connection"]["connectionKey"]
|
|
7395
|
+
connection = next(
|
|
7396
|
+
value
|
|
7397
|
+
for value in vars_json["auth_tenant_connection_metadata"]
|
|
7398
|
+
if value["connectionKey"] == connection_key
|
|
7399
|
+
)
|
|
7400
|
+
user_pool_id = connection["cognitoUserPoolId"]
|
|
7401
|
+
provider_name = connection["cognitoIdentityProviderName"]
|
|
7402
|
+
existing = _describe_cognito_identity_provider(user_pool_id, provider_name)
|
|
7403
|
+
|
|
7404
|
+
if action == "disable":
|
|
7405
|
+
return {
|
|
7406
|
+
"action": action,
|
|
7407
|
+
"connectionKey": connection_key,
|
|
7408
|
+
"providerName": provider_name,
|
|
7409
|
+
"status": "pending_route_removal",
|
|
7410
|
+
}
|
|
7411
|
+
|
|
7412
|
+
# Validation intentionally reads the secret as well: a provider whose
|
|
7413
|
+
# referenced credential is inaccessible or malformed is not valid.
|
|
7414
|
+
client_secret = _read_tenant_entra_secret(connection)
|
|
7415
|
+
if action == "validate":
|
|
7416
|
+
if existing is None:
|
|
7417
|
+
raise RuntimeError("Tenant Entra Cognito identity provider does not exist")
|
|
7418
|
+
_assert_tenant_entra_provider_matches(existing, connection)
|
|
7419
|
+
return {
|
|
7420
|
+
"action": action,
|
|
7421
|
+
"connectionKey": connection_key,
|
|
7422
|
+
"providerName": provider_name,
|
|
7423
|
+
"status": "valid",
|
|
7424
|
+
}
|
|
7425
|
+
|
|
7426
|
+
cli_input = {
|
|
7427
|
+
"UserPoolId": user_pool_id,
|
|
7428
|
+
"ProviderName": provider_name,
|
|
7429
|
+
"ProviderDetails": {
|
|
7430
|
+
"client_id": connection["clientId"],
|
|
7431
|
+
"client_secret": client_secret,
|
|
7432
|
+
"attributes_request_method": "GET",
|
|
7433
|
+
"oidc_issuer": connection["issuerUrl"],
|
|
7434
|
+
"authorize_scopes": "openid email profile",
|
|
7435
|
+
},
|
|
7436
|
+
"AttributeMapping": {
|
|
7437
|
+
"email": "preferred_username",
|
|
7438
|
+
"name": "name",
|
|
7439
|
+
"username": "sub",
|
|
7440
|
+
"custom:entra_tenant_id": "tid",
|
|
7441
|
+
"custom:entra_object_id": "oid",
|
|
7442
|
+
},
|
|
7443
|
+
"IdpIdentifiers": [],
|
|
7444
|
+
}
|
|
7445
|
+
aws_action = "update-identity-provider" if existing else "create-identity-provider"
|
|
7446
|
+
try:
|
|
7447
|
+
subprocess.run(
|
|
7448
|
+
[
|
|
7449
|
+
"aws",
|
|
7450
|
+
"cognito-idp",
|
|
7451
|
+
aws_action,
|
|
7452
|
+
"--cli-input-json",
|
|
7453
|
+
"file:///dev/stdin",
|
|
7454
|
+
"--output",
|
|
7455
|
+
"json",
|
|
7456
|
+
],
|
|
7457
|
+
input=json.dumps(cli_input, separators=(",", ":")),
|
|
7458
|
+
text=True,
|
|
7459
|
+
check=True,
|
|
7460
|
+
stdout=subprocess.DEVNULL,
|
|
7461
|
+
stderr=subprocess.DEVNULL,
|
|
7462
|
+
)
|
|
7463
|
+
except subprocess.CalledProcessError as error:
|
|
7464
|
+
raise RuntimeError("Tenant Entra Cognito reconciliation failed") from error
|
|
7465
|
+
described = _describe_cognito_identity_provider(user_pool_id, provider_name)
|
|
7466
|
+
_assert_tenant_entra_provider_matches(described, connection)
|
|
7467
|
+
return {
|
|
7468
|
+
"action": action,
|
|
7469
|
+
"connectionKey": connection_key,
|
|
7470
|
+
"providerName": provider_name,
|
|
7471
|
+
"status": "reconciled",
|
|
7472
|
+
}
|
|
7473
|
+
|
|
7474
|
+
|
|
7475
|
+
def finalize_disabled_identity_provider(payload, vars_json):
|
|
7476
|
+
operation = identity_provider_operation(payload)
|
|
7477
|
+
if operation is None or str(operation.get("action") or "").lower() != "disable":
|
|
7478
|
+
return None
|
|
7479
|
+
connection = operation["connection"]
|
|
7480
|
+
user_pool_id = next(
|
|
7481
|
+
value["cognitoUserPoolId"]
|
|
7482
|
+
for value in vars_json["auth_tenant_connection_metadata"]
|
|
7483
|
+
if value["connectionKey"] == connection["connectionKey"]
|
|
7484
|
+
)
|
|
7485
|
+
provider_name = connection["providerName"]
|
|
7486
|
+
if _describe_cognito_identity_provider(user_pool_id, provider_name) is not None:
|
|
7487
|
+
try:
|
|
7488
|
+
output(
|
|
7489
|
+
[
|
|
7490
|
+
"aws",
|
|
7491
|
+
"cognito-idp",
|
|
7492
|
+
"delete-identity-provider",
|
|
7493
|
+
"--user-pool-id",
|
|
7494
|
+
user_pool_id,
|
|
7495
|
+
"--provider-name",
|
|
7496
|
+
provider_name,
|
|
7497
|
+
],
|
|
7498
|
+
stderr=subprocess.DEVNULL,
|
|
7499
|
+
)
|
|
7500
|
+
except subprocess.CalledProcessError as error:
|
|
7501
|
+
raise RuntimeError("Tenant Entra Cognito provider could not be disabled") from error
|
|
7502
|
+
return {
|
|
7503
|
+
"action": "disable",
|
|
7504
|
+
"connectionKey": connection["connectionKey"],
|
|
7505
|
+
"providerName": provider_name,
|
|
7506
|
+
"status": "disabled",
|
|
7507
|
+
}
|
|
7508
|
+
|
|
7509
|
+
|
|
7510
|
+
def record_identity_provider_operation_state(payload, vars_json):
|
|
7511
|
+
operation = identity_provider_operation(payload)
|
|
7512
|
+
if operation is None:
|
|
7513
|
+
return
|
|
7514
|
+
current = auth_reconciliation_state(vars_json["stage"])
|
|
7515
|
+
expected = int(operation["expectedPreviousRevision"])
|
|
7516
|
+
if int(current.get("desiredRevision") or 0) != expected:
|
|
7517
|
+
raise RuntimeError("Identity-provider desired revision changed during reconciliation")
|
|
7518
|
+
current["desiredRevision"] = int(operation["revision"])
|
|
7519
|
+
current["tenantConnections"] = [
|
|
7520
|
+
{key: value for key, value in connection.items() if key != "clientSecretVersionStage"}
|
|
7521
|
+
for connection in vars_json["auth_tenant_connection_metadata"]
|
|
7522
|
+
]
|
|
7523
|
+
run(
|
|
7524
|
+
[
|
|
7525
|
+
"aws",
|
|
7526
|
+
"ssm",
|
|
7527
|
+
"put-parameter",
|
|
7528
|
+
"--overwrite",
|
|
7529
|
+
"--type",
|
|
7530
|
+
"String",
|
|
7531
|
+
"--name",
|
|
7532
|
+
f"/thinkwork/{vars_json['stage']}/auth/reconciliation/state",
|
|
7533
|
+
"--value",
|
|
7534
|
+
json.dumps(current, separators=(",", ":")),
|
|
7535
|
+
]
|
|
7536
|
+
)
|
|
7537
|
+
|
|
7538
|
+
|
|
7539
|
+
def reconcile_native_auth_metadata(outputs_path, vars_json):
|
|
7540
|
+
outputs = json.loads(outputs_path.read_text(encoding="utf-8"))
|
|
7541
|
+
previous = auth_reconciliation_state(vars_json["stage"])
|
|
7542
|
+
payload, next_state = native_auth_reconciliation_payload(outputs, vars_json, previous)
|
|
7543
|
+
if payload is None:
|
|
7544
|
+
return {"status": "unchanged", "revision": previous.get("revision", 0)}
|
|
7545
|
+
api_endpoint = str(outputs.get("api_endpoint", {}).get("value") or "").rstrip("/")
|
|
7546
|
+
api_auth_secret = str(vars_json.get("api_auth_secret") or "")
|
|
7547
|
+
if not api_endpoint or not api_auth_secret:
|
|
7548
|
+
raise RuntimeError("Native auth reconciliation requires API endpoint and service auth")
|
|
7549
|
+
request = urllib.request.Request(
|
|
7550
|
+
f"{api_endpoint}/api/auth/providers/reconcile",
|
|
7551
|
+
data=json.dumps(payload, separators=(",", ":")).encode("utf-8"),
|
|
7552
|
+
method="POST",
|
|
7553
|
+
headers={
|
|
7554
|
+
"Authorization": f"Bearer {api_auth_secret}",
|
|
7555
|
+
"Content-Type": "application/json",
|
|
7556
|
+
"Accept": "application/json",
|
|
7557
|
+
},
|
|
7558
|
+
)
|
|
7559
|
+
try:
|
|
7560
|
+
with urllib.request.urlopen(request, timeout=30) as response:
|
|
7561
|
+
result = json.loads(response.read().decode("utf-8") or "{}")
|
|
7562
|
+
except urllib.error.HTTPError as error:
|
|
7563
|
+
safe_body = error.read().decode("utf-8", errors="replace")[:1000]
|
|
7564
|
+
raise RuntimeError(
|
|
7565
|
+
f"Native auth metadata reconciliation failed with HTTP {error.code}: {safe_body}"
|
|
7566
|
+
) from error
|
|
7567
|
+
run(
|
|
7568
|
+
[
|
|
7569
|
+
"aws",
|
|
7570
|
+
"ssm",
|
|
7571
|
+
"put-parameter",
|
|
7572
|
+
"--overwrite",
|
|
7573
|
+
"--type",
|
|
7574
|
+
"String",
|
|
7575
|
+
"--name",
|
|
7576
|
+
f"/thinkwork/{vars_json['stage']}/auth/reconciliation/state",
|
|
7577
|
+
"--value",
|
|
7578
|
+
json.dumps(next_state, separators=(",", ":")),
|
|
7579
|
+
]
|
|
7580
|
+
)
|
|
7581
|
+
return {
|
|
7582
|
+
"status": str(result.get("status") or "applied"),
|
|
7583
|
+
"revision": int(result.get("revision") or payload["revision"]),
|
|
7584
|
+
"manifestFingerprint": payload["manifestFingerprint"],
|
|
7585
|
+
}
|
|
7586
|
+
|
|
7587
|
+
|
|
4214
7588
|
def sync_static(outputs_path, static_files, vars_json, artifact_names=None):
|
|
4215
7589
|
outputs = json.loads(outputs_path.read_text(encoding="utf-8"))
|
|
4216
7590
|
artifact_names = set(artifact_names or [])
|
|
4217
7591
|
syncs = [
|
|
4218
7592
|
("web", "app_bucket_name", "app_distribution_id"),
|
|
4219
|
-
("docs", "docs_bucket_name", "docs_distribution_id"),
|
|
4220
7593
|
]
|
|
4221
7594
|
for artifact_name, bucket_output, distribution_output in syncs:
|
|
4222
7595
|
if artifact_names and artifact_name not in artifact_names:
|
|
@@ -4346,7 +7719,7 @@ def write_deployment_status_pointer(status, vars_json=None, terraform_exit_code=
|
|
|
4346
7719
|
action = os.environ.get("THINKWORK_DEPLOYMENT_ACTION")
|
|
4347
7720
|
if os.environ.get("THINKWORK_MANAGED_APP_OPERATION") == "true":
|
|
4348
7721
|
return
|
|
4349
|
-
if not bucket or action not in {"deploy", "update", "web"}:
|
|
7722
|
+
if not bucket or action not in {"deploy", "update", "apply", "web"}:
|
|
4350
7723
|
return
|
|
4351
7724
|
vars_json = vars_json or {}
|
|
4352
7725
|
previous = read_current_status_pointer(bucket)
|
|
@@ -4486,7 +7859,16 @@ def main():
|
|
|
4486
7859
|
action = os.environ.get("THINKWORK_DEPLOYMENT_ACTION") or payload.get("action") or "deploy"
|
|
4487
7860
|
if action == "teardown":
|
|
4488
7861
|
action = "destroy"
|
|
4489
|
-
if action not in {
|
|
7862
|
+
if action not in {
|
|
7863
|
+
"deploy",
|
|
7864
|
+
"update",
|
|
7865
|
+
"destroy",
|
|
7866
|
+
"plan",
|
|
7867
|
+
"apply",
|
|
7868
|
+
"recover",
|
|
7869
|
+
"status",
|
|
7870
|
+
"web",
|
|
7871
|
+
}:
|
|
4490
7872
|
raise RuntimeError(f"Unsupported deployment action: {action}")
|
|
4491
7873
|
os.environ["THINKWORK_DEPLOYMENT_ACTION"] = action
|
|
4492
7874
|
configure_managed_app_evidence_prefix(payload)
|
|
@@ -4506,16 +7888,45 @@ def main():
|
|
|
4506
7888
|
)
|
|
4507
7889
|
return 0
|
|
4508
7890
|
|
|
7891
|
+
if action == "recover":
|
|
7892
|
+
CONTROLLER_EVIDENCE = {
|
|
7893
|
+
"incidentRecovery": recover_tei_v380_incident(payload),
|
|
7894
|
+
}
|
|
7895
|
+
write_evidence(
|
|
7896
|
+
"succeeded",
|
|
7897
|
+
{
|
|
7898
|
+
"stage": "tei-e2e",
|
|
7899
|
+
"account_id": "637423202447",
|
|
7900
|
+
"region": "us-east-1",
|
|
7901
|
+
},
|
|
7902
|
+
0,
|
|
7903
|
+
)
|
|
7904
|
+
return 0
|
|
7905
|
+
|
|
7906
|
+
terraform_phase = validate_terraform_execution_phase(payload, action)
|
|
7907
|
+
planned_action = requested_plan_action(payload, action)
|
|
7908
|
+
execution_effect_action = planned_action if terraform_phase == "apply" else action
|
|
4509
7909
|
runner_secrets = secret_payload(payload)
|
|
4510
7910
|
web_only = is_web_only_operation(payload, action)
|
|
7911
|
+
identity_operation = identity_provider_operation(payload)
|
|
4511
7912
|
static_files = {}
|
|
4512
|
-
|
|
4513
|
-
|
|
4514
|
-
|
|
4515
|
-
|
|
4516
|
-
|
|
7913
|
+
release_request = release_sync_request(execution_effect_action, payload, identity_operation)
|
|
7914
|
+
control_runtime_operation = requires_agentcore_control_runtime(
|
|
7915
|
+
execution_effect_action, payload, identity_operation
|
|
7916
|
+
)
|
|
7917
|
+
if release_request is not None:
|
|
7918
|
+
static_files = sync_release_artifacts(**release_request)
|
|
7919
|
+
elif control_runtime_operation:
|
|
7920
|
+
# Plan and destroy still execute AgentCore local-exec wrappers for
|
|
7921
|
+
# existing resources. Download the content-addressed platform bundle
|
|
7922
|
+
# without publishing application artifacts so those paths never depend
|
|
7923
|
+
# on an ambient CodeBuild SDK.
|
|
7924
|
+
sync_release_artifacts(artifact_types=set())
|
|
4517
7925
|
managed_app_artifacts = stage_managed_app_release_artifacts(action, payload)
|
|
4518
7926
|
vars_json = write_runner_files(payload, runner_secrets)
|
|
7927
|
+
set_database_access_mode(vars_json)
|
|
7928
|
+
if control_runtime_operation:
|
|
7929
|
+
prepare_agentcore_control_runtime()
|
|
4519
7930
|
controller_summary = controller_input_summary(payload)
|
|
4520
7931
|
CONTROLLER_EVIDENCE = {
|
|
4521
7932
|
"inputSummary": controller_summary,
|
|
@@ -4551,6 +7962,11 @@ def main():
|
|
|
4551
7962
|
write_evidence("succeeded", vars_json, 0)
|
|
4552
7963
|
return 0
|
|
4553
7964
|
|
|
7965
|
+
if identity_operation is not None:
|
|
7966
|
+
CONTROLLER_EVIDENCE["identityProvider"] = reconcile_identity_provider_resource(
|
|
7967
|
+
payload, vars_json
|
|
7968
|
+
)
|
|
7969
|
+
|
|
4554
7970
|
configure_cloudflare_provider_auth(vars_json["stage"])
|
|
4555
7971
|
configure_terraform_provider_mirror()
|
|
4556
7972
|
run(["terraform", "init", "-backend-config=backend.hcl", "-no-color"], cwd=TF)
|
|
@@ -4564,41 +7980,60 @@ def main():
|
|
|
4564
7980
|
)
|
|
4565
7981
|
if selected.returncode != 0:
|
|
4566
7982
|
run(["terraform", "workspace", "new", workspace, "-no-color"], cwd=TF)
|
|
7983
|
+
state_identity = terraform_state_identity()
|
|
4567
7984
|
target_args = managed_app_terraform_target_args(payload)
|
|
4568
|
-
|
|
4569
|
-
|
|
4570
|
-
|
|
4571
|
-
|
|
4572
|
-
|
|
7985
|
+
pre_app_schema = (
|
|
7986
|
+
prepare_additive_schema_before_app(payload, vars_json)
|
|
7987
|
+
if terraform_phase in {"plan", "legacy"}
|
|
7988
|
+
else None
|
|
7989
|
+
)
|
|
7990
|
+
if terraform_phase == "legacy" and should_reconcile_native_auth_schema(
|
|
7991
|
+
terraform_phase, planned_action, payload
|
|
7992
|
+
):
|
|
7993
|
+
CONTROLLER_EVIDENCE["cognitoSchema"] = ensure_native_auth_custom_attributes(
|
|
7994
|
+
current_terraform_outputs(vars_json["stage"])
|
|
4573
7995
|
)
|
|
4574
|
-
|
|
4575
|
-
|
|
4576
|
-
|
|
4577
|
-
|
|
4578
|
-
|
|
4579
|
-
|
|
7996
|
+
applied_plan = None
|
|
7997
|
+
if terraform_phase == "apply":
|
|
7998
|
+
applied_plan = validate_and_materialize_approved_plan(payload, state_identity)
|
|
7999
|
+
planned_action = applied_plan.get("plannedAction")
|
|
8000
|
+
if planned_action not in {"deploy", "update", "destroy"}:
|
|
8001
|
+
raise RuntimeError("Approved plan has an invalid plannedAction")
|
|
8002
|
+
TERRAFORM_EVIDENCE["approvedPlan"] = applied_plan
|
|
8003
|
+
if should_reconcile_native_auth_schema(terraform_phase, planned_action, payload):
|
|
8004
|
+
CONTROLLER_EVIDENCE["cognitoSchema"] = ensure_native_auth_custom_attributes(
|
|
8005
|
+
current_terraform_outputs(vars_json["stage"])
|
|
4580
8006
|
)
|
|
4581
|
-
|
|
4582
|
-
|
|
4583
|
-
else:
|
|
4584
|
-
plan = subprocess.run(
|
|
4585
|
-
["terraform", "plan", *target_args, "-out=tfplan", "-no-color"],
|
|
8007
|
+
result = subprocess.run(
|
|
8008
|
+
["terraform", "apply", "-no-color", "tfplan"],
|
|
4586
8009
|
cwd=TF,
|
|
4587
8010
|
text=True,
|
|
4588
8011
|
)
|
|
4589
|
-
|
|
4590
|
-
|
|
4591
|
-
|
|
4592
|
-
|
|
4593
|
-
|
|
4594
|
-
|
|
4595
|
-
|
|
4596
|
-
|
|
4597
|
-
|
|
4598
|
-
|
|
8012
|
+
elif pre_app_schema is not None and pre_app_schema.returncode != 0:
|
|
8013
|
+
result = pre_app_schema
|
|
8014
|
+
else:
|
|
8015
|
+
result = execute_terraform_plan_phase(
|
|
8016
|
+
payload,
|
|
8017
|
+
state_identity,
|
|
8018
|
+
planned_action,
|
|
8019
|
+
terraform_phase,
|
|
8020
|
+
target_args,
|
|
8021
|
+
)
|
|
4599
8022
|
|
|
4600
8023
|
outputs_path = TF / "outputs.json"
|
|
4601
|
-
if
|
|
8024
|
+
if (
|
|
8025
|
+
result.returncode == 0
|
|
8026
|
+
and terraform_phase in {"apply", "legacy"}
|
|
8027
|
+
and planned_action
|
|
8028
|
+
in {
|
|
8029
|
+
"deploy",
|
|
8030
|
+
"update",
|
|
8031
|
+
}
|
|
8032
|
+
):
|
|
8033
|
+
if identity_operation is not None:
|
|
8034
|
+
finalized = finalize_disabled_identity_provider(payload, vars_json)
|
|
8035
|
+
if finalized is not None:
|
|
8036
|
+
CONTROLLER_EVIDENCE["identityProvider"] = finalized
|
|
4602
8037
|
write_outputs_after_apply(payload, vars_json, outputs_path)
|
|
4603
8038
|
if is_managed_app_operation(payload):
|
|
4604
8039
|
sync_twenty_thinkwork_app(
|
|
@@ -4608,7 +8043,21 @@ def main():
|
|
|
4608
8043
|
runner_secrets,
|
|
4609
8044
|
managed_app_artifacts,
|
|
4610
8045
|
)
|
|
8046
|
+
elif identity_operation is not None:
|
|
8047
|
+
CONTROLLER_EVIDENCE["authReconciliation"] = reconcile_native_auth_metadata(
|
|
8048
|
+
outputs_path, vars_json
|
|
8049
|
+
)
|
|
8050
|
+
promoted = promote_tenant_entra_pending_secret(payload, vars_json)
|
|
8051
|
+
if promoted is not None:
|
|
8052
|
+
CONTROLLER_EVIDENCE["identityProviderSecret"] = promoted
|
|
8053
|
+
write_outputs_to_ssm(outputs_path, vars_json)
|
|
8054
|
+
record_identity_provider_operation_state(payload, vars_json)
|
|
4611
8055
|
else:
|
|
8056
|
+
CONTROLLER_EVIDENCE["authReconciliation"] = reconcile_native_auth_metadata(
|
|
8057
|
+
outputs_path, vars_json
|
|
8058
|
+
)
|
|
8059
|
+
# Destructive retirement migrations are permitted only after the
|
|
8060
|
+
# deployed native routes have reconciled successfully.
|
|
4612
8061
|
push_database_schema(outputs_path, vars_json)
|
|
4613
8062
|
ensure_first_admin(outputs_path, vars_json, payload, runner_secrets)
|
|
4614
8063
|
write_outputs_to_ssm(outputs_path, vars_json)
|
|
@@ -4619,6 +8068,13 @@ def main():
|
|
|
4619
8068
|
selected_controller_release,
|
|
4620
8069
|
)
|
|
4621
8070
|
sync_static(outputs_path, static_files, vars_json)
|
|
8071
|
+
if control_runtime_operation:
|
|
8072
|
+
pi_runtime_evidence = reconcile_agentcore_pi_runtime(vars_json, payload)
|
|
8073
|
+
if pi_runtime_evidence is not None:
|
|
8074
|
+
CONTROLLER_EVIDENCE["agentcorePiRuntime"] = write_json_evidence_artifact(
|
|
8075
|
+
"agentcore-pi-runtime.json",
|
|
8076
|
+
pi_runtime_evidence,
|
|
8077
|
+
)
|
|
4622
8078
|
try:
|
|
4623
8079
|
self_update_runner_script()
|
|
4624
8080
|
except Exception as self_update_error:
|