thinkwork-cli 0.13.1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (262) hide show
  1. package/README.md +3 -14
  2. package/dist/{api-client-4DZ266J5.js → api-client-4P4JZOBX.js} +3 -1
  3. package/dist/bowser-6Z6JFAIR.js +2822 -0
  4. package/dist/{chunk-Y2WHJECR.js → chunk-4OMDWYZQ.js} +5 -44
  5. package/dist/chunk-AD2NIWB5.js +6516 -0
  6. package/dist/chunk-DV3MRBR5.js +43 -0
  7. package/dist/chunk-EI5ZX7CN.js +522 -0
  8. package/dist/chunk-EUJ7X6YD.js +126 -0
  9. package/dist/chunk-GG5WKBOX.js +436 -0
  10. package/dist/chunk-I64EZSU5.js +358 -0
  11. package/dist/chunk-KMIJ5KGT.js +128 -0
  12. package/dist/chunk-MWQ7RO76.js +137 -0
  13. package/dist/chunk-OCPHU34C.js +996 -0
  14. package/dist/chunk-OLUQ4U2Z.js +1728 -0
  15. package/dist/chunk-XYM7PE2A.js +2264 -0
  16. package/dist/chunk-YQCYD6NE.js +3299 -0
  17. package/dist/cli.js +49156 -31257
  18. package/dist/commands/enterprise/templates/deploy-repo/scripts/apply-release.mjs +0 -1
  19. package/dist/commands/enterprise/templates/deploy-repo/scripts/smoke.mjs +0 -1
  20. package/dist/commands/enterprise/templates/deploy-repo/terraform/main.tf +0 -14
  21. package/dist/dist-es-2OTNTL4N.js +315 -0
  22. package/dist/dist-es-2XVAH4BQ.js +14726 -0
  23. package/dist/dist-es-3DPDUJAH.js +2000 -0
  24. package/dist/dist-es-BNZTBBVC.js +176 -0
  25. package/dist/dist-es-CD2RBDAC.js +88 -0
  26. package/dist/dist-es-I4FW57PS.js +88 -0
  27. package/dist/dist-es-KL7Y5NB2.js +377 -0
  28. package/dist/dist-es-KVFWJNDY.js +318 -0
  29. package/dist/dist-es-PS73DWNL.js +70 -0
  30. package/dist/dist-es-S3NZQFGT.js +170 -0
  31. package/dist/dist-es-SGCA4P32.js +48 -0
  32. package/dist/dist-es-WBM6J4RA.js +70 -0
  33. package/dist/dist-es-ZGVADYIF.js +485 -0
  34. package/dist/dist-es-ZKC2QKD2.js +377 -0
  35. package/dist/drizzle/0076_scheduled_jobs_marco_backfill.sql +2 -1
  36. package/dist/drizzle/0079_seed_tenant_customize_catalog.sql +1 -0
  37. package/dist/drizzle/0090_brain_schema_extraction.sql +10 -0
  38. package/dist/drizzle/0132_drop_computer_tables.sql +3 -0
  39. package/dist/drizzle/0160_compliance_event_types_plugins.sql +2 -0
  40. package/dist/drizzle/0214_think137_drop_judge_evidence_roi.sql +1 -0
  41. package/dist/drizzle/0216_drop_agent_loop_versions_goal_worker_policy.sql +1 -0
  42. package/dist/drizzle/0227_analyst_reader_role.sql +17 -20
  43. package/dist/drizzle/0230_analyst_rls.sql +28 -36
  44. package/dist/drizzle/0250_kg_schema_extraction.sql +27 -0
  45. package/dist/drizzle/0251_brain_stragglers.sql +21 -0
  46. package/dist/drizzle/0253_drop_brain_substrate.sql +2 -0
  47. package/dist/drizzle/0257_agents_runtime_allow_harness.sql +24 -0
  48. package/dist/drizzle/0261_native_auth_control_plane.sql +332 -0
  49. package/dist/drizzle/0262_auth_subscription_tickets.sql +59 -0
  50. package/dist/drizzle/0263_agentcore_runtime_identifier.sql +69 -0
  51. package/dist/drizzle/0263_drop_workos_auth_runtime.sql +236 -0
  52. package/dist/drizzle/0264_auth_enrollment_attempt_limits.sql +15 -0
  53. package/dist/drizzle/0265_eval_runs_requester_identity.sql +28 -0
  54. package/dist/drizzle/0265_require_fully_valid_auth_publication.sql +27 -0
  55. package/dist/drizzle/0266_auth_identity_recovery_grants.sql +9 -0
  56. package/dist/drizzle/0266_eval_profiles_runtime_type.sql +12 -0
  57. package/dist/drizzle/0267_ontology_candidate_rejections.sql +53 -0
  58. package/dist/drizzle/0268_identity_crosswalk_routing.sql +182 -0
  59. package/dist/drizzle/0269_ontology_system_map.sql +31 -0
  60. package/dist/drizzle/0270_tool_execution_ledger.sql +115 -0
  61. package/dist/drizzle/0271_ontology_twin_declarations.sql +47 -0
  62. package/dist/drizzle/0272_identity_graph_projection_cursor.sql +23 -0
  63. package/dist/drizzle/0273_twin_materialization_suggestions.sql +29 -0
  64. package/dist/drizzle/0274_tenant_mcp_twin_keys.sql +59 -0
  65. package/dist/drizzle/0275_identity_bulk_rebuild_fence.sql +30 -0
  66. package/dist/drizzle/0276_tool_execution_ledger_allow_cascade_delete.sql +18 -0
  67. package/dist/drizzle/0277_knowledge_base_sources.sql +107 -0
  68. package/dist/drizzle/0278_kb_page_transcription.sql +55 -0
  69. package/dist/drizzle/0279_mcp_servers_off_plugin_provenance.sql +93 -0
  70. package/dist/drizzle/0280_work_item_external_refs_drop_plane_provider.sql +46 -0
  71. package/dist/drizzle/0281_tenant_mcp_twin_keys_expiry_suffix.sql +9 -0
  72. package/dist/drizzle/0282_tenant_mcp_twin_keys_grants.sql +24 -0
  73. package/dist/drizzle/0283_agents_agentcore_runtime_dispatch.sql +13 -0
  74. package/dist/drizzle/0284_user_brain_claims.sql +69 -0
  75. package/dist/drizzle/0285_tenant_mcp_twin_keys_trusted_subsystem.sql +30 -0
  76. package/dist/drizzle/0286_brain_analytics_key.sql +26 -0
  77. package/dist/drizzle/0287_brain_subject.sql +25 -0
  78. package/dist/drizzle/0288_multi_lane_identities.sql +25 -0
  79. package/dist/event-streams-7JNLJEEL.js +894 -0
  80. package/dist/event-streams-KDF5RWR6.js +1515 -0
  81. package/dist/loadSso-ETCXTR4I.js +576 -0
  82. package/dist/loadSso-UBD7KUHE.js +583 -0
  83. package/dist/plugins/README.md +2 -2
  84. package/dist/plugins/catalog/package.json +1 -5
  85. package/dist/plugins/catalog/src/__tests__/build-catalog.test.ts +6 -4
  86. package/dist/plugins/catalog/src/__tests__/catalog.test.ts +21 -21
  87. package/dist/plugins/catalog/src/__tests__/contracts.test.ts +102 -94
  88. package/dist/plugins/catalog/src/__tests__/plugin-package.test.ts +7 -8
  89. package/dist/plugins/catalog/src/__tests__/plugin-registry.test.ts +0 -20
  90. package/dist/plugins/catalog/src/contracts.ts +17 -179
  91. package/dist/plugins/catalog/src/registry/generated-first-party.ts +0 -24
  92. package/dist/plugins/n8n/test/integrated-app-contract.test.ts +1 -1
  93. package/dist/plugins/twenty/package.json +2 -1
  94. package/dist/plugins/twenty/scripts/LAMBDA_DEPLOY.md +114 -0
  95. package/dist/plugins/twenty/scripts/build-lambda.sh +29 -0
  96. package/dist/plugins/twenty/scripts/configure-product-views.ts +195 -0
  97. package/dist/plugins/twenty/scripts/lambda-handler.ts +49 -0
  98. package/dist/plugins/twenty/scripts/lib/__tests__/load-attachments.test.ts +38 -0
  99. package/dist/plugins/twenty/scripts/lib/__tests__/load-records.test.ts +145 -2
  100. package/dist/plugins/twenty/scripts/lib/__tests__/mappers.test.ts +397 -151
  101. package/dist/plugins/twenty/scripts/lib/__tests__/members-ensure.test.ts +28 -129
  102. package/dist/plugins/twenty/scripts/lib/__tests__/provision-members-db.test.ts +10 -0
  103. package/dist/plugins/twenty/scripts/lib/__tests__/schema-ensure.test.ts +13 -8
  104. package/dist/plugins/twenty/scripts/lib/lastmile-reader.ts +222 -2
  105. package/dist/plugins/twenty/scripts/lib/load-attachments.ts +29 -0
  106. package/dist/plugins/twenty/scripts/lib/load-records.ts +160 -43
  107. package/dist/plugins/twenty/scripts/lib/mappers.ts +368 -152
  108. package/dist/plugins/twenty/scripts/lib/members-ensure.ts +43 -268
  109. package/dist/plugins/twenty/scripts/lib/provision-members-db.ts +57 -0
  110. package/dist/plugins/twenty/scripts/lib/schema-ensure.ts +136 -31
  111. package/dist/plugins/twenty/scripts/migrate-lastmile.ts +254 -152
  112. package/dist/plugins/twenty/scripts/provision-twenty-members.ts +44 -1
  113. package/dist/plugins/twenty/scripts/purge-lastmile-import.ts +178 -0
  114. package/dist/plugins/twenty/scripts/restructure-products.ts +312 -0
  115. package/dist/plugins/twenty/scripts/retag-stage-values.ts +362 -0
  116. package/dist/plugins/twenty/test/manifest.test.ts +2 -4
  117. package/dist/scripts/post-deploy.sh +79 -1
  118. package/dist/signin-5QZNRSOO.js +821 -0
  119. package/dist/sso-oidc-XK673P54.js +837 -0
  120. package/dist/sts-OXI2UALF.js +1113 -0
  121. package/dist/sts-SQW4NBK3.js +1117 -0
  122. package/dist/terraform/examples/greenfield/main.tf +391 -176
  123. package/dist/terraform/examples/greenfield/terraform.tfvars.example +26 -8
  124. package/dist/terraform/modules/app/agentcore-admin/main.tf +47 -0
  125. package/dist/terraform/modules/app/agentcore-code-interpreter/README.md +39 -47
  126. package/dist/terraform/modules/app/agentcore-code-interpreter/main.tf +13 -66
  127. package/dist/terraform/modules/app/agentcore-identity/main.tf +253 -0
  128. package/dist/terraform/modules/app/agentcore-identity/outputs.tf +51 -0
  129. package/dist/terraform/modules/app/agentcore-identity/scripts/bootstrap_twenty_oauth_client.sh +239 -0
  130. package/dist/terraform/modules/app/agentcore-identity/scripts/delete_identity.sh +22 -0
  131. package/dist/terraform/modules/app/agentcore-identity/scripts/delete_twenty_identity.sh +16 -0
  132. package/dist/terraform/modules/app/agentcore-identity/scripts/delete_workload_identity.sh +19 -0
  133. package/dist/terraform/modules/app/agentcore-identity/scripts/ensure_workload_identity.sh +29 -0
  134. package/dist/terraform/modules/app/agentcore-identity/scripts/read_identity.sh +55 -0
  135. package/dist/terraform/modules/app/agentcore-identity/scripts/reconcile_identity.sh +73 -0
  136. package/dist/terraform/modules/app/agentcore-identity/scripts/reconcile_twenty_identity.sh +45 -0
  137. package/dist/terraform/modules/app/agentcore-identity/scripts/reconcile_twenty_provider.mjs +123 -0
  138. package/dist/terraform/modules/app/agentcore-identity/tests/tainted-marker/after.tf +15 -0
  139. package/dist/terraform/modules/app/agentcore-identity/tests/tainted-marker/before.tf +7 -0
  140. package/dist/terraform/modules/app/agentcore-identity/tests/tainted-marker/identity-after.tf +8 -0
  141. package/dist/terraform/modules/app/agentcore-identity/tests/tainted-marker/identity-before.tf +9 -0
  142. package/dist/terraform/modules/app/agentcore-identity/tests/tainted-marker/thinkwork-after.tf +3 -0
  143. package/dist/terraform/modules/app/agentcore-identity/tests/tainted-marker/thinkwork-before.tf +3 -0
  144. package/dist/terraform/modules/app/agentcore-identity/variables.tf +60 -0
  145. package/dist/terraform/modules/app/agentcore-memory/README.md +46 -9
  146. package/dist/terraform/modules/app/agentcore-memory/main.tf +27 -10
  147. package/dist/terraform/modules/app/agentcore-memory/scripts/create_or_find_memory.sh +156 -24
  148. package/dist/terraform/modules/app/agentcore-pi/README.md +1 -3
  149. package/dist/terraform/modules/app/agentcore-pi/main.tf +42 -52
  150. package/dist/terraform/modules/app/agentcore-pi/outputs.tf +3 -3
  151. package/dist/terraform/modules/app/agentcore-pi/scripts/reconcile_pi_runtime.mjs +175 -0
  152. package/dist/terraform/modules/app/agentcore-pi/variables.tf +35 -28
  153. package/dist/terraform/modules/app/agentcore-platform/main.tf +68 -0
  154. package/dist/terraform/modules/app/agentcore-runtime/main.tf +4 -20
  155. package/dist/terraform/modules/app/appsync-subscriptions/main.tf +127 -21
  156. package/dist/terraform/modules/app/appsync-subscriptions/outputs.tf +0 -6
  157. package/dist/terraform/modules/app/appsync-subscriptions/variables.tf +5 -0
  158. package/dist/terraform/modules/app/capability-broker/main.tf +2 -2
  159. package/dist/terraform/modules/app/capability-broker/variables.tf +11 -0
  160. package/dist/terraform/modules/app/deployment-control-plane/README.md +264 -0
  161. package/dist/terraform/modules/app/deployment-control-plane/buildspec.yml +4 -1
  162. package/dist/terraform/modules/app/deployment-control-plane/main.tf +94 -0
  163. package/dist/terraform/modules/app/deployment-control-plane/outputs.tf +5 -0
  164. package/dist/terraform/modules/app/deployment-control-plane/runner.py +3689 -233
  165. package/dist/terraform/modules/app/deployment-control-plane/test_runner_bundle.py +2579 -140
  166. package/dist/terraform/modules/app/deployment-control-plane/variables.tf +33 -0
  167. package/dist/terraform/modules/app/lambda-api/auth-state.tf +72 -0
  168. package/dist/terraform/modules/app/lambda-api/chat-latency-observability.tf +466 -0
  169. package/dist/terraform/modules/app/lambda-api/dispatch-dlq.tf +55 -0
  170. package/dist/terraform/modules/app/lambda-api/handlers.tf +531 -619
  171. package/dist/terraform/modules/app/lambda-api/iam-grouped.tf +350 -241
  172. package/dist/terraform/modules/app/lambda-api/main.tf +75 -13
  173. package/dist/terraform/modules/app/lambda-api/memory-alarms.tf +3 -28
  174. package/dist/terraform/modules/app/lambda-api/oauth-secrets.tf +1 -2
  175. package/dist/terraform/modules/app/lambda-api/outputs.tf +17 -10
  176. package/dist/terraform/modules/app/lambda-api/runtime-config.tf +3 -21
  177. package/dist/terraform/modules/app/lambda-api/turn-assertion.tf +45 -0
  178. package/dist/terraform/modules/app/lambda-api/variables.tf +307 -121
  179. package/dist/{plugins/n8n/terraform → terraform/modules/app}/n8n/README.md +1 -1
  180. package/dist/{plugins/n8n/terraform → terraform/modules/app}/n8n/main.tf +18 -18
  181. package/dist/terraform/modules/app/sandbox-log-scrubber/README.md +11 -10
  182. package/dist/terraform/modules/app/sandbox-log-scrubber/main.tf +5 -5
  183. package/dist/terraform/modules/app/static-site/main.tf +1 -1
  184. package/dist/terraform/modules/app/www-dns/main.tf +9 -27
  185. package/dist/terraform/modules/app/www-dns/variables.tf +4 -10
  186. package/dist/terraform/modules/data/aurora-postgres/README.md +34 -0
  187. package/dist/terraform/modules/data/aurora-postgres/alarms.tf +82 -0
  188. package/dist/terraform/modules/data/aurora-postgres/main.tf +27 -57
  189. package/dist/terraform/modules/data/aurora-postgres/outputs.tf +0 -9
  190. package/dist/terraform/modules/data/aurora-postgres/variables.tf +41 -0
  191. package/dist/terraform/modules/data/s3-buckets/main.tf +5 -6
  192. package/dist/terraform/modules/foundation/cognito/main.tf +367 -72
  193. package/dist/terraform/modules/foundation/cognito/outputs.tf +47 -0
  194. package/dist/terraform/modules/foundation/cognito/variables.tf +131 -9
  195. package/dist/terraform/modules/thinkwork/README.md +5 -1
  196. package/dist/terraform/modules/thinkwork/main.tf +459 -234
  197. package/dist/terraform/modules/thinkwork/outputs.tf +44 -62
  198. package/dist/terraform/modules/thinkwork/variables.tf +353 -89
  199. package/dist/terraform/schema.graphql +69 -36
  200. package/dist/workspace-defaults/files/INSTRUCTIONS.md +9 -11
  201. package/dist/workspace-defaults/files/MEMORY_GUIDE.md +38 -35
  202. package/dist/workspace-defaults/files/ROUTER.md +1 -1
  203. package/dist/workspace-defaults/files/TOOLS.md +9 -39
  204. package/dist/workspace-defaults/files/USER.md +1 -0
  205. package/dist/workspace-defaults/files/agents/analyst/INSTRUCTIONS.md +6 -3
  206. package/dist/workspace-defaults/files/catalog-skills/brain-brief-builder/SKILL.md +165 -0
  207. package/dist/workspace-defaults/files/catalog-skills/n8n-workflow-operator/SKILL.md +75 -0
  208. package/dist/workspace-defaults/files/catalog-skills/n8n-workflow-operator/references/mcp-tooling.md +40 -0
  209. package/dist/workspace-defaults/files/catalog-skills/n8n-workflow-operator/references/validation-and-handoff.md +40 -0
  210. package/dist/workspace-defaults/files/catalog-skills/n8n-workflow-operator/references/workflow-authoring.md +46 -0
  211. package/dist/workspace-defaults/files/skills/artifact-builder/references/crm-dashboard.md +1 -1
  212. package/dist/workspace-defaults/files/skills/document-composer/SKILL.md +22 -3
  213. package/package.json +3 -2
  214. package/dist/drizzle/0261_thread_turns_origin_turn_index.sql +0 -11
  215. package/dist/plugins/company-data/README.md +0 -49
  216. package/dist/plugins/company-data/package.json +0 -21
  217. package/dist/plugins/company-data/src/index.ts +0 -30
  218. package/dist/plugins/company-data/src/manifest.ts +0 -26
  219. package/dist/plugins/company-data/test/manifest.test.ts +0 -116
  220. package/dist/plugins/company-data/tsconfig.json +0 -8
  221. package/dist/plugins/company-etl/README.md +0 -46
  222. package/dist/plugins/company-etl/package.json +0 -21
  223. package/dist/plugins/company-etl/src/index.ts +0 -29
  224. package/dist/plugins/company-etl/src/manifest.ts +0 -26
  225. package/dist/plugins/company-etl/test/manifest.test.ts +0 -108
  226. package/dist/plugins/company-etl/tsconfig.json +0 -8
  227. package/dist/plugins/lastmile/README.md +0 -50
  228. package/dist/plugins/lastmile/package.json +0 -23
  229. package/dist/plugins/lastmile/smoke/lastmile-plugin-smoke.mjs +0 -661
  230. package/dist/plugins/lastmile/src/api/tasks-adapter.ts +0 -572
  231. package/dist/plugins/lastmile/src/discovery.fixture.ts +0 -64
  232. package/dist/plugins/lastmile/src/index.ts +0 -46
  233. package/dist/plugins/lastmile/src/manifest.ts +0 -139
  234. package/dist/plugins/lastmile/test/api/tasks-adapter.test.ts +0 -266
  235. package/dist/plugins/lastmile/test/discovery.test.ts +0 -88
  236. package/dist/plugins/lastmile/tsconfig.json +0 -8
  237. package/dist/plugins/n8n/smoke/n8n-managed-app-smoke.mjs +0 -409
  238. package/dist/plugins/twenty/smoke/twenty-managed-app-smoke.mjs +0 -359
  239. package/dist/plugins/workos-auth/package.json +0 -22
  240. package/dist/plugins/workos-auth/src/index.ts +0 -30
  241. package/dist/plugins/workos-auth/src/manifest.ts +0 -40
  242. package/dist/plugins/workos-auth/src/provider-contract.ts +0 -39
  243. package/dist/plugins/workos-auth/test/manifest.test.ts +0 -94
  244. package/dist/plugins/workos-auth/tsconfig.json +0 -8
  245. package/dist/terraform/modules/app/agentcore-code-interpreter/Dockerfile.sandbox-base +0 -61
  246. package/dist/terraform/modules/app/agentcore-code-interpreter/sandbox/sitecustomize.py +0 -227
  247. package/dist/terraform/modules/app/agentcore-code-interpreter/sandbox/test_sitecustomize.py +0 -277
  248. package/dist/terraform/modules/app/agentcore-code-interpreter/scripts/build_and_push_sandbox_base.sh +0 -70
  249. package/dist/terraform/modules/app/agentcore-harness/main.tf +0 -101
  250. package/dist/terraform/modules/app/agentcore-harness/outputs.tf +0 -9
  251. package/dist/terraform/modules/app/agentcore-harness/variables.tf +0 -25
  252. package/dist/terraform/modules/app/hindsight-memory/README.md +0 -89
  253. package/dist/terraform/modules/app/hindsight-memory/main.tf +0 -411
  254. package/dist/terraform/modules/data/bedrock-knowledge-base/main.tf +0 -102
  255. /package/dist/{plugins/n8n/terraform → terraform/modules/app}/n8n/outputs.tf +0 -0
  256. /package/dist/{plugins/n8n/terraform → terraform/modules/app}/n8n/scripts/sync-database.py +0 -0
  257. /package/dist/{plugins/n8n/terraform → terraform/modules/app}/n8n/scripts/test_sync_database.py +0 -0
  258. /package/dist/{plugins/n8n/terraform → terraform/modules/app}/n8n/variables.tf +0 -0
  259. /package/dist/{plugins/twenty/terraform → terraform/modules/app}/twenty/README.md +0 -0
  260. /package/dist/{plugins/twenty/terraform → terraform/modules/app}/twenty/main.tf +0 -0
  261. /package/dist/{plugins/twenty/terraform → terraform/modules/app}/twenty/outputs.tf +0 -0
  262. /package/dist/{plugins/twenty/terraform → terraform/modules/app}/twenty/variables.tf +0 -0
@@ -7,8 +7,11 @@ import re
7
7
  import secrets
8
8
  import subprocess
9
9
  import tarfile
10
+ import time
11
+ import urllib.error
10
12
  import urllib.parse
11
13
  import urllib.request
14
+ import uuid
12
15
  from datetime import UTC, datetime
13
16
  from pathlib import Path
14
17
  from tempfile import TemporaryDirectory
@@ -42,11 +45,66 @@ RELEASE_MANIFEST_TRUST_POLICIES = {
42
45
  POST_SEED_MIGRATIONS = [
43
46
  "0155_tenant_model_catalog.sql",
44
47
  ]
48
+ AUTH_RETIREMENT_MIGRATION_MARKER = "-- deployment-phase: auth-retired"
49
+ PLATFORM_MIGRATION_LEDGER_CUTOFF = 260
45
50
  MIGRATION_MARKER_KINDS = [
46
51
  ("-- creates-column:", "column"),
47
52
  ("-- creates-constraint:", "constraint"),
48
53
  ("-- creates:", "object"),
49
54
  ]
55
+ NATIVE_AUTH_CUSTOM_ATTRIBUTES = [
56
+ {
57
+ "AttributeDataType": "String",
58
+ "DeveloperOnlyAttribute": False,
59
+ "Mutable": True,
60
+ "Name": "entra_tenant_id",
61
+ "Required": False,
62
+ "StringAttributeConstraints": {"MinLength": "0", "MaxLength": "36"},
63
+ },
64
+ {
65
+ "AttributeDataType": "String",
66
+ "DeveloperOnlyAttribute": False,
67
+ "Mutable": True,
68
+ "Name": "entra_object_id",
69
+ "Required": False,
70
+ "StringAttributeConstraints": {"MinLength": "0", "MaxLength": "36"},
71
+ },
72
+ ]
73
+ AGENTCORE_CONTROL_SDK_VERSION = "3.1089.0"
74
+ TERRAFORM_PLAN_APPROVAL_CONTRACT = "thinkwork.terraform.saved-plan-approval.v1"
75
+ TERRAFORM_PLAN_DESCRIPTOR_NAME = "terraform-plan-approval.json"
76
+ TERRAFORM_SAVED_PLAN_NAME = "terraform-plan.bin"
77
+ TEI_V380_RECOVERY_CONTRACT = "thinkwork.incident.tei-v380-proof-plane-recovery.v1"
78
+ TEI_V380_RECOVERY_INCIDENT_ID = "tei-v380-proof-plane-partial-apply"
79
+ TEI_V380_REJECTED_PLAN_SHA256 = "ba7b0ffa099f725b4f32e394446ee43693fe3dd4b7a9eb639d47eb833d40db84"
80
+ TEI_V380_STATE_LINEAGE = "61f4093e-41da-ac29-852a-267bc853e24f"
81
+ TEI_V380_STATE_SERIAL = 819
82
+ TEI_V380_STATE_SHA256 = "cc99973e7b80e1a0c6e1097d889e006e5e2610e236ed730a02ca5f3457b99d7f"
83
+ TEI_V380_STATE_BUCKET = "tei-thinkwork-terraform-state"
84
+ TEI_V380_STATE_KEY = "env:/tei-e2e/thinkwork/tei-e2e/terraform.tfstate"
85
+ TEI_V380_LOCK_TABLE = "tei-thinkwork-terraform-locks"
86
+ TEI_V380_LOCK_KEY = f"{TEI_V380_STATE_BUCKET}/{TEI_V380_STATE_KEY}"
87
+ TEI_V380_LOCK_ID = "04b3e9a1-3de6-c668-87e7-43d94280605e"
88
+ TEI_V380_STATE_DIGEST = "7fe214eeaae491f5c1d8d792bddbbb19"
89
+ TEI_V380_SECRET_ARN = (
90
+ "arn:aws:secretsmanager:us-east-1:637423202447:secret:"
91
+ "thinkwork/tei-e2e/agentcore-identity/twenty-crm-oauth-client-7OeAlG"
92
+ )
93
+ TEI_V380_SECRET_NAME = "thinkwork/tei-e2e/agentcore-identity/twenty-crm-oauth-client"
94
+ TEI_V380_SECRET_ADDRESS = (
95
+ "module.thinkwork.module.agentcore_proof_identity."
96
+ "aws_secretsmanager_secret.twenty_oauth_client[0]"
97
+ )
98
+ TEI_V380_KMS_KEY_ID = "8798ab48-cd60-477d-a96e-c2d56bae7b40"
99
+ TEI_V380_KMS_KEY_ARN = "arn:aws:kms:us-east-1:637423202447:key/8798ab48-cd60-477d-a96e-c2d56bae7b40"
100
+ TEI_V380_KMS_KEY_ADDRESS = 'module.thinkwork.module.api.aws_kms_key.agentcore_turn_assertion["v1"]'
101
+ TEI_V380_KMS_ALIAS = "alias/thinkwork-tei-e2e-agentcore-turn-assertion-v1"
102
+ TEI_V380_KMS_ALIAS_ADDRESS = (
103
+ 'module.thinkwork.module.api.aws_kms_alias.agentcore_turn_assertion["v1"]'
104
+ )
105
+ TEI_V380_OWNER_ALLOWLIST = (
106
+ "0ea2f626-8b40-43ca-b7f1-3601b2a6bef8,f498f488-70a1-709b-24d7-f895f2164301"
107
+ )
50
108
 
51
109
 
52
110
  def run(args, **kwargs):
@@ -65,6 +123,10 @@ def sha256_file(path):
65
123
  return digest.hexdigest()
66
124
 
67
125
 
126
+ def sha256_bytes(value):
127
+ return hashlib.sha256(value).hexdigest()
128
+
129
+
68
130
  def read_json_env(name, default):
69
131
  value = os.environ.get(name)
70
132
  if not value:
@@ -381,6 +443,118 @@ def safe_extract_tar_file(archive_path, destination):
381
443
  tar.extractall(destination, members=members)
382
444
 
383
445
 
446
+ def validate_agentcore_node_runtime(version):
447
+ normalized = str(version or "").strip().removeprefix("v")
448
+ try:
449
+ major = int(normalized.split(".", 1)[0])
450
+ except (TypeError, ValueError):
451
+ raise RuntimeError(
452
+ f"Cannot determine Node.js version for AgentCore runtime: {version}"
453
+ ) from None
454
+ if major < 22:
455
+ raise RuntimeError(
456
+ "AgentCore control runtime requires Node.js 22 or newer; "
457
+ f"deployment runner is using {version}"
458
+ )
459
+ return f"v{normalized}"
460
+
461
+
462
+ def prepare_agentcore_control_runtime():
463
+ global RELEASE_EVIDENCE
464
+ node_version = validate_agentcore_node_runtime(output(["node", "--version"]))
465
+ runner_bundle = bundle_extract_dir({"name": "platform"}) / "runner"
466
+ manifest_path = runner_bundle / "agentcore-control-runtime.json"
467
+ if not manifest_path.is_file():
468
+ raise RuntimeError("Release bundle is missing the AgentCore control runtime manifest")
469
+
470
+ runtime_manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
471
+ expected_sdk = runtime_manifest.get("sdk") or {}
472
+ if (
473
+ expected_sdk.get("package") != "@aws-sdk/client-bedrock-agentcore-control"
474
+ or expected_sdk.get("version") != AGENTCORE_CONTROL_SDK_VERSION
475
+ ):
476
+ raise RuntimeError("Release does not pin the required AgentCore control SDK version")
477
+
478
+ runtime_dir = safe_join(
479
+ runner_bundle,
480
+ str(runtime_manifest.get("directory") or "agentcore-control-runtime"),
481
+ )
482
+ if not runtime_dir.is_dir():
483
+ raise RuntimeError("Release bundle has no AgentCore control runtime directory")
484
+ runtime_files = runtime_manifest.get("files") or []
485
+ if not runtime_files:
486
+ raise RuntimeError("AgentCore control runtime manifest has no files")
487
+ runtime_paths = [str(item.get("path") or "") for item in runtime_files]
488
+ # THINK-324: the managed harness is retired — harness-lifecycle.js is
489
+ # absent from new releases but tolerated in legacy bundles.
490
+ required_entrypoints = {
491
+ "preflight.js",
492
+ "reconcile_twenty_provider.js",
493
+ }
494
+ required_files = required_entrypoints | {"package.json"}
495
+ if len(runtime_paths) != len(set(runtime_paths)) or not required_files.issubset(runtime_paths):
496
+ raise RuntimeError("AgentCore control runtime manifest has an invalid file set")
497
+ for runtime_file in runtime_files:
498
+ file_path = safe_join(runtime_dir, str(runtime_file.get("path") or ""))
499
+ if not file_path.is_file() or sha256_file(file_path) != runtime_file.get("sha256"):
500
+ raise RuntimeError(
501
+ f"AgentCore control runtime file digest mismatch: {runtime_file.get('path')}"
502
+ )
503
+ actual_runtime_paths = {
504
+ str(path.relative_to(runtime_dir)) for path in runtime_dir.rglob("*") if path.is_file()
505
+ }
506
+ if actual_runtime_paths != set(runtime_paths):
507
+ raise RuntimeError("AgentCore control runtime contains unmanifested or missing files")
508
+
509
+ module_boundary = json.loads((runtime_dir / "package.json").read_text(encoding="utf-8"))
510
+ if module_boundary != {"type": "module"}:
511
+ raise RuntimeError("AgentCore control runtime has an invalid ESM module boundary")
512
+
513
+ preflight = json.loads(output(["node", str(runtime_dir / "preflight.js")]))
514
+ if preflight != expected_sdk:
515
+ raise RuntimeError("AgentCore control runtime preflight differs from its manifest")
516
+ # reconcile_pi_runtime.js (THINK-584 U5) is preflighted only when present:
517
+ # a newer seeded runner.py may run against a pre-U5 release bundle (the
518
+ # runner-py-bootstrap-skew trap), and the Pi runtime reconcile step skips
519
+ # loudly in that case rather than failing bundle validation here.
520
+ optional_entrypoints = ("reconcile_pi_runtime.js",)
521
+ entrypoint_preflights = [
522
+ json.loads(
523
+ output(
524
+ [
525
+ "node",
526
+ str(runtime_dir / entrypoint),
527
+ "--runtime-preflight",
528
+ ]
529
+ )
530
+ )
531
+ for entrypoint in ("reconcile_twenty_provider.js", *optional_entrypoints)
532
+ if entrypoint not in optional_entrypoints or (runtime_dir / entrypoint).is_file()
533
+ ]
534
+ if any(not item.get("sdkImportReady") for item in entrypoint_preflights):
535
+ raise RuntimeError("An AgentCore control runtime entrypoint cannot import its SDK")
536
+ os.environ["THINKWORK_AGENTCORE_CONTROL_RUNTIME_DIR"] = str(runtime_dir)
537
+ evidence = {
538
+ "manifestSha256": sha256_file(manifest_path),
539
+ "sdkPackage": preflight["package"],
540
+ "sdkVersion": preflight["version"],
541
+ "nodeVersion": node_version,
542
+ "bundledEntrypoints": sorted(
543
+ item["path"]
544
+ for item in runtime_files
545
+ if item.get("path") in required_entrypoints or item.get("path") in optional_entrypoints
546
+ ),
547
+ "bundledRuntimeVerified": True,
548
+ "entrypointPreflights": entrypoint_preflights,
549
+ }
550
+ RELEASE_EVIDENCE["agentCoreControlRuntime"] = evidence
551
+ print(
552
+ "[runner] AgentCore control runtime ready: "
553
+ f"sdk={evidence['sdkVersion']} manifest={evidence['manifestSha256']}"
554
+ )
555
+ return evidence
556
+
557
+
384
558
  def release_artifacts_by_name(manifest):
385
559
  return {
386
560
  artifact.get("name"): artifact
@@ -500,12 +674,18 @@ def write_json_evidence_artifact(name, payload):
500
674
  }
501
675
 
502
676
 
677
+ # vars_json keys the runner consumes itself; they are not variables of the
678
+ # generated Terraform root and are left out of terraform.auto.tfvars.json.
679
+ RUNNER_ONLY_VARS = frozenset({"db_access"})
680
+
681
+
503
682
  def redacted_tfvars(vars_json):
504
683
  redacted = dict(vars_json)
505
684
  for key in [
506
685
  "api_auth_secret",
507
686
  "db_password",
508
687
  "google_oauth_client_secret",
688
+ "microsoft_oauth_client_secret",
509
689
  ]:
510
690
  if key in redacted:
511
691
  redacted[key] = "[redacted]"
@@ -648,6 +828,259 @@ def terraform_plan_summary(plan_json):
648
828
  }
649
829
 
650
830
 
831
+ def terraform_plan_delete_addresses(plan_json):
832
+ return [
833
+ change.get("address")
834
+ for change in plan_json.get("resource_changes", [])
835
+ if "delete" in (change.get("change", {}).get("actions") or [])
836
+ ]
837
+
838
+
839
+ def deployment_config_for_approval(payload):
840
+ """Return the immutable deployment intent shared by plan and apply phases.
841
+
842
+ Controller/session routing fields necessarily change between executions.
843
+ Everything that can influence the generated Terraform configuration stays
844
+ in this projection, including release pins and preserved configuration.
845
+ """
846
+ reviewed = json.loads(json.dumps(payload))
847
+ for key in [
848
+ "action",
849
+ "phase",
850
+ "sessionId",
851
+ "session",
852
+ "evidence",
853
+ "approvedPlan",
854
+ ]:
855
+ reviewed.pop(key, None)
856
+ operation = reviewed.get("operation")
857
+ if isinstance(operation, dict):
858
+ for key in ["action", "plan", "apply", "approvedPlan"]:
859
+ operation.pop(key, None)
860
+ return reviewed
861
+
862
+
863
+ def deployment_config_sha256(payload):
864
+ return sha256_bytes(stable_json_bytes(deployment_config_for_approval(payload)))
865
+
866
+
867
+ def controller_input_sha256(payload):
868
+ return sha256_bytes(stable_json_bytes(payload))
869
+
870
+
871
+ def raw_controller_input_sha256(payload):
872
+ raw = os.environ.get("THINKWORK_DEPLOYMENT_INPUT")
873
+ if raw:
874
+ return sha256_bytes(raw.encode("utf-8"))
875
+ return controller_input_sha256(payload)
876
+
877
+
878
+ def requested_plan_action(payload, action):
879
+ if action in {"deploy", "update", "destroy"}:
880
+ return action
881
+ operation = payload.get("operation")
882
+ if isinstance(operation, dict):
883
+ candidate = operation.get("targetAction") or operation.get("requestedAction")
884
+ if candidate in {"deploy", "update", "destroy"}:
885
+ return candidate
886
+ if action == "plan":
887
+ return "update"
888
+ return action
889
+
890
+
891
+ def validate_terraform_execution_phase(payload, action):
892
+ """Select the explicit saved-plan protocol or the compatible legacy path.
893
+
894
+ The incident recovery always uses separate ``plan`` and ``apply``
895
+ executions. Existing callers still send ``deploy``/``update``/``destroy``
896
+ while their API and CLI approval surfaces are migrated, so those actions
897
+ retain their established combined behavior instead of becoming an
898
+ accidental platform-wide outage.
899
+ """
900
+ operation_value = payload.get("operation")
901
+ operation_is_structured = isinstance(operation_value, dict)
902
+ operation = operation_value
903
+ if not operation_is_structured:
904
+ operation = {}
905
+ requested_apply = operation.get("apply")
906
+ requested_plan = operation.get("plan")
907
+
908
+ if action == "apply":
909
+ if requested_apply is not True or requested_plan is not False:
910
+ raise RuntimeError(
911
+ "Saved-plan apply requires operation.apply=true and operation.plan=false"
912
+ )
913
+ approved = payload.get("approvedPlan")
914
+ if not isinstance(approved, dict):
915
+ raise RuntimeError("Saved-plan apply requires approvedPlan")
916
+ return "apply"
917
+
918
+ if action in {"deploy", "update", "destroy"}:
919
+ return "legacy"
920
+
921
+ if action == "plan":
922
+ if not operation_is_structured:
923
+ # Managed-application jobs predate the structured foundation
924
+ # operation object and already use action=plan as a strict
925
+ # plan-only phase. Preserve that safe contract.
926
+ return "plan"
927
+ if requested_apply is not False:
928
+ raise RuntimeError(
929
+ "Terraform planning never implies approval: operation.apply must be false; "
930
+ "apply the accepted saved plan in a separate action=apply execution"
931
+ )
932
+ if requested_plan is not True:
933
+ raise RuntimeError("Terraform plan phase requires operation.plan=true")
934
+ return "plan"
935
+
936
+ return "none"
937
+
938
+
939
+ def canonical_state_identity_bytes(state):
940
+ """`terraform state pull` re-renders the state document on every run and
941
+ serializes the render-only check_results block in nondeterministic map
942
+ order, so identical remote state produces different raw bytes across
943
+ runs. Hash a canonical JSON form without check_results; lineage/serial
944
+ stay the authoritative change signal for the infrastructure content."""
945
+ comparable = {key: value for key, value in state.items() if key != "check_results"}
946
+ return json.dumps(comparable, sort_keys=True, separators=(",", ":")).encode("utf-8")
947
+
948
+
949
+ def terraform_state_identity():
950
+ raw = output(["terraform", "state", "pull"], cwd=TF)
951
+ state = json.loads(raw or "{}")
952
+ lineage = state.get("lineage")
953
+ serial = state.get("serial")
954
+ if not isinstance(lineage, str) or not lineage:
955
+ raise RuntimeError("Terraform state has no lineage; refusing saved-plan operation")
956
+ if not isinstance(serial, int) or serial < 0:
957
+ raise RuntimeError("Terraform state has no valid serial; refusing saved-plan operation")
958
+ return {
959
+ "lineage": lineage,
960
+ "serial": serial,
961
+ "sha256": sha256_bytes(canonical_state_identity_bytes(state)),
962
+ "terraformVersion": state.get("terraform_version"),
963
+ }
964
+
965
+
966
+ def _validated_evidence_s3_uri(uri, label):
967
+ parsed = urllib.parse.urlparse(require_string(uri, label))
968
+ expected_bucket = os.environ.get("THINKWORK_EVIDENCE_BUCKET")
969
+ if parsed.scheme != "s3" or not parsed.netloc or not parsed.path.lstrip("/"):
970
+ raise RuntimeError(f"{label} must be an s3:// evidence URI")
971
+ if expected_bucket and parsed.netloc != expected_bucket:
972
+ raise RuntimeError(f"{label} must use the configured deployment evidence bucket")
973
+ return uri
974
+
975
+
976
+ def _download_approved_artifact(uri, destination, expected_sha256, label):
977
+ _validated_evidence_s3_uri(uri, label)
978
+ expected = require_string(expected_sha256, f"{label}.sha256").lower()
979
+ if not re.fullmatch(r"[0-9a-f]{64}", expected):
980
+ raise RuntimeError(f"{label}.sha256 must be a lowercase SHA-256 digest")
981
+ run(["aws", "s3", "cp", uri, str(destination)])
982
+ actual = sha256_file(destination)
983
+ if actual != expected:
984
+ raise RuntimeError(f"{label} digest mismatch: expected {expected}, got {actual}")
985
+ return actual
986
+
987
+
988
+ def load_approved_plan_descriptor(payload):
989
+ approved = payload.get("approvedPlan")
990
+ if not isinstance(approved, dict):
991
+ raise RuntimeError("Saved-plan apply requires approvedPlan")
992
+ descriptor_ref = approved.get("descriptor")
993
+ if not isinstance(descriptor_ref, dict):
994
+ raise RuntimeError("approvedPlan.descriptor is required")
995
+ descriptor_path = WORK / TERRAFORM_PLAN_DESCRIPTOR_NAME
996
+ _download_approved_artifact(
997
+ descriptor_ref.get("s3Uri"),
998
+ descriptor_path,
999
+ descriptor_ref.get("sha256"),
1000
+ "approvedPlan.descriptor",
1001
+ )
1002
+ descriptor = json.loads(descriptor_path.read_text(encoding="utf-8"))
1003
+ if descriptor.get("contract") != TERRAFORM_PLAN_APPROVAL_CONTRACT:
1004
+ raise RuntimeError("Approved plan descriptor contract is unsupported")
1005
+ return descriptor, descriptor_ref
1006
+
1007
+
1008
+ def comparable_terraform_plan(value, *, _top_level=True, _parent_key=None):
1009
+ """Normalize render-only Terraform JSON variance without changing plan decisions."""
1010
+ if isinstance(value, dict):
1011
+ return {
1012
+ key: comparable_terraform_plan(item, _top_level=False, _parent_key=key)
1013
+ for key, item in value.items()
1014
+ if not (_top_level and key == "timestamp")
1015
+ }
1016
+ if isinstance(value, list):
1017
+ normalized = [comparable_terraform_plan(item, _top_level=False) for item in value]
1018
+ if _parent_key == "references" and all(isinstance(item, str) for item in normalized):
1019
+ return sorted(normalized)
1020
+ return normalized
1021
+ return value
1022
+
1023
+
1024
+ def validate_and_materialize_approved_plan(payload, state_identity):
1025
+ descriptor, descriptor_ref = load_approved_plan_descriptor(payload)
1026
+ if descriptor.get("deploymentConfigSha256") != deployment_config_sha256(payload):
1027
+ raise RuntimeError("Approved plan deployment configuration does not match apply input")
1028
+ if descriptor.get("release") != release_selection(payload):
1029
+ raise RuntimeError("Approved plan release pin does not match apply input")
1030
+ if descriptor.get("state") != state_identity:
1031
+ raise RuntimeError("Terraform state changed after plan approval; refusing apply")
1032
+
1033
+ tfvars_sha256 = sha256_file(TF / "terraform.auto.tfvars.json")
1034
+ if descriptor.get("terraformVariablesSha256") != tfvars_sha256:
1035
+ raise RuntimeError("Approved plan Terraform variables do not match apply execution")
1036
+
1037
+ saved_plan = descriptor.get("savedPlan")
1038
+ plan_json = descriptor.get("jsonPlan")
1039
+ if not isinstance(saved_plan, dict) or not isinstance(plan_json, dict):
1040
+ raise RuntimeError("Approved plan descriptor is missing saved plan artifacts")
1041
+ destination = TF / "tfplan"
1042
+ _download_approved_artifact(
1043
+ saved_plan.get("s3Uri"),
1044
+ destination,
1045
+ saved_plan.get("sha256"),
1046
+ "approvedPlan.savedPlan",
1047
+ )
1048
+ accepted_json = WORK / "accepted-terraform-plan.json"
1049
+ _download_approved_artifact(
1050
+ plan_json.get("s3Uri"),
1051
+ accepted_json,
1052
+ plan_json.get("sha256"),
1053
+ "approvedPlan.jsonPlan",
1054
+ )
1055
+ rendered = WORK / "approved-terraform-plan.json"
1056
+ with rendered.open("w", encoding="utf-8") as handle:
1057
+ run(["terraform", "show", "-json", "tfplan"], cwd=TF, stdout=handle)
1058
+ accepted_plan = json.loads(accepted_json.read_text(encoding="utf-8"))
1059
+ rendered_plan = json.loads(rendered.read_text(encoding="utf-8"))
1060
+ if comparable_terraform_plan(rendered_plan) != comparable_terraform_plan(accepted_plan):
1061
+ raise RuntimeError("Approved saved plan no longer renders to the accepted JSON plan")
1062
+ if payload.get("incidentRecovery") is not None:
1063
+ delete_addresses = terraform_plan_delete_addresses(rendered_plan)
1064
+ if delete_addresses:
1065
+ raise RuntimeError(
1066
+ "TEI v380 recovery approved plan contains delete actions; refusing apply"
1067
+ )
1068
+
1069
+ return {
1070
+ "descriptor": {
1071
+ "fileName": TERRAFORM_PLAN_DESCRIPTOR_NAME,
1072
+ "sha256": descriptor_ref.get("sha256"),
1073
+ "s3Uri": descriptor_ref.get("s3Uri"),
1074
+ },
1075
+ "savedPlan": saved_plan,
1076
+ "jsonPlan": plan_json,
1077
+ "summary": descriptor.get("summary"),
1078
+ "plannedAction": descriptor.get("plannedAction"),
1079
+ "sourceSessionId": descriptor.get("sessionId"),
1080
+ "state": state_identity,
1081
+ }
1082
+
1083
+
651
1084
  def managed_app_terraform_target_args(payload):
652
1085
  app_key = payload.get("appKey")
653
1086
  if app_key == "n8n":
@@ -662,6 +1095,43 @@ def managed_app_terraform_target_args(payload):
662
1095
  return []
663
1096
 
664
1097
 
1098
+ def execute_terraform_plan_phase(
1099
+ payload,
1100
+ state_identity,
1101
+ planned_action,
1102
+ terraform_phase,
1103
+ target_args,
1104
+ ):
1105
+ """Create plan evidence and apply only for the compatible legacy phase."""
1106
+ plan_args = ["terraform", "plan"]
1107
+ if planned_action == "destroy":
1108
+ plan_args.append("-destroy")
1109
+ plan_args.extend([*target_args, "-out=tfplan", "-no-color"])
1110
+ plan = subprocess.run(plan_args, cwd=TF, text=True)
1111
+ if plan.returncode != 0:
1112
+ return plan
1113
+ TERRAFORM_EVIDENCE["plan"] = write_terraform_plan_evidence(
1114
+ payload,
1115
+ state_identity,
1116
+ planned_action,
1117
+ )
1118
+ if terraform_phase != "legacy":
1119
+ return plan
1120
+ return subprocess.run(
1121
+ ["terraform", "apply", "-auto-approve", "-no-color", "tfplan"],
1122
+ cwd=TF,
1123
+ text=True,
1124
+ )
1125
+
1126
+
1127
+ def should_reconcile_native_auth_schema(terraform_phase, planned_action, payload):
1128
+ return (
1129
+ terraform_phase in {"apply", "legacy"}
1130
+ and planned_action in {"deploy", "update"}
1131
+ and not is_managed_app_operation(payload)
1132
+ )
1133
+
1134
+
665
1135
  def truthy(value):
666
1136
  if isinstance(value, bool):
667
1137
  return value
@@ -792,6 +1262,28 @@ def is_web_only_operation(payload, action=None):
792
1262
  return component == "web" or truthy(payload.get("webOnly"))
793
1263
 
794
1264
 
1265
+ def release_sync_request(action, payload, identity_operation):
1266
+ if (
1267
+ action not in {"deploy", "update", "apply", "web"}
1268
+ or is_managed_app_operation(payload)
1269
+ or identity_operation is not None
1270
+ ):
1271
+ return None
1272
+ web_only = is_web_only_operation(payload, action)
1273
+ return {
1274
+ "artifact_types": {"static-site"} if web_only else None,
1275
+ "artifact_names": {"web"} if web_only else None,
1276
+ }
1277
+
1278
+
1279
+ def requires_agentcore_control_runtime(action, payload, identity_operation):
1280
+ return (
1281
+ action in {"deploy", "update", "plan", "apply", "destroy"}
1282
+ and not is_managed_app_operation(payload)
1283
+ and identity_operation is None
1284
+ )
1285
+
1286
+
795
1287
  def configure_managed_app_evidence_prefix(payload):
796
1288
  if not is_managed_app_operation(payload):
797
1289
  return
@@ -907,21 +1399,54 @@ def cloudfront_alias_items(value):
907
1399
  return []
908
1400
 
909
1401
 
910
- def write_terraform_plan_evidence(payload=None):
1402
+ def write_terraform_plan_evidence(payload=None, state_identity=None, planned_action="update"):
1403
+ payload = payload or {}
1404
+ state_identity = state_identity or terraform_state_identity()
911
1405
  plan_path = Path("terraform-plan.json")
912
1406
  with plan_path.open("w", encoding="utf-8") as handle:
913
1407
  run(["terraform", "show", "-json", "tfplan"], cwd=TF, stdout=handle)
914
1408
  plan_json = json.loads(plan_path.read_text(encoding="utf-8"))
915
- validate_managed_app_plan_scope(payload or {}, plan_json)
916
- validate_environment_plan_scope(payload or {}, plan_json)
1409
+ validate_managed_app_plan_scope(payload, plan_json)
1410
+ validate_environment_plan_scope(payload, plan_json)
917
1411
  artifact = {
918
1412
  "fileName": plan_path.name,
919
1413
  "sha256": sha256_file(plan_path),
920
1414
  "s3Uri": upload_evidence_artifact(plan_path),
921
1415
  }
1416
+ saved_plan_path = TF / "tfplan"
1417
+ saved_plan = {
1418
+ "fileName": TERRAFORM_SAVED_PLAN_NAME,
1419
+ "sha256": sha256_file(saved_plan_path),
1420
+ "s3Uri": upload_evidence_artifact(saved_plan_path, TERRAFORM_SAVED_PLAN_NAME),
1421
+ }
1422
+ descriptor = {
1423
+ "schemaVersion": 1,
1424
+ "contract": TERRAFORM_PLAN_APPROVAL_CONTRACT,
1425
+ "createdAt": datetime.now(UTC).isoformat(),
1426
+ "status": "awaiting-approval",
1427
+ "sessionId": payload.get("sessionId") or os.environ.get("THINKWORK_DEPLOYMENT_SESSION_ID"),
1428
+ "plannedAction": planned_action,
1429
+ "controllerInputSha256": controller_input_sha256(payload),
1430
+ "rawControllerInputSha256": raw_controller_input_sha256(payload),
1431
+ "deploymentConfigSha256": deployment_config_sha256(payload),
1432
+ "terraformVariablesSha256": sha256_file(TF / "terraform.auto.tfvars.json"),
1433
+ "release": release_selection(payload),
1434
+ "state": state_identity,
1435
+ "jsonPlan": artifact,
1436
+ "savedPlan": saved_plan,
1437
+ "summary": terraform_plan_summary(plan_json),
1438
+ }
1439
+ descriptor_artifact = write_json_evidence_artifact(
1440
+ TERRAFORM_PLAN_DESCRIPTOR_NAME,
1441
+ descriptor,
1442
+ )
922
1443
  return {
923
1444
  "artifact": artifact,
924
- "summary": terraform_plan_summary(plan_json),
1445
+ "savedPlan": saved_plan,
1446
+ "approvalDescriptor": descriptor_artifact,
1447
+ "summary": descriptor["summary"],
1448
+ "state": state_identity,
1449
+ "status": "awaiting-approval",
925
1450
  }
926
1451
 
927
1452
 
@@ -969,6 +1494,38 @@ def safe_get_bool(runner_secrets, payload, name, default=False):
969
1494
  return default
970
1495
 
971
1496
 
1497
+ def safe_get_list(runner_secrets, payload, name, default=None):
1498
+ """List analogue of safe_get_bool's precedence. Generated-root
1499
+ variables typed list(string) reject bare strings, so string values
1500
+ (comma-separated, common in Secrets Manager JSON) are split here and
1501
+ the wiring point always emits a real list of non-empty strings."""
1502
+ for source in (runner_secrets, payload):
1503
+ value = source.get(name)
1504
+ if isinstance(value, list):
1505
+ return [str(item).strip() for item in value if str(item).strip()]
1506
+ if isinstance(value, str) and value.strip():
1507
+ return [part.strip() for part in value.split(",") if part.strip()]
1508
+ return list(default or [])
1509
+
1510
+
1511
+ def safe_get_ingress_cidrs(runner_secrets, payload, name, default=None):
1512
+ """safe_get_list precedence, but an explicitly PRESENT empty value ([] or
1513
+ "") wins as "no ingress" instead of falling through to the default —
1514
+ closing the database port (THINK-1082) is spelled dbPublicIngressCidrs: []
1515
+ and must not silently re-seed the world-open default."""
1516
+ for source in (runner_secrets, payload):
1517
+ if name not in source:
1518
+ continue
1519
+ value = source.get(name)
1520
+ if isinstance(value, list):
1521
+ return [str(item).strip() for item in value if str(item).strip()]
1522
+ if isinstance(value, str):
1523
+ return [part.strip() for part in value.split(",") if part.strip()]
1524
+ if value is None:
1525
+ continue
1526
+ return list(default or [])
1527
+
1528
+
972
1529
  def current_terraform_state(stage):
973
1530
  bucket = os.environ.get("THINKWORK_TERRAFORM_STATE_BUCKET")
974
1531
  if not bucket:
@@ -988,96 +1545,760 @@ def current_terraform_state(stage):
988
1545
  return {}
989
1546
 
990
1547
 
991
- def current_terraform_outputs(stage):
992
- outputs = current_terraform_state(stage).get("outputs")
993
- return outputs if isinstance(outputs, dict) else {}
1548
+ def _terraform_state_object_keys(stage):
1549
+ return [
1550
+ f"env:/{stage}/thinkwork/{stage}/terraform.tfstate",
1551
+ f"thinkwork/{stage}/terraform.tfstate",
1552
+ ]
994
1553
 
995
1554
 
996
- def cloudflare_api_token(stage):
997
- existing = os.environ.get("CLOUDFLARE_API_TOKEN")
998
- if existing:
999
- return existing
1000
- parameter_name = f"/thinkwork/{stage}/cloudflare-namespace-token"
1001
- try:
1002
- token = output(
1555
+ def terraform_state_object_snapshot(stage):
1556
+ bucket = require_string(
1557
+ os.environ.get("THINKWORK_TERRAFORM_STATE_BUCKET"),
1558
+ "THINKWORK_TERRAFORM_STATE_BUCKET",
1559
+ )
1560
+ for key in _terraform_state_object_keys(stage):
1561
+ try:
1562
+ head = json.loads(
1563
+ output(
1564
+ [
1565
+ "aws",
1566
+ "s3api",
1567
+ "head-object",
1568
+ "--bucket",
1569
+ bucket,
1570
+ "--key",
1571
+ key,
1572
+ "--output",
1573
+ "json",
1574
+ ]
1575
+ )
1576
+ )
1577
+ body = subprocess.check_output(
1578
+ ["aws", "s3", "cp", f"s3://{bucket}/{key}", "-"],
1579
+ )
1580
+ except (subprocess.CalledProcessError, json.JSONDecodeError):
1581
+ continue
1582
+ state = json.loads(body)
1583
+ return {
1584
+ "bucket": bucket,
1585
+ "key": key,
1586
+ "etag": str(head.get("ETag") or "").strip('"'),
1587
+ "size": int(head.get("ContentLength") or len(body)),
1588
+ "lastModified": head.get("LastModified"),
1589
+ "serverSideEncryption": head.get("ServerSideEncryption"),
1590
+ "versionId": head.get("VersionId"),
1591
+ "sha256": sha256_bytes(body),
1592
+ "lineage": state.get("lineage"),
1593
+ "serial": state.get("serial"),
1594
+ "terraformVersion": state.get("terraform_version"),
1595
+ }
1596
+ raise RuntimeError(f"Terraform state object for stage {stage} was not found")
1597
+
1598
+
1599
+ def _incident_operation(payload):
1600
+ operation = payload.get("operation")
1601
+ if not isinstance(operation, dict) or operation.get("kind") != "incident_recovery":
1602
+ raise RuntimeError("Incident recovery requires operation.kind=incident_recovery")
1603
+ return operation
1604
+
1605
+
1606
+ def _require_exact_incident_payload_aliases(payload, aliases, expected, label):
1607
+ present = {name: payload.get(name) for name in aliases if name in payload}
1608
+ if not present or any(value != expected for value in present.values()):
1609
+ raise RuntimeError(f"TEI v380 recovery {label} does not match exact incident scope")
1610
+
1611
+
1612
+ def tei_v380_recovery_scope():
1613
+ return {
1614
+ "contract": TEI_V380_RECOVERY_CONTRACT,
1615
+ "incidentId": TEI_V380_RECOVERY_INCIDENT_ID,
1616
+ "rejectedPlanSha256": TEI_V380_REJECTED_PLAN_SHA256,
1617
+ "stage": "tei-e2e",
1618
+ "accountId": "637423202447",
1619
+ "region": "us-east-1",
1620
+ "state": {
1621
+ "bucket": TEI_V380_STATE_BUCKET,
1622
+ "key": TEI_V380_STATE_KEY,
1623
+ "lineage": TEI_V380_STATE_LINEAGE,
1624
+ "serial": TEI_V380_STATE_SERIAL,
1625
+ "sha256": TEI_V380_STATE_SHA256,
1626
+ },
1627
+ "lock": {
1628
+ "table": TEI_V380_LOCK_TABLE,
1629
+ "key": TEI_V380_LOCK_KEY,
1630
+ "id": TEI_V380_LOCK_ID,
1631
+ "operation": "OperationTypeApply",
1632
+ "terraformVersion": "1.8.5",
1633
+ "digest": TEI_V380_STATE_DIGEST,
1634
+ },
1635
+ "secret": {
1636
+ "address": TEI_V380_SECRET_ADDRESS,
1637
+ "arn": TEI_V380_SECRET_ARN,
1638
+ "name": TEI_V380_SECRET_NAME,
1639
+ },
1640
+ "kmsKey": {
1641
+ "address": TEI_V380_KMS_KEY_ADDRESS,
1642
+ "id": TEI_V380_KMS_KEY_ID,
1643
+ "arn": TEI_V380_KMS_KEY_ARN,
1644
+ "aliasAddress": TEI_V380_KMS_ALIAS_ADDRESS,
1645
+ "alias": TEI_V380_KMS_ALIAS,
1646
+ },
1647
+ }
1648
+
1649
+
1650
+ def validate_tei_v380_recovery_contract(payload):
1651
+ operation = _incident_operation(payload)
1652
+ expected = tei_v380_recovery_scope()
1653
+ actual = {
1654
+ key: operation.get(key)
1655
+ for key in [
1656
+ "contract",
1657
+ "incidentId",
1658
+ "rejectedPlanSha256",
1659
+ "stage",
1660
+ "accountId",
1661
+ "region",
1662
+ "state",
1663
+ "lock",
1664
+ "secret",
1665
+ "kmsKey",
1666
+ ]
1667
+ }
1668
+ if actual != expected:
1669
+ raise RuntimeError("TEI v380 recovery contract does not match the exact incident scope")
1670
+ expected_operation_keys = {"kind", *expected.keys()}
1671
+ if set(operation) != expected_operation_keys:
1672
+ raise RuntimeError("TEI v380 recovery operation contains unexpected fields")
1673
+ _require_exact_incident_payload_aliases(
1674
+ payload, ("environmentName", "stage"), "tei-e2e", "stage"
1675
+ )
1676
+ _require_exact_incident_payload_aliases(
1677
+ payload, ("awsAccountId", "accountId"), "637423202447", "account"
1678
+ )
1679
+ _require_exact_incident_payload_aliases(payload, ("awsRegion", "region"), "us-east-1", "region")
1680
+ return expected
1681
+
1682
+
1683
+ def assert_incident_recovery_operation_is_exclusive():
1684
+ current_execution = os.environ.get("THINKWORK_DEPLOYMENT_EXECUTION_ARN")
1685
+ state_machine_arn = require_string(
1686
+ os.environ.get("THINKWORK_DEPLOYMENT_STATE_MACHINE_ARN"),
1687
+ "THINKWORK_DEPLOYMENT_STATE_MACHINE_ARN",
1688
+ )
1689
+ executions = json.loads(
1690
+ output(
1003
1691
  [
1004
1692
  "aws",
1005
- "ssm",
1006
- "get-parameter",
1007
- "--name",
1008
- parameter_name,
1009
- "--with-decryption",
1010
- "--query",
1011
- "Parameter.Value",
1693
+ "stepfunctions",
1694
+ "list-executions",
1695
+ "--state-machine-arn",
1696
+ state_machine_arn,
1697
+ "--status-filter",
1698
+ "RUNNING",
1012
1699
  "--output",
1013
- "text",
1700
+ "json",
1014
1701
  ]
1015
1702
  )
1016
- except Exception:
1017
- return ""
1018
- if token:
1019
- os.environ["CLOUDFLARE_API_TOKEN"] = token
1020
- return token
1021
-
1022
-
1023
- def configure_cloudflare_provider_auth(stage):
1024
- cloudflare_api_token(stage)
1703
+ or "{}"
1704
+ ).get("executions", [])
1705
+ if current_execution:
1706
+ other_executions = [
1707
+ item.get("executionArn")
1708
+ for item in executions
1709
+ if item.get("executionArn") != current_execution
1710
+ ]
1711
+ else:
1712
+ # Established control planes do not receive Execution.Id until this
1713
+ # release updates their state-machine definition. Permit exactly the
1714
+ # sole running controller execution during that bounded transition.
1715
+ # Zero executions would mean the recovery was launched outside the
1716
+ # managed controller; more than one is ambiguous and unsafe.
1717
+ if len(executions) != 1:
1718
+ raise RuntimeError(
1719
+ "Incident recovery without execution identity requires exactly one "
1720
+ "running deployment execution"
1721
+ )
1722
+ current_execution = executions[0].get("executionArn")
1723
+ other_executions = []
1724
+ if other_executions:
1725
+ raise RuntimeError("Another deployment execution is running; recovery refused")
1726
+
1727
+ project = require_string(
1728
+ os.environ.get("THINKWORK_DEPLOYMENT_RUNNER_PROJECT_NAME"),
1729
+ "THINKWORK_DEPLOYMENT_RUNNER_PROJECT_NAME",
1730
+ )
1731
+ current_build = os.environ.get("CODEBUILD_BUILD_ID")
1732
+ if not current_build:
1733
+ raise RuntimeError("Incident recovery must run inside the deployment CodeBuild")
1734
+ build_ids = json.loads(
1735
+ output(
1736
+ [
1737
+ "aws",
1738
+ "codebuild",
1739
+ "list-builds-for-project",
1740
+ "--project-name",
1741
+ project,
1742
+ "--sort-order",
1743
+ "DESCENDING",
1744
+ "--no-paginate",
1745
+ "--output",
1746
+ "json",
1747
+ ]
1748
+ )
1749
+ or "{}"
1750
+ ).get("ids", [])[:100]
1751
+ if build_ids:
1752
+ builds = json.loads(
1753
+ output(
1754
+ [
1755
+ "aws",
1756
+ "codebuild",
1757
+ "batch-get-builds",
1758
+ "--ids",
1759
+ *build_ids,
1760
+ "--output",
1761
+ "json",
1762
+ ]
1763
+ )
1764
+ or "{}"
1765
+ ).get("builds", [])
1766
+ other_builds = [
1767
+ item.get("id")
1768
+ for item in builds
1769
+ if item.get("buildStatus") == "IN_PROGRESS" and item.get("id") != current_build
1770
+ ]
1771
+ if other_builds:
1772
+ raise RuntimeError("Another deployment CodeBuild is running; recovery refused")
1773
+ return {
1774
+ "currentExecutionArn": current_execution,
1775
+ "currentBuildId": current_build,
1776
+ "otherRunningExecutions": 0,
1777
+ "otherRunningBuilds": 0,
1778
+ }
1025
1779
 
1026
1780
 
1027
- def cloudflare_zone_id_for_hostname(stage, hostname):
1028
- configured = os.environ.get("THINKWORK_CLOUDFLARE_ZONE_ID", "")
1029
- if configured:
1030
- return configured
1031
- hostname = (hostname or "").strip().lower().rstrip(".")
1032
- if not hostname:
1033
- return ""
1034
- token = cloudflare_api_token(stage)
1035
- if not token:
1036
- return ""
1037
- request = urllib.request.Request(
1038
- "https://api.cloudflare.com/client/v4/zones?per_page=50",
1039
- headers={"Authorization": f"Bearer {token}"},
1040
- )
1041
- try:
1042
- with urllib.request.urlopen(request, timeout=30) as response:
1043
- body = json.loads(response.read().decode("utf-8"))
1044
- except Exception:
1045
- return ""
1046
- zones = body.get("result")
1047
- if not isinstance(zones, list):
1048
- return ""
1049
- matches = []
1050
- for zone in zones:
1051
- if not isinstance(zone, dict):
1052
- continue
1053
- name = str(zone.get("name") or "").strip().lower().rstrip(".")
1054
- zone_id = str(zone.get("id") or "").strip()
1055
- if not name or not zone_id:
1056
- continue
1057
- if hostname == name or hostname.endswith(f".{name}"):
1058
- matches.append((len(name), zone_id))
1059
- if not matches:
1060
- return ""
1061
- return sorted(matches, reverse=True)[0][1]
1781
+ def _redacted_secret_description(description):
1782
+ return {
1783
+ "arn": description.get("ARN"),
1784
+ "name": description.get("Name"),
1785
+ "deletedDate": description.get("DeletedDate"),
1786
+ "kmsKeyId": description.get("KmsKeyId"),
1787
+ "rotationEnabled": bool(description.get("RotationEnabled", False)),
1788
+ }
1062
1789
 
1063
1790
 
1064
- def state_output(outputs, name, default=None):
1065
- value = outputs.get(name)
1066
- if isinstance(value, dict) and "value" in value:
1067
- return value["value"]
1068
- return default
1791
+ def _redacted_kms_description(description):
1792
+ metadata = description.get("KeyMetadata") or {}
1793
+ return {
1794
+ "keyId": metadata.get("KeyId"),
1795
+ "arn": metadata.get("Arn"),
1796
+ "keyState": metadata.get("KeyState"),
1797
+ "deletionDate": metadata.get("DeletionDate"),
1798
+ "enabled": metadata.get("Enabled"),
1799
+ "keyManager": metadata.get("KeyManager"),
1800
+ "origin": metadata.get("Origin"),
1801
+ }
1069
1802
 
1070
1803
 
1071
- def string_state_output(outputs, name):
1072
- value = state_output(outputs, name, "")
1073
- return value.strip() if isinstance(value, str) else ""
1804
+ def _wait_for_json_description(fetch, accept, description, attempts=20, delay_seconds=1):
1805
+ """Bound eventual-consistency waits without ever reading secret values."""
1806
+ latest = None
1807
+ for attempt in range(attempts):
1808
+ latest = fetch()
1809
+ if accept(latest):
1810
+ return latest
1811
+ if attempt + 1 < attempts:
1812
+ time.sleep(delay_seconds)
1813
+ raise RuntimeError(f"Timed out waiting for {description}")
1074
1814
 
1075
1815
 
1076
- def bool_state_output(outputs, name):
1077
- return bool(state_output(outputs, name, False))
1816
+ def _dynamodb_string(item, name):
1817
+ value = (item.get(name) or {}).get("S")
1818
+ return value if isinstance(value, str) else None
1078
1819
 
1079
1820
 
1080
- def has_state_output(outputs, name):
1821
+ def tei_v380_lock_snapshot():
1822
+ def get_item(lock_key):
1823
+ raw = output(
1824
+ [
1825
+ "aws",
1826
+ "dynamodb",
1827
+ "get-item",
1828
+ "--table-name",
1829
+ TEI_V380_LOCK_TABLE,
1830
+ "--key",
1831
+ json.dumps({"LockID": {"S": lock_key}}, separators=(",", ":")),
1832
+ "--consistent-read",
1833
+ "--output",
1834
+ "json",
1835
+ ]
1836
+ )
1837
+ return json.loads(raw or "{}").get("Item") or {}
1838
+
1839
+ lock_item = get_item(TEI_V380_LOCK_KEY)
1840
+ digest_item = get_item(f"{TEI_V380_LOCK_KEY}-md5")
1841
+ info_raw = _dynamodb_string(lock_item, "Info")
1842
+ info = json.loads(info_raw) if info_raw else None
1843
+ if info is not None and not isinstance(info, dict):
1844
+ raise RuntimeError("TEI v380 Terraform lock Info is malformed")
1845
+ return {
1846
+ "table": TEI_V380_LOCK_TABLE,
1847
+ "key": TEI_V380_LOCK_KEY,
1848
+ "present": bool(lock_item),
1849
+ "info": {
1850
+ "id": (info or {}).get("ID"),
1851
+ "operation": (info or {}).get("Operation"),
1852
+ "version": (info or {}).get("Version"),
1853
+ "created": (info or {}).get("Created"),
1854
+ "path": (info or {}).get("Path"),
1855
+ }
1856
+ if info
1857
+ else None,
1858
+ "digestKey": f"{TEI_V380_LOCK_KEY}-md5",
1859
+ "digest": _dynamodb_string(digest_item, "Digest"),
1860
+ }
1861
+
1862
+
1863
+ def force_unlock_tei_v380_state():
1864
+ """Use Terraform's backend-aware unlock path; never delete DynamoDB rows directly."""
1865
+ before = tei_v380_lock_snapshot()
1866
+ expected_info = {
1867
+ "id": TEI_V380_LOCK_ID,
1868
+ "operation": "OperationTypeApply",
1869
+ "version": "1.8.5",
1870
+ "path": TEI_V380_LOCK_KEY,
1871
+ }
1872
+ actual_info = before.get("info") or {}
1873
+ if before.get("digest") != TEI_V380_STATE_DIGEST:
1874
+ raise RuntimeError("TEI v380 Terraform state digest changed; unlock refused")
1875
+ if before.get("present") is not True:
1876
+ return {
1877
+ "method": "terraform-force-unlock",
1878
+ "lockId": TEI_V380_LOCK_ID,
1879
+ "before": before,
1880
+ "after": before,
1881
+ "alreadyUnlocked": True,
1882
+ "directDynamoDbMutation": False,
1883
+ }
1884
+ if any(actual_info.get(key) != value for key, value in expected_info.items()):
1885
+ raise RuntimeError("TEI v380 stale Terraform lock identity does not match")
1886
+
1887
+ unlock_dir = WORK / "tei-v380-force-unlock"
1888
+ unlock_dir.mkdir(parents=True, exist_ok=True)
1889
+ (unlock_dir / "main.tf").write_text(
1890
+ 'terraform {\n required_version = ">= 1.8.5"\n backend "s3" {}\n}\n',
1891
+ encoding="utf-8",
1892
+ )
1893
+ (unlock_dir / "backend.hcl").write_text(
1894
+ "\n".join(
1895
+ [
1896
+ f"bucket = {hcl_string(TEI_V380_STATE_BUCKET)}",
1897
+ f"key = {hcl_string(TEI_V380_STATE_KEY)}",
1898
+ 'region = "us-east-1"',
1899
+ f"dynamodb_table = {hcl_string(TEI_V380_LOCK_TABLE)}",
1900
+ "encrypt = true",
1901
+ "",
1902
+ ]
1903
+ ),
1904
+ encoding="utf-8",
1905
+ )
1906
+ run(
1907
+ [
1908
+ "terraform",
1909
+ "init",
1910
+ "-reconfigure",
1911
+ "-backend-config=backend.hcl",
1912
+ "-input=false",
1913
+ "-no-color",
1914
+ ],
1915
+ cwd=unlock_dir,
1916
+ )
1917
+ run(
1918
+ ["terraform", "force-unlock", "-force", TEI_V380_LOCK_ID],
1919
+ cwd=unlock_dir,
1920
+ )
1921
+ after = tei_v380_lock_snapshot()
1922
+ if after.get("present") is not False:
1923
+ raise RuntimeError("TEI v380 stale Terraform lock remains after force-unlock")
1924
+ if after.get("digest") != before.get("digest"):
1925
+ raise RuntimeError("Terraform state digest changed during force-unlock")
1926
+ return {
1927
+ "method": "terraform-force-unlock",
1928
+ "lockId": TEI_V380_LOCK_ID,
1929
+ "before": before,
1930
+ "after": after,
1931
+ "alreadyUnlocked": False,
1932
+ "directDynamoDbMutation": False,
1933
+ }
1934
+
1935
+
1936
+ def recover_tei_v380_incident(payload):
1937
+ validate_tei_v380_recovery_contract(payload)
1938
+ exclusivity = assert_incident_recovery_operation_is_exclusive()
1939
+ caller_account = output(
1940
+ ["aws", "sts", "get-caller-identity", "--query", "Account", "--output", "text"]
1941
+ )
1942
+ if caller_account != "637423202447":
1943
+ raise RuntimeError("TEI v380 recovery caller is in the wrong AWS account")
1944
+
1945
+ before_state = terraform_state_object_snapshot("tei-e2e")
1946
+ for key, expected in [
1947
+ ("bucket", TEI_V380_STATE_BUCKET),
1948
+ ("key", TEI_V380_STATE_KEY),
1949
+ ("lineage", TEI_V380_STATE_LINEAGE),
1950
+ ("serial", TEI_V380_STATE_SERIAL),
1951
+ ("sha256", TEI_V380_STATE_SHA256),
1952
+ ]:
1953
+ if before_state.get(key) != expected:
1954
+ raise RuntimeError(f"TEI v380 recovery state {key} changed; recovery refused")
1955
+
1956
+ progress = {
1957
+ "schemaVersion": 1,
1958
+ "contract": TEI_V380_RECOVERY_CONTRACT,
1959
+ "incidentId": TEI_V380_RECOVERY_INCIDENT_ID,
1960
+ "rejectedPlanSha256": TEI_V380_REJECTED_PLAN_SHA256,
1961
+ "status": "preflight-passed",
1962
+ "startedAt": datetime.now(UTC).isoformat(),
1963
+ "exclusiveOperation": exclusivity,
1964
+ "stateBefore": before_state,
1965
+ "terraformStateMutation": False,
1966
+ "secretValueRead": False,
1967
+ }
1968
+ write_json_evidence_artifact("incident-recovery-progress.json", progress)
1969
+
1970
+ secret_before_raw = json.loads(
1971
+ output(
1972
+ [
1973
+ "aws",
1974
+ "secretsmanager",
1975
+ "describe-secret",
1976
+ "--secret-id",
1977
+ TEI_V380_SECRET_ARN,
1978
+ "--output",
1979
+ "json",
1980
+ ]
1981
+ )
1982
+ )
1983
+ if (
1984
+ secret_before_raw.get("ARN") != TEI_V380_SECRET_ARN
1985
+ or secret_before_raw.get("Name") != TEI_V380_SECRET_NAME
1986
+ ):
1987
+ raise RuntimeError("TEI v380 recovery secret identity does not match")
1988
+ if secret_before_raw.get("DeletedDate"):
1989
+ run(
1990
+ [
1991
+ "aws",
1992
+ "secretsmanager",
1993
+ "restore-secret",
1994
+ "--secret-id",
1995
+ TEI_V380_SECRET_ARN,
1996
+ ],
1997
+ stdout=subprocess.DEVNULL,
1998
+ )
1999
+
2000
+ def describe_secret():
2001
+ return json.loads(
2002
+ output(
2003
+ [
2004
+ "aws",
2005
+ "secretsmanager",
2006
+ "describe-secret",
2007
+ "--secret-id",
2008
+ TEI_V380_SECRET_ARN,
2009
+ "--output",
2010
+ "json",
2011
+ ]
2012
+ )
2013
+ )
2014
+
2015
+ secret_after_raw = _wait_for_json_description(
2016
+ describe_secret,
2017
+ lambda value: not value.get("DeletedDate"),
2018
+ "restored TEI Twenty OAuth client secret",
2019
+ )
2020
+ if (
2021
+ secret_after_raw.get("ARN") != TEI_V380_SECRET_ARN
2022
+ or secret_after_raw.get("Name") != TEI_V380_SECRET_NAME
2023
+ ):
2024
+ raise RuntimeError("Restored TEI v380 recovery secret identity does not match")
2025
+ progress["secret"] = {
2026
+ "address": TEI_V380_SECRET_ADDRESS,
2027
+ "before": _redacted_secret_description(secret_before_raw),
2028
+ "after": _redacted_secret_description(secret_after_raw),
2029
+ "status": "restored",
2030
+ }
2031
+ progress["status"] = "secret-restored"
2032
+ write_json_evidence_artifact("incident-recovery-progress.json", progress)
2033
+
2034
+ kms_before_raw = json.loads(
2035
+ output(
2036
+ [
2037
+ "aws",
2038
+ "kms",
2039
+ "describe-key",
2040
+ "--key-id",
2041
+ TEI_V380_KMS_KEY_ID,
2042
+ "--output",
2043
+ "json",
2044
+ ]
2045
+ )
2046
+ )
2047
+ kms_before = _redacted_kms_description(kms_before_raw)
2048
+ if (
2049
+ kms_before.get("keyId") != TEI_V380_KMS_KEY_ID
2050
+ or kms_before.get("arn") != TEI_V380_KMS_KEY_ARN
2051
+ ):
2052
+ raise RuntimeError("TEI v380 recovery KMS key identity does not match")
2053
+ if kms_before.get("keyState") == "PendingDeletion":
2054
+ run(
2055
+ ["aws", "kms", "cancel-key-deletion", "--key-id", TEI_V380_KMS_KEY_ID],
2056
+ stdout=subprocess.DEVNULL,
2057
+ )
2058
+
2059
+ def describe_kms_key():
2060
+ return json.loads(
2061
+ output(
2062
+ [
2063
+ "aws",
2064
+ "kms",
2065
+ "describe-key",
2066
+ "--key-id",
2067
+ TEI_V380_KMS_KEY_ID,
2068
+ "--output",
2069
+ "json",
2070
+ ]
2071
+ )
2072
+ )
2073
+
2074
+ kms_mid_raw = _wait_for_json_description(
2075
+ describe_kms_key,
2076
+ lambda value: _redacted_kms_description(value).get("keyState") in {"Disabled", "Enabled"},
2077
+ "cancelled TEI turn-assertion KMS deletion",
2078
+ )
2079
+ if _redacted_kms_description(kms_mid_raw).get("keyState") == "Disabled":
2080
+ run(
2081
+ ["aws", "kms", "enable-key", "--key-id", TEI_V380_KMS_KEY_ID],
2082
+ stdout=subprocess.DEVNULL,
2083
+ )
2084
+ kms_after_raw = _wait_for_json_description(
2085
+ describe_kms_key,
2086
+ lambda value: (
2087
+ _redacted_kms_description(value).get("keyState") == "Enabled"
2088
+ and _redacted_kms_description(value).get("enabled") is True
2089
+ ),
2090
+ "enabled TEI turn-assertion KMS key",
2091
+ )
2092
+ kms_after = _redacted_kms_description(kms_after_raw)
2093
+ if (
2094
+ kms_after.get("keyId") != TEI_V380_KMS_KEY_ID
2095
+ or kms_after.get("arn") != TEI_V380_KMS_KEY_ARN
2096
+ ):
2097
+ raise RuntimeError("Recovered TEI v380 KMS key identity does not match")
2098
+ progress["kmsKey"] = {
2099
+ "address": TEI_V380_KMS_KEY_ADDRESS,
2100
+ "aliasAddress": TEI_V380_KMS_ALIAS_ADDRESS,
2101
+ "alias": TEI_V380_KMS_ALIAS,
2102
+ "before": kms_before,
2103
+ "after": kms_after,
2104
+ "status": "enabled",
2105
+ }
2106
+ progress["status"] = "resources-recovered"
2107
+ write_json_evidence_artifact("incident-recovery-progress.json", progress)
2108
+
2109
+ progress["terraformLock"] = force_unlock_tei_v380_state()
2110
+ progress["status"] = "resources-recovered-and-state-unlocked"
2111
+ write_json_evidence_artifact("incident-recovery-progress.json", progress)
2112
+
2113
+ after_state = terraform_state_object_snapshot("tei-e2e")
2114
+ if after_state != before_state:
2115
+ raise RuntimeError("Terraform state object changed during incident recovery")
2116
+ progress.update(
2117
+ {
2118
+ "status": "succeeded",
2119
+ "completedAt": datetime.now(UTC).isoformat(),
2120
+ "stateAfter": after_state,
2121
+ "terraformStateMutation": False,
2122
+ "secretValueRead": False,
2123
+ "nextStep": "proof-enabled-plan-with-imports",
2124
+ }
2125
+ )
2126
+ artifact = write_json_evidence_artifact("incident-recovery.json", progress)
2127
+ return {"proof": progress, "artifact": artifact}
2128
+
2129
+
2130
+ def tei_v380_recovery_import_blocks(payload, vars_json):
2131
+ recovery = payload.get("incidentRecovery")
2132
+ if recovery is None:
2133
+ return ""
2134
+ if not isinstance(recovery, dict):
2135
+ raise RuntimeError("incidentRecovery must be an object")
2136
+ evidence_ref = recovery.get("evidence")
2137
+ actual_scope = {key: recovery.get(key) for key in tei_v380_recovery_scope()}
2138
+ if actual_scope != tei_v380_recovery_scope():
2139
+ raise RuntimeError("Incident recovery import scope does not match TEI v380 incident")
2140
+ if not isinstance(evidence_ref, dict):
2141
+ raise RuntimeError("incidentRecovery.evidence is required")
2142
+
2143
+ # The decoded Terraform state is used elsewhere to preserve outputs and
2144
+ # guardrails, but it does not carry an object digest. Recovery approval is
2145
+ # explicitly bound to the S3 state object's SHA/lineage/serial, so compare
2146
+ # against the authoritative object snapshot instead of the decoded state.
2147
+ current_state = terraform_state_object_snapshot(
2148
+ vars_json.get("stage") or recovery.get("stage") or ""
2149
+ )
2150
+
2151
+ evidence_path = WORK / "accepted-incident-recovery.json"
2152
+ _download_approved_artifact(
2153
+ evidence_ref.get("s3Uri"),
2154
+ evidence_path,
2155
+ evidence_ref.get("sha256"),
2156
+ "incidentRecovery.evidence",
2157
+ )
2158
+ evidence = json.loads(evidence_path.read_text(encoding="utf-8"))
2159
+ if (
2160
+ evidence.get("contract") != TEI_V380_RECOVERY_CONTRACT
2161
+ or evidence.get("incidentId") != TEI_V380_RECOVERY_INCIDENT_ID
2162
+ or evidence.get("rejectedPlanSha256") != TEI_V380_REJECTED_PLAN_SHA256
2163
+ or evidence.get("status") != "succeeded"
2164
+ or evidence.get("terraformStateMutation") is not False
2165
+ or evidence.get("secretValueRead") is not False
2166
+ or evidence.get("stateBefore") != evidence.get("stateAfter")
2167
+ ):
2168
+ raise RuntimeError("Incident recovery evidence is incomplete or unsafe")
2169
+ evidence_state = evidence.get("stateAfter") or {}
2170
+ if (
2171
+ evidence_state.get("lineage") != TEI_V380_STATE_LINEAGE
2172
+ or evidence_state.get("serial") != TEI_V380_STATE_SERIAL
2173
+ or evidence_state.get("sha256") != TEI_V380_STATE_SHA256
2174
+ ):
2175
+ raise RuntimeError("Incident recovery evidence state identity does not match")
2176
+ if (
2177
+ current_state.get("lineage") != TEI_V380_STATE_LINEAGE
2178
+ or current_state.get("serial") != TEI_V380_STATE_SERIAL
2179
+ or current_state.get("sha256") != TEI_V380_STATE_SHA256
2180
+ ):
2181
+ raise RuntimeError("Current Terraform state moved after incident recovery")
2182
+
2183
+ if vars_json.get("enable_agentcore_multiplayer_proof") is not True:
2184
+ raise RuntimeError("Incident recovery import requires AgentCore proof enabled")
2185
+ if vars_json.get("agentcore_multiplayer_proof_tenant_slug") != "tei":
2186
+ raise RuntimeError("Incident recovery import requires the exact TEI tenant slug")
2187
+ if vars_json.get("agentcore_multiplayer_proof_owner_allowlist") != TEI_V380_OWNER_ALLOWLIST:
2188
+ raise RuntimeError("Incident recovery import requires the exact owner allowlist")
2189
+ if vars_json.get("finalize_auth_retirement") is not False:
2190
+ raise RuntimeError("Incident recovery import cannot finalize auth retirement")
2191
+ if vars_json.get("auth_retirement_phase") != "coexistence":
2192
+ raise RuntimeError("Incident recovery import requires auth coexistence")
2193
+ if vars_json.get("microsoft_oauth_tenant") != "organizations":
2194
+ raise RuntimeError("Incident recovery import requires Microsoft organizations tenant")
2195
+
2196
+ return f"""
2197
+ # TEI v380 incident recovery imports. These addresses and IDs are bound to the
2198
+ # independently accepted recovery evidence above. They recover ownership of
2199
+ # the original secret and KMS key without reading or replacing either value.
2200
+ import {{
2201
+ to = {TEI_V380_SECRET_ADDRESS}
2202
+ id = {hcl_string(TEI_V380_SECRET_ARN)}
2203
+ }}
2204
+
2205
+ import {{
2206
+ to = {TEI_V380_KMS_KEY_ADDRESS}
2207
+ id = {hcl_string(TEI_V380_KMS_KEY_ID)}
2208
+ }}
2209
+ """
2210
+
2211
+
2212
+ def current_terraform_outputs(stage):
2213
+ outputs = current_terraform_state(stage).get("outputs")
2214
+ return outputs if isinstance(outputs, dict) else {}
2215
+
2216
+
2217
+ def cloudflare_api_token(stage):
2218
+ existing = os.environ.get("CLOUDFLARE_API_TOKEN")
2219
+ if existing:
2220
+ return existing
2221
+ parameter_name = f"/thinkwork/{stage}/cloudflare-namespace-token"
2222
+ try:
2223
+ token = output(
2224
+ [
2225
+ "aws",
2226
+ "ssm",
2227
+ "get-parameter",
2228
+ "--name",
2229
+ parameter_name,
2230
+ "--with-decryption",
2231
+ "--query",
2232
+ "Parameter.Value",
2233
+ "--output",
2234
+ "text",
2235
+ ]
2236
+ )
2237
+ except Exception:
2238
+ return ""
2239
+ if token:
2240
+ os.environ["CLOUDFLARE_API_TOKEN"] = token
2241
+ return token
2242
+
2243
+
2244
+ def configure_cloudflare_provider_auth(stage):
2245
+ cloudflare_api_token(stage)
2246
+
2247
+
2248
+ def cloudflare_zone_id_for_hostname(stage, hostname):
2249
+ configured = os.environ.get("THINKWORK_CLOUDFLARE_ZONE_ID", "")
2250
+ if configured:
2251
+ return configured
2252
+ hostname = (hostname or "").strip().lower().rstrip(".")
2253
+ if not hostname:
2254
+ return ""
2255
+ token = cloudflare_api_token(stage)
2256
+ if not token:
2257
+ return ""
2258
+ request = urllib.request.Request(
2259
+ "https://api.cloudflare.com/client/v4/zones?per_page=50",
2260
+ headers={"Authorization": f"Bearer {token}"},
2261
+ )
2262
+ try:
2263
+ with urllib.request.urlopen(request, timeout=30) as response:
2264
+ body = json.loads(response.read().decode("utf-8"))
2265
+ except Exception:
2266
+ return ""
2267
+ zones = body.get("result")
2268
+ if not isinstance(zones, list):
2269
+ return ""
2270
+ matches = []
2271
+ for zone in zones:
2272
+ if not isinstance(zone, dict):
2273
+ continue
2274
+ name = str(zone.get("name") or "").strip().lower().rstrip(".")
2275
+ zone_id = str(zone.get("id") or "").strip()
2276
+ if not name or not zone_id:
2277
+ continue
2278
+ if hostname == name or hostname.endswith(f".{name}"):
2279
+ matches.append((len(name), zone_id))
2280
+ if not matches:
2281
+ return ""
2282
+ return sorted(matches, reverse=True)[0][1]
2283
+
2284
+
2285
+ def state_output(outputs, name, default=None):
2286
+ value = outputs.get(name)
2287
+ if isinstance(value, dict) and "value" in value:
2288
+ return value["value"]
2289
+ return default
2290
+
2291
+
2292
+ def string_state_output(outputs, name):
2293
+ value = state_output(outputs, name, "")
2294
+ return value.strip() if isinstance(value, str) else ""
2295
+
2296
+
2297
+ def bool_state_output(outputs, name):
2298
+ return bool(state_output(outputs, name, False))
2299
+
2300
+
2301
+ def has_state_output(outputs, name):
1081
2302
  value = outputs.get(name)
1082
2303
  return isinstance(value, dict) and "value" in value
1083
2304
 
@@ -1233,6 +2454,42 @@ def config_string_list(desired_config, key, env_name, default=None):
1233
2454
  return list(default or [])
1234
2455
 
1235
2456
 
2457
+ def coerce_string_list(value):
2458
+ """Normalize a runner-secrets / payload value into a list[str].
2459
+
2460
+ Accepts a native list, a JSON-array string (``["a","b"]``), or a
2461
+ comma-separated string. Blank entries are dropped. Anything else yields
2462
+ an empty list. Order and duplicates are preserved for the caller; the
2463
+ Terraform side applies distinct().
2464
+ """
2465
+ if isinstance(value, list):
2466
+ return [str(item).strip() for item in value if str(item).strip()]
2467
+ if isinstance(value, str):
2468
+ text = value.strip()
2469
+ if not text:
2470
+ return []
2471
+ if text.startswith("["):
2472
+ try:
2473
+ parsed = json.loads(text)
2474
+ except ValueError:
2475
+ parsed = None
2476
+ if isinstance(parsed, list):
2477
+ return [str(item).strip() for item in parsed if str(item).strip()]
2478
+ return [item.strip() for item in text.split(",") if item.strip()]
2479
+ return []
2480
+
2481
+
2482
+ def runner_secret_string_list(runner_secrets, payload, *keys):
2483
+ """First non-empty list found across runner_secrets then payload for keys."""
2484
+ for source in (runner_secrets or {}, payload or {}):
2485
+ for key in keys:
2486
+ if key in source:
2487
+ result = coerce_string_list(source.get(key))
2488
+ if result:
2489
+ return result
2490
+ return []
2491
+
2492
+
1236
2493
  def required_config_value(
1237
2494
  desired_config,
1238
2495
  manifest_images,
@@ -1951,19 +3208,13 @@ def managed_app_terraform_overrides(payload, stage, account_id, current_outputs,
1951
3208
  # plans a destroy of a cert still attached to the n8n ALB (observed:
1952
3209
  # McPherson update to v0.1.0-canary.314). When state shows the root
1953
3210
  # managed cert, reproduce the install-time inputs instead.
1954
- managed_n8n_cert = state_root_resource_attributes(
1955
- current_state, "aws_acm_certificate", "n8n"
1956
- )
3211
+ managed_n8n_cert = state_root_resource_attributes(current_state, "aws_acm_certificate", "n8n")
1957
3212
  if managed_n8n_cert:
1958
3213
  overrides["n8n_domain"] = (
1959
- managed_n8n_cert.get("domain_name")
1960
- or url_hostname(overrides["n8n_public_url"])
1961
- or ""
3214
+ managed_n8n_cert.get("domain_name") or url_hostname(overrides["n8n_public_url"]) or ""
1962
3215
  )
1963
3216
  overrides["n8n_certificate_arn"] = ""
1964
- foundation_n8n_dns_name = overrides["n8n_domain"] or url_hostname(
1965
- overrides["n8n_public_url"]
1966
- )
3217
+ foundation_n8n_dns_name = overrides["n8n_domain"] or url_hostname(overrides["n8n_public_url"])
1967
3218
  if overrides["n8n_provisioned"] and foundation_n8n_dns_name:
1968
3219
  # State-derived zone lookup goes blank once the cloudflare_record
1969
3220
  # resources are absent (e.g. a prior partial destroy) — fall back
@@ -2081,7 +3332,9 @@ def resolve_agentcore_pi_source_image_uri(payload):
2081
3332
  safe_get(operation, "action", default=safe_get(payload, "action", "phase", default=""))
2082
3333
  ).lower()
2083
3334
  kind = str(safe_get(operation, "kind", default="")).lower()
2084
- is_customer_update = action == "update" or kind == "foundation"
3335
+ is_customer_update = (
3336
+ action == "update" or kind == "foundation"
3337
+ ) and kind != "identity_provider"
2085
3338
  explicit = safe_get(payload, "agentcorePiSourceImageUri", default="")
2086
3339
  if explicit:
2087
3340
  if is_customer_update:
@@ -2104,31 +3357,209 @@ def resolve_agentcore_pi_source_image_uri(payload):
2104
3357
  return release_runtime_image("agentcore-pi-amd64")
2105
3358
 
2106
3359
 
2107
- def release_git_sha():
2108
- if not MANIFEST.exists():
2109
- return ""
2110
- manifest = json.loads(MANIFEST.read_text(encoding="utf-8"))
2111
- release = manifest.get("release")
2112
- if isinstance(release, dict):
2113
- git_sha = release.get("gitSha")
2114
- return git_sha if isinstance(git_sha, str) else ""
2115
- return ""
3360
+ def escape_env_control_chars(value):
3361
+ # AgentCore rejects env values with raw control characters (the PEM's
3362
+ # newlines); mirror them as literal escapes exactly like the dev-pipeline
3363
+ # env mirror does (scripts/update-agentcore-runtime-image.sh). Consumers
3364
+ # un-escape (the Pi container's publicKeyPemFromEnv normalizes \n).
3365
+ return value.replace("\r", "\\r").replace("\n", "\\n").replace("\t", "\\t")
2116
3366
 
2117
3367
 
2118
- def ensure_release_manifest_available(manifest_url, manifest_sha256):
2119
- if not manifest_url and not MANIFEST.exists():
2120
- return
2121
- if not MANIFEST.exists():
2122
- download(manifest_url, MANIFEST)
2123
- verify_release_manifest_digest(MANIFEST, manifest_sha256)
3368
+ def reconcile_agentcore_pi_runtime(vars_json, payload):
3369
+ """Create/update the Pi Bedrock AgentCore Runtime after a successful apply
3370
+ (THINK-584 U5). Pins the runtime to the immutable digest of the customer's
3371
+ mirrored Pi image, mirrors the Pi Lambda env atomically (abort on empty),
3372
+ and records the runtime id in SSM. Secret env values never appear in logs
3373
+ or argv — the request travels to the node entrypoint on stdin."""
3374
+ stage = vars_json["stage"]
3375
+ region = os.environ.get("AWS_REGION") or safe_get(payload, "awsRegion", default="us-east-1")
3376
+ runtime_bundle_dir = os.environ.get("THINKWORK_AGENTCORE_CONTROL_RUNTIME_DIR", "")
3377
+ entrypoint = (
3378
+ Path(runtime_bundle_dir) / "reconcile_pi_runtime.js" if runtime_bundle_dir else None
3379
+ )
3380
+ if entrypoint is None or not entrypoint.is_file():
3381
+ print(
3382
+ "[runner] AgentCore Pi runtime reconcile SKIPPED: release bundle predates "
3383
+ "reconcile_pi_runtime.js (THINK-584 U5). Re-seed the runner bundle to enable "
3384
+ "the customer runtime-update path."
3385
+ )
3386
+ return None
2124
3387
 
3388
+ lambda_name = f"thinkwork-{stage}-agentcore-pi"
3389
+ fn = json.loads(
3390
+ output(
3391
+ [
3392
+ "aws",
3393
+ "lambda",
3394
+ "get-function-configuration",
3395
+ "--function-name",
3396
+ lambda_name,
3397
+ "--region",
3398
+ region,
3399
+ ]
3400
+ )
3401
+ )
3402
+ environment = {
3403
+ key: escape_env_control_chars(value) if isinstance(value, str) else value
3404
+ for key, value in ((fn.get("Environment") or {}).get("Variables") or {}).items()
3405
+ }
3406
+ if not environment:
3407
+ raise RuntimeError(
3408
+ f"AgentCore Pi runtime reconcile: {lambda_name} reported an EMPTY environment; "
3409
+ "mirroring it would strand the runtime (atomic-or-abort)."
3410
+ )
3411
+ # THINK-586 U7 (KTD6): runtime-only overlay — the warm-session cache
3412
+ # factory gates on this var; it must never enter the Lambda terraform env.
3413
+ environment["AGENTCORE_RUNTIME_SESSION_CACHE"] = "1"
3414
+ role_arn = fn.get("Role") or ""
3415
+ account_id = (fn.get("FunctionArn") or "").split(":")[4] if fn.get("FunctionArn") else ""
3416
+ if not role_arn or not account_id:
3417
+ raise RuntimeError("AgentCore Pi runtime reconcile: Lambda did not report Role/ARN")
3418
+
3419
+ # AgentCore runtimes are arm64-only ("Supported platforms: [arm64]" —
3420
+ # first tei-e2e canary, 2026-08-04). The Lambda's amd64 pin
3421
+ # (agentcorePiSourceImageUri) is the WRONG architecture for the runtime;
3422
+ # the runtime image is mirrored into the customer ECR under the release
3423
+ # version tag by the release workflow's `mirror-customer-images` job
3424
+ # (THINK-616; targets listed in .github/release-mirror-targets.json).
3425
+ release_version = os.environ.get("THINKWORK_RELEASE_VERSION", "") or str(
3426
+ safe_get(payload, "releaseVersion", default="")
3427
+ )
3428
+ if not release_version:
3429
+ raise RuntimeError(
3430
+ "AgentCore Pi runtime reconcile: no release version available to "
3431
+ "resolve the arm64 runtime image tag."
3432
+ )
3433
+ image_tag = f"{release_version}-pi-arm64"
3434
+ repository = f"thinkwork-{stage}-agentcore"
3435
+ digest = output(
3436
+ [
3437
+ "aws",
3438
+ "ecr",
3439
+ "describe-images",
3440
+ "--repository-name",
3441
+ repository,
3442
+ "--region",
3443
+ region,
3444
+ "--image-ids",
3445
+ f"imageTag={image_tag}",
3446
+ "--query",
3447
+ "imageDetails[0].imageDigest",
3448
+ "--output",
3449
+ "text",
3450
+ ]
3451
+ )
3452
+ if not re.fullmatch(r"sha256:[0-9a-f]{64}", digest):
3453
+ raise RuntimeError(
3454
+ f"AgentCore Pi runtime reconcile: could not resolve an immutable digest for "
3455
+ f"{repository}:{image_tag} (got '{digest}'). Mirror the release's "
3456
+ "arm64 Pi runtime image into this customer's ECR under that tag "
3457
+ "before retrying."
3458
+ )
3459
+ image_uri = f"{account_id}.dkr.ecr.{region}.amazonaws.com/{repository}@{digest}"
2125
3460
 
2126
- def source_repo_and_ref(module_source, release_version):
2127
- source = module_source.removeprefix("git::")
2128
- source_path, _, query = source.partition("?")
2129
- params = urllib.parse.parse_qs(query)
2130
- ref = params.get("ref", [release_version])[0]
2131
- if source_path == "thinkwork-ai/thinkwork/aws":
3461
+ ssm_name = f"/thinkwork/{stage}/agentcore/runtime-id-pi"
3462
+ runtime_id = ""
3463
+ try:
3464
+ runtime_id = output(
3465
+ [
3466
+ "aws",
3467
+ "ssm",
3468
+ "get-parameter",
3469
+ "--name",
3470
+ ssm_name,
3471
+ "--region",
3472
+ region,
3473
+ "--query",
3474
+ "Parameter.Value",
3475
+ "--output",
3476
+ "text",
3477
+ ],
3478
+ stderr=subprocess.DEVNULL,
3479
+ )
3480
+ except subprocess.CalledProcessError:
3481
+ runtime_id = ""
3482
+ if runtime_id == "None":
3483
+ runtime_id = ""
3484
+
3485
+ # AgentCore runtime names must match [a-zA-Z][a-zA-Z0-9_]{0,47} — no
3486
+ # hyphens. Customer stages like "tei-e2e" broke the create with a
3487
+ # ValidationException (first live canary of this step, 2026-08-04).
3488
+ runtime_stage = re.sub(r"[^a-zA-Z0-9_]", "_", stage)
3489
+ request = {
3490
+ "region": region,
3491
+ "runtimeName": f"thinkwork_{runtime_stage}_pi",
3492
+ "runtimeId": runtime_id,
3493
+ "roleArn": role_arn,
3494
+ "imageUri": image_uri,
3495
+ "environment": environment,
3496
+ "waitSeconds": 900,
3497
+ }
3498
+ print(
3499
+ f"[runner] Reconciling AgentCore Pi runtime to {image_uri} "
3500
+ f"({len(environment)} env vars mirrored; values not logged)"
3501
+ )
3502
+ reconciled = subprocess.run(
3503
+ ["node", str(entrypoint)],
3504
+ input=json.dumps(request),
3505
+ capture_output=True,
3506
+ text=True,
3507
+ )
3508
+ if reconciled.returncode != 0:
3509
+ raise RuntimeError(
3510
+ "AgentCore Pi runtime reconcile failed: "
3511
+ f"{(reconciled.stderr or '').strip() or f'exit {reconciled.returncode}'}"
3512
+ )
3513
+ result = json.loads(reconciled.stdout)
3514
+ run(
3515
+ [
3516
+ "aws",
3517
+ "ssm",
3518
+ "put-parameter",
3519
+ "--name",
3520
+ ssm_name,
3521
+ "--value",
3522
+ str(result["runtimeId"]),
3523
+ "--type",
3524
+ "String",
3525
+ "--overwrite",
3526
+ "--region",
3527
+ region,
3528
+ ],
3529
+ stdout=subprocess.DEVNULL,
3530
+ )
3531
+ print(
3532
+ f"[runner] AgentCore Pi runtime ready: {result['runtimeId']} "
3533
+ f"v{result.get('version')} ({'created' if result.get('created') else 'updated'})"
3534
+ )
3535
+ return result
3536
+
3537
+
3538
+ def release_git_sha():
3539
+ if not MANIFEST.exists():
3540
+ return ""
3541
+ manifest = json.loads(MANIFEST.read_text(encoding="utf-8"))
3542
+ release = manifest.get("release")
3543
+ if isinstance(release, dict):
3544
+ git_sha = release.get("gitSha")
3545
+ return git_sha if isinstance(git_sha, str) else ""
3546
+ return ""
3547
+
3548
+
3549
+ def ensure_release_manifest_available(manifest_url, manifest_sha256):
3550
+ if not manifest_url and not MANIFEST.exists():
3551
+ return
3552
+ if not MANIFEST.exists():
3553
+ download(manifest_url, MANIFEST)
3554
+ verify_release_manifest_digest(MANIFEST, manifest_sha256)
3555
+
3556
+
3557
+ def source_repo_and_ref(module_source, release_version):
3558
+ source = module_source.removeprefix("git::")
3559
+ source_path, _, query = source.partition("?")
3560
+ params = urllib.parse.parse_qs(query)
3561
+ ref = params.get("ref", [release_version])[0]
3562
+ if source_path == "thinkwork-ai/thinkwork/aws":
2132
3563
  repo = "https://github.com/thinkwork-ai/thinkwork.git"
2133
3564
  ref = release_git_sha() or release_version
2134
3565
  elif source_path.startswith("github.com/"):
@@ -2203,6 +3634,10 @@ def checkout_source(module_source, release_version):
2203
3634
 
2204
3635
 
2205
3636
  def database_url_from_outputs(outputs):
3637
+ """The database handle every schema step uses: a `postgresql://` URL for
3638
+ psql, or a DataApiDatabase when this run's dbAccess is `data-api`."""
3639
+ if DATABASE_ACCESS_MODE == "data-api":
3640
+ return DataApiDatabase.from_outputs(outputs)
2206
3641
  endpoint = outputs.get("db_cluster_endpoint", {}).get("value")
2207
3642
  secret_arn = outputs.get("db_secret_arn", {}).get("value")
2208
3643
  database_name = outputs.get("database_name", {}).get("value") or "thinkwork"
@@ -2240,18 +3675,698 @@ def psql_env(database_url):
2240
3675
  return env
2241
3676
 
2242
3677
 
3678
+ DATABASE_URL_PASSWORD_RE = re.compile(r"(://[^/:@\s]*:)[^@\s]*(@)")
3679
+
3680
+
3681
+ def redact_database_url(value):
3682
+ """Replace the password segment of any `scheme://user:password@host` URL
3683
+ in `value` with `***`. Applied to everything a database URL can reach in a
3684
+ log or exception message: a failed psql surfaces its argv (URL included)
3685
+ through CalledProcessError, which CodeBuild prints to CloudWatch."""
3686
+ if isinstance(value, (list, tuple)):
3687
+ return [redact_database_url(item) for item in value]
3688
+ if not isinstance(value, str):
3689
+ return value
3690
+ return DATABASE_URL_PASSWORD_RE.sub(r"\1***\2", value)
3691
+
3692
+
3693
+ def _redacted_psql_error(error):
3694
+ """Re-raise a psql CalledProcessError with the connection password
3695
+ redacted from its command. Same exception type, same returncode, same
3696
+ captured output — only the secret is gone."""
3697
+ return subprocess.CalledProcessError(
3698
+ error.returncode,
3699
+ redact_database_url(error.cmd),
3700
+ output=redact_database_url(error.output),
3701
+ stderr=redact_database_url(error.stderr),
3702
+ )
3703
+
3704
+
2243
3705
  def psql(database_url, sql=None, file=None, variables=None):
3706
+ if isinstance(database_url, DataApiDatabase):
3707
+ return database_url.execute(sql=sql, file=file, variables=variables)
2244
3708
  args = ["psql", database_url, "-v", "ON_ERROR_STOP=1"]
2245
3709
  for key, value in (variables or {}).items():
2246
3710
  args.extend(["-v", f"{key}={value}"])
2247
- if file:
2248
- args.extend(["-f", str(file)])
2249
- return run(args)
2250
- return subprocess.run(args, input=sql, check=True, text=True)
3711
+ try:
3712
+ if file:
3713
+ args.extend(["-f", str(file)])
3714
+ return run(args)
3715
+ return subprocess.run(args, input=sql, check=True, text=True)
3716
+ except subprocess.CalledProcessError as error:
3717
+ raise _redacted_psql_error(error) from None
2251
3718
 
2252
3719
 
2253
3720
  def psql_output(database_url, sql):
2254
- return output(["psql", database_url, "-tAc", sql])
3721
+ if isinstance(database_url, DataApiDatabase):
3722
+ return database_url.scalar(sql)
3723
+ try:
3724
+ return output(["psql", database_url, "-tAc", sql])
3725
+ except subprocess.CalledProcessError as error:
3726
+ raise _redacted_psql_error(error) from None
3727
+
3728
+
3729
+ ################################################################################
3730
+ # Database access over the RDS Data API (THINK-1082)
3731
+ #
3732
+ # The runner is the last client of the Aurora cluster's public endpoint. With
3733
+ # every API Lambda inside the VPC, `psql` from this out-of-VPC CodeBuild is
3734
+ # the only reason the world-open 0.0.0.0/0 -> 5432 rule still exists. Setting
3735
+ # `dbAccess: "data-api"` (runner-secrets document, or the controller payload)
3736
+ # routes every psql / psql_output call below through the RDS Data API instead:
3737
+ # HTTPS, IAM-authorised, no network path — the lane the Brain already uses for
3738
+ # its own migrations against these clusters. The call sites do not change;
3739
+ # `database_url_from_outputs` hands back a DataApiDatabase handle instead of a
3740
+ # URL and `psql` / `psql_output` dispatch on it.
3741
+ #
3742
+ # What the Data API does not give us and how it is bridged here:
3743
+ # - one statement per ExecuteStatement -> scripts are split into statements
3744
+ # (dollar-quote / quote / comment aware; drizzle `--> statement-breakpoint`
3745
+ # markers are honoured as boundaries too);
3746
+ # - no psql variables -> `:'name'`, `:"name"` and `:name` are substituted for
3747
+ # the variables the runner passes (`stage`, the compliance passwords) — in
3748
+ # code only, never inside quotes, never touching `::casts`;
3749
+ # - no psql meta-commands -> `\set`, `\echo`/`\qecho`, `\if :{?name}` /
3750
+ # `\else` / `\endif` and `\quit` (the forms the migration files use) are
3751
+ # interpreted; anything else is refused by name;
3752
+ # - 45 s per call -> DDL runs with continueAfterTimeout so a slow statement
3753
+ # is never aborted server-side, and every script runs inside ONE Data API
3754
+ # transaction (BeginTransaction ... CommitTransaction) so its statements
3755
+ # serialise on one connection and a statement that outlived its call can
3756
+ # never race the next one. A script's own top-level BEGIN/COMMIT map onto
3757
+ # that transaction; `CREATE INDEX CONCURRENTLY` (which refuses to run in a
3758
+ # transaction block) commits the open one and runs standalone.
3759
+ # - ON_ERROR_STOP -> the first failing statement stops the script, rolls the
3760
+ # transaction back and raises DatabaseCommandError, a CalledProcessError
3761
+ # subclass, so every existing `except subprocess.CalledProcessError`
3762
+ # keeps working and the migration ledger is recorded only after success.
3763
+ ################################################################################
3764
+
3765
+ DATABASE_ACCESS_MODES = ("psql", "data-api")
3766
+ DATABASE_ACCESS_MODE = "psql"
3767
+ RDS_DATA_API_MAX_SQL_BYTES = 65536
3768
+ STATEMENT_BREAKPOINT = "--> statement-breakpoint"
3769
+
3770
+
3771
+ def set_database_access_mode(vars_json):
3772
+ """Select psql (today) or the RDS Data API for every database step of
3773
+ this run; logged once so the build log proves which lane ran."""
3774
+ global DATABASE_ACCESS_MODE
3775
+ mode = str((vars_json or {}).get("db_access") or "psql")
3776
+ if mode not in DATABASE_ACCESS_MODES:
3777
+ raise RuntimeError(
3778
+ f"dbAccess must be one of {', '.join(DATABASE_ACCESS_MODES)}; got {mode!r}"
3779
+ )
3780
+ DATABASE_ACCESS_MODE = mode
3781
+ print(f"[runner] database access: {mode}")
3782
+ return mode
3783
+
3784
+
3785
+ def safe_get_db_access(runner_secrets, payload):
3786
+ """`dbAccess` rides the same precedence as apiLambdasInVpc: the
3787
+ runner-secrets document is the durable home, the controller payload can
3788
+ override per deploy. Runner-only — it never reaches the Terraform root."""
3789
+ value = safe_get(runner_secrets, "dbAccess", default=safe_get(payload, "dbAccess", default=""))
3790
+ value = value.strip().lower() or "psql"
3791
+ if value not in DATABASE_ACCESS_MODES:
3792
+ raise RuntimeError(
3793
+ f"dbAccess must be one of {', '.join(DATABASE_ACCESS_MODES)}; got {value!r}"
3794
+ )
3795
+ return value
3796
+
3797
+
3798
+ class DatabaseCommandError(subprocess.CalledProcessError):
3799
+ """A failed Data API statement. Subclasses CalledProcessError so every
3800
+ caller that catches psql failures handles it identically; `cmd` names the
3801
+ lane and the script, never a connection string or a secret."""
3802
+
3803
+ def __init__(self, message, label):
3804
+ super().__init__(1, ["rds-data", "execute-statement", label], output="", stderr=message)
3805
+ self.message = message
3806
+
3807
+ def __str__(self):
3808
+ return f"rds-data failed for {self.cmd[2]}: {self.message}"
3809
+
3810
+
3811
+ def _sql_segments(text):
3812
+ """Split SQL text into (kind, chunk) segments: 'code', 'squote',
3813
+ 'dquote', 'dollar', 'line_comment', 'block_comment' and 'meta' (a psql
3814
+ meta-command: an unquoted backslash starting a line, through the end of
3815
+ that line). Concatenating the chunks reproduces the input byte for byte."""
3816
+ segments = []
3817
+ i = 0
3818
+ n = len(text)
3819
+ code_start = 0
3820
+
3821
+ def flush_code(end):
3822
+ if end > code_start:
3823
+ segments.append(("code", text[code_start:end]))
3824
+
3825
+ while i < n:
3826
+ ch = text[i]
3827
+ if ch == "\\" and not text[text.rfind("\n", 0, i) + 1 : i].strip():
3828
+ flush_code(i)
3829
+ end = text.find("\n", i)
3830
+ end = n if end == -1 else end
3831
+ segments.append(("meta", text[i:end]))
3832
+ i = code_start = end
3833
+ continue
3834
+ if ch == "-" and text.startswith("--", i):
3835
+ flush_code(i)
3836
+ end = text.find("\n", i)
3837
+ end = n if end == -1 else end
3838
+ segments.append(("line_comment", text[i:end]))
3839
+ i = code_start = end
3840
+ continue
3841
+ if ch == "/" and text.startswith("/*", i):
3842
+ flush_code(i)
3843
+ depth = 0
3844
+ j = i
3845
+ while j < n:
3846
+ if text.startswith("/*", j):
3847
+ depth += 1
3848
+ j += 2
3849
+ elif text.startswith("*/", j):
3850
+ depth -= 1
3851
+ j += 2
3852
+ if depth == 0:
3853
+ break
3854
+ else:
3855
+ j += 1
3856
+ segments.append(("block_comment", text[i:j]))
3857
+ i = code_start = j
3858
+ continue
3859
+ if ch in "'\"" and PSQL_QUOTED_VARIABLE_RE.match(text, i) and _is_variable_colon(text, i):
3860
+ # psql's :'name' / :"name" — interpolation syntax, not a literal
3861
+ i = PSQL_QUOTED_VARIABLE_RE.match(text, i).end()
3862
+ continue
3863
+ if ch == "'":
3864
+ flush_code(i)
3865
+ escaped = (
3866
+ i > 0
3867
+ and text[i - 1] in "eE"
3868
+ and (i < 2 or not (text[i - 2].isalnum() or text[i - 2] == "_"))
3869
+ )
3870
+ j = i + 1
3871
+ while j < n:
3872
+ if escaped and text[j] == "\\":
3873
+ j += 2
3874
+ continue
3875
+ if text[j] == "'":
3876
+ if j + 1 < n and text[j + 1] == "'":
3877
+ j += 2
3878
+ continue
3879
+ j += 1
3880
+ break
3881
+ j += 1
3882
+ segments.append(("squote", text[i:j]))
3883
+ i = code_start = j
3884
+ continue
3885
+ if ch == '"':
3886
+ flush_code(i)
3887
+ j = i + 1
3888
+ while j < n:
3889
+ if text[j] == '"':
3890
+ if j + 1 < n and text[j + 1] == '"':
3891
+ j += 2
3892
+ continue
3893
+ j += 1
3894
+ break
3895
+ j += 1
3896
+ segments.append(("dquote", text[i:j]))
3897
+ i = code_start = j
3898
+ continue
3899
+ if ch == "$":
3900
+ match = DOLLAR_QUOTE_RE.match(text, i)
3901
+ if match and not (i > 0 and (text[i - 1].isalnum() or text[i - 1] == "_")):
3902
+ tag = match.group(0)
3903
+ flush_code(i)
3904
+ end = text.find(tag, i + len(tag))
3905
+ end = n if end == -1 else end + len(tag)
3906
+ segments.append(("dollar", text[i:end]))
3907
+ i = code_start = end
3908
+ continue
3909
+ i += 1
3910
+ flush_code(n)
3911
+ return segments
3912
+
3913
+
3914
+ DOLLAR_QUOTE_RE = re.compile(r"\$[A-Za-z_][A-Za-z0-9_]*\$|\$\$")
3915
+ PSQL_QUOTED_VARIABLE_RE = re.compile(r"'\w+'|\"\w+\"")
3916
+
3917
+
3918
+ def _is_variable_colon(text, i):
3919
+ """True when the quote at `i` follows a psql variable colon: a `:` that
3920
+ is not part of a `::` cast or of an identifier."""
3921
+ return (
3922
+ i > 0
3923
+ and text[i - 1] == ":"
3924
+ and (i < 2 or not (text[i - 2] == ":" or text[i - 2].isalnum() or text[i - 2] == "_"))
3925
+ )
3926
+
3927
+
3928
+ PSQL_META_RE = re.compile(r"^\s*\\(\w+)\s*(.*?)\s*$")
3929
+ PSQL_VARIABLE_RE = re.compile(r"(?<![:\w]):(?:'(\w+)'|\"(\w+)\"|(\w+))")
3930
+
3931
+
3932
+ def split_sql_script(text):
3933
+ """Split a psql script into ordered items: ('sql', statement) for each
3934
+ top-level `;`-terminated statement (drizzle `--> statement-breakpoint`
3935
+ markers are boundaries too) and ('meta', line) for each psql meta-command
3936
+ line (an unquoted backslash at the start of a line). Comment-only and
3937
+ blank chunks are dropped; comments inside a statement travel with it."""
3938
+ items = []
3939
+ buffer = []
3940
+ has_code = [False]
3941
+
3942
+ def emit():
3943
+ statement = "".join(buffer).strip()
3944
+ if has_code[0] and statement:
3945
+ items.append(("sql", statement))
3946
+ buffer.clear()
3947
+ has_code[0] = False
3948
+
3949
+ for kind, chunk in _sql_segments(text):
3950
+ if kind == "meta":
3951
+ items.append(("meta", chunk.strip()))
3952
+ continue
3953
+ if kind == "line_comment":
3954
+ if chunk.strip() == STATEMENT_BREAKPOINT:
3955
+ emit()
3956
+ else:
3957
+ buffer.append(chunk)
3958
+ continue
3959
+ if kind != "code":
3960
+ buffer.append(chunk)
3961
+ has_code[0] = True
3962
+ continue
3963
+ start = 0
3964
+ for i, ch in enumerate(chunk):
3965
+ if ch == ";":
3966
+ buffer.append(chunk[start:i])
3967
+ emit()
3968
+ start = i + 1
3969
+ elif not ch.isspace():
3970
+ has_code[0] = True
3971
+ buffer.append(chunk[start:])
3972
+ emit()
3973
+ return items
3974
+
3975
+
3976
+ def substitute_psql_variables(sql, variables):
3977
+ """psql `-v` interpolation for the variables given: `:'name'` becomes a
3978
+ SQL string literal, `:"name"` an identifier, bare `:name` the raw value.
3979
+ Only code is touched — never quoted text, comments or `::type` casts —
3980
+ and only names that were actually passed (psql leaves the rest alone)."""
3981
+ if not variables:
3982
+ return sql
3983
+
3984
+ def replace(match):
3985
+ literal, identifier, raw = match.groups()
3986
+ name = literal or identifier or raw
3987
+ if name not in variables:
3988
+ return match.group(0)
3989
+ value = str(variables[name])
3990
+ if literal:
3991
+ return "'" + value.replace("'", "''") + "'"
3992
+ if identifier:
3993
+ return '"' + value.replace('"', '""') + '"'
3994
+ return value
3995
+
3996
+ return "".join(
3997
+ PSQL_VARIABLE_RE.sub(replace, chunk) if kind == "code" else chunk
3998
+ for kind, chunk in _sql_segments(sql)
3999
+ )
4000
+
4001
+
4002
+ def _leading_keywords(statement, count=2):
4003
+ words = []
4004
+ for kind, chunk in _sql_segments(statement):
4005
+ if kind in ("line_comment", "block_comment"):
4006
+ continue
4007
+ if kind != "code":
4008
+ break
4009
+ words.extend(chunk.split())
4010
+ if len(words) >= count:
4011
+ break
4012
+ return tuple(word.upper().rstrip(";") for word in words[:count])
4013
+
4014
+
4015
+ def _statement_runs_outside_transaction(statement):
4016
+ """Postgres refuses these inside a transaction block; psql (autocommit)
4017
+ runs them standalone, so the Data API lane does too."""
4018
+ keywords = _leading_keywords(statement, 4)
4019
+ return "CONCURRENTLY" in keywords or keywords[:1] == ("VACUUM",)
4020
+
4021
+
4022
+ def _rds_data_cell_text(cell):
4023
+ """psql -tA rendering of one Data API field: NULL is empty, booleans are
4024
+ t/f, numbers print plainly, everything else is its string form."""
4025
+ if not isinstance(cell, dict) or cell.get("isNull"):
4026
+ return ""
4027
+ if "booleanValue" in cell:
4028
+ return "t" if cell["booleanValue"] else "f"
4029
+ for key in ("stringValue", "longValue", "doubleValue"):
4030
+ if key in cell:
4031
+ return str(cell[key])
4032
+ if "arrayValue" in cell:
4033
+ values = next(iter(cell["arrayValue"].values()), [])
4034
+ return "{" + ",".join(str(value) for value in values) + "}"
4035
+ if "blobValue" in cell:
4036
+ value = cell["blobValue"]
4037
+ return value.decode("utf-8", "replace") if isinstance(value, bytes) else str(value)
4038
+ return ""
4039
+
4040
+
4041
+ class AwsCliRdsDataClient:
4042
+ """boto3-shaped rds-data client backed by the AWS CLI, for a build image
4043
+ without boto3 (the runner already shells out to `aws` everywhere)."""
4044
+
4045
+ def __init__(self, region):
4046
+ self.region = region
4047
+
4048
+ def _call(self, operation, params):
4049
+ args = ["aws", "rds-data", operation, "--cli-input-json", json.dumps(params)]
4050
+ if self.region:
4051
+ args.extend(["--region", self.region])
4052
+ try:
4053
+ body = subprocess.check_output(args, text=True, stderr=subprocess.PIPE)
4054
+ except subprocess.CalledProcessError as error:
4055
+ message = (error.stderr or "").strip() or f"aws rds-data {operation} failed"
4056
+ raise RuntimeError(message) from None
4057
+ return json.loads(body or "{}")
4058
+
4059
+ def execute_statement(self, **params):
4060
+ return self._call("execute-statement", params)
4061
+
4062
+ def begin_transaction(self, **params):
4063
+ return self._call("begin-transaction", params)
4064
+
4065
+ def commit_transaction(self, **params):
4066
+ return self._call("commit-transaction", params)
4067
+
4068
+ def rollback_transaction(self, **params):
4069
+ return self._call("rollback-transaction", params)
4070
+
4071
+
4072
+ def rds_data_client(region):
4073
+ try:
4074
+ import boto3 # noqa: PLC0415 — optional on the build image
4075
+ except ImportError:
4076
+ return AwsCliRdsDataClient(region)
4077
+ return boto3.client("rds-data", region_name=region) if region else boto3.client("rds-data")
4078
+
4079
+
4080
+ def _cluster_arn_from_outputs(outputs):
4081
+ arn = (outputs.get("db_cluster_arn") or {}).get("value")
4082
+ if arn:
4083
+ return str(arn)
4084
+ endpoint = str((outputs.get("db_cluster_endpoint") or {}).get("value") or "")
4085
+ labels = endpoint.split(".")
4086
+ if len(labels) < 5 or labels[-2] != "amazonaws" or labels[-3] != "rds":
4087
+ raise RuntimeError(
4088
+ "Terraform outputs carry no db_cluster_arn and the endpoint "
4089
+ f"{endpoint!r} is not an RDS hostname; cannot address the RDS Data API"
4090
+ )
4091
+ region = labels[-4]
4092
+ account = output(
4093
+ ["aws", "sts", "get-caller-identity", "--query", "Account", "--output", "text"]
4094
+ )
4095
+ return f"arn:aws:rds:{region}:{account}:cluster:{labels[0]}"
4096
+
4097
+
4098
+ class DataApiDatabase:
4099
+ """The database handle in data-api mode: what `database_url_from_outputs`
4100
+ returns instead of a URL, and what `psql` / `psql_output` dispatch on."""
4101
+
4102
+ def __init__(self, cluster_arn, secret_arn, database_name, client=None, region=None):
4103
+ self.cluster_arn = cluster_arn
4104
+ self.secret_arn = secret_arn
4105
+ self.database_name = database_name
4106
+ self.region = region or os.environ.get("AWS_REGION") or cluster_arn.split(":")[3]
4107
+ self._client = client
4108
+
4109
+ @classmethod
4110
+ def from_outputs(cls, outputs, client=None):
4111
+ secret_arn = (outputs.get("db_secret_arn") or {}).get("value")
4112
+ if not secret_arn:
4113
+ raise RuntimeError("Terraform outputs missing database secret ARN")
4114
+ database_name = (outputs.get("database_name") or {}).get("value") or "thinkwork"
4115
+ return cls(_cluster_arn_from_outputs(outputs), str(secret_arn), str(database_name), client)
4116
+
4117
+ def __repr__(self):
4118
+ return f"DataApiDatabase({self.cluster_arn}, database={self.database_name})"
4119
+
4120
+ __str__ = __repr__
4121
+
4122
+ @property
4123
+ def client(self):
4124
+ if self._client is None:
4125
+ self._client = rds_data_client(self.region)
4126
+ return self._client
4127
+
4128
+ def _params(self, sql, transaction_id=None, continue_after_timeout=False):
4129
+ params = {
4130
+ "resourceArn": self.cluster_arn,
4131
+ "secretArn": self.secret_arn,
4132
+ "database": self.database_name,
4133
+ "sql": sql,
4134
+ }
4135
+ if transaction_id:
4136
+ params["transactionId"] = transaction_id
4137
+ if continue_after_timeout:
4138
+ params["continueAfterTimeout"] = True
4139
+ return params
4140
+
4141
+ def scalar(self, sql):
4142
+ """`psql -tAc`: every row on its own line, columns `|`-joined, NULL
4143
+ empty, booleans t/f, whole result stripped."""
4144
+ try:
4145
+ response = self.client.execute_statement(**self._params(sql))
4146
+ except Exception as error: # noqa: BLE001 — every SDK/CLI failure lands here
4147
+ raise DatabaseCommandError(str(error), "query") from None
4148
+ rows = [
4149
+ "|".join(_rds_data_cell_text(cell) for cell in row)
4150
+ for row in response.get("records") or []
4151
+ ]
4152
+ return "\n".join(rows).strip()
4153
+
4154
+ def execute(self, sql=None, file=None, variables=None):
4155
+ """`psql -v ON_ERROR_STOP=1 [-v k=v] -f file | -c sql`."""
4156
+ if file is not None:
4157
+ label = Path(file).name
4158
+ text = Path(file).read_text(encoding="utf-8")
4159
+ else:
4160
+ label = "inline sql"
4161
+ text = sql or ""
4162
+ variables = {str(key): str(value) for key, value in (variables or {}).items()}
4163
+ statements = self._plan(text, label, variables)
4164
+ self._run(statements, label, variables)
4165
+
4166
+ def _plan(self, text, label, variables):
4167
+ """Interpret meta-commands in order and resolve the statements that
4168
+ will run; refuses the script before anything runs when it needs a
4169
+ meta-command this lane does not implement or a statement the Data
4170
+ API cannot carry."""
4171
+ planned = []
4172
+ conditions = []
4173
+ for kind, item in split_sql_script(text):
4174
+ active = all(conditions)
4175
+ if kind == "meta":
4176
+ match = PSQL_META_RE.match(item)
4177
+ command, argument = (match.group(1), match.group(2)) if match else (item, "")
4178
+ if command == "if":
4179
+ defined = re.fullmatch(r":\{\?(\w+)\}", argument)
4180
+ if not defined:
4181
+ raise DatabaseCommandError(
4182
+ f"unsupported psql meta-command in Data API mode: {item}", label
4183
+ )
4184
+ conditions.append(defined.group(1) in variables)
4185
+ elif command == "else":
4186
+ if not conditions:
4187
+ raise DatabaseCommandError(f"\\else without \\if in {label}", label)
4188
+ conditions[-1] = not conditions[-1]
4189
+ elif command == "endif":
4190
+ if not conditions:
4191
+ raise DatabaseCommandError(f"\\endif without \\if in {label}", label)
4192
+ conditions.pop()
4193
+ elif not active:
4194
+ continue
4195
+ elif command == "set":
4196
+ parts = argument.split(None, 1)
4197
+ if parts:
4198
+ value = parts[1].strip() if len(parts) > 1 else ""
4199
+ if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"":
4200
+ value = value[1:-1]
4201
+ variables[parts[0]] = value
4202
+ elif command in ("echo", "qecho"):
4203
+ message = argument.strip("'")
4204
+ print(f"[psql:{label}] {message}")
4205
+ elif command in ("quit", "q"):
4206
+ # psql: \quit in a script ends that script, exit status 0
4207
+ # (it takes no argument; `\quit 1` still exits 0).
4208
+ print(f"[psql:{label}] \\quit — remaining statements skipped")
4209
+ conditions.clear()
4210
+ break
4211
+ else:
4212
+ raise DatabaseCommandError(
4213
+ f"unsupported psql meta-command in Data API mode: {item}", label
4214
+ )
4215
+ continue
4216
+ if not active:
4217
+ continue
4218
+ statement = substitute_psql_variables(item, variables)
4219
+ if len(
4220
+ statement.encode("utf-8")
4221
+ ) > RDS_DATA_API_MAX_SQL_BYTES and _statement_runs_outside_transaction(statement):
4222
+ raise DatabaseCommandError(
4223
+ f"statement {len(planned) + 1} exceeds the RDS Data API's "
4224
+ f"{RDS_DATA_API_MAX_SQL_BYTES}-byte limit and cannot run inside a "
4225
+ "transaction; split it at `;` boundaries",
4226
+ label,
4227
+ )
4228
+ planned.append(statement)
4229
+ if conditions:
4230
+ raise DatabaseCommandError(f"\\if without \\endif in {label}", label)
4231
+ return planned
4232
+
4233
+ def _run(self, statements, label, variables):
4234
+ transaction_id = None
4235
+ secrets_to_redact = [value for value in variables.values() if len(value) >= 8]
4236
+
4237
+ def redact(message):
4238
+ for secret in secrets_to_redact:
4239
+ message = message.replace(secret, "***")
4240
+ return message
4241
+
4242
+ def begin():
4243
+ return self.client.begin_transaction(
4244
+ resourceArn=self.cluster_arn, secretArn=self.secret_arn, database=self.database_name
4245
+ )["transactionId"]
4246
+
4247
+ try:
4248
+ for index, statement in enumerate(statements, start=1):
4249
+ head = _leading_keywords(statement)
4250
+ if head[:1] in {("BEGIN",), ("START",)}:
4251
+ if transaction_id is None:
4252
+ transaction_id = begin()
4253
+ continue
4254
+ if head[:1] in {("COMMIT",), ("END",)}:
4255
+ if transaction_id is not None:
4256
+ self.client.commit_transaction(
4257
+ resourceArn=self.cluster_arn,
4258
+ secretArn=self.secret_arn,
4259
+ transactionId=transaction_id,
4260
+ )
4261
+ transaction_id = None
4262
+ continue
4263
+ if head[:1] == ("ROLLBACK",):
4264
+ if transaction_id is not None:
4265
+ self.client.rollback_transaction(
4266
+ resourceArn=self.cluster_arn,
4267
+ secretArn=self.secret_arn,
4268
+ transactionId=transaction_id,
4269
+ )
4270
+ transaction_id = None
4271
+ continue
4272
+ if _statement_runs_outside_transaction(statement):
4273
+ if transaction_id is not None:
4274
+ self.client.commit_transaction(
4275
+ resourceArn=self.cluster_arn,
4276
+ secretArn=self.secret_arn,
4277
+ transactionId=transaction_id,
4278
+ )
4279
+ transaction_id = None
4280
+ self._execute_one(statement, None, index, label, redact)
4281
+ continue
4282
+ if transaction_id is None:
4283
+ transaction_id = begin()
4284
+ self._execute_one(statement, transaction_id, index, label, redact)
4285
+ if transaction_id is not None:
4286
+ self.client.commit_transaction(
4287
+ resourceArn=self.cluster_arn,
4288
+ secretArn=self.secret_arn,
4289
+ transactionId=transaction_id,
4290
+ )
4291
+ except BaseException:
4292
+ if transaction_id is not None:
4293
+ try:
4294
+ self.client.rollback_transaction(
4295
+ resourceArn=self.cluster_arn,
4296
+ secretArn=self.secret_arn,
4297
+ transactionId=transaction_id,
4298
+ )
4299
+ except Exception: # noqa: BLE001 — rollback is best-effort
4300
+ pass
4301
+ raise
4302
+
4303
+ def _execute_one(self, statement, transaction_id, index, label, redact):
4304
+ if len(statement.encode("utf-8")) > RDS_DATA_API_MAX_SQL_BYTES:
4305
+ self._execute_oversized(statement, transaction_id, index, label, redact)
4306
+ return
4307
+ try:
4308
+ self.client.execute_statement(
4309
+ **self._params(statement, transaction_id, continue_after_timeout=True)
4310
+ )
4311
+ except Exception as error: # noqa: BLE001 — every SDK/CLI failure lands here
4312
+ first_line = statement.strip().splitlines()[0][:120]
4313
+ raise DatabaseCommandError(
4314
+ redact(f"statement {index} ({first_line}): {error}"), label
4315
+ ) from None
4316
+
4317
+ def _execute_oversized(self, statement, transaction_id, index, label, redact):
4318
+ """A statement over the Data API's per-call limit (0230_analyst_rls
4319
+ carries an 85 KB DO block) is shipped in pieces into a temp table on
4320
+ the transaction's connection, reassembled and run there with EXECUTE,
4321
+ then the temp table is dropped. Same statement, same transaction."""
4322
+ if transaction_id is None:
4323
+ raise DatabaseCommandError(
4324
+ f"statement {index} exceeds {RDS_DATA_API_MAX_SQL_BYTES} bytes and "
4325
+ "cannot run inside a transaction",
4326
+ label,
4327
+ )
4328
+ tag = "$chunk$"
4329
+ suffix = 0
4330
+ while tag in statement:
4331
+ suffix += 1
4332
+ tag = f"$chunk{suffix}$"
4333
+ table = "pg_temp.runner_oversized_statement"
4334
+ pieces = []
4335
+ current = []
4336
+ size = 0
4337
+ for char in statement:
4338
+ width = len(char.encode("utf-8"))
4339
+ if size + width > RDS_DATA_API_MAX_SQL_BYTES - 4096:
4340
+ pieces.append("".join(current))
4341
+ current, size = [], 0
4342
+ current.append(char)
4343
+ size += width
4344
+ if current:
4345
+ pieces.append("".join(current))
4346
+ steps = [f"CREATE TEMP TABLE {table.split('.')[1]} (n integer PRIMARY KEY, chunk text)"]
4347
+ steps.extend(
4348
+ f"INSERT INTO {table} (n, chunk) VALUES ({number}, {tag}{piece}{tag})"
4349
+ for number, piece in enumerate(pieces)
4350
+ )
4351
+ steps.append(
4352
+ "DO $runner$ BEGIN EXECUTE (SELECT string_agg(chunk, '' ORDER BY n) "
4353
+ f"FROM {table}); END $runner$"
4354
+ )
4355
+ steps.append(f"DROP TABLE {table}")
4356
+ print(
4357
+ f"[psql:{label}] statement {index} is {len(statement.encode('utf-8'))} bytes; "
4358
+ f"shipping it to the server in {len(pieces)} pieces"
4359
+ )
4360
+ for step in steps:
4361
+ try:
4362
+ self.client.execute_statement(
4363
+ **self._params(step, transaction_id, continue_after_timeout=True)
4364
+ )
4365
+ except Exception as error: # noqa: BLE001 — every SDK/CLI failure lands here
4366
+ first_line = statement.strip().splitlines()[0][:120]
4367
+ raise DatabaseCommandError(
4368
+ redact(f"statement {index} ({first_line}): {error}"), label
4369
+ ) from None
2255
4370
 
2256
4371
 
2257
4372
  def put_secret_value(secret_id, payload):
@@ -2298,9 +4413,24 @@ def ensure_compliance_roles(database_url, outputs, vars_json):
2298
4413
  )
2299
4414
 
2300
4415
 
2301
- def migration_files():
4416
+ def migration_files(vars_json=None):
2302
4417
  migrations = SOURCE / "packages/database-pg/drizzle"
2303
- return sorted(path for path in migrations.glob("*.sql") if "rollback" not in path.name)
4418
+ allow_auth_retirement = bool(
4419
+ isinstance(vars_json, dict)
4420
+ and vars_json.get("auth_retirement_phase") == "retired"
4421
+ and vars_json.get("finalize_auth_retirement") is True
4422
+ )
4423
+ paths = []
4424
+ for path in migrations.glob("*.sql"):
4425
+ if "rollback" in path.name:
4426
+ continue
4427
+ if (
4428
+ AUTH_RETIREMENT_MIGRATION_MARKER in path.read_text(encoding="utf-8")
4429
+ and not allow_auth_retirement
4430
+ ):
4431
+ continue
4432
+ paths.append(path)
4433
+ return sorted(paths)
2304
4434
 
2305
4435
 
2306
4436
  def apply_migration_file(database_url, outputs, vars_json, path):
@@ -2334,7 +4464,7 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto;
2334
4464
  CREATE EXTENSION IF NOT EXISTS vector;
2335
4465
  """,
2336
4466
  )
2337
- for path in migration_files():
4467
+ for path in migration_files(vars_json):
2338
4468
  apply_migration_file(database_url, outputs, vars_json, path)
2339
4469
 
2340
4470
 
@@ -2454,7 +4584,7 @@ def platform_migration_object_present(database_url, kind, name):
2454
4584
  return None
2455
4585
 
2456
4586
 
2457
- def backfill_platform_migration_ledger(database_url):
4587
+ def backfill_platform_migration_ledger(database_url, vars_json):
2458
4588
  """One-time ledger bootstrap for environments installed before the ledger
2459
4589
  existed. Every file shipping with this release is recorded as assumed
2460
4590
  applied — auto-re-running old files is unsafe (markers can name objects
@@ -2463,11 +4593,29 @@ def backfill_platform_migration_ledger(database_url):
2463
4593
  transition apply through the exact pending path."""
2464
4594
  verified = []
2465
4595
  assumed = []
2466
- for path in migration_files():
4596
+ for path in migration_files(vars_json):
4597
+ match = re.match(r"^(\d{4})_", path.name)
4598
+ migration_number = int(match.group(1)) if match else None
2467
4599
  verdicts = [
2468
4600
  platform_migration_object_present(database_url, kind, name)
2469
4601
  for kind, name in declared_migration_objects(path)
2470
4602
  ]
4603
+ # The ledger ships with the native-auth release. Files through 0260 may
4604
+ # already have been applied by the old deploy paths and cannot safely be
4605
+ # replayed. Files introduced with this release must never be silently
4606
+ # assumed: record them only when every declared object is present;
4607
+ # otherwise leave them pending so apply_pending_platform_migrations runs
4608
+ # them before the app is published.
4609
+ if (
4610
+ migration_number is not None
4611
+ and migration_number > PLATFORM_MIGRATION_LEDGER_CUTOFF
4612
+ and (not verdicts or any(verdict is not True for verdict in verdicts))
4613
+ ):
4614
+ print(
4615
+ f"[migrations] transition: {path.name} is newer than the pre-ledger cutoff "
4616
+ "and remains pending"
4617
+ )
4618
+ continue
2471
4619
  if any(verdict is False for verdict in verdicts):
2472
4620
  print(
2473
4621
  f"[migrations] transition WARNING: {path.name} declares objects missing from "
@@ -2484,7 +4632,7 @@ def backfill_platform_migration_ledger(database_url):
2484
4632
 
2485
4633
  def apply_pending_platform_migrations(database_url, outputs, vars_json):
2486
4634
  recorded = recorded_platform_migrations(database_url)
2487
- for path in migration_files():
4635
+ for path in migration_files(vars_json):
2488
4636
  if path.name in recorded:
2489
4637
  continue
2490
4638
  print(f"[migrations] applying {path.name}")
@@ -2923,31 +5071,6 @@ def pg_literal(value):
2923
5071
  return "'" + value.replace("'", "''") + "'"
2924
5072
 
2925
5073
 
2926
- def ensure_hindsight_database(database_url, vars_json):
2927
- """THINK-220: the dedicated Hindsight database is bootstrap SQL — Terraform
2928
- cannot CREATE DATABASE inside the cluster. Runs post-apply; the Hindsight
2929
- service may crash-loop briefly on a first enablement until this creates
2930
- the database, then ECS stabilizes on its own. Vanilla boot migrations
2931
- (RUN_MIGRATIONS_ON_STARTUP) build the schema, including the maintenance
2932
- discovery functions the repair migration installs on base-schema runs."""
2933
- db_name = (vars_json.get("hindsight_database_name") or "").strip()
2934
- if not db_name:
2935
- return
2936
- if not re.fullmatch(r"[a-z][a-z0-9_]*", db_name):
2937
- raise RuntimeError(f"invalid hindsight_database_name: {db_name!r}")
2938
- admin_url = re.sub(r"/[^/?]+(\?|$)", r"/postgres\1", database_url, count=1)
2939
- exists = psql_output(
2940
- admin_url,
2941
- f"SELECT 1 FROM pg_database WHERE datname = '{db_name}'",
2942
- ).strip()
2943
- if not exists:
2944
- print(f"[hindsight] creating database {db_name}")
2945
- psql(admin_url, sql=f'CREATE DATABASE "{db_name}"')
2946
- hindsight_url = re.sub(r"/[^/?]+(\?|$)", f"/{db_name}\\1", database_url, count=1)
2947
- psql(hindsight_url, sql="CREATE EXTENSION IF NOT EXISTS vector;")
2948
- print(f"[hindsight] database {db_name} ready (pgvector ensured)")
2949
-
2950
-
2951
5074
  def push_database_schema(outputs_path, vars_json):
2952
5075
  outputs = json.loads(outputs_path.read_text(encoding="utf-8"))
2953
5076
  checkout_source(
@@ -2956,20 +5079,21 @@ def push_database_schema(outputs_path, vars_json):
2956
5079
  os.environ.get("THINKWORK_RELEASE_VERSION", "main"),
2957
5080
  )
2958
5081
  database_url = database_url_from_outputs(outputs)
2959
- ensure_hindsight_database(database_url, vars_json)
2960
- fresh = not psql_output(database_url, "SELECT to_regclass('public.tenants')").strip()
5082
+ fresh = not psql_output(database_url, "SELECT to_regclass('public.tenants')::text").strip()
2961
5083
  ledger_present = bool(
2962
- psql_output(database_url, "SELECT to_regclass('public.platform_schema_migrations')").strip()
5084
+ psql_output(
5085
+ database_url, "SELECT to_regclass('public.platform_schema_migrations')::text"
5086
+ ).strip()
2963
5087
  )
2964
5088
  if fresh:
2965
5089
  initialize_greenfield_database(database_url, outputs, vars_json)
2966
5090
  ensure_migration_ledger(database_url)
2967
5091
  if fresh:
2968
5092
  record_platform_migrations(
2969
- database_url, [path.name for path in migration_files()], "greenfield"
5093
+ database_url, [path.name for path in migration_files(vars_json)], "greenfield"
2970
5094
  )
2971
5095
  elif not ledger_present:
2972
- backfill_platform_migration_ledger(database_url)
5096
+ backfill_platform_migration_ledger(database_url, vars_json)
2973
5097
  apply_pending_platform_migrations(database_url, outputs, vars_json)
2974
5098
  seed_platform_bootstrap_defaults(database_url)
2975
5099
  migrations = SOURCE / "packages/database-pg/drizzle"
@@ -2980,6 +5104,55 @@ def push_database_schema(outputs_path, vars_json):
2980
5104
  apply_migration_file(database_url, outputs, vars_json, path)
2981
5105
 
2982
5106
 
5107
+ def prepare_additive_schema_before_app(payload, vars_json):
5108
+ """Materialize foundation/data dependencies and apply additive schema before
5109
+ the full app plan can publish Lambdas that consume it. Retirement stays
5110
+ disabled here and, when explicitly requested, runs only after the app
5111
+ retirement phase has deployed."""
5112
+ action = str(payload.get("action") or "").lower()
5113
+ if (
5114
+ action not in {"deploy", "update"}
5115
+ or is_managed_app_operation(payload)
5116
+ or identity_provider_operation(payload) is not None
5117
+ ):
5118
+ return None
5119
+ target = "module.thinkwork.module.database"
5120
+ plan = subprocess.run(
5121
+ [
5122
+ "terraform",
5123
+ "plan",
5124
+ f"-target={target}",
5125
+ "-out=tfplan-data",
5126
+ "-no-color",
5127
+ ],
5128
+ cwd=TF,
5129
+ text=True,
5130
+ )
5131
+ if plan.returncode != 0:
5132
+ return plan
5133
+ applied = subprocess.run(
5134
+ ["terraform", "apply", "-auto-approve", "-no-color", "tfplan-data"],
5135
+ cwd=TF,
5136
+ text=True,
5137
+ )
5138
+ if applied.returncode != 0:
5139
+ return applied
5140
+ outputs_path = TF / "pre-app-outputs.json"
5141
+ outputs_path.write_text(
5142
+ output(["terraform", "output", "-json"], cwd=TF),
5143
+ encoding="utf-8",
5144
+ )
5145
+ additive_vars = dict(vars_json)
5146
+ additive_vars["finalize_auth_retirement"] = False
5147
+ push_database_schema(outputs_path, additive_vars)
5148
+ TERRAFORM_EVIDENCE["schemaOrdering"] = {
5149
+ "status": "succeeded",
5150
+ "target": target,
5151
+ "phase": "foundation-data-additive-before-app",
5152
+ }
5153
+ return applied
5154
+
5155
+
2983
5156
  def write_runner_files(payload, runner_secrets):
2984
5157
  preserved_config = payload.get("preservedConfig")
2985
5158
  if not isinstance(preserved_config, dict):
@@ -3065,43 +5238,180 @@ def write_runner_files(payload, runner_secrets):
3065
5238
  if not api_auth_secret:
3066
5239
  api_auth_secret = secrets.token_urlsafe(48)
3067
5240
 
5241
+ auth_state = auth_reconciliation_state(stage)
5242
+ tenant_auth_metadata = identity_provider_desired_connections(
5243
+ payload,
5244
+ stage=stage,
5245
+ account_id=account_id,
5246
+ region=region,
5247
+ previous_state=auth_state,
5248
+ current_outputs=current_outputs,
5249
+ )
5250
+ prior_phase_output = current_outputs.get("auth_retirement_phase")
5251
+ prior_phase = prior_phase_output.get("value") if isinstance(prior_phase_output, dict) else None
5252
+ if prior_phase not in {"coexistence", "cutover", "retired"}:
5253
+ # An existing pre-native-auth stack must enter through coexistence.
5254
+ # A truly empty state is greenfield and has no WorkOS runtime to keep.
5255
+ prior_phase = "coexistence" if current_state else "retired"
5256
+ auth_retirement_phase = safe_get(
5257
+ runner_secrets,
5258
+ "authRetirementPhase",
5259
+ default=safe_get(
5260
+ reviewed_payload,
5261
+ "authRetirementPhase",
5262
+ default=prior_phase,
5263
+ ),
5264
+ )
5265
+ if auth_retirement_phase not in {"coexistence", "cutover", "retired"}:
5266
+ raise RuntimeError("authRetirementPhase must be coexistence, cutover, or retired")
5267
+ auth_migration_recovery_deadline = safe_get(
5268
+ runner_secrets,
5269
+ "authMigrationRecoveryDeadline",
5270
+ default=safe_get(
5271
+ reviewed_payload,
5272
+ "authMigrationRecoveryDeadline",
5273
+ default=os.environ.get("AUTH_MIGRATION_RECOVERY_DEADLINE", ""),
5274
+ ),
5275
+ ).strip()
5276
+ if auth_migration_recovery_deadline:
5277
+ try:
5278
+ parsed_recovery_deadline = datetime.fromisoformat(
5279
+ auth_migration_recovery_deadline.replace("Z", "+00:00")
5280
+ )
5281
+ if parsed_recovery_deadline.tzinfo is None:
5282
+ raise ValueError("timezone required")
5283
+ except ValueError as exc:
5284
+ raise RuntimeError(
5285
+ "authMigrationRecoveryDeadline must be an RFC3339 timestamp"
5286
+ ) from exc
5287
+ if auth_retirement_phase == "coexistence" and not auth_migration_recovery_deadline:
5288
+ raise RuntimeError(
5289
+ "authMigrationRecoveryDeadline is required while authRetirementPhase is coexistence"
5290
+ )
5291
+ finalize_auth_retirement = payload.get("finalizeAuthRetirement") is True
3068
5292
  vars_json = {
3069
5293
  "stage": stage,
3070
5294
  "region": region,
3071
5295
  "account_id": account_id,
3072
5296
  "db_password": db_password,
3073
5297
  "api_auth_secret": api_auth_secret,
5298
+ "auth_retirement_phase": auth_retirement_phase,
5299
+ "auth_migration_recovery_deadline": auth_migration_recovery_deadline,
5300
+ "finalize_auth_retirement": finalize_auth_retirement,
3074
5301
  "database_engine": safe_get(
3075
5302
  reviewed_payload,
3076
5303
  "databaseEngine",
3077
5304
  default="aurora-serverless",
3078
5305
  ),
3079
- "enable_hindsight": safe_get_bool(
3080
- {},
5306
+ # External S3 KB source R20: customer-owned buckets granted to the
5307
+ # KB service role for s3-connect sources. Per-environment infra
5308
+ # posture — durable home is the runner-secrets document; a payload
5309
+ # list can override per-deploy. Accepts a JSON list or a
5310
+ # comma-separated string (Secrets Manager values often arrive as
5311
+ # strings).
5312
+ "external_kb_source_arns": safe_get_list(
5313
+ runner_secrets,
3081
5314
  reviewed_payload,
3082
- "enableHindsight",
5315
+ "externalKbSourceArns",
5316
+ ),
5317
+ # VPC endpoint diet (cost review 2026-08-28): which AWS services get
5318
+ # a private interface endpoint, and one ENI or one per AZ. The
5319
+ # private subnets are NAT-routed, so this is network posture, not
5320
+ # reachability. Environment-level — durable home is the
5321
+ # runner-secrets document; the payload can override per-deploy.
5322
+ # Absent = the module's legacy full set across every AZ; the literal
5323
+ # "none" = no interface endpoints at all.
5324
+ "vpc_interface_endpoint_services": safe_get_list(
5325
+ runner_secrets,
5326
+ reviewed_payload,
5327
+ "vpcInterfaceEndpointServices",
5328
+ ),
5329
+ "vpc_interface_endpoint_single_az": safe_get_bool(
5330
+ runner_secrets,
5331
+ reviewed_payload,
5332
+ "vpcInterfaceEndpointSingleAz",
5333
+ default=False,
5334
+ ),
5335
+ # AgentCore Harness is an environment-level deployment choice. The
5336
+ # controller carries these reviewed, non-secret values forward so a
5337
+ # release update cannot silently remove the managed runtime plane.
5338
+ # The registry module retains its rollout-era multiplayer-proof input
5339
+ # names for compatibility; all provisioning still happens in the
5340
+ # normal Terraform apply owned by this controller.
5341
+ # THINK-1082: run the deployment runner inside the VPC so the psql
5342
+ # schema step reaches Aurora by security group instead of the
5343
+ # world-open 0.0.0.0/0 -> 5432 rule (turns on the VPC NAT gateway).
5344
+ # Environment-level network posture — durable home is the
5345
+ # runner-secrets document (deploymentRunnerInVpc); the controller
5346
+ # payload can override per-deploy. Ship-inert default.
5347
+ "deployment_runner_in_vpc": safe_get_bool(
5348
+ runner_secrets,
5349
+ reviewed_payload,
5350
+ "deploymentRunnerInVpc",
5351
+ default=False,
5352
+ ),
5353
+ # THINK-1082: every database-connected Lambda (api handlers,
5354
+ # agentcore-admin, Pi, broker) inside the VPC behind the api-lambda
5355
+ # security group, granted 5432 on the DB group — so the world-open
5356
+ # 0.0.0.0/0 -> 5432 rule can close without taking the app down.
5357
+ # Same homes as deploymentRunnerInVpc: runner-secrets document is
5358
+ # durable, payload overrides per-deploy. Ship-inert default.
5359
+ "api_lambdas_in_vpc": safe_get_bool(
5360
+ runner_secrets,
5361
+ reviewed_payload,
5362
+ "apiLambdasInVpc",
3083
5363
  default=False,
3084
5364
  ),
3085
- # Analyst data-path Lambda VPC egress (stable NAT EIP for external
3086
- # database IP allowlists). Per-environment infra posture, so the
3087
- # durable home is the runner-secrets document; a payload boolean
3088
- # can override per-deploy.
3089
- "analyst_lambda_vpc_egress": safe_get_bool(
5365
+ # THINK-1082: database exposure. Defaults reproduce today's posture
5366
+ # (public endpoint + world-open 5432 seeded on a new DB security
5367
+ # group). Closed per stage only after both *InVpc flags are live.
5368
+ # dbPublicIngressCidrs accepts a JSON list or a comma-separated
5369
+ # string; an explicit empty string / [] means "seed no rule".
5370
+ "db_publicly_accessible": safe_get_bool(
5371
+ runner_secrets,
5372
+ reviewed_payload,
5373
+ "dbPubliclyAccessible",
5374
+ default=True,
5375
+ ),
5376
+ "db_public_ingress_cidrs": safe_get_ingress_cidrs(
3090
5377
  runner_secrets,
3091
5378
  reviewed_payload,
3092
- "analystLambdaVpcEgress",
5379
+ "dbPublicIngressCidrs",
5380
+ default=["0.0.0.0/0"],
5381
+ ),
5382
+ # THINK-1082: how this runner reaches the database for its schema
5383
+ # steps — `psql` (public endpoint, today) or `data-api` (RDS Data API:
5384
+ # HTTPS + IAM, no network path, so the runner can stay outside the
5385
+ # VPC while the public 5432 rule closes). Runner-only: it never
5386
+ # reaches the generated Terraform root (see RUNNER_ONLY_VARS).
5387
+ "db_access": safe_get_db_access(runner_secrets, reviewed_payload),
5388
+ "enable_agentcore_multiplayer_proof": safe_get_bool(
5389
+ {},
5390
+ reviewed_payload,
5391
+ "enableAgentCoreHarness",
3093
5392
  default=False,
3094
5393
  ),
3095
- # THINK-220: dedicated Hindsight database on the stage cluster.
3096
- # Empty keeps the legacy hindsight-schema layout; set (e.g.
3097
- # "thinkwork_hindsight") points the Hindsight service and platform
3098
- # readers at that database's public schema, where upstream's
3099
- # maintenance discovery actually runs.
3100
- "hindsight_database_name": safe_get(
5394
+ "agentcore_multiplayer_proof_tenant_slug": safe_get(
3101
5395
  reviewed_payload,
3102
- "hindsightDatabaseName",
5396
+ "agentCoreHarnessTenantSlug",
3103
5397
  default="",
3104
5398
  ),
5399
+ "agentcore_multiplayer_proof_owner_allowlist": safe_get(
5400
+ reviewed_payload,
5401
+ "agentCoreHarnessOwnerAllowlist",
5402
+ default="",
5403
+ ),
5404
+ # THINK-585/587 — stage kill-switch for AgentCore Runtime chat
5405
+ # dispatch. Environment posture: durable home is the runner-secrets
5406
+ # document (agentcoreRuntimeDispatchEnabled); a reviewed payload can
5407
+ # override per-deploy. Dark by default — flipping a customer stage is
5408
+ # an explicit operator action.
5409
+ "agentcore_runtime_dispatch_enabled": safe_get_bool(
5410
+ runner_secrets,
5411
+ reviewed_payload,
5412
+ "agentcoreRuntimeDispatchEnabled",
5413
+ default=False,
5414
+ ),
3105
5415
  "platform_operator_emails": safe_get(
3106
5416
  runner_secrets,
3107
5417
  "adminEmail",
@@ -3131,6 +5441,38 @@ def write_runner_files(payload, runner_secrets):
3131
5441
  default="",
3132
5442
  ),
3133
5443
  ),
5444
+ "microsoft_oauth_client_id": safe_get(
5445
+ runner_secrets,
5446
+ "microsoftOauthClientId",
5447
+ default=safe_get(
5448
+ reviewed_payload,
5449
+ "microsoftOauthClientId",
5450
+ default="",
5451
+ ),
5452
+ ),
5453
+ "microsoft_oauth_client_secret": safe_get(
5454
+ runner_secrets,
5455
+ "microsoftOauthClientSecret",
5456
+ default=safe_get(
5457
+ reviewed_payload,
5458
+ "microsoftOauthClientSecret",
5459
+ default="",
5460
+ ),
5461
+ ),
5462
+ "microsoft_oauth_tenant": safe_get(
5463
+ runner_secrets,
5464
+ "microsoftOauthTenant",
5465
+ default=safe_get(
5466
+ reviewed_payload,
5467
+ "microsoftOauthTenant",
5468
+ default="",
5469
+ ),
5470
+ ),
5471
+ # Secret-free desired metadata. The compact Terraform projection only
5472
+ # creates provider-specific public app clients; the runner reconciles
5473
+ # the secret-bearing Cognito IdP before Terraform plans those clients.
5474
+ "tenant_entra_connections": tenant_entra_terraform_projection(tenant_auth_metadata),
5475
+ "auth_tenant_connection_metadata": tenant_auth_metadata,
3134
5476
  "cognito_email_source_arn": safe_get(
3135
5477
  runner_secrets,
3136
5478
  "cognitoEmailSourceArn",
@@ -3185,6 +5527,20 @@ def write_runner_files(payload, runner_secrets):
3185
5527
  "customerDomainLegacyRetired",
3186
5528
  default=False,
3187
5529
  ),
5530
+ # Extra web-family OAuth callback/logout origins (e.g. the customer's
5531
+ # Company Brain console CloudFront domain, hosted in their own account).
5532
+ # Persisted in runner-secrets so they survive every controller apply
5533
+ # instead of being stripped back to the computed defaults.
5534
+ "additional_admin_callback_urls": runner_secret_string_list(
5535
+ runner_secrets,
5536
+ reviewed_payload,
5537
+ "adminCallbackUrls",
5538
+ ),
5539
+ "additional_admin_logout_urls": runner_secret_string_list(
5540
+ runner_secrets,
5541
+ reviewed_payload,
5542
+ "adminLogoutUrls",
5543
+ ),
3188
5544
  "lambda_artifact_bucket": os.environ["THINKWORK_RELEASE_ARTIFACT_BUCKET"],
3189
5545
  "lambda_artifact_prefix": f"releases/{release_version}/lambdas",
3190
5546
  "deployment_release_version": release_version,
@@ -3201,19 +5557,89 @@ def write_runner_files(payload, runner_secrets):
3201
5557
  "deployment_terraform_module_source": terraform_module_source,
3202
5558
  "deployment_terraform_module_version": terraform_module_version,
3203
5559
  "agentcore_pi_source_image_uri": resolve_agentcore_pi_source_image_uri(payload),
3204
- }
3205
- enforce_customer_domain_preservation(
3206
- current_outputs,
3207
- vars_json,
3208
- payload,
3209
- runner_secrets,
3210
- )
3211
- vars_json.update(
3212
- managed_app_terraform_overrides(payload, stage, account_id, current_outputs, current_state)
3213
- )
3214
-
3215
- TF.mkdir(parents=True, exist_ok=True)
3216
- (TF / "backend.hcl").write_text(
5560
+ # Account-level Lambda memory ceiling. Legacy AWS accounts cap
5561
+ # function memory at 3008 MB (raised only via support case); set the
5562
+ # runner-secrets key lambdaMaxMemoryMb on such accounts so handlers
5563
+ # that request more are clamped instead of failing mid-apply.
5564
+ "lambda_max_memory_mb": safe_get(
5565
+ runner_secrets,
5566
+ "lambdaMaxMemoryMb",
5567
+ default=safe_get(reviewed_payload, "lambdaMaxMemoryMb", default=10240),
5568
+ ),
5569
+ # Digital Twin Neptune wiring (THINK-334): values come from the
5570
+ # etl-platform neptune stack outputs, written into the runner-secrets
5571
+ # document by `thinkwork twin install`. Empty = twin disabled.
5572
+ "neptune_endpoint": safe_get(
5573
+ runner_secrets,
5574
+ "neptuneEndpoint",
5575
+ default=safe_get(reviewed_payload, "neptuneEndpoint", default=""),
5576
+ ),
5577
+ "neptune_cluster_resource_id": safe_get(
5578
+ runner_secrets,
5579
+ "neptuneClusterResourceId",
5580
+ default=safe_get(reviewed_payload, "neptuneClusterResourceId", default=""),
5581
+ ),
5582
+ "neptune_client_security_group_id": safe_get(
5583
+ runner_secrets,
5584
+ "neptuneClientSecurityGroupId",
5585
+ default=safe_get(reviewed_payload, "neptuneClientSecurityGroupId", default=""),
5586
+ ),
5587
+ "neptune_load_bucket": safe_get(
5588
+ runner_secrets,
5589
+ "neptuneLoadBucket",
5590
+ default=safe_get(reviewed_payload, "neptuneLoadBucket", default=""),
5591
+ ),
5592
+ "neptune_loader_role_arn": safe_get(
5593
+ runner_secrets,
5594
+ "neptuneLoaderRoleArn",
5595
+ default=safe_get(reviewed_payload, "neptuneLoaderRoleArn", default=""),
5596
+ ),
5597
+ # Platform Brain MCP endpoint (consolidation U14). Without this
5598
+ # mapping, customer stages never set BRAIN_MCP_URL and twin-connector
5599
+ # provisioning falls back to the retired product /mcp/twin route.
5600
+ "brain_mcp_url": safe_get(
5601
+ runner_secrets,
5602
+ "brainMcpUrl",
5603
+ default=safe_get(reviewed_payload, "brainMcpUrl", default=""),
5604
+ ),
5605
+ # Company Brain platform-agent machine lane (THINK-628/THINK-710).
5606
+ # Secrets Manager ARN of etl-platform/brain-mcp/m2m/platform-agent in
5607
+ # this account; grants the shared API Lambda role read so MCP config
5608
+ # build can mint Cognito bearers for the Brain connector. Environment
5609
+ # posture: durable home is the runner-secrets document. Empty leaves
5610
+ # the tkt_ lane untouched.
5611
+ "brain_m2m_platform_agent_secret_arn": safe_get(
5612
+ runner_secrets,
5613
+ "brainM2mPlatformAgentSecretArn",
5614
+ default=safe_get(reviewed_payload, "brainM2mPlatformAgentSecretArn", default=""),
5615
+ ),
5616
+ # Brain ops-api flag intake (THINK-781): base URL + agent-identity
5617
+ # m2m secret ARN for flagThreadToBrain's POST /flags. Empty leaves
5618
+ # the Send-to-the-Brain action reporting "not configured".
5619
+ "brain_ops_api_url": safe_get(
5620
+ runner_secrets,
5621
+ "brainOpsApiUrl",
5622
+ default=safe_get(reviewed_payload, "brainOpsApiUrl", default=""),
5623
+ ),
5624
+ "brain_ops_m2m_secret_arn": safe_get(
5625
+ runner_secrets,
5626
+ "brainOpsM2mSecretArn",
5627
+ default=safe_get(reviewed_payload, "brainOpsM2mSecretArn", default=""),
5628
+ ),
5629
+ }
5630
+ enforce_customer_domain_preservation(
5631
+ current_outputs,
5632
+ vars_json,
5633
+ payload,
5634
+ runner_secrets,
5635
+ )
5636
+ vars_json.update(
5637
+ managed_app_terraform_overrides(payload, stage, account_id, current_outputs, current_state)
5638
+ )
5639
+ incident_import_blocks = tei_v380_recovery_import_blocks(payload, vars_json)
5640
+
5641
+ TF.mkdir(parents=True, exist_ok=True)
5642
+ (TF / "backend.hcl").write_text(
3217
5643
  "\n".join(
3218
5644
  [
3219
5645
  f"bucket = {hcl_string(os.environ['THINKWORK_TERRAFORM_STATE_BUCKET'])}",
@@ -3227,13 +5653,20 @@ def write_runner_files(payload, runner_secrets):
3227
5653
  encoding="utf-8",
3228
5654
  )
3229
5655
  (TF / "terraform.auto.tfvars.json").write_text(
3230
- json.dumps(vars_json, indent=2, sort_keys=True),
5656
+ json.dumps(
5657
+ {key: value for key, value in vars_json.items() if key not in RUNNER_ONLY_VARS},
5658
+ indent=2,
5659
+ sort_keys=True,
5660
+ ),
3231
5661
  encoding="utf-8",
3232
5662
  )
3233
5663
  (TF / "main.tf").write_text(
3234
5664
  f"""
3235
5665
  terraform {{
3236
- required_version = ">= 1.5"
5666
+ # The managed deployment runtime is pinned to this floor. Terraform 1.8.5
5667
+ # adds declarative state-only removal, used below to forget the original
5668
+ # non-owning AgentCore reconciliation marker without a destroy action.
5669
+ required_version = ">= 1.8.5"
3237
5670
 
3238
5671
  backend "s3" {{}}
3239
5672
 
@@ -3267,6 +5700,21 @@ provider "aws" {{
3267
5700
  # even though they do not change those records.
3268
5701
  provider "cloudflare" {{}}
3269
5702
 
5703
+ # The original AgentCore Twenty reconciliation marker never owned external
5704
+ # cleanup, but an interrupted local-exec may have left it tainted. Forget every
5705
+ # counted instance from managed-deployment state without refresh, provider RPC,
5706
+ # or destroy provisioners. The child module owns the replacement idempotent
5707
+ # reconciliation marker under a new address.
5708
+ removed {{
5709
+ from = module.thinkwork.module.agentcore_proof_identity.terraform_data.twenty_identity_lifecycle
5710
+
5711
+ lifecycle {{
5712
+ destroy = false
5713
+ }}
5714
+ }}
5715
+
5716
+ {incident_import_blocks}
5717
+
3270
5718
  variable "stage" {{
3271
5719
  type = string
3272
5720
  }}
@@ -3289,24 +5737,105 @@ variable "api_auth_secret" {{
3289
5737
  sensitive = true
3290
5738
  }}
3291
5739
 
5740
+ variable "auth_retirement_phase" {{
5741
+ type = string
5742
+ default = "coexistence"
5743
+ }}
5744
+
5745
+ variable "auth_migration_recovery_deadline" {{
5746
+ type = string
5747
+ default = ""
5748
+ }}
5749
+
3292
5750
  variable "database_engine" {{
3293
5751
  type = string
3294
5752
  }}
3295
5753
 
3296
- variable "enable_hindsight" {{
3297
- type = bool
5754
+ variable "external_kb_source_arns" {{
5755
+ type = list(string)
5756
+ default = []
5757
+ }}
5758
+
5759
+ variable "vpc_interface_endpoint_services" {{
5760
+ type = list(string)
5761
+ default = []
3298
5762
  }}
3299
5763
 
3300
- variable "analyst_lambda_vpc_egress" {{
5764
+ variable "vpc_interface_endpoint_single_az" {{
3301
5765
  type = bool
3302
5766
  default = false
3303
5767
  }}
3304
5768
 
3305
- variable "hindsight_database_name" {{
5769
+ variable "lambda_max_memory_mb" {{
5770
+ type = number
5771
+ default = 10240
5772
+ }}
5773
+
5774
+ variable "neptune_endpoint" {{
5775
+ type = string
5776
+ default = ""
5777
+ }}
5778
+
5779
+ variable "neptune_cluster_resource_id" {{
3306
5780
  type = string
3307
5781
  default = ""
3308
5782
  }}
3309
5783
 
5784
+ variable "neptune_client_security_group_id" {{
5785
+ type = string
5786
+ default = ""
5787
+ }}
5788
+
5789
+ variable "neptune_load_bucket" {{
5790
+ type = string
5791
+ default = ""
5792
+ }}
5793
+
5794
+ variable "neptune_loader_role_arn" {{
5795
+ type = string
5796
+ default = ""
5797
+ }}
5798
+
5799
+ variable "brain_mcp_url" {{
5800
+ type = string
5801
+ default = ""
5802
+ }}
5803
+
5804
+ variable "brain_m2m_platform_agent_secret_arn" {{
5805
+ type = string
5806
+ default = ""
5807
+ }}
5808
+
5809
+ variable "brain_ops_api_url" {{
5810
+ type = string
5811
+ default = ""
5812
+ }}
5813
+
5814
+ variable "brain_ops_m2m_secret_arn" {{
5815
+ type = string
5816
+ default = ""
5817
+ }}
5818
+
5819
+ variable "enable_agentcore_multiplayer_proof" {{
5820
+ type = bool
5821
+ default = false
5822
+ }}
5823
+
5824
+ variable "agentcore_multiplayer_proof_tenant_slug" {{
5825
+ type = string
5826
+ default = ""
5827
+ }}
5828
+
5829
+ variable "agentcore_multiplayer_proof_owner_allowlist" {{
5830
+ type = string
5831
+ default = ""
5832
+ }}
5833
+
5834
+ variable "agentcore_runtime_dispatch_enabled" {{
5835
+ type = bool
5836
+ default = false
5837
+ }}
5838
+
3310
5839
  variable "platform_operator_emails" {{
3311
5840
  type = string
3312
5841
  }}
@@ -3343,6 +5872,16 @@ variable "customer_domain_legacy_retired" {{
3343
5872
  type = bool
3344
5873
  }}
3345
5874
 
5875
+ variable "additional_admin_callback_urls" {{
5876
+ type = list(string)
5877
+ default = []
5878
+ }}
5879
+
5880
+ variable "additional_admin_logout_urls" {{
5881
+ type = list(string)
5882
+ default = []
5883
+ }}
5884
+
3346
5885
  variable "google_oauth_client_id" {{
3347
5886
  type = string
3348
5887
  }}
@@ -3352,6 +5891,35 @@ variable "google_oauth_client_secret" {{
3352
5891
  sensitive = true
3353
5892
  }}
3354
5893
 
5894
+ variable "microsoft_oauth_client_id" {{
5895
+ type = string
5896
+ }}
5897
+
5898
+ variable "microsoft_oauth_client_secret" {{
5899
+ type = string
5900
+ sensitive = true
5901
+ }}
5902
+
5903
+ variable "microsoft_oauth_tenant" {{
5904
+ type = string
5905
+ default = ""
5906
+ }}
5907
+
5908
+ variable "tenant_entra_connections" {{
5909
+ type = list(object({{
5910
+ connection_key = string
5911
+ tenant_id = string
5912
+ provider_name = string
5913
+ display_name = string
5914
+ }}))
5915
+ default = []
5916
+ }}
5917
+
5918
+ variable "auth_tenant_connection_metadata" {{
5919
+ type = list(any)
5920
+ default = []
5921
+ }}
5922
+
3355
5923
  variable "lambda_artifact_bucket" {{
3356
5924
  type = string
3357
5925
  }}
@@ -3412,6 +5980,26 @@ variable "enable_deployment_control_plane" {{
3412
5980
  type = bool
3413
5981
  }}
3414
5982
 
5983
+ variable "deployment_runner_in_vpc" {{
5984
+ type = bool
5985
+ default = false
5986
+ }}
5987
+
5988
+ variable "api_lambdas_in_vpc" {{
5989
+ type = bool
5990
+ default = false
5991
+ }}
5992
+
5993
+ variable "db_publicly_accessible" {{
5994
+ type = bool
5995
+ default = true
5996
+ }}
5997
+
5998
+ variable "db_public_ingress_cidrs" {{
5999
+ type = list(string)
6000
+ default = ["0.0.0.0/0"]
6001
+ }}
6002
+
3415
6003
  variable "twenty_provisioned" {{
3416
6004
  type = bool
3417
6005
  }}
@@ -3672,10 +6260,16 @@ module "thinkwork" {{
3672
6260
 
3673
6261
  db_password = var.db_password
3674
6262
  api_auth_secret = var.api_auth_secret
6263
+ auth_retirement_phase = var.auth_retirement_phase
6264
+ auth_migration_recovery_deadline = var.auth_migration_recovery_deadline
3675
6265
  database_engine = var.database_engine
3676
6266
 
3677
6267
  google_oauth_client_id = var.google_oauth_client_id
3678
6268
  google_oauth_client_secret = var.google_oauth_client_secret
6269
+ microsoft_oauth_client_id = var.microsoft_oauth_client_id
6270
+ microsoft_oauth_client_secret = var.microsoft_oauth_client_secret
6271
+ microsoft_oauth_tenant = var.microsoft_oauth_tenant
6272
+ tenant_entra_connections = var.tenant_entra_connections
3679
6273
  platform_operator_emails = var.platform_operator_emails
3680
6274
 
3681
6275
  cognito_email_source_arn = var.cognito_email_source_arn
@@ -3689,15 +6283,36 @@ module "thinkwork" {{
3689
6283
  customer_domain_delegated = var.customer_domain_delegated
3690
6284
  customer_domain_legacy_retired = var.customer_domain_legacy_retired
3691
6285
 
6286
+ additional_admin_callback_urls = var.additional_admin_callback_urls
6287
+ additional_admin_logout_urls = var.additional_admin_logout_urls
6288
+
3692
6289
  lambda_artifact_bucket = var.lambda_artifact_bucket
3693
6290
  lambda_artifact_prefix = var.lambda_artifact_prefix
3694
6291
  require_lambda_artifacts = true
3695
6292
  agentcore_pi_source_image_uri = var.agentcore_pi_source_image_uri
3696
6293
 
3697
- enable_hindsight = var.enable_hindsight
3698
- analyst_lambda_vpc_egress = var.analyst_lambda_vpc_egress
3699
- hindsight_database_name = var.hindsight_database_name
6294
+ external_kb_source_arns = var.external_kb_source_arns
6295
+ vpc_interface_endpoint_services = (
6296
+ length(var.vpc_interface_endpoint_services) == 0
6297
+ ? null
6298
+ : (contains(var.vpc_interface_endpoint_services, "none") ? [] : var.vpc_interface_endpoint_services)
6299
+ )
6300
+ vpc_interface_endpoint_single_az = var.vpc_interface_endpoint_single_az
6301
+ lambda_max_memory_mb = var.lambda_max_memory_mb
6302
+ neptune_endpoint = var.neptune_endpoint
6303
+ neptune_cluster_resource_id = var.neptune_cluster_resource_id
6304
+ neptune_client_security_group_id = var.neptune_client_security_group_id
6305
+ neptune_load_bucket = var.neptune_load_bucket
6306
+ neptune_loader_role_arn = var.neptune_loader_role_arn
6307
+ brain_mcp_url = var.brain_mcp_url
6308
+ brain_m2m_platform_agent_secret_arn = var.brain_m2m_platform_agent_secret_arn
6309
+ brain_ops_api_url = var.brain_ops_api_url
6310
+ brain_ops_m2m_secret_arn = var.brain_ops_m2m_secret_arn
3700
6311
  enable_workspace_orchestration = true
6312
+ enable_agentcore_multiplayer_proof = var.enable_agentcore_multiplayer_proof
6313
+ agentcore_multiplayer_proof_tenant_slug = var.agentcore_multiplayer_proof_tenant_slug
6314
+ agentcore_multiplayer_proof_owner_allowlist = var.agentcore_multiplayer_proof_owner_allowlist
6315
+ agentcore_runtime_dispatch_enabled = var.agentcore_runtime_dispatch_enabled
3701
6316
 
3702
6317
  twenty_provisioned = var.twenty_provisioned
3703
6318
  twenty_runtime_enabled = var.twenty_runtime_enabled
@@ -3747,6 +6362,10 @@ module "thinkwork" {{
3747
6362
  enable_slack_workspace_app = false
3748
6363
 
3749
6364
  enable_deployment_control_plane = var.enable_deployment_control_plane
6365
+ deployment_runner_in_vpc = var.deployment_runner_in_vpc
6366
+ api_lambdas_in_vpc = var.api_lambdas_in_vpc
6367
+ db_publicly_accessible = var.db_publicly_accessible
6368
+ db_public_ingress_cidrs = var.db_public_ingress_cidrs
3750
6369
  deployment_control_plane_create_secret_placeholders = var.deployment_control_plane_create_secret_placeholders
3751
6370
  deployment_state_machine_arn = var.deployment_state_machine_arn
3752
6371
  deployment_evidence_bucket = var.deployment_evidence_bucket
@@ -3778,21 +6397,18 @@ output "app_distribution_id" {{ value = module.thinkwork.app_distribution_id }}
3778
6397
  output "api_endpoint" {{ value = module.thinkwork.api_endpoint }}
3779
6398
  output "appsync_api_url" {{ value = module.thinkwork.appsync_api_url }}
3780
6399
  output "appsync_realtime_url" {{ value = module.thinkwork.appsync_realtime_url }}
3781
- output "appsync_api_key" {{
3782
- value = module.thinkwork.appsync_api_key
3783
- sensitive = true
3784
- }}
3785
6400
  output "auth_domain" {{ value = module.thinkwork.auth_domain }}
3786
6401
  output "customer_domain" {{ value = module.thinkwork.customer_domain }}
3787
6402
  output "customer_domain_name_servers" {{ value = module.thinkwork.customer_domain_name_servers }}
6403
+ output "db_cluster_arn" {{ value = module.thinkwork.db_cluster_arn }}
3788
6404
  output "db_cluster_endpoint" {{ value = module.thinkwork.db_cluster_endpoint }}
3789
6405
  output "db_secret_arn" {{ value = module.thinkwork.db_secret_arn }}
3790
6406
  output "database_name" {{ value = module.thinkwork.database_name }}
3791
6407
  output "user_pool_id" {{ value = module.thinkwork.user_pool_id }}
3792
6408
  output "admin_client_id" {{ value = module.thinkwork.admin_client_id }}
3793
- output "docs_bucket_name" {{ value = module.thinkwork.docs_bucket_name }}
3794
- output "docs_distribution_id" {{ value = module.thinkwork.docs_distribution_id }}
3795
- output "docs_distribution_domain" {{ value = module.thinkwork.docs_distribution_domain }}
6409
+ output "web_local_client_id" {{ value = module.thinkwork.web_local_client_id }}
6410
+ output "auth_route_clients" {{ value = module.thinkwork.auth_route_clients }}
6411
+ output "auth_retirement_phase" {{ value = module.thinkwork.auth_retirement_phase }}
3796
6412
  output "twenty_provisioned" {{ value = module.thinkwork.twenty_provisioned }}
3797
6413
  output "twenty_runtime_enabled" {{ value = module.thinkwork.twenty_runtime_enabled }}
3798
6414
  output "twenty_url" {{ value = module.thinkwork.twenty_url }}
@@ -3833,7 +6449,7 @@ output "deployment_appconfig_configuration_profile_id" {{ value = module.thinkwo
3833
6449
 
3834
6450
  def sync_release_artifacts(artifact_types=None, artifact_names=None):
3835
6451
  global RELEASE_EVIDENCE
3836
- artifact_types = set(artifact_types or {"lambda", "static-site"})
6452
+ artifact_types = {"lambda", "static-site"} if artifact_types is None else set(artifact_types)
3837
6453
  artifact_names = set(artifact_names or [])
3838
6454
  manifest_url = os.environ.get("THINKWORK_RELEASE_MANIFEST_URL")
3839
6455
  expected = os.environ.get("THINKWORK_RELEASE_MANIFEST_SHA256", "").lower()
@@ -4004,6 +6620,69 @@ def write_current_outputs_from_state(stage, outputs_path):
4004
6620
  return outputs
4005
6621
 
4006
6622
 
6623
+ def ensure_native_auth_custom_attributes(current_outputs):
6624
+ """Add the native Entra claims to an existing pool before app clients use them."""
6625
+ user_pool_id = str(_current_output_value(current_outputs, "user_pool_id") or "").strip()
6626
+ if not user_pool_id:
6627
+ return {"status": "skipped", "reason": "user-pool-not-created"}
6628
+
6629
+ def describe_attribute_names():
6630
+ raw = output(
6631
+ [
6632
+ "aws",
6633
+ "cognito-idp",
6634
+ "describe-user-pool",
6635
+ "--user-pool-id",
6636
+ user_pool_id,
6637
+ "--output",
6638
+ "json",
6639
+ ]
6640
+ )
6641
+ described = json.loads(raw or "{}")
6642
+ attributes = described.get("UserPool", {}).get("SchemaAttributes", [])
6643
+ return {
6644
+ str(attribute.get("Name") or "")
6645
+ for attribute in attributes
6646
+ if isinstance(attribute, dict)
6647
+ }
6648
+
6649
+ existing_names = describe_attribute_names()
6650
+ missing = [
6651
+ attribute
6652
+ for attribute in NATIVE_AUTH_CUSTOM_ATTRIBUTES
6653
+ if f"custom:{attribute['Name']}" not in existing_names
6654
+ ]
6655
+ if not missing:
6656
+ return {"status": "current", "userPoolId": user_pool_id, "added": []}
6657
+
6658
+ request = {"UserPoolId": user_pool_id, "CustomAttributes": missing}
6659
+ run(
6660
+ [
6661
+ "aws",
6662
+ "cognito-idp",
6663
+ "add-custom-attributes",
6664
+ "--cli-input-json",
6665
+ json.dumps(request, separators=(",", ":"), sort_keys=True),
6666
+ ],
6667
+ stdout=subprocess.DEVNULL,
6668
+ )
6669
+ reconciled_names = describe_attribute_names()
6670
+ unresolved = [
6671
+ attribute["Name"]
6672
+ for attribute in missing
6673
+ if f"custom:{attribute['Name']}" not in reconciled_names
6674
+ ]
6675
+ if unresolved:
6676
+ raise RuntimeError(
6677
+ "Cognito native-auth custom attributes were not added: " + ", ".join(unresolved)
6678
+ )
6679
+ return {
6680
+ "status": "reconciled",
6681
+ "userPoolId": user_pool_id,
6682
+ "added": [attribute["Name"] for attribute in missing],
6683
+ }
6684
+
6685
+
4007
6686
  def write_outputs_to_ssm(outputs_path, vars_json):
4008
6687
  outputs = json.loads(outputs_path.read_text(encoding="utf-8"))
4009
6688
  profile, web_env = runtime_profile(outputs, vars_json)
@@ -4013,7 +6692,6 @@ def write_outputs_to_ssm(outputs_path, vars_json):
4013
6692
  "profile/graphql-http-url": None,
4014
6693
  "profile/appsync-url": "appsync_api_url",
4015
6694
  "profile/appsync-realtime-url": "appsync_realtime_url",
4016
- "profile/appsync-api-key": "appsync_api_key",
4017
6695
  "profile/cognito-domain": None,
4018
6696
  "profile/cognito-user-pool-id": "user_pool_id",
4019
6697
  "profile/cognito-client-id": "admin_client_id",
@@ -4154,7 +6832,6 @@ def runtime_profile(outputs, vars_json):
4154
6832
  "graphqlHttpUrl": graphql_http_url,
4155
6833
  "appsyncUrl": output_value("appsync_api_url"),
4156
6834
  "appsyncRealtimeUrl": output_value("appsync_realtime_url"),
4157
- "appsyncApiKey": output_value("appsync_api_key"),
4158
6835
  "cognitoDomain": cognito_domain,
4159
6836
  "cognitoUserPoolId": output_value("user_pool_id"),
4160
6837
  "cognitoClientId": output_value("admin_client_id"),
@@ -4185,7 +6862,6 @@ def runtime_profile(outputs, vars_json):
4185
6862
  "VITE_GRAPHQL_HTTP_URL": profile["graphqlHttpUrl"],
4186
6863
  "VITE_GRAPHQL_URL": profile["appsyncUrl"],
4187
6864
  "VITE_GRAPHQL_WS_URL": profile["appsyncRealtimeUrl"],
4188
- "VITE_GRAPHQL_API_KEY": profile["appsyncApiKey"],
4189
6865
  "VITE_COGNITO_DOMAIN": profile["cognitoDomain"],
4190
6866
  "VITE_COGNITO_USER_POOL_ID": profile["cognitoUserPoolId"],
4191
6867
  "VITE_COGNITO_CLIENT_ID": profile["cognitoClientId"],
@@ -4211,12 +6887,709 @@ def runtime_profile(outputs, vars_json):
4211
6887
  return profile, web_env + "\n"
4212
6888
 
4213
6889
 
6890
+ IDENTITY_PROVIDER_ACTIONS = {"create", "validate", "rotate", "disable"}
6891
+ ENTRA_TENANT_GUID_RE = re.compile(
6892
+ r"^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$",
6893
+ re.IGNORECASE,
6894
+ )
6895
+ ENTRA_PROVIDER_NAME_RE = re.compile(r"^Entra_[a-f0-9]{16}_[a-f0-9]{8}$")
6896
+
6897
+
6898
+ def identity_provider_operation(payload):
6899
+ operation = payload.get("operation")
6900
+ if not isinstance(operation, dict) or operation.get("kind") != "identity_provider":
6901
+ return None
6902
+ return operation
6903
+
6904
+
6905
+ def _identity_string(value, label):
6906
+ if not isinstance(value, str) or not value.strip():
6907
+ raise RuntimeError(f"Identity-provider operation requires {label}")
6908
+ return value.strip()
6909
+
6910
+
6911
+ def _current_output_value(current_outputs, key):
6912
+ value = current_outputs.get(key) if isinstance(current_outputs, dict) else None
6913
+ if isinstance(value, dict) and "value" in value:
6914
+ return value.get("value")
6915
+ return value
6916
+
6917
+
6918
+ def validate_identity_provider_secret_arn(secret_arn, stage, account_id, region):
6919
+ expected = f"arn:aws:secretsmanager:{region}:{account_id}:secret:thinkwork/{stage}/auth/entra/"
6920
+ if not isinstance(secret_arn, str) or not secret_arn.startswith(expected):
6921
+ raise RuntimeError(
6922
+ "Tenant Entra secret ARN must belong to this account, region, stage, "
6923
+ "and thinkwork/<stage>/auth/entra namespace"
6924
+ )
6925
+ return secret_arn
6926
+
6927
+
6928
+ def validate_identity_provider_connection(raw, stage, account_id, region, user_pool_id):
6929
+ if not isinstance(raw, dict):
6930
+ raise RuntimeError("Identity-provider operation requires safe connection metadata")
6931
+ tenant_id = _identity_string(raw.get("tenantId"), "connection.tenantId").lower()
6932
+ if not ENTRA_TENANT_GUID_RE.fullmatch(tenant_id):
6933
+ raise RuntimeError("Tenant Entra directory ID must be a GUID")
6934
+ provider_name = _identity_string(raw.get("providerName"), "connection.providerName")
6935
+ if not ENTRA_PROVIDER_NAME_RE.fullmatch(provider_name):
6936
+ raise RuntimeError("Tenant Entra provider name is not deterministic")
6937
+ connection_key = f"microsoft:tenant:{tenant_id}"
6938
+ if raw.get("connectionKey") != connection_key:
6939
+ raise RuntimeError("Tenant Entra connection key does not match its directory ID")
6940
+ client_id = _identity_string(raw.get("clientId"), "connection.clientId")
6941
+ display_name = _identity_string(raw.get("displayName"), "connection.displayName")
6942
+ secret_arn = validate_identity_provider_secret_arn(
6943
+ raw.get("clientSecretRef"), stage, account_id, region
6944
+ )
6945
+ client_secret_version_stage = raw.get("clientSecretVersionStage")
6946
+ if client_secret_version_stage is not None and client_secret_version_stage != "AWSPENDING":
6947
+ raise RuntimeError("Tenant Entra clientSecretVersionStage must be AWSPENDING when supplied")
6948
+ bindings = raw.get("tenantBindings")
6949
+ if not isinstance(bindings, list) or len(bindings) != 1:
6950
+ raise RuntimeError("Tenant Entra connection requires exactly one tenant binding")
6951
+ binding = bindings[0]
6952
+ if not isinstance(binding, dict):
6953
+ raise RuntimeError("Tenant Entra binding must be an object")
6954
+ thinkwork_tenant_id = _identity_string(binding.get("tenantId"), "tenant binding ID")
6955
+ try:
6956
+ uuid.UUID(thinkwork_tenant_id)
6957
+ except (ValueError, AttributeError) as error:
6958
+ raise RuntimeError("ThinkWork tenant binding ID must be a UUID") from error
6959
+ hostnames = binding.get("hostnames")
6960
+ if not isinstance(hostnames, list) or not hostnames:
6961
+ raise RuntimeError("Tenant Entra binding requires at least one hostname")
6962
+ normalized_hostnames = []
6963
+ for value in hostnames:
6964
+ hostname = _identity_string(value, "tenant binding hostname").lower().rstrip(".")
6965
+ if not re.fullmatch(r"[a-z0-9](?:[a-z0-9.-]{0,251}[a-z0-9])?", hostname):
6966
+ raise RuntimeError(f"Invalid tenant auth hostname: {hostname}")
6967
+ normalized_hostnames.append(hostname)
6968
+ issuer_url = f"https://login.microsoftonline.com/{tenant_id}/v2.0"
6969
+ if raw.get("issuerUrl") != issuer_url:
6970
+ raise RuntimeError("Tenant Entra issuer must use its tenant GUID v2.0 authority")
6971
+ return {
6972
+ "connectionKey": connection_key,
6973
+ "providerKey": "microsoft",
6974
+ "providerKind": "microsoft_tenant",
6975
+ "displayName": display_name,
6976
+ "lifecycleState": "native",
6977
+ "cognitoUserPoolId": _identity_string(user_pool_id, "Cognito user pool ID"),
6978
+ "cognitoIdentityProviderName": provider_name,
6979
+ "issuerUrl": issuer_url,
6980
+ "clientId": client_id,
6981
+ "clientSecretRef": secret_arn,
6982
+ **(
6983
+ {"clientSecretVersionStage": client_secret_version_stage}
6984
+ if client_secret_version_stage is not None
6985
+ else {}
6986
+ ),
6987
+ "authorizeScopes": "openid email profile",
6988
+ "tenantBindings": [
6989
+ {
6990
+ "tenantId": thinkwork_tenant_id,
6991
+ "label": _identity_string(binding.get("label"), "tenant binding label"),
6992
+ "hostnames": sorted(set(normalized_hostnames)),
6993
+ "status": "enabled",
6994
+ }
6995
+ ],
6996
+ # Kept in runner/SSM safe state for the Terraform projection only.
6997
+ "tenantDirectoryId": tenant_id,
6998
+ }
6999
+
7000
+
7001
+ def identity_provider_desired_connections(
7002
+ payload, *, stage, account_id, region, previous_state, current_outputs
7003
+ ):
7004
+ previous = previous_state.get("tenantConnections") or []
7005
+ if not isinstance(previous, list):
7006
+ raise RuntimeError("Auth reconciliation tenantConnections state must be an array")
7007
+ desired = [dict(value) for value in previous if isinstance(value, dict)]
7008
+ operation = identity_provider_operation(payload)
7009
+ if operation is None:
7010
+ return desired
7011
+
7012
+ action = str(operation.get("action") or "").lower()
7013
+ if action not in IDENTITY_PROVIDER_ACTIONS:
7014
+ raise RuntimeError(f"Unsupported identity-provider action: {action}")
7015
+ expected_previous = int(operation.get("expectedPreviousRevision", -1))
7016
+ revision = int(operation.get("revision", -1))
7017
+ current_revision = int(previous_state.get("desiredRevision") or 0)
7018
+ if expected_previous != current_revision or revision != current_revision + 1:
7019
+ raise RuntimeError(f"Identity-provider revision conflict: expected {current_revision + 1}")
7020
+
7021
+ user_pool_id = _current_output_value(current_outputs, "user_pool_id")
7022
+ connection = validate_identity_provider_connection(
7023
+ operation.get("connection"), stage, account_id, region, user_pool_id
7024
+ )
7025
+ client_secret_version_stage = connection.get("clientSecretVersionStage")
7026
+ if action == "rotate" and client_secret_version_stage != "AWSPENDING":
7027
+ raise RuntimeError("Tenant Entra rotation requires clientSecretVersionStage AWSPENDING")
7028
+ if action != "rotate" and client_secret_version_stage is not None:
7029
+ raise RuntimeError(
7030
+ "clientSecretVersionStage AWSPENDING is valid only for Tenant Entra rotation"
7031
+ )
7032
+ existing = next(
7033
+ (value for value in desired if value.get("connectionKey") == connection["connectionKey"]),
7034
+ None,
7035
+ )
7036
+ if action == "create" and existing and existing.get("lifecycleState") != "denied":
7037
+ raise RuntimeError("Tenant Entra connection already exists; use rotate or validate")
7038
+ if action in {"validate", "rotate", "disable"} and not existing:
7039
+ raise RuntimeError(f"Tenant Entra connection does not exist for {action}")
7040
+ if action in {"validate", "rotate", "disable"} and existing != connection:
7041
+ # Disable may change lifecycle; rotate may add only its runner-owned
7042
+ # pending-version label. Safe connection metadata must remain exact.
7043
+ comparable = dict(existing)
7044
+ comparable["lifecycleState"] = "native"
7045
+ submitted_comparable = dict(connection)
7046
+ comparable.pop("clientSecretVersionStage", None)
7047
+ submitted_comparable.pop("clientSecretVersionStage", None)
7048
+ if comparable != submitted_comparable:
7049
+ raise RuntimeError(
7050
+ "Identity-provider metadata differs from the active desired connection"
7051
+ )
7052
+
7053
+ desired = [
7054
+ value for value in desired if value.get("connectionKey") != connection["connectionKey"]
7055
+ ]
7056
+ if action == "disable":
7057
+ connection["lifecycleState"] = "denied"
7058
+ connection["tenantBindings"][0]["status"] = "disabled"
7059
+ desired.append(connection)
7060
+ return sorted(desired, key=lambda value: value["connectionKey"])
7061
+
7062
+
7063
+ def tenant_entra_terraform_projection(connections):
7064
+ projected = []
7065
+ for connection in connections:
7066
+ if connection.get("lifecycleState") != "native":
7067
+ continue
7068
+ projected.append(
7069
+ {
7070
+ "connection_key": connection["connectionKey"],
7071
+ "tenant_id": connection["tenantDirectoryId"],
7072
+ "provider_name": connection["cognitoIdentityProviderName"],
7073
+ "display_name": connection["displayName"],
7074
+ }
7075
+ )
7076
+ return projected
7077
+
7078
+
7079
+ def native_auth_reconciliation_payload(outputs, vars_json, previous_state=None):
7080
+ """Build the complete secret-free Cognito desired set from Terraform outputs."""
7081
+ previous_state = previous_state if isinstance(previous_state, dict) else {}
7082
+ route_manifest = outputs.get("auth_route_clients", {}).get("value") or {}
7083
+ if not isinstance(route_manifest, dict):
7084
+ raise RuntimeError("auth_route_clients Terraform output must be an object")
7085
+ user_pool_id = str(outputs.get("user_pool_id", {}).get("value") or "")
7086
+ if not user_pool_id:
7087
+ raise RuntimeError("Native auth reconciliation requires user_pool_id output")
7088
+
7089
+ routes = []
7090
+ provider_client_ids = {}
7091
+ for manifest_key, raw in sorted(route_manifest.items()):
7092
+ if not isinstance(raw, dict):
7093
+ raise RuntimeError(f"Invalid auth route manifest entry: {manifest_key}")
7094
+ provider_names = [str(value) for value in raw.get("provider_names") or []]
7095
+ client_id = str(raw.get("client_id") or "")
7096
+ for provider_name in provider_names:
7097
+ provider_client_ids.setdefault(provider_name, []).append(client_id)
7098
+ routes.append(
7099
+ {
7100
+ "routeKey": str(raw.get("route_key") or ""),
7101
+ "clientFamily": str(raw.get("client_family") or ""),
7102
+ "cognitoUserPoolId": user_pool_id,
7103
+ "cognitoAppClientId": client_id,
7104
+ "providerNames": provider_names,
7105
+ "explicitAuthFlows": [str(value) for value in raw.get("explicit_auth_flows") or []],
7106
+ "redirectUris": [str(value) for value in raw.get("callback_urls") or []],
7107
+ "logoutUris": [str(value) for value in raw.get("logout_urls") or []],
7108
+ "lifecycleState": str(raw.get("lifecycle_state") or "native"),
7109
+ }
7110
+ )
7111
+
7112
+ connections = [
7113
+ {
7114
+ "connectionKey": "local",
7115
+ "providerKey": "cognito",
7116
+ "providerKind": "local",
7117
+ "displayName": "Email and password",
7118
+ "lifecycleState": "native",
7119
+ "cognitoUserPoolId": user_pool_id,
7120
+ "cognitoIdentityProviderName": "COGNITO",
7121
+ "authorizeScopes": "openid email profile",
7122
+ "tenantBindings": [],
7123
+ }
7124
+ ]
7125
+ if provider_client_ids.get("Google"):
7126
+ connections.append(
7127
+ {
7128
+ "connectionKey": "google",
7129
+ "providerKey": "google",
7130
+ "providerKind": "google",
7131
+ "displayName": "Google",
7132
+ "lifecycleState": "native",
7133
+ "cognitoUserPoolId": user_pool_id,
7134
+ "cognitoIdentityProviderName": "Google",
7135
+ "authorizeScopes": "openid email profile",
7136
+ "tenantBindings": [],
7137
+ }
7138
+ )
7139
+ if provider_client_ids.get("MicrosoftOrganizations"):
7140
+ microsoft_tenant = str(vars_json.get("microsoft_oauth_tenant") or "").lower()
7141
+ if microsoft_tenant not in {"common", "organizations", "consumers"} and not re.fullmatch(
7142
+ r"[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}",
7143
+ microsoft_tenant,
7144
+ ):
7145
+ raise RuntimeError(
7146
+ "Microsoft native auth reconciliation requires microsoft_oauth_tenant to be"
7147
+ " an Entra directory GUID or one of: common, organizations, consumers"
7148
+ )
7149
+ connections.append(
7150
+ {
7151
+ "connectionKey": "microsoft:organizations",
7152
+ "providerKey": "microsoft",
7153
+ "providerKind": "microsoft_organizations",
7154
+ "displayName": "Microsoft",
7155
+ "lifecycleState": "native",
7156
+ "cognitoUserPoolId": user_pool_id,
7157
+ "cognitoIdentityProviderName": "MicrosoftOrganizations",
7158
+ "issuerUrl": (f"https://login.microsoftonline.com/{microsoft_tenant}/v2.0"),
7159
+ "authorizeScopes": "openid email profile",
7160
+ "tenantBindings": [],
7161
+ }
7162
+ )
7163
+
7164
+ # Identity-provider operations store the complete tenant connection set in
7165
+ # safe SSM state. Standard releases carry it forward; omission is not
7166
+ # interpreted as deletion.
7167
+ tenant_connections = vars_json.get("auth_tenant_connection_metadata")
7168
+ if tenant_connections is None:
7169
+ tenant_connections = previous_state.get("tenantConnections") or []
7170
+ for connection in tenant_connections:
7171
+ if isinstance(connection, dict):
7172
+ # tenantDirectoryId is runner-owned projection metadata, not part
7173
+ # of the public/API reconciliation contract.
7174
+ connections.append(
7175
+ {
7176
+ key: value
7177
+ for key, value in connection.items()
7178
+ if key not in {"tenantDirectoryId", "clientSecretVersionStage"}
7179
+ }
7180
+ )
7181
+
7182
+ desired = {
7183
+ "connections": sorted(connections, key=lambda item: item["connectionKey"]),
7184
+ "routeClients": sorted(
7185
+ routes,
7186
+ key=lambda item: f"{item['routeKey']}:{item['clientFamily']}",
7187
+ ),
7188
+ }
7189
+ desired_fingerprint = hashlib.sha256(canonical_json(desired).encode("utf-8")).hexdigest()
7190
+ if desired_fingerprint == previous_state.get("desiredFingerprint"):
7191
+ return None, previous_state
7192
+
7193
+ previous_revision = int(previous_state.get("revision") or 0)
7194
+ payload = {
7195
+ "stage": vars_json["stage"],
7196
+ "awsAccountId": vars_json["account_id"],
7197
+ "awsRegion": vars_json["region"],
7198
+ "revision": previous_revision + 1,
7199
+ "expectedPreviousRevision": previous_revision,
7200
+ "idempotencyKey": str(
7201
+ uuid.uuid5(
7202
+ uuid.NAMESPACE_URL,
7203
+ f"thinkwork-auth:{vars_json['stage']}:{previous_revision + 1}:{desired_fingerprint}",
7204
+ )
7205
+ ),
7206
+ "connections": desired["connections"],
7207
+ "routeClients": desired["routeClients"],
7208
+ }
7209
+ payload["manifestFingerprint"] = hashlib.sha256(
7210
+ json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
7211
+ ).hexdigest()
7212
+ next_state = {
7213
+ "revision": payload["revision"],
7214
+ "desiredFingerprint": desired_fingerprint,
7215
+ "tenantConnections": tenant_connections,
7216
+ "desiredRevision": previous_state.get("desiredRevision", 0),
7217
+ }
7218
+ return payload, next_state
7219
+
7220
+
7221
+ def auth_reconciliation_state(stage):
7222
+ name = f"/thinkwork/{stage}/auth/reconciliation/state"
7223
+ try:
7224
+ encoded = output(
7225
+ [
7226
+ "aws",
7227
+ "ssm",
7228
+ "get-parameter",
7229
+ "--name",
7230
+ name,
7231
+ "--query",
7232
+ "Parameter.Value",
7233
+ "--output",
7234
+ "text",
7235
+ ]
7236
+ )
7237
+ return json.loads(encoded)
7238
+ except Exception:
7239
+ return {}
7240
+
7241
+
7242
+ def _describe_cognito_identity_provider(user_pool_id, provider_name):
7243
+ try:
7244
+ raw = output(
7245
+ [
7246
+ "aws",
7247
+ "cognito-idp",
7248
+ "describe-identity-provider",
7249
+ "--user-pool-id",
7250
+ user_pool_id,
7251
+ "--provider-name",
7252
+ provider_name,
7253
+ "--output",
7254
+ "json",
7255
+ ],
7256
+ stderr=subprocess.PIPE,
7257
+ )
7258
+ value = json.loads(raw or "{}")
7259
+ return value.get("IdentityProvider") if isinstance(value, dict) else None
7260
+ except subprocess.CalledProcessError as error:
7261
+ stderr = str(getattr(error, "stderr", "") or "")
7262
+ if "ResourceNotFoundException" in stderr:
7263
+ return None
7264
+ raise RuntimeError(
7265
+ "Tenant Entra Cognito identity provider could not be described"
7266
+ ) from error
7267
+ except json.JSONDecodeError as error:
7268
+ raise RuntimeError(
7269
+ "Tenant Entra Cognito identity provider returned malformed metadata"
7270
+ ) from error
7271
+
7272
+
7273
+ def _read_tenant_entra_secret(connection):
7274
+ try:
7275
+ args = [
7276
+ "aws",
7277
+ "secretsmanager",
7278
+ "get-secret-value",
7279
+ "--secret-id",
7280
+ connection["clientSecretRef"],
7281
+ ]
7282
+ version_stage = connection.get("clientSecretVersionStage")
7283
+ if version_stage:
7284
+ args.extend(["--version-stage", version_stage])
7285
+ args.extend(
7286
+ [
7287
+ "--query",
7288
+ "SecretString",
7289
+ "--output",
7290
+ "text",
7291
+ ]
7292
+ )
7293
+ body = output(
7294
+ args,
7295
+ stderr=subprocess.DEVNULL,
7296
+ )
7297
+ secret = json.loads(body or "{}")
7298
+ except (subprocess.CalledProcessError, json.JSONDecodeError) as error:
7299
+ raise RuntimeError("Tenant Entra secret could not be read") from error
7300
+ client_id = secret.get("clientId") if isinstance(secret, dict) else None
7301
+ client_secret = secret.get("clientSecret") if isinstance(secret, dict) else None
7302
+ if (
7303
+ client_id != connection["clientId"]
7304
+ or not isinstance(client_secret, str)
7305
+ or not client_secret
7306
+ ):
7307
+ raise RuntimeError("Tenant Entra secret document is missing matching credentials")
7308
+ return client_secret
7309
+
7310
+
7311
+ def promote_tenant_entra_pending_secret(payload, vars_json):
7312
+ operation = identity_provider_operation(payload)
7313
+ if operation is None or str(operation.get("action") or "").lower() != "rotate":
7314
+ return None
7315
+ connection_key = operation["connection"]["connectionKey"]
7316
+ connection = next(
7317
+ value
7318
+ for value in vars_json["auth_tenant_connection_metadata"]
7319
+ if value["connectionKey"] == connection_key
7320
+ )
7321
+ if connection.get("clientSecretVersionStage") != "AWSPENDING":
7322
+ raise RuntimeError("Tenant Entra rotation is missing its pending secret version")
7323
+ raw = output(
7324
+ [
7325
+ "aws",
7326
+ "secretsmanager",
7327
+ "describe-secret",
7328
+ "--secret-id",
7329
+ connection["clientSecretRef"],
7330
+ "--query",
7331
+ "VersionIdsToStages",
7332
+ "--output",
7333
+ "json",
7334
+ ],
7335
+ stderr=subprocess.DEVNULL,
7336
+ )
7337
+ versions = json.loads(raw or "{}")
7338
+ pending = next(
7339
+ (version for version, stages in versions.items() if "AWSPENDING" in stages),
7340
+ None,
7341
+ )
7342
+ current = next(
7343
+ (version for version, stages in versions.items() if "AWSCURRENT" in stages),
7344
+ None,
7345
+ )
7346
+ if not pending:
7347
+ raise RuntimeError("Tenant Entra pending secret version was not found")
7348
+ if pending != current:
7349
+ args = [
7350
+ "aws",
7351
+ "secretsmanager",
7352
+ "update-secret-version-stage",
7353
+ "--secret-id",
7354
+ connection["clientSecretRef"],
7355
+ "--version-stage",
7356
+ "AWSCURRENT",
7357
+ "--move-to-version-id",
7358
+ pending,
7359
+ ]
7360
+ if current:
7361
+ args.extend(["--remove-from-version-id", current])
7362
+ output(args, stderr=subprocess.DEVNULL)
7363
+ return {"status": "promoted", "connectionKey": connection_key}
7364
+
7365
+
7366
+ def _assert_tenant_entra_provider_matches(existing, connection):
7367
+ details = existing.get("ProviderDetails") if isinstance(existing, dict) else None
7368
+ mapping = existing.get("AttributeMapping") if isinstance(existing, dict) else None
7369
+ expected_mapping = {
7370
+ "email": "preferred_username",
7371
+ "name": "name",
7372
+ "username": "sub",
7373
+ "custom:entra_tenant_id": "tid",
7374
+ "custom:entra_object_id": "oid",
7375
+ }
7376
+ if (
7377
+ not isinstance(existing, dict)
7378
+ or existing.get("ProviderName") != connection["cognitoIdentityProviderName"]
7379
+ or existing.get("ProviderType") != "OIDC"
7380
+ or not isinstance(details, dict)
7381
+ or details.get("client_id") != connection["clientId"]
7382
+ or str(details.get("oidc_issuer") or "").rstrip("/") != connection["issuerUrl"].rstrip("/")
7383
+ or details.get("authorize_scopes") != "openid email profile"
7384
+ or mapping != expected_mapping
7385
+ ):
7386
+ raise RuntimeError("Tenant Entra Cognito identity provider has drifted")
7387
+
7388
+
7389
+ def reconcile_identity_provider_resource(payload, vars_json):
7390
+ operation = identity_provider_operation(payload)
7391
+ if operation is None:
7392
+ return None
7393
+ action = str(operation["action"]).lower()
7394
+ connection_key = operation["connection"]["connectionKey"]
7395
+ connection = next(
7396
+ value
7397
+ for value in vars_json["auth_tenant_connection_metadata"]
7398
+ if value["connectionKey"] == connection_key
7399
+ )
7400
+ user_pool_id = connection["cognitoUserPoolId"]
7401
+ provider_name = connection["cognitoIdentityProviderName"]
7402
+ existing = _describe_cognito_identity_provider(user_pool_id, provider_name)
7403
+
7404
+ if action == "disable":
7405
+ return {
7406
+ "action": action,
7407
+ "connectionKey": connection_key,
7408
+ "providerName": provider_name,
7409
+ "status": "pending_route_removal",
7410
+ }
7411
+
7412
+ # Validation intentionally reads the secret as well: a provider whose
7413
+ # referenced credential is inaccessible or malformed is not valid.
7414
+ client_secret = _read_tenant_entra_secret(connection)
7415
+ if action == "validate":
7416
+ if existing is None:
7417
+ raise RuntimeError("Tenant Entra Cognito identity provider does not exist")
7418
+ _assert_tenant_entra_provider_matches(existing, connection)
7419
+ return {
7420
+ "action": action,
7421
+ "connectionKey": connection_key,
7422
+ "providerName": provider_name,
7423
+ "status": "valid",
7424
+ }
7425
+
7426
+ cli_input = {
7427
+ "UserPoolId": user_pool_id,
7428
+ "ProviderName": provider_name,
7429
+ "ProviderDetails": {
7430
+ "client_id": connection["clientId"],
7431
+ "client_secret": client_secret,
7432
+ "attributes_request_method": "GET",
7433
+ "oidc_issuer": connection["issuerUrl"],
7434
+ "authorize_scopes": "openid email profile",
7435
+ },
7436
+ "AttributeMapping": {
7437
+ "email": "preferred_username",
7438
+ "name": "name",
7439
+ "username": "sub",
7440
+ "custom:entra_tenant_id": "tid",
7441
+ "custom:entra_object_id": "oid",
7442
+ },
7443
+ "IdpIdentifiers": [],
7444
+ }
7445
+ aws_action = "update-identity-provider" if existing else "create-identity-provider"
7446
+ try:
7447
+ subprocess.run(
7448
+ [
7449
+ "aws",
7450
+ "cognito-idp",
7451
+ aws_action,
7452
+ "--cli-input-json",
7453
+ "file:///dev/stdin",
7454
+ "--output",
7455
+ "json",
7456
+ ],
7457
+ input=json.dumps(cli_input, separators=(",", ":")),
7458
+ text=True,
7459
+ check=True,
7460
+ stdout=subprocess.DEVNULL,
7461
+ stderr=subprocess.DEVNULL,
7462
+ )
7463
+ except subprocess.CalledProcessError as error:
7464
+ raise RuntimeError("Tenant Entra Cognito reconciliation failed") from error
7465
+ described = _describe_cognito_identity_provider(user_pool_id, provider_name)
7466
+ _assert_tenant_entra_provider_matches(described, connection)
7467
+ return {
7468
+ "action": action,
7469
+ "connectionKey": connection_key,
7470
+ "providerName": provider_name,
7471
+ "status": "reconciled",
7472
+ }
7473
+
7474
+
7475
+ def finalize_disabled_identity_provider(payload, vars_json):
7476
+ operation = identity_provider_operation(payload)
7477
+ if operation is None or str(operation.get("action") or "").lower() != "disable":
7478
+ return None
7479
+ connection = operation["connection"]
7480
+ user_pool_id = next(
7481
+ value["cognitoUserPoolId"]
7482
+ for value in vars_json["auth_tenant_connection_metadata"]
7483
+ if value["connectionKey"] == connection["connectionKey"]
7484
+ )
7485
+ provider_name = connection["providerName"]
7486
+ if _describe_cognito_identity_provider(user_pool_id, provider_name) is not None:
7487
+ try:
7488
+ output(
7489
+ [
7490
+ "aws",
7491
+ "cognito-idp",
7492
+ "delete-identity-provider",
7493
+ "--user-pool-id",
7494
+ user_pool_id,
7495
+ "--provider-name",
7496
+ provider_name,
7497
+ ],
7498
+ stderr=subprocess.DEVNULL,
7499
+ )
7500
+ except subprocess.CalledProcessError as error:
7501
+ raise RuntimeError("Tenant Entra Cognito provider could not be disabled") from error
7502
+ return {
7503
+ "action": "disable",
7504
+ "connectionKey": connection["connectionKey"],
7505
+ "providerName": provider_name,
7506
+ "status": "disabled",
7507
+ }
7508
+
7509
+
7510
+ def record_identity_provider_operation_state(payload, vars_json):
7511
+ operation = identity_provider_operation(payload)
7512
+ if operation is None:
7513
+ return
7514
+ current = auth_reconciliation_state(vars_json["stage"])
7515
+ expected = int(operation["expectedPreviousRevision"])
7516
+ if int(current.get("desiredRevision") or 0) != expected:
7517
+ raise RuntimeError("Identity-provider desired revision changed during reconciliation")
7518
+ current["desiredRevision"] = int(operation["revision"])
7519
+ current["tenantConnections"] = [
7520
+ {key: value for key, value in connection.items() if key != "clientSecretVersionStage"}
7521
+ for connection in vars_json["auth_tenant_connection_metadata"]
7522
+ ]
7523
+ run(
7524
+ [
7525
+ "aws",
7526
+ "ssm",
7527
+ "put-parameter",
7528
+ "--overwrite",
7529
+ "--type",
7530
+ "String",
7531
+ "--name",
7532
+ f"/thinkwork/{vars_json['stage']}/auth/reconciliation/state",
7533
+ "--value",
7534
+ json.dumps(current, separators=(",", ":")),
7535
+ ]
7536
+ )
7537
+
7538
+
7539
+ def reconcile_native_auth_metadata(outputs_path, vars_json):
7540
+ outputs = json.loads(outputs_path.read_text(encoding="utf-8"))
7541
+ previous = auth_reconciliation_state(vars_json["stage"])
7542
+ payload, next_state = native_auth_reconciliation_payload(outputs, vars_json, previous)
7543
+ if payload is None:
7544
+ return {"status": "unchanged", "revision": previous.get("revision", 0)}
7545
+ api_endpoint = str(outputs.get("api_endpoint", {}).get("value") or "").rstrip("/")
7546
+ api_auth_secret = str(vars_json.get("api_auth_secret") or "")
7547
+ if not api_endpoint or not api_auth_secret:
7548
+ raise RuntimeError("Native auth reconciliation requires API endpoint and service auth")
7549
+ request = urllib.request.Request(
7550
+ f"{api_endpoint}/api/auth/providers/reconcile",
7551
+ data=json.dumps(payload, separators=(",", ":")).encode("utf-8"),
7552
+ method="POST",
7553
+ headers={
7554
+ "Authorization": f"Bearer {api_auth_secret}",
7555
+ "Content-Type": "application/json",
7556
+ "Accept": "application/json",
7557
+ },
7558
+ )
7559
+ try:
7560
+ with urllib.request.urlopen(request, timeout=30) as response:
7561
+ result = json.loads(response.read().decode("utf-8") or "{}")
7562
+ except urllib.error.HTTPError as error:
7563
+ safe_body = error.read().decode("utf-8", errors="replace")[:1000]
7564
+ raise RuntimeError(
7565
+ f"Native auth metadata reconciliation failed with HTTP {error.code}: {safe_body}"
7566
+ ) from error
7567
+ run(
7568
+ [
7569
+ "aws",
7570
+ "ssm",
7571
+ "put-parameter",
7572
+ "--overwrite",
7573
+ "--type",
7574
+ "String",
7575
+ "--name",
7576
+ f"/thinkwork/{vars_json['stage']}/auth/reconciliation/state",
7577
+ "--value",
7578
+ json.dumps(next_state, separators=(",", ":")),
7579
+ ]
7580
+ )
7581
+ return {
7582
+ "status": str(result.get("status") or "applied"),
7583
+ "revision": int(result.get("revision") or payload["revision"]),
7584
+ "manifestFingerprint": payload["manifestFingerprint"],
7585
+ }
7586
+
7587
+
4214
7588
  def sync_static(outputs_path, static_files, vars_json, artifact_names=None):
4215
7589
  outputs = json.loads(outputs_path.read_text(encoding="utf-8"))
4216
7590
  artifact_names = set(artifact_names or [])
4217
7591
  syncs = [
4218
7592
  ("web", "app_bucket_name", "app_distribution_id"),
4219
- ("docs", "docs_bucket_name", "docs_distribution_id"),
4220
7593
  ]
4221
7594
  for artifact_name, bucket_output, distribution_output in syncs:
4222
7595
  if artifact_names and artifact_name not in artifact_names:
@@ -4346,7 +7719,7 @@ def write_deployment_status_pointer(status, vars_json=None, terraform_exit_code=
4346
7719
  action = os.environ.get("THINKWORK_DEPLOYMENT_ACTION")
4347
7720
  if os.environ.get("THINKWORK_MANAGED_APP_OPERATION") == "true":
4348
7721
  return
4349
- if not bucket or action not in {"deploy", "update", "web"}:
7722
+ if not bucket or action not in {"deploy", "update", "apply", "web"}:
4350
7723
  return
4351
7724
  vars_json = vars_json or {}
4352
7725
  previous = read_current_status_pointer(bucket)
@@ -4486,7 +7859,16 @@ def main():
4486
7859
  action = os.environ.get("THINKWORK_DEPLOYMENT_ACTION") or payload.get("action") or "deploy"
4487
7860
  if action == "teardown":
4488
7861
  action = "destroy"
4489
- if action not in {"deploy", "update", "destroy", "plan", "status", "web"}:
7862
+ if action not in {
7863
+ "deploy",
7864
+ "update",
7865
+ "destroy",
7866
+ "plan",
7867
+ "apply",
7868
+ "recover",
7869
+ "status",
7870
+ "web",
7871
+ }:
4490
7872
  raise RuntimeError(f"Unsupported deployment action: {action}")
4491
7873
  os.environ["THINKWORK_DEPLOYMENT_ACTION"] = action
4492
7874
  configure_managed_app_evidence_prefix(payload)
@@ -4506,16 +7888,45 @@ def main():
4506
7888
  )
4507
7889
  return 0
4508
7890
 
7891
+ if action == "recover":
7892
+ CONTROLLER_EVIDENCE = {
7893
+ "incidentRecovery": recover_tei_v380_incident(payload),
7894
+ }
7895
+ write_evidence(
7896
+ "succeeded",
7897
+ {
7898
+ "stage": "tei-e2e",
7899
+ "account_id": "637423202447",
7900
+ "region": "us-east-1",
7901
+ },
7902
+ 0,
7903
+ )
7904
+ return 0
7905
+
7906
+ terraform_phase = validate_terraform_execution_phase(payload, action)
7907
+ planned_action = requested_plan_action(payload, action)
7908
+ execution_effect_action = planned_action if terraform_phase == "apply" else action
4509
7909
  runner_secrets = secret_payload(payload)
4510
7910
  web_only = is_web_only_operation(payload, action)
7911
+ identity_operation = identity_provider_operation(payload)
4511
7912
  static_files = {}
4512
- if action in {"deploy", "update", "web"} and not is_managed_app_operation(payload):
4513
- static_files = sync_release_artifacts(
4514
- artifact_types={"static-site"} if web_only else None,
4515
- artifact_names={"web"} if web_only else None,
4516
- )
7913
+ release_request = release_sync_request(execution_effect_action, payload, identity_operation)
7914
+ control_runtime_operation = requires_agentcore_control_runtime(
7915
+ execution_effect_action, payload, identity_operation
7916
+ )
7917
+ if release_request is not None:
7918
+ static_files = sync_release_artifacts(**release_request)
7919
+ elif control_runtime_operation:
7920
+ # Plan and destroy still execute AgentCore local-exec wrappers for
7921
+ # existing resources. Download the content-addressed platform bundle
7922
+ # without publishing application artifacts so those paths never depend
7923
+ # on an ambient CodeBuild SDK.
7924
+ sync_release_artifacts(artifact_types=set())
4517
7925
  managed_app_artifacts = stage_managed_app_release_artifacts(action, payload)
4518
7926
  vars_json = write_runner_files(payload, runner_secrets)
7927
+ set_database_access_mode(vars_json)
7928
+ if control_runtime_operation:
7929
+ prepare_agentcore_control_runtime()
4519
7930
  controller_summary = controller_input_summary(payload)
4520
7931
  CONTROLLER_EVIDENCE = {
4521
7932
  "inputSummary": controller_summary,
@@ -4551,6 +7962,11 @@ def main():
4551
7962
  write_evidence("succeeded", vars_json, 0)
4552
7963
  return 0
4553
7964
 
7965
+ if identity_operation is not None:
7966
+ CONTROLLER_EVIDENCE["identityProvider"] = reconcile_identity_provider_resource(
7967
+ payload, vars_json
7968
+ )
7969
+
4554
7970
  configure_cloudflare_provider_auth(vars_json["stage"])
4555
7971
  configure_terraform_provider_mirror()
4556
7972
  run(["terraform", "init", "-backend-config=backend.hcl", "-no-color"], cwd=TF)
@@ -4564,41 +7980,60 @@ def main():
4564
7980
  )
4565
7981
  if selected.returncode != 0:
4566
7982
  run(["terraform", "workspace", "new", workspace, "-no-color"], cwd=TF)
7983
+ state_identity = terraform_state_identity()
4567
7984
  target_args = managed_app_terraform_target_args(payload)
4568
- if action == "destroy":
4569
- plan = subprocess.run(
4570
- ["terraform", "plan", "-destroy", *target_args, "-out=tfplan", "-no-color"],
4571
- cwd=TF,
4572
- text=True,
7985
+ pre_app_schema = (
7986
+ prepare_additive_schema_before_app(payload, vars_json)
7987
+ if terraform_phase in {"plan", "legacy"}
7988
+ else None
7989
+ )
7990
+ if terraform_phase == "legacy" and should_reconcile_native_auth_schema(
7991
+ terraform_phase, planned_action, payload
7992
+ ):
7993
+ CONTROLLER_EVIDENCE["cognitoSchema"] = ensure_native_auth_custom_attributes(
7994
+ current_terraform_outputs(vars_json["stage"])
4573
7995
  )
4574
- if plan.returncode == 0:
4575
- TERRAFORM_EVIDENCE["plan"] = write_terraform_plan_evidence(payload)
4576
- result = subprocess.run(
4577
- ["terraform", "apply", "-auto-approve", "-no-color", "tfplan"],
4578
- cwd=TF,
4579
- text=True,
7996
+ applied_plan = None
7997
+ if terraform_phase == "apply":
7998
+ applied_plan = validate_and_materialize_approved_plan(payload, state_identity)
7999
+ planned_action = applied_plan.get("plannedAction")
8000
+ if planned_action not in {"deploy", "update", "destroy"}:
8001
+ raise RuntimeError("Approved plan has an invalid plannedAction")
8002
+ TERRAFORM_EVIDENCE["approvedPlan"] = applied_plan
8003
+ if should_reconcile_native_auth_schema(terraform_phase, planned_action, payload):
8004
+ CONTROLLER_EVIDENCE["cognitoSchema"] = ensure_native_auth_custom_attributes(
8005
+ current_terraform_outputs(vars_json["stage"])
4580
8006
  )
4581
- else:
4582
- result = plan
4583
- else:
4584
- plan = subprocess.run(
4585
- ["terraform", "plan", *target_args, "-out=tfplan", "-no-color"],
8007
+ result = subprocess.run(
8008
+ ["terraform", "apply", "-no-color", "tfplan"],
4586
8009
  cwd=TF,
4587
8010
  text=True,
4588
8011
  )
4589
- if plan.returncode == 0:
4590
- TERRAFORM_EVIDENCE["plan"] = write_terraform_plan_evidence(payload)
4591
- if action == "plan" or plan.returncode != 0:
4592
- result = plan
4593
- else:
4594
- result = subprocess.run(
4595
- ["terraform", "apply", "-auto-approve", "-no-color", "tfplan"],
4596
- cwd=TF,
4597
- text=True,
4598
- )
8012
+ elif pre_app_schema is not None and pre_app_schema.returncode != 0:
8013
+ result = pre_app_schema
8014
+ else:
8015
+ result = execute_terraform_plan_phase(
8016
+ payload,
8017
+ state_identity,
8018
+ planned_action,
8019
+ terraform_phase,
8020
+ target_args,
8021
+ )
4599
8022
 
4600
8023
  outputs_path = TF / "outputs.json"
4601
- if result.returncode == 0 and action in {"deploy", "update"}:
8024
+ if (
8025
+ result.returncode == 0
8026
+ and terraform_phase in {"apply", "legacy"}
8027
+ and planned_action
8028
+ in {
8029
+ "deploy",
8030
+ "update",
8031
+ }
8032
+ ):
8033
+ if identity_operation is not None:
8034
+ finalized = finalize_disabled_identity_provider(payload, vars_json)
8035
+ if finalized is not None:
8036
+ CONTROLLER_EVIDENCE["identityProvider"] = finalized
4602
8037
  write_outputs_after_apply(payload, vars_json, outputs_path)
4603
8038
  if is_managed_app_operation(payload):
4604
8039
  sync_twenty_thinkwork_app(
@@ -4608,7 +8043,21 @@ def main():
4608
8043
  runner_secrets,
4609
8044
  managed_app_artifacts,
4610
8045
  )
8046
+ elif identity_operation is not None:
8047
+ CONTROLLER_EVIDENCE["authReconciliation"] = reconcile_native_auth_metadata(
8048
+ outputs_path, vars_json
8049
+ )
8050
+ promoted = promote_tenant_entra_pending_secret(payload, vars_json)
8051
+ if promoted is not None:
8052
+ CONTROLLER_EVIDENCE["identityProviderSecret"] = promoted
8053
+ write_outputs_to_ssm(outputs_path, vars_json)
8054
+ record_identity_provider_operation_state(payload, vars_json)
4611
8055
  else:
8056
+ CONTROLLER_EVIDENCE["authReconciliation"] = reconcile_native_auth_metadata(
8057
+ outputs_path, vars_json
8058
+ )
8059
+ # Destructive retirement migrations are permitted only after the
8060
+ # deployed native routes have reconciled successfully.
4612
8061
  push_database_schema(outputs_path, vars_json)
4613
8062
  ensure_first_admin(outputs_path, vars_json, payload, runner_secrets)
4614
8063
  write_outputs_to_ssm(outputs_path, vars_json)
@@ -4619,6 +8068,13 @@ def main():
4619
8068
  selected_controller_release,
4620
8069
  )
4621
8070
  sync_static(outputs_path, static_files, vars_json)
8071
+ if control_runtime_operation:
8072
+ pi_runtime_evidence = reconcile_agentcore_pi_runtime(vars_json, payload)
8073
+ if pi_runtime_evidence is not None:
8074
+ CONTROLLER_EVIDENCE["agentcorePiRuntime"] = write_json_evidence_artifact(
8075
+ "agentcore-pi-runtime.json",
8076
+ pi_runtime_evidence,
8077
+ )
4622
8078
  try:
4623
8079
  self_update_runner_script()
4624
8080
  except Exception as self_update_error: