thinkwork-cli 0.13.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (231) hide show
  1. package/README.md +3 -14
  2. package/dist/{api-client-JYZXYQJR.js → api-client-4P4JZOBX.js} +1 -1
  3. package/dist/{chunk-Z2F6RJFR.js → chunk-4OMDWYZQ.js} +4 -5
  4. package/dist/{chunk-ILZKG5WV.js → chunk-AD2NIWB5.js} +434 -69
  5. package/dist/chunk-MWQ7RO76.js +137 -0
  6. package/dist/{chunk-DVPBI5HM.js → chunk-OLUQ4U2Z.js} +26 -26
  7. package/dist/{chunk-QOERT44A.js → chunk-YQCYD6NE.js} +57 -44
  8. package/dist/cli.js +4400 -16431
  9. package/dist/commands/enterprise/templates/deploy-repo/scripts/apply-release.mjs +0 -1
  10. package/dist/commands/enterprise/templates/deploy-repo/scripts/smoke.mjs +0 -1
  11. package/dist/commands/enterprise/templates/deploy-repo/terraform/main.tf +0 -14
  12. package/dist/{dist-es-CRV46D5B.js → dist-es-2OTNTL4N.js} +4 -4
  13. package/dist/{dist-es-ENBUC3JP.js → dist-es-2XVAH4BQ.js} +2 -2
  14. package/dist/{dist-es-DCY4IC3Y.js → dist-es-3DPDUJAH.js} +2 -2
  15. package/dist/{dist-es-G7PTG75M.js → dist-es-BNZTBBVC.js} +1 -1
  16. package/dist/{dist-es-GKDBE7E6.js → dist-es-CD2RBDAC.js} +1 -1
  17. package/dist/{dist-es-R7FGL2IB.js → dist-es-I4FW57PS.js} +1 -1
  18. package/dist/{dist-es-USORBXZ5.js → dist-es-KVFWJNDY.js} +4 -4
  19. package/dist/{dist-es-HY3535OI.js → dist-es-PS73DWNL.js} +2 -2
  20. package/dist/{dist-es-LDE437AM.js → dist-es-S3NZQFGT.js} +1 -1
  21. package/dist/{dist-es-G54LBMRN.js → dist-es-SGCA4P32.js} +1 -1
  22. package/dist/{dist-es-Q57SZXZ7.js → dist-es-WBM6J4RA.js} +2 -2
  23. package/dist/{dist-es-P26FWFLT.js → dist-es-ZGVADYIF.js} +8 -8
  24. package/dist/drizzle/0227_analyst_reader_role.sql +17 -26
  25. package/dist/drizzle/0230_analyst_rls.sql +28 -45
  26. package/dist/drizzle/0261_native_auth_control_plane.sql +7 -3
  27. package/dist/drizzle/0270_tool_execution_ledger.sql +115 -0
  28. package/dist/drizzle/0271_ontology_twin_declarations.sql +47 -0
  29. package/dist/drizzle/0272_identity_graph_projection_cursor.sql +23 -0
  30. package/dist/drizzle/0273_twin_materialization_suggestions.sql +29 -0
  31. package/dist/drizzle/0274_tenant_mcp_twin_keys.sql +59 -0
  32. package/dist/drizzle/0275_identity_bulk_rebuild_fence.sql +30 -0
  33. package/dist/drizzle/0276_tool_execution_ledger_allow_cascade_delete.sql +18 -0
  34. package/dist/drizzle/0277_knowledge_base_sources.sql +107 -0
  35. package/dist/drizzle/0278_kb_page_transcription.sql +55 -0
  36. package/dist/drizzle/0279_mcp_servers_off_plugin_provenance.sql +93 -0
  37. package/dist/drizzle/0280_work_item_external_refs_drop_plane_provider.sql +46 -0
  38. package/dist/drizzle/0281_tenant_mcp_twin_keys_expiry_suffix.sql +9 -0
  39. package/dist/drizzle/0282_tenant_mcp_twin_keys_grants.sql +24 -0
  40. package/dist/drizzle/0283_agents_agentcore_runtime_dispatch.sql +13 -0
  41. package/dist/drizzle/0284_user_brain_claims.sql +69 -0
  42. package/dist/drizzle/0285_tenant_mcp_twin_keys_trusted_subsystem.sql +30 -0
  43. package/dist/drizzle/0286_brain_analytics_key.sql +26 -0
  44. package/dist/drizzle/0287_brain_subject.sql +25 -0
  45. package/dist/drizzle/0288_multi_lane_identities.sql +25 -0
  46. package/dist/{loadSso-DJVB7BXE.js → loadSso-ETCXTR4I.js} +2 -2
  47. package/dist/{loadSso-JAMNA3WY.js → loadSso-UBD7KUHE.js} +2 -2
  48. package/dist/plugins/README.md +2 -2
  49. package/dist/plugins/catalog/package.json +0 -3
  50. package/dist/plugins/catalog/src/__tests__/build-catalog.test.ts +6 -3
  51. package/dist/plugins/catalog/src/__tests__/catalog.test.ts +21 -21
  52. package/dist/plugins/catalog/src/__tests__/contracts.test.ts +102 -25
  53. package/dist/plugins/catalog/src/__tests__/plugin-package.test.ts +7 -6
  54. package/dist/plugins/catalog/src/__tests__/plugin-registry.test.ts +0 -15
  55. package/dist/plugins/catalog/src/contracts.ts +15 -0
  56. package/dist/plugins/catalog/src/registry/generated-first-party.ts +0 -18
  57. package/dist/plugins/n8n/test/integrated-app-contract.test.ts +1 -1
  58. package/dist/plugins/twenty/package.json +2 -1
  59. package/dist/plugins/twenty/scripts/LAMBDA_DEPLOY.md +114 -0
  60. package/dist/plugins/twenty/scripts/build-lambda.sh +29 -0
  61. package/dist/plugins/twenty/scripts/configure-product-views.ts +195 -0
  62. package/dist/plugins/twenty/scripts/lambda-handler.ts +49 -0
  63. package/dist/plugins/twenty/scripts/lib/__tests__/load-attachments.test.ts +38 -0
  64. package/dist/plugins/twenty/scripts/lib/__tests__/load-records.test.ts +145 -2
  65. package/dist/plugins/twenty/scripts/lib/__tests__/mappers.test.ts +397 -151
  66. package/dist/plugins/twenty/scripts/lib/__tests__/members-ensure.test.ts +28 -129
  67. package/dist/plugins/twenty/scripts/lib/__tests__/provision-members-db.test.ts +10 -0
  68. package/dist/plugins/twenty/scripts/lib/__tests__/schema-ensure.test.ts +13 -8
  69. package/dist/plugins/twenty/scripts/lib/lastmile-reader.ts +222 -2
  70. package/dist/plugins/twenty/scripts/lib/load-attachments.ts +29 -0
  71. package/dist/plugins/twenty/scripts/lib/load-records.ts +160 -43
  72. package/dist/plugins/twenty/scripts/lib/mappers.ts +368 -152
  73. package/dist/plugins/twenty/scripts/lib/members-ensure.ts +43 -268
  74. package/dist/plugins/twenty/scripts/lib/provision-members-db.ts +57 -0
  75. package/dist/plugins/twenty/scripts/lib/schema-ensure.ts +136 -31
  76. package/dist/plugins/twenty/scripts/migrate-lastmile.ts +254 -152
  77. package/dist/plugins/twenty/scripts/provision-twenty-members.ts +44 -1
  78. package/dist/plugins/twenty/scripts/purge-lastmile-import.ts +178 -0
  79. package/dist/plugins/twenty/scripts/restructure-products.ts +312 -0
  80. package/dist/plugins/twenty/scripts/retag-stage-values.ts +362 -0
  81. package/dist/plugins/twenty/test/manifest.test.ts +2 -4
  82. package/dist/scripts/post-deploy.sh +79 -1
  83. package/dist/{signin-UGDWTGS4.js → signin-5QZNRSOO.js} +2 -2
  84. package/dist/{sso-oidc-Q4VXANXY.js → sso-oidc-XK673P54.js} +2 -2
  85. package/dist/{sts-CWLSE3GH.js → sts-OXI2UALF.js} +6 -134
  86. package/dist/{sts-DNZLNVOV.js → sts-SQW4NBK3.js} +6 -134
  87. package/dist/terraform/examples/greenfield/main.tf +233 -175
  88. package/dist/terraform/examples/greenfield/terraform.tfvars.example +12 -8
  89. package/dist/terraform/modules/app/agentcore-admin/main.tf +47 -0
  90. package/dist/terraform/modules/app/agentcore-code-interpreter/README.md +39 -47
  91. package/dist/terraform/modules/app/agentcore-code-interpreter/main.tf +13 -66
  92. package/dist/terraform/modules/app/agentcore-identity/main.tf +106 -26
  93. package/dist/terraform/modules/app/agentcore-identity/outputs.tf +6 -6
  94. package/dist/terraform/modules/app/agentcore-identity/scripts/bootstrap_twenty_oauth_client.sh +216 -99
  95. package/dist/terraform/modules/app/agentcore-identity/scripts/delete_identity.sh +6 -9
  96. package/dist/terraform/modules/app/agentcore-identity/scripts/delete_workload_identity.sh +19 -0
  97. package/dist/terraform/modules/app/agentcore-identity/scripts/ensure_workload_identity.sh +29 -0
  98. package/dist/terraform/modules/app/agentcore-identity/scripts/read_identity.sh +22 -7
  99. package/dist/terraform/modules/app/agentcore-identity/scripts/reconcile_twenty_identity.sh +17 -46
  100. package/dist/terraform/modules/app/agentcore-identity/scripts/reconcile_twenty_provider.mjs +65 -3
  101. package/dist/terraform/modules/app/agentcore-identity/tests/tainted-marker/after.tf +15 -0
  102. package/dist/terraform/modules/app/agentcore-identity/tests/tainted-marker/before.tf +7 -0
  103. package/dist/terraform/modules/app/agentcore-identity/tests/tainted-marker/identity-after.tf +8 -0
  104. package/dist/terraform/modules/app/agentcore-identity/tests/tainted-marker/identity-before.tf +9 -0
  105. package/dist/terraform/modules/app/agentcore-identity/tests/tainted-marker/thinkwork-after.tf +3 -0
  106. package/dist/terraform/modules/app/agentcore-identity/tests/tainted-marker/thinkwork-before.tf +3 -0
  107. package/dist/terraform/modules/app/agentcore-identity/variables.tf +6 -0
  108. package/dist/terraform/modules/app/agentcore-memory/README.md +46 -9
  109. package/dist/terraform/modules/app/agentcore-memory/main.tf +27 -10
  110. package/dist/terraform/modules/app/agentcore-memory/scripts/create_or_find_memory.sh +156 -24
  111. package/dist/terraform/modules/app/agentcore-pi/README.md +1 -3
  112. package/dist/terraform/modules/app/agentcore-pi/main.tf +42 -52
  113. package/dist/terraform/modules/app/agentcore-pi/outputs.tf +3 -3
  114. package/dist/terraform/modules/app/agentcore-pi/scripts/reconcile_pi_runtime.mjs +175 -0
  115. package/dist/terraform/modules/app/agentcore-pi/variables.tf +35 -28
  116. package/dist/terraform/modules/app/agentcore-platform/main.tf +68 -0
  117. package/dist/terraform/modules/app/agentcore-runtime/main.tf +4 -20
  118. package/dist/terraform/modules/app/capability-broker/main.tf +2 -2
  119. package/dist/terraform/modules/app/capability-broker/variables.tf +11 -0
  120. package/dist/terraform/modules/app/deployment-control-plane/README.md +264 -0
  121. package/dist/terraform/modules/app/deployment-control-plane/buildspec.yml +4 -1
  122. package/dist/terraform/modules/app/deployment-control-plane/main.tf +94 -0
  123. package/dist/terraform/modules/app/deployment-control-plane/outputs.tf +5 -0
  124. package/dist/terraform/modules/app/deployment-control-plane/runner.py +2641 -124
  125. package/dist/terraform/modules/app/deployment-control-plane/test_runner_bundle.py +2034 -154
  126. package/dist/terraform/modules/app/deployment-control-plane/variables.tf +33 -0
  127. package/dist/terraform/modules/app/lambda-api/auth-state.tf +72 -0
  128. package/dist/terraform/modules/app/lambda-api/chat-latency-observability.tf +466 -0
  129. package/dist/terraform/modules/app/lambda-api/dispatch-dlq.tf +55 -0
  130. package/dist/terraform/modules/app/lambda-api/handlers.tf +411 -723
  131. package/dist/terraform/modules/app/lambda-api/iam-grouped.tf +262 -198
  132. package/dist/terraform/modules/app/lambda-api/main.tf +51 -13
  133. package/dist/terraform/modules/app/lambda-api/mcp-oauth.tf +4 -181
  134. package/dist/terraform/modules/app/lambda-api/memory-alarms.tf +3 -28
  135. package/dist/terraform/modules/app/lambda-api/oauth-secrets.tf +1 -2
  136. package/dist/terraform/modules/app/lambda-api/outputs.tf +17 -45
  137. package/dist/terraform/modules/app/lambda-api/turn-assertion.tf +45 -0
  138. package/dist/terraform/modules/app/lambda-api/variables.tf +219 -126
  139. package/dist/{plugins/n8n/terraform → terraform/modules/app}/n8n/README.md +1 -1
  140. package/dist/{plugins/n8n/terraform → terraform/modules/app}/n8n/main.tf +18 -18
  141. package/dist/terraform/modules/app/sandbox-log-scrubber/README.md +11 -10
  142. package/dist/terraform/modules/app/sandbox-log-scrubber/main.tf +5 -5
  143. package/dist/terraform/modules/app/static-site/main.tf +1 -1
  144. package/dist/terraform/modules/app/www-dns/main.tf +9 -27
  145. package/dist/terraform/modules/app/www-dns/variables.tf +4 -10
  146. package/dist/terraform/modules/data/aurora-postgres/README.md +34 -0
  147. package/dist/terraform/modules/data/aurora-postgres/alarms.tf +82 -0
  148. package/dist/terraform/modules/data/aurora-postgres/main.tf +27 -57
  149. package/dist/terraform/modules/data/aurora-postgres/outputs.tf +0 -9
  150. package/dist/terraform/modules/data/aurora-postgres/variables.tf +41 -0
  151. package/dist/terraform/modules/data/s3-buckets/main.tf +5 -6
  152. package/dist/terraform/modules/foundation/cognito/main.tf +81 -8
  153. package/dist/terraform/modules/foundation/cognito/outputs.tf +5 -0
  154. package/dist/terraform/modules/foundation/cognito/variables.tf +24 -12
  155. package/dist/terraform/modules/thinkwork/README.md +5 -1
  156. package/dist/terraform/modules/thinkwork/main.tf +376 -374
  157. package/dist/terraform/modules/thinkwork/outputs.tf +12 -158
  158. package/dist/terraform/modules/thinkwork/variables.tf +246 -89
  159. package/dist/workspace-defaults/files/INSTRUCTIONS.md +9 -11
  160. package/dist/workspace-defaults/files/MEMORY_GUIDE.md +38 -35
  161. package/dist/workspace-defaults/files/ROUTER.md +1 -1
  162. package/dist/workspace-defaults/files/TOOLS.md +9 -39
  163. package/dist/workspace-defaults/files/agents/analyst/INSTRUCTIONS.md +6 -3
  164. package/dist/workspace-defaults/files/catalog-skills/brain-brief-builder/SKILL.md +165 -0
  165. package/dist/workspace-defaults/files/catalog-skills/n8n-workflow-operator/SKILL.md +75 -0
  166. package/dist/workspace-defaults/files/catalog-skills/n8n-workflow-operator/references/mcp-tooling.md +40 -0
  167. package/dist/workspace-defaults/files/catalog-skills/n8n-workflow-operator/references/validation-and-handoff.md +40 -0
  168. package/dist/workspace-defaults/files/catalog-skills/n8n-workflow-operator/references/workflow-authoring.md +46 -0
  169. package/dist/workspace-defaults/files/skills/artifact-builder/references/crm-dashboard.md +1 -1
  170. package/dist/workspace-defaults/files/skills/document-composer/SKILL.md +22 -3
  171. package/package.json +1 -1
  172. package/dist/drizzle/0261_managed_harness_multiplayer.sql +0 -313
  173. package/dist/drizzle/0262_parallel_harness_threads.sql +0 -9
  174. package/dist/drizzle/0264_harness_tool_execution_ledger.sql +0 -113
  175. package/dist/plugins/company-data/README.md +0 -49
  176. package/dist/plugins/company-data/package.json +0 -21
  177. package/dist/plugins/company-data/src/index.ts +0 -30
  178. package/dist/plugins/company-data/src/manifest.ts +0 -26
  179. package/dist/plugins/company-data/test/manifest.test.ts +0 -116
  180. package/dist/plugins/company-data/tsconfig.json +0 -8
  181. package/dist/plugins/company-etl/README.md +0 -46
  182. package/dist/plugins/company-etl/package.json +0 -21
  183. package/dist/plugins/company-etl/src/index.ts +0 -29
  184. package/dist/plugins/company-etl/src/manifest.ts +0 -26
  185. package/dist/plugins/company-etl/test/manifest.test.ts +0 -108
  186. package/dist/plugins/company-etl/tsconfig.json +0 -8
  187. package/dist/plugins/lastmile/README.md +0 -50
  188. package/dist/plugins/lastmile/package.json +0 -23
  189. package/dist/plugins/lastmile/smoke/lastmile-plugin-smoke.mjs +0 -661
  190. package/dist/plugins/lastmile/src/api/tasks-adapter.ts +0 -572
  191. package/dist/plugins/lastmile/src/discovery.fixture.ts +0 -64
  192. package/dist/plugins/lastmile/src/index.ts +0 -46
  193. package/dist/plugins/lastmile/src/manifest.ts +0 -139
  194. package/dist/plugins/lastmile/test/api/tasks-adapter.test.ts +0 -266
  195. package/dist/plugins/lastmile/test/discovery.test.ts +0 -88
  196. package/dist/plugins/lastmile/tsconfig.json +0 -8
  197. package/dist/plugins/n8n/smoke/n8n-managed-app-smoke.mjs +0 -409
  198. package/dist/plugins/twenty/smoke/twenty-managed-app-smoke.mjs +0 -359
  199. package/dist/terraform/modules/app/agentcore-code-interpreter/Dockerfile.sandbox-base +0 -61
  200. package/dist/terraform/modules/app/agentcore-code-interpreter/sandbox/sitecustomize.py +0 -227
  201. package/dist/terraform/modules/app/agentcore-code-interpreter/sandbox/test_sitecustomize.py +0 -277
  202. package/dist/terraform/modules/app/agentcore-code-interpreter/scripts/build_and_push_sandbox_base.sh +0 -70
  203. package/dist/terraform/modules/app/agentcore-gateway/main.tf +0 -305
  204. package/dist/terraform/modules/app/agentcore-gateway/outputs.tf +0 -44
  205. package/dist/terraform/modules/app/agentcore-gateway/scripts/delete_gateway.sh +0 -93
  206. package/dist/terraform/modules/app/agentcore-gateway/scripts/read_gateway.sh +0 -33
  207. package/dist/terraform/modules/app/agentcore-gateway/scripts/reconcile_gateway.sh +0 -543
  208. package/dist/terraform/modules/app/agentcore-gateway/variables.tf +0 -70
  209. package/dist/terraform/modules/app/agentcore-harness/main.tf +0 -330
  210. package/dist/terraform/modules/app/agentcore-harness/outputs.tf +0 -72
  211. package/dist/terraform/modules/app/agentcore-harness/scripts/delete_harness.sh +0 -5
  212. package/dist/terraform/modules/app/agentcore-harness/scripts/harness-lifecycle.mjs +0 -430
  213. package/dist/terraform/modules/app/agentcore-harness/scripts/harness-readback.mjs +0 -180
  214. package/dist/terraform/modules/app/agentcore-harness/scripts/harness-tool-contract.mjs +0 -193
  215. package/dist/terraform/modules/app/agentcore-harness/scripts/prune_harness.sh +0 -5
  216. package/dist/terraform/modules/app/agentcore-harness/scripts/read_harness.sh +0 -5
  217. package/dist/terraform/modules/app/agentcore-harness/scripts/reconcile_harness.sh +0 -5
  218. package/dist/terraform/modules/app/agentcore-harness/variables.tf +0 -96
  219. package/dist/terraform/modules/app/hindsight-memory/README.md +0 -89
  220. package/dist/terraform/modules/app/hindsight-memory/main.tf +0 -411
  221. package/dist/terraform/modules/app/lambda-api/agentcore-harness-alarms.tf +0 -88
  222. package/dist/terraform/modules/app/lambda-api/agentcore-identity-alarms.tf +0 -91
  223. package/dist/terraform/modules/data/bedrock-knowledge-base/main.tf +0 -102
  224. /package/dist/{plugins/n8n/terraform → terraform/modules/app}/n8n/outputs.tf +0 -0
  225. /package/dist/{plugins/n8n/terraform → terraform/modules/app}/n8n/scripts/sync-database.py +0 -0
  226. /package/dist/{plugins/n8n/terraform → terraform/modules/app}/n8n/scripts/test_sync_database.py +0 -0
  227. /package/dist/{plugins/n8n/terraform → terraform/modules/app}/n8n/variables.tf +0 -0
  228. /package/dist/{plugins/twenty/terraform → terraform/modules/app}/twenty/README.md +0 -0
  229. /package/dist/{plugins/twenty/terraform → terraform/modules/app}/twenty/main.tf +0 -0
  230. /package/dist/{plugins/twenty/terraform → terraform/modules/app}/twenty/outputs.tf +0 -0
  231. /package/dist/{plugins/twenty/terraform → terraform/modules/app}/twenty/variables.tf +0 -0
@@ -7,6 +7,7 @@ import re
7
7
  import secrets
8
8
  import subprocess
9
9
  import tarfile
10
+ import time
10
11
  import urllib.error
11
12
  import urllib.parse
12
13
  import urllib.request
@@ -51,6 +52,59 @@ MIGRATION_MARKER_KINDS = [
51
52
  ("-- creates-constraint:", "constraint"),
52
53
  ("-- creates:", "object"),
53
54
  ]
55
+ NATIVE_AUTH_CUSTOM_ATTRIBUTES = [
56
+ {
57
+ "AttributeDataType": "String",
58
+ "DeveloperOnlyAttribute": False,
59
+ "Mutable": True,
60
+ "Name": "entra_tenant_id",
61
+ "Required": False,
62
+ "StringAttributeConstraints": {"MinLength": "0", "MaxLength": "36"},
63
+ },
64
+ {
65
+ "AttributeDataType": "String",
66
+ "DeveloperOnlyAttribute": False,
67
+ "Mutable": True,
68
+ "Name": "entra_object_id",
69
+ "Required": False,
70
+ "StringAttributeConstraints": {"MinLength": "0", "MaxLength": "36"},
71
+ },
72
+ ]
73
+ AGENTCORE_CONTROL_SDK_VERSION = "3.1089.0"
74
+ TERRAFORM_PLAN_APPROVAL_CONTRACT = "thinkwork.terraform.saved-plan-approval.v1"
75
+ TERRAFORM_PLAN_DESCRIPTOR_NAME = "terraform-plan-approval.json"
76
+ TERRAFORM_SAVED_PLAN_NAME = "terraform-plan.bin"
77
+ TEI_V380_RECOVERY_CONTRACT = "thinkwork.incident.tei-v380-proof-plane-recovery.v1"
78
+ TEI_V380_RECOVERY_INCIDENT_ID = "tei-v380-proof-plane-partial-apply"
79
+ TEI_V380_REJECTED_PLAN_SHA256 = "ba7b0ffa099f725b4f32e394446ee43693fe3dd4b7a9eb639d47eb833d40db84"
80
+ TEI_V380_STATE_LINEAGE = "61f4093e-41da-ac29-852a-267bc853e24f"
81
+ TEI_V380_STATE_SERIAL = 819
82
+ TEI_V380_STATE_SHA256 = "cc99973e7b80e1a0c6e1097d889e006e5e2610e236ed730a02ca5f3457b99d7f"
83
+ TEI_V380_STATE_BUCKET = "tei-thinkwork-terraform-state"
84
+ TEI_V380_STATE_KEY = "env:/tei-e2e/thinkwork/tei-e2e/terraform.tfstate"
85
+ TEI_V380_LOCK_TABLE = "tei-thinkwork-terraform-locks"
86
+ TEI_V380_LOCK_KEY = f"{TEI_V380_STATE_BUCKET}/{TEI_V380_STATE_KEY}"
87
+ TEI_V380_LOCK_ID = "04b3e9a1-3de6-c668-87e7-43d94280605e"
88
+ TEI_V380_STATE_DIGEST = "7fe214eeaae491f5c1d8d792bddbbb19"
89
+ TEI_V380_SECRET_ARN = (
90
+ "arn:aws:secretsmanager:us-east-1:637423202447:secret:"
91
+ "thinkwork/tei-e2e/agentcore-identity/twenty-crm-oauth-client-7OeAlG"
92
+ )
93
+ TEI_V380_SECRET_NAME = "thinkwork/tei-e2e/agentcore-identity/twenty-crm-oauth-client"
94
+ TEI_V380_SECRET_ADDRESS = (
95
+ "module.thinkwork.module.agentcore_proof_identity."
96
+ "aws_secretsmanager_secret.twenty_oauth_client[0]"
97
+ )
98
+ TEI_V380_KMS_KEY_ID = "8798ab48-cd60-477d-a96e-c2d56bae7b40"
99
+ TEI_V380_KMS_KEY_ARN = "arn:aws:kms:us-east-1:637423202447:key/8798ab48-cd60-477d-a96e-c2d56bae7b40"
100
+ TEI_V380_KMS_KEY_ADDRESS = 'module.thinkwork.module.api.aws_kms_key.agentcore_turn_assertion["v1"]'
101
+ TEI_V380_KMS_ALIAS = "alias/thinkwork-tei-e2e-agentcore-turn-assertion-v1"
102
+ TEI_V380_KMS_ALIAS_ADDRESS = (
103
+ 'module.thinkwork.module.api.aws_kms_alias.agentcore_turn_assertion["v1"]'
104
+ )
105
+ TEI_V380_OWNER_ALLOWLIST = (
106
+ "0ea2f626-8b40-43ca-b7f1-3601b2a6bef8,f498f488-70a1-709b-24d7-f895f2164301"
107
+ )
54
108
 
55
109
 
56
110
  def run(args, **kwargs):
@@ -69,6 +123,10 @@ def sha256_file(path):
69
123
  return digest.hexdigest()
70
124
 
71
125
 
126
+ def sha256_bytes(value):
127
+ return hashlib.sha256(value).hexdigest()
128
+
129
+
72
130
  def read_json_env(name, default):
73
131
  value = os.environ.get(name)
74
132
  if not value:
@@ -385,6 +443,118 @@ def safe_extract_tar_file(archive_path, destination):
385
443
  tar.extractall(destination, members=members)
386
444
 
387
445
 
446
+ def validate_agentcore_node_runtime(version):
447
+ normalized = str(version or "").strip().removeprefix("v")
448
+ try:
449
+ major = int(normalized.split(".", 1)[0])
450
+ except (TypeError, ValueError):
451
+ raise RuntimeError(
452
+ f"Cannot determine Node.js version for AgentCore runtime: {version}"
453
+ ) from None
454
+ if major < 22:
455
+ raise RuntimeError(
456
+ "AgentCore control runtime requires Node.js 22 or newer; "
457
+ f"deployment runner is using {version}"
458
+ )
459
+ return f"v{normalized}"
460
+
461
+
462
+ def prepare_agentcore_control_runtime():
463
+ global RELEASE_EVIDENCE
464
+ node_version = validate_agentcore_node_runtime(output(["node", "--version"]))
465
+ runner_bundle = bundle_extract_dir({"name": "platform"}) / "runner"
466
+ manifest_path = runner_bundle / "agentcore-control-runtime.json"
467
+ if not manifest_path.is_file():
468
+ raise RuntimeError("Release bundle is missing the AgentCore control runtime manifest")
469
+
470
+ runtime_manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
471
+ expected_sdk = runtime_manifest.get("sdk") or {}
472
+ if (
473
+ expected_sdk.get("package") != "@aws-sdk/client-bedrock-agentcore-control"
474
+ or expected_sdk.get("version") != AGENTCORE_CONTROL_SDK_VERSION
475
+ ):
476
+ raise RuntimeError("Release does not pin the required AgentCore control SDK version")
477
+
478
+ runtime_dir = safe_join(
479
+ runner_bundle,
480
+ str(runtime_manifest.get("directory") or "agentcore-control-runtime"),
481
+ )
482
+ if not runtime_dir.is_dir():
483
+ raise RuntimeError("Release bundle has no AgentCore control runtime directory")
484
+ runtime_files = runtime_manifest.get("files") or []
485
+ if not runtime_files:
486
+ raise RuntimeError("AgentCore control runtime manifest has no files")
487
+ runtime_paths = [str(item.get("path") or "") for item in runtime_files]
488
+ # THINK-324: the managed harness is retired — harness-lifecycle.js is
489
+ # absent from new releases but tolerated in legacy bundles.
490
+ required_entrypoints = {
491
+ "preflight.js",
492
+ "reconcile_twenty_provider.js",
493
+ }
494
+ required_files = required_entrypoints | {"package.json"}
495
+ if len(runtime_paths) != len(set(runtime_paths)) or not required_files.issubset(runtime_paths):
496
+ raise RuntimeError("AgentCore control runtime manifest has an invalid file set")
497
+ for runtime_file in runtime_files:
498
+ file_path = safe_join(runtime_dir, str(runtime_file.get("path") or ""))
499
+ if not file_path.is_file() or sha256_file(file_path) != runtime_file.get("sha256"):
500
+ raise RuntimeError(
501
+ f"AgentCore control runtime file digest mismatch: {runtime_file.get('path')}"
502
+ )
503
+ actual_runtime_paths = {
504
+ str(path.relative_to(runtime_dir)) for path in runtime_dir.rglob("*") if path.is_file()
505
+ }
506
+ if actual_runtime_paths != set(runtime_paths):
507
+ raise RuntimeError("AgentCore control runtime contains unmanifested or missing files")
508
+
509
+ module_boundary = json.loads((runtime_dir / "package.json").read_text(encoding="utf-8"))
510
+ if module_boundary != {"type": "module"}:
511
+ raise RuntimeError("AgentCore control runtime has an invalid ESM module boundary")
512
+
513
+ preflight = json.loads(output(["node", str(runtime_dir / "preflight.js")]))
514
+ if preflight != expected_sdk:
515
+ raise RuntimeError("AgentCore control runtime preflight differs from its manifest")
516
+ # reconcile_pi_runtime.js (THINK-584 U5) is preflighted only when present:
517
+ # a newer seeded runner.py may run against a pre-U5 release bundle (the
518
+ # runner-py-bootstrap-skew trap), and the Pi runtime reconcile step skips
519
+ # loudly in that case rather than failing bundle validation here.
520
+ optional_entrypoints = ("reconcile_pi_runtime.js",)
521
+ entrypoint_preflights = [
522
+ json.loads(
523
+ output(
524
+ [
525
+ "node",
526
+ str(runtime_dir / entrypoint),
527
+ "--runtime-preflight",
528
+ ]
529
+ )
530
+ )
531
+ for entrypoint in ("reconcile_twenty_provider.js", *optional_entrypoints)
532
+ if entrypoint not in optional_entrypoints or (runtime_dir / entrypoint).is_file()
533
+ ]
534
+ if any(not item.get("sdkImportReady") for item in entrypoint_preflights):
535
+ raise RuntimeError("An AgentCore control runtime entrypoint cannot import its SDK")
536
+ os.environ["THINKWORK_AGENTCORE_CONTROL_RUNTIME_DIR"] = str(runtime_dir)
537
+ evidence = {
538
+ "manifestSha256": sha256_file(manifest_path),
539
+ "sdkPackage": preflight["package"],
540
+ "sdkVersion": preflight["version"],
541
+ "nodeVersion": node_version,
542
+ "bundledEntrypoints": sorted(
543
+ item["path"]
544
+ for item in runtime_files
545
+ if item.get("path") in required_entrypoints or item.get("path") in optional_entrypoints
546
+ ),
547
+ "bundledRuntimeVerified": True,
548
+ "entrypointPreflights": entrypoint_preflights,
549
+ }
550
+ RELEASE_EVIDENCE["agentCoreControlRuntime"] = evidence
551
+ print(
552
+ "[runner] AgentCore control runtime ready: "
553
+ f"sdk={evidence['sdkVersion']} manifest={evidence['manifestSha256']}"
554
+ )
555
+ return evidence
556
+
557
+
388
558
  def release_artifacts_by_name(manifest):
389
559
  return {
390
560
  artifact.get("name"): artifact
@@ -504,6 +674,11 @@ def write_json_evidence_artifact(name, payload):
504
674
  }
505
675
 
506
676
 
677
+ # vars_json keys the runner consumes itself; they are not variables of the
678
+ # generated Terraform root and are left out of terraform.auto.tfvars.json.
679
+ RUNNER_ONLY_VARS = frozenset({"db_access"})
680
+
681
+
507
682
  def redacted_tfvars(vars_json):
508
683
  redacted = dict(vars_json)
509
684
  for key in [
@@ -653,6 +828,259 @@ def terraform_plan_summary(plan_json):
653
828
  }
654
829
 
655
830
 
831
+ def terraform_plan_delete_addresses(plan_json):
832
+ return [
833
+ change.get("address")
834
+ for change in plan_json.get("resource_changes", [])
835
+ if "delete" in (change.get("change", {}).get("actions") or [])
836
+ ]
837
+
838
+
839
+ def deployment_config_for_approval(payload):
840
+ """Return the immutable deployment intent shared by plan and apply phases.
841
+
842
+ Controller/session routing fields necessarily change between executions.
843
+ Everything that can influence the generated Terraform configuration stays
844
+ in this projection, including release pins and preserved configuration.
845
+ """
846
+ reviewed = json.loads(json.dumps(payload))
847
+ for key in [
848
+ "action",
849
+ "phase",
850
+ "sessionId",
851
+ "session",
852
+ "evidence",
853
+ "approvedPlan",
854
+ ]:
855
+ reviewed.pop(key, None)
856
+ operation = reviewed.get("operation")
857
+ if isinstance(operation, dict):
858
+ for key in ["action", "plan", "apply", "approvedPlan"]:
859
+ operation.pop(key, None)
860
+ return reviewed
861
+
862
+
863
+ def deployment_config_sha256(payload):
864
+ return sha256_bytes(stable_json_bytes(deployment_config_for_approval(payload)))
865
+
866
+
867
+ def controller_input_sha256(payload):
868
+ return sha256_bytes(stable_json_bytes(payload))
869
+
870
+
871
+ def raw_controller_input_sha256(payload):
872
+ raw = os.environ.get("THINKWORK_DEPLOYMENT_INPUT")
873
+ if raw:
874
+ return sha256_bytes(raw.encode("utf-8"))
875
+ return controller_input_sha256(payload)
876
+
877
+
878
+ def requested_plan_action(payload, action):
879
+ if action in {"deploy", "update", "destroy"}:
880
+ return action
881
+ operation = payload.get("operation")
882
+ if isinstance(operation, dict):
883
+ candidate = operation.get("targetAction") or operation.get("requestedAction")
884
+ if candidate in {"deploy", "update", "destroy"}:
885
+ return candidate
886
+ if action == "plan":
887
+ return "update"
888
+ return action
889
+
890
+
891
+ def validate_terraform_execution_phase(payload, action):
892
+ """Select the explicit saved-plan protocol or the compatible legacy path.
893
+
894
+ The incident recovery always uses separate ``plan`` and ``apply``
895
+ executions. Existing callers still send ``deploy``/``update``/``destroy``
896
+ while their API and CLI approval surfaces are migrated, so those actions
897
+ retain their established combined behavior instead of becoming an
898
+ accidental platform-wide outage.
899
+ """
900
+ operation_value = payload.get("operation")
901
+ operation_is_structured = isinstance(operation_value, dict)
902
+ operation = operation_value
903
+ if not operation_is_structured:
904
+ operation = {}
905
+ requested_apply = operation.get("apply")
906
+ requested_plan = operation.get("plan")
907
+
908
+ if action == "apply":
909
+ if requested_apply is not True or requested_plan is not False:
910
+ raise RuntimeError(
911
+ "Saved-plan apply requires operation.apply=true and operation.plan=false"
912
+ )
913
+ approved = payload.get("approvedPlan")
914
+ if not isinstance(approved, dict):
915
+ raise RuntimeError("Saved-plan apply requires approvedPlan")
916
+ return "apply"
917
+
918
+ if action in {"deploy", "update", "destroy"}:
919
+ return "legacy"
920
+
921
+ if action == "plan":
922
+ if not operation_is_structured:
923
+ # Managed-application jobs predate the structured foundation
924
+ # operation object and already use action=plan as a strict
925
+ # plan-only phase. Preserve that safe contract.
926
+ return "plan"
927
+ if requested_apply is not False:
928
+ raise RuntimeError(
929
+ "Terraform planning never implies approval: operation.apply must be false; "
930
+ "apply the accepted saved plan in a separate action=apply execution"
931
+ )
932
+ if requested_plan is not True:
933
+ raise RuntimeError("Terraform plan phase requires operation.plan=true")
934
+ return "plan"
935
+
936
+ return "none"
937
+
938
+
939
+ def canonical_state_identity_bytes(state):
940
+ """`terraform state pull` re-renders the state document on every run and
941
+ serializes the render-only check_results block in nondeterministic map
942
+ order, so identical remote state produces different raw bytes across
943
+ runs. Hash a canonical JSON form without check_results; lineage/serial
944
+ stay the authoritative change signal for the infrastructure content."""
945
+ comparable = {key: value for key, value in state.items() if key != "check_results"}
946
+ return json.dumps(comparable, sort_keys=True, separators=(",", ":")).encode("utf-8")
947
+
948
+
949
+ def terraform_state_identity():
950
+ raw = output(["terraform", "state", "pull"], cwd=TF)
951
+ state = json.loads(raw or "{}")
952
+ lineage = state.get("lineage")
953
+ serial = state.get("serial")
954
+ if not isinstance(lineage, str) or not lineage:
955
+ raise RuntimeError("Terraform state has no lineage; refusing saved-plan operation")
956
+ if not isinstance(serial, int) or serial < 0:
957
+ raise RuntimeError("Terraform state has no valid serial; refusing saved-plan operation")
958
+ return {
959
+ "lineage": lineage,
960
+ "serial": serial,
961
+ "sha256": sha256_bytes(canonical_state_identity_bytes(state)),
962
+ "terraformVersion": state.get("terraform_version"),
963
+ }
964
+
965
+
966
+ def _validated_evidence_s3_uri(uri, label):
967
+ parsed = urllib.parse.urlparse(require_string(uri, label))
968
+ expected_bucket = os.environ.get("THINKWORK_EVIDENCE_BUCKET")
969
+ if parsed.scheme != "s3" or not parsed.netloc or not parsed.path.lstrip("/"):
970
+ raise RuntimeError(f"{label} must be an s3:// evidence URI")
971
+ if expected_bucket and parsed.netloc != expected_bucket:
972
+ raise RuntimeError(f"{label} must use the configured deployment evidence bucket")
973
+ return uri
974
+
975
+
976
+ def _download_approved_artifact(uri, destination, expected_sha256, label):
977
+ _validated_evidence_s3_uri(uri, label)
978
+ expected = require_string(expected_sha256, f"{label}.sha256").lower()
979
+ if not re.fullmatch(r"[0-9a-f]{64}", expected):
980
+ raise RuntimeError(f"{label}.sha256 must be a lowercase SHA-256 digest")
981
+ run(["aws", "s3", "cp", uri, str(destination)])
982
+ actual = sha256_file(destination)
983
+ if actual != expected:
984
+ raise RuntimeError(f"{label} digest mismatch: expected {expected}, got {actual}")
985
+ return actual
986
+
987
+
988
+ def load_approved_plan_descriptor(payload):
989
+ approved = payload.get("approvedPlan")
990
+ if not isinstance(approved, dict):
991
+ raise RuntimeError("Saved-plan apply requires approvedPlan")
992
+ descriptor_ref = approved.get("descriptor")
993
+ if not isinstance(descriptor_ref, dict):
994
+ raise RuntimeError("approvedPlan.descriptor is required")
995
+ descriptor_path = WORK / TERRAFORM_PLAN_DESCRIPTOR_NAME
996
+ _download_approved_artifact(
997
+ descriptor_ref.get("s3Uri"),
998
+ descriptor_path,
999
+ descriptor_ref.get("sha256"),
1000
+ "approvedPlan.descriptor",
1001
+ )
1002
+ descriptor = json.loads(descriptor_path.read_text(encoding="utf-8"))
1003
+ if descriptor.get("contract") != TERRAFORM_PLAN_APPROVAL_CONTRACT:
1004
+ raise RuntimeError("Approved plan descriptor contract is unsupported")
1005
+ return descriptor, descriptor_ref
1006
+
1007
+
1008
+ def comparable_terraform_plan(value, *, _top_level=True, _parent_key=None):
1009
+ """Normalize render-only Terraform JSON variance without changing plan decisions."""
1010
+ if isinstance(value, dict):
1011
+ return {
1012
+ key: comparable_terraform_plan(item, _top_level=False, _parent_key=key)
1013
+ for key, item in value.items()
1014
+ if not (_top_level and key == "timestamp")
1015
+ }
1016
+ if isinstance(value, list):
1017
+ normalized = [comparable_terraform_plan(item, _top_level=False) for item in value]
1018
+ if _parent_key == "references" and all(isinstance(item, str) for item in normalized):
1019
+ return sorted(normalized)
1020
+ return normalized
1021
+ return value
1022
+
1023
+
1024
+ def validate_and_materialize_approved_plan(payload, state_identity):
1025
+ descriptor, descriptor_ref = load_approved_plan_descriptor(payload)
1026
+ if descriptor.get("deploymentConfigSha256") != deployment_config_sha256(payload):
1027
+ raise RuntimeError("Approved plan deployment configuration does not match apply input")
1028
+ if descriptor.get("release") != release_selection(payload):
1029
+ raise RuntimeError("Approved plan release pin does not match apply input")
1030
+ if descriptor.get("state") != state_identity:
1031
+ raise RuntimeError("Terraform state changed after plan approval; refusing apply")
1032
+
1033
+ tfvars_sha256 = sha256_file(TF / "terraform.auto.tfvars.json")
1034
+ if descriptor.get("terraformVariablesSha256") != tfvars_sha256:
1035
+ raise RuntimeError("Approved plan Terraform variables do not match apply execution")
1036
+
1037
+ saved_plan = descriptor.get("savedPlan")
1038
+ plan_json = descriptor.get("jsonPlan")
1039
+ if not isinstance(saved_plan, dict) or not isinstance(plan_json, dict):
1040
+ raise RuntimeError("Approved plan descriptor is missing saved plan artifacts")
1041
+ destination = TF / "tfplan"
1042
+ _download_approved_artifact(
1043
+ saved_plan.get("s3Uri"),
1044
+ destination,
1045
+ saved_plan.get("sha256"),
1046
+ "approvedPlan.savedPlan",
1047
+ )
1048
+ accepted_json = WORK / "accepted-terraform-plan.json"
1049
+ _download_approved_artifact(
1050
+ plan_json.get("s3Uri"),
1051
+ accepted_json,
1052
+ plan_json.get("sha256"),
1053
+ "approvedPlan.jsonPlan",
1054
+ )
1055
+ rendered = WORK / "approved-terraform-plan.json"
1056
+ with rendered.open("w", encoding="utf-8") as handle:
1057
+ run(["terraform", "show", "-json", "tfplan"], cwd=TF, stdout=handle)
1058
+ accepted_plan = json.loads(accepted_json.read_text(encoding="utf-8"))
1059
+ rendered_plan = json.loads(rendered.read_text(encoding="utf-8"))
1060
+ if comparable_terraform_plan(rendered_plan) != comparable_terraform_plan(accepted_plan):
1061
+ raise RuntimeError("Approved saved plan no longer renders to the accepted JSON plan")
1062
+ if payload.get("incidentRecovery") is not None:
1063
+ delete_addresses = terraform_plan_delete_addresses(rendered_plan)
1064
+ if delete_addresses:
1065
+ raise RuntimeError(
1066
+ "TEI v380 recovery approved plan contains delete actions; refusing apply"
1067
+ )
1068
+
1069
+ return {
1070
+ "descriptor": {
1071
+ "fileName": TERRAFORM_PLAN_DESCRIPTOR_NAME,
1072
+ "sha256": descriptor_ref.get("sha256"),
1073
+ "s3Uri": descriptor_ref.get("s3Uri"),
1074
+ },
1075
+ "savedPlan": saved_plan,
1076
+ "jsonPlan": plan_json,
1077
+ "summary": descriptor.get("summary"),
1078
+ "plannedAction": descriptor.get("plannedAction"),
1079
+ "sourceSessionId": descriptor.get("sessionId"),
1080
+ "state": state_identity,
1081
+ }
1082
+
1083
+
656
1084
  def managed_app_terraform_target_args(payload):
657
1085
  app_key = payload.get("appKey")
658
1086
  if app_key == "n8n":
@@ -667,6 +1095,43 @@ def managed_app_terraform_target_args(payload):
667
1095
  return []
668
1096
 
669
1097
 
1098
+ def execute_terraform_plan_phase(
1099
+ payload,
1100
+ state_identity,
1101
+ planned_action,
1102
+ terraform_phase,
1103
+ target_args,
1104
+ ):
1105
+ """Create plan evidence and apply only for the compatible legacy phase."""
1106
+ plan_args = ["terraform", "plan"]
1107
+ if planned_action == "destroy":
1108
+ plan_args.append("-destroy")
1109
+ plan_args.extend([*target_args, "-out=tfplan", "-no-color"])
1110
+ plan = subprocess.run(plan_args, cwd=TF, text=True)
1111
+ if plan.returncode != 0:
1112
+ return plan
1113
+ TERRAFORM_EVIDENCE["plan"] = write_terraform_plan_evidence(
1114
+ payload,
1115
+ state_identity,
1116
+ planned_action,
1117
+ )
1118
+ if terraform_phase != "legacy":
1119
+ return plan
1120
+ return subprocess.run(
1121
+ ["terraform", "apply", "-auto-approve", "-no-color", "tfplan"],
1122
+ cwd=TF,
1123
+ text=True,
1124
+ )
1125
+
1126
+
1127
+ def should_reconcile_native_auth_schema(terraform_phase, planned_action, payload):
1128
+ return (
1129
+ terraform_phase in {"apply", "legacy"}
1130
+ and planned_action in {"deploy", "update"}
1131
+ and not is_managed_app_operation(payload)
1132
+ )
1133
+
1134
+
670
1135
  def truthy(value):
671
1136
  if isinstance(value, bool):
672
1137
  return value
@@ -797,6 +1262,28 @@ def is_web_only_operation(payload, action=None):
797
1262
  return component == "web" or truthy(payload.get("webOnly"))
798
1263
 
799
1264
 
1265
+ def release_sync_request(action, payload, identity_operation):
1266
+ if (
1267
+ action not in {"deploy", "update", "apply", "web"}
1268
+ or is_managed_app_operation(payload)
1269
+ or identity_operation is not None
1270
+ ):
1271
+ return None
1272
+ web_only = is_web_only_operation(payload, action)
1273
+ return {
1274
+ "artifact_types": {"static-site"} if web_only else None,
1275
+ "artifact_names": {"web"} if web_only else None,
1276
+ }
1277
+
1278
+
1279
+ def requires_agentcore_control_runtime(action, payload, identity_operation):
1280
+ return (
1281
+ action in {"deploy", "update", "plan", "apply", "destroy"}
1282
+ and not is_managed_app_operation(payload)
1283
+ and identity_operation is None
1284
+ )
1285
+
1286
+
800
1287
  def configure_managed_app_evidence_prefix(payload):
801
1288
  if not is_managed_app_operation(payload):
802
1289
  return
@@ -912,21 +1399,54 @@ def cloudfront_alias_items(value):
912
1399
  return []
913
1400
 
914
1401
 
915
- def write_terraform_plan_evidence(payload=None):
1402
+ def write_terraform_plan_evidence(payload=None, state_identity=None, planned_action="update"):
1403
+ payload = payload or {}
1404
+ state_identity = state_identity or terraform_state_identity()
916
1405
  plan_path = Path("terraform-plan.json")
917
1406
  with plan_path.open("w", encoding="utf-8") as handle:
918
1407
  run(["terraform", "show", "-json", "tfplan"], cwd=TF, stdout=handle)
919
1408
  plan_json = json.loads(plan_path.read_text(encoding="utf-8"))
920
- validate_managed_app_plan_scope(payload or {}, plan_json)
921
- validate_environment_plan_scope(payload or {}, plan_json)
1409
+ validate_managed_app_plan_scope(payload, plan_json)
1410
+ validate_environment_plan_scope(payload, plan_json)
922
1411
  artifact = {
923
1412
  "fileName": plan_path.name,
924
1413
  "sha256": sha256_file(plan_path),
925
1414
  "s3Uri": upload_evidence_artifact(plan_path),
926
1415
  }
1416
+ saved_plan_path = TF / "tfplan"
1417
+ saved_plan = {
1418
+ "fileName": TERRAFORM_SAVED_PLAN_NAME,
1419
+ "sha256": sha256_file(saved_plan_path),
1420
+ "s3Uri": upload_evidence_artifact(saved_plan_path, TERRAFORM_SAVED_PLAN_NAME),
1421
+ }
1422
+ descriptor = {
1423
+ "schemaVersion": 1,
1424
+ "contract": TERRAFORM_PLAN_APPROVAL_CONTRACT,
1425
+ "createdAt": datetime.now(UTC).isoformat(),
1426
+ "status": "awaiting-approval",
1427
+ "sessionId": payload.get("sessionId") or os.environ.get("THINKWORK_DEPLOYMENT_SESSION_ID"),
1428
+ "plannedAction": planned_action,
1429
+ "controllerInputSha256": controller_input_sha256(payload),
1430
+ "rawControllerInputSha256": raw_controller_input_sha256(payload),
1431
+ "deploymentConfigSha256": deployment_config_sha256(payload),
1432
+ "terraformVariablesSha256": sha256_file(TF / "terraform.auto.tfvars.json"),
1433
+ "release": release_selection(payload),
1434
+ "state": state_identity,
1435
+ "jsonPlan": artifact,
1436
+ "savedPlan": saved_plan,
1437
+ "summary": terraform_plan_summary(plan_json),
1438
+ }
1439
+ descriptor_artifact = write_json_evidence_artifact(
1440
+ TERRAFORM_PLAN_DESCRIPTOR_NAME,
1441
+ descriptor,
1442
+ )
927
1443
  return {
928
1444
  "artifact": artifact,
929
- "summary": terraform_plan_summary(plan_json),
1445
+ "savedPlan": saved_plan,
1446
+ "approvalDescriptor": descriptor_artifact,
1447
+ "summary": descriptor["summary"],
1448
+ "state": state_identity,
1449
+ "status": "awaiting-approval",
930
1450
  }
931
1451
 
932
1452
 
@@ -974,6 +1494,38 @@ def safe_get_bool(runner_secrets, payload, name, default=False):
974
1494
  return default
975
1495
 
976
1496
 
1497
+ def safe_get_list(runner_secrets, payload, name, default=None):
1498
+ """List analogue of safe_get_bool's precedence. Generated-root
1499
+ variables typed list(string) reject bare strings, so string values
1500
+ (comma-separated, common in Secrets Manager JSON) are split here and
1501
+ the wiring point always emits a real list of non-empty strings."""
1502
+ for source in (runner_secrets, payload):
1503
+ value = source.get(name)
1504
+ if isinstance(value, list):
1505
+ return [str(item).strip() for item in value if str(item).strip()]
1506
+ if isinstance(value, str) and value.strip():
1507
+ return [part.strip() for part in value.split(",") if part.strip()]
1508
+ return list(default or [])
1509
+
1510
+
1511
+ def safe_get_ingress_cidrs(runner_secrets, payload, name, default=None):
1512
+ """safe_get_list precedence, but an explicitly PRESENT empty value ([] or
1513
+ "") wins as "no ingress" instead of falling through to the default —
1514
+ closing the database port (THINK-1082) is spelled dbPublicIngressCidrs: []
1515
+ and must not silently re-seed the world-open default."""
1516
+ for source in (runner_secrets, payload):
1517
+ if name not in source:
1518
+ continue
1519
+ value = source.get(name)
1520
+ if isinstance(value, list):
1521
+ return [str(item).strip() for item in value if str(item).strip()]
1522
+ if isinstance(value, str):
1523
+ return [part.strip() for part in value.split(",") if part.strip()]
1524
+ if value is None:
1525
+ continue
1526
+ return list(default or [])
1527
+
1528
+
977
1529
  def current_terraform_state(stage):
978
1530
  bucket = os.environ.get("THINKWORK_TERRAFORM_STATE_BUCKET")
979
1531
  if not bucket:
@@ -993,6 +1545,670 @@ def current_terraform_state(stage):
993
1545
  return {}
994
1546
 
995
1547
 
1548
+ def _terraform_state_object_keys(stage):
1549
+ return [
1550
+ f"env:/{stage}/thinkwork/{stage}/terraform.tfstate",
1551
+ f"thinkwork/{stage}/terraform.tfstate",
1552
+ ]
1553
+
1554
+
1555
+ def terraform_state_object_snapshot(stage):
1556
+ bucket = require_string(
1557
+ os.environ.get("THINKWORK_TERRAFORM_STATE_BUCKET"),
1558
+ "THINKWORK_TERRAFORM_STATE_BUCKET",
1559
+ )
1560
+ for key in _terraform_state_object_keys(stage):
1561
+ try:
1562
+ head = json.loads(
1563
+ output(
1564
+ [
1565
+ "aws",
1566
+ "s3api",
1567
+ "head-object",
1568
+ "--bucket",
1569
+ bucket,
1570
+ "--key",
1571
+ key,
1572
+ "--output",
1573
+ "json",
1574
+ ]
1575
+ )
1576
+ )
1577
+ body = subprocess.check_output(
1578
+ ["aws", "s3", "cp", f"s3://{bucket}/{key}", "-"],
1579
+ )
1580
+ except (subprocess.CalledProcessError, json.JSONDecodeError):
1581
+ continue
1582
+ state = json.loads(body)
1583
+ return {
1584
+ "bucket": bucket,
1585
+ "key": key,
1586
+ "etag": str(head.get("ETag") or "").strip('"'),
1587
+ "size": int(head.get("ContentLength") or len(body)),
1588
+ "lastModified": head.get("LastModified"),
1589
+ "serverSideEncryption": head.get("ServerSideEncryption"),
1590
+ "versionId": head.get("VersionId"),
1591
+ "sha256": sha256_bytes(body),
1592
+ "lineage": state.get("lineage"),
1593
+ "serial": state.get("serial"),
1594
+ "terraformVersion": state.get("terraform_version"),
1595
+ }
1596
+ raise RuntimeError(f"Terraform state object for stage {stage} was not found")
1597
+
1598
+
1599
+ def _incident_operation(payload):
1600
+ operation = payload.get("operation")
1601
+ if not isinstance(operation, dict) or operation.get("kind") != "incident_recovery":
1602
+ raise RuntimeError("Incident recovery requires operation.kind=incident_recovery")
1603
+ return operation
1604
+
1605
+
1606
+ def _require_exact_incident_payload_aliases(payload, aliases, expected, label):
1607
+ present = {name: payload.get(name) for name in aliases if name in payload}
1608
+ if not present or any(value != expected for value in present.values()):
1609
+ raise RuntimeError(f"TEI v380 recovery {label} does not match exact incident scope")
1610
+
1611
+
1612
+ def tei_v380_recovery_scope():
1613
+ return {
1614
+ "contract": TEI_V380_RECOVERY_CONTRACT,
1615
+ "incidentId": TEI_V380_RECOVERY_INCIDENT_ID,
1616
+ "rejectedPlanSha256": TEI_V380_REJECTED_PLAN_SHA256,
1617
+ "stage": "tei-e2e",
1618
+ "accountId": "637423202447",
1619
+ "region": "us-east-1",
1620
+ "state": {
1621
+ "bucket": TEI_V380_STATE_BUCKET,
1622
+ "key": TEI_V380_STATE_KEY,
1623
+ "lineage": TEI_V380_STATE_LINEAGE,
1624
+ "serial": TEI_V380_STATE_SERIAL,
1625
+ "sha256": TEI_V380_STATE_SHA256,
1626
+ },
1627
+ "lock": {
1628
+ "table": TEI_V380_LOCK_TABLE,
1629
+ "key": TEI_V380_LOCK_KEY,
1630
+ "id": TEI_V380_LOCK_ID,
1631
+ "operation": "OperationTypeApply",
1632
+ "terraformVersion": "1.8.5",
1633
+ "digest": TEI_V380_STATE_DIGEST,
1634
+ },
1635
+ "secret": {
1636
+ "address": TEI_V380_SECRET_ADDRESS,
1637
+ "arn": TEI_V380_SECRET_ARN,
1638
+ "name": TEI_V380_SECRET_NAME,
1639
+ },
1640
+ "kmsKey": {
1641
+ "address": TEI_V380_KMS_KEY_ADDRESS,
1642
+ "id": TEI_V380_KMS_KEY_ID,
1643
+ "arn": TEI_V380_KMS_KEY_ARN,
1644
+ "aliasAddress": TEI_V380_KMS_ALIAS_ADDRESS,
1645
+ "alias": TEI_V380_KMS_ALIAS,
1646
+ },
1647
+ }
1648
+
1649
+
1650
+ def validate_tei_v380_recovery_contract(payload):
1651
+ operation = _incident_operation(payload)
1652
+ expected = tei_v380_recovery_scope()
1653
+ actual = {
1654
+ key: operation.get(key)
1655
+ for key in [
1656
+ "contract",
1657
+ "incidentId",
1658
+ "rejectedPlanSha256",
1659
+ "stage",
1660
+ "accountId",
1661
+ "region",
1662
+ "state",
1663
+ "lock",
1664
+ "secret",
1665
+ "kmsKey",
1666
+ ]
1667
+ }
1668
+ if actual != expected:
1669
+ raise RuntimeError("TEI v380 recovery contract does not match the exact incident scope")
1670
+ expected_operation_keys = {"kind", *expected.keys()}
1671
+ if set(operation) != expected_operation_keys:
1672
+ raise RuntimeError("TEI v380 recovery operation contains unexpected fields")
1673
+ _require_exact_incident_payload_aliases(
1674
+ payload, ("environmentName", "stage"), "tei-e2e", "stage"
1675
+ )
1676
+ _require_exact_incident_payload_aliases(
1677
+ payload, ("awsAccountId", "accountId"), "637423202447", "account"
1678
+ )
1679
+ _require_exact_incident_payload_aliases(payload, ("awsRegion", "region"), "us-east-1", "region")
1680
+ return expected
1681
+
1682
+
1683
+ def assert_incident_recovery_operation_is_exclusive():
1684
+ current_execution = os.environ.get("THINKWORK_DEPLOYMENT_EXECUTION_ARN")
1685
+ state_machine_arn = require_string(
1686
+ os.environ.get("THINKWORK_DEPLOYMENT_STATE_MACHINE_ARN"),
1687
+ "THINKWORK_DEPLOYMENT_STATE_MACHINE_ARN",
1688
+ )
1689
+ executions = json.loads(
1690
+ output(
1691
+ [
1692
+ "aws",
1693
+ "stepfunctions",
1694
+ "list-executions",
1695
+ "--state-machine-arn",
1696
+ state_machine_arn,
1697
+ "--status-filter",
1698
+ "RUNNING",
1699
+ "--output",
1700
+ "json",
1701
+ ]
1702
+ )
1703
+ or "{}"
1704
+ ).get("executions", [])
1705
+ if current_execution:
1706
+ other_executions = [
1707
+ item.get("executionArn")
1708
+ for item in executions
1709
+ if item.get("executionArn") != current_execution
1710
+ ]
1711
+ else:
1712
+ # Established control planes do not receive Execution.Id until this
1713
+ # release updates their state-machine definition. Permit exactly the
1714
+ # sole running controller execution during that bounded transition.
1715
+ # Zero executions would mean the recovery was launched outside the
1716
+ # managed controller; more than one is ambiguous and unsafe.
1717
+ if len(executions) != 1:
1718
+ raise RuntimeError(
1719
+ "Incident recovery without execution identity requires exactly one "
1720
+ "running deployment execution"
1721
+ )
1722
+ current_execution = executions[0].get("executionArn")
1723
+ other_executions = []
1724
+ if other_executions:
1725
+ raise RuntimeError("Another deployment execution is running; recovery refused")
1726
+
1727
+ project = require_string(
1728
+ os.environ.get("THINKWORK_DEPLOYMENT_RUNNER_PROJECT_NAME"),
1729
+ "THINKWORK_DEPLOYMENT_RUNNER_PROJECT_NAME",
1730
+ )
1731
+ current_build = os.environ.get("CODEBUILD_BUILD_ID")
1732
+ if not current_build:
1733
+ raise RuntimeError("Incident recovery must run inside the deployment CodeBuild")
1734
+ build_ids = json.loads(
1735
+ output(
1736
+ [
1737
+ "aws",
1738
+ "codebuild",
1739
+ "list-builds-for-project",
1740
+ "--project-name",
1741
+ project,
1742
+ "--sort-order",
1743
+ "DESCENDING",
1744
+ "--no-paginate",
1745
+ "--output",
1746
+ "json",
1747
+ ]
1748
+ )
1749
+ or "{}"
1750
+ ).get("ids", [])[:100]
1751
+ if build_ids:
1752
+ builds = json.loads(
1753
+ output(
1754
+ [
1755
+ "aws",
1756
+ "codebuild",
1757
+ "batch-get-builds",
1758
+ "--ids",
1759
+ *build_ids,
1760
+ "--output",
1761
+ "json",
1762
+ ]
1763
+ )
1764
+ or "{}"
1765
+ ).get("builds", [])
1766
+ other_builds = [
1767
+ item.get("id")
1768
+ for item in builds
1769
+ if item.get("buildStatus") == "IN_PROGRESS" and item.get("id") != current_build
1770
+ ]
1771
+ if other_builds:
1772
+ raise RuntimeError("Another deployment CodeBuild is running; recovery refused")
1773
+ return {
1774
+ "currentExecutionArn": current_execution,
1775
+ "currentBuildId": current_build,
1776
+ "otherRunningExecutions": 0,
1777
+ "otherRunningBuilds": 0,
1778
+ }
1779
+
1780
+
1781
+ def _redacted_secret_description(description):
1782
+ return {
1783
+ "arn": description.get("ARN"),
1784
+ "name": description.get("Name"),
1785
+ "deletedDate": description.get("DeletedDate"),
1786
+ "kmsKeyId": description.get("KmsKeyId"),
1787
+ "rotationEnabled": bool(description.get("RotationEnabled", False)),
1788
+ }
1789
+
1790
+
1791
+ def _redacted_kms_description(description):
1792
+ metadata = description.get("KeyMetadata") or {}
1793
+ return {
1794
+ "keyId": metadata.get("KeyId"),
1795
+ "arn": metadata.get("Arn"),
1796
+ "keyState": metadata.get("KeyState"),
1797
+ "deletionDate": metadata.get("DeletionDate"),
1798
+ "enabled": metadata.get("Enabled"),
1799
+ "keyManager": metadata.get("KeyManager"),
1800
+ "origin": metadata.get("Origin"),
1801
+ }
1802
+
1803
+
1804
+ def _wait_for_json_description(fetch, accept, description, attempts=20, delay_seconds=1):
1805
+ """Bound eventual-consistency waits without ever reading secret values."""
1806
+ latest = None
1807
+ for attempt in range(attempts):
1808
+ latest = fetch()
1809
+ if accept(latest):
1810
+ return latest
1811
+ if attempt + 1 < attempts:
1812
+ time.sleep(delay_seconds)
1813
+ raise RuntimeError(f"Timed out waiting for {description}")
1814
+
1815
+
1816
+ def _dynamodb_string(item, name):
1817
+ value = (item.get(name) or {}).get("S")
1818
+ return value if isinstance(value, str) else None
1819
+
1820
+
1821
+ def tei_v380_lock_snapshot():
1822
+ def get_item(lock_key):
1823
+ raw = output(
1824
+ [
1825
+ "aws",
1826
+ "dynamodb",
1827
+ "get-item",
1828
+ "--table-name",
1829
+ TEI_V380_LOCK_TABLE,
1830
+ "--key",
1831
+ json.dumps({"LockID": {"S": lock_key}}, separators=(",", ":")),
1832
+ "--consistent-read",
1833
+ "--output",
1834
+ "json",
1835
+ ]
1836
+ )
1837
+ return json.loads(raw or "{}").get("Item") or {}
1838
+
1839
+ lock_item = get_item(TEI_V380_LOCK_KEY)
1840
+ digest_item = get_item(f"{TEI_V380_LOCK_KEY}-md5")
1841
+ info_raw = _dynamodb_string(lock_item, "Info")
1842
+ info = json.loads(info_raw) if info_raw else None
1843
+ if info is not None and not isinstance(info, dict):
1844
+ raise RuntimeError("TEI v380 Terraform lock Info is malformed")
1845
+ return {
1846
+ "table": TEI_V380_LOCK_TABLE,
1847
+ "key": TEI_V380_LOCK_KEY,
1848
+ "present": bool(lock_item),
1849
+ "info": {
1850
+ "id": (info or {}).get("ID"),
1851
+ "operation": (info or {}).get("Operation"),
1852
+ "version": (info or {}).get("Version"),
1853
+ "created": (info or {}).get("Created"),
1854
+ "path": (info or {}).get("Path"),
1855
+ }
1856
+ if info
1857
+ else None,
1858
+ "digestKey": f"{TEI_V380_LOCK_KEY}-md5",
1859
+ "digest": _dynamodb_string(digest_item, "Digest"),
1860
+ }
1861
+
1862
+
1863
+ def force_unlock_tei_v380_state():
1864
+ """Use Terraform's backend-aware unlock path; never delete DynamoDB rows directly."""
1865
+ before = tei_v380_lock_snapshot()
1866
+ expected_info = {
1867
+ "id": TEI_V380_LOCK_ID,
1868
+ "operation": "OperationTypeApply",
1869
+ "version": "1.8.5",
1870
+ "path": TEI_V380_LOCK_KEY,
1871
+ }
1872
+ actual_info = before.get("info") or {}
1873
+ if before.get("digest") != TEI_V380_STATE_DIGEST:
1874
+ raise RuntimeError("TEI v380 Terraform state digest changed; unlock refused")
1875
+ if before.get("present") is not True:
1876
+ return {
1877
+ "method": "terraform-force-unlock",
1878
+ "lockId": TEI_V380_LOCK_ID,
1879
+ "before": before,
1880
+ "after": before,
1881
+ "alreadyUnlocked": True,
1882
+ "directDynamoDbMutation": False,
1883
+ }
1884
+ if any(actual_info.get(key) != value for key, value in expected_info.items()):
1885
+ raise RuntimeError("TEI v380 stale Terraform lock identity does not match")
1886
+
1887
+ unlock_dir = WORK / "tei-v380-force-unlock"
1888
+ unlock_dir.mkdir(parents=True, exist_ok=True)
1889
+ (unlock_dir / "main.tf").write_text(
1890
+ 'terraform {\n required_version = ">= 1.8.5"\n backend "s3" {}\n}\n',
1891
+ encoding="utf-8",
1892
+ )
1893
+ (unlock_dir / "backend.hcl").write_text(
1894
+ "\n".join(
1895
+ [
1896
+ f"bucket = {hcl_string(TEI_V380_STATE_BUCKET)}",
1897
+ f"key = {hcl_string(TEI_V380_STATE_KEY)}",
1898
+ 'region = "us-east-1"',
1899
+ f"dynamodb_table = {hcl_string(TEI_V380_LOCK_TABLE)}",
1900
+ "encrypt = true",
1901
+ "",
1902
+ ]
1903
+ ),
1904
+ encoding="utf-8",
1905
+ )
1906
+ run(
1907
+ [
1908
+ "terraform",
1909
+ "init",
1910
+ "-reconfigure",
1911
+ "-backend-config=backend.hcl",
1912
+ "-input=false",
1913
+ "-no-color",
1914
+ ],
1915
+ cwd=unlock_dir,
1916
+ )
1917
+ run(
1918
+ ["terraform", "force-unlock", "-force", TEI_V380_LOCK_ID],
1919
+ cwd=unlock_dir,
1920
+ )
1921
+ after = tei_v380_lock_snapshot()
1922
+ if after.get("present") is not False:
1923
+ raise RuntimeError("TEI v380 stale Terraform lock remains after force-unlock")
1924
+ if after.get("digest") != before.get("digest"):
1925
+ raise RuntimeError("Terraform state digest changed during force-unlock")
1926
+ return {
1927
+ "method": "terraform-force-unlock",
1928
+ "lockId": TEI_V380_LOCK_ID,
1929
+ "before": before,
1930
+ "after": after,
1931
+ "alreadyUnlocked": False,
1932
+ "directDynamoDbMutation": False,
1933
+ }
1934
+
1935
+
1936
+ def recover_tei_v380_incident(payload):
1937
+ validate_tei_v380_recovery_contract(payload)
1938
+ exclusivity = assert_incident_recovery_operation_is_exclusive()
1939
+ caller_account = output(
1940
+ ["aws", "sts", "get-caller-identity", "--query", "Account", "--output", "text"]
1941
+ )
1942
+ if caller_account != "637423202447":
1943
+ raise RuntimeError("TEI v380 recovery caller is in the wrong AWS account")
1944
+
1945
+ before_state = terraform_state_object_snapshot("tei-e2e")
1946
+ for key, expected in [
1947
+ ("bucket", TEI_V380_STATE_BUCKET),
1948
+ ("key", TEI_V380_STATE_KEY),
1949
+ ("lineage", TEI_V380_STATE_LINEAGE),
1950
+ ("serial", TEI_V380_STATE_SERIAL),
1951
+ ("sha256", TEI_V380_STATE_SHA256),
1952
+ ]:
1953
+ if before_state.get(key) != expected:
1954
+ raise RuntimeError(f"TEI v380 recovery state {key} changed; recovery refused")
1955
+
1956
+ progress = {
1957
+ "schemaVersion": 1,
1958
+ "contract": TEI_V380_RECOVERY_CONTRACT,
1959
+ "incidentId": TEI_V380_RECOVERY_INCIDENT_ID,
1960
+ "rejectedPlanSha256": TEI_V380_REJECTED_PLAN_SHA256,
1961
+ "status": "preflight-passed",
1962
+ "startedAt": datetime.now(UTC).isoformat(),
1963
+ "exclusiveOperation": exclusivity,
1964
+ "stateBefore": before_state,
1965
+ "terraformStateMutation": False,
1966
+ "secretValueRead": False,
1967
+ }
1968
+ write_json_evidence_artifact("incident-recovery-progress.json", progress)
1969
+
1970
+ secret_before_raw = json.loads(
1971
+ output(
1972
+ [
1973
+ "aws",
1974
+ "secretsmanager",
1975
+ "describe-secret",
1976
+ "--secret-id",
1977
+ TEI_V380_SECRET_ARN,
1978
+ "--output",
1979
+ "json",
1980
+ ]
1981
+ )
1982
+ )
1983
+ if (
1984
+ secret_before_raw.get("ARN") != TEI_V380_SECRET_ARN
1985
+ or secret_before_raw.get("Name") != TEI_V380_SECRET_NAME
1986
+ ):
1987
+ raise RuntimeError("TEI v380 recovery secret identity does not match")
1988
+ if secret_before_raw.get("DeletedDate"):
1989
+ run(
1990
+ [
1991
+ "aws",
1992
+ "secretsmanager",
1993
+ "restore-secret",
1994
+ "--secret-id",
1995
+ TEI_V380_SECRET_ARN,
1996
+ ],
1997
+ stdout=subprocess.DEVNULL,
1998
+ )
1999
+
2000
+ def describe_secret():
2001
+ return json.loads(
2002
+ output(
2003
+ [
2004
+ "aws",
2005
+ "secretsmanager",
2006
+ "describe-secret",
2007
+ "--secret-id",
2008
+ TEI_V380_SECRET_ARN,
2009
+ "--output",
2010
+ "json",
2011
+ ]
2012
+ )
2013
+ )
2014
+
2015
+ secret_after_raw = _wait_for_json_description(
2016
+ describe_secret,
2017
+ lambda value: not value.get("DeletedDate"),
2018
+ "restored TEI Twenty OAuth client secret",
2019
+ )
2020
+ if (
2021
+ secret_after_raw.get("ARN") != TEI_V380_SECRET_ARN
2022
+ or secret_after_raw.get("Name") != TEI_V380_SECRET_NAME
2023
+ ):
2024
+ raise RuntimeError("Restored TEI v380 recovery secret identity does not match")
2025
+ progress["secret"] = {
2026
+ "address": TEI_V380_SECRET_ADDRESS,
2027
+ "before": _redacted_secret_description(secret_before_raw),
2028
+ "after": _redacted_secret_description(secret_after_raw),
2029
+ "status": "restored",
2030
+ }
2031
+ progress["status"] = "secret-restored"
2032
+ write_json_evidence_artifact("incident-recovery-progress.json", progress)
2033
+
2034
+ kms_before_raw = json.loads(
2035
+ output(
2036
+ [
2037
+ "aws",
2038
+ "kms",
2039
+ "describe-key",
2040
+ "--key-id",
2041
+ TEI_V380_KMS_KEY_ID,
2042
+ "--output",
2043
+ "json",
2044
+ ]
2045
+ )
2046
+ )
2047
+ kms_before = _redacted_kms_description(kms_before_raw)
2048
+ if (
2049
+ kms_before.get("keyId") != TEI_V380_KMS_KEY_ID
2050
+ or kms_before.get("arn") != TEI_V380_KMS_KEY_ARN
2051
+ ):
2052
+ raise RuntimeError("TEI v380 recovery KMS key identity does not match")
2053
+ if kms_before.get("keyState") == "PendingDeletion":
2054
+ run(
2055
+ ["aws", "kms", "cancel-key-deletion", "--key-id", TEI_V380_KMS_KEY_ID],
2056
+ stdout=subprocess.DEVNULL,
2057
+ )
2058
+
2059
+ def describe_kms_key():
2060
+ return json.loads(
2061
+ output(
2062
+ [
2063
+ "aws",
2064
+ "kms",
2065
+ "describe-key",
2066
+ "--key-id",
2067
+ TEI_V380_KMS_KEY_ID,
2068
+ "--output",
2069
+ "json",
2070
+ ]
2071
+ )
2072
+ )
2073
+
2074
+ kms_mid_raw = _wait_for_json_description(
2075
+ describe_kms_key,
2076
+ lambda value: _redacted_kms_description(value).get("keyState") in {"Disabled", "Enabled"},
2077
+ "cancelled TEI turn-assertion KMS deletion",
2078
+ )
2079
+ if _redacted_kms_description(kms_mid_raw).get("keyState") == "Disabled":
2080
+ run(
2081
+ ["aws", "kms", "enable-key", "--key-id", TEI_V380_KMS_KEY_ID],
2082
+ stdout=subprocess.DEVNULL,
2083
+ )
2084
+ kms_after_raw = _wait_for_json_description(
2085
+ describe_kms_key,
2086
+ lambda value: (
2087
+ _redacted_kms_description(value).get("keyState") == "Enabled"
2088
+ and _redacted_kms_description(value).get("enabled") is True
2089
+ ),
2090
+ "enabled TEI turn-assertion KMS key",
2091
+ )
2092
+ kms_after = _redacted_kms_description(kms_after_raw)
2093
+ if (
2094
+ kms_after.get("keyId") != TEI_V380_KMS_KEY_ID
2095
+ or kms_after.get("arn") != TEI_V380_KMS_KEY_ARN
2096
+ ):
2097
+ raise RuntimeError("Recovered TEI v380 KMS key identity does not match")
2098
+ progress["kmsKey"] = {
2099
+ "address": TEI_V380_KMS_KEY_ADDRESS,
2100
+ "aliasAddress": TEI_V380_KMS_ALIAS_ADDRESS,
2101
+ "alias": TEI_V380_KMS_ALIAS,
2102
+ "before": kms_before,
2103
+ "after": kms_after,
2104
+ "status": "enabled",
2105
+ }
2106
+ progress["status"] = "resources-recovered"
2107
+ write_json_evidence_artifact("incident-recovery-progress.json", progress)
2108
+
2109
+ progress["terraformLock"] = force_unlock_tei_v380_state()
2110
+ progress["status"] = "resources-recovered-and-state-unlocked"
2111
+ write_json_evidence_artifact("incident-recovery-progress.json", progress)
2112
+
2113
+ after_state = terraform_state_object_snapshot("tei-e2e")
2114
+ if after_state != before_state:
2115
+ raise RuntimeError("Terraform state object changed during incident recovery")
2116
+ progress.update(
2117
+ {
2118
+ "status": "succeeded",
2119
+ "completedAt": datetime.now(UTC).isoformat(),
2120
+ "stateAfter": after_state,
2121
+ "terraformStateMutation": False,
2122
+ "secretValueRead": False,
2123
+ "nextStep": "proof-enabled-plan-with-imports",
2124
+ }
2125
+ )
2126
+ artifact = write_json_evidence_artifact("incident-recovery.json", progress)
2127
+ return {"proof": progress, "artifact": artifact}
2128
+
2129
+
2130
+ def tei_v380_recovery_import_blocks(payload, vars_json):
2131
+ recovery = payload.get("incidentRecovery")
2132
+ if recovery is None:
2133
+ return ""
2134
+ if not isinstance(recovery, dict):
2135
+ raise RuntimeError("incidentRecovery must be an object")
2136
+ evidence_ref = recovery.get("evidence")
2137
+ actual_scope = {key: recovery.get(key) for key in tei_v380_recovery_scope()}
2138
+ if actual_scope != tei_v380_recovery_scope():
2139
+ raise RuntimeError("Incident recovery import scope does not match TEI v380 incident")
2140
+ if not isinstance(evidence_ref, dict):
2141
+ raise RuntimeError("incidentRecovery.evidence is required")
2142
+
2143
+ # The decoded Terraform state is used elsewhere to preserve outputs and
2144
+ # guardrails, but it does not carry an object digest. Recovery approval is
2145
+ # explicitly bound to the S3 state object's SHA/lineage/serial, so compare
2146
+ # against the authoritative object snapshot instead of the decoded state.
2147
+ current_state = terraform_state_object_snapshot(
2148
+ vars_json.get("stage") or recovery.get("stage") or ""
2149
+ )
2150
+
2151
+ evidence_path = WORK / "accepted-incident-recovery.json"
2152
+ _download_approved_artifact(
2153
+ evidence_ref.get("s3Uri"),
2154
+ evidence_path,
2155
+ evidence_ref.get("sha256"),
2156
+ "incidentRecovery.evidence",
2157
+ )
2158
+ evidence = json.loads(evidence_path.read_text(encoding="utf-8"))
2159
+ if (
2160
+ evidence.get("contract") != TEI_V380_RECOVERY_CONTRACT
2161
+ or evidence.get("incidentId") != TEI_V380_RECOVERY_INCIDENT_ID
2162
+ or evidence.get("rejectedPlanSha256") != TEI_V380_REJECTED_PLAN_SHA256
2163
+ or evidence.get("status") != "succeeded"
2164
+ or evidence.get("terraformStateMutation") is not False
2165
+ or evidence.get("secretValueRead") is not False
2166
+ or evidence.get("stateBefore") != evidence.get("stateAfter")
2167
+ ):
2168
+ raise RuntimeError("Incident recovery evidence is incomplete or unsafe")
2169
+ evidence_state = evidence.get("stateAfter") or {}
2170
+ if (
2171
+ evidence_state.get("lineage") != TEI_V380_STATE_LINEAGE
2172
+ or evidence_state.get("serial") != TEI_V380_STATE_SERIAL
2173
+ or evidence_state.get("sha256") != TEI_V380_STATE_SHA256
2174
+ ):
2175
+ raise RuntimeError("Incident recovery evidence state identity does not match")
2176
+ if (
2177
+ current_state.get("lineage") != TEI_V380_STATE_LINEAGE
2178
+ or current_state.get("serial") != TEI_V380_STATE_SERIAL
2179
+ or current_state.get("sha256") != TEI_V380_STATE_SHA256
2180
+ ):
2181
+ raise RuntimeError("Current Terraform state moved after incident recovery")
2182
+
2183
+ if vars_json.get("enable_agentcore_multiplayer_proof") is not True:
2184
+ raise RuntimeError("Incident recovery import requires AgentCore proof enabled")
2185
+ if vars_json.get("agentcore_multiplayer_proof_tenant_slug") != "tei":
2186
+ raise RuntimeError("Incident recovery import requires the exact TEI tenant slug")
2187
+ if vars_json.get("agentcore_multiplayer_proof_owner_allowlist") != TEI_V380_OWNER_ALLOWLIST:
2188
+ raise RuntimeError("Incident recovery import requires the exact owner allowlist")
2189
+ if vars_json.get("finalize_auth_retirement") is not False:
2190
+ raise RuntimeError("Incident recovery import cannot finalize auth retirement")
2191
+ if vars_json.get("auth_retirement_phase") != "coexistence":
2192
+ raise RuntimeError("Incident recovery import requires auth coexistence")
2193
+ if vars_json.get("microsoft_oauth_tenant") != "organizations":
2194
+ raise RuntimeError("Incident recovery import requires Microsoft organizations tenant")
2195
+
2196
+ return f"""
2197
+ # TEI v380 incident recovery imports. These addresses and IDs are bound to the
2198
+ # independently accepted recovery evidence above. They recover ownership of
2199
+ # the original secret and KMS key without reading or replacing either value.
2200
+ import {{
2201
+ to = {TEI_V380_SECRET_ADDRESS}
2202
+ id = {hcl_string(TEI_V380_SECRET_ARN)}
2203
+ }}
2204
+
2205
+ import {{
2206
+ to = {TEI_V380_KMS_KEY_ADDRESS}
2207
+ id = {hcl_string(TEI_V380_KMS_KEY_ID)}
2208
+ }}
2209
+ """
2210
+
2211
+
996
2212
  def current_terraform_outputs(stage):
997
2213
  outputs = current_terraform_state(stage).get("outputs")
998
2214
  return outputs if isinstance(outputs, dict) else {}
@@ -1238,6 +2454,42 @@ def config_string_list(desired_config, key, env_name, default=None):
1238
2454
  return list(default or [])
1239
2455
 
1240
2456
 
2457
+ def coerce_string_list(value):
2458
+ """Normalize a runner-secrets / payload value into a list[str].
2459
+
2460
+ Accepts a native list, a JSON-array string (``["a","b"]``), or a
2461
+ comma-separated string. Blank entries are dropped. Anything else yields
2462
+ an empty list. Order and duplicates are preserved for the caller; the
2463
+ Terraform side applies distinct().
2464
+ """
2465
+ if isinstance(value, list):
2466
+ return [str(item).strip() for item in value if str(item).strip()]
2467
+ if isinstance(value, str):
2468
+ text = value.strip()
2469
+ if not text:
2470
+ return []
2471
+ if text.startswith("["):
2472
+ try:
2473
+ parsed = json.loads(text)
2474
+ except ValueError:
2475
+ parsed = None
2476
+ if isinstance(parsed, list):
2477
+ return [str(item).strip() for item in parsed if str(item).strip()]
2478
+ return [item.strip() for item in text.split(",") if item.strip()]
2479
+ return []
2480
+
2481
+
2482
+ def runner_secret_string_list(runner_secrets, payload, *keys):
2483
+ """First non-empty list found across runner_secrets then payload for keys."""
2484
+ for source in (runner_secrets or {}, payload or {}):
2485
+ for key in keys:
2486
+ if key in source:
2487
+ result = coerce_string_list(source.get(key))
2488
+ if result:
2489
+ return result
2490
+ return []
2491
+
2492
+
1241
2493
  def required_config_value(
1242
2494
  desired_config,
1243
2495
  manifest_images,
@@ -2105,6 +3357,184 @@ def resolve_agentcore_pi_source_image_uri(payload):
2105
3357
  return release_runtime_image("agentcore-pi-amd64")
2106
3358
 
2107
3359
 
3360
+ def escape_env_control_chars(value):
3361
+ # AgentCore rejects env values with raw control characters (the PEM's
3362
+ # newlines); mirror them as literal escapes exactly like the dev-pipeline
3363
+ # env mirror does (scripts/update-agentcore-runtime-image.sh). Consumers
3364
+ # un-escape (the Pi container's publicKeyPemFromEnv normalizes \n).
3365
+ return value.replace("\r", "\\r").replace("\n", "\\n").replace("\t", "\\t")
3366
+
3367
+
3368
+ def reconcile_agentcore_pi_runtime(vars_json, payload):
3369
+ """Create/update the Pi Bedrock AgentCore Runtime after a successful apply
3370
+ (THINK-584 U5). Pins the runtime to the immutable digest of the customer's
3371
+ mirrored Pi image, mirrors the Pi Lambda env atomically (abort on empty),
3372
+ and records the runtime id in SSM. Secret env values never appear in logs
3373
+ or argv — the request travels to the node entrypoint on stdin."""
3374
+ stage = vars_json["stage"]
3375
+ region = os.environ.get("AWS_REGION") or safe_get(payload, "awsRegion", default="us-east-1")
3376
+ runtime_bundle_dir = os.environ.get("THINKWORK_AGENTCORE_CONTROL_RUNTIME_DIR", "")
3377
+ entrypoint = (
3378
+ Path(runtime_bundle_dir) / "reconcile_pi_runtime.js" if runtime_bundle_dir else None
3379
+ )
3380
+ if entrypoint is None or not entrypoint.is_file():
3381
+ print(
3382
+ "[runner] AgentCore Pi runtime reconcile SKIPPED: release bundle predates "
3383
+ "reconcile_pi_runtime.js (THINK-584 U5). Re-seed the runner bundle to enable "
3384
+ "the customer runtime-update path."
3385
+ )
3386
+ return None
3387
+
3388
+ lambda_name = f"thinkwork-{stage}-agentcore-pi"
3389
+ fn = json.loads(
3390
+ output(
3391
+ [
3392
+ "aws",
3393
+ "lambda",
3394
+ "get-function-configuration",
3395
+ "--function-name",
3396
+ lambda_name,
3397
+ "--region",
3398
+ region,
3399
+ ]
3400
+ )
3401
+ )
3402
+ environment = {
3403
+ key: escape_env_control_chars(value) if isinstance(value, str) else value
3404
+ for key, value in ((fn.get("Environment") or {}).get("Variables") or {}).items()
3405
+ }
3406
+ if not environment:
3407
+ raise RuntimeError(
3408
+ f"AgentCore Pi runtime reconcile: {lambda_name} reported an EMPTY environment; "
3409
+ "mirroring it would strand the runtime (atomic-or-abort)."
3410
+ )
3411
+ # THINK-586 U7 (KTD6): runtime-only overlay — the warm-session cache
3412
+ # factory gates on this var; it must never enter the Lambda terraform env.
3413
+ environment["AGENTCORE_RUNTIME_SESSION_CACHE"] = "1"
3414
+ role_arn = fn.get("Role") or ""
3415
+ account_id = (fn.get("FunctionArn") or "").split(":")[4] if fn.get("FunctionArn") else ""
3416
+ if not role_arn or not account_id:
3417
+ raise RuntimeError("AgentCore Pi runtime reconcile: Lambda did not report Role/ARN")
3418
+
3419
+ # AgentCore runtimes are arm64-only ("Supported platforms: [arm64]" —
3420
+ # first tei-e2e canary, 2026-08-04). The Lambda's amd64 pin
3421
+ # (agentcorePiSourceImageUri) is the WRONG architecture for the runtime;
3422
+ # the runtime image is mirrored into the customer ECR under the release
3423
+ # version tag by the release workflow's `mirror-customer-images` job
3424
+ # (THINK-616; targets listed in .github/release-mirror-targets.json).
3425
+ release_version = os.environ.get("THINKWORK_RELEASE_VERSION", "") or str(
3426
+ safe_get(payload, "releaseVersion", default="")
3427
+ )
3428
+ if not release_version:
3429
+ raise RuntimeError(
3430
+ "AgentCore Pi runtime reconcile: no release version available to "
3431
+ "resolve the arm64 runtime image tag."
3432
+ )
3433
+ image_tag = f"{release_version}-pi-arm64"
3434
+ repository = f"thinkwork-{stage}-agentcore"
3435
+ digest = output(
3436
+ [
3437
+ "aws",
3438
+ "ecr",
3439
+ "describe-images",
3440
+ "--repository-name",
3441
+ repository,
3442
+ "--region",
3443
+ region,
3444
+ "--image-ids",
3445
+ f"imageTag={image_tag}",
3446
+ "--query",
3447
+ "imageDetails[0].imageDigest",
3448
+ "--output",
3449
+ "text",
3450
+ ]
3451
+ )
3452
+ if not re.fullmatch(r"sha256:[0-9a-f]{64}", digest):
3453
+ raise RuntimeError(
3454
+ f"AgentCore Pi runtime reconcile: could not resolve an immutable digest for "
3455
+ f"{repository}:{image_tag} (got '{digest}'). Mirror the release's "
3456
+ "arm64 Pi runtime image into this customer's ECR under that tag "
3457
+ "before retrying."
3458
+ )
3459
+ image_uri = f"{account_id}.dkr.ecr.{region}.amazonaws.com/{repository}@{digest}"
3460
+
3461
+ ssm_name = f"/thinkwork/{stage}/agentcore/runtime-id-pi"
3462
+ runtime_id = ""
3463
+ try:
3464
+ runtime_id = output(
3465
+ [
3466
+ "aws",
3467
+ "ssm",
3468
+ "get-parameter",
3469
+ "--name",
3470
+ ssm_name,
3471
+ "--region",
3472
+ region,
3473
+ "--query",
3474
+ "Parameter.Value",
3475
+ "--output",
3476
+ "text",
3477
+ ],
3478
+ stderr=subprocess.DEVNULL,
3479
+ )
3480
+ except subprocess.CalledProcessError:
3481
+ runtime_id = ""
3482
+ if runtime_id == "None":
3483
+ runtime_id = ""
3484
+
3485
+ # AgentCore runtime names must match [a-zA-Z][a-zA-Z0-9_]{0,47} — no
3486
+ # hyphens. Customer stages like "tei-e2e" broke the create with a
3487
+ # ValidationException (first live canary of this step, 2026-08-04).
3488
+ runtime_stage = re.sub(r"[^a-zA-Z0-9_]", "_", stage)
3489
+ request = {
3490
+ "region": region,
3491
+ "runtimeName": f"thinkwork_{runtime_stage}_pi",
3492
+ "runtimeId": runtime_id,
3493
+ "roleArn": role_arn,
3494
+ "imageUri": image_uri,
3495
+ "environment": environment,
3496
+ "waitSeconds": 900,
3497
+ }
3498
+ print(
3499
+ f"[runner] Reconciling AgentCore Pi runtime to {image_uri} "
3500
+ f"({len(environment)} env vars mirrored; values not logged)"
3501
+ )
3502
+ reconciled = subprocess.run(
3503
+ ["node", str(entrypoint)],
3504
+ input=json.dumps(request),
3505
+ capture_output=True,
3506
+ text=True,
3507
+ )
3508
+ if reconciled.returncode != 0:
3509
+ raise RuntimeError(
3510
+ "AgentCore Pi runtime reconcile failed: "
3511
+ f"{(reconciled.stderr or '').strip() or f'exit {reconciled.returncode}'}"
3512
+ )
3513
+ result = json.loads(reconciled.stdout)
3514
+ run(
3515
+ [
3516
+ "aws",
3517
+ "ssm",
3518
+ "put-parameter",
3519
+ "--name",
3520
+ ssm_name,
3521
+ "--value",
3522
+ str(result["runtimeId"]),
3523
+ "--type",
3524
+ "String",
3525
+ "--overwrite",
3526
+ "--region",
3527
+ region,
3528
+ ],
3529
+ stdout=subprocess.DEVNULL,
3530
+ )
3531
+ print(
3532
+ f"[runner] AgentCore Pi runtime ready: {result['runtimeId']} "
3533
+ f"v{result.get('version')} ({'created' if result.get('created') else 'updated'})"
3534
+ )
3535
+ return result
3536
+
3537
+
2108
3538
  def release_git_sha():
2109
3539
  if not MANIFEST.exists():
2110
3540
  return ""
@@ -2204,6 +3634,10 @@ def checkout_source(module_source, release_version):
2204
3634
 
2205
3635
 
2206
3636
  def database_url_from_outputs(outputs):
3637
+ """The database handle every schema step uses: a `postgresql://` URL for
3638
+ psql, or a DataApiDatabase when this run's dbAccess is `data-api`."""
3639
+ if DATABASE_ACCESS_MODE == "data-api":
3640
+ return DataApiDatabase.from_outputs(outputs)
2207
3641
  endpoint = outputs.get("db_cluster_endpoint", {}).get("value")
2208
3642
  secret_arn = outputs.get("db_secret_arn", {}).get("value")
2209
3643
  database_name = outputs.get("database_name", {}).get("value") or "thinkwork"
@@ -2235,24 +3669,704 @@ def database_url_from_outputs(outputs):
2235
3669
  )
2236
3670
 
2237
3671
 
2238
- def psql_env(database_url):
2239
- env = os.environ.copy()
2240
- env["DATABASE_URL"] = database_url
2241
- return env
3672
+ def psql_env(database_url):
3673
+ env = os.environ.copy()
3674
+ env["DATABASE_URL"] = database_url
3675
+ return env
3676
+
3677
+
3678
+ DATABASE_URL_PASSWORD_RE = re.compile(r"(://[^/:@\s]*:)[^@\s]*(@)")
3679
+
3680
+
3681
+ def redact_database_url(value):
3682
+ """Replace the password segment of any `scheme://user:password@host` URL
3683
+ in `value` with `***`. Applied to everything a database URL can reach in a
3684
+ log or exception message: a failed psql surfaces its argv (URL included)
3685
+ through CalledProcessError, which CodeBuild prints to CloudWatch."""
3686
+ if isinstance(value, (list, tuple)):
3687
+ return [redact_database_url(item) for item in value]
3688
+ if not isinstance(value, str):
3689
+ return value
3690
+ return DATABASE_URL_PASSWORD_RE.sub(r"\1***\2", value)
3691
+
3692
+
3693
+ def _redacted_psql_error(error):
3694
+ """Re-raise a psql CalledProcessError with the connection password
3695
+ redacted from its command. Same exception type, same returncode, same
3696
+ captured output — only the secret is gone."""
3697
+ return subprocess.CalledProcessError(
3698
+ error.returncode,
3699
+ redact_database_url(error.cmd),
3700
+ output=redact_database_url(error.output),
3701
+ stderr=redact_database_url(error.stderr),
3702
+ )
3703
+
3704
+
3705
+ def psql(database_url, sql=None, file=None, variables=None):
3706
+ if isinstance(database_url, DataApiDatabase):
3707
+ return database_url.execute(sql=sql, file=file, variables=variables)
3708
+ args = ["psql", database_url, "-v", "ON_ERROR_STOP=1"]
3709
+ for key, value in (variables or {}).items():
3710
+ args.extend(["-v", f"{key}={value}"])
3711
+ try:
3712
+ if file:
3713
+ args.extend(["-f", str(file)])
3714
+ return run(args)
3715
+ return subprocess.run(args, input=sql, check=True, text=True)
3716
+ except subprocess.CalledProcessError as error:
3717
+ raise _redacted_psql_error(error) from None
3718
+
3719
+
3720
+ def psql_output(database_url, sql):
3721
+ if isinstance(database_url, DataApiDatabase):
3722
+ return database_url.scalar(sql)
3723
+ try:
3724
+ return output(["psql", database_url, "-tAc", sql])
3725
+ except subprocess.CalledProcessError as error:
3726
+ raise _redacted_psql_error(error) from None
3727
+
3728
+
3729
+ ################################################################################
3730
+ # Database access over the RDS Data API (THINK-1082)
3731
+ #
3732
+ # The runner is the last client of the Aurora cluster's public endpoint. With
3733
+ # every API Lambda inside the VPC, `psql` from this out-of-VPC CodeBuild is
3734
+ # the only reason the world-open 0.0.0.0/0 -> 5432 rule still exists. Setting
3735
+ # `dbAccess: "data-api"` (runner-secrets document, or the controller payload)
3736
+ # routes every psql / psql_output call below through the RDS Data API instead:
3737
+ # HTTPS, IAM-authorised, no network path — the lane the Brain already uses for
3738
+ # its own migrations against these clusters. The call sites do not change;
3739
+ # `database_url_from_outputs` hands back a DataApiDatabase handle instead of a
3740
+ # URL and `psql` / `psql_output` dispatch on it.
3741
+ #
3742
+ # What the Data API does not give us and how it is bridged here:
3743
+ # - one statement per ExecuteStatement -> scripts are split into statements
3744
+ # (dollar-quote / quote / comment aware; drizzle `--> statement-breakpoint`
3745
+ # markers are honoured as boundaries too);
3746
+ # - no psql variables -> `:'name'`, `:"name"` and `:name` are substituted for
3747
+ # the variables the runner passes (`stage`, the compliance passwords) — in
3748
+ # code only, never inside quotes, never touching `::casts`;
3749
+ # - no psql meta-commands -> `\set`, `\echo`/`\qecho`, `\if :{?name}` /
3750
+ # `\else` / `\endif` and `\quit` (the forms the migration files use) are
3751
+ # interpreted; anything else is refused by name;
3752
+ # - 45 s per call -> DDL runs with continueAfterTimeout so a slow statement
3753
+ # is never aborted server-side, and every script runs inside ONE Data API
3754
+ # transaction (BeginTransaction ... CommitTransaction) so its statements
3755
+ # serialise on one connection and a statement that outlived its call can
3756
+ # never race the next one. A script's own top-level BEGIN/COMMIT map onto
3757
+ # that transaction; `CREATE INDEX CONCURRENTLY` (which refuses to run in a
3758
+ # transaction block) commits the open one and runs standalone.
3759
+ # - ON_ERROR_STOP -> the first failing statement stops the script, rolls the
3760
+ # transaction back and raises DatabaseCommandError, a CalledProcessError
3761
+ # subclass, so every existing `except subprocess.CalledProcessError`
3762
+ # keeps working and the migration ledger is recorded only after success.
3763
+ ################################################################################
3764
+
3765
+ DATABASE_ACCESS_MODES = ("psql", "data-api")
3766
+ DATABASE_ACCESS_MODE = "psql"
3767
+ RDS_DATA_API_MAX_SQL_BYTES = 65536
3768
+ STATEMENT_BREAKPOINT = "--> statement-breakpoint"
3769
+
3770
+
3771
+ def set_database_access_mode(vars_json):
3772
+ """Select psql (today) or the RDS Data API for every database step of
3773
+ this run; logged once so the build log proves which lane ran."""
3774
+ global DATABASE_ACCESS_MODE
3775
+ mode = str((vars_json or {}).get("db_access") or "psql")
3776
+ if mode not in DATABASE_ACCESS_MODES:
3777
+ raise RuntimeError(
3778
+ f"dbAccess must be one of {', '.join(DATABASE_ACCESS_MODES)}; got {mode!r}"
3779
+ )
3780
+ DATABASE_ACCESS_MODE = mode
3781
+ print(f"[runner] database access: {mode}")
3782
+ return mode
3783
+
3784
+
3785
+ def safe_get_db_access(runner_secrets, payload):
3786
+ """`dbAccess` rides the same precedence as apiLambdasInVpc: the
3787
+ runner-secrets document is the durable home, the controller payload can
3788
+ override per deploy. Runner-only — it never reaches the Terraform root."""
3789
+ value = safe_get(runner_secrets, "dbAccess", default=safe_get(payload, "dbAccess", default=""))
3790
+ value = value.strip().lower() or "psql"
3791
+ if value not in DATABASE_ACCESS_MODES:
3792
+ raise RuntimeError(
3793
+ f"dbAccess must be one of {', '.join(DATABASE_ACCESS_MODES)}; got {value!r}"
3794
+ )
3795
+ return value
3796
+
3797
+
3798
+ class DatabaseCommandError(subprocess.CalledProcessError):
3799
+ """A failed Data API statement. Subclasses CalledProcessError so every
3800
+ caller that catches psql failures handles it identically; `cmd` names the
3801
+ lane and the script, never a connection string or a secret."""
3802
+
3803
+ def __init__(self, message, label):
3804
+ super().__init__(1, ["rds-data", "execute-statement", label], output="", stderr=message)
3805
+ self.message = message
3806
+
3807
+ def __str__(self):
3808
+ return f"rds-data failed for {self.cmd[2]}: {self.message}"
3809
+
3810
+
3811
+ def _sql_segments(text):
3812
+ """Split SQL text into (kind, chunk) segments: 'code', 'squote',
3813
+ 'dquote', 'dollar', 'line_comment', 'block_comment' and 'meta' (a psql
3814
+ meta-command: an unquoted backslash starting a line, through the end of
3815
+ that line). Concatenating the chunks reproduces the input byte for byte."""
3816
+ segments = []
3817
+ i = 0
3818
+ n = len(text)
3819
+ code_start = 0
3820
+
3821
+ def flush_code(end):
3822
+ if end > code_start:
3823
+ segments.append(("code", text[code_start:end]))
3824
+
3825
+ while i < n:
3826
+ ch = text[i]
3827
+ if ch == "\\" and not text[text.rfind("\n", 0, i) + 1 : i].strip():
3828
+ flush_code(i)
3829
+ end = text.find("\n", i)
3830
+ end = n if end == -1 else end
3831
+ segments.append(("meta", text[i:end]))
3832
+ i = code_start = end
3833
+ continue
3834
+ if ch == "-" and text.startswith("--", i):
3835
+ flush_code(i)
3836
+ end = text.find("\n", i)
3837
+ end = n if end == -1 else end
3838
+ segments.append(("line_comment", text[i:end]))
3839
+ i = code_start = end
3840
+ continue
3841
+ if ch == "/" and text.startswith("/*", i):
3842
+ flush_code(i)
3843
+ depth = 0
3844
+ j = i
3845
+ while j < n:
3846
+ if text.startswith("/*", j):
3847
+ depth += 1
3848
+ j += 2
3849
+ elif text.startswith("*/", j):
3850
+ depth -= 1
3851
+ j += 2
3852
+ if depth == 0:
3853
+ break
3854
+ else:
3855
+ j += 1
3856
+ segments.append(("block_comment", text[i:j]))
3857
+ i = code_start = j
3858
+ continue
3859
+ if ch in "'\"" and PSQL_QUOTED_VARIABLE_RE.match(text, i) and _is_variable_colon(text, i):
3860
+ # psql's :'name' / :"name" — interpolation syntax, not a literal
3861
+ i = PSQL_QUOTED_VARIABLE_RE.match(text, i).end()
3862
+ continue
3863
+ if ch == "'":
3864
+ flush_code(i)
3865
+ escaped = (
3866
+ i > 0
3867
+ and text[i - 1] in "eE"
3868
+ and (i < 2 or not (text[i - 2].isalnum() or text[i - 2] == "_"))
3869
+ )
3870
+ j = i + 1
3871
+ while j < n:
3872
+ if escaped and text[j] == "\\":
3873
+ j += 2
3874
+ continue
3875
+ if text[j] == "'":
3876
+ if j + 1 < n and text[j + 1] == "'":
3877
+ j += 2
3878
+ continue
3879
+ j += 1
3880
+ break
3881
+ j += 1
3882
+ segments.append(("squote", text[i:j]))
3883
+ i = code_start = j
3884
+ continue
3885
+ if ch == '"':
3886
+ flush_code(i)
3887
+ j = i + 1
3888
+ while j < n:
3889
+ if text[j] == '"':
3890
+ if j + 1 < n and text[j + 1] == '"':
3891
+ j += 2
3892
+ continue
3893
+ j += 1
3894
+ break
3895
+ j += 1
3896
+ segments.append(("dquote", text[i:j]))
3897
+ i = code_start = j
3898
+ continue
3899
+ if ch == "$":
3900
+ match = DOLLAR_QUOTE_RE.match(text, i)
3901
+ if match and not (i > 0 and (text[i - 1].isalnum() or text[i - 1] == "_")):
3902
+ tag = match.group(0)
3903
+ flush_code(i)
3904
+ end = text.find(tag, i + len(tag))
3905
+ end = n if end == -1 else end + len(tag)
3906
+ segments.append(("dollar", text[i:end]))
3907
+ i = code_start = end
3908
+ continue
3909
+ i += 1
3910
+ flush_code(n)
3911
+ return segments
3912
+
3913
+
3914
+ DOLLAR_QUOTE_RE = re.compile(r"\$[A-Za-z_][A-Za-z0-9_]*\$|\$\$")
3915
+ PSQL_QUOTED_VARIABLE_RE = re.compile(r"'\w+'|\"\w+\"")
3916
+
3917
+
3918
+ def _is_variable_colon(text, i):
3919
+ """True when the quote at `i` follows a psql variable colon: a `:` that
3920
+ is not part of a `::` cast or of an identifier."""
3921
+ return (
3922
+ i > 0
3923
+ and text[i - 1] == ":"
3924
+ and (i < 2 or not (text[i - 2] == ":" or text[i - 2].isalnum() or text[i - 2] == "_"))
3925
+ )
3926
+
3927
+
3928
+ PSQL_META_RE = re.compile(r"^\s*\\(\w+)\s*(.*?)\s*$")
3929
+ PSQL_VARIABLE_RE = re.compile(r"(?<![:\w]):(?:'(\w+)'|\"(\w+)\"|(\w+))")
3930
+
3931
+
3932
+ def split_sql_script(text):
3933
+ """Split a psql script into ordered items: ('sql', statement) for each
3934
+ top-level `;`-terminated statement (drizzle `--> statement-breakpoint`
3935
+ markers are boundaries too) and ('meta', line) for each psql meta-command
3936
+ line (an unquoted backslash at the start of a line). Comment-only and
3937
+ blank chunks are dropped; comments inside a statement travel with it."""
3938
+ items = []
3939
+ buffer = []
3940
+ has_code = [False]
3941
+
3942
+ def emit():
3943
+ statement = "".join(buffer).strip()
3944
+ if has_code[0] and statement:
3945
+ items.append(("sql", statement))
3946
+ buffer.clear()
3947
+ has_code[0] = False
3948
+
3949
+ for kind, chunk in _sql_segments(text):
3950
+ if kind == "meta":
3951
+ items.append(("meta", chunk.strip()))
3952
+ continue
3953
+ if kind == "line_comment":
3954
+ if chunk.strip() == STATEMENT_BREAKPOINT:
3955
+ emit()
3956
+ else:
3957
+ buffer.append(chunk)
3958
+ continue
3959
+ if kind != "code":
3960
+ buffer.append(chunk)
3961
+ has_code[0] = True
3962
+ continue
3963
+ start = 0
3964
+ for i, ch in enumerate(chunk):
3965
+ if ch == ";":
3966
+ buffer.append(chunk[start:i])
3967
+ emit()
3968
+ start = i + 1
3969
+ elif not ch.isspace():
3970
+ has_code[0] = True
3971
+ buffer.append(chunk[start:])
3972
+ emit()
3973
+ return items
3974
+
3975
+
3976
+ def substitute_psql_variables(sql, variables):
3977
+ """psql `-v` interpolation for the variables given: `:'name'` becomes a
3978
+ SQL string literal, `:"name"` an identifier, bare `:name` the raw value.
3979
+ Only code is touched — never quoted text, comments or `::type` casts —
3980
+ and only names that were actually passed (psql leaves the rest alone)."""
3981
+ if not variables:
3982
+ return sql
3983
+
3984
+ def replace(match):
3985
+ literal, identifier, raw = match.groups()
3986
+ name = literal or identifier or raw
3987
+ if name not in variables:
3988
+ return match.group(0)
3989
+ value = str(variables[name])
3990
+ if literal:
3991
+ return "'" + value.replace("'", "''") + "'"
3992
+ if identifier:
3993
+ return '"' + value.replace('"', '""') + '"'
3994
+ return value
3995
+
3996
+ return "".join(
3997
+ PSQL_VARIABLE_RE.sub(replace, chunk) if kind == "code" else chunk
3998
+ for kind, chunk in _sql_segments(sql)
3999
+ )
4000
+
4001
+
4002
+ def _leading_keywords(statement, count=2):
4003
+ words = []
4004
+ for kind, chunk in _sql_segments(statement):
4005
+ if kind in ("line_comment", "block_comment"):
4006
+ continue
4007
+ if kind != "code":
4008
+ break
4009
+ words.extend(chunk.split())
4010
+ if len(words) >= count:
4011
+ break
4012
+ return tuple(word.upper().rstrip(";") for word in words[:count])
4013
+
4014
+
4015
+ def _statement_runs_outside_transaction(statement):
4016
+ """Postgres refuses these inside a transaction block; psql (autocommit)
4017
+ runs them standalone, so the Data API lane does too."""
4018
+ keywords = _leading_keywords(statement, 4)
4019
+ return "CONCURRENTLY" in keywords or keywords[:1] == ("VACUUM",)
4020
+
4021
+
4022
+ def _rds_data_cell_text(cell):
4023
+ """psql -tA rendering of one Data API field: NULL is empty, booleans are
4024
+ t/f, numbers print plainly, everything else is its string form."""
4025
+ if not isinstance(cell, dict) or cell.get("isNull"):
4026
+ return ""
4027
+ if "booleanValue" in cell:
4028
+ return "t" if cell["booleanValue"] else "f"
4029
+ for key in ("stringValue", "longValue", "doubleValue"):
4030
+ if key in cell:
4031
+ return str(cell[key])
4032
+ if "arrayValue" in cell:
4033
+ values = next(iter(cell["arrayValue"].values()), [])
4034
+ return "{" + ",".join(str(value) for value in values) + "}"
4035
+ if "blobValue" in cell:
4036
+ value = cell["blobValue"]
4037
+ return value.decode("utf-8", "replace") if isinstance(value, bytes) else str(value)
4038
+ return ""
4039
+
4040
+
4041
+ class AwsCliRdsDataClient:
4042
+ """boto3-shaped rds-data client backed by the AWS CLI, for a build image
4043
+ without boto3 (the runner already shells out to `aws` everywhere)."""
4044
+
4045
+ def __init__(self, region):
4046
+ self.region = region
4047
+
4048
+ def _call(self, operation, params):
4049
+ args = ["aws", "rds-data", operation, "--cli-input-json", json.dumps(params)]
4050
+ if self.region:
4051
+ args.extend(["--region", self.region])
4052
+ try:
4053
+ body = subprocess.check_output(args, text=True, stderr=subprocess.PIPE)
4054
+ except subprocess.CalledProcessError as error:
4055
+ message = (error.stderr or "").strip() or f"aws rds-data {operation} failed"
4056
+ raise RuntimeError(message) from None
4057
+ return json.loads(body or "{}")
4058
+
4059
+ def execute_statement(self, **params):
4060
+ return self._call("execute-statement", params)
4061
+
4062
+ def begin_transaction(self, **params):
4063
+ return self._call("begin-transaction", params)
4064
+
4065
+ def commit_transaction(self, **params):
4066
+ return self._call("commit-transaction", params)
4067
+
4068
+ def rollback_transaction(self, **params):
4069
+ return self._call("rollback-transaction", params)
4070
+
4071
+
4072
+ def rds_data_client(region):
4073
+ try:
4074
+ import boto3 # noqa: PLC0415 — optional on the build image
4075
+ except ImportError:
4076
+ return AwsCliRdsDataClient(region)
4077
+ return boto3.client("rds-data", region_name=region) if region else boto3.client("rds-data")
4078
+
4079
+
4080
+ def _cluster_arn_from_outputs(outputs):
4081
+ arn = (outputs.get("db_cluster_arn") or {}).get("value")
4082
+ if arn:
4083
+ return str(arn)
4084
+ endpoint = str((outputs.get("db_cluster_endpoint") or {}).get("value") or "")
4085
+ labels = endpoint.split(".")
4086
+ if len(labels) < 5 or labels[-2] != "amazonaws" or labels[-3] != "rds":
4087
+ raise RuntimeError(
4088
+ "Terraform outputs carry no db_cluster_arn and the endpoint "
4089
+ f"{endpoint!r} is not an RDS hostname; cannot address the RDS Data API"
4090
+ )
4091
+ region = labels[-4]
4092
+ account = output(
4093
+ ["aws", "sts", "get-caller-identity", "--query", "Account", "--output", "text"]
4094
+ )
4095
+ return f"arn:aws:rds:{region}:{account}:cluster:{labels[0]}"
4096
+
4097
+
4098
+ class DataApiDatabase:
4099
+ """The database handle in data-api mode: what `database_url_from_outputs`
4100
+ returns instead of a URL, and what `psql` / `psql_output` dispatch on."""
4101
+
4102
+ def __init__(self, cluster_arn, secret_arn, database_name, client=None, region=None):
4103
+ self.cluster_arn = cluster_arn
4104
+ self.secret_arn = secret_arn
4105
+ self.database_name = database_name
4106
+ self.region = region or os.environ.get("AWS_REGION") or cluster_arn.split(":")[3]
4107
+ self._client = client
4108
+
4109
+ @classmethod
4110
+ def from_outputs(cls, outputs, client=None):
4111
+ secret_arn = (outputs.get("db_secret_arn") or {}).get("value")
4112
+ if not secret_arn:
4113
+ raise RuntimeError("Terraform outputs missing database secret ARN")
4114
+ database_name = (outputs.get("database_name") or {}).get("value") or "thinkwork"
4115
+ return cls(_cluster_arn_from_outputs(outputs), str(secret_arn), str(database_name), client)
4116
+
4117
+ def __repr__(self):
4118
+ return f"DataApiDatabase({self.cluster_arn}, database={self.database_name})"
4119
+
4120
+ __str__ = __repr__
4121
+
4122
+ @property
4123
+ def client(self):
4124
+ if self._client is None:
4125
+ self._client = rds_data_client(self.region)
4126
+ return self._client
4127
+
4128
+ def _params(self, sql, transaction_id=None, continue_after_timeout=False):
4129
+ params = {
4130
+ "resourceArn": self.cluster_arn,
4131
+ "secretArn": self.secret_arn,
4132
+ "database": self.database_name,
4133
+ "sql": sql,
4134
+ }
4135
+ if transaction_id:
4136
+ params["transactionId"] = transaction_id
4137
+ if continue_after_timeout:
4138
+ params["continueAfterTimeout"] = True
4139
+ return params
4140
+
4141
+ def scalar(self, sql):
4142
+ """`psql -tAc`: every row on its own line, columns `|`-joined, NULL
4143
+ empty, booleans t/f, whole result stripped."""
4144
+ try:
4145
+ response = self.client.execute_statement(**self._params(sql))
4146
+ except Exception as error: # noqa: BLE001 — every SDK/CLI failure lands here
4147
+ raise DatabaseCommandError(str(error), "query") from None
4148
+ rows = [
4149
+ "|".join(_rds_data_cell_text(cell) for cell in row)
4150
+ for row in response.get("records") or []
4151
+ ]
4152
+ return "\n".join(rows).strip()
4153
+
4154
+ def execute(self, sql=None, file=None, variables=None):
4155
+ """`psql -v ON_ERROR_STOP=1 [-v k=v] -f file | -c sql`."""
4156
+ if file is not None:
4157
+ label = Path(file).name
4158
+ text = Path(file).read_text(encoding="utf-8")
4159
+ else:
4160
+ label = "inline sql"
4161
+ text = sql or ""
4162
+ variables = {str(key): str(value) for key, value in (variables or {}).items()}
4163
+ statements = self._plan(text, label, variables)
4164
+ self._run(statements, label, variables)
4165
+
4166
+ def _plan(self, text, label, variables):
4167
+ """Interpret meta-commands in order and resolve the statements that
4168
+ will run; refuses the script before anything runs when it needs a
4169
+ meta-command this lane does not implement or a statement the Data
4170
+ API cannot carry."""
4171
+ planned = []
4172
+ conditions = []
4173
+ for kind, item in split_sql_script(text):
4174
+ active = all(conditions)
4175
+ if kind == "meta":
4176
+ match = PSQL_META_RE.match(item)
4177
+ command, argument = (match.group(1), match.group(2)) if match else (item, "")
4178
+ if command == "if":
4179
+ defined = re.fullmatch(r":\{\?(\w+)\}", argument)
4180
+ if not defined:
4181
+ raise DatabaseCommandError(
4182
+ f"unsupported psql meta-command in Data API mode: {item}", label
4183
+ )
4184
+ conditions.append(defined.group(1) in variables)
4185
+ elif command == "else":
4186
+ if not conditions:
4187
+ raise DatabaseCommandError(f"\\else without \\if in {label}", label)
4188
+ conditions[-1] = not conditions[-1]
4189
+ elif command == "endif":
4190
+ if not conditions:
4191
+ raise DatabaseCommandError(f"\\endif without \\if in {label}", label)
4192
+ conditions.pop()
4193
+ elif not active:
4194
+ continue
4195
+ elif command == "set":
4196
+ parts = argument.split(None, 1)
4197
+ if parts:
4198
+ value = parts[1].strip() if len(parts) > 1 else ""
4199
+ if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"":
4200
+ value = value[1:-1]
4201
+ variables[parts[0]] = value
4202
+ elif command in ("echo", "qecho"):
4203
+ message = argument.strip("'")
4204
+ print(f"[psql:{label}] {message}")
4205
+ elif command in ("quit", "q"):
4206
+ # psql: \quit in a script ends that script, exit status 0
4207
+ # (it takes no argument; `\quit 1` still exits 0).
4208
+ print(f"[psql:{label}] \\quit — remaining statements skipped")
4209
+ conditions.clear()
4210
+ break
4211
+ else:
4212
+ raise DatabaseCommandError(
4213
+ f"unsupported psql meta-command in Data API mode: {item}", label
4214
+ )
4215
+ continue
4216
+ if not active:
4217
+ continue
4218
+ statement = substitute_psql_variables(item, variables)
4219
+ if len(
4220
+ statement.encode("utf-8")
4221
+ ) > RDS_DATA_API_MAX_SQL_BYTES and _statement_runs_outside_transaction(statement):
4222
+ raise DatabaseCommandError(
4223
+ f"statement {len(planned) + 1} exceeds the RDS Data API's "
4224
+ f"{RDS_DATA_API_MAX_SQL_BYTES}-byte limit and cannot run inside a "
4225
+ "transaction; split it at `;` boundaries",
4226
+ label,
4227
+ )
4228
+ planned.append(statement)
4229
+ if conditions:
4230
+ raise DatabaseCommandError(f"\\if without \\endif in {label}", label)
4231
+ return planned
4232
+
4233
+ def _run(self, statements, label, variables):
4234
+ transaction_id = None
4235
+ secrets_to_redact = [value for value in variables.values() if len(value) >= 8]
2242
4236
 
4237
+ def redact(message):
4238
+ for secret in secrets_to_redact:
4239
+ message = message.replace(secret, "***")
4240
+ return message
2243
4241
 
2244
- def psql(database_url, sql=None, file=None, variables=None):
2245
- args = ["psql", database_url, "-v", "ON_ERROR_STOP=1"]
2246
- for key, value in (variables or {}).items():
2247
- args.extend(["-v", f"{key}={value}"])
2248
- if file:
2249
- args.extend(["-f", str(file)])
2250
- return run(args)
2251
- return subprocess.run(args, input=sql, check=True, text=True)
4242
+ def begin():
4243
+ return self.client.begin_transaction(
4244
+ resourceArn=self.cluster_arn, secretArn=self.secret_arn, database=self.database_name
4245
+ )["transactionId"]
2252
4246
 
4247
+ try:
4248
+ for index, statement in enumerate(statements, start=1):
4249
+ head = _leading_keywords(statement)
4250
+ if head[:1] in {("BEGIN",), ("START",)}:
4251
+ if transaction_id is None:
4252
+ transaction_id = begin()
4253
+ continue
4254
+ if head[:1] in {("COMMIT",), ("END",)}:
4255
+ if transaction_id is not None:
4256
+ self.client.commit_transaction(
4257
+ resourceArn=self.cluster_arn,
4258
+ secretArn=self.secret_arn,
4259
+ transactionId=transaction_id,
4260
+ )
4261
+ transaction_id = None
4262
+ continue
4263
+ if head[:1] == ("ROLLBACK",):
4264
+ if transaction_id is not None:
4265
+ self.client.rollback_transaction(
4266
+ resourceArn=self.cluster_arn,
4267
+ secretArn=self.secret_arn,
4268
+ transactionId=transaction_id,
4269
+ )
4270
+ transaction_id = None
4271
+ continue
4272
+ if _statement_runs_outside_transaction(statement):
4273
+ if transaction_id is not None:
4274
+ self.client.commit_transaction(
4275
+ resourceArn=self.cluster_arn,
4276
+ secretArn=self.secret_arn,
4277
+ transactionId=transaction_id,
4278
+ )
4279
+ transaction_id = None
4280
+ self._execute_one(statement, None, index, label, redact)
4281
+ continue
4282
+ if transaction_id is None:
4283
+ transaction_id = begin()
4284
+ self._execute_one(statement, transaction_id, index, label, redact)
4285
+ if transaction_id is not None:
4286
+ self.client.commit_transaction(
4287
+ resourceArn=self.cluster_arn,
4288
+ secretArn=self.secret_arn,
4289
+ transactionId=transaction_id,
4290
+ )
4291
+ except BaseException:
4292
+ if transaction_id is not None:
4293
+ try:
4294
+ self.client.rollback_transaction(
4295
+ resourceArn=self.cluster_arn,
4296
+ secretArn=self.secret_arn,
4297
+ transactionId=transaction_id,
4298
+ )
4299
+ except Exception: # noqa: BLE001 — rollback is best-effort
4300
+ pass
4301
+ raise
2253
4302
 
2254
- def psql_output(database_url, sql):
2255
- return output(["psql", database_url, "-tAc", sql])
4303
+ def _execute_one(self, statement, transaction_id, index, label, redact):
4304
+ if len(statement.encode("utf-8")) > RDS_DATA_API_MAX_SQL_BYTES:
4305
+ self._execute_oversized(statement, transaction_id, index, label, redact)
4306
+ return
4307
+ try:
4308
+ self.client.execute_statement(
4309
+ **self._params(statement, transaction_id, continue_after_timeout=True)
4310
+ )
4311
+ except Exception as error: # noqa: BLE001 — every SDK/CLI failure lands here
4312
+ first_line = statement.strip().splitlines()[0][:120]
4313
+ raise DatabaseCommandError(
4314
+ redact(f"statement {index} ({first_line}): {error}"), label
4315
+ ) from None
4316
+
4317
+ def _execute_oversized(self, statement, transaction_id, index, label, redact):
4318
+ """A statement over the Data API's per-call limit (0230_analyst_rls
4319
+ carries an 85 KB DO block) is shipped in pieces into a temp table on
4320
+ the transaction's connection, reassembled and run there with EXECUTE,
4321
+ then the temp table is dropped. Same statement, same transaction."""
4322
+ if transaction_id is None:
4323
+ raise DatabaseCommandError(
4324
+ f"statement {index} exceeds {RDS_DATA_API_MAX_SQL_BYTES} bytes and "
4325
+ "cannot run inside a transaction",
4326
+ label,
4327
+ )
4328
+ tag = "$chunk$"
4329
+ suffix = 0
4330
+ while tag in statement:
4331
+ suffix += 1
4332
+ tag = f"$chunk{suffix}$"
4333
+ table = "pg_temp.runner_oversized_statement"
4334
+ pieces = []
4335
+ current = []
4336
+ size = 0
4337
+ for char in statement:
4338
+ width = len(char.encode("utf-8"))
4339
+ if size + width > RDS_DATA_API_MAX_SQL_BYTES - 4096:
4340
+ pieces.append("".join(current))
4341
+ current, size = [], 0
4342
+ current.append(char)
4343
+ size += width
4344
+ if current:
4345
+ pieces.append("".join(current))
4346
+ steps = [f"CREATE TEMP TABLE {table.split('.')[1]} (n integer PRIMARY KEY, chunk text)"]
4347
+ steps.extend(
4348
+ f"INSERT INTO {table} (n, chunk) VALUES ({number}, {tag}{piece}{tag})"
4349
+ for number, piece in enumerate(pieces)
4350
+ )
4351
+ steps.append(
4352
+ "DO $runner$ BEGIN EXECUTE (SELECT string_agg(chunk, '' ORDER BY n) "
4353
+ f"FROM {table}); END $runner$"
4354
+ )
4355
+ steps.append(f"DROP TABLE {table}")
4356
+ print(
4357
+ f"[psql:{label}] statement {index} is {len(statement.encode('utf-8'))} bytes; "
4358
+ f"shipping it to the server in {len(pieces)} pieces"
4359
+ )
4360
+ for step in steps:
4361
+ try:
4362
+ self.client.execute_statement(
4363
+ **self._params(step, transaction_id, continue_after_timeout=True)
4364
+ )
4365
+ except Exception as error: # noqa: BLE001 — every SDK/CLI failure lands here
4366
+ first_line = statement.strip().splitlines()[0][:120]
4367
+ raise DatabaseCommandError(
4368
+ redact(f"statement {index} ({first_line}): {error}"), label
4369
+ ) from None
2256
4370
 
2257
4371
 
2258
4372
  def put_secret_value(secret_id, payload):
@@ -2957,31 +5071,6 @@ def pg_literal(value):
2957
5071
  return "'" + value.replace("'", "''") + "'"
2958
5072
 
2959
5073
 
2960
- def ensure_hindsight_database(database_url, vars_json):
2961
- """THINK-220: the dedicated Hindsight database is bootstrap SQL — Terraform
2962
- cannot CREATE DATABASE inside the cluster. Runs post-apply; the Hindsight
2963
- service may crash-loop briefly on a first enablement until this creates
2964
- the database, then ECS stabilizes on its own. Vanilla boot migrations
2965
- (RUN_MIGRATIONS_ON_STARTUP) build the schema, including the maintenance
2966
- discovery functions the repair migration installs on base-schema runs."""
2967
- db_name = (vars_json.get("hindsight_database_name") or "").strip()
2968
- if not db_name:
2969
- return
2970
- if not re.fullmatch(r"[a-z][a-z0-9_]*", db_name):
2971
- raise RuntimeError(f"invalid hindsight_database_name: {db_name!r}")
2972
- admin_url = re.sub(r"/[^/?]+(\?|$)", r"/postgres\1", database_url, count=1)
2973
- exists = psql_output(
2974
- admin_url,
2975
- f"SELECT 1 FROM pg_database WHERE datname = '{db_name}'",
2976
- ).strip()
2977
- if not exists:
2978
- print(f"[hindsight] creating database {db_name}")
2979
- psql(admin_url, sql=f'CREATE DATABASE "{db_name}"')
2980
- hindsight_url = re.sub(r"/[^/?]+(\?|$)", f"/{db_name}\\1", database_url, count=1)
2981
- psql(hindsight_url, sql="CREATE EXTENSION IF NOT EXISTS vector;")
2982
- print(f"[hindsight] database {db_name} ready (pgvector ensured)")
2983
-
2984
-
2985
5074
  def push_database_schema(outputs_path, vars_json):
2986
5075
  outputs = json.loads(outputs_path.read_text(encoding="utf-8"))
2987
5076
  checkout_source(
@@ -2990,10 +5079,11 @@ def push_database_schema(outputs_path, vars_json):
2990
5079
  os.environ.get("THINKWORK_RELEASE_VERSION", "main"),
2991
5080
  )
2992
5081
  database_url = database_url_from_outputs(outputs)
2993
- ensure_hindsight_database(database_url, vars_json)
2994
- fresh = not psql_output(database_url, "SELECT to_regclass('public.tenants')").strip()
5082
+ fresh = not psql_output(database_url, "SELECT to_regclass('public.tenants')::text").strip()
2995
5083
  ledger_present = bool(
2996
- psql_output(database_url, "SELECT to_regclass('public.platform_schema_migrations')").strip()
5084
+ psql_output(
5085
+ database_url, "SELECT to_regclass('public.platform_schema_migrations')::text"
5086
+ ).strip()
2997
5087
  )
2998
5088
  if fresh:
2999
5089
  initialize_greenfield_database(database_url, outputs, vars_json)
@@ -3158,11 +5248,7 @@ def write_runner_files(payload, runner_secrets):
3158
5248
  current_outputs=current_outputs,
3159
5249
  )
3160
5250
  prior_phase_output = current_outputs.get("auth_retirement_phase")
3161
- prior_phase = (
3162
- prior_phase_output.get("value")
3163
- if isinstance(prior_phase_output, dict)
3164
- else None
3165
- )
5251
+ prior_phase = prior_phase_output.get("value") if isinstance(prior_phase_output, dict) else None
3166
5252
  if prior_phase not in {"coexistence", "cutover", "retired"}:
3167
5253
  # An existing pre-native-auth stack must enter through coexistence.
3168
5254
  # A truly empty state is greenfield and has no WorkOS runtime to keep.
@@ -3217,32 +5303,115 @@ def write_runner_files(payload, runner_secrets):
3217
5303
  "databaseEngine",
3218
5304
  default="aurora-serverless",
3219
5305
  ),
3220
- "enable_hindsight": safe_get_bool(
3221
- {},
5306
+ # External S3 KB source R20: customer-owned buckets granted to the
5307
+ # KB service role for s3-connect sources. Per-environment infra
5308
+ # posture — durable home is the runner-secrets document; a payload
5309
+ # list can override per-deploy. Accepts a JSON list or a
5310
+ # comma-separated string (Secrets Manager values often arrive as
5311
+ # strings).
5312
+ "external_kb_source_arns": safe_get_list(
5313
+ runner_secrets,
5314
+ reviewed_payload,
5315
+ "externalKbSourceArns",
5316
+ ),
5317
+ # VPC endpoint diet (cost review 2026-08-28): which AWS services get
5318
+ # a private interface endpoint, and one ENI or one per AZ. The
5319
+ # private subnets are NAT-routed, so this is network posture, not
5320
+ # reachability. Environment-level — durable home is the
5321
+ # runner-secrets document; the payload can override per-deploy.
5322
+ # Absent = the module's legacy full set across every AZ; the literal
5323
+ # "none" = no interface endpoints at all.
5324
+ "vpc_interface_endpoint_services": safe_get_list(
5325
+ runner_secrets,
5326
+ reviewed_payload,
5327
+ "vpcInterfaceEndpointServices",
5328
+ ),
5329
+ "vpc_interface_endpoint_single_az": safe_get_bool(
5330
+ runner_secrets,
5331
+ reviewed_payload,
5332
+ "vpcInterfaceEndpointSingleAz",
5333
+ default=False,
5334
+ ),
5335
+ # AgentCore Harness is an environment-level deployment choice. The
5336
+ # controller carries these reviewed, non-secret values forward so a
5337
+ # release update cannot silently remove the managed runtime plane.
5338
+ # The registry module retains its rollout-era multiplayer-proof input
5339
+ # names for compatibility; all provisioning still happens in the
5340
+ # normal Terraform apply owned by this controller.
5341
+ # THINK-1082: run the deployment runner inside the VPC so the psql
5342
+ # schema step reaches Aurora by security group instead of the
5343
+ # world-open 0.0.0.0/0 -> 5432 rule (turns on the VPC NAT gateway).
5344
+ # Environment-level network posture — durable home is the
5345
+ # runner-secrets document (deploymentRunnerInVpc); the controller
5346
+ # payload can override per-deploy. Ship-inert default.
5347
+ "deployment_runner_in_vpc": safe_get_bool(
5348
+ runner_secrets,
5349
+ reviewed_payload,
5350
+ "deploymentRunnerInVpc",
5351
+ default=False,
5352
+ ),
5353
+ # THINK-1082: every database-connected Lambda (api handlers,
5354
+ # agentcore-admin, Pi, broker) inside the VPC behind the api-lambda
5355
+ # security group, granted 5432 on the DB group — so the world-open
5356
+ # 0.0.0.0/0 -> 5432 rule can close without taking the app down.
5357
+ # Same homes as deploymentRunnerInVpc: runner-secrets document is
5358
+ # durable, payload overrides per-deploy. Ship-inert default.
5359
+ "api_lambdas_in_vpc": safe_get_bool(
5360
+ runner_secrets,
3222
5361
  reviewed_payload,
3223
- "enableHindsight",
5362
+ "apiLambdasInVpc",
3224
5363
  default=False,
3225
5364
  ),
3226
- # Analyst data-path Lambda VPC egress (stable NAT EIP for external
3227
- # database IP allowlists). Per-environment infra posture, so the
3228
- # durable home is the runner-secrets document; a payload boolean
3229
- # can override per-deploy.
3230
- "analyst_lambda_vpc_egress": safe_get_bool(
5365
+ # THINK-1082: database exposure. Defaults reproduce today's posture
5366
+ # (public endpoint + world-open 5432 seeded on a new DB security
5367
+ # group). Closed per stage only after both *InVpc flags are live.
5368
+ # dbPublicIngressCidrs accepts a JSON list or a comma-separated
5369
+ # string; an explicit empty string / [] means "seed no rule".
5370
+ "db_publicly_accessible": safe_get_bool(
5371
+ runner_secrets,
5372
+ reviewed_payload,
5373
+ "dbPubliclyAccessible",
5374
+ default=True,
5375
+ ),
5376
+ "db_public_ingress_cidrs": safe_get_ingress_cidrs(
3231
5377
  runner_secrets,
3232
5378
  reviewed_payload,
3233
- "analystLambdaVpcEgress",
5379
+ "dbPublicIngressCidrs",
5380
+ default=["0.0.0.0/0"],
5381
+ ),
5382
+ # THINK-1082: how this runner reaches the database for its schema
5383
+ # steps — `psql` (public endpoint, today) or `data-api` (RDS Data API:
5384
+ # HTTPS + IAM, no network path, so the runner can stay outside the
5385
+ # VPC while the public 5432 rule closes). Runner-only: it never
5386
+ # reaches the generated Terraform root (see RUNNER_ONLY_VARS).
5387
+ "db_access": safe_get_db_access(runner_secrets, reviewed_payload),
5388
+ "enable_agentcore_multiplayer_proof": safe_get_bool(
5389
+ {},
5390
+ reviewed_payload,
5391
+ "enableAgentCoreHarness",
3234
5392
  default=False,
3235
5393
  ),
3236
- # THINK-220: dedicated Hindsight database on the stage cluster.
3237
- # Empty keeps the legacy hindsight-schema layout; set (e.g.
3238
- # "thinkwork_hindsight") points the Hindsight service and platform
3239
- # readers at that database's public schema, where upstream's
3240
- # maintenance discovery actually runs.
3241
- "hindsight_database_name": safe_get(
5394
+ "agentcore_multiplayer_proof_tenant_slug": safe_get(
5395
+ reviewed_payload,
5396
+ "agentCoreHarnessTenantSlug",
5397
+ default="",
5398
+ ),
5399
+ "agentcore_multiplayer_proof_owner_allowlist": safe_get(
3242
5400
  reviewed_payload,
3243
- "hindsightDatabaseName",
5401
+ "agentCoreHarnessOwnerAllowlist",
3244
5402
  default="",
3245
5403
  ),
5404
+ # THINK-585/587 — stage kill-switch for AgentCore Runtime chat
5405
+ # dispatch. Environment posture: durable home is the runner-secrets
5406
+ # document (agentcoreRuntimeDispatchEnabled); a reviewed payload can
5407
+ # override per-deploy. Dark by default — flipping a customer stage is
5408
+ # an explicit operator action.
5409
+ "agentcore_runtime_dispatch_enabled": safe_get_bool(
5410
+ runner_secrets,
5411
+ reviewed_payload,
5412
+ "agentcoreRuntimeDispatchEnabled",
5413
+ default=False,
5414
+ ),
3246
5415
  "platform_operator_emails": safe_get(
3247
5416
  runner_secrets,
3248
5417
  "adminEmail",
@@ -3358,6 +5527,20 @@ def write_runner_files(payload, runner_secrets):
3358
5527
  "customerDomainLegacyRetired",
3359
5528
  default=False,
3360
5529
  ),
5530
+ # Extra web-family OAuth callback/logout origins (e.g. the customer's
5531
+ # Company Brain console CloudFront domain, hosted in their own account).
5532
+ # Persisted in runner-secrets so they survive every controller apply
5533
+ # instead of being stripped back to the computed defaults.
5534
+ "additional_admin_callback_urls": runner_secret_string_list(
5535
+ runner_secrets,
5536
+ reviewed_payload,
5537
+ "adminCallbackUrls",
5538
+ ),
5539
+ "additional_admin_logout_urls": runner_secret_string_list(
5540
+ runner_secrets,
5541
+ reviewed_payload,
5542
+ "adminLogoutUrls",
5543
+ ),
3361
5544
  "lambda_artifact_bucket": os.environ["THINKWORK_RELEASE_ARTIFACT_BUCKET"],
3362
5545
  "lambda_artifact_prefix": f"releases/{release_version}/lambdas",
3363
5546
  "deployment_release_version": release_version,
@@ -3374,6 +5557,75 @@ def write_runner_files(payload, runner_secrets):
3374
5557
  "deployment_terraform_module_source": terraform_module_source,
3375
5558
  "deployment_terraform_module_version": terraform_module_version,
3376
5559
  "agentcore_pi_source_image_uri": resolve_agentcore_pi_source_image_uri(payload),
5560
+ # Account-level Lambda memory ceiling. Legacy AWS accounts cap
5561
+ # function memory at 3008 MB (raised only via support case); set the
5562
+ # runner-secrets key lambdaMaxMemoryMb on such accounts so handlers
5563
+ # that request more are clamped instead of failing mid-apply.
5564
+ "lambda_max_memory_mb": safe_get(
5565
+ runner_secrets,
5566
+ "lambdaMaxMemoryMb",
5567
+ default=safe_get(reviewed_payload, "lambdaMaxMemoryMb", default=10240),
5568
+ ),
5569
+ # Digital Twin Neptune wiring (THINK-334): values come from the
5570
+ # etl-platform neptune stack outputs, written into the runner-secrets
5571
+ # document by `thinkwork twin install`. Empty = twin disabled.
5572
+ "neptune_endpoint": safe_get(
5573
+ runner_secrets,
5574
+ "neptuneEndpoint",
5575
+ default=safe_get(reviewed_payload, "neptuneEndpoint", default=""),
5576
+ ),
5577
+ "neptune_cluster_resource_id": safe_get(
5578
+ runner_secrets,
5579
+ "neptuneClusterResourceId",
5580
+ default=safe_get(reviewed_payload, "neptuneClusterResourceId", default=""),
5581
+ ),
5582
+ "neptune_client_security_group_id": safe_get(
5583
+ runner_secrets,
5584
+ "neptuneClientSecurityGroupId",
5585
+ default=safe_get(reviewed_payload, "neptuneClientSecurityGroupId", default=""),
5586
+ ),
5587
+ "neptune_load_bucket": safe_get(
5588
+ runner_secrets,
5589
+ "neptuneLoadBucket",
5590
+ default=safe_get(reviewed_payload, "neptuneLoadBucket", default=""),
5591
+ ),
5592
+ "neptune_loader_role_arn": safe_get(
5593
+ runner_secrets,
5594
+ "neptuneLoaderRoleArn",
5595
+ default=safe_get(reviewed_payload, "neptuneLoaderRoleArn", default=""),
5596
+ ),
5597
+ # Platform Brain MCP endpoint (consolidation U14). Without this
5598
+ # mapping, customer stages never set BRAIN_MCP_URL and twin-connector
5599
+ # provisioning falls back to the retired product /mcp/twin route.
5600
+ "brain_mcp_url": safe_get(
5601
+ runner_secrets,
5602
+ "brainMcpUrl",
5603
+ default=safe_get(reviewed_payload, "brainMcpUrl", default=""),
5604
+ ),
5605
+ # Company Brain platform-agent machine lane (THINK-628/THINK-710).
5606
+ # Secrets Manager ARN of etl-platform/brain-mcp/m2m/platform-agent in
5607
+ # this account; grants the shared API Lambda role read so MCP config
5608
+ # build can mint Cognito bearers for the Brain connector. Environment
5609
+ # posture: durable home is the runner-secrets document. Empty leaves
5610
+ # the tkt_ lane untouched.
5611
+ "brain_m2m_platform_agent_secret_arn": safe_get(
5612
+ runner_secrets,
5613
+ "brainM2mPlatformAgentSecretArn",
5614
+ default=safe_get(reviewed_payload, "brainM2mPlatformAgentSecretArn", default=""),
5615
+ ),
5616
+ # Brain ops-api flag intake (THINK-781): base URL + agent-identity
5617
+ # m2m secret ARN for flagThreadToBrain's POST /flags. Empty leaves
5618
+ # the Send-to-the-Brain action reporting "not configured".
5619
+ "brain_ops_api_url": safe_get(
5620
+ runner_secrets,
5621
+ "brainOpsApiUrl",
5622
+ default=safe_get(reviewed_payload, "brainOpsApiUrl", default=""),
5623
+ ),
5624
+ "brain_ops_m2m_secret_arn": safe_get(
5625
+ runner_secrets,
5626
+ "brainOpsM2mSecretArn",
5627
+ default=safe_get(reviewed_payload, "brainOpsM2mSecretArn", default=""),
5628
+ ),
3377
5629
  }
3378
5630
  enforce_customer_domain_preservation(
3379
5631
  current_outputs,
@@ -3384,6 +5636,7 @@ def write_runner_files(payload, runner_secrets):
3384
5636
  vars_json.update(
3385
5637
  managed_app_terraform_overrides(payload, stage, account_id, current_outputs, current_state)
3386
5638
  )
5639
+ incident_import_blocks = tei_v380_recovery_import_blocks(payload, vars_json)
3387
5640
 
3388
5641
  TF.mkdir(parents=True, exist_ok=True)
3389
5642
  (TF / "backend.hcl").write_text(
@@ -3400,13 +5653,20 @@ def write_runner_files(payload, runner_secrets):
3400
5653
  encoding="utf-8",
3401
5654
  )
3402
5655
  (TF / "terraform.auto.tfvars.json").write_text(
3403
- json.dumps(vars_json, indent=2, sort_keys=True),
5656
+ json.dumps(
5657
+ {key: value for key, value in vars_json.items() if key not in RUNNER_ONLY_VARS},
5658
+ indent=2,
5659
+ sort_keys=True,
5660
+ ),
3404
5661
  encoding="utf-8",
3405
5662
  )
3406
5663
  (TF / "main.tf").write_text(
3407
5664
  f"""
3408
5665
  terraform {{
3409
- required_version = ">= 1.5"
5666
+ # The managed deployment runtime is pinned to this floor. Terraform 1.8.5
5667
+ # adds declarative state-only removal, used below to forget the original
5668
+ # non-owning AgentCore reconciliation marker without a destroy action.
5669
+ required_version = ">= 1.8.5"
3410
5670
 
3411
5671
  backend "s3" {{}}
3412
5672
 
@@ -3440,6 +5700,21 @@ provider "aws" {{
3440
5700
  # even though they do not change those records.
3441
5701
  provider "cloudflare" {{}}
3442
5702
 
5703
+ # The original AgentCore Twenty reconciliation marker never owned external
5704
+ # cleanup, but an interrupted local-exec may have left it tainted. Forget every
5705
+ # counted instance from managed-deployment state without refresh, provider RPC,
5706
+ # or destroy provisioners. The child module owns the replacement idempotent
5707
+ # reconciliation marker under a new address.
5708
+ removed {{
5709
+ from = module.thinkwork.module.agentcore_proof_identity.terraform_data.twenty_identity_lifecycle
5710
+
5711
+ lifecycle {{
5712
+ destroy = false
5713
+ }}
5714
+ }}
5715
+
5716
+ {incident_import_blocks}
5717
+
3443
5718
  variable "stage" {{
3444
5719
  type = string
3445
5720
  }}
@@ -3476,20 +5751,91 @@ variable "database_engine" {{
3476
5751
  type = string
3477
5752
  }}
3478
5753
 
3479
- variable "enable_hindsight" {{
3480
- type = bool
5754
+ variable "external_kb_source_arns" {{
5755
+ type = list(string)
5756
+ default = []
5757
+ }}
5758
+
5759
+ variable "vpc_interface_endpoint_services" {{
5760
+ type = list(string)
5761
+ default = []
5762
+ }}
5763
+
5764
+ variable "vpc_interface_endpoint_single_az" {{
5765
+ type = bool
5766
+ default = false
5767
+ }}
5768
+
5769
+ variable "lambda_max_memory_mb" {{
5770
+ type = number
5771
+ default = 10240
5772
+ }}
5773
+
5774
+ variable "neptune_endpoint" {{
5775
+ type = string
5776
+ default = ""
5777
+ }}
5778
+
5779
+ variable "neptune_cluster_resource_id" {{
5780
+ type = string
5781
+ default = ""
5782
+ }}
5783
+
5784
+ variable "neptune_client_security_group_id" {{
5785
+ type = string
5786
+ default = ""
5787
+ }}
5788
+
5789
+ variable "neptune_load_bucket" {{
5790
+ type = string
5791
+ default = ""
5792
+ }}
5793
+
5794
+ variable "neptune_loader_role_arn" {{
5795
+ type = string
5796
+ default = ""
5797
+ }}
5798
+
5799
+ variable "brain_mcp_url" {{
5800
+ type = string
5801
+ default = ""
5802
+ }}
5803
+
5804
+ variable "brain_m2m_platform_agent_secret_arn" {{
5805
+ type = string
5806
+ default = ""
5807
+ }}
5808
+
5809
+ variable "brain_ops_api_url" {{
5810
+ type = string
5811
+ default = ""
5812
+ }}
5813
+
5814
+ variable "brain_ops_m2m_secret_arn" {{
5815
+ type = string
5816
+ default = ""
3481
5817
  }}
3482
5818
 
3483
- variable "analyst_lambda_vpc_egress" {{
5819
+ variable "enable_agentcore_multiplayer_proof" {{
3484
5820
  type = bool
3485
5821
  default = false
3486
5822
  }}
3487
5823
 
3488
- variable "hindsight_database_name" {{
5824
+ variable "agentcore_multiplayer_proof_tenant_slug" {{
5825
+ type = string
5826
+ default = ""
5827
+ }}
5828
+
5829
+ variable "agentcore_multiplayer_proof_owner_allowlist" {{
3489
5830
  type = string
3490
5831
  default = ""
3491
5832
  }}
3492
5833
 
5834
+ variable "agentcore_runtime_dispatch_enabled" {{
5835
+ type = bool
5836
+ default = false
5837
+ }}
5838
+
3493
5839
  variable "platform_operator_emails" {{
3494
5840
  type = string
3495
5841
  }}
@@ -3526,6 +5872,16 @@ variable "customer_domain_legacy_retired" {{
3526
5872
  type = bool
3527
5873
  }}
3528
5874
 
5875
+ variable "additional_admin_callback_urls" {{
5876
+ type = list(string)
5877
+ default = []
5878
+ }}
5879
+
5880
+ variable "additional_admin_logout_urls" {{
5881
+ type = list(string)
5882
+ default = []
5883
+ }}
5884
+
3529
5885
  variable "google_oauth_client_id" {{
3530
5886
  type = string
3531
5887
  }}
@@ -3624,6 +5980,26 @@ variable "enable_deployment_control_plane" {{
3624
5980
  type = bool
3625
5981
  }}
3626
5982
 
5983
+ variable "deployment_runner_in_vpc" {{
5984
+ type = bool
5985
+ default = false
5986
+ }}
5987
+
5988
+ variable "api_lambdas_in_vpc" {{
5989
+ type = bool
5990
+ default = false
5991
+ }}
5992
+
5993
+ variable "db_publicly_accessible" {{
5994
+ type = bool
5995
+ default = true
5996
+ }}
5997
+
5998
+ variable "db_public_ingress_cidrs" {{
5999
+ type = list(string)
6000
+ default = ["0.0.0.0/0"]
6001
+ }}
6002
+
3627
6003
  variable "twenty_provisioned" {{
3628
6004
  type = bool
3629
6005
  }}
@@ -3907,15 +6283,36 @@ module "thinkwork" {{
3907
6283
  customer_domain_delegated = var.customer_domain_delegated
3908
6284
  customer_domain_legacy_retired = var.customer_domain_legacy_retired
3909
6285
 
6286
+ additional_admin_callback_urls = var.additional_admin_callback_urls
6287
+ additional_admin_logout_urls = var.additional_admin_logout_urls
6288
+
3910
6289
  lambda_artifact_bucket = var.lambda_artifact_bucket
3911
6290
  lambda_artifact_prefix = var.lambda_artifact_prefix
3912
6291
  require_lambda_artifacts = true
3913
6292
  agentcore_pi_source_image_uri = var.agentcore_pi_source_image_uri
3914
6293
 
3915
- enable_hindsight = var.enable_hindsight
3916
- analyst_lambda_vpc_egress = var.analyst_lambda_vpc_egress
3917
- hindsight_database_name = var.hindsight_database_name
6294
+ external_kb_source_arns = var.external_kb_source_arns
6295
+ vpc_interface_endpoint_services = (
6296
+ length(var.vpc_interface_endpoint_services) == 0
6297
+ ? null
6298
+ : (contains(var.vpc_interface_endpoint_services, "none") ? [] : var.vpc_interface_endpoint_services)
6299
+ )
6300
+ vpc_interface_endpoint_single_az = var.vpc_interface_endpoint_single_az
6301
+ lambda_max_memory_mb = var.lambda_max_memory_mb
6302
+ neptune_endpoint = var.neptune_endpoint
6303
+ neptune_cluster_resource_id = var.neptune_cluster_resource_id
6304
+ neptune_client_security_group_id = var.neptune_client_security_group_id
6305
+ neptune_load_bucket = var.neptune_load_bucket
6306
+ neptune_loader_role_arn = var.neptune_loader_role_arn
6307
+ brain_mcp_url = var.brain_mcp_url
6308
+ brain_m2m_platform_agent_secret_arn = var.brain_m2m_platform_agent_secret_arn
6309
+ brain_ops_api_url = var.brain_ops_api_url
6310
+ brain_ops_m2m_secret_arn = var.brain_ops_m2m_secret_arn
3918
6311
  enable_workspace_orchestration = true
6312
+ enable_agentcore_multiplayer_proof = var.enable_agentcore_multiplayer_proof
6313
+ agentcore_multiplayer_proof_tenant_slug = var.agentcore_multiplayer_proof_tenant_slug
6314
+ agentcore_multiplayer_proof_owner_allowlist = var.agentcore_multiplayer_proof_owner_allowlist
6315
+ agentcore_runtime_dispatch_enabled = var.agentcore_runtime_dispatch_enabled
3919
6316
 
3920
6317
  twenty_provisioned = var.twenty_provisioned
3921
6318
  twenty_runtime_enabled = var.twenty_runtime_enabled
@@ -3965,6 +6362,10 @@ module "thinkwork" {{
3965
6362
  enable_slack_workspace_app = false
3966
6363
 
3967
6364
  enable_deployment_control_plane = var.enable_deployment_control_plane
6365
+ deployment_runner_in_vpc = var.deployment_runner_in_vpc
6366
+ api_lambdas_in_vpc = var.api_lambdas_in_vpc
6367
+ db_publicly_accessible = var.db_publicly_accessible
6368
+ db_public_ingress_cidrs = var.db_public_ingress_cidrs
3968
6369
  deployment_control_plane_create_secret_placeholders = var.deployment_control_plane_create_secret_placeholders
3969
6370
  deployment_state_machine_arn = var.deployment_state_machine_arn
3970
6371
  deployment_evidence_bucket = var.deployment_evidence_bucket
@@ -3999,6 +6400,7 @@ output "appsync_realtime_url" {{ value = module.thinkwork.appsync_realtime_url }
3999
6400
  output "auth_domain" {{ value = module.thinkwork.auth_domain }}
4000
6401
  output "customer_domain" {{ value = module.thinkwork.customer_domain }}
4001
6402
  output "customer_domain_name_servers" {{ value = module.thinkwork.customer_domain_name_servers }}
6403
+ output "db_cluster_arn" {{ value = module.thinkwork.db_cluster_arn }}
4002
6404
  output "db_cluster_endpoint" {{ value = module.thinkwork.db_cluster_endpoint }}
4003
6405
  output "db_secret_arn" {{ value = module.thinkwork.db_secret_arn }}
4004
6406
  output "database_name" {{ value = module.thinkwork.database_name }}
@@ -4007,9 +6409,6 @@ output "admin_client_id" {{ value = module.thinkwork.admin_client_id }}
4007
6409
  output "web_local_client_id" {{ value = module.thinkwork.web_local_client_id }}
4008
6410
  output "auth_route_clients" {{ value = module.thinkwork.auth_route_clients }}
4009
6411
  output "auth_retirement_phase" {{ value = module.thinkwork.auth_retirement_phase }}
4010
- output "docs_bucket_name" {{ value = module.thinkwork.docs_bucket_name }}
4011
- output "docs_distribution_id" {{ value = module.thinkwork.docs_distribution_id }}
4012
- output "docs_distribution_domain" {{ value = module.thinkwork.docs_distribution_domain }}
4013
6412
  output "twenty_provisioned" {{ value = module.thinkwork.twenty_provisioned }}
4014
6413
  output "twenty_runtime_enabled" {{ value = module.thinkwork.twenty_runtime_enabled }}
4015
6414
  output "twenty_url" {{ value = module.thinkwork.twenty_url }}
@@ -4050,7 +6449,7 @@ output "deployment_appconfig_configuration_profile_id" {{ value = module.thinkwo
4050
6449
 
4051
6450
  def sync_release_artifacts(artifact_types=None, artifact_names=None):
4052
6451
  global RELEASE_EVIDENCE
4053
- artifact_types = set(artifact_types or {"lambda", "static-site"})
6452
+ artifact_types = {"lambda", "static-site"} if artifact_types is None else set(artifact_types)
4054
6453
  artifact_names = set(artifact_names or [])
4055
6454
  manifest_url = os.environ.get("THINKWORK_RELEASE_MANIFEST_URL")
4056
6455
  expected = os.environ.get("THINKWORK_RELEASE_MANIFEST_SHA256", "").lower()
@@ -4221,6 +6620,69 @@ def write_current_outputs_from_state(stage, outputs_path):
4221
6620
  return outputs
4222
6621
 
4223
6622
 
6623
+ def ensure_native_auth_custom_attributes(current_outputs):
6624
+ """Add the native Entra claims to an existing pool before app clients use them."""
6625
+ user_pool_id = str(_current_output_value(current_outputs, "user_pool_id") or "").strip()
6626
+ if not user_pool_id:
6627
+ return {"status": "skipped", "reason": "user-pool-not-created"}
6628
+
6629
+ def describe_attribute_names():
6630
+ raw = output(
6631
+ [
6632
+ "aws",
6633
+ "cognito-idp",
6634
+ "describe-user-pool",
6635
+ "--user-pool-id",
6636
+ user_pool_id,
6637
+ "--output",
6638
+ "json",
6639
+ ]
6640
+ )
6641
+ described = json.loads(raw or "{}")
6642
+ attributes = described.get("UserPool", {}).get("SchemaAttributes", [])
6643
+ return {
6644
+ str(attribute.get("Name") or "")
6645
+ for attribute in attributes
6646
+ if isinstance(attribute, dict)
6647
+ }
6648
+
6649
+ existing_names = describe_attribute_names()
6650
+ missing = [
6651
+ attribute
6652
+ for attribute in NATIVE_AUTH_CUSTOM_ATTRIBUTES
6653
+ if f"custom:{attribute['Name']}" not in existing_names
6654
+ ]
6655
+ if not missing:
6656
+ return {"status": "current", "userPoolId": user_pool_id, "added": []}
6657
+
6658
+ request = {"UserPoolId": user_pool_id, "CustomAttributes": missing}
6659
+ run(
6660
+ [
6661
+ "aws",
6662
+ "cognito-idp",
6663
+ "add-custom-attributes",
6664
+ "--cli-input-json",
6665
+ json.dumps(request, separators=(",", ":"), sort_keys=True),
6666
+ ],
6667
+ stdout=subprocess.DEVNULL,
6668
+ )
6669
+ reconciled_names = describe_attribute_names()
6670
+ unresolved = [
6671
+ attribute["Name"]
6672
+ for attribute in missing
6673
+ if f"custom:{attribute['Name']}" not in reconciled_names
6674
+ ]
6675
+ if unresolved:
6676
+ raise RuntimeError(
6677
+ "Cognito native-auth custom attributes were not added: " + ", ".join(unresolved)
6678
+ )
6679
+ return {
6680
+ "status": "reconciled",
6681
+ "userPoolId": user_pool_id,
6682
+ "added": [attribute["Name"] for attribute in missing],
6683
+ }
6684
+
6685
+
4224
6686
  def write_outputs_to_ssm(outputs_path, vars_json):
4225
6687
  outputs = json.loads(outputs_path.read_text(encoding="utf-8"))
4226
6688
  profile, web_env = runtime_profile(outputs, vars_json)
@@ -4676,9 +7138,13 @@ def native_auth_reconciliation_payload(outputs, vars_json, previous_state=None):
4676
7138
  )
4677
7139
  if provider_client_ids.get("MicrosoftOrganizations"):
4678
7140
  microsoft_tenant = str(vars_json.get("microsoft_oauth_tenant") or "").lower()
4679
- if not microsoft_tenant:
7141
+ if microsoft_tenant not in {"common", "organizations", "consumers"} and not re.fullmatch(
7142
+ r"[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}",
7143
+ microsoft_tenant,
7144
+ ):
4680
7145
  raise RuntimeError(
4681
- "Microsoft native auth reconciliation requires microsoft_oauth_tenant"
7146
+ "Microsoft native auth reconciliation requires microsoft_oauth_tenant to be"
7147
+ " an Entra directory GUID or one of: common, organizations, consumers"
4682
7148
  )
4683
7149
  connections.append(
4684
7150
  {
@@ -5124,7 +7590,6 @@ def sync_static(outputs_path, static_files, vars_json, artifact_names=None):
5124
7590
  artifact_names = set(artifact_names or [])
5125
7591
  syncs = [
5126
7592
  ("web", "app_bucket_name", "app_distribution_id"),
5127
- ("docs", "docs_bucket_name", "docs_distribution_id"),
5128
7593
  ]
5129
7594
  for artifact_name, bucket_output, distribution_output in syncs:
5130
7595
  if artifact_names and artifact_name not in artifact_names:
@@ -5254,7 +7719,7 @@ def write_deployment_status_pointer(status, vars_json=None, terraform_exit_code=
5254
7719
  action = os.environ.get("THINKWORK_DEPLOYMENT_ACTION")
5255
7720
  if os.environ.get("THINKWORK_MANAGED_APP_OPERATION") == "true":
5256
7721
  return
5257
- if not bucket or action not in {"deploy", "update", "web"}:
7722
+ if not bucket or action not in {"deploy", "update", "apply", "web"}:
5258
7723
  return
5259
7724
  vars_json = vars_json or {}
5260
7725
  previous = read_current_status_pointer(bucket)
@@ -5394,7 +7859,16 @@ def main():
5394
7859
  action = os.environ.get("THINKWORK_DEPLOYMENT_ACTION") or payload.get("action") or "deploy"
5395
7860
  if action == "teardown":
5396
7861
  action = "destroy"
5397
- if action not in {"deploy", "update", "destroy", "plan", "status", "web"}:
7862
+ if action not in {
7863
+ "deploy",
7864
+ "update",
7865
+ "destroy",
7866
+ "plan",
7867
+ "apply",
7868
+ "recover",
7869
+ "status",
7870
+ "web",
7871
+ }:
5398
7872
  raise RuntimeError(f"Unsupported deployment action: {action}")
5399
7873
  os.environ["THINKWORK_DEPLOYMENT_ACTION"] = action
5400
7874
  configure_managed_app_evidence_prefix(payload)
@@ -5414,21 +7888,45 @@ def main():
5414
7888
  )
5415
7889
  return 0
5416
7890
 
7891
+ if action == "recover":
7892
+ CONTROLLER_EVIDENCE = {
7893
+ "incidentRecovery": recover_tei_v380_incident(payload),
7894
+ }
7895
+ write_evidence(
7896
+ "succeeded",
7897
+ {
7898
+ "stage": "tei-e2e",
7899
+ "account_id": "637423202447",
7900
+ "region": "us-east-1",
7901
+ },
7902
+ 0,
7903
+ )
7904
+ return 0
7905
+
7906
+ terraform_phase = validate_terraform_execution_phase(payload, action)
7907
+ planned_action = requested_plan_action(payload, action)
7908
+ execution_effect_action = planned_action if terraform_phase == "apply" else action
5417
7909
  runner_secrets = secret_payload(payload)
5418
7910
  web_only = is_web_only_operation(payload, action)
5419
7911
  identity_operation = identity_provider_operation(payload)
5420
7912
  static_files = {}
5421
- if (
5422
- action in {"deploy", "update", "web"}
5423
- and not is_managed_app_operation(payload)
5424
- and identity_operation is None
5425
- ):
5426
- static_files = sync_release_artifacts(
5427
- artifact_types={"static-site"} if web_only else None,
5428
- artifact_names={"web"} if web_only else None,
5429
- )
7913
+ release_request = release_sync_request(execution_effect_action, payload, identity_operation)
7914
+ control_runtime_operation = requires_agentcore_control_runtime(
7915
+ execution_effect_action, payload, identity_operation
7916
+ )
7917
+ if release_request is not None:
7918
+ static_files = sync_release_artifacts(**release_request)
7919
+ elif control_runtime_operation:
7920
+ # Plan and destroy still execute AgentCore local-exec wrappers for
7921
+ # existing resources. Download the content-addressed platform bundle
7922
+ # without publishing application artifacts so those paths never depend
7923
+ # on an ambient CodeBuild SDK.
7924
+ sync_release_artifacts(artifact_types=set())
5430
7925
  managed_app_artifacts = stage_managed_app_release_artifacts(action, payload)
5431
7926
  vars_json = write_runner_files(payload, runner_secrets)
7927
+ set_database_access_mode(vars_json)
7928
+ if control_runtime_operation:
7929
+ prepare_agentcore_control_runtime()
5432
7930
  controller_summary = controller_input_summary(payload)
5433
7931
  CONTROLLER_EVIDENCE = {
5434
7932
  "inputSummary": controller_summary,
@@ -5482,44 +7980,56 @@ def main():
5482
7980
  )
5483
7981
  if selected.returncode != 0:
5484
7982
  run(["terraform", "workspace", "new", workspace, "-no-color"], cwd=TF)
7983
+ state_identity = terraform_state_identity()
5485
7984
  target_args = managed_app_terraform_target_args(payload)
5486
- pre_app_schema = prepare_additive_schema_before_app(payload, vars_json)
5487
- if pre_app_schema is not None and pre_app_schema.returncode != 0:
5488
- result = pre_app_schema
5489
- elif action == "destroy":
5490
- plan = subprocess.run(
5491
- ["terraform", "plan", "-destroy", *target_args, "-out=tfplan", "-no-color"],
5492
- cwd=TF,
5493
- text=True,
7985
+ pre_app_schema = (
7986
+ prepare_additive_schema_before_app(payload, vars_json)
7987
+ if terraform_phase in {"plan", "legacy"}
7988
+ else None
7989
+ )
7990
+ if terraform_phase == "legacy" and should_reconcile_native_auth_schema(
7991
+ terraform_phase, planned_action, payload
7992
+ ):
7993
+ CONTROLLER_EVIDENCE["cognitoSchema"] = ensure_native_auth_custom_attributes(
7994
+ current_terraform_outputs(vars_json["stage"])
5494
7995
  )
5495
- if plan.returncode == 0:
5496
- TERRAFORM_EVIDENCE["plan"] = write_terraform_plan_evidence(payload)
5497
- result = subprocess.run(
5498
- ["terraform", "apply", "-auto-approve", "-no-color", "tfplan"],
5499
- cwd=TF,
5500
- text=True,
7996
+ applied_plan = None
7997
+ if terraform_phase == "apply":
7998
+ applied_plan = validate_and_materialize_approved_plan(payload, state_identity)
7999
+ planned_action = applied_plan.get("plannedAction")
8000
+ if planned_action not in {"deploy", "update", "destroy"}:
8001
+ raise RuntimeError("Approved plan has an invalid plannedAction")
8002
+ TERRAFORM_EVIDENCE["approvedPlan"] = applied_plan
8003
+ if should_reconcile_native_auth_schema(terraform_phase, planned_action, payload):
8004
+ CONTROLLER_EVIDENCE["cognitoSchema"] = ensure_native_auth_custom_attributes(
8005
+ current_terraform_outputs(vars_json["stage"])
5501
8006
  )
5502
- else:
5503
- result = plan
5504
- else:
5505
- plan = subprocess.run(
5506
- ["terraform", "plan", *target_args, "-out=tfplan", "-no-color"],
8007
+ result = subprocess.run(
8008
+ ["terraform", "apply", "-no-color", "tfplan"],
5507
8009
  cwd=TF,
5508
8010
  text=True,
5509
8011
  )
5510
- if plan.returncode == 0:
5511
- TERRAFORM_EVIDENCE["plan"] = write_terraform_plan_evidence(payload)
5512
- if action == "plan" or plan.returncode != 0:
5513
- result = plan
5514
- else:
5515
- result = subprocess.run(
5516
- ["terraform", "apply", "-auto-approve", "-no-color", "tfplan"],
5517
- cwd=TF,
5518
- text=True,
5519
- )
8012
+ elif pre_app_schema is not None and pre_app_schema.returncode != 0:
8013
+ result = pre_app_schema
8014
+ else:
8015
+ result = execute_terraform_plan_phase(
8016
+ payload,
8017
+ state_identity,
8018
+ planned_action,
8019
+ terraform_phase,
8020
+ target_args,
8021
+ )
5520
8022
 
5521
8023
  outputs_path = TF / "outputs.json"
5522
- if result.returncode == 0 and action in {"deploy", "update"}:
8024
+ if (
8025
+ result.returncode == 0
8026
+ and terraform_phase in {"apply", "legacy"}
8027
+ and planned_action
8028
+ in {
8029
+ "deploy",
8030
+ "update",
8031
+ }
8032
+ ):
5523
8033
  if identity_operation is not None:
5524
8034
  finalized = finalize_disabled_identity_provider(payload, vars_json)
5525
8035
  if finalized is not None:
@@ -5558,6 +8068,13 @@ def main():
5558
8068
  selected_controller_release,
5559
8069
  )
5560
8070
  sync_static(outputs_path, static_files, vars_json)
8071
+ if control_runtime_operation:
8072
+ pi_runtime_evidence = reconcile_agentcore_pi_runtime(vars_json, payload)
8073
+ if pi_runtime_evidence is not None:
8074
+ CONTROLLER_EVIDENCE["agentcorePiRuntime"] = write_json_evidence_artifact(
8075
+ "agentcore-pi-runtime.json",
8076
+ pi_runtime_evidence,
8077
+ )
5561
8078
  try:
5562
8079
  self_update_runner_script()
5563
8080
  except Exception as self_update_error: