graphcore-kernel 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (343) hide show
  1. graphcore/__init__.py +5 -0
  2. graphcore/agent_frontend/__init__.py +8 -0
  3. graphcore/agent_frontend/api.py +69 -0
  4. graphcore/agent_frontend/base/__init__.py +11 -0
  5. graphcore/agent_frontend/base/views.py +167 -0
  6. graphcore/agent_frontend/client.py +31 -0
  7. graphcore/agent_frontend/context.py +157 -0
  8. graphcore/agent_frontend/errors.py +12 -0
  9. graphcore/agent_frontend/loading.py +54 -0
  10. graphcore/agent_frontend/registry.py +42 -0
  11. graphcore/agent_frontend/render.py +207 -0
  12. graphcore/agent_frontend/server.py +103 -0
  13. graphcore/agent_frontend/signatures.py +35 -0
  14. graphcore/agent_frontend/testing.py +45 -0
  15. graphcore/agent_frontend/tests/__init__.py +0 -0
  16. graphcore/agent_frontend/tests/fakes.py +114 -0
  17. graphcore/agent_frontend/tests/test_frontend.py +447 -0
  18. graphcore/agent_frontend/tests/test_loading.py +83 -0
  19. graphcore/agent_frontend/tests/test_server.py +109 -0
  20. graphcore/agent_frontend/tools.py +137 -0
  21. graphcore/api/__init__.py +4 -0
  22. graphcore/api/graph_api.py +378 -0
  23. graphcore/api/predicate_codec.py +81 -0
  24. graphcore/api/tests/__init__.py +0 -0
  25. graphcore/api/tests/test_changed_entities.py +109 -0
  26. graphcore/api/tests/test_graph_api.py +290 -0
  27. graphcore/api/tests/test_graph_api_access.py +94 -0
  28. graphcore/api/tests/test_graph_api_grants.py +90 -0
  29. graphcore/api/tests/test_predicate_codec.py +83 -0
  30. graphcore/api/tests/test_subtree_query.py +79 -0
  31. graphcore/apps.py +71 -0
  32. graphcore/auth/__init__.py +1 -0
  33. graphcore/auth/authn/README.md +178 -0
  34. graphcore/auth/authn/__init__.py +3 -0
  35. graphcore/auth/authn/authenticator.py +120 -0
  36. graphcore/auth/authn/onboarding.py +105 -0
  37. graphcore/auth/authn/principals.py +20 -0
  38. graphcore/auth/authn/resource_metadata.py +43 -0
  39. graphcore/auth/authn/service_accounts.py +96 -0
  40. graphcore/auth/authn/testing.py +96 -0
  41. graphcore/auth/authn/tests/__init__.py +0 -0
  42. graphcore/auth/authn/tests/test_authenticator.py +145 -0
  43. graphcore/auth/authn/tests/test_onboarding.py +145 -0
  44. graphcore/auth/authn/tests/test_service_accounts.py +180 -0
  45. graphcore/auth/authn/users.py +59 -0
  46. graphcore/auth/authz/__init__.py +0 -0
  47. graphcore/auth/authz/access_port.py +120 -0
  48. graphcore/auth/authz/adapter/__init__.py +0 -0
  49. graphcore/auth/authz/adapter/default_access_adapter.py +556 -0
  50. graphcore/auth/authz/adapter/tests/__init__.py +0 -0
  51. graphcore/auth/authz/adapter/tests/test_default_adapter.py +543 -0
  52. graphcore/auth/authz/adapter/tests/test_entity_write_hooks.py +207 -0
  53. graphcore/auth/authz/adapter/tests/test_group_types.py +70 -0
  54. graphcore/auth/authz/adapter/tests/test_sync_support.py +209 -0
  55. graphcore/auth/authz/defaults.py +26 -0
  56. graphcore/auth/authz/kernel_policy.py +191 -0
  57. graphcore/auth/authz/kernel_writes.py +31 -0
  58. graphcore/auth/authz/lookups.py +25 -0
  59. graphcore/auth/authz/policy.py +403 -0
  60. graphcore/auth/authz/tests/__init__.py +0 -0
  61. graphcore/auth/authz/tests/test_can_contain.py +150 -0
  62. graphcore/auth/authz/tests/test_contained_by.py +167 -0
  63. graphcore/auth/authz/tests/test_creation.py +391 -0
  64. graphcore/auth/authz/tests/test_delete_permission.py +82 -0
  65. graphcore/auth/authz/tests/test_granted_to_me.py +160 -0
  66. graphcore/auth/authz/tests/test_integration.py +637 -0
  67. graphcore/auth/authz/tests/test_invites.py +184 -0
  68. graphcore/auth/authz/tests/test_kernel_policy.py +140 -0
  69. graphcore/auth/authz/tests/test_lockout.py +132 -0
  70. graphcore/auth/authz/tests/test_policy.py +329 -0
  71. graphcore/auth/authz/tests/test_primitives.py +73 -0
  72. graphcore/auth/authz/tests/test_principal_creation.py +62 -0
  73. graphcore/bootstrap/__init__.py +0 -0
  74. graphcore/bootstrap/deployment.py +67 -0
  75. graphcore/bootstrap/signals.py +16 -0
  76. graphcore/bootstrap/tests/__init__.py +0 -0
  77. graphcore/bootstrap/tests/test_deployment_seed.py +154 -0
  78. graphcore/broadcast/__init__.py +4 -0
  79. graphcore/broadcast/broadcaster.py +201 -0
  80. graphcore/broadcast/channels.py +56 -0
  81. graphcore/broadcast/tests/__init__.py +0 -0
  82. graphcore/broadcast/tests/test_broadcaster.py +444 -0
  83. graphcore/celery.py +14 -0
  84. graphcore/docs/AGENT_FRONTEND_TDD.md +389 -0
  85. graphcore/docs/ARCHITECTURE_PRINCIPLES.md +152 -0
  86. graphcore/docs/AUTHORIZATION_TECHNICAL_TDD.md +1379 -0
  87. graphcore/docs/ENTITY_REFERENCE_TDD.md +561 -0
  88. graphcore/docs/GRAPHAPI_FRONT_DOOR.md +173 -0
  89. graphcore/docs/RUNTIME.md +90 -0
  90. graphcore/docs/SCHEDULING_TDD.md +244 -0
  91. graphcore/docs/STRATEGY_EXECUTION_TDD.md +1043 -0
  92. graphcore/docs/VIEW_ENTITY_TDD.md +522 -0
  93. graphcore/docs/VIEW_SYSTEM_TDD.md +307 -0
  94. graphcore/docs/__init__.py +0 -0
  95. graphcore/docs/doc_topics.py +97 -0
  96. graphcore/docs/docs_discovery.py +64 -0
  97. graphcore/docs/public_api.py +56 -0
  98. graphcore/domain/__init__.py +0 -0
  99. graphcore/domain/entity/Entity.py +204 -0
  100. graphcore/domain/entity/__init__.py +0 -0
  101. graphcore/domain/entity/deployment/Deployment.py +25 -0
  102. graphcore/domain/entity/deployment/__init__.py +0 -0
  103. graphcore/domain/entity/grant/Grant.py +31 -0
  104. graphcore/domain/entity/grant/__init__.py +0 -0
  105. graphcore/domain/entity/group/Group.py +48 -0
  106. graphcore/domain/entity/group/README.md +88 -0
  107. graphcore/domain/entity/group/__init__.py +0 -0
  108. graphcore/domain/entity/invite/Invite.py +31 -0
  109. graphcore/domain/entity/invite/__init__.py +0 -0
  110. graphcore/domain/entity/request_template/RequestTemplate.py +48 -0
  111. graphcore/domain/entity/request_template/__init__.py +0 -0
  112. graphcore/domain/entity/request_template/tests/__init__.py +0 -0
  113. graphcore/domain/entity/request_template/tests/test_request_template.py +52 -0
  114. graphcore/domain/entity/service_account/ServiceAccount.py +41 -0
  115. graphcore/domain/entity/service_account/__init__.py +0 -0
  116. graphcore/domain/entity/strategy_request/StrategyRequest.py +61 -0
  117. graphcore/domain/entity/strategy_request/__init__.py +0 -0
  118. graphcore/domain/entity/strategy_request/tests/__init__.py +0 -0
  119. graphcore/domain/entity/strategy_request/tests/test_strategy_request_entity.py +35 -0
  120. graphcore/domain/entity/tests/__init__.py +0 -0
  121. graphcore/domain/entity/tests/test_entity.py +227 -0
  122. graphcore/domain/entity/tests/test_grant_group_entities.py +36 -0
  123. graphcore/domain/entity/tests/test_write_once_fields.py +98 -0
  124. graphcore/domain/entity/user/User.py +39 -0
  125. graphcore/domain/entity/user/__init__.py +0 -0
  126. graphcore/domain/entity/user/tests/__init__.py +0 -0
  127. graphcore/domain/entity/user/tests/test_user_entity.py +33 -0
  128. graphcore/domain/entity/view/View.py +37 -0
  129. graphcore/domain/entity/view/__init__.py +0 -0
  130. graphcore/domain/entity/view/position.py +78 -0
  131. graphcore/domain/entity/view/tests/__init__.py +0 -0
  132. graphcore/domain/entity/view/tests/test_position.py +71 -0
  133. graphcore/domain/entity/view/tests/test_view_entity.py +134 -0
  134. graphcore/domain/strategy/BaseStrategy.py +759 -0
  135. graphcore/domain/strategy/DeploymentStrategies.py +55 -0
  136. graphcore/domain/strategy/GrantStrategies.py +189 -0
  137. graphcore/domain/strategy/GroupStrategies.py +193 -0
  138. graphcore/domain/strategy/InviteStrategies.py +84 -0
  139. graphcore/domain/strategy/RequestTemplateStrategies.py +126 -0
  140. graphcore/domain/strategy/ViewStrategies.py +219 -0
  141. graphcore/domain/strategy/__init__.py +0 -0
  142. graphcore/domain/strategy/runner.py +84 -0
  143. graphcore/domain/strategy/session.py +56 -0
  144. graphcore/domain/strategy/tests/BaseStrategyTest.py +167 -0
  145. graphcore/domain/strategy/tests/__init__.py +0 -0
  146. graphcore/domain/strategy/tests/test_BaseStrategies.py +870 -0
  147. graphcore/domain/strategy/tests/test_grant_strategies.py +245 -0
  148. graphcore/domain/strategy/tests/test_group_strategies.py +99 -0
  149. graphcore/domain/strategy/tests/test_runner.py +220 -0
  150. graphcore/domain/strategy/tests/test_strategy_locks.py +446 -0
  151. graphcore/domain/strategy/tests/test_utils.py +95 -0
  152. graphcore/domain/strategy/tests/test_view_strategies.py +436 -0
  153. graphcore/errors.py +145 -0
  154. graphcore/execution/README.md +97 -0
  155. graphcore/execution/StrategyExecutorService.py +333 -0
  156. graphcore/execution/__init__.py +0 -0
  157. graphcore/execution/dispatcher.py +19 -0
  158. graphcore/execution/environments/__init__.py +0 -0
  159. graphcore/execution/environments/celery.py +59 -0
  160. graphcore/execution/environments/in_process.py +55 -0
  161. graphcore/execution/local_session.py +117 -0
  162. graphcore/execution/redis_run_events.py +74 -0
  163. graphcore/execution/requests.py +22 -0
  164. graphcore/execution/run_events.py +85 -0
  165. graphcore/execution/strategy_context.py +54 -0
  166. graphcore/execution/tasks.py +34 -0
  167. graphcore/execution/tests/__init__.py +5 -0
  168. graphcore/execution/tests/contract.py +171 -0
  169. graphcore/execution/tests/support.py +310 -0
  170. graphcore/execution/tests/test_dispatchers.py +76 -0
  171. graphcore/execution/tests/test_executor.py +368 -0
  172. graphcore/execution/tests/test_local_session.py +95 -0
  173. graphcore/execution/tests/test_reconcile_runs.py +134 -0
  174. graphcore/execution/tests/test_run_events.py +106 -0
  175. graphcore/execution/tests/test_strategy_context.py +62 -0
  176. graphcore/gateway/__init__.py +1 -0
  177. graphcore/gateway/remote.py +35 -0
  178. graphcore/gateway/tests/__init__.py +0 -0
  179. graphcore/gateway/tests/test_remote.py +30 -0
  180. graphcore/graph/EntityService.py +710 -0
  181. graphcore/graph/__init__.py +0 -0
  182. graphcore/graph/entity_cache.py +152 -0
  183. graphcore/graph/tests/__init__.py +4 -0
  184. graphcore/graph/tests/test_EntityService.py +908 -0
  185. graphcore/graph/tests/test_EntityService_list.py +170 -0
  186. graphcore/graph/tests/test_EntityService_model_resolution.py +339 -0
  187. graphcore/graph/tests/test_EntityService_referrers.py +236 -0
  188. graphcore/graph/tests/test_EntityService_routing.py +119 -0
  189. graphcore/graph/tests/test_dependent_on.py +141 -0
  190. graphcore/graph/tests/test_engine_is_type_agnostic.py +25 -0
  191. graphcore/graph/tests/test_engine_resolves_read_permission.py +93 -0
  192. graphcore/graph/tests/test_entity_cache.py +159 -0
  193. graphcore/graph/tests/test_null_entity_cache.py +12 -0
  194. graphcore/management/__init__.py +0 -0
  195. graphcore/management/commands/__init__.py +0 -0
  196. graphcore/management/commands/dev_reset.py +18 -0
  197. graphcore/management/commands/graphcore_createadmin.py +65 -0
  198. graphcore/management/commands/graphcore_policy.py +59 -0
  199. graphcore/management/commands/graphcore_status.py +98 -0
  200. graphcore/management/tests/__init__.py +0 -0
  201. graphcore/management/tests/test_createadmin.py +78 -0
  202. graphcore/management/tests/test_policy_command.py +46 -0
  203. graphcore/management/tests/test_status_command.py +85 -0
  204. graphcore/migrations/0001_initial.py +186 -0
  205. graphcore/migrations/0002_invite.py +26 -0
  206. graphcore/migrations/__init__.py +0 -0
  207. graphcore/models.py +11 -0
  208. graphcore/persistence/ENTITY_REPOSITORY_TECHNICAL_TDD.md +396 -0
  209. graphcore/persistence/__init__.py +6 -0
  210. graphcore/persistence/django/__init__.py +0 -0
  211. graphcore/persistence/django/backends.py +43 -0
  212. graphcore/persistence/django/bindings.py +28 -0
  213. graphcore/persistence/django/cache.py +38 -0
  214. graphcore/persistence/django/compiler.py +214 -0
  215. graphcore/persistence/django/identity.py +18 -0
  216. graphcore/persistence/django/models.py +252 -0
  217. graphcore/persistence/django/repository.py +525 -0
  218. graphcore/persistence/django/tests/__init__.py +0 -0
  219. graphcore/persistence/django/tests/test_backends.py +68 -0
  220. graphcore/persistence/django/tests/test_compiler.py +269 -0
  221. graphcore/persistence/django/tests/test_repository.py +472 -0
  222. graphcore/persistence/django/tests/test_shared_column.py +120 -0
  223. graphcore/persistence/django/tests/test_threads.py +50 -0
  224. graphcore/persistence/django/tests/test_update_shape.py +175 -0
  225. graphcore/persistence/django/threads.py +38 -0
  226. graphcore/persistence/repository.py +172 -0
  227. graphcore/persistence/tests/__init__.py +0 -0
  228. graphcore/persistence/tests/contract.py +688 -0
  229. graphcore/persistence/tests/evaluate.py +51 -0
  230. graphcore/persistence/tests/fixtures.py +32 -0
  231. graphcore/persistence/tests/memory.py +263 -0
  232. graphcore/persistence/tests/test_memory_repository.py +10 -0
  233. graphcore/persistence/tests/test_query_types.py +127 -0
  234. graphcore/public/__init__.py +9 -0
  235. graphcore/public/api.py +9 -0
  236. graphcore/public/apps.py +10 -0
  237. graphcore/public/auth/__init__.py +1 -0
  238. graphcore/public/auth/grants.py +7 -0
  239. graphcore/public/auth/onboarding.py +8 -0
  240. graphcore/public/auth/perms.py +7 -0
  241. graphcore/public/auth/policy.py +10 -0
  242. graphcore/public/auth/roles.py +8 -0
  243. graphcore/public/auth/subjects.py +7 -0
  244. graphcore/public/entities/__init__.py +32 -0
  245. graphcore/public/errors.py +19 -0
  246. graphcore/public/persistence/__init__.py +44 -0
  247. graphcore/public/persistence/django.py +24 -0
  248. graphcore/public/scheduling.py +23 -0
  249. graphcore/public/strategies/__init__.py +85 -0
  250. graphcore/public/test_settings.py +16 -0
  251. graphcore/public/testing.py +47 -0
  252. graphcore/registry/EntityDirectory.py +246 -0
  253. graphcore/registry/README.md +147 -0
  254. graphcore/registry/StrategyDirectory.py +308 -0
  255. graphcore/registry/TYPE_METADATA_PLACEMENT.md +66 -0
  256. graphcore/registry/__init__.py +0 -0
  257. graphcore/registry/app_discovery.py +108 -0
  258. graphcore/registry/decorators.py +77 -0
  259. graphcore/registry/discovery.py +111 -0
  260. graphcore/registry/registry.py +79 -0
  261. graphcore/registry/tests/__init__.py +0 -0
  262. graphcore/registry/tests/test_agent_frontend_discovery.py +56 -0
  263. graphcore/registry/tests/test_app_config.py +36 -0
  264. graphcore/registry/tests/test_discovery.py +147 -0
  265. graphcore/registry/tests/test_entity_directory_storage.py +73 -0
  266. graphcore/registry/tests/test_import_package.py +87 -0
  267. graphcore/registry/tests/test_principal_flag.py +67 -0
  268. graphcore/registry/tests/test_strategy_directory.py +29 -0
  269. graphcore/routes/__init__.py +116 -0
  270. graphcore/routes/tests/__init__.py +0 -0
  271. graphcore/routes/tests/test_routes.py +90 -0
  272. graphcore/runtime.py +243 -0
  273. graphcore/scheduling/__init__.py +0 -0
  274. graphcore/scheduling/beat_scheduler.py +164 -0
  275. graphcore/scheduling/beat_tasks.py +63 -0
  276. graphcore/scheduling/scheduler.py +37 -0
  277. graphcore/scheduling/tests/__init__.py +0 -0
  278. graphcore/scheduling/tests/contract.py +136 -0
  279. graphcore/scheduling/tests/memory.py +42 -0
  280. graphcore/scheduling/tests/test_beat_scheduler.py +87 -0
  281. graphcore/scheduling/tests/test_memory_scheduler.py +15 -0
  282. graphcore/scheduling/tests/test_ticks.py +143 -0
  283. graphcore/tasks.py +9 -0
  284. graphcore/test_settings.py +78 -0
  285. graphcore/testing/__init__.py +0 -0
  286. graphcore/testing/case.py +112 -0
  287. graphcore/testing/runner.py +32 -0
  288. graphcore/testing/settings.py +26 -0
  289. graphcore/testing/tests/__init__.py +0 -0
  290. graphcore/testing/tests/agent_view_fixture.py +19 -0
  291. graphcore/testing/tests/test_case_and_runner.py +219 -0
  292. graphcore/testing/tests/test_settings_for_apps.py +50 -0
  293. graphcore/tests/__init__.py +0 -0
  294. graphcore/tests/ids.py +14 -0
  295. graphcore/tests/public_api_manifest.py +102 -0
  296. graphcore/tests/runs.py +7 -0
  297. graphcore/tests/test_app_discovery.py +90 -0
  298. graphcore/tests/test_composition_boundaries.py +231 -0
  299. graphcore/tests/test_docs_discovery.py +108 -0
  300. graphcore/tests/test_no_print.py +70 -0
  301. graphcore/tests/test_packaging.py +64 -0
  302. graphcore/tests/test_public_api_boundaries.py +229 -0
  303. graphcore/tests/test_public_api_contract.py +62 -0
  304. graphcore/tests/test_public_api_docs.py +28 -0
  305. graphcore/tests/test_runtime.py +42 -0
  306. graphcore/tests/test_shared_type_boundaries.py +128 -0
  307. graphcore/tests/test_shared_types.py +144 -0
  308. graphcore/tests/test_test_settings.py +45 -0
  309. graphcore/transport/__init__.py +0 -0
  310. graphcore/transport/agent_asgi.py +68 -0
  311. graphcore/transport/agent_client.py +85 -0
  312. graphcore/transport/channel_layer.py +68 -0
  313. graphcore/transport/consumers.py +507 -0
  314. graphcore/transport/routing.py +6 -0
  315. graphcore/transport/tests/__init__.py +0 -0
  316. graphcore/transport/tests/test_agent_client.py +31 -0
  317. graphcore/transport/tests/test_api_views.py +733 -0
  318. graphcore/transport/tests/test_channel_layer.py +96 -0
  319. graphcore/transport/tests/test_consumers.py +963 -0
  320. graphcore/transport/tests/test_resource_metadata.py +67 -0
  321. graphcore/transport/urls.py +29 -0
  322. graphcore/transport/views.py +358 -0
  323. graphcore/types/__init__.py +4 -0
  324. graphcore/types/access.py +114 -0
  325. graphcore/types/api.py +76 -0
  326. graphcore/types/entities.py +100 -0
  327. graphcore/types/entity_id.py +85 -0
  328. graphcore/types/execution.py +41 -0
  329. graphcore/types/identity.py +20 -0
  330. graphcore/types/onboarding.py +20 -0
  331. graphcore/types/query.py +116 -0
  332. graphcore/types/schedules.py +91 -0
  333. graphcore/types/strategy.py +73 -0
  334. graphcore/types/tests/__init__.py +0 -0
  335. graphcore/types/tests/test_entity_id.py +50 -0
  336. graphcore/urls.py +5 -0
  337. graphcore_kernel-0.1.0.dist-info/METADATA +94 -0
  338. graphcore_kernel-0.1.0.dist-info/RECORD +343 -0
  339. graphcore_kernel-0.1.0.dist-info/WHEEL +5 -0
  340. graphcore_kernel-0.1.0.dist-info/entry_points.txt +2 -0
  341. graphcore_kernel-0.1.0.dist-info/licenses/LICENSE +202 -0
  342. graphcore_kernel-0.1.0.dist-info/licenses/NOTICE +2 -0
  343. graphcore_kernel-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1043 @@
1
+ # Strategy Execution TDD
2
+
3
+ ## Introduction
4
+
5
+ Strategies are the only writes in GraphCore. This document specifies how a
6
+ strategy request is run: how the kernel decides where it runs, how a running
7
+ strategy reads and writes the graph, and how a run is reported back to its
8
+ caller.
9
+
10
+ The design separates three concerns:
11
+
12
+ - **What a strategy is.** A strategy is domain code. It declares its
13
+ workload and talks to the graph through one interface, the
14
+ `StrategySession`. It carries no knowledge of the process it runs in.
15
+ - **Where a strategy runs.** Infrastructure. A `Dispatcher` port hands one
16
+ attempt to an execution environment, which runs it with the
17
+ `StrategyRunner` and reports the result back.
18
+ - **What a run guarantees.** Kernel policy. `StrategyExecutorService`
19
+ authorizes, records, retries, and reports, in the kernel, for every
20
+ environment.
21
+
22
+ An execution environment either holds a runtime or does not. One with a
23
+ runtime reaches the graph through its own `EntityService`; one without
24
+ reaches it through the kernel's execution transport. The runner, the
25
+ session contract, and the way results are reported are the same in both,
26
+ so moving an environment from one kind to the other changes its entry point
27
+ and nothing else. The kernel ships two environments, the event loop of the
28
+ submitting process and a Celery process, both with a runtime. A project
29
+ adds an environment by supplying a workload, a dispatcher, and an entry
30
+ point, without changing kernel or strategy code.
31
+
32
+ ## Definitions
33
+
34
+ - **Strategy**: a subclass of `Strategy` (`domain/strategy/BaseStrategy.py`).
35
+ Its `apply()` reads and writes the graph to fulfil one request. Kernel and
36
+ application strategies are the same kind of object.
37
+ - **Strategy request**: a `StrategyRequest`, an ordinary top-level entity
38
+ naming a strategy, its parameters, its target, and its outcome. It runs as
39
+ its `owner`. A `StrategyRequestPayload` is the client-safe input a caller
40
+ creates one from.
41
+ - **Request template**: a `RequestTemplate`, an ordinary top-level entity
42
+ describing a request to make later, optionally on a schedule
43
+ (`docs/SCHEDULING_TDD.md`). It is never run itself.
44
+ - **Workload**: the `Workload` value a strategy declares, naming the kind of
45
+ environment its `apply()` needs. The set is open: the kernel defines
46
+ `Workload.IO` (the default) and `Workload.CPU`; a project defines more.
47
+ - **Runtime**: the `Runtime` that `build_runtime` constructs
48
+ (`graphcore/runtime.py`): one process's live service graph, holding the
49
+ registry, repository, cache, access port, broadcaster, scheduler,
50
+ `EntityService`, and `StrategyExecutorService`. Building one requires the
51
+ deployment's database, cache, and broker connections.
52
+ - **Kernel process**: a process holding a runtime. The ASGI process is one;
53
+ so is the Celery process.
54
+ - **Front door**: `StrategyExecutorService.execute_request`, the single entry
55
+ point for submitting a `StrategyRequest`.
56
+ - **Run**: everything that happens to one request between submission and
57
+ its terminal status, across all attempts.
58
+ - **Attempt**: one execution of a request's strategy, numbered from 1,
59
+ described by an `ExecutionAttempt` and reported by an `AttemptResult`.
60
+ - **Execution environment**: where an attempt runs. Defined by a dispatcher
61
+ (how attempts reach it) and an entry point (the code that builds a session
62
+ and runs the runner). An environment with a runtime is a kernel process;
63
+ one without reaches the kernel over the execution transport.
64
+ - **Dispatcher**: an implementation of the `Dispatcher` port. It delivers an
65
+ attempt to an execution environment and returns.
66
+ - **Strategy runner**: `StrategyRunner`, the domain-level code that performs
67
+ one attempt. It depends only on `StrategySession` and `ChangeTracker`.
68
+ - **Strategy session**: an implementation of `StrategySession`, bound to one
69
+ attempt. It is the strategy's and the runner's only path to the graph.
70
+ `LocalStrategySession` reaches the graph through a runtime's
71
+ `EntityService`; `RemoteStrategySession` reaches it through a
72
+ `KernelClient`.
73
+ - **Change tracker**: a `ChangeTracker`, built for one attempt, holding the
74
+ last persisted state of every entity the attempt's session returned or
75
+ saved.
76
+ - **Execution transport**: the kernel's HTTP routes that serve environments
77
+ without a runtime: the session operations, `finish_attempt`, and nested
78
+ `execute`. Mounted separately from client transports and never reachable
79
+ by clients.
80
+ - **Environment credential**: what authenticates an environment without a
81
+ runtime to the execution transport. It identifies the environment, not a
82
+ user.
83
+ - **Kernel client**: `KernelClient`, the client for the execution transport.
84
+ - **Fence**: the attempt number a request currently holds. A session write
85
+ or an attempt result carrying any other attempt number is refused.
86
+ - **Claim**: the entry point's conditional write that moves a dispatched
87
+ attempt from `pending` to `running` before running it. Only one delivery
88
+ of an attempt claims it.
89
+ - **Deadline**: `attempt_deadline` on a request. While `pending`, the time
90
+ by which an environment must claim the attempt; while `running`, the time
91
+ by which the claimed attempt must report.
92
+ - **Run reconciler**: the kernel task that sends again, re-dispatches, or
93
+ fails requests whose deadline has passed.
94
+
95
+ ## Requirements
96
+
97
+ 1. A strategy's code is identical whichever execution environment runs it.
98
+ Placement is decided by declared workload and deployment configuration,
99
+ never by the strategy's shape.
100
+ 2. `domain/strategy/` imports no kernel service and no infrastructure. It
101
+ owns the `StrategySession` interface and depends only on that; adapters
102
+ implement the interface over kernel services or a kernel client
103
+ (dependency inversion, as with `AccessPort`).
104
+ 3. Every strategy, kernel or application, reaches the graph through the same
105
+ `StrategySession` contract. No strategy holds a capability its principal
106
+ lacks.
107
+ 4. GraphAPI exposes no operation that exists to serve strategy execution.
108
+ Clients write only by executing strategies.
109
+ 5. Every save made by a strategy is authorized by the persistence chokepoint
110
+ against the running strategy's class and the request's owner, whatever
111
+ environment the strategy runs in.
112
+ 6. Authorization, request bookkeeping, retries, and reporting have exactly
113
+ one implementation, in the kernel. An execution environment performs one
114
+ attempt and nothing else.
115
+ 7. An environment without a runtime can run attempts holding no database
116
+ credentials and no broker credentials, with the strategy code and
117
+ `graphcore.public` as its only kernel dependencies. Whether an
118
+ environment holds a runtime is its own decision and changes only its
119
+ entry point.
120
+ 8. The execution environment for a workload is chosen in the composition
121
+ root and can change without changing kernel or strategy code.
122
+ 9. A deployment that maps a declared workload to no dispatcher fails at
123
+ startup, not on the first request.
124
+ 10. A bearer token never crosses a process boundary.
125
+ 11. No execution environment is trusted. An environment can act only for an
126
+ attempt the kernel dispatched, only while that attempt is current, and
127
+ only as that request's owner.
128
+ 12. No strategy runs with kernel-internal authority. Kernel code opens
129
+ `kernel_internal_context()` for its own writes (deployment seeding,
130
+ principal creation, onboarding, the executor's request writes), and
131
+ never while a strategy context is active.
132
+ 13. Every submitted request is saved before it is dispatched, and its
133
+ outcome is saved on it.
134
+ 14. A run does not depend on the process that submitted it. A submitting
135
+ process can restart while an attempt is in flight without losing the
136
+ run.
137
+ 15. A run whose environment fails is detected and either re-dispatched or
138
+ failed, without user action.
139
+
140
+ ## Architecture
141
+
142
+ ### Components
143
+
144
+ ```
145
+ kernel process (ASGI or Celery) environment without a runtime
146
+ ┌────────────────────────────────────────────┐ ┌─────────────────────────────────────────┐
147
+ │ caller ─► executor.execute_request │ │ entry point(data) │
148
+ │ │ save (status = pending, │ │ attempt = ExecutionAttempt.from_dict │
149
+ │ │ attempt = 1, deadline) │ │ kernel.claim_attempt(attempt) │
150
+ │ │ or, on a retry, │ │ tracker = ChangeTracker() │
151
+ │ executor.start_attempt │ │ session = RemoteStrategySession( │
152
+ │ │ save (attempt = n, deadline) │ ──────► │ attempt, tracker, kernel) │
153
+ │ ▼ │ │ result = runner.run( │
154
+ │ dispatchers[workload].dispatch │ │ attempt, session, tracker)│
155
+ │ │ │ kernel.finish_attempt(result) │
156
+ │ entry point (InProcessDispatcher._run, │ │ │
157
+ │ celery run_attempt) │ │ RemoteStrategySession │
158
+ │ executor.claim_attempt(attempt) │ │ get / list / save / execute ──────────┼─┐
159
+ │ save (status = running, deadline) │ └─────────────────────────────────────────┘ │
160
+ │ tracker = ChangeTracker() │ │
161
+ │ session = LocalStrategySession( │ execution transport (kernel process) │
162
+ │ attempt, tracker, entity_service, │ ┌─────────────────────────────────────────┐ │
163
+ │ executor, registry) │ │ environment credential ─► │◄┘
164
+ │ result = runner.run( │ │ LocalStrategySession(attempt, …).<op> │
165
+ │ attempt, session, tracker) │ │ claim / finish_attempt ─► executor │
166
+ │ executor.finish_attempt(result) │ └─────────────────────────────────────────┘
167
+ │ │
168
+ │ executor.finish_attempt │
169
+ │ fence check; save (succeeded | failed) │
170
+ │ or start next attempt; callbacks │
171
+ │ │
172
+ │ LocalStrategySession │
173
+ │ fence check ─► strategy_context ─► │
174
+ │ EntityService / front door │
175
+ │ │
176
+ │ reconcile_runs (periodic) │
177
+ └────────────────────────────────────────────┘
178
+ ```
179
+
180
+ The runner, the strategy, the tracker, the auto-save, and `finish_attempt`
181
+ are identical in every environment. What differs is the session adapter the
182
+ entry point constructs: the local one holds the runtime's services, the
183
+ remote one holds a `KernelClient`. The fence check and `strategy_context`
184
+ live in `LocalStrategySession`; the execution transport serves a remote
185
+ session by building a local one for the same attempt.
186
+
187
+ | Component | Package | Kind | Responsibility |
188
+ |---|---|---|---|
189
+ | `Strategy` | `domain/strategy/` | domain | Strategy behavior; declares `workload` and `timeout` |
190
+ | `StrategySession` | `domain/strategy/session.py` | port (owned by the domain) | The strategy's view of the graph |
191
+ | `StrategyRunner`, `ChangeTracker` | `domain/strategy/runner.py` | domain | Performs one attempt through a session |
192
+ | `LocalStrategySession` | `execution/local_session.py` | adapter | The session over a runtime's `EntityService` and front door |
193
+ | `RemoteStrategySession`, `KernelClient` | `execution/remote_session.py` | adapter | The session over the execution transport |
194
+ | execution transport | `transport/execution.py` | transport | Serves environments without a runtime |
195
+ | `Workload` | `types/strategy.py` | shared type | Declared environment kind |
196
+ | `StrategyRequestPayload` | `types/strategy.py` | shared type | Client-safe description of a request |
197
+ | `RequestStatus` | `types/strategy.py` | shared type | A request's status |
198
+ | `ExecutionAttempt`, `AttemptResult` | `types/execution.py` | shared types | What crosses the dispatcher port, both ways |
199
+ | `new_strategy_request` | `execution/requests.py` | kernel function | The one way a `StrategyRequest` is built |
200
+ | `StrategyExecutorService` | `execution/StrategyExecutorService.py` | kernel service | Front door, attempts, results, callbacks |
201
+ | `Dispatcher` | `execution/dispatcher.py` | port | Delivery of an attempt to an environment |
202
+ | in-process environment | `execution/environments/in_process.py` | adapter and entry point | `InProcessDispatcher`; runs the attempt on this process's event loop |
203
+ | Celery environment | `execution/environments/celery.py` | adapter and entry point | `CeleryDispatcher` and the `run_attempt` task |
204
+ | `RunEvents` | `execution/run_events.py` | port | Tells a waiting caller its run has ended |
205
+ | `InMemoryRunEvents` | `execution/run_events.py` | adapter | `RunEvents` within one process |
206
+ | `RedisRunEvents` | `execution/redis_run_events.py` | adapter | `RunEvents` across processes over Redis pub/sub |
207
+ | `reconcile_runs` | `execution/tasks.py` | kernel task | Run reconciliation |
208
+
209
+ ### Declaring a workload and a timeout
210
+
211
+ ```python
212
+ @dataclass(frozen=True)
213
+ class Workload:
214
+ name: str
215
+
216
+ Workload.IO = Workload('io')
217
+ Workload.CPU = Workload('cpu')
218
+
219
+ class Strategy(ABC):
220
+ workload: Workload = Workload.IO
221
+ timeout: int | None = None # seconds per attempt; None means the deployment default
222
+ ```
223
+
224
+ A workload describes the strategy's needs, not a place. A project defines
225
+ its own (`SANDBOX = Workload('sandbox')`) next to the strategies that
226
+ declare it. The composition root binds workloads to dispatchers:
227
+
228
+ ```python
229
+ executor = StrategyExecutorService(registry, entity_service, default_timeout=600,
230
+ max_attempts=6, dispatch_timeout=300)
231
+ executor.bind_dispatchers({ # default_dispatchers
232
+ Workload.IO: InProcessDispatcher(executor, entity_service, registry),
233
+ Workload.CPU: CeleryDispatcher(),
234
+ })
235
+ ```
236
+
237
+ A Celery worker is the CPU environment, so its runtime binds
238
+ `worker_dispatchers` instead: CPU runs on the worker's own loop, beside IO.
239
+ A CPU strategy that waits on a nested CPU request then runs it beside
240
+ itself instead of waiting for a second worker slot, which a busy pool may
241
+ never free before the parent's deadline. The worker says what it is when it
242
+ starts (`serve_as_worker()`, from Celery's `worker_init`), before its
243
+ runtime is built; the runtime factory receives it as `worker=True`.
244
+
245
+ Invariants:
246
+
247
+ - `build_runtime` raises if a registered strategy declares a workload absent
248
+ from `dispatchers`.
249
+ - Adding a workload is defining a value and a mapping entry. The executor
250
+ never branches on a workload value.
251
+ - `timeout` is declarative metadata, like `workload` and
252
+ `read_only_strategy`: read by the executor, never acted on by the
253
+ strategy.
254
+
255
+ ### The strategy session
256
+
257
+ ```python
258
+ class StrategySession(ABC):
259
+ principal: Principal
260
+
261
+ def create(self, entity_cls: type[E], **fields) -> E: ...
262
+ async def get(self, entity_id: str) -> Entity: ...
263
+ async def get_many(self, entity_ids: list[str]) -> dict[str, Entity]: ...
264
+ async def list(self, entity_filter: EntityFilter, cursor: str | None = None,
265
+ limit: int = 25) -> EntityPage[Entity]: ...
266
+ async def save(self, entity: E) -> E: ...
267
+ async def delete_tree(self, root_id: str) -> list[str]: ...
268
+ async def execute(self, payload: StrategyRequestPayload, *, wait: bool = True,
269
+ callback_channel: str | None = None,
270
+ ) -> tuple[StrategyRequest, Entity | None]: ...
271
+ ```
272
+
273
+ The port holds only what strategies call. Strategies and the runner are its
274
+ only users.
275
+
276
+ A session is bound to one attempt. Its principal is `Principal(owner_id)`
277
+ from the attempt, identity-only; strategy code never passes a principal or
278
+ a strategy identity to it.
279
+
280
+ Contract:
281
+
282
+ - `create` constructs an entity with an identifier minted for this
283
+ deployment. The k-th `create` of an attempt mints
284
+ `derived_entity_id(attempt.backend_id, f"{request_id}/{k}")`, so every
285
+ attempt of a request mints the same identifiers in the same call order.
286
+ It does not persist.
287
+ - The first `save` in an attempt of an entity that attempt's `create`
288
+ minted replaces the entity an earlier attempt of the request saved under
289
+ that identifier, if one did. A retried `apply()` therefore recreates its
290
+ entities instead of duplicating them or conflicting with them.
291
+ - `get` raises `NotFound` or `Forbidden`. `get_many` returns only the
292
+ entities the principal may read.
293
+ - `list` is principal-scoped and has `EntityService.list_entities`
294
+ semantics.
295
+ - `save` persists through the persistence chokepoint and raises
296
+ `ConcurrentModificationError` on a stale version.
297
+ - `delete_tree` deletes through the deletion chokepoint.
298
+ - `execute` creates a `StrategyRequest` from the payload and submits it
299
+ through a front door as the session's principal. The nested request is
300
+ authorized on its own, and behaves as the front door does for any caller:
301
+ with `wait=True` it returns once the run succeeds or raises the error the
302
+ run failed with. The calling strategy decides whether to catch it; if it
303
+ does not, its own run fails with that error.
304
+ - Every write is fenced: `save` or `delete_tree` takes effect only while
305
+ the attempt's request is `running` with the attempt's number, or the
306
+ write is refused with `StaleAttempt`. The check and the write are atomic.
307
+ - A strategy calls `save` whenever it wants a change persisted mid-run.
308
+ Each save is broadcast, so saves are how a strategy streams progress to
309
+ clients, and each is a checkpoint. Saving is never required: the runner
310
+ persists the returned target for the strategy (see The runner).
311
+ - A session is built with the attempt's `ChangeTracker` and records in it
312
+ the state of every entity it returns from `get`, `get_many`, or `list`,
313
+ or persists with `save`. This is a requirement on session adapters, not
314
+ an operation strategies see.
315
+ - Each `save` commits on return. A session is not a transaction.
316
+ - Every read is scoped to the session's principal. No session operation
317
+ reads, writes, or checks access on behalf of anyone else.
318
+ - A strategy makes no authorization decision of its own. A rule about who
319
+ may set a field (for example, only a deployment administrator may give a
320
+ group a `key`) belongs to the type's access policy, as a
321
+ `deployment_fields` entry naming the permission on the deployment that
322
+ setting the field requires, and is enforced at the persistence
323
+ chokepoint.
324
+
325
+ `Strategy` exposes the session as `self.session`, alongside
326
+ `self.strategy_request`, `self.entity`, `self.principal`, and
327
+ `self.registry` (read-only type metadata). The helpers
328
+ `self.get_entity`, `self.save_entity`, and `self.execute_request` delegate to
329
+ the session. A strategy holds no reference to `EntityService` or
330
+ `StrategyExecutorService`.
331
+
332
+ ```python
333
+ class SummarizeNotes(Strategy):
334
+ async def apply(self):
335
+ page = await self.session.list(EntityFilter(type_ids=frozenset({'Note'}),
336
+ parent_id=self.entity.entity_id))
337
+ summary = self.session.create(Summary, parent_id=self.entity.entity_id)
338
+ summary.text = summarize(page.items)
339
+ await self.session.save(summary)
340
+ return self.strategy_request, self.entity
341
+ ```
342
+
343
+ ### The session adapters
344
+
345
+ ```python
346
+ class LocalStrategySession(StrategySession):
347
+ def __init__(self, attempt: ExecutionAttempt, tracker: ChangeTracker,
348
+ entity_service: EntityService, executor: StrategyExecutorService,
349
+ registry: Registry): ...
350
+
351
+ class RemoteStrategySession(StrategySession):
352
+ def __init__(self, attempt: ExecutionAttempt, tracker: ChangeTracker,
353
+ kernel: KernelClient): ...
354
+ ```
355
+
356
+ Both take the attempt and the tracker. The rest is how the session reaches
357
+ the graph.
358
+
359
+ `LocalStrategySession` reads and saves through `entity_service` as the
360
+ attempt's principal, submits nested requests through `executor`'s front
361
+ door, and resolves the strategy class from `registry` by
362
+ `attempt.strategy_name`. Each write passes the fence to `EntityService` as
363
+ a repository `Precondition` on the request (`status = running`,
364
+ `attempt = attempt.attempt`), which the repository checks in the write's
365
+ own transaction, and runs inside `strategy_context(strategy_cls)`. A
366
+ `PreconditionFailed` becomes `StaleAttempt`.
367
+
368
+ `RemoteStrategySession` forwards every operation to `kernel`, with the
369
+ attempt attached. It opens no context and checks no fence; the kernel side
370
+ of the execution transport does both, through a `LocalStrategySession`.
371
+ Entities cross the wire as `to_dict()` and are rehydrated with `from_dict`.
372
+
373
+ `KernelClient(base_url, environment_credential)` is the client for the
374
+ execution transport. It is the only thing an environment without a runtime
375
+ is configured with.
376
+
377
+ Invariants:
378
+
379
+ - A session adapter is constructed only by an environment's entry point, or
380
+ by the execution transport on behalf of one.
381
+ - The two adapters pass the same `StrategySessionContract`.
382
+ - Neither adapter holds a bearer token. The remote one holds the
383
+ deployment's environment credential, which authenticates the environment,
384
+ not a user.
385
+
386
+ ### The execution transport
387
+
388
+ `transport/execution.py` mounts routes for environments without a runtime,
389
+ separately from the client transports, under a path that client routing
390
+ never exposes. It is an HTTP transport like the client one, doing wire
391
+ format and error mapping; the logic behind every route is
392
+ `LocalStrategySession` or the executor. Every route takes the attempt
393
+ (`request_id`, `attempt`) and is authenticated at two levels:
394
+
395
+ 1. **The environment.** The request carries the deployment's environment
396
+ credential. The shipped default is a shared secret
397
+ (`GRAPHCORE_ENVIRONMENT_SECRET`); a deployment substitutes mTLS or a
398
+ network boundary through its runtime factory.
399
+ 2. **The attempt.** The named request must exist and be `running` with the
400
+ named attempt number. Otherwise the call is refused with `StaleAttempt`.
401
+ This is the fence, applied to every call, reads included.
402
+
403
+ | Route | Does |
404
+ |---|---|
405
+ | `get`, `get_many`, `list`, `save`, `delete_tree`, `execute` | builds a `LocalStrategySession` for the attempt and calls the same operation on it |
406
+ | `finish_attempt` | passes the `AttemptResult` to this process's executor |
407
+
408
+ Invariants:
409
+
410
+ - The execution transport makes no authorization decision of its own beyond
411
+ the environment credential. The fence, the strategy context, and the
412
+ principal come from `LocalStrategySession`, which takes them from the
413
+ persisted request, never from the caller.
414
+ - GraphAPI and the client transports expose none of these routes.
415
+ - A nested `execute` from a remote session is submitted and dispatched by
416
+ the kernel process serving the transport. The remote environment never
417
+ blocks on anything it submitted.
418
+
419
+ ### Strategy requests and request templates
420
+
421
+ A `StrategyRequest` is one run: it is submitted to the front door and runs
422
+ immediately. A `RequestTemplate` describes a request to make later and is
423
+ never run itself; requests are created from it, for example on each tick of
424
+ its schedule (`docs/SCHEDULING_TDD.md`). Both are ordinary top-level
425
+ entities: created with `EntityService.create`, saved through `EntityService`,
426
+ with the deployment as parent whatever their target.
427
+
428
+ | Field | `StrategyRequest` | `RequestTemplate` |
429
+ |---|---|---|
430
+ | `strategy_name`, `param_config`, `target_entity_id` | what it runs | what each request made from it runs |
431
+ | `owner` | the principal it runs as | the principal its requests run as |
432
+ | `callbacks` | payloads submitted, as the same principal, after it succeeds | copied to each request made from it |
433
+ | `callback_channel`, `client_request_id` | where and how to report completion | — |
434
+ | `status` | a `RequestStatus` | — |
435
+ | `attempt`, `attempt_deadline` | the current attempt number and its deadline | — |
436
+ | `ret_val`, `error`, `error_code`, `error_details` | the outcome; a failure keeps its error's message, `code`, and details | — |
437
+ | `template_id` | the template it was made from, if any | — |
438
+ | `cron`, `enabled` | — | its optional schedule |
439
+
440
+ `StrategyRequestPayload` is the client-safe description of a request:
441
+ `strategy_name`, `param_config`, `target_entity_id`, `callbacks`, and
442
+ `client_request_id`. Every `StrategyRequest` is built from one by
443
+ `new_strategy_request(entity_service, payload, template_id=None)`, which
444
+ creates the entity with `EntityService.create` and copies the payload's
445
+ fields onto it. The caller that submits a request builds it:
446
+
447
+ | Caller | Payload | `template_id` |
448
+ |---|---|---|
449
+ | GraphAPI | the client's | none |
450
+ | `StrategySession.execute` | the strategy's (`request_constructor`) | none |
451
+ | `finish_attempt` | each of a succeeded request's `callbacks`, with `source_ret_val` added to its `param_config` | none |
452
+ | a scheduler's tick | `template.to_payload()` | the template's id |
453
+
454
+ `template_id` is never read from a payload, so a client cannot claim a
455
+ request was made from a template.
456
+
457
+ A `RequestTemplate` is created by `CreateRequestTemplate`, an
458
+ ordinary strategy, from its `param_config` (`strategy_name`, `param_config`,
459
+ `target_entity_id`, and optionally `callbacks`, `cron`, `enabled`), and saved
460
+ through its session.
461
+
462
+ ```python
463
+ RequestStatus = Literal['pending', 'running', 'succeeded', 'failed']
464
+ ```
465
+
466
+ A request is `pending` from the front door's save until an environment
467
+ claims an attempt, and again whenever a next attempt is dispatched;
468
+ `running` while a claimed attempt is current; and `succeeded` or `failed`
469
+ once the run ends. `succeeded` and `failed` are terminal. Like the kernel's
470
+ other closed value sets, `RequestStatus` is a `Literal` alias, not an
471
+ `Enum`; unlike `Workload`, no project adds values to it.
472
+
473
+ `RequestStatus` is the only status vocabulary for a run. GraphAPI's
474
+ `StrategyOutcome.status` is the request's `status` at the time the front
475
+ door returns: `pending` or `running` with `wait=False`, `succeeded` or
476
+ `failed` with `wait=True`. A client reads the same values on the outcome,
477
+ on the request entity, and on the request's broadcasts.
478
+
479
+ Request saves are broadcast like any entity save, so a client can follow a
480
+ request's status by subscribing to it.
481
+
482
+ Invariants:
483
+
484
+ - A request runs as its `owner`, and nothing else names who it runs as. The
485
+ persistence chokepoint sets `owner` to the acting principal when an entity
486
+ is created and restores it from storage on every later save, so neither a
487
+ caller, a client payload, nor a strategy can choose or change it. The same
488
+ holds for a template's `owner`.
489
+ - `RequestTemplate` declares `dependent_on = ('target_entity_id',)`:
490
+ deleting a target deletes the templates that target it. `StrategyRequest`
491
+ declares none, so deleting a target keeps the history of runs on it,
492
+ including the run doing the deleting.
493
+ - No strategy writes a `StrategyRequest`. The executor creates it and writes
494
+ `status`, `attempt`, `attempt_deadline`, `ret_val`, and the error fields, in
495
+ `kernel_internal_context()`, never inside a strategy context. A strategy
496
+ sets `self.strategy_request.ret_val`; it reaches the request through the
497
+ `AttemptResult`.
498
+ - `CreateRequestTemplate` and `ManageRequestTemplate` are
499
+ the only strategies the access policy allows to write a `RequestTemplate`.
500
+ Both reject an unregistered `strategy_name` and an invalid `cron` before
501
+ saving.
502
+ - Requests and templates are readable by their owner only.
503
+
504
+ ### Attempts and attempt results
505
+
506
+ ```python
507
+ @dataclass(frozen=True)
508
+ class ExecutionAttempt:
509
+ request_id: str
510
+ attempt: int
511
+ strategy_name: str
512
+ target_entity_id: str | None
513
+ owner_id: str
514
+ backend_id: str # for identifier minting in `create`
515
+
516
+ @dataclass(frozen=True)
517
+ class AttemptResult:
518
+ request_id: str
519
+ attempt: int
520
+ outcome: Literal['succeeded', 'conflict', 'failed']
521
+ ret_val: dict | None = None
522
+ error: dict | None = None # the failing KernelError.to_dict()
523
+ ```
524
+
525
+ Both have `to_dict()` and `from_dict()`, like every shared type.
526
+
527
+ Invariants:
528
+
529
+ - The `to_dict()` form never carries a token, a live entity, or a pickled object.
530
+ The environment acts as `Principal(owner_id)`, identity-only, so an
531
+ out-of-process attempt cannot perform delegated backend-to-backend calls.
532
+ - An attempt carries the request by reference and the fields the runner
533
+ needs to start without a kernel. Everything else the runner reads comes
534
+ through its session.
535
+ - An `AttemptResult` never carries an entity. The target's final state is
536
+ written by the runner through its session, not returned.
537
+
538
+ ### The dispatcher port
539
+
540
+ ```python
541
+ class Dispatcher(ABC):
542
+ @abstractmethod
543
+ async def dispatch(self, attempt: ExecutionAttempt) -> None:
544
+ """Deliver the attempt to the environment and return once it is accepted."""
545
+
546
+ async def drain(self) -> None:
547
+ """Wait for attempts this process is running on the current loop."""
548
+ ```
549
+
550
+ Contract:
551
+
552
+ - `dispatch` returns when the environment has accepted the attempt, not when
553
+ it finishes. It raises only if the attempt could not be delivered.
554
+ - The result of the attempt reaches the kernel through
555
+ `StrategyExecutorService.finish_attempt`, however the environment gets it
556
+ there. `dispatch` never returns a result.
557
+ - A dispatcher performs no persistence, authorization, or reporting.
558
+ - A dispatcher may deliver an attempt more than once. The entry point's
559
+ claim makes every delivery after the first run nothing.
560
+ - `drain` returns once the attempts this process started on the current
561
+ event loop have finished. An entry point that owns its loop, such as a
562
+ Celery task, drains every dispatcher before returning, so an IO child
563
+ submitted by a CPU strategy is not lost when the loop closes.
564
+
565
+ ### Execution environments
566
+
567
+ An environment is one module holding its dispatcher and its entry point:
568
+ the two halves of one serialization, `to_dict()` on the kernel side and
569
+ `from_dict()` on the environment's. The entry point builds a
570
+ `ChangeTracker` and a session, runs the runner, and reports the result. Which
571
+ session it builds follows one rule:
572
+
573
+ - An environment with a runtime builds `LocalStrategySession` from that
574
+ runtime's services and calls `finish_attempt` on its own executor.
575
+ - An environment without a runtime builds `RemoteStrategySession` over a
576
+ `KernelClient` and posts its result to the execution transport.
577
+
578
+ Nothing else differs. An environment moves from one kind to the other by
579
+ changing its entry point.
580
+
581
+ #### In-process (`execution/environments/in_process.py`)
582
+
583
+ The kernel process's own event loop, so the dispatcher is also the entry
584
+ point:
585
+
586
+ ```python
587
+ class InProcessDispatcher(Dispatcher):
588
+ def __init__(self, executor, entity_service, registry): ...
589
+
590
+ async def dispatch(self, attempt):
591
+ asyncio.create_task(self._run(attempt))
592
+
593
+ async def _run(self, attempt):
594
+ if not await self.executor.claim_attempt(attempt):
595
+ return
596
+ tracker = ChangeTracker()
597
+ session = LocalStrategySession(attempt, tracker, self.entity_service,
598
+ self.executor, self.registry)
599
+ result = await StrategyRunner(self.registry).run(attempt, session, tracker)
600
+ await self.executor.finish_attempt(result)
601
+ ```
602
+
603
+ A failure the runner does not catch becomes a `failed` result.
604
+
605
+ #### Celery (`execution/environments/celery.py`)
606
+
607
+ A Celery process that holds a runtime. The dispatcher enqueues the attempt's
608
+ `to_dict()` form from a plain executor thread, so an eager-mode task runs on
609
+ its own event loop, as in a worker, never on the caller's. The task is the
610
+ entry point:
611
+
612
+ ```python
613
+ class CeleryDispatcher(Dispatcher):
614
+ async def dispatch(self, attempt):
615
+ await asyncio.get_running_loop().run_in_executor(
616
+ None, run_attempt.delay, attempt.to_dict())
617
+
618
+ @shared_task(name='graphcore.tasks.run_attempt')
619
+ def run_attempt(data):
620
+ rt = get_runtime()
621
+ attempt = ExecutionAttempt.from_dict(data)
622
+
623
+ async def run():
624
+ if not await rt.executor.claim_attempt(attempt):
625
+ return
626
+ tracker = ChangeTracker()
627
+ session = LocalStrategySession(attempt, tracker, rt.entity_service, rt.executor, rt.registry)
628
+ result = await StrategyRunner(rt.registry).run(attempt, session, tracker)
629
+ await rt.executor.finish_attempt(result)
630
+ await settle(rt.executor) # every dispatcher's drain()
631
+
632
+ async_to_sync(run)()
633
+ ```
634
+
635
+ Every kernel task has an explicit `graphcore.tasks.*` name, so the name a
636
+ producer enqueues never depends on a module path.
637
+
638
+ The same process consumes the kernel's periodic tasks (`reconcile_runs`,
639
+ and the scheduler's tasks, `docs/SCHEDULING_TDD.md`).
640
+
641
+ #### An environment without a runtime
642
+
643
+ A project's sandbox or GPU host supplies its own dispatcher and an entry
644
+ point of the same shape, built from `graphcore.public`:
645
+
646
+ ```python
647
+ kernel = KernelClient(settings.GRAPHCORE_KERNEL_URL, settings.GRAPHCORE_ENVIRONMENT_SECRET)
648
+ runner = StrategyRunner(registry) # the project's registered strategies
649
+
650
+ def run_attempt(data):
651
+ attempt = ExecutionAttempt.from_dict(data)
652
+ if not kernel.claim_attempt(attempt):
653
+ return
654
+ tracker = ChangeTracker()
655
+ session = RemoteStrategySession(attempt, tracker, kernel)
656
+ result = async_to_sync(runner.run)(attempt, session, tracker)
657
+ kernel.finish_attempt(result)
658
+ ```
659
+
660
+ Its only configuration is the kernel URL and the environment credential.
661
+ The strategy code must be importable there; nothing else from the kernel is
662
+ needed.
663
+
664
+ Invariants:
665
+
666
+ - Only the composition root constructs a dispatcher.
667
+ - `execution/environments/celery.py` is the only module in `execution/`
668
+ that imports Celery, apart from `execution/tasks.py`.
669
+ - An entry point never writes a `StrategyRequest` and never opens
670
+ `kernel_internal_context()`; it claims through `claim_attempt` and
671
+ reports through `finish_attempt`.
672
+ - An entry point runs an attempt only after claiming it.
673
+
674
+ ### The runner
675
+
676
+ `StrategyRunner(registry).run(attempt, session, tracker) -> AttemptResult`
677
+ performs one attempt. It is domain code: it imports `Strategy`, `StrategySession`,
678
+ `ChangeTracker`, and the registry, and nothing from the kernel's services.
679
+
680
+ 1. Load the target through the session, if the request has one; a missing
681
+ target is a `failed` result.
682
+ 2. Construct the strategy with the session, validate `param_config` against
683
+ its schema, check `verify_executable()`, and await `apply()`. An invalid
684
+ `param_config` fails with the path of the offending parameter.
685
+ 3. If the strategy is not `read_only_strategy` and
686
+ `tracker.is_unsaved(target)`, save the returned target through the
687
+ session. This is the auto-save: the same path, chokepoint, and fence as
688
+ any save the strategy made itself, so a target the strategy already saved,
689
+ or never changed, is not saved again.
690
+ 4. Run `cleanup()`; a cleanup error is logged, never raised.
691
+ 5. Return `succeeded` with `strategy_request.ret_val`, `conflict` on
692
+ `ConcurrentModificationError` from `apply()` or the auto-save, or `failed`
693
+ on any other exception. A failure carries its error as `to_dict()`; an
694
+ exception that is not a `KernelError` is reported as `StrategyFailed`.
695
+ `StaleAttempt` is `failed`: the kernel has already moved on.
696
+
697
+ ```python
698
+ class ChangeTracker:
699
+ def record(self, entity: Entity) -> None: ... # called by session adapters
700
+ def is_unsaved(self, entity: Entity) -> bool: ... # called by the runner
701
+ ```
702
+
703
+ `is_unsaved(entity)` is true when the entity differs from its recorded state
704
+ or has none.
705
+
706
+ Invariants:
707
+
708
+ - A strategy is constructed only by the runner. The runner opens no
709
+ context: the session opens `strategy_context` around each write, on the
710
+ kernel side for a remote session.
711
+ - The runner writes only through the session. It never writes a
712
+ `StrategyRequest` and never opens `kernel_internal_context()`.
713
+ - `apply()` runs once per attempt. A strategy is safe to run more than once.
714
+ - The runner performs no authorization and no retry. Both belong to the
715
+ kernel.
716
+
717
+ ### The executor
718
+
719
+ `StrategyExecutorService` lives in every kernel process. It owns three
720
+ operations.
721
+
722
+ #### The front door
723
+
724
+ `execute_request(request, principal, wait=True, callback_channel=None)
725
+ -> tuple[StrategyRequest, Entity | None]`:
726
+
727
+ 1. Refuse a request whose id already exists. The front door only creates
728
+ requests.
729
+ 2. Resolve the strategy class; an unregistered name raises `InvalidRequest`.
730
+ 3. Authorize (`_can_execute`): allowlist on the target's type, read
731
+ permission and the strategy's required permission on the persisted
732
+ target, or on the deployment for an untargeted strategy.
733
+ 4. Record `callback_channel` on the request and save it with
734
+ `status = pending`, `attempt = 1`, and
735
+ `attempt_deadline = now + dispatch_timeout` as `principal`, in
736
+ `kernel_internal_context()`. The chokepoint sets its `owner` to
737
+ `principal`. Should the process stop before step 5, the reconciler sends
738
+ attempt 1 at that deadline.
739
+ 5. Dispatch attempt 1 as Starting an attempt's step 3 does, without its
740
+ re-authorization: step 3 above just authorized the request's owner.
741
+ 6. With `wait=False`, return `(request, None)`. With `wait=True`, wait for
742
+ the request to reach a terminal status (see Waiting). If it `succeeded`,
743
+ return `(request, target)`, where `target` is the target reloaded as the
744
+ principal, or `None` for an untargeted request. If it `failed`, raise
745
+ `request.failure()`: the `KernelError` of the class whose `code` the
746
+ request recorded, with its message and details and the request's id as
747
+ `strategy_request_id`, or `StrategyFailed` for an unexpected error.
748
+
749
+ The front door does not know who called it. A client transport, a
750
+ strategy's nested `execute`, and a scheduler's tick all get the same
751
+ result, and each decides what a raised error means to it. GraphAPI turns it
752
+ into a `failed` `StrategyOutcome`.
753
+
754
+ A request refused at steps 1 to 3 is not saved.
755
+
756
+ #### Starting an attempt
757
+
758
+ `start_attempt(request)` sends attempt `request.attempt + 1`; the
759
+ reconciler also resends an unclaimed attempt. Sending attempt `n`:
760
+
761
+ 1. Re-run `_can_execute` for the request's owner against current persisted
762
+ state. On refusal, finish the run as `failed`.
763
+ 2. In `kernel_internal_context()`, save the request with
764
+ `status = pending`, `attempt = n`, and
765
+ `attempt_deadline = now + dispatch_timeout`.
766
+ 3. Build the `ExecutionAttempt` and call
767
+ `dispatchers[strategy_cls.workload].dispatch(attempt)`. If dispatch
768
+ raises, finish the run as `failed`.
769
+
770
+ #### Claiming an attempt
771
+
772
+ `claim_attempt(attempt: ExecutionAttempt) -> bool`, called by the entry
773
+ point before it runs the attempt:
774
+
775
+ 1. Load the request. If it is not `pending` with `attempt.attempt`, return
776
+ `False`.
777
+ 2. In `kernel_internal_context()`, save it with `status = running` and
778
+ `attempt_deadline = now + (strategy.timeout or default_timeout)`. A
779
+ version conflict means another delivery claimed it first: return
780
+ `False`.
781
+ 3. Return `True`.
782
+
783
+ The strategy's timeout therefore counts from when an environment starts the
784
+ attempt, not from when it was queued. An entry point given `False` runs
785
+ nothing and reports nothing.
786
+
787
+ #### Finishing an attempt
788
+
789
+ `finish_attempt(result: AttemptResult)`:
790
+
791
+ 1. Load the request. If it is not `running` or `result.attempt !=
792
+ request.attempt`, discard the result and return. This is the fence: a
793
+ result from an attempt the reconciler has superseded, or a duplicate
794
+ delivery, has no effect.
795
+ 2. On `succeeded`: in `kernel_internal_context()`, save
796
+ `status = succeeded` and `ret_val`; create a request from each of the
797
+ request's `callbacks` and submit it through the front door as the owner
798
+ with `wait=False`; send `strategy_complete` with `success: True` to
799
+ `callback_channel`, echoing `client_request_id`.
800
+ 3. On `conflict`: if `request.attempt < max_attempts`, wait
801
+ `0.1 * 2**attempt` seconds plus up to 0.1 seconds of jitter, then
802
+ `start_attempt(request)`. Otherwise finish as `failed`.
803
+ 4. On `failed`: in `kernel_internal_context()`, save `status = failed` and
804
+ the error from `KernelError.from_dict(result.error)` as `error`,
805
+ `error_code`, and `error_details`; send `strategy_complete` with
806
+ `success: False`, the error, and its `code`, echoing `client_request_id`.
807
+
808
+ Every step is idempotent for a given `(request_id, attempt)`.
809
+
810
+ #### Waiting
811
+
812
+ Waiting is event-driven through the `RunEvents` port:
813
+
814
+ ```python
815
+ class RunEvents(ABC):
816
+ def subscribe(self, request_id) -> AsyncContextManager[RunSubscription]: ...
817
+ async def publish(self, request_id) -> None: ... # never raises
818
+
819
+ class RunSubscription:
820
+ async def wait(self, until: Optional[float]) -> None: ...
821
+ ```
822
+
823
+ `wait=True` subscribes to the request before dispatching, then loops:
824
+ read the request; return it if its status is terminal; if it is past its
825
+ `attempt_deadline`, reconcile it (see Run reconciliation) and read
826
+ again; otherwise wait on the subscription until it is notified or the
827
+ clock reaches `attempt_deadline`. Whichever process saves a terminal
828
+ status publishes the request's id after the save.
829
+
830
+ A notification carries only the id, never the outcome: the waiter always
831
+ reads the request, so a duplicate, early, or lost notification changes
832
+ nothing but when it reads. The attempt deadline is the only timer. A lost
833
+ notification delays the waiter to that deadline, at which it reads the
834
+ request, finds it terminal, and returns.
835
+
836
+ `InMemoryRunEvents` notifies subscribers in its own process, from any
837
+ thread or event loop. `RedisRunEvents` also publishes on one Redis channel
838
+ and runs one listener thread per process, started on its first
839
+ subscription. Each time the listener subscribes or resubscribes, it
840
+ notifies every subscriber in its process, so each re-reads its request and
841
+ nothing published while the listener was disconnected is relied on.
842
+
843
+ Invariants:
844
+
845
+ - Every attempt is authorized in the kernel when it starts, so an owner who
846
+ lost access stops running, including on reconciled re-dispatch.
847
+ - `kernel_internal_context()` raises when opened while a strategy context is
848
+ active. The executor opens it only to write requests.
849
+ - The `strategy_complete` message has one shape for every environment,
850
+ including `client_request_id`.
851
+ - Nothing in the submitting process holds the run. The result is handled by
852
+ whichever kernel process receives it.
853
+
854
+ ### Run reconciliation
855
+
856
+ Requests are the single source of truth for runs; the executor reconciles
857
+ toward them (`ARCHITECTURE_PRINCIPLES.md`, rule 8). `finish_attempt` only
858
+ runs if the environment lives long enough to report, so the
859
+ `graphcore.tasks.reconcile_runs` task runs in the Celery process when a
860
+ worker starts and every `GRAPHCORE_RECONCILE_INTERVAL` seconds (300 by
861
+ default), through the `graphcore_reconcile_runs` periodic task that
862
+ `BeatScheduler` ensures alongside its own (`docs/SCHEDULING_TDD.md`). For each request with `status` `pending` or `running` and
863
+ `attempt_deadline` in the past:
864
+
865
+ - if `pending`, no environment claimed the attempt, or the front door
866
+ stopped before dispatching: send attempt `attempt` again. The
867
+ same attempt number is sent, so a slow queue costs the run no attempts,
868
+ and whichever delivery claims first runs;
869
+ - if `running` and `attempt < max_attempts`, `start_attempt(request)`;
870
+ - otherwise finish the run as `failed` with error `attempt timed out`.
871
+
872
+ Because `start_attempt` increments `attempt`, any late result or write from
873
+ the superseded attempt is refused by the fence.
874
+
875
+ Invariants:
876
+
877
+ - A request never stays `pending` or `running` past its deadline plus one
878
+ reconcile interval. A request with a waiter is reconciled by that waiter
879
+ at its deadline.
880
+ - A re-dispatched attempt is authorized and fenced exactly like a first one.
881
+
882
+ ### Composition
883
+
884
+ `build_runtime` receives the dispatcher mapping and the executor's settings
885
+ and constructs `StrategyExecutorService(registry, entity_service,
886
+ default_timeout, max_attempts, run_events)`, then calls `executor.bind_dispatchers`
887
+ with the mapping a dispatcher factory `(executor, entity_service, registry)`
888
+ returns, since an in-process dispatcher reports back to the executor.
889
+ `bind_dispatchers` is called only there, and it raises if a registered
890
+ strategy's workload has no dispatcher. `InProcessDispatcher` is
891
+ constructed there with the executor, `entity_service`, and the registry.
892
+ `run_events` defaults to `InMemoryRunEvents`. The Django runtime chooses it
893
+ from `CELERY_BROKER_URL`: `memory://` (Celery eager, one process) gives
894
+ `InMemoryRunEvents`, `redis://` or `rediss://` gives `RedisRunEvents` on
895
+ the broker's Redis, and any other broker fails at startup, since a run
896
+ finished by a worker would never wake its waiter. A project's runtime
897
+ factory can pass its own adapter.
898
+ `transport/execution.py` is mounted by the ASGI process's URL
899
+ configuration, behind the environment credential.
900
+
901
+ `Workload`, `Dispatcher`, `ExecutionAttempt`, `AttemptResult`,
902
+ `StrategySession`, `StrategyRunner`, `ChangeTracker`,
903
+ `RemoteStrategySession`, and `KernelClient` are exported from
904
+ `graphcore.public`, so a project's runtime factory
905
+ (`GRAPHCORE_RUNTIME_FACTORY`) can supply its own dispatchers and a project's
906
+ environment can run an entry point of its own.
907
+
908
+ Boundary tests in `tests/test_composition_boundaries.py` hold:
909
+
910
+ - Only the composition root constructs a `Dispatcher` adapter or
911
+ `RedisRunEvents`.
912
+ - Only `execution/environments/celery.py` and `execution/tasks.py` import
913
+ Celery within `execution/`.
914
+ - `domain/strategy/` imports nothing from `graph/`, `execution/`,
915
+ `persistence/`, `broadcast/`, `scheduling/`, or `auth/authz/adapter/`.
916
+ `StrategyRunner` is in `domain/strategy/` and is held to the same rule.
917
+ - `execution/remote_session.py` imports nothing from `graph/`,
918
+ `persistence/`, `broadcast/`, `scheduling/`, or `auth/`, and never calls
919
+ `get_runtime()`.
920
+
921
+ Conformance suites, exported from `graphcore.public.testing`:
922
+
923
+ - `DispatcherContract`: `dispatch` returns before the attempt finishes;
924
+ the attempt's result reaches `finish_attempt`; a failing runner yields a
925
+ `failed` result. Runs against `InProcessDispatcher` and against
926
+ `CeleryDispatcher` with Celery in eager mode.
927
+ - `StrategySessionContract`: the session contract above, including the
928
+ fence, against `LocalStrategySession` and against `RemoteStrategySession`
929
+ over the execution transport.
930
+
931
+ ### Worked example: a CPU workload
932
+
933
+ ```python
934
+ @register_strategy(Document)
935
+ class EmbedDocument(Strategy):
936
+ workload = Workload.CPU
937
+ timeout = 1800
938
+
939
+ async def apply(self):
940
+ chunks = await self.session.list(EntityFilter(type_ids=frozenset({'Chunk'}),
941
+ parent_id=self.entity.entity_id))
942
+ self.entity.embedding = embed([c.text for c in chunks.items])
943
+ return self.strategy_request, self.entity
944
+ ```
945
+
946
+ 1. A client executes `EmbedDocument` through GraphAPI. The front door
947
+ authorizes it, saves the request with `status = pending`, `attempt = 1`
948
+ and a dispatch deadline, and `CeleryDispatcher` enqueues the attempt.
949
+ 2. The Celery process's `run_attempt` claims attempt 1, which saves it
950
+ `running` with `attempt_deadline` 30 minutes out, then builds a
951
+ `LocalStrategySession` for it from its runtime and runs the runner. `apply()`'s `list` and
952
+ the auto-save go through that session, which checks the fence, opens
953
+ `strategy_context(EmbedDocument)`, and writes through the chokepoint as
954
+ the request's owner.
955
+ 3. `run_attempt` passes its `AttemptResult` to its own executor's
956
+ `finish_attempt`, which saves `succeeded` and sends `strategy_complete`
957
+ to the client's channel.
958
+ 4. With `wait=True`, the ASGI process's front door sees the request's
959
+ `succeeded` save on its subscription, reloads the target, and returns
960
+ `(request, entity)`.
961
+
962
+ If the Celery process dies during step 2, `reconcile_runs` finds the request
963
+ past its deadline; `start_attempt` saves it with `attempt = 2` and a new
964
+ `attempt_deadline`, and dispatches again. A save from
965
+ the dead attempt, should it ever arrive, is refused by the fence.
966
+
967
+ The strategy's code is the same with `workload = Workload.IO`; only the
968
+ dispatcher differs, and the attempt runs in the ASGI process. It is also the
969
+ same in an environment without a runtime; only the entry point differs.
970
+
971
+ ### Trade-offs
972
+
973
+ - **Every remote read and write is a round trip.** An environment without
974
+ a runtime reaches the graph through the execution transport. For CPU
975
+ workloads, which read once, compute, and save once, the cost is noise; a
976
+ strategy that performs many small reads is an IO workload.
977
+ - **`apply()` reruns on conflict and on timeout.** Every attempt repeats the
978
+ whole strategy, including expensive CPU work. Strategies with costly
979
+ `apply()` bodies keep their contended write small and late, and declare a
980
+ `timeout` that covers their worst case.
981
+ - **Sessions are a contract, not a sandbox.** In a kernel process, strategy
982
+ code can import kernel modules. The session is the supported surface for
983
+ strategies; the boundary tests enforce it for the kernel's own code. In
984
+ an environment without a runtime, the session is the only path there is.
985
+ - **Out-of-process attempts are identity-only.** A CPU workload cannot make
986
+ delegated backend-to-backend calls, which follows from requirement 10.
987
+ - **The Celery process holds a runtime.** It has database and broker
988
+ credentials and could reach the graph around its session; the boundary
989
+ tests, not isolation, keep it on the session. Moving it to the remote
990
+ session is an entry-point change, taken when isolation matters more than
991
+ the round trips.
992
+ - **Request writes per run.** A run costs a `pending` save, a claim save per
993
+ attempt, a dispatch save per retry, and a terminal save, each broadcast, in
994
+ exchange for a durable, observable history of every run, a timeout that
995
+ excludes queue time, and a reconciler that needs no other state.
996
+ - **A slow queue is sent duplicates.** An attempt still queued at its
997
+ dispatch deadline is sent again, once per deadline, until a delivery
998
+ claims it; every later delivery is a claim that fails and runs nothing.
999
+ With no worker consuming, a run stays `pending` rather than failing.
1000
+ - **CPU work in a worker stays in that worker.** A CPU request submitted
1001
+ from a Celery worker, nested, scheduled, or reconciled, runs on that
1002
+ worker's loop rather than on another worker, so the task that submitted it
1003
+ lasts until it finishes.
1004
+ - **Created identifiers follow from the request.** Anyone who can read a
1005
+ request can compute the identifiers its strategy will create. Taking one
1006
+ first makes that run fail; it grants nothing, since the run's save is
1007
+ still authorized against the stored entity.
1008
+ - **A fence per write.** Every session write locks the request's row for
1009
+ the length of its transaction, so writes of one attempt serialize on it,
1010
+ and a reconciler's `start_attempt` waits for an in-flight write to commit.
1011
+ Every execution-transport call also checks the fence, reads included.
1012
+ - **Waiting across processes needs a broker.** A waiter in one process
1013
+ learns of a run finished in another only through a cross-process
1014
+ `RunEvents` adapter, so the Django runtime refuses a broker it has no
1015
+ adapter for. Each process holds one listener connection to Redis for as
1016
+ long as it has had a waiter.
1017
+ - **Strategies see only what their principal sees.** A kernel strategy
1018
+ validates against principal-scoped reads like any other. An administrator
1019
+ can name, for example, only groups they can read as new-user defaults.
1020
+ - **A scheduled tick is submitted from the Celery process.** An IO strategy
1021
+ on a schedule therefore runs on that process's event loop through its
1022
+ `InProcessDispatcher`, not in the ASGI process. Workload decides the kind
1023
+ of environment, not a specific process.
1024
+
1025
+ ### Open Items
1026
+
1027
+ - **Environment authentication.** The shipped default is a shared secret.
1028
+ Whether the kernel also issues per-attempt tokens so an environment
1029
+ credential alone cannot act for an attempt it was not dispatched, and
1030
+ whether mTLS is the shipped default for deployments with a network
1031
+ boundary.
1032
+ - **Authorization at template creation.** Creating a template checks only
1033
+ that its creator may create a `RequestTemplate`. Whether the owner may
1034
+ execute the named strategy on its target is checked for each request made
1035
+ from it. Whether to also check it when the template is created, and
1036
+ through which session operation.
1037
+ - **Request retention.** Requests accumulate. Whether to prune finished
1038
+ ones, after how long, and whether pruning is a reconciler over requests.
1039
+ - **Reconcile interval and default timeout.** Whether 600 seconds per
1040
+ attempt and the reconcile interval are exposed as settings, and what
1041
+ defaults suit deployments with long-running CPU strategies.
1042
+ - **GPU and other kernel workloads.** The kernel defines further `Workload`
1043
+ values only when it ships a dispatcher for them.