graphcore-kernel 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (343) hide show
  1. graphcore/__init__.py +5 -0
  2. graphcore/agent_frontend/__init__.py +8 -0
  3. graphcore/agent_frontend/api.py +69 -0
  4. graphcore/agent_frontend/base/__init__.py +11 -0
  5. graphcore/agent_frontend/base/views.py +167 -0
  6. graphcore/agent_frontend/client.py +31 -0
  7. graphcore/agent_frontend/context.py +157 -0
  8. graphcore/agent_frontend/errors.py +12 -0
  9. graphcore/agent_frontend/loading.py +54 -0
  10. graphcore/agent_frontend/registry.py +42 -0
  11. graphcore/agent_frontend/render.py +207 -0
  12. graphcore/agent_frontend/server.py +103 -0
  13. graphcore/agent_frontend/signatures.py +35 -0
  14. graphcore/agent_frontend/testing.py +45 -0
  15. graphcore/agent_frontend/tests/__init__.py +0 -0
  16. graphcore/agent_frontend/tests/fakes.py +114 -0
  17. graphcore/agent_frontend/tests/test_frontend.py +447 -0
  18. graphcore/agent_frontend/tests/test_loading.py +83 -0
  19. graphcore/agent_frontend/tests/test_server.py +109 -0
  20. graphcore/agent_frontend/tools.py +137 -0
  21. graphcore/api/__init__.py +4 -0
  22. graphcore/api/graph_api.py +378 -0
  23. graphcore/api/predicate_codec.py +81 -0
  24. graphcore/api/tests/__init__.py +0 -0
  25. graphcore/api/tests/test_changed_entities.py +109 -0
  26. graphcore/api/tests/test_graph_api.py +290 -0
  27. graphcore/api/tests/test_graph_api_access.py +94 -0
  28. graphcore/api/tests/test_graph_api_grants.py +90 -0
  29. graphcore/api/tests/test_predicate_codec.py +83 -0
  30. graphcore/api/tests/test_subtree_query.py +79 -0
  31. graphcore/apps.py +71 -0
  32. graphcore/auth/__init__.py +1 -0
  33. graphcore/auth/authn/README.md +178 -0
  34. graphcore/auth/authn/__init__.py +3 -0
  35. graphcore/auth/authn/authenticator.py +120 -0
  36. graphcore/auth/authn/onboarding.py +105 -0
  37. graphcore/auth/authn/principals.py +20 -0
  38. graphcore/auth/authn/resource_metadata.py +43 -0
  39. graphcore/auth/authn/service_accounts.py +96 -0
  40. graphcore/auth/authn/testing.py +96 -0
  41. graphcore/auth/authn/tests/__init__.py +0 -0
  42. graphcore/auth/authn/tests/test_authenticator.py +145 -0
  43. graphcore/auth/authn/tests/test_onboarding.py +145 -0
  44. graphcore/auth/authn/tests/test_service_accounts.py +180 -0
  45. graphcore/auth/authn/users.py +59 -0
  46. graphcore/auth/authz/__init__.py +0 -0
  47. graphcore/auth/authz/access_port.py +120 -0
  48. graphcore/auth/authz/adapter/__init__.py +0 -0
  49. graphcore/auth/authz/adapter/default_access_adapter.py +556 -0
  50. graphcore/auth/authz/adapter/tests/__init__.py +0 -0
  51. graphcore/auth/authz/adapter/tests/test_default_adapter.py +543 -0
  52. graphcore/auth/authz/adapter/tests/test_entity_write_hooks.py +207 -0
  53. graphcore/auth/authz/adapter/tests/test_group_types.py +70 -0
  54. graphcore/auth/authz/adapter/tests/test_sync_support.py +209 -0
  55. graphcore/auth/authz/defaults.py +26 -0
  56. graphcore/auth/authz/kernel_policy.py +191 -0
  57. graphcore/auth/authz/kernel_writes.py +31 -0
  58. graphcore/auth/authz/lookups.py +25 -0
  59. graphcore/auth/authz/policy.py +403 -0
  60. graphcore/auth/authz/tests/__init__.py +0 -0
  61. graphcore/auth/authz/tests/test_can_contain.py +150 -0
  62. graphcore/auth/authz/tests/test_contained_by.py +167 -0
  63. graphcore/auth/authz/tests/test_creation.py +391 -0
  64. graphcore/auth/authz/tests/test_delete_permission.py +82 -0
  65. graphcore/auth/authz/tests/test_granted_to_me.py +160 -0
  66. graphcore/auth/authz/tests/test_integration.py +637 -0
  67. graphcore/auth/authz/tests/test_invites.py +184 -0
  68. graphcore/auth/authz/tests/test_kernel_policy.py +140 -0
  69. graphcore/auth/authz/tests/test_lockout.py +132 -0
  70. graphcore/auth/authz/tests/test_policy.py +329 -0
  71. graphcore/auth/authz/tests/test_primitives.py +73 -0
  72. graphcore/auth/authz/tests/test_principal_creation.py +62 -0
  73. graphcore/bootstrap/__init__.py +0 -0
  74. graphcore/bootstrap/deployment.py +67 -0
  75. graphcore/bootstrap/signals.py +16 -0
  76. graphcore/bootstrap/tests/__init__.py +0 -0
  77. graphcore/bootstrap/tests/test_deployment_seed.py +154 -0
  78. graphcore/broadcast/__init__.py +4 -0
  79. graphcore/broadcast/broadcaster.py +201 -0
  80. graphcore/broadcast/channels.py +56 -0
  81. graphcore/broadcast/tests/__init__.py +0 -0
  82. graphcore/broadcast/tests/test_broadcaster.py +444 -0
  83. graphcore/celery.py +14 -0
  84. graphcore/docs/AGENT_FRONTEND_TDD.md +389 -0
  85. graphcore/docs/ARCHITECTURE_PRINCIPLES.md +152 -0
  86. graphcore/docs/AUTHORIZATION_TECHNICAL_TDD.md +1379 -0
  87. graphcore/docs/ENTITY_REFERENCE_TDD.md +561 -0
  88. graphcore/docs/GRAPHAPI_FRONT_DOOR.md +173 -0
  89. graphcore/docs/RUNTIME.md +90 -0
  90. graphcore/docs/SCHEDULING_TDD.md +244 -0
  91. graphcore/docs/STRATEGY_EXECUTION_TDD.md +1043 -0
  92. graphcore/docs/VIEW_ENTITY_TDD.md +522 -0
  93. graphcore/docs/VIEW_SYSTEM_TDD.md +307 -0
  94. graphcore/docs/__init__.py +0 -0
  95. graphcore/docs/doc_topics.py +97 -0
  96. graphcore/docs/docs_discovery.py +64 -0
  97. graphcore/docs/public_api.py +56 -0
  98. graphcore/domain/__init__.py +0 -0
  99. graphcore/domain/entity/Entity.py +204 -0
  100. graphcore/domain/entity/__init__.py +0 -0
  101. graphcore/domain/entity/deployment/Deployment.py +25 -0
  102. graphcore/domain/entity/deployment/__init__.py +0 -0
  103. graphcore/domain/entity/grant/Grant.py +31 -0
  104. graphcore/domain/entity/grant/__init__.py +0 -0
  105. graphcore/domain/entity/group/Group.py +48 -0
  106. graphcore/domain/entity/group/README.md +88 -0
  107. graphcore/domain/entity/group/__init__.py +0 -0
  108. graphcore/domain/entity/invite/Invite.py +31 -0
  109. graphcore/domain/entity/invite/__init__.py +0 -0
  110. graphcore/domain/entity/request_template/RequestTemplate.py +48 -0
  111. graphcore/domain/entity/request_template/__init__.py +0 -0
  112. graphcore/domain/entity/request_template/tests/__init__.py +0 -0
  113. graphcore/domain/entity/request_template/tests/test_request_template.py +52 -0
  114. graphcore/domain/entity/service_account/ServiceAccount.py +41 -0
  115. graphcore/domain/entity/service_account/__init__.py +0 -0
  116. graphcore/domain/entity/strategy_request/StrategyRequest.py +61 -0
  117. graphcore/domain/entity/strategy_request/__init__.py +0 -0
  118. graphcore/domain/entity/strategy_request/tests/__init__.py +0 -0
  119. graphcore/domain/entity/strategy_request/tests/test_strategy_request_entity.py +35 -0
  120. graphcore/domain/entity/tests/__init__.py +0 -0
  121. graphcore/domain/entity/tests/test_entity.py +227 -0
  122. graphcore/domain/entity/tests/test_grant_group_entities.py +36 -0
  123. graphcore/domain/entity/tests/test_write_once_fields.py +98 -0
  124. graphcore/domain/entity/user/User.py +39 -0
  125. graphcore/domain/entity/user/__init__.py +0 -0
  126. graphcore/domain/entity/user/tests/__init__.py +0 -0
  127. graphcore/domain/entity/user/tests/test_user_entity.py +33 -0
  128. graphcore/domain/entity/view/View.py +37 -0
  129. graphcore/domain/entity/view/__init__.py +0 -0
  130. graphcore/domain/entity/view/position.py +78 -0
  131. graphcore/domain/entity/view/tests/__init__.py +0 -0
  132. graphcore/domain/entity/view/tests/test_position.py +71 -0
  133. graphcore/domain/entity/view/tests/test_view_entity.py +134 -0
  134. graphcore/domain/strategy/BaseStrategy.py +759 -0
  135. graphcore/domain/strategy/DeploymentStrategies.py +55 -0
  136. graphcore/domain/strategy/GrantStrategies.py +189 -0
  137. graphcore/domain/strategy/GroupStrategies.py +193 -0
  138. graphcore/domain/strategy/InviteStrategies.py +84 -0
  139. graphcore/domain/strategy/RequestTemplateStrategies.py +126 -0
  140. graphcore/domain/strategy/ViewStrategies.py +219 -0
  141. graphcore/domain/strategy/__init__.py +0 -0
  142. graphcore/domain/strategy/runner.py +84 -0
  143. graphcore/domain/strategy/session.py +56 -0
  144. graphcore/domain/strategy/tests/BaseStrategyTest.py +167 -0
  145. graphcore/domain/strategy/tests/__init__.py +0 -0
  146. graphcore/domain/strategy/tests/test_BaseStrategies.py +870 -0
  147. graphcore/domain/strategy/tests/test_grant_strategies.py +245 -0
  148. graphcore/domain/strategy/tests/test_group_strategies.py +99 -0
  149. graphcore/domain/strategy/tests/test_runner.py +220 -0
  150. graphcore/domain/strategy/tests/test_strategy_locks.py +446 -0
  151. graphcore/domain/strategy/tests/test_utils.py +95 -0
  152. graphcore/domain/strategy/tests/test_view_strategies.py +436 -0
  153. graphcore/errors.py +145 -0
  154. graphcore/execution/README.md +97 -0
  155. graphcore/execution/StrategyExecutorService.py +333 -0
  156. graphcore/execution/__init__.py +0 -0
  157. graphcore/execution/dispatcher.py +19 -0
  158. graphcore/execution/environments/__init__.py +0 -0
  159. graphcore/execution/environments/celery.py +59 -0
  160. graphcore/execution/environments/in_process.py +55 -0
  161. graphcore/execution/local_session.py +117 -0
  162. graphcore/execution/redis_run_events.py +74 -0
  163. graphcore/execution/requests.py +22 -0
  164. graphcore/execution/run_events.py +85 -0
  165. graphcore/execution/strategy_context.py +54 -0
  166. graphcore/execution/tasks.py +34 -0
  167. graphcore/execution/tests/__init__.py +5 -0
  168. graphcore/execution/tests/contract.py +171 -0
  169. graphcore/execution/tests/support.py +310 -0
  170. graphcore/execution/tests/test_dispatchers.py +76 -0
  171. graphcore/execution/tests/test_executor.py +368 -0
  172. graphcore/execution/tests/test_local_session.py +95 -0
  173. graphcore/execution/tests/test_reconcile_runs.py +134 -0
  174. graphcore/execution/tests/test_run_events.py +106 -0
  175. graphcore/execution/tests/test_strategy_context.py +62 -0
  176. graphcore/gateway/__init__.py +1 -0
  177. graphcore/gateway/remote.py +35 -0
  178. graphcore/gateway/tests/__init__.py +0 -0
  179. graphcore/gateway/tests/test_remote.py +30 -0
  180. graphcore/graph/EntityService.py +710 -0
  181. graphcore/graph/__init__.py +0 -0
  182. graphcore/graph/entity_cache.py +152 -0
  183. graphcore/graph/tests/__init__.py +4 -0
  184. graphcore/graph/tests/test_EntityService.py +908 -0
  185. graphcore/graph/tests/test_EntityService_list.py +170 -0
  186. graphcore/graph/tests/test_EntityService_model_resolution.py +339 -0
  187. graphcore/graph/tests/test_EntityService_referrers.py +236 -0
  188. graphcore/graph/tests/test_EntityService_routing.py +119 -0
  189. graphcore/graph/tests/test_dependent_on.py +141 -0
  190. graphcore/graph/tests/test_engine_is_type_agnostic.py +25 -0
  191. graphcore/graph/tests/test_engine_resolves_read_permission.py +93 -0
  192. graphcore/graph/tests/test_entity_cache.py +159 -0
  193. graphcore/graph/tests/test_null_entity_cache.py +12 -0
  194. graphcore/management/__init__.py +0 -0
  195. graphcore/management/commands/__init__.py +0 -0
  196. graphcore/management/commands/dev_reset.py +18 -0
  197. graphcore/management/commands/graphcore_createadmin.py +65 -0
  198. graphcore/management/commands/graphcore_policy.py +59 -0
  199. graphcore/management/commands/graphcore_status.py +98 -0
  200. graphcore/management/tests/__init__.py +0 -0
  201. graphcore/management/tests/test_createadmin.py +78 -0
  202. graphcore/management/tests/test_policy_command.py +46 -0
  203. graphcore/management/tests/test_status_command.py +85 -0
  204. graphcore/migrations/0001_initial.py +186 -0
  205. graphcore/migrations/0002_invite.py +26 -0
  206. graphcore/migrations/__init__.py +0 -0
  207. graphcore/models.py +11 -0
  208. graphcore/persistence/ENTITY_REPOSITORY_TECHNICAL_TDD.md +396 -0
  209. graphcore/persistence/__init__.py +6 -0
  210. graphcore/persistence/django/__init__.py +0 -0
  211. graphcore/persistence/django/backends.py +43 -0
  212. graphcore/persistence/django/bindings.py +28 -0
  213. graphcore/persistence/django/cache.py +38 -0
  214. graphcore/persistence/django/compiler.py +214 -0
  215. graphcore/persistence/django/identity.py +18 -0
  216. graphcore/persistence/django/models.py +252 -0
  217. graphcore/persistence/django/repository.py +525 -0
  218. graphcore/persistence/django/tests/__init__.py +0 -0
  219. graphcore/persistence/django/tests/test_backends.py +68 -0
  220. graphcore/persistence/django/tests/test_compiler.py +269 -0
  221. graphcore/persistence/django/tests/test_repository.py +472 -0
  222. graphcore/persistence/django/tests/test_shared_column.py +120 -0
  223. graphcore/persistence/django/tests/test_threads.py +50 -0
  224. graphcore/persistence/django/tests/test_update_shape.py +175 -0
  225. graphcore/persistence/django/threads.py +38 -0
  226. graphcore/persistence/repository.py +172 -0
  227. graphcore/persistence/tests/__init__.py +0 -0
  228. graphcore/persistence/tests/contract.py +688 -0
  229. graphcore/persistence/tests/evaluate.py +51 -0
  230. graphcore/persistence/tests/fixtures.py +32 -0
  231. graphcore/persistence/tests/memory.py +263 -0
  232. graphcore/persistence/tests/test_memory_repository.py +10 -0
  233. graphcore/persistence/tests/test_query_types.py +127 -0
  234. graphcore/public/__init__.py +9 -0
  235. graphcore/public/api.py +9 -0
  236. graphcore/public/apps.py +10 -0
  237. graphcore/public/auth/__init__.py +1 -0
  238. graphcore/public/auth/grants.py +7 -0
  239. graphcore/public/auth/onboarding.py +8 -0
  240. graphcore/public/auth/perms.py +7 -0
  241. graphcore/public/auth/policy.py +10 -0
  242. graphcore/public/auth/roles.py +8 -0
  243. graphcore/public/auth/subjects.py +7 -0
  244. graphcore/public/entities/__init__.py +32 -0
  245. graphcore/public/errors.py +19 -0
  246. graphcore/public/persistence/__init__.py +44 -0
  247. graphcore/public/persistence/django.py +24 -0
  248. graphcore/public/scheduling.py +23 -0
  249. graphcore/public/strategies/__init__.py +85 -0
  250. graphcore/public/test_settings.py +16 -0
  251. graphcore/public/testing.py +47 -0
  252. graphcore/registry/EntityDirectory.py +246 -0
  253. graphcore/registry/README.md +147 -0
  254. graphcore/registry/StrategyDirectory.py +308 -0
  255. graphcore/registry/TYPE_METADATA_PLACEMENT.md +66 -0
  256. graphcore/registry/__init__.py +0 -0
  257. graphcore/registry/app_discovery.py +108 -0
  258. graphcore/registry/decorators.py +77 -0
  259. graphcore/registry/discovery.py +111 -0
  260. graphcore/registry/registry.py +79 -0
  261. graphcore/registry/tests/__init__.py +0 -0
  262. graphcore/registry/tests/test_agent_frontend_discovery.py +56 -0
  263. graphcore/registry/tests/test_app_config.py +36 -0
  264. graphcore/registry/tests/test_discovery.py +147 -0
  265. graphcore/registry/tests/test_entity_directory_storage.py +73 -0
  266. graphcore/registry/tests/test_import_package.py +87 -0
  267. graphcore/registry/tests/test_principal_flag.py +67 -0
  268. graphcore/registry/tests/test_strategy_directory.py +29 -0
  269. graphcore/routes/__init__.py +116 -0
  270. graphcore/routes/tests/__init__.py +0 -0
  271. graphcore/routes/tests/test_routes.py +90 -0
  272. graphcore/runtime.py +243 -0
  273. graphcore/scheduling/__init__.py +0 -0
  274. graphcore/scheduling/beat_scheduler.py +164 -0
  275. graphcore/scheduling/beat_tasks.py +63 -0
  276. graphcore/scheduling/scheduler.py +37 -0
  277. graphcore/scheduling/tests/__init__.py +0 -0
  278. graphcore/scheduling/tests/contract.py +136 -0
  279. graphcore/scheduling/tests/memory.py +42 -0
  280. graphcore/scheduling/tests/test_beat_scheduler.py +87 -0
  281. graphcore/scheduling/tests/test_memory_scheduler.py +15 -0
  282. graphcore/scheduling/tests/test_ticks.py +143 -0
  283. graphcore/tasks.py +9 -0
  284. graphcore/test_settings.py +78 -0
  285. graphcore/testing/__init__.py +0 -0
  286. graphcore/testing/case.py +112 -0
  287. graphcore/testing/runner.py +32 -0
  288. graphcore/testing/settings.py +26 -0
  289. graphcore/testing/tests/__init__.py +0 -0
  290. graphcore/testing/tests/agent_view_fixture.py +19 -0
  291. graphcore/testing/tests/test_case_and_runner.py +219 -0
  292. graphcore/testing/tests/test_settings_for_apps.py +50 -0
  293. graphcore/tests/__init__.py +0 -0
  294. graphcore/tests/ids.py +14 -0
  295. graphcore/tests/public_api_manifest.py +102 -0
  296. graphcore/tests/runs.py +7 -0
  297. graphcore/tests/test_app_discovery.py +90 -0
  298. graphcore/tests/test_composition_boundaries.py +231 -0
  299. graphcore/tests/test_docs_discovery.py +108 -0
  300. graphcore/tests/test_no_print.py +70 -0
  301. graphcore/tests/test_packaging.py +64 -0
  302. graphcore/tests/test_public_api_boundaries.py +229 -0
  303. graphcore/tests/test_public_api_contract.py +62 -0
  304. graphcore/tests/test_public_api_docs.py +28 -0
  305. graphcore/tests/test_runtime.py +42 -0
  306. graphcore/tests/test_shared_type_boundaries.py +128 -0
  307. graphcore/tests/test_shared_types.py +144 -0
  308. graphcore/tests/test_test_settings.py +45 -0
  309. graphcore/transport/__init__.py +0 -0
  310. graphcore/transport/agent_asgi.py +68 -0
  311. graphcore/transport/agent_client.py +85 -0
  312. graphcore/transport/channel_layer.py +68 -0
  313. graphcore/transport/consumers.py +507 -0
  314. graphcore/transport/routing.py +6 -0
  315. graphcore/transport/tests/__init__.py +0 -0
  316. graphcore/transport/tests/test_agent_client.py +31 -0
  317. graphcore/transport/tests/test_api_views.py +733 -0
  318. graphcore/transport/tests/test_channel_layer.py +96 -0
  319. graphcore/transport/tests/test_consumers.py +963 -0
  320. graphcore/transport/tests/test_resource_metadata.py +67 -0
  321. graphcore/transport/urls.py +29 -0
  322. graphcore/transport/views.py +358 -0
  323. graphcore/types/__init__.py +4 -0
  324. graphcore/types/access.py +114 -0
  325. graphcore/types/api.py +76 -0
  326. graphcore/types/entities.py +100 -0
  327. graphcore/types/entity_id.py +85 -0
  328. graphcore/types/execution.py +41 -0
  329. graphcore/types/identity.py +20 -0
  330. graphcore/types/onboarding.py +20 -0
  331. graphcore/types/query.py +116 -0
  332. graphcore/types/schedules.py +91 -0
  333. graphcore/types/strategy.py +73 -0
  334. graphcore/types/tests/__init__.py +0 -0
  335. graphcore/types/tests/test_entity_id.py +50 -0
  336. graphcore/urls.py +5 -0
  337. graphcore_kernel-0.1.0.dist-info/METADATA +94 -0
  338. graphcore_kernel-0.1.0.dist-info/RECORD +343 -0
  339. graphcore_kernel-0.1.0.dist-info/WHEEL +5 -0
  340. graphcore_kernel-0.1.0.dist-info/entry_points.txt +2 -0
  341. graphcore_kernel-0.1.0.dist-info/licenses/LICENSE +202 -0
  342. graphcore_kernel-0.1.0.dist-info/licenses/NOTICE +2 -0
  343. graphcore_kernel-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,333 @@
1
+ """The kernel's single implementation of authorization, request
2
+ bookkeeping, retries, and reporting for strategy runs
3
+ (docs/STRATEGY_EXECUTION_TDD.md, The executor)."""
4
+ from __future__ import annotations
5
+
6
+ import asyncio
7
+ import copy
8
+ import logging
9
+ import random
10
+ import time
11
+ from typing import Mapping, Optional
12
+
13
+ from graphcore.domain.entity.Entity import Entity
14
+ from graphcore.domain.entity.deployment.Deployment import Deployment
15
+ from graphcore.domain.entity.strategy_request.StrategyRequest import StrategyRequest
16
+ from graphcore.errors import (
17
+ ConcurrentModificationError, ConfigurationError, Forbidden, InvalidRequest, KernelError, NotFound,
18
+ StrategyFailed,
19
+ )
20
+ from graphcore.execution.dispatcher import Dispatcher
21
+ from graphcore.execution.requests import new_strategy_request
22
+ from graphcore.execution.run_events import InMemoryRunEvents, RunEvents, RunSubscription
23
+ from graphcore.execution.strategy_context import kernel_internal_context
24
+ from graphcore.graph.EntityService import EntityService
25
+ from graphcore.registry.registry import Registry
26
+ from graphcore.types.execution import AttemptResult, ExecutionAttempt
27
+ from graphcore.types.identity import Principal
28
+ from graphcore.types.query import AllOf, Equal, InValues
29
+ from graphcore.types.strategy import TERMINAL_STATUSES, StrategyRequestPayload, Workload
30
+
31
+ logger = logging.getLogger(__name__)
32
+
33
+
34
+ class StrategyExecutorService:
35
+ """Front door (`execute_request`), `start_attempt`, `claim_attempt`,
36
+ and `finish_attempt`. Nothing here holds a run: every step reads the
37
+ persisted request, so any kernel process can take the next one."""
38
+
39
+ def __init__(self, registry: Registry, entity_service: EntityService,
40
+ default_timeout: int = 600, max_attempts: int = 6,
41
+ run_events: Optional[RunEvents] = None, dispatch_timeout: int = 300):
42
+ self.registry = registry
43
+ self.entity_service = entity_service
44
+ self.default_timeout = default_timeout
45
+ self.max_attempts = max_attempts
46
+ self.dispatch_timeout = dispatch_timeout
47
+ self.run_events = run_events or InMemoryRunEvents()
48
+ self.dispatchers: dict[Workload, Dispatcher] = {}
49
+
50
+ def bind_dispatchers(self, dispatchers: Mapping[Workload, Dispatcher]) -> None:
51
+ """Called once by the composition root. A registered strategy whose
52
+ workload has no dispatcher fails here, not on its first request."""
53
+ missing = {
54
+ strategy_cls.workload
55
+ for classes in self.registry.strategies.get_all_strategies().values()
56
+ for strategy_cls in classes
57
+ } - set(dispatchers)
58
+ if missing:
59
+ raise ConfigurationError(
60
+ f"No dispatcher for workload(s): {', '.join(sorted(w.name for w in missing))}"
61
+ )
62
+ self.dispatchers = dict(dispatchers)
63
+
64
+ # --- the front door ---------------------------------------------------
65
+
66
+ async def execute_request(
67
+ self,
68
+ request: StrategyRequest,
69
+ principal: Principal,
70
+ wait: bool = True,
71
+ callback_channel: Optional[str] = None,
72
+ ) -> tuple[StrategyRequest, Optional[Entity]]:
73
+ """Authorize, save `pending` as attempt 1, and dispatch it: the first
74
+ attempt is authorized here, not again by `_send`. A request
75
+ refused before the save is never saved. With `wait`, return once the
76
+ run has succeeded, with the target reloaded as `principal`, or raise
77
+ the error the run failed with."""
78
+ if await self.entity_service.entity_exists_in_db(request.entity_id):
79
+ raise InvalidRequest(f"Strategy request {request.entity_id} already exists")
80
+ strategy_cls = self.registry.strategies.require(request.strategy_name)
81
+ if not request.target_entity_id and not self._runs_untargeted(strategy_cls):
82
+ raise InvalidRequest("target_entity_id is required")
83
+ await self._can_execute(strategy_cls, self._authorized_on(request), principal)
84
+
85
+ request.callback_channel = callback_channel
86
+ request.status = 'pending'
87
+ request.attempt = 1
88
+ request.attempt_deadline = time.time() + self.dispatch_timeout
89
+ with kernel_internal_context():
90
+ request = await self.entity_service.save_entity(request, principal, existing=None)
91
+
92
+ if not wait:
93
+ await self._dispatch(request, strategy_cls)
94
+ return request, None
95
+ async with self.run_events.subscribe(request.entity_id) as subscription:
96
+ await self._dispatch(request, strategy_cls)
97
+ request = await self._until_terminal(request.entity_id, subscription)
98
+
99
+ if request.status == 'failed':
100
+ raise request.failure()
101
+ target = None
102
+ if request.target_entity_id:
103
+ try:
104
+ target = await self.entity_service.get_entity(request.target_entity_id, principal)
105
+ except (NotFound, Forbidden):
106
+ target = None
107
+ return request, target
108
+
109
+ async def submit(self, payload: StrategyRequestPayload, principal: Principal, wait: bool = True,
110
+ callback_channel: Optional[str] = None):
111
+ """Build a request from `payload` and execute it."""
112
+ return await self.execute_request(
113
+ new_strategy_request(self.entity_service, payload), principal,
114
+ wait=wait, callback_channel=callback_channel,
115
+ )
116
+
117
+ def _authorized_on(self, request: StrategyRequest) -> str:
118
+ """An untargeted request is authorized on the deployment, as an
119
+ entity with no parent is at the persistence chokepoint."""
120
+ return request.target_entity_id or self.entity_service.deployment_entity_id
121
+
122
+ def _runs_untargeted(self, strategy_cls) -> bool:
123
+ allowed = self.registry.policy.allowed_strategies(Deployment)
124
+ return allowed is not None and strategy_cls in allowed
125
+
126
+ async def _can_execute(self, strategy_cls, target_entity_id: str, principal: Principal) -> None:
127
+ """Strategy in the target type's allowlist, and the principal holds
128
+ the type's read permission and the strategy's required permission on
129
+ the persisted target."""
130
+ target = await self.entity_service.repository.load(target_entity_id)
131
+ if target is None:
132
+ raise NotFound(f"Entity with ID {target_entity_id} not found")
133
+
134
+ entity_type = self.registry.entities.get_entity_class(target.entity_type_id)
135
+ if entity_type is None:
136
+ raise InvalidRequest(f"Entity type {target.entity_type_id} is not registered.")
137
+
138
+ self.registry.policy.require_listed(entity_type, strategy_cls, 'target')
139
+
140
+ read_perm = self.registry.policy.read_permission(entity_type)
141
+ perm = self.registry.policy.required_permission(entity_type, strategy_cls)
142
+ for needed in dict.fromkeys((read_perm, perm)):
143
+ if not await self.entity_service.access_adapter.allows(principal, needed, target_entity_id):
144
+ raise Forbidden(
145
+ f"Strategy {strategy_cls.__name__} is not allowed on entity {target_entity_id}: "
146
+ f"user {principal.principal_entity_id} does not have {type(needed).__name__}"
147
+ )
148
+
149
+ # --- attempts ---------------------------------------------------------
150
+
151
+ async def start_attempt(self, request: StrategyRequest) -> None:
152
+ """Dispatch the request's next attempt."""
153
+ await self._send(request, request.attempt + 1)
154
+
155
+ async def _send(self, request: StrategyRequest, attempt_number: int) -> None:
156
+ """Re-authorize the owner against current state, save `pending` with
157
+ `attempt_number` and a dispatch deadline, and dispatch by workload."""
158
+ owner = Principal(principal_entity_id=request.owner)
159
+ try:
160
+ strategy_cls = self.registry.strategies.require(request.strategy_name)
161
+ await self._can_execute(strategy_cls, self._authorized_on(request), owner)
162
+ except Exception as exc:
163
+ await self._finish(request, 'failed', error=KernelError.of(exc))
164
+ return
165
+
166
+ stored = copy.copy(request)
167
+ request.status = 'pending'
168
+ request.attempt = attempt_number
169
+ request.attempt_deadline = time.time() + self.dispatch_timeout
170
+ with kernel_internal_context():
171
+ request = await self.entity_service.save_entity(request, owner, existing=stored)
172
+ await self._dispatch(request, strategy_cls)
173
+
174
+ async def _dispatch(self, request: StrategyRequest, strategy_cls) -> None:
175
+ """Hand the request's saved `pending` attempt to its workload's environment."""
176
+ attempt = ExecutionAttempt(
177
+ request_id=request.entity_id, attempt=request.attempt,
178
+ strategy_name=request.strategy_name, target_entity_id=request.target_entity_id,
179
+ owner_id=request.owner, backend_id=self.entity_service.backend_id,
180
+ )
181
+ try:
182
+ await self.dispatchers[strategy_cls.workload].dispatch(attempt)
183
+ except Exception as exc:
184
+ logger.exception("Could not dispatch attempt %s of %s", attempt.attempt, attempt.request_id)
185
+ await self._finish(request, 'failed', error=StrategyFailed(f"dispatch failed: {exc}"))
186
+
187
+ async def claim_attempt(self, attempt: ExecutionAttempt) -> bool:
188
+ """Called by an entry point before it runs `attempt`: save `running`
189
+ with the strategy's deadline, counted from now. False when the
190
+ request does not hold this attempt as `pending`, including when
191
+ another delivery of it claimed it first; the entry point then runs
192
+ nothing."""
193
+ request = await self.entity_service.repository.load(attempt.request_id)
194
+ if request is None or request.status != 'pending' or request.attempt != attempt.attempt:
195
+ return False
196
+ strategy_cls = self.registry.strategies.by_name(request.strategy_name)
197
+ stored = copy.copy(request)
198
+ request.status = 'running'
199
+ request.attempt_deadline = time.time() + (
200
+ getattr(strategy_cls, 'timeout', None) or self.default_timeout
201
+ )
202
+ try:
203
+ with kernel_internal_context():
204
+ await self.entity_service.save_entity(
205
+ request, Principal(principal_entity_id=request.owner), existing=stored,
206
+ )
207
+ except ConcurrentModificationError:
208
+ return False
209
+ return True
210
+
211
+ async def finish_attempt(self, result: AttemptResult) -> None:
212
+ """Apply one attempt's result. A result for an attempt the request no
213
+ longer holds, or a repeat delivery, has no effect."""
214
+ request = await self.entity_service.repository.load(result.request_id)
215
+ if request is None or request.status != 'running' or request.attempt != result.attempt:
216
+ logger.info("Discarding stale result for attempt %s of %s", result.attempt, result.request_id)
217
+ return
218
+
219
+ if result.outcome == 'succeeded':
220
+ await self._finish(request, 'succeeded', ret_val=result.ret_val or {})
221
+ elif result.outcome == 'conflict' and request.attempt < self.max_attempts:
222
+ await self._backoff(request.attempt)
223
+ await self.start_attempt(request)
224
+ else:
225
+ await self._finish(request, 'failed', error=KernelError.from_dict(result.error or {}))
226
+
227
+ async def _backoff(self, attempt: int) -> None:
228
+ await asyncio.sleep(0.1 * 2 ** attempt + random.uniform(0, 0.1))
229
+
230
+ async def reconcile_runs(self) -> int:
231
+ """Reconcile every unfinished request past its deadline. Returns how
232
+ many it acted on."""
233
+ repository = self.entity_service.repository
234
+ unfinished = await repository.list_ids(AllOf((
235
+ Equal('entity_type_id', StrategyRequest.get_entity_type_id()),
236
+ InValues('status', frozenset(UNFINISHED_STATUSES)),
237
+ )))
238
+ acted = 0
239
+ for request in (await repository.load_many(unfinished)).values():
240
+ if _overdue(request):
241
+ acted += await self._reconcile(request)
242
+ return acted
243
+
244
+ async def _reconcile(self, request: StrategyRequest) -> int:
245
+ """Send an unclaimed attempt again, as the same attempt; start the
246
+ next attempt of an overdue running one, or fail the run once out of
247
+ attempts. 0 when another writer changed the request first."""
248
+ try:
249
+ if request.status == 'pending':
250
+ await self._send(request, request.attempt)
251
+ elif request.attempt < self.max_attempts:
252
+ await self.start_attempt(request)
253
+ else:
254
+ await self._finish(request, 'failed', error=StrategyFailed('attempt timed out'))
255
+ return 1
256
+ except ConcurrentModificationError:
257
+ logger.info("Request %s changed while reconciling; left to its writer", request.entity_id)
258
+ return 0
259
+
260
+ async def _finish(self, request: StrategyRequest, status: str, ret_val: Optional[dict] = None,
261
+ error: Optional[KernelError] = None) -> None:
262
+ owner = Principal(principal_entity_id=request.owner)
263
+ stored = copy.copy(request)
264
+ request.status = status
265
+ request.ret_val = ret_val or {}
266
+ request.error = error.message if error else None
267
+ request.error_code = error.code if error else None
268
+ request.error_details = dict(error.details) if error else {}
269
+ request.attempt_deadline = None
270
+ with kernel_internal_context():
271
+ request = await self.entity_service.save_entity(request, owner, existing=stored)
272
+ await self.run_events.publish(request.entity_id)
273
+
274
+ if status == 'succeeded':
275
+ for callback in request.callbacks or []:
276
+ await self._submit_callback(request, callback, owner)
277
+ if request.callback_channel:
278
+ await self._report(request)
279
+
280
+ async def _submit_callback(self, request: StrategyRequest, callback: dict, owner: Principal) -> None:
281
+ try:
282
+ payload = StrategyRequestPayload.from_json(callback)
283
+ payload.param_config = {**payload.param_config, 'source_ret_val': request.ret_val}
284
+ payload.client_request_id = None
285
+ await self.submit(payload, owner, wait=False)
286
+ except Exception:
287
+ logger.exception("Callback %r of request %s was not submitted", callback, request.entity_id)
288
+
289
+ async def _report(self, request: StrategyRequest) -> None:
290
+ """One `strategy_complete` shape for every environment."""
291
+ message = {
292
+ 'type': 'strategy_complete',
293
+ 'request_id': request.entity_id,
294
+ 'target_entity_id': request.target_entity_id,
295
+ 'client_request_id': request.client_request_id,
296
+ 'status': request.status,
297
+ 'success': request.status == 'succeeded',
298
+ }
299
+ if request.status == 'succeeded':
300
+ message['ret_val'] = request.ret_val
301
+ else:
302
+ message['error'] = request.error
303
+ message['code'] = request.error_code
304
+ message['details'] = request.error_details or {}
305
+ try:
306
+ await self.entity_service.broadcaster.send(request.callback_channel, message)
307
+ except Exception:
308
+ logger.exception("strategy_complete for %s was not delivered", request.entity_id)
309
+
310
+ # --- waiting ----------------------------------------------------------
311
+
312
+ async def _until_terminal(self, request_id: str, subscription: RunSubscription) -> StrategyRequest:
313
+ """Read the request whenever its run's event arrives or its attempt's
314
+ deadline passes. A waiter past the deadline reconciles the run
315
+ itself, so it never depends on the periodic reconciler."""
316
+ while True:
317
+ request = await self.entity_service.repository.load(request_id)
318
+ if request is None:
319
+ raise NotFound(f"Strategy request {request_id} no longer exists")
320
+ if request.status in TERMINAL_STATUSES:
321
+ return request
322
+ if _overdue(request):
323
+ await self._reconcile(request)
324
+ continue
325
+ await subscription.wait(until=request.attempt_deadline)
326
+
327
+
328
+ UNFINISHED_STATUSES = ('pending', 'running')
329
+
330
+
331
+ def _overdue(request: StrategyRequest) -> bool:
332
+ return (request.status in UNFINISHED_STATUSES and request.attempt_deadline is not None
333
+ and request.attempt_deadline <= time.time())
File without changes
@@ -0,0 +1,19 @@
1
+ """The port that hands one attempt to an execution environment
2
+ (docs/STRATEGY_EXECUTION_TDD.md, The dispatcher port)."""
3
+ from __future__ import annotations
4
+
5
+ from abc import ABC, abstractmethod
6
+
7
+ from graphcore.types.execution import ExecutionAttempt
8
+
9
+
10
+ class Dispatcher(ABC):
11
+ @abstractmethod
12
+ async def dispatch(self, attempt: ExecutionAttempt) -> None:
13
+ """Deliver the attempt to the environment and return once it is
14
+ accepted. Raises only if it could not be delivered. The result
15
+ reaches `StrategyExecutorService.finish_attempt`, never this call."""
16
+
17
+ async def drain(self) -> None:
18
+ """Wait for attempts this process is running on the current event
19
+ loop. An entry point that owns its loop calls it before returning."""
File without changes
@@ -0,0 +1,59 @@
1
+ """A Celery process holding a runtime (docs/STRATEGY_EXECUTION_TDD.md,
2
+ Celery). The dispatcher enqueues the attempt's `to_dict()` form; the
3
+ `run_attempt` task is the entry point."""
4
+ from __future__ import annotations
5
+
6
+ import asyncio
7
+ import threading
8
+
9
+ from asgiref.sync import async_to_sync
10
+ from celery import shared_task
11
+ from celery.signals import worker_init
12
+
13
+ from graphcore.domain.strategy.runner import ChangeTracker, StrategyRunner
14
+ from graphcore.errors import KernelError
15
+ from graphcore.execution.dispatcher import Dispatcher
16
+ from graphcore.execution.local_session import LocalStrategySession
17
+ from graphcore.runtime import get_runtime, serve_as_worker
18
+ from graphcore.types.execution import AttemptResult, ExecutionAttempt
19
+
20
+
21
+ @worker_init.connect
22
+ def _serve_as_worker(**kwargs) -> None:
23
+ # A worker on a thread shares its process with a web server (`graphcore run --debug`).
24
+ if threading.current_thread() is threading.main_thread():
25
+ serve_as_worker()
26
+
27
+
28
+ class CeleryDispatcher(Dispatcher):
29
+ async def dispatch(self, attempt: ExecutionAttempt) -> None:
30
+ # A plain executor thread, not sync_to_async: in eager mode the task
31
+ # then runs on its own loop, as in a worker, not on the caller's.
32
+ await asyncio.get_running_loop().run_in_executor(None, run_attempt.delay, attempt.to_dict())
33
+
34
+
35
+ async def settle(executor) -> None:
36
+ """Let in-process attempts this loop started finish before the task's
37
+ loop closes, e.g. an IO child submitted by a CPU strategy."""
38
+ for dispatcher in executor.dispatchers.values():
39
+ await dispatcher.drain()
40
+
41
+
42
+ @shared_task(name='graphcore.tasks.run_attempt')
43
+ def run_attempt(data: dict) -> None:
44
+ rt = get_runtime()
45
+ attempt = ExecutionAttempt.from_dict(data)
46
+
47
+ async def run():
48
+ if not await rt.executor.claim_attempt(attempt):
49
+ return
50
+ tracker = ChangeTracker()
51
+ session = LocalStrategySession(attempt, tracker, rt.entity_service, rt.executor, rt.registry)
52
+ try:
53
+ result = await StrategyRunner(rt.registry).run(attempt, session, tracker)
54
+ except Exception as exc:
55
+ result = AttemptResult(attempt.request_id, attempt.attempt, 'failed', error=KernelError.of(exc).to_dict())
56
+ await rt.executor.finish_attempt(result)
57
+ await settle(rt.executor)
58
+
59
+ async_to_sync(run)()
@@ -0,0 +1,55 @@
1
+ """The submitting process's own event loop: the dispatcher is also the
2
+ entry point (docs/STRATEGY_EXECUTION_TDD.md, In-process)."""
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import logging
7
+
8
+ from graphcore.domain.strategy.runner import ChangeTracker, StrategyRunner
9
+ from graphcore.errors import KernelError
10
+ from graphcore.execution.dispatcher import Dispatcher
11
+ from graphcore.execution.local_session import LocalStrategySession
12
+ from graphcore.types.execution import AttemptResult, ExecutionAttempt
13
+
14
+ logger = logging.getLogger(__name__)
15
+
16
+
17
+ class InProcessDispatcher(Dispatcher):
18
+ def __init__(self, executor, entity_service, registry):
19
+ self.executor = executor
20
+ self.entity_service = entity_service
21
+ self.registry = registry
22
+ self._tasks: set[asyncio.Task] = set()
23
+
24
+ async def dispatch(self, attempt: ExecutionAttempt) -> None:
25
+ task = asyncio.create_task(self._run(attempt))
26
+ self._tasks.add(task)
27
+ task.add_done_callback(self._tasks.discard)
28
+
29
+ async def _run(self, attempt: ExecutionAttempt) -> None:
30
+ self.entity_service.repository.use_shared_threads()
31
+ try:
32
+ if not await self.executor.claim_attempt(attempt):
33
+ return
34
+ except Exception:
35
+ logger.exception("Could not claim attempt %s of %s", attempt.attempt, attempt.request_id)
36
+ return
37
+ try:
38
+ tracker = ChangeTracker()
39
+ session = LocalStrategySession(attempt, tracker, self.entity_service, self.executor, self.registry)
40
+ result = await StrategyRunner(self.registry).run(attempt, session, tracker)
41
+ except Exception as exc:
42
+ logger.exception("In-process attempt %s of %s failed", attempt.attempt, attempt.request_id)
43
+ result = AttemptResult(attempt.request_id, attempt.attempt, 'failed', error=KernelError.of(exc).to_dict())
44
+ try:
45
+ await self.executor.finish_attempt(result)
46
+ except Exception:
47
+ logger.exception("Could not finish attempt %s of %s", attempt.attempt, attempt.request_id)
48
+
49
+ async def drain(self) -> None:
50
+ loop = asyncio.get_running_loop()
51
+ while True:
52
+ pending = [t for t in self._tasks if not t.done() and t.get_loop() is loop]
53
+ if not pending:
54
+ return
55
+ await asyncio.gather(*pending, return_exceptions=True)
@@ -0,0 +1,117 @@
1
+ """`StrategySession` over a runtime's `EntityService` and the executor's
2
+ front door (docs/STRATEGY_EXECUTION_TDD.md, The session adapters)."""
3
+ from __future__ import annotations
4
+
5
+ from typing import Optional
6
+
7
+ from graphcore.domain.entity.strategy_request.StrategyRequest import StrategyRequest
8
+ from graphcore.domain.strategy.runner import ChangeTracker
9
+ from graphcore.domain.strategy.session import StrategySession
10
+ from graphcore.errors import PreconditionFailed, StaleAttempt
11
+ from graphcore.execution.requests import new_strategy_request
12
+ from graphcore.execution.strategy_context import strategy_context
13
+ from graphcore.persistence.repository import Precondition
14
+ from graphcore.types.entities import EntityFilter, Tombstone
15
+ from graphcore.types.entity_id import derived_entity_id
16
+ from graphcore.types.execution import ExecutionAttempt
17
+ from graphcore.types.identity import Principal
18
+ from graphcore.types.query import AllOf, Equal
19
+ from graphcore.types.strategy import StrategyRequestPayload
20
+
21
+
22
+ class LocalStrategySession(StrategySession):
23
+ """Reads and writes as the attempt's owner. Every write is fenced and
24
+ runs inside the strategy's context, so the persistence chokepoint
25
+ authorizes it against the strategy's class."""
26
+
27
+ def __init__(self, attempt: ExecutionAttempt, tracker: ChangeTracker, entity_service,
28
+ executor, registry):
29
+ self.attempt = attempt
30
+ self.tracker = tracker
31
+ self.entity_service = entity_service
32
+ self.executor = executor
33
+ self.registry = registry
34
+ self.principal = Principal(principal_entity_id=attempt.owner_id)
35
+ self._strategy_cls = None
36
+ self._created = 0
37
+ self._unsaved_creations: set[str] = set()
38
+
39
+ def _strategy(self):
40
+ if self._strategy_cls is None:
41
+ self._strategy_cls = self.registry.strategies.require(self.attempt.strategy_name)
42
+ return self._strategy_cls
43
+
44
+ def _fence(self) -> Precondition:
45
+ return Precondition(self.attempt.request_id, AllOf((
46
+ Equal('entity_type_id', StrategyRequest.get_entity_type_id()),
47
+ Equal('status', 'running'),
48
+ Equal('attempt', self.attempt.attempt),
49
+ )))
50
+
51
+ def _stale(self) -> StaleAttempt:
52
+ return StaleAttempt(
53
+ f"Attempt {self.attempt.attempt} of request {self.attempt.request_id} is no longer current"
54
+ )
55
+
56
+ def create(self, entity_cls, **fields):
57
+ entity_id = derived_entity_id(self.attempt.backend_id, f"{self.attempt.request_id}/{self._created}")
58
+ self._created += 1
59
+ self._unsaved_creations.add(entity_id)
60
+ entity = entity_cls(entity_id=entity_id)
61
+ parent_id = fields.pop('parent_id', None)
62
+ if parent_id is not None:
63
+ entity.set_parent(parent_id)
64
+ for name, value in fields.items():
65
+ setattr(entity, name, value)
66
+ return entity
67
+
68
+ async def get(self, entity_id: str):
69
+ entity = await self.entity_service.get_entity(entity_id, self.principal)
70
+ self.tracker.record(entity)
71
+ return entity
72
+
73
+ async def get_many(self, entity_ids: list[str]) -> dict:
74
+ found = await self.entity_service.get_entities(entity_ids, self.principal)
75
+ entities = {eid: e for eid, e in found.items() if not isinstance(e, Tombstone)}
76
+ for entity in entities.values():
77
+ self.tracker.record(entity)
78
+ return entities
79
+
80
+ async def list(self, entity_filter: EntityFilter, cursor: Optional[str] = None, limit: int = 25):
81
+ page = await self.entity_service.list_entities(self.principal, entity_filter, cursor=cursor, limit=limit)
82
+ for entity in page.entities:
83
+ self.tracker.record(entity)
84
+ return page
85
+
86
+ async def save(self, entity):
87
+ if entity.entity_id in self._unsaved_creations and entity.version == 0:
88
+ await self._replace_earlier_attempts(entity)
89
+ try:
90
+ with strategy_context(self._strategy()):
91
+ saved = await self.entity_service.save_entity(entity, self.principal, precondition=self._fence())
92
+ except PreconditionFailed as exc:
93
+ raise self._stale() from exc
94
+ self._unsaved_creations.discard(entity.entity_id)
95
+ self.tracker.record(saved)
96
+ return saved
97
+
98
+ async def _replace_earlier_attempts(self, entity) -> None:
99
+ """An earlier attempt of this request may have saved the entity
100
+ `create` minted again; this attempt's first save replaces it."""
101
+ stored = await self.entity_service.repository.load(entity.entity_id)
102
+ if stored is not None and type(stored) is type(entity):
103
+ entity.version = stored.version
104
+
105
+ async def delete_tree(self, root_id: str) -> list[str]:
106
+ try:
107
+ with strategy_context(self._strategy()):
108
+ return await self.entity_service.delete_entity_tree(root_id, self.principal, precondition=self._fence())
109
+ except PreconditionFailed as exc:
110
+ raise self._stale() from exc
111
+
112
+ async def execute(self, payload: StrategyRequestPayload, *, wait: bool = True,
113
+ callback_channel: Optional[str] = None):
114
+ request = new_strategy_request(self.entity_service, payload)
115
+ return await self.executor.execute_request(
116
+ request, self.principal, wait=wait, callback_channel=callback_channel,
117
+ )
@@ -0,0 +1,74 @@
1
+ """`RunEvents` across processes over Redis pub/sub
2
+ (docs/STRATEGY_EXECUTION_TDD.md, Waiting)."""
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import logging
7
+ import os
8
+ import threading
9
+ import time
10
+
11
+ from graphcore.execution.run_events import RunEvents
12
+
13
+ logger = logging.getLogger(__name__)
14
+
15
+ CHANNEL = 'graphcore.run_finished'
16
+
17
+
18
+ class RedisRunEvents(RunEvents):
19
+ """Each process publishes finished request ids on one channel and runs
20
+ one listener thread, started on its first subscription, that wakes the
21
+ process's subscribers. Whenever the listener (re)subscribes, it wakes
22
+ every subscriber so each re-reads its request: nothing published while
23
+ it was not listening is relied on."""
24
+
25
+ def __init__(self, url: str, channel: str = CHANNEL, reconnect_delay: float = 1.0):
26
+ super().__init__()
27
+ self.url = url
28
+ self.channel = channel
29
+ self.reconnect_delay = reconnect_delay
30
+ self._pid = None
31
+ self._client = None
32
+ self._listener = None
33
+
34
+ def _redis(self):
35
+ import redis
36
+
37
+ # Per process: a forked worker must not share its parent's sockets.
38
+ if self._pid != os.getpid():
39
+ self._pid = os.getpid()
40
+ self._client = redis.Redis.from_url(self.url, decode_responses=True)
41
+ self._listener = None
42
+ return self._client
43
+
44
+ def _subscribed(self) -> None:
45
+ client = self._redis()
46
+ with self._lock:
47
+ if self._listener is None or not self._listener.is_alive():
48
+ self._listener = threading.Thread(
49
+ target=self._listen, args=(client,), name='graphcore-run-events', daemon=True,
50
+ )
51
+ self._listener.start()
52
+
53
+ def _listen(self, client) -> None:
54
+ while True:
55
+ try:
56
+ pubsub = client.pubsub()
57
+ pubsub.subscribe(self.channel)
58
+ for message in pubsub.listen():
59
+ if message['type'] == 'subscribe':
60
+ self.notify_local()
61
+ elif message['type'] == 'message':
62
+ self.notify_local(message['data'])
63
+ except Exception:
64
+ logger.warning("Run events listener lost %s; reconnecting", self.channel, exc_info=True)
65
+ time.sleep(self.reconnect_delay)
66
+
67
+ async def publish(self, request_id: str) -> None:
68
+ self.notify_local(request_id)
69
+ try:
70
+ await asyncio.get_running_loop().run_in_executor(
71
+ None, self._redis().publish, self.channel, request_id,
72
+ )
73
+ except Exception:
74
+ logger.exception("Run event for %s was not published", request_id)