graphcore-kernel 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (343) hide show
  1. graphcore/__init__.py +5 -0
  2. graphcore/agent_frontend/__init__.py +8 -0
  3. graphcore/agent_frontend/api.py +69 -0
  4. graphcore/agent_frontend/base/__init__.py +11 -0
  5. graphcore/agent_frontend/base/views.py +167 -0
  6. graphcore/agent_frontend/client.py +31 -0
  7. graphcore/agent_frontend/context.py +157 -0
  8. graphcore/agent_frontend/errors.py +12 -0
  9. graphcore/agent_frontend/loading.py +54 -0
  10. graphcore/agent_frontend/registry.py +42 -0
  11. graphcore/agent_frontend/render.py +207 -0
  12. graphcore/agent_frontend/server.py +103 -0
  13. graphcore/agent_frontend/signatures.py +35 -0
  14. graphcore/agent_frontend/testing.py +45 -0
  15. graphcore/agent_frontend/tests/__init__.py +0 -0
  16. graphcore/agent_frontend/tests/fakes.py +114 -0
  17. graphcore/agent_frontend/tests/test_frontend.py +447 -0
  18. graphcore/agent_frontend/tests/test_loading.py +83 -0
  19. graphcore/agent_frontend/tests/test_server.py +109 -0
  20. graphcore/agent_frontend/tools.py +137 -0
  21. graphcore/api/__init__.py +4 -0
  22. graphcore/api/graph_api.py +378 -0
  23. graphcore/api/predicate_codec.py +81 -0
  24. graphcore/api/tests/__init__.py +0 -0
  25. graphcore/api/tests/test_changed_entities.py +109 -0
  26. graphcore/api/tests/test_graph_api.py +290 -0
  27. graphcore/api/tests/test_graph_api_access.py +94 -0
  28. graphcore/api/tests/test_graph_api_grants.py +90 -0
  29. graphcore/api/tests/test_predicate_codec.py +83 -0
  30. graphcore/api/tests/test_subtree_query.py +79 -0
  31. graphcore/apps.py +71 -0
  32. graphcore/auth/__init__.py +1 -0
  33. graphcore/auth/authn/README.md +178 -0
  34. graphcore/auth/authn/__init__.py +3 -0
  35. graphcore/auth/authn/authenticator.py +120 -0
  36. graphcore/auth/authn/onboarding.py +105 -0
  37. graphcore/auth/authn/principals.py +20 -0
  38. graphcore/auth/authn/resource_metadata.py +43 -0
  39. graphcore/auth/authn/service_accounts.py +96 -0
  40. graphcore/auth/authn/testing.py +96 -0
  41. graphcore/auth/authn/tests/__init__.py +0 -0
  42. graphcore/auth/authn/tests/test_authenticator.py +145 -0
  43. graphcore/auth/authn/tests/test_onboarding.py +145 -0
  44. graphcore/auth/authn/tests/test_service_accounts.py +180 -0
  45. graphcore/auth/authn/users.py +59 -0
  46. graphcore/auth/authz/__init__.py +0 -0
  47. graphcore/auth/authz/access_port.py +120 -0
  48. graphcore/auth/authz/adapter/__init__.py +0 -0
  49. graphcore/auth/authz/adapter/default_access_adapter.py +556 -0
  50. graphcore/auth/authz/adapter/tests/__init__.py +0 -0
  51. graphcore/auth/authz/adapter/tests/test_default_adapter.py +543 -0
  52. graphcore/auth/authz/adapter/tests/test_entity_write_hooks.py +207 -0
  53. graphcore/auth/authz/adapter/tests/test_group_types.py +70 -0
  54. graphcore/auth/authz/adapter/tests/test_sync_support.py +209 -0
  55. graphcore/auth/authz/defaults.py +26 -0
  56. graphcore/auth/authz/kernel_policy.py +191 -0
  57. graphcore/auth/authz/kernel_writes.py +31 -0
  58. graphcore/auth/authz/lookups.py +25 -0
  59. graphcore/auth/authz/policy.py +403 -0
  60. graphcore/auth/authz/tests/__init__.py +0 -0
  61. graphcore/auth/authz/tests/test_can_contain.py +150 -0
  62. graphcore/auth/authz/tests/test_contained_by.py +167 -0
  63. graphcore/auth/authz/tests/test_creation.py +391 -0
  64. graphcore/auth/authz/tests/test_delete_permission.py +82 -0
  65. graphcore/auth/authz/tests/test_granted_to_me.py +160 -0
  66. graphcore/auth/authz/tests/test_integration.py +637 -0
  67. graphcore/auth/authz/tests/test_invites.py +184 -0
  68. graphcore/auth/authz/tests/test_kernel_policy.py +140 -0
  69. graphcore/auth/authz/tests/test_lockout.py +132 -0
  70. graphcore/auth/authz/tests/test_policy.py +329 -0
  71. graphcore/auth/authz/tests/test_primitives.py +73 -0
  72. graphcore/auth/authz/tests/test_principal_creation.py +62 -0
  73. graphcore/bootstrap/__init__.py +0 -0
  74. graphcore/bootstrap/deployment.py +67 -0
  75. graphcore/bootstrap/signals.py +16 -0
  76. graphcore/bootstrap/tests/__init__.py +0 -0
  77. graphcore/bootstrap/tests/test_deployment_seed.py +154 -0
  78. graphcore/broadcast/__init__.py +4 -0
  79. graphcore/broadcast/broadcaster.py +201 -0
  80. graphcore/broadcast/channels.py +56 -0
  81. graphcore/broadcast/tests/__init__.py +0 -0
  82. graphcore/broadcast/tests/test_broadcaster.py +444 -0
  83. graphcore/celery.py +14 -0
  84. graphcore/docs/AGENT_FRONTEND_TDD.md +389 -0
  85. graphcore/docs/ARCHITECTURE_PRINCIPLES.md +152 -0
  86. graphcore/docs/AUTHORIZATION_TECHNICAL_TDD.md +1379 -0
  87. graphcore/docs/ENTITY_REFERENCE_TDD.md +561 -0
  88. graphcore/docs/GRAPHAPI_FRONT_DOOR.md +173 -0
  89. graphcore/docs/RUNTIME.md +90 -0
  90. graphcore/docs/SCHEDULING_TDD.md +244 -0
  91. graphcore/docs/STRATEGY_EXECUTION_TDD.md +1043 -0
  92. graphcore/docs/VIEW_ENTITY_TDD.md +522 -0
  93. graphcore/docs/VIEW_SYSTEM_TDD.md +307 -0
  94. graphcore/docs/__init__.py +0 -0
  95. graphcore/docs/doc_topics.py +97 -0
  96. graphcore/docs/docs_discovery.py +64 -0
  97. graphcore/docs/public_api.py +56 -0
  98. graphcore/domain/__init__.py +0 -0
  99. graphcore/domain/entity/Entity.py +204 -0
  100. graphcore/domain/entity/__init__.py +0 -0
  101. graphcore/domain/entity/deployment/Deployment.py +25 -0
  102. graphcore/domain/entity/deployment/__init__.py +0 -0
  103. graphcore/domain/entity/grant/Grant.py +31 -0
  104. graphcore/domain/entity/grant/__init__.py +0 -0
  105. graphcore/domain/entity/group/Group.py +48 -0
  106. graphcore/domain/entity/group/README.md +88 -0
  107. graphcore/domain/entity/group/__init__.py +0 -0
  108. graphcore/domain/entity/invite/Invite.py +31 -0
  109. graphcore/domain/entity/invite/__init__.py +0 -0
  110. graphcore/domain/entity/request_template/RequestTemplate.py +48 -0
  111. graphcore/domain/entity/request_template/__init__.py +0 -0
  112. graphcore/domain/entity/request_template/tests/__init__.py +0 -0
  113. graphcore/domain/entity/request_template/tests/test_request_template.py +52 -0
  114. graphcore/domain/entity/service_account/ServiceAccount.py +41 -0
  115. graphcore/domain/entity/service_account/__init__.py +0 -0
  116. graphcore/domain/entity/strategy_request/StrategyRequest.py +61 -0
  117. graphcore/domain/entity/strategy_request/__init__.py +0 -0
  118. graphcore/domain/entity/strategy_request/tests/__init__.py +0 -0
  119. graphcore/domain/entity/strategy_request/tests/test_strategy_request_entity.py +35 -0
  120. graphcore/domain/entity/tests/__init__.py +0 -0
  121. graphcore/domain/entity/tests/test_entity.py +227 -0
  122. graphcore/domain/entity/tests/test_grant_group_entities.py +36 -0
  123. graphcore/domain/entity/tests/test_write_once_fields.py +98 -0
  124. graphcore/domain/entity/user/User.py +39 -0
  125. graphcore/domain/entity/user/__init__.py +0 -0
  126. graphcore/domain/entity/user/tests/__init__.py +0 -0
  127. graphcore/domain/entity/user/tests/test_user_entity.py +33 -0
  128. graphcore/domain/entity/view/View.py +37 -0
  129. graphcore/domain/entity/view/__init__.py +0 -0
  130. graphcore/domain/entity/view/position.py +78 -0
  131. graphcore/domain/entity/view/tests/__init__.py +0 -0
  132. graphcore/domain/entity/view/tests/test_position.py +71 -0
  133. graphcore/domain/entity/view/tests/test_view_entity.py +134 -0
  134. graphcore/domain/strategy/BaseStrategy.py +759 -0
  135. graphcore/domain/strategy/DeploymentStrategies.py +55 -0
  136. graphcore/domain/strategy/GrantStrategies.py +189 -0
  137. graphcore/domain/strategy/GroupStrategies.py +193 -0
  138. graphcore/domain/strategy/InviteStrategies.py +84 -0
  139. graphcore/domain/strategy/RequestTemplateStrategies.py +126 -0
  140. graphcore/domain/strategy/ViewStrategies.py +219 -0
  141. graphcore/domain/strategy/__init__.py +0 -0
  142. graphcore/domain/strategy/runner.py +84 -0
  143. graphcore/domain/strategy/session.py +56 -0
  144. graphcore/domain/strategy/tests/BaseStrategyTest.py +167 -0
  145. graphcore/domain/strategy/tests/__init__.py +0 -0
  146. graphcore/domain/strategy/tests/test_BaseStrategies.py +870 -0
  147. graphcore/domain/strategy/tests/test_grant_strategies.py +245 -0
  148. graphcore/domain/strategy/tests/test_group_strategies.py +99 -0
  149. graphcore/domain/strategy/tests/test_runner.py +220 -0
  150. graphcore/domain/strategy/tests/test_strategy_locks.py +446 -0
  151. graphcore/domain/strategy/tests/test_utils.py +95 -0
  152. graphcore/domain/strategy/tests/test_view_strategies.py +436 -0
  153. graphcore/errors.py +145 -0
  154. graphcore/execution/README.md +97 -0
  155. graphcore/execution/StrategyExecutorService.py +333 -0
  156. graphcore/execution/__init__.py +0 -0
  157. graphcore/execution/dispatcher.py +19 -0
  158. graphcore/execution/environments/__init__.py +0 -0
  159. graphcore/execution/environments/celery.py +59 -0
  160. graphcore/execution/environments/in_process.py +55 -0
  161. graphcore/execution/local_session.py +117 -0
  162. graphcore/execution/redis_run_events.py +74 -0
  163. graphcore/execution/requests.py +22 -0
  164. graphcore/execution/run_events.py +85 -0
  165. graphcore/execution/strategy_context.py +54 -0
  166. graphcore/execution/tasks.py +34 -0
  167. graphcore/execution/tests/__init__.py +5 -0
  168. graphcore/execution/tests/contract.py +171 -0
  169. graphcore/execution/tests/support.py +310 -0
  170. graphcore/execution/tests/test_dispatchers.py +76 -0
  171. graphcore/execution/tests/test_executor.py +368 -0
  172. graphcore/execution/tests/test_local_session.py +95 -0
  173. graphcore/execution/tests/test_reconcile_runs.py +134 -0
  174. graphcore/execution/tests/test_run_events.py +106 -0
  175. graphcore/execution/tests/test_strategy_context.py +62 -0
  176. graphcore/gateway/__init__.py +1 -0
  177. graphcore/gateway/remote.py +35 -0
  178. graphcore/gateway/tests/__init__.py +0 -0
  179. graphcore/gateway/tests/test_remote.py +30 -0
  180. graphcore/graph/EntityService.py +710 -0
  181. graphcore/graph/__init__.py +0 -0
  182. graphcore/graph/entity_cache.py +152 -0
  183. graphcore/graph/tests/__init__.py +4 -0
  184. graphcore/graph/tests/test_EntityService.py +908 -0
  185. graphcore/graph/tests/test_EntityService_list.py +170 -0
  186. graphcore/graph/tests/test_EntityService_model_resolution.py +339 -0
  187. graphcore/graph/tests/test_EntityService_referrers.py +236 -0
  188. graphcore/graph/tests/test_EntityService_routing.py +119 -0
  189. graphcore/graph/tests/test_dependent_on.py +141 -0
  190. graphcore/graph/tests/test_engine_is_type_agnostic.py +25 -0
  191. graphcore/graph/tests/test_engine_resolves_read_permission.py +93 -0
  192. graphcore/graph/tests/test_entity_cache.py +159 -0
  193. graphcore/graph/tests/test_null_entity_cache.py +12 -0
  194. graphcore/management/__init__.py +0 -0
  195. graphcore/management/commands/__init__.py +0 -0
  196. graphcore/management/commands/dev_reset.py +18 -0
  197. graphcore/management/commands/graphcore_createadmin.py +65 -0
  198. graphcore/management/commands/graphcore_policy.py +59 -0
  199. graphcore/management/commands/graphcore_status.py +98 -0
  200. graphcore/management/tests/__init__.py +0 -0
  201. graphcore/management/tests/test_createadmin.py +78 -0
  202. graphcore/management/tests/test_policy_command.py +46 -0
  203. graphcore/management/tests/test_status_command.py +85 -0
  204. graphcore/migrations/0001_initial.py +186 -0
  205. graphcore/migrations/0002_invite.py +26 -0
  206. graphcore/migrations/__init__.py +0 -0
  207. graphcore/models.py +11 -0
  208. graphcore/persistence/ENTITY_REPOSITORY_TECHNICAL_TDD.md +396 -0
  209. graphcore/persistence/__init__.py +6 -0
  210. graphcore/persistence/django/__init__.py +0 -0
  211. graphcore/persistence/django/backends.py +43 -0
  212. graphcore/persistence/django/bindings.py +28 -0
  213. graphcore/persistence/django/cache.py +38 -0
  214. graphcore/persistence/django/compiler.py +214 -0
  215. graphcore/persistence/django/identity.py +18 -0
  216. graphcore/persistence/django/models.py +252 -0
  217. graphcore/persistence/django/repository.py +525 -0
  218. graphcore/persistence/django/tests/__init__.py +0 -0
  219. graphcore/persistence/django/tests/test_backends.py +68 -0
  220. graphcore/persistence/django/tests/test_compiler.py +269 -0
  221. graphcore/persistence/django/tests/test_repository.py +472 -0
  222. graphcore/persistence/django/tests/test_shared_column.py +120 -0
  223. graphcore/persistence/django/tests/test_threads.py +50 -0
  224. graphcore/persistence/django/tests/test_update_shape.py +175 -0
  225. graphcore/persistence/django/threads.py +38 -0
  226. graphcore/persistence/repository.py +172 -0
  227. graphcore/persistence/tests/__init__.py +0 -0
  228. graphcore/persistence/tests/contract.py +688 -0
  229. graphcore/persistence/tests/evaluate.py +51 -0
  230. graphcore/persistence/tests/fixtures.py +32 -0
  231. graphcore/persistence/tests/memory.py +263 -0
  232. graphcore/persistence/tests/test_memory_repository.py +10 -0
  233. graphcore/persistence/tests/test_query_types.py +127 -0
  234. graphcore/public/__init__.py +9 -0
  235. graphcore/public/api.py +9 -0
  236. graphcore/public/apps.py +10 -0
  237. graphcore/public/auth/__init__.py +1 -0
  238. graphcore/public/auth/grants.py +7 -0
  239. graphcore/public/auth/onboarding.py +8 -0
  240. graphcore/public/auth/perms.py +7 -0
  241. graphcore/public/auth/policy.py +10 -0
  242. graphcore/public/auth/roles.py +8 -0
  243. graphcore/public/auth/subjects.py +7 -0
  244. graphcore/public/entities/__init__.py +32 -0
  245. graphcore/public/errors.py +19 -0
  246. graphcore/public/persistence/__init__.py +44 -0
  247. graphcore/public/persistence/django.py +24 -0
  248. graphcore/public/scheduling.py +23 -0
  249. graphcore/public/strategies/__init__.py +85 -0
  250. graphcore/public/test_settings.py +16 -0
  251. graphcore/public/testing.py +47 -0
  252. graphcore/registry/EntityDirectory.py +246 -0
  253. graphcore/registry/README.md +147 -0
  254. graphcore/registry/StrategyDirectory.py +308 -0
  255. graphcore/registry/TYPE_METADATA_PLACEMENT.md +66 -0
  256. graphcore/registry/__init__.py +0 -0
  257. graphcore/registry/app_discovery.py +108 -0
  258. graphcore/registry/decorators.py +77 -0
  259. graphcore/registry/discovery.py +111 -0
  260. graphcore/registry/registry.py +79 -0
  261. graphcore/registry/tests/__init__.py +0 -0
  262. graphcore/registry/tests/test_agent_frontend_discovery.py +56 -0
  263. graphcore/registry/tests/test_app_config.py +36 -0
  264. graphcore/registry/tests/test_discovery.py +147 -0
  265. graphcore/registry/tests/test_entity_directory_storage.py +73 -0
  266. graphcore/registry/tests/test_import_package.py +87 -0
  267. graphcore/registry/tests/test_principal_flag.py +67 -0
  268. graphcore/registry/tests/test_strategy_directory.py +29 -0
  269. graphcore/routes/__init__.py +116 -0
  270. graphcore/routes/tests/__init__.py +0 -0
  271. graphcore/routes/tests/test_routes.py +90 -0
  272. graphcore/runtime.py +243 -0
  273. graphcore/scheduling/__init__.py +0 -0
  274. graphcore/scheduling/beat_scheduler.py +164 -0
  275. graphcore/scheduling/beat_tasks.py +63 -0
  276. graphcore/scheduling/scheduler.py +37 -0
  277. graphcore/scheduling/tests/__init__.py +0 -0
  278. graphcore/scheduling/tests/contract.py +136 -0
  279. graphcore/scheduling/tests/memory.py +42 -0
  280. graphcore/scheduling/tests/test_beat_scheduler.py +87 -0
  281. graphcore/scheduling/tests/test_memory_scheduler.py +15 -0
  282. graphcore/scheduling/tests/test_ticks.py +143 -0
  283. graphcore/tasks.py +9 -0
  284. graphcore/test_settings.py +78 -0
  285. graphcore/testing/__init__.py +0 -0
  286. graphcore/testing/case.py +112 -0
  287. graphcore/testing/runner.py +32 -0
  288. graphcore/testing/settings.py +26 -0
  289. graphcore/testing/tests/__init__.py +0 -0
  290. graphcore/testing/tests/agent_view_fixture.py +19 -0
  291. graphcore/testing/tests/test_case_and_runner.py +219 -0
  292. graphcore/testing/tests/test_settings_for_apps.py +50 -0
  293. graphcore/tests/__init__.py +0 -0
  294. graphcore/tests/ids.py +14 -0
  295. graphcore/tests/public_api_manifest.py +102 -0
  296. graphcore/tests/runs.py +7 -0
  297. graphcore/tests/test_app_discovery.py +90 -0
  298. graphcore/tests/test_composition_boundaries.py +231 -0
  299. graphcore/tests/test_docs_discovery.py +108 -0
  300. graphcore/tests/test_no_print.py +70 -0
  301. graphcore/tests/test_packaging.py +64 -0
  302. graphcore/tests/test_public_api_boundaries.py +229 -0
  303. graphcore/tests/test_public_api_contract.py +62 -0
  304. graphcore/tests/test_public_api_docs.py +28 -0
  305. graphcore/tests/test_runtime.py +42 -0
  306. graphcore/tests/test_shared_type_boundaries.py +128 -0
  307. graphcore/tests/test_shared_types.py +144 -0
  308. graphcore/tests/test_test_settings.py +45 -0
  309. graphcore/transport/__init__.py +0 -0
  310. graphcore/transport/agent_asgi.py +68 -0
  311. graphcore/transport/agent_client.py +85 -0
  312. graphcore/transport/channel_layer.py +68 -0
  313. graphcore/transport/consumers.py +507 -0
  314. graphcore/transport/routing.py +6 -0
  315. graphcore/transport/tests/__init__.py +0 -0
  316. graphcore/transport/tests/test_agent_client.py +31 -0
  317. graphcore/transport/tests/test_api_views.py +733 -0
  318. graphcore/transport/tests/test_channel_layer.py +96 -0
  319. graphcore/transport/tests/test_consumers.py +963 -0
  320. graphcore/transport/tests/test_resource_metadata.py +67 -0
  321. graphcore/transport/urls.py +29 -0
  322. graphcore/transport/views.py +358 -0
  323. graphcore/types/__init__.py +4 -0
  324. graphcore/types/access.py +114 -0
  325. graphcore/types/api.py +76 -0
  326. graphcore/types/entities.py +100 -0
  327. graphcore/types/entity_id.py +85 -0
  328. graphcore/types/execution.py +41 -0
  329. graphcore/types/identity.py +20 -0
  330. graphcore/types/onboarding.py +20 -0
  331. graphcore/types/query.py +116 -0
  332. graphcore/types/schedules.py +91 -0
  333. graphcore/types/strategy.py +73 -0
  334. graphcore/types/tests/__init__.py +0 -0
  335. graphcore/types/tests/test_entity_id.py +50 -0
  336. graphcore/urls.py +5 -0
  337. graphcore_kernel-0.1.0.dist-info/METADATA +94 -0
  338. graphcore_kernel-0.1.0.dist-info/RECORD +343 -0
  339. graphcore_kernel-0.1.0.dist-info/WHEEL +5 -0
  340. graphcore_kernel-0.1.0.dist-info/entry_points.txt +2 -0
  341. graphcore_kernel-0.1.0.dist-info/licenses/LICENSE +202 -0
  342. graphcore_kernel-0.1.0.dist-info/licenses/NOTICE +2 -0
  343. graphcore_kernel-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,76 @@
1
+ """Both kernel dispatchers against the dispatcher contract, and which one
2
+ the composition root maps each workload to."""
3
+ import threading
4
+ from unittest.mock import patch
5
+
6
+ from asgiref.sync import async_to_sync
7
+ from celery.signals import worker_init
8
+ from django.test import SimpleTestCase
9
+
10
+ from graphcore.execution.environments.celery import CeleryDispatcher
11
+ from graphcore.execution.environments.in_process import InProcessDispatcher
12
+ from graphcore.execution.tests.contract import DispatcherContract
13
+ from graphcore.execution.tests.support import TEST_REGISTRY, ExecutionTestCase, Fail, Increment
14
+ from graphcore import runtime
15
+ from graphcore.errors import ConfigurationError
16
+ from graphcore.runtime import default_dispatchers, worker_dispatchers
17
+ from graphcore.types.strategy import Workload
18
+
19
+
20
+ class _Fixture(ExecutionTestCase):
21
+ def setUp(self):
22
+ super().setUp()
23
+ self.dispatcher = self.make_dispatcher()
24
+ self.executor.bind_dispatchers({Workload.IO: self.dispatcher, Workload.CPU: self.dispatcher})
25
+
26
+ async def arrange(self):
27
+ await self.seed()
28
+ self.owner = self.alice
29
+ self.succeeding = Increment.request_constructor(self.probe.entity_id)
30
+ self.failing = Fail.request_constructor(self.probe.entity_id)
31
+
32
+
33
+ def _sync(cls):
34
+ for name in dir(cls):
35
+ if name.startswith('test_'):
36
+ setattr(cls, name, async_to_sync(getattr(cls, name)))
37
+ return cls
38
+
39
+
40
+ @_sync
41
+ class InProcessDispatcherTests(_Fixture, DispatcherContract):
42
+ def make_dispatcher(self):
43
+ return InProcessDispatcher(self.executor, self.service, TEST_REGISTRY)
44
+
45
+
46
+ @_sync
47
+ class CeleryDispatcherTests(_Fixture, DispatcherContract):
48
+ def make_dispatcher(self):
49
+ return CeleryDispatcher()
50
+
51
+
52
+ class WorkerPlacementTests(SimpleTestCase):
53
+ def test_a_worker_runs_cpu_beside_io_and_every_other_process_queues_it(self):
54
+ self.assertIsInstance(default_dispatchers(None, None, TEST_REGISTRY)[Workload.CPU], CeleryDispatcher)
55
+ dispatchers = worker_dispatchers(None, None, TEST_REGISTRY)
56
+ self.assertIsInstance(dispatchers[Workload.CPU], InProcessDispatcher)
57
+ self.assertIs(dispatchers[Workload.CPU], dispatchers[Workload.IO])
58
+
59
+ def test_a_starting_worker_builds_its_runtime_as_a_worker(self):
60
+ with patch.object(runtime, '_worker', False), patch.object(runtime, '_runtime', None), \
61
+ patch.object(runtime, 'configured_runtime') as configured:
62
+ worker_init.send(sender=None)
63
+ runtime.get_runtime()
64
+ configured.assert_called_once_with(worker=True)
65
+
66
+ def test_a_process_becomes_a_worker_only_before_building_its_runtime(self):
67
+ with patch.object(runtime, '_worker', False), patch.object(runtime, '_runtime', object()):
68
+ with self.assertRaises(ConfigurationError):
69
+ runtime.serve_as_worker()
70
+
71
+ def test_a_worker_sharing_its_process_with_a_web_server_leaves_it_unmarked(self):
72
+ with patch.object(runtime, '_worker', False), patch.object(runtime, '_runtime', None):
73
+ thread = threading.Thread(target=lambda: worker_init.send(sender=None))
74
+ thread.start()
75
+ thread.join()
76
+ self.assertFalse(runtime._worker)
@@ -0,0 +1,368 @@
1
+ """The executor's front door, attempts, retries, and reporting
2
+ (docs/STRATEGY_EXECUTION_TDD.md, The executor)."""
3
+ import asyncio
4
+ from unittest.mock import patch
5
+
6
+ from asgiref.sync import async_to_sync
7
+
8
+ from graphcore.errors import ConfigurationError, Conflict, Forbidden, InvalidRequest, NotFound, StrategyFailed
9
+ from graphcore.execution.requests import new_strategy_request
10
+ from graphcore.execution.tests.support import (
11
+ AlwaysConflict, ConflictOnce, CpuFail, CpuIncrement, CpuNested, CpuRefuse, ExecutionTestCase, Fail, Increment,
12
+ Claimed, Nested, NoChange, NotExecutable, ReadOnly, Record, Recording, Refuse, Untargeted, Validated, WriteOther,
13
+ )
14
+ from graphcore.types.execution import AttemptResult
15
+ from graphcore.types.strategy import StrategyRequestPayload, Workload
16
+
17
+
18
+ class FrontDoorTests(ExecutionTestCase):
19
+ def _record_statuses(self):
20
+ statuses = []
21
+ real = self.service.save_entity
22
+
23
+ async def save(entity, principal, **kwargs):
24
+ saved = await real(entity, principal, **kwargs)
25
+ if hasattr(entity, 'status') and hasattr(entity, 'attempt'):
26
+ statuses.append((saved.status, saved.attempt))
27
+ return saved
28
+
29
+ patcher = patch.object(self.service, 'save_entity', side_effect=save)
30
+ patcher.start()
31
+ self.addCleanup(patcher.stop)
32
+ return statuses
33
+
34
+ @async_to_sync
35
+ async def test_an_io_run_goes_pending_running_succeeded(self):
36
+ await self.seed()
37
+ statuses = self._record_statuses()
38
+ request, target = await self.submit(Increment)
39
+ self.assertEqual(statuses, [('pending', 1), ('running', 1), ('succeeded', 1)])
40
+ self.assertEqual(request.status, 'succeeded')
41
+ self.assertEqual(request.ret_val, {'counter': 1})
42
+ self.assertEqual(request.owner, self.alice.principal_entity_id)
43
+ self.assertEqual(target.get_attribute('counter'), 1)
44
+ self.assertEqual(len(self.io.attempts), 1)
45
+ self.assertEqual(self.cpu.attempts, [])
46
+
47
+ @async_to_sync
48
+ async def test_a_cpu_run_is_dispatched_to_its_workload_and_awaited(self):
49
+ await self.seed()
50
+ request, target = await self.submit(CpuIncrement)
51
+ self.assertEqual(request.status, 'succeeded')
52
+ self.assertEqual(target.get_attribute('counter'), 1)
53
+ self.assertEqual([a.request_id for a in self.cpu.attempts], [request.entity_id])
54
+ self.assertEqual(self.io.attempts, [])
55
+
56
+ @async_to_sync
57
+ async def test_without_wait_the_request_is_returned_before_it_finishes(self):
58
+ await self.seed()
59
+ request, target = await self.submit(Increment, wait=False)
60
+ self.assertIsNone(target)
61
+ self.assertIn(request.status, ('pending', 'running'))
62
+ await self.settle()
63
+ self.assertEqual((await self.load(request.entity_id)).status, 'succeeded')
64
+ self.assertEqual(await self.counter(), 1)
65
+
66
+ @async_to_sync
67
+ async def test_a_failed_run_records_its_error_and_writes_nothing(self):
68
+ await self.seed()
69
+ Fail.cleaned = CpuFail.cleaned = 0
70
+ for strategy in (Fail, CpuFail):
71
+ request = await self.failure(StrategyFailed, strategy)
72
+ self.assertEqual((request.status, request.error, request.error_code), ('failed', 'boom', 'strategy_failed'))
73
+ self.assertIsNone(await self.counter())
74
+ self.assertEqual((Fail.cleaned, CpuFail.cleaned), (1, 1))
75
+
76
+ @async_to_sync
77
+ async def test_an_apps_refusal_records_its_code_and_details_without_a_retry(self):
78
+ await self.seed()
79
+ for strategy in (Refuse, CpuRefuse):
80
+ request = await self.failure(Conflict, strategy)
81
+ self.assertEqual((request.status, request.error, request.error_code, request.attempt),
82
+ ('failed', 'stale', 'conflict', 1))
83
+ self.assertEqual(request.error_details, {'invitation': {'answers': {'dinner': 'accepted'}}})
84
+
85
+ @async_to_sync
86
+ async def test_a_request_id_runs_once(self):
87
+ await self.seed()
88
+ request = new_strategy_request(self.service, self.payload(Increment))
89
+ await self.executor.execute_request(request, self.alice)
90
+ with self.assertRaisesRegex(InvalidRequest, 'already exists'):
91
+ await self.executor.execute_request(request, self.alice)
92
+ self.assertEqual(await self.counter(), 1)
93
+
94
+ @async_to_sync
95
+ async def test_refused_requests_are_never_saved(self):
96
+ await self.seed()
97
+ cases = [
98
+ (Forbidden, lambda: self.submit(Increment, principal=self.bob)),
99
+ (NotFound, lambda: self.submit(Increment, target=self.service.create(type(self.probe)).entity_id)),
100
+ (InvalidRequest, lambda: self.executor.submit(
101
+ StrategyRequestPayload(strategy_name='Unregistered', target_entity_id=self.probe.entity_id),
102
+ self.alice)),
103
+ (InvalidRequest, lambda: self.executor.submit(
104
+ StrategyRequestPayload(strategy_name='Increment', target_entity_id=None), self.alice)),
105
+ ]
106
+ for error, attempt in cases:
107
+ with self.assertRaises(error):
108
+ await attempt()
109
+ self.assertEqual(await self.requests(), [])
110
+
111
+ @async_to_sync
112
+ async def test_an_untargeted_strategy_runs_on_the_deployment(self):
113
+ await self.seed()
114
+ request, target = await self.executor.submit(Untargeted.request_constructor(), self.alice)
115
+ self.assertEqual(request.status, 'succeeded')
116
+ self.assertEqual(request.ret_val, {'untargeted': True})
117
+ self.assertIsNone(target)
118
+
119
+ @async_to_sync
120
+ async def test_unchanged_and_read_only_targets_are_not_saved(self):
121
+ await self.seed()
122
+ for strategy in (NoChange, ReadOnly):
123
+ request, _ = await self.submit(strategy)
124
+ self.assertEqual(request.status, 'succeeded')
125
+ self.assertEqual((await self.load(self.probe.entity_id)).version, self.probe.version)
126
+
127
+ @async_to_sync
128
+ async def test_boundary_checks_fail_the_run_before_apply(self):
129
+ await self.seed()
130
+ request = await self.failure(InvalidRequest, Validated, n='three')
131
+ self.assertEqual(request.error, "Invalid param_config for Validated: n: 'three' is not of type 'integer'")
132
+ request = await self.failure(StrategyFailed, NotExecutable)
133
+ self.assertEqual((request.status, request.error), ('failed', 'probe is not ready'))
134
+ request, _ = await self.submit(Validated, n=3)
135
+ self.assertEqual(request.status, 'succeeded')
136
+
137
+ @async_to_sync
138
+ async def test_a_strategy_writes_only_what_its_owner_may(self):
139
+ await self.seed()
140
+ theirs = await self.make_probe(self.bob)
141
+ for claim_kernel in (False, True):
142
+ await self.failure(Forbidden, WriteOther, other=theirs.entity_id, claim_kernel=claim_kernel)
143
+ self.assertIsNone(await self.counter(theirs.entity_id))
144
+
145
+ @async_to_sync
146
+ async def test_many_concurrent_runs_all_finish(self):
147
+ await self.seed()
148
+ probes = [await self.make_probe(self.alice) for _ in range(20)]
149
+ results = await asyncio.gather(*(self.submit(Increment, target=p.entity_id) for p in probes))
150
+ self.assertEqual({r.status for r, _ in results}, {'succeeded'})
151
+ for probe in probes:
152
+ self.assertEqual(await self.counter(probe.entity_id), 1)
153
+
154
+
155
+ class AttemptTests(ExecutionTestCase):
156
+ @async_to_sync
157
+ async def test_a_conflict_is_retried_with_backoff(self):
158
+ await self.seed()
159
+ request, target = await self.submit(ConflictOnce)
160
+ self.assertEqual((request.status, request.attempt), ('succeeded', 2))
161
+ self.assertEqual(target.get_attribute('counter'), 1)
162
+ self.assertEqual(self.backoffs, [1])
163
+
164
+ @async_to_sync
165
+ async def test_conflicts_fail_the_run_after_max_attempts(self):
166
+ await self.seed()
167
+ self.executor.max_attempts = 3
168
+ request = await self.failure(Conflict, AlwaysConflict)
169
+ self.assertEqual((request.status, request.attempt), ('failed', 3))
170
+ self.assertEqual(request.error, 'always loses')
171
+ self.assertEqual(self.backoffs, [1, 2])
172
+
173
+ @async_to_sync
174
+ async def test_access_is_checked_again_at_each_attempt(self):
175
+ await self.seed()
176
+ shared = await self.make_probe(self.alice)
177
+ await self.runtime.graph_api.grant(self.alice, shared.entity_id, self.bob.principal_entity_id, 'editor')
178
+ self.executor.bind_dispatchers({Workload.IO: Recording(), Workload.CPU: Recording()})
179
+ request, _ = await self.submit(Increment, principal=self.bob, target=shared.entity_id, wait=False)
180
+ self.executor.bind_dispatchers({Workload.IO: self.io, Workload.CPU: self.cpu})
181
+ await self.runtime.graph_api.revoke(self.alice, shared.entity_id, self.bob.principal_entity_id)
182
+
183
+ await self.executor.start_attempt(await self.load(request.entity_id))
184
+
185
+ request = await self.load(request.entity_id)
186
+ self.assertEqual(request.status, 'failed')
187
+ self.assertIn('not allowed', request.error)
188
+ self.assertIsNone(await self.counter(shared.entity_id))
189
+
190
+ @async_to_sync
191
+ async def test_a_stale_or_repeated_result_has_no_effect(self):
192
+ await self.seed()
193
+ held = Claimed(self.executor)
194
+ self.executor.bind_dispatchers({Workload.IO: held, Workload.CPU: held})
195
+ request, _ = await self.submit(Increment, wait=False)
196
+ await self.executor.finish_attempt(AttemptResult(request.entity_id, 0, 'succeeded', ret_val={'x': 1}))
197
+ self.assertEqual((await self.load(request.entity_id)).status, 'running')
198
+
199
+ done = AttemptResult(request.entity_id, 1, 'succeeded', ret_val={'x': 2})
200
+ await self.executor.finish_attempt(done)
201
+ finished = await self.load(request.entity_id)
202
+ await self.executor.finish_attempt(AttemptResult(request.entity_id, 1, 'failed', error='late'))
203
+ again = await self.load(request.entity_id)
204
+ self.assertEqual((again.status, again.ret_val, again.version), ('succeeded', {'x': 2}, finished.version))
205
+
206
+ @async_to_sync
207
+ async def test_a_dispatch_failure_fails_the_run(self):
208
+ await self.seed()
209
+
210
+ class Down(Recording):
211
+ async def dispatch(self, attempt):
212
+ raise ConnectionError('broker down')
213
+
214
+ self.executor.bind_dispatchers({Workload.IO: self.io, Workload.CPU: Down()})
215
+ request = await self.failure(StrategyFailed, CpuIncrement)
216
+ self.assertIn('broker down', request.error)
217
+
218
+ def test_every_registered_workload_needs_a_dispatcher(self):
219
+ with self.assertRaises(ConfigurationError):
220
+ self.executor.bind_dispatchers({Workload.IO: Recording()})
221
+
222
+
223
+ class WaitingTests(ExecutionTestCase):
224
+ @async_to_sync
225
+ async def test_a_waiter_returns_when_its_run_finishes_elsewhere(self):
226
+ await self.seed()
227
+ held = Claimed(self.executor)
228
+ self.executor.bind_dispatchers({Workload.IO: held, Workload.CPU: held})
229
+ waiter = asyncio.ensure_future(self.submit(Increment))
230
+ while not held.attempts:
231
+ await asyncio.sleep(0)
232
+ await self.executor.finish_attempt(AttemptResult(held.attempts[0].request_id, 1, 'succeeded', ret_val={'x': 1}))
233
+ request, _ = await asyncio.wait_for(waiter, 5)
234
+ self.assertEqual((request.status, request.ret_val), ('succeeded', {'x': 1}))
235
+
236
+ @async_to_sync
237
+ async def test_a_waiter_reconciles_its_own_overdue_run(self):
238
+ await self.seed()
239
+ held = Claimed(self.executor)
240
+ self.executor.bind_dispatchers({Workload.IO: held, Workload.CPU: held})
241
+ self.executor.default_timeout = 0.1
242
+ self.executor.max_attempts = 2
243
+ request = await asyncio.wait_for(self.failure(StrategyFailed, Increment), 5)
244
+ self.assertEqual((request.attempt, request.error), (2, 'attempt timed out'))
245
+ self.assertEqual(len(held.attempts), 2)
246
+
247
+ @async_to_sync
248
+ async def test_a_waiter_sends_an_unclaimed_attempt_again_as_the_same_attempt(self):
249
+ await self.seed()
250
+ lost = Recording()
251
+ self.executor.bind_dispatchers({Workload.IO: lost, Workload.CPU: lost})
252
+ self.executor.dispatch_timeout = 0.1
253
+ waiter = asyncio.ensure_future(self.submit(Increment))
254
+ while len(lost.attempts) < 2:
255
+ await asyncio.sleep(0.01)
256
+ self.executor.bind_dispatchers({Workload.IO: self.io, Workload.CPU: self.cpu})
257
+ self.assertEqual([a.attempt for a in lost.attempts[:2]], [1, 1])
258
+ await self.io.dispatch(lost.attempts[0])
259
+ request, target = await asyncio.wait_for(waiter, 5)
260
+ self.assertEqual((request.status, request.attempt), ('succeeded', 1))
261
+ self.assertEqual(target.get_attribute('counter'), 1)
262
+
263
+
264
+ class ReportingTests(ExecutionTestCase):
265
+ def _capture(self):
266
+ sent = []
267
+
268
+ async def send(channel, message):
269
+ sent.append((channel, message))
270
+
271
+ patcher = patch.object(self.service.broadcaster, 'send', side_effect=send)
272
+ patcher.start()
273
+ self.addCleanup(patcher.stop)
274
+ return sent
275
+
276
+ @async_to_sync
277
+ async def test_strategy_complete_carries_the_client_request_id_in_every_environment(self):
278
+ await self.seed()
279
+ sent = self._capture()
280
+ for strategy in (Increment, CpuIncrement):
281
+ payload = self.payload(strategy)
282
+ payload.client_request_id = f'client-{strategy.__name__}'
283
+ request, _ = await self.executor.submit(payload, self.alice, callback_channel='chan')
284
+ channel, message = sent[-1]
285
+ self.assertEqual(channel, 'chan')
286
+ self.assertEqual(message, {
287
+ 'type': 'strategy_complete', 'request_id': request.entity_id,
288
+ 'target_entity_id': self.probe.entity_id, 'client_request_id': f'client-{strategy.__name__}',
289
+ 'status': 'succeeded', 'success': True, 'ret_val': request.ret_val,
290
+ })
291
+
292
+ @async_to_sync
293
+ async def test_a_failure_is_reported_with_its_error(self):
294
+ await self.seed()
295
+ sent = self._capture()
296
+ for strategy in (Fail, CpuFail):
297
+ await self.failure(StrategyFailed, strategy, callback_channel='chan')
298
+ message = sent[-1][1]
299
+ self.assertEqual(
300
+ (message['status'], message['success'], message['error'], message['code']),
301
+ ('failed', False, 'boom', 'strategy_failed'),
302
+ )
303
+ self.assertNotIn('ret_val', message)
304
+ self.assertEqual(message['details'], {})
305
+
306
+ @async_to_sync
307
+ async def test_a_refusal_is_reported_with_its_details(self):
308
+ await self.seed()
309
+ sent = self._capture()
310
+ await self.failure(Conflict, Refuse, callback_channel='chan')
311
+ message = sent[-1][1]
312
+ self.assertEqual((message['code'], message['details']),
313
+ ('conflict', {'invitation': {'answers': {'dinner': 'accepted'}}}))
314
+
315
+ @async_to_sync
316
+ async def test_without_a_channel_nothing_is_sent(self):
317
+ await self.seed()
318
+ sent = self._capture()
319
+ await self.submit(Increment)
320
+ self.assertFalse([m for _, m in sent if m.get('type') == 'strategy_complete'])
321
+
322
+ @async_to_sync
323
+ async def test_callbacks_run_as_the_owner_with_the_source_result(self):
324
+ await self.seed()
325
+ other = await self.make_probe(self.alice)
326
+ payload = self.payload(Increment)
327
+ payload.callbacks = [Record.request_constructor(other.entity_id).to_dict()]
328
+ payload.client_request_id = 'not-copied'
329
+ request, _ = await self.executor.submit(payload, self.alice)
330
+ await self.settle()
331
+
332
+ callback = next(r for r in await self.requests() if r.strategy_name == 'Record')
333
+ self.assertEqual(callback.status, 'succeeded')
334
+ self.assertEqual(callback.owner, self.alice.principal_entity_id)
335
+ self.assertEqual(callback.param_config['source_ret_val'], request.ret_val)
336
+ self.assertIsNone(callback.client_request_id)
337
+ self.assertEqual((await self.load(other.entity_id)).get_attribute('received'), {'counter': 1})
338
+
339
+ @async_to_sync
340
+ async def test_a_failed_run_submits_no_callbacks(self):
341
+ await self.seed()
342
+ payload = self.payload(Fail)
343
+ payload.callbacks = [Record.request_constructor(self.probe.entity_id).to_dict()]
344
+ with self.assertRaises(StrategyFailed):
345
+ await self.executor.submit(payload, self.alice)
346
+ await self.settle()
347
+ self.assertEqual([r.strategy_name for r in await self.requests()], ['Fail'])
348
+
349
+
350
+ class NestedRequestTests(ExecutionTestCase):
351
+ @async_to_sync
352
+ async def test_a_nested_request_runs_as_the_owner_in_any_environment(self):
353
+ await self.seed()
354
+ for parent, child in ((Nested, 'CpuIncrement'), (CpuNested, 'Increment'), (Nested, 'Increment')):
355
+ request, _ = await self.submit(parent, child=child)
356
+ self.assertEqual(request.status, 'succeeded', request.error)
357
+ self.assertEqual(request.ret_val['child_status'], 'succeeded')
358
+ nested = await self.load(request.ret_val['child_id'])
359
+ self.assertEqual((nested.status, nested.owner), ('succeeded', self.alice.principal_entity_id))
360
+ self.assertEqual(await self.counter(), 3)
361
+
362
+ @async_to_sync
363
+ async def test_a_nested_request_without_wait_still_finishes(self):
364
+ await self.seed()
365
+ request, _ = await self.submit(Nested, child='Increment', child_wait=False)
366
+ await self.settle()
367
+ self.assertEqual((await self.load(request.ret_val['child_id'])).status, 'succeeded')
368
+ self.assertEqual(await self.counter(), 1)
@@ -0,0 +1,95 @@
1
+ """`LocalStrategySession` against the session contract, and what only the
2
+ local adapter promises: the change tracker and the strategy context."""
3
+ from unittest.mock import patch
4
+
5
+ from asgiref.sync import async_to_sync
6
+
7
+ from graphcore.domain.strategy.runner import ChangeTracker
8
+ from graphcore.execution.local_session import LocalStrategySession
9
+ from graphcore.execution.strategy_context import current_strategy, kernel_internal_context
10
+ from graphcore.execution.tests.contract import StrategySessionContract
11
+ from graphcore.execution.tests.support import (
12
+ Claimed, ExecutionTestCase, Increment, SeeContext, TEST_REGISTRY, _Probe,
13
+ )
14
+ from graphcore.types.strategy import Workload
15
+
16
+
17
+ class LocalSessionFixture(ExecutionTestCase):
18
+ async def arrange(self):
19
+ await self.seed()
20
+ self.owner, self.stranger = self.alice, self.bob
21
+ self.entity_cls = _Probe
22
+ self.writer, self.reader = 'Increment', 'ReadOnly'
23
+ self.nested = Increment.request_constructor(self.probe.entity_id)
24
+
25
+ async def seed_entity(self, owner):
26
+ return await self.make_probe(owner)
27
+
28
+ async def open_session(self, strategy_name, tracker=None):
29
+ held = Claimed(self.executor)
30
+ self.executor.bind_dispatchers({Workload.IO: held, Workload.CPU: held})
31
+ strategy = TEST_REGISTRY.strategies.by_name(strategy_name)
32
+ await self.executor.submit(strategy.request_constructor(self.probe.entity_id), self.owner, wait=False)
33
+ self.executor.bind_dispatchers({Workload.IO: self.io, Workload.CPU: self.cpu})
34
+ return LocalStrategySession(held.attempts[-1], tracker or ChangeTracker(), self.service,
35
+ self.executor, TEST_REGISTRY)
36
+
37
+ async def retry(self, session):
38
+ held = Claimed(self.executor)
39
+ self.executor.bind_dispatchers({Workload.IO: held, Workload.CPU: held})
40
+ await self.executor.start_attempt(await self.load(session.attempt.request_id))
41
+ self.executor.bind_dispatchers({Workload.IO: self.io, Workload.CPU: self.cpu})
42
+ return LocalStrategySession(held.attempts[-1], ChangeTracker(), self.service,
43
+ self.executor, TEST_REGISTRY)
44
+
45
+ async def supersede(self, session):
46
+ request = await self.load(session.attempt.request_id)
47
+ request.attempt += 1
48
+ with kernel_internal_context():
49
+ await self.service.save_entity(request, session.principal)
50
+
51
+ def backend_id(self):
52
+ return self.service.backend_id
53
+
54
+
55
+ def _sync(cls):
56
+ for name in dir(cls):
57
+ if name.startswith('test_'):
58
+ setattr(cls, name, async_to_sync(getattr(cls, name)))
59
+ return cls
60
+
61
+
62
+ @_sync
63
+ class LocalStrategySessionContractTests(LocalSessionFixture, StrategySessionContract):
64
+ pass
65
+
66
+
67
+ @_sync
68
+ class LocalStrategySessionTests(LocalSessionFixture):
69
+ async def test_the_tracker_holds_what_was_read_and_saved(self):
70
+ await self.arrange()
71
+ tracker = ChangeTracker()
72
+ session = await self.open_session('Increment', tracker)
73
+ probe = await session.get(self.probe.entity_id)
74
+ self.assertFalse(tracker.is_unsaved(probe))
75
+ probe.set_attribute('counter', 7)
76
+ self.assertTrue(tracker.is_unsaved(probe))
77
+ saved = await session.save(probe)
78
+ self.assertFalse(tracker.is_unsaved(saved))
79
+
80
+ async def test_each_write_and_only_writes_run_in_the_strategy_context(self):
81
+ await self.arrange()
82
+ SeeContext.seen = []
83
+ during_save = []
84
+ real = self.service.save_entity
85
+
86
+ async def save(entity, principal, **kwargs):
87
+ if isinstance(entity, _Probe):
88
+ during_save.append(current_strategy())
89
+ return await real(entity, principal, **kwargs)
90
+
91
+ with patch.object(self.service, 'save_entity', side_effect=save):
92
+ request, _ = await self.submit(SeeContext)
93
+ self.assertEqual(request.status, 'succeeded')
94
+ self.assertEqual(SeeContext.seen, [None, None])
95
+ self.assertEqual(during_save, [SeeContext])
@@ -0,0 +1,134 @@
1
+ """Run reconciliation: an attempt past its deadline is re-dispatched, and
2
+ its late writes and results are refused (docs/STRATEGY_EXECUTION_TDD.md,
3
+ Run reconciliation)."""
4
+ import time
5
+ from unittest.mock import patch
6
+
7
+ from asgiref.sync import async_to_sync
8
+
9
+ from graphcore.domain.strategy.runner import ChangeTracker
10
+ from graphcore.errors import StaleAttempt
11
+ from graphcore.execution.local_session import LocalStrategySession
12
+ from graphcore.execution.strategy_context import kernel_internal_context
13
+ from graphcore.execution.tests.support import (
14
+ TEST_REGISTRY, Claimed, CpuIncrement, ExecutionTestCase, Increment, Recording,
15
+ )
16
+ from graphcore.types.execution import AttemptResult
17
+ from graphcore.types.strategy import Workload
18
+
19
+
20
+ class ReconcileRunsTests(ExecutionTestCase):
21
+ async def _lost(self, strategy=Increment):
22
+ """A running request whose attempt never reports, past its deadline."""
23
+ lost = Claimed(self.executor)
24
+ self.executor.bind_dispatchers({Workload.IO: lost, Workload.CPU: lost})
25
+ request, _ = await self.submit(strategy, wait=False)
26
+ self.executor.bind_dispatchers({Workload.IO: self.io, Workload.CPU: self.cpu})
27
+ await self._expire(request.entity_id)
28
+ return lost.attempts[0]
29
+
30
+ async def _expire(self, request_id):
31
+ request = await self.load(request_id)
32
+ request.attempt_deadline = time.time() - 1
33
+ with kernel_internal_context():
34
+ await self.service.save_entity(request, self.alice)
35
+
36
+ @async_to_sync
37
+ async def test_an_expired_attempt_is_dispatched_again(self):
38
+ await self.seed()
39
+ for strategy in (Increment, CpuIncrement):
40
+ first = await self._lost(strategy)
41
+ self.assertEqual(await self.executor.reconcile_runs(), 1)
42
+ await self.settle()
43
+ request = await self.load(first.request_id)
44
+ self.assertEqual((request.status, request.attempt), ('succeeded', 2))
45
+ self.assertEqual(await self.counter(), 2)
46
+
47
+ @async_to_sync
48
+ async def test_the_expired_attempt_writes_nothing_and_its_result_is_discarded(self):
49
+ await self.seed()
50
+ first = await self._lost()
51
+ late = LocalStrategySession(first, ChangeTracker(), self.service, self.executor, TEST_REGISTRY)
52
+ probe = await late.get(self.probe.entity_id)
53
+ await self.executor.reconcile_runs()
54
+ await self.settle()
55
+
56
+ probe.set_attribute('counter', 100)
57
+ with self.assertRaises(StaleAttempt):
58
+ await late.save(probe)
59
+ await self.executor.finish_attempt(AttemptResult(first.request_id, 1, 'failed', error='late'))
60
+ request = await self.load(first.request_id)
61
+ self.assertEqual((request.status, request.attempt, request.error), ('succeeded', 2, None))
62
+ self.assertEqual(await self.counter(), 1)
63
+
64
+ @async_to_sync
65
+ async def test_an_expired_last_attempt_fails_the_run(self):
66
+ await self.seed()
67
+ self.executor.max_attempts = 1
68
+ first = await self._lost()
69
+ self.assertEqual(await self.executor.reconcile_runs(), 1)
70
+ request = await self.load(first.request_id)
71
+ self.assertEqual((request.status, request.error, request.attempt_deadline), ('failed', 'attempt timed out', None))
72
+
73
+ @async_to_sync
74
+ async def test_attempts_within_their_deadline_and_finished_runs_are_left_alone(self):
75
+ await self.seed()
76
+ held = Recording()
77
+ self.executor.bind_dispatchers({Workload.IO: held, Workload.CPU: held})
78
+ running, _ = await self.submit(Increment, wait=False)
79
+ self.executor.bind_dispatchers({Workload.IO: self.io, Workload.CPU: self.cpu})
80
+ done, _ = await self.submit(Increment)
81
+ self.assertEqual(await self.executor.reconcile_runs(), 0)
82
+ self.assertEqual((await self.load(running.entity_id)).attempt, 1)
83
+ self.assertEqual((await self.load(done.entity_id)).status, 'succeeded')
84
+
85
+ @async_to_sync
86
+ async def test_a_deadline_follows_the_strategy_timeout(self):
87
+ await self.seed()
88
+ held = Claimed(self.executor)
89
+ self.executor.bind_dispatchers({Workload.IO: held, Workload.CPU: held})
90
+ self.executor.default_timeout = 1000
91
+ before = time.time()
92
+ request, _ = await self.submit(Increment, wait=False)
93
+ deadline = (await self.load(request.entity_id)).attempt_deadline
94
+ self.assertGreaterEqual(deadline, before + 1000)
95
+ Increment.timeout = 5
96
+ try:
97
+ request, _ = await self.submit(Increment, wait=False)
98
+ finally:
99
+ Increment.timeout = None
100
+ self.assertLess((await self.load(request.entity_id)).attempt_deadline, before + 100)
101
+
102
+ @async_to_sync
103
+ async def test_a_request_saved_but_never_started_is_started(self):
104
+ await self.seed()
105
+ with patch.object(self.executor, '_dispatch'):
106
+ request, _ = await self.submit(Increment, wait=False)
107
+ await self._expire(request.entity_id)
108
+ self.assertEqual(await self.executor.reconcile_runs(), 1)
109
+ await self.settle()
110
+ request = await self.load(request.entity_id)
111
+ self.assertEqual((request.status, request.attempt), ('succeeded', 1))
112
+
113
+ @async_to_sync
114
+ async def test_an_unclaimed_attempt_is_sent_again_and_runs_once(self):
115
+ await self.seed()
116
+ lost = Recording()
117
+ self.executor.bind_dispatchers({Workload.IO: lost, Workload.CPU: lost})
118
+ request, _ = await self.submit(Increment, wait=False)
119
+ await self._expire(request.entity_id)
120
+ self.assertEqual(await self.executor.reconcile_runs(), 1)
121
+ self.assertEqual([a.attempt for a in lost.attempts], [1, 1])
122
+ for attempt in lost.attempts:
123
+ await self.io.dispatch(attempt)
124
+ await self.settle()
125
+ request = await self.load(request.entity_id)
126
+ self.assertEqual((request.status, request.attempt), ('succeeded', 1))
127
+ self.assertEqual(await self.counter(), 1)
128
+
129
+ def test_the_reconcile_task_runs_the_executor(self):
130
+ from graphcore.execution.tasks import reconcile_runs
131
+ async_to_sync(self.seed)()
132
+ first = async_to_sync(self._lost)()
133
+ self.assertEqual(reconcile_runs.delay().get(), 1)
134
+ self.assertEqual(async_to_sync(self.load)(first.request_id).status, 'succeeded')