graphcore-kernel 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (343) hide show
  1. graphcore/__init__.py +5 -0
  2. graphcore/agent_frontend/__init__.py +8 -0
  3. graphcore/agent_frontend/api.py +69 -0
  4. graphcore/agent_frontend/base/__init__.py +11 -0
  5. graphcore/agent_frontend/base/views.py +167 -0
  6. graphcore/agent_frontend/client.py +31 -0
  7. graphcore/agent_frontend/context.py +157 -0
  8. graphcore/agent_frontend/errors.py +12 -0
  9. graphcore/agent_frontend/loading.py +54 -0
  10. graphcore/agent_frontend/registry.py +42 -0
  11. graphcore/agent_frontend/render.py +207 -0
  12. graphcore/agent_frontend/server.py +103 -0
  13. graphcore/agent_frontend/signatures.py +35 -0
  14. graphcore/agent_frontend/testing.py +45 -0
  15. graphcore/agent_frontend/tests/__init__.py +0 -0
  16. graphcore/agent_frontend/tests/fakes.py +114 -0
  17. graphcore/agent_frontend/tests/test_frontend.py +447 -0
  18. graphcore/agent_frontend/tests/test_loading.py +83 -0
  19. graphcore/agent_frontend/tests/test_server.py +109 -0
  20. graphcore/agent_frontend/tools.py +137 -0
  21. graphcore/api/__init__.py +4 -0
  22. graphcore/api/graph_api.py +378 -0
  23. graphcore/api/predicate_codec.py +81 -0
  24. graphcore/api/tests/__init__.py +0 -0
  25. graphcore/api/tests/test_changed_entities.py +109 -0
  26. graphcore/api/tests/test_graph_api.py +290 -0
  27. graphcore/api/tests/test_graph_api_access.py +94 -0
  28. graphcore/api/tests/test_graph_api_grants.py +90 -0
  29. graphcore/api/tests/test_predicate_codec.py +83 -0
  30. graphcore/api/tests/test_subtree_query.py +79 -0
  31. graphcore/apps.py +71 -0
  32. graphcore/auth/__init__.py +1 -0
  33. graphcore/auth/authn/README.md +178 -0
  34. graphcore/auth/authn/__init__.py +3 -0
  35. graphcore/auth/authn/authenticator.py +120 -0
  36. graphcore/auth/authn/onboarding.py +105 -0
  37. graphcore/auth/authn/principals.py +20 -0
  38. graphcore/auth/authn/resource_metadata.py +43 -0
  39. graphcore/auth/authn/service_accounts.py +96 -0
  40. graphcore/auth/authn/testing.py +96 -0
  41. graphcore/auth/authn/tests/__init__.py +0 -0
  42. graphcore/auth/authn/tests/test_authenticator.py +145 -0
  43. graphcore/auth/authn/tests/test_onboarding.py +145 -0
  44. graphcore/auth/authn/tests/test_service_accounts.py +180 -0
  45. graphcore/auth/authn/users.py +59 -0
  46. graphcore/auth/authz/__init__.py +0 -0
  47. graphcore/auth/authz/access_port.py +120 -0
  48. graphcore/auth/authz/adapter/__init__.py +0 -0
  49. graphcore/auth/authz/adapter/default_access_adapter.py +556 -0
  50. graphcore/auth/authz/adapter/tests/__init__.py +0 -0
  51. graphcore/auth/authz/adapter/tests/test_default_adapter.py +543 -0
  52. graphcore/auth/authz/adapter/tests/test_entity_write_hooks.py +207 -0
  53. graphcore/auth/authz/adapter/tests/test_group_types.py +70 -0
  54. graphcore/auth/authz/adapter/tests/test_sync_support.py +209 -0
  55. graphcore/auth/authz/defaults.py +26 -0
  56. graphcore/auth/authz/kernel_policy.py +191 -0
  57. graphcore/auth/authz/kernel_writes.py +31 -0
  58. graphcore/auth/authz/lookups.py +25 -0
  59. graphcore/auth/authz/policy.py +403 -0
  60. graphcore/auth/authz/tests/__init__.py +0 -0
  61. graphcore/auth/authz/tests/test_can_contain.py +150 -0
  62. graphcore/auth/authz/tests/test_contained_by.py +167 -0
  63. graphcore/auth/authz/tests/test_creation.py +391 -0
  64. graphcore/auth/authz/tests/test_delete_permission.py +82 -0
  65. graphcore/auth/authz/tests/test_granted_to_me.py +160 -0
  66. graphcore/auth/authz/tests/test_integration.py +637 -0
  67. graphcore/auth/authz/tests/test_invites.py +184 -0
  68. graphcore/auth/authz/tests/test_kernel_policy.py +140 -0
  69. graphcore/auth/authz/tests/test_lockout.py +132 -0
  70. graphcore/auth/authz/tests/test_policy.py +329 -0
  71. graphcore/auth/authz/tests/test_primitives.py +73 -0
  72. graphcore/auth/authz/tests/test_principal_creation.py +62 -0
  73. graphcore/bootstrap/__init__.py +0 -0
  74. graphcore/bootstrap/deployment.py +67 -0
  75. graphcore/bootstrap/signals.py +16 -0
  76. graphcore/bootstrap/tests/__init__.py +0 -0
  77. graphcore/bootstrap/tests/test_deployment_seed.py +154 -0
  78. graphcore/broadcast/__init__.py +4 -0
  79. graphcore/broadcast/broadcaster.py +201 -0
  80. graphcore/broadcast/channels.py +56 -0
  81. graphcore/broadcast/tests/__init__.py +0 -0
  82. graphcore/broadcast/tests/test_broadcaster.py +444 -0
  83. graphcore/celery.py +14 -0
  84. graphcore/docs/AGENT_FRONTEND_TDD.md +389 -0
  85. graphcore/docs/ARCHITECTURE_PRINCIPLES.md +152 -0
  86. graphcore/docs/AUTHORIZATION_TECHNICAL_TDD.md +1379 -0
  87. graphcore/docs/ENTITY_REFERENCE_TDD.md +561 -0
  88. graphcore/docs/GRAPHAPI_FRONT_DOOR.md +173 -0
  89. graphcore/docs/RUNTIME.md +90 -0
  90. graphcore/docs/SCHEDULING_TDD.md +244 -0
  91. graphcore/docs/STRATEGY_EXECUTION_TDD.md +1043 -0
  92. graphcore/docs/VIEW_ENTITY_TDD.md +522 -0
  93. graphcore/docs/VIEW_SYSTEM_TDD.md +307 -0
  94. graphcore/docs/__init__.py +0 -0
  95. graphcore/docs/doc_topics.py +97 -0
  96. graphcore/docs/docs_discovery.py +64 -0
  97. graphcore/docs/public_api.py +56 -0
  98. graphcore/domain/__init__.py +0 -0
  99. graphcore/domain/entity/Entity.py +204 -0
  100. graphcore/domain/entity/__init__.py +0 -0
  101. graphcore/domain/entity/deployment/Deployment.py +25 -0
  102. graphcore/domain/entity/deployment/__init__.py +0 -0
  103. graphcore/domain/entity/grant/Grant.py +31 -0
  104. graphcore/domain/entity/grant/__init__.py +0 -0
  105. graphcore/domain/entity/group/Group.py +48 -0
  106. graphcore/domain/entity/group/README.md +88 -0
  107. graphcore/domain/entity/group/__init__.py +0 -0
  108. graphcore/domain/entity/invite/Invite.py +31 -0
  109. graphcore/domain/entity/invite/__init__.py +0 -0
  110. graphcore/domain/entity/request_template/RequestTemplate.py +48 -0
  111. graphcore/domain/entity/request_template/__init__.py +0 -0
  112. graphcore/domain/entity/request_template/tests/__init__.py +0 -0
  113. graphcore/domain/entity/request_template/tests/test_request_template.py +52 -0
  114. graphcore/domain/entity/service_account/ServiceAccount.py +41 -0
  115. graphcore/domain/entity/service_account/__init__.py +0 -0
  116. graphcore/domain/entity/strategy_request/StrategyRequest.py +61 -0
  117. graphcore/domain/entity/strategy_request/__init__.py +0 -0
  118. graphcore/domain/entity/strategy_request/tests/__init__.py +0 -0
  119. graphcore/domain/entity/strategy_request/tests/test_strategy_request_entity.py +35 -0
  120. graphcore/domain/entity/tests/__init__.py +0 -0
  121. graphcore/domain/entity/tests/test_entity.py +227 -0
  122. graphcore/domain/entity/tests/test_grant_group_entities.py +36 -0
  123. graphcore/domain/entity/tests/test_write_once_fields.py +98 -0
  124. graphcore/domain/entity/user/User.py +39 -0
  125. graphcore/domain/entity/user/__init__.py +0 -0
  126. graphcore/domain/entity/user/tests/__init__.py +0 -0
  127. graphcore/domain/entity/user/tests/test_user_entity.py +33 -0
  128. graphcore/domain/entity/view/View.py +37 -0
  129. graphcore/domain/entity/view/__init__.py +0 -0
  130. graphcore/domain/entity/view/position.py +78 -0
  131. graphcore/domain/entity/view/tests/__init__.py +0 -0
  132. graphcore/domain/entity/view/tests/test_position.py +71 -0
  133. graphcore/domain/entity/view/tests/test_view_entity.py +134 -0
  134. graphcore/domain/strategy/BaseStrategy.py +759 -0
  135. graphcore/domain/strategy/DeploymentStrategies.py +55 -0
  136. graphcore/domain/strategy/GrantStrategies.py +189 -0
  137. graphcore/domain/strategy/GroupStrategies.py +193 -0
  138. graphcore/domain/strategy/InviteStrategies.py +84 -0
  139. graphcore/domain/strategy/RequestTemplateStrategies.py +126 -0
  140. graphcore/domain/strategy/ViewStrategies.py +219 -0
  141. graphcore/domain/strategy/__init__.py +0 -0
  142. graphcore/domain/strategy/runner.py +84 -0
  143. graphcore/domain/strategy/session.py +56 -0
  144. graphcore/domain/strategy/tests/BaseStrategyTest.py +167 -0
  145. graphcore/domain/strategy/tests/__init__.py +0 -0
  146. graphcore/domain/strategy/tests/test_BaseStrategies.py +870 -0
  147. graphcore/domain/strategy/tests/test_grant_strategies.py +245 -0
  148. graphcore/domain/strategy/tests/test_group_strategies.py +99 -0
  149. graphcore/domain/strategy/tests/test_runner.py +220 -0
  150. graphcore/domain/strategy/tests/test_strategy_locks.py +446 -0
  151. graphcore/domain/strategy/tests/test_utils.py +95 -0
  152. graphcore/domain/strategy/tests/test_view_strategies.py +436 -0
  153. graphcore/errors.py +145 -0
  154. graphcore/execution/README.md +97 -0
  155. graphcore/execution/StrategyExecutorService.py +333 -0
  156. graphcore/execution/__init__.py +0 -0
  157. graphcore/execution/dispatcher.py +19 -0
  158. graphcore/execution/environments/__init__.py +0 -0
  159. graphcore/execution/environments/celery.py +59 -0
  160. graphcore/execution/environments/in_process.py +55 -0
  161. graphcore/execution/local_session.py +117 -0
  162. graphcore/execution/redis_run_events.py +74 -0
  163. graphcore/execution/requests.py +22 -0
  164. graphcore/execution/run_events.py +85 -0
  165. graphcore/execution/strategy_context.py +54 -0
  166. graphcore/execution/tasks.py +34 -0
  167. graphcore/execution/tests/__init__.py +5 -0
  168. graphcore/execution/tests/contract.py +171 -0
  169. graphcore/execution/tests/support.py +310 -0
  170. graphcore/execution/tests/test_dispatchers.py +76 -0
  171. graphcore/execution/tests/test_executor.py +368 -0
  172. graphcore/execution/tests/test_local_session.py +95 -0
  173. graphcore/execution/tests/test_reconcile_runs.py +134 -0
  174. graphcore/execution/tests/test_run_events.py +106 -0
  175. graphcore/execution/tests/test_strategy_context.py +62 -0
  176. graphcore/gateway/__init__.py +1 -0
  177. graphcore/gateway/remote.py +35 -0
  178. graphcore/gateway/tests/__init__.py +0 -0
  179. graphcore/gateway/tests/test_remote.py +30 -0
  180. graphcore/graph/EntityService.py +710 -0
  181. graphcore/graph/__init__.py +0 -0
  182. graphcore/graph/entity_cache.py +152 -0
  183. graphcore/graph/tests/__init__.py +4 -0
  184. graphcore/graph/tests/test_EntityService.py +908 -0
  185. graphcore/graph/tests/test_EntityService_list.py +170 -0
  186. graphcore/graph/tests/test_EntityService_model_resolution.py +339 -0
  187. graphcore/graph/tests/test_EntityService_referrers.py +236 -0
  188. graphcore/graph/tests/test_EntityService_routing.py +119 -0
  189. graphcore/graph/tests/test_dependent_on.py +141 -0
  190. graphcore/graph/tests/test_engine_is_type_agnostic.py +25 -0
  191. graphcore/graph/tests/test_engine_resolves_read_permission.py +93 -0
  192. graphcore/graph/tests/test_entity_cache.py +159 -0
  193. graphcore/graph/tests/test_null_entity_cache.py +12 -0
  194. graphcore/management/__init__.py +0 -0
  195. graphcore/management/commands/__init__.py +0 -0
  196. graphcore/management/commands/dev_reset.py +18 -0
  197. graphcore/management/commands/graphcore_createadmin.py +65 -0
  198. graphcore/management/commands/graphcore_policy.py +59 -0
  199. graphcore/management/commands/graphcore_status.py +98 -0
  200. graphcore/management/tests/__init__.py +0 -0
  201. graphcore/management/tests/test_createadmin.py +78 -0
  202. graphcore/management/tests/test_policy_command.py +46 -0
  203. graphcore/management/tests/test_status_command.py +85 -0
  204. graphcore/migrations/0001_initial.py +186 -0
  205. graphcore/migrations/0002_invite.py +26 -0
  206. graphcore/migrations/__init__.py +0 -0
  207. graphcore/models.py +11 -0
  208. graphcore/persistence/ENTITY_REPOSITORY_TECHNICAL_TDD.md +396 -0
  209. graphcore/persistence/__init__.py +6 -0
  210. graphcore/persistence/django/__init__.py +0 -0
  211. graphcore/persistence/django/backends.py +43 -0
  212. graphcore/persistence/django/bindings.py +28 -0
  213. graphcore/persistence/django/cache.py +38 -0
  214. graphcore/persistence/django/compiler.py +214 -0
  215. graphcore/persistence/django/identity.py +18 -0
  216. graphcore/persistence/django/models.py +252 -0
  217. graphcore/persistence/django/repository.py +525 -0
  218. graphcore/persistence/django/tests/__init__.py +0 -0
  219. graphcore/persistence/django/tests/test_backends.py +68 -0
  220. graphcore/persistence/django/tests/test_compiler.py +269 -0
  221. graphcore/persistence/django/tests/test_repository.py +472 -0
  222. graphcore/persistence/django/tests/test_shared_column.py +120 -0
  223. graphcore/persistence/django/tests/test_threads.py +50 -0
  224. graphcore/persistence/django/tests/test_update_shape.py +175 -0
  225. graphcore/persistence/django/threads.py +38 -0
  226. graphcore/persistence/repository.py +172 -0
  227. graphcore/persistence/tests/__init__.py +0 -0
  228. graphcore/persistence/tests/contract.py +688 -0
  229. graphcore/persistence/tests/evaluate.py +51 -0
  230. graphcore/persistence/tests/fixtures.py +32 -0
  231. graphcore/persistence/tests/memory.py +263 -0
  232. graphcore/persistence/tests/test_memory_repository.py +10 -0
  233. graphcore/persistence/tests/test_query_types.py +127 -0
  234. graphcore/public/__init__.py +9 -0
  235. graphcore/public/api.py +9 -0
  236. graphcore/public/apps.py +10 -0
  237. graphcore/public/auth/__init__.py +1 -0
  238. graphcore/public/auth/grants.py +7 -0
  239. graphcore/public/auth/onboarding.py +8 -0
  240. graphcore/public/auth/perms.py +7 -0
  241. graphcore/public/auth/policy.py +10 -0
  242. graphcore/public/auth/roles.py +8 -0
  243. graphcore/public/auth/subjects.py +7 -0
  244. graphcore/public/entities/__init__.py +32 -0
  245. graphcore/public/errors.py +19 -0
  246. graphcore/public/persistence/__init__.py +44 -0
  247. graphcore/public/persistence/django.py +24 -0
  248. graphcore/public/scheduling.py +23 -0
  249. graphcore/public/strategies/__init__.py +85 -0
  250. graphcore/public/test_settings.py +16 -0
  251. graphcore/public/testing.py +47 -0
  252. graphcore/registry/EntityDirectory.py +246 -0
  253. graphcore/registry/README.md +147 -0
  254. graphcore/registry/StrategyDirectory.py +308 -0
  255. graphcore/registry/TYPE_METADATA_PLACEMENT.md +66 -0
  256. graphcore/registry/__init__.py +0 -0
  257. graphcore/registry/app_discovery.py +108 -0
  258. graphcore/registry/decorators.py +77 -0
  259. graphcore/registry/discovery.py +111 -0
  260. graphcore/registry/registry.py +79 -0
  261. graphcore/registry/tests/__init__.py +0 -0
  262. graphcore/registry/tests/test_agent_frontend_discovery.py +56 -0
  263. graphcore/registry/tests/test_app_config.py +36 -0
  264. graphcore/registry/tests/test_discovery.py +147 -0
  265. graphcore/registry/tests/test_entity_directory_storage.py +73 -0
  266. graphcore/registry/tests/test_import_package.py +87 -0
  267. graphcore/registry/tests/test_principal_flag.py +67 -0
  268. graphcore/registry/tests/test_strategy_directory.py +29 -0
  269. graphcore/routes/__init__.py +116 -0
  270. graphcore/routes/tests/__init__.py +0 -0
  271. graphcore/routes/tests/test_routes.py +90 -0
  272. graphcore/runtime.py +243 -0
  273. graphcore/scheduling/__init__.py +0 -0
  274. graphcore/scheduling/beat_scheduler.py +164 -0
  275. graphcore/scheduling/beat_tasks.py +63 -0
  276. graphcore/scheduling/scheduler.py +37 -0
  277. graphcore/scheduling/tests/__init__.py +0 -0
  278. graphcore/scheduling/tests/contract.py +136 -0
  279. graphcore/scheduling/tests/memory.py +42 -0
  280. graphcore/scheduling/tests/test_beat_scheduler.py +87 -0
  281. graphcore/scheduling/tests/test_memory_scheduler.py +15 -0
  282. graphcore/scheduling/tests/test_ticks.py +143 -0
  283. graphcore/tasks.py +9 -0
  284. graphcore/test_settings.py +78 -0
  285. graphcore/testing/__init__.py +0 -0
  286. graphcore/testing/case.py +112 -0
  287. graphcore/testing/runner.py +32 -0
  288. graphcore/testing/settings.py +26 -0
  289. graphcore/testing/tests/__init__.py +0 -0
  290. graphcore/testing/tests/agent_view_fixture.py +19 -0
  291. graphcore/testing/tests/test_case_and_runner.py +219 -0
  292. graphcore/testing/tests/test_settings_for_apps.py +50 -0
  293. graphcore/tests/__init__.py +0 -0
  294. graphcore/tests/ids.py +14 -0
  295. graphcore/tests/public_api_manifest.py +102 -0
  296. graphcore/tests/runs.py +7 -0
  297. graphcore/tests/test_app_discovery.py +90 -0
  298. graphcore/tests/test_composition_boundaries.py +231 -0
  299. graphcore/tests/test_docs_discovery.py +108 -0
  300. graphcore/tests/test_no_print.py +70 -0
  301. graphcore/tests/test_packaging.py +64 -0
  302. graphcore/tests/test_public_api_boundaries.py +229 -0
  303. graphcore/tests/test_public_api_contract.py +62 -0
  304. graphcore/tests/test_public_api_docs.py +28 -0
  305. graphcore/tests/test_runtime.py +42 -0
  306. graphcore/tests/test_shared_type_boundaries.py +128 -0
  307. graphcore/tests/test_shared_types.py +144 -0
  308. graphcore/tests/test_test_settings.py +45 -0
  309. graphcore/transport/__init__.py +0 -0
  310. graphcore/transport/agent_asgi.py +68 -0
  311. graphcore/transport/agent_client.py +85 -0
  312. graphcore/transport/channel_layer.py +68 -0
  313. graphcore/transport/consumers.py +507 -0
  314. graphcore/transport/routing.py +6 -0
  315. graphcore/transport/tests/__init__.py +0 -0
  316. graphcore/transport/tests/test_agent_client.py +31 -0
  317. graphcore/transport/tests/test_api_views.py +733 -0
  318. graphcore/transport/tests/test_channel_layer.py +96 -0
  319. graphcore/transport/tests/test_consumers.py +963 -0
  320. graphcore/transport/tests/test_resource_metadata.py +67 -0
  321. graphcore/transport/urls.py +29 -0
  322. graphcore/transport/views.py +358 -0
  323. graphcore/types/__init__.py +4 -0
  324. graphcore/types/access.py +114 -0
  325. graphcore/types/api.py +76 -0
  326. graphcore/types/entities.py +100 -0
  327. graphcore/types/entity_id.py +85 -0
  328. graphcore/types/execution.py +41 -0
  329. graphcore/types/identity.py +20 -0
  330. graphcore/types/onboarding.py +20 -0
  331. graphcore/types/query.py +116 -0
  332. graphcore/types/schedules.py +91 -0
  333. graphcore/types/strategy.py +73 -0
  334. graphcore/types/tests/__init__.py +0 -0
  335. graphcore/types/tests/test_entity_id.py +50 -0
  336. graphcore/urls.py +5 -0
  337. graphcore_kernel-0.1.0.dist-info/METADATA +94 -0
  338. graphcore_kernel-0.1.0.dist-info/RECORD +343 -0
  339. graphcore_kernel-0.1.0.dist-info/WHEEL +5 -0
  340. graphcore_kernel-0.1.0.dist-info/entry_points.txt +2 -0
  341. graphcore_kernel-0.1.0.dist-info/licenses/LICENSE +202 -0
  342. graphcore_kernel-0.1.0.dist-info/licenses/NOTICE +2 -0
  343. graphcore_kernel-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,116 @@
1
+ """The route table: every address a client can show, owned by the server."""
2
+ from __future__ import annotations
3
+
4
+ from contextlib import contextmanager
5
+ from dataclasses import dataclass
6
+ from typing import Optional
7
+
8
+ from graphcore.errors import NotFound
9
+ from graphcore.types.entity_id import is_entity_id
10
+
11
+ ENTITY_PARAM = '{entity}'
12
+ KERNEL_APP_ID = 'graphcore'
13
+
14
+
15
+ @dataclass(frozen=True)
16
+ class Route:
17
+ pattern: str
18
+ view_name: str
19
+ app_id: str
20
+
21
+ def __post_init__(self) -> None:
22
+ pattern = self.pattern
23
+ if not pattern.startswith('/') or '|' in pattern or '@' in pattern:
24
+ raise ValueError(f"invalid route pattern {pattern!r}")
25
+ if pattern.count(ENTITY_PARAM) > 1:
26
+ raise ValueError(f"route {pattern!r} has more than one {ENTITY_PARAM}")
27
+ if any('{' in s and s != ENTITY_PARAM for s in pattern.split('/')):
28
+ raise ValueError(f"route {pattern!r} has an unknown parameter")
29
+ if not self.view_name or '|' in self.view_name or '@' in self.view_name:
30
+ raise ValueError(f"invalid view name {self.view_name!r} for route {pattern!r}")
31
+
32
+ @property
33
+ def segments(self) -> list[str]:
34
+ return [s for s in self.pattern.split('/') if s]
35
+
36
+
37
+ @dataclass(frozen=True)
38
+ class Resolved:
39
+ view_name: str
40
+ entity_id: Optional[str] = None
41
+
42
+
43
+ def _slots(route: Route) -> list[Optional[str]]:
44
+ """One slot per address segment; None is a wildcard (`{entity}` takes two)."""
45
+ out: list[Optional[str]] = []
46
+ for segment in route.segments:
47
+ out.extend([None, None] if segment == ENTITY_PARAM else [segment])
48
+ return out
49
+
50
+
51
+ def _collides(a: Route, b: Route) -> bool:
52
+ sa, sb = _slots(a), _slots(b)
53
+ return len(sa) == len(sb) and all(x is None or y is None or x == y for x, y in zip(sa, sb))
54
+
55
+
56
+ class RouteTable:
57
+ def __init__(self) -> None:
58
+ self._routes: list[Route] = []
59
+ self._declaring_app_id = KERNEL_APP_ID
60
+ self._closed = False
61
+ self.add('/', 'home')
62
+ self.add('/e/{entity}', 'entity')
63
+
64
+ @contextmanager
65
+ def declaring_app(self, app_id: str):
66
+ previous, self._declaring_app_id = self._declaring_app_id, app_id
67
+ try:
68
+ yield
69
+ finally:
70
+ self._declaring_app_id = previous
71
+
72
+ def add(self, pattern: str, view_name: str) -> None:
73
+ if self._closed:
74
+ raise RuntimeError('the route table is finalized')
75
+ self._routes.append(Route(pattern, view_name, self._declaring_app_id))
76
+
77
+ def finalize(self) -> None:
78
+ for i, a in enumerate(self._routes):
79
+ for b in self._routes[i + 1:]:
80
+ if _collides(a, b):
81
+ raise ValueError(
82
+ f"routes {a.pattern!r} ({a.app_id}) and {b.pattern!r} ({b.app_id}) "
83
+ "can match the same address"
84
+ )
85
+ self._closed = True
86
+
87
+ def all(self) -> list[Route]:
88
+ return list(self._routes)
89
+
90
+ def resolve(self, address: str) -> Resolved:
91
+ parts = [s for s in address.split('/') if s]
92
+ for route in self._routes:
93
+ entity_id = self._match(route.segments, parts)
94
+ if entity_id is not False:
95
+ return Resolved(route.view_name, entity_id)
96
+ raise NotFound(f"no route matches {address!r}")
97
+
98
+ @staticmethod
99
+ def _match(pattern: list[str], parts: list[str]):
100
+ """The matched entity id (or None), or False for no match."""
101
+ entity_id = None
102
+ i = 0
103
+ for segment in pattern:
104
+ if segment == ENTITY_PARAM:
105
+ if i + 2 > len(parts):
106
+ return False
107
+ candidate = f'gc://{parts[i]}/{parts[i + 1]}'
108
+ if not is_entity_id(candidate):
109
+ return False
110
+ entity_id = candidate
111
+ i += 2
112
+ else:
113
+ if i >= len(parts) or parts[i] != segment:
114
+ return False
115
+ i += 1
116
+ return entity_id if i == len(parts) else False
File without changes
@@ -0,0 +1,90 @@
1
+ import json
2
+
3
+ from django.test import RequestFactory, SimpleTestCase, TestCase
4
+
5
+ from graphcore.errors import NotFound
6
+ from graphcore.routes import Route, RouteTable
7
+ from graphcore.transport import views
8
+
9
+ BACKEND, LOCAL = 'a' * 10, 'b' * 12
10
+ ENTITY = f'gc://{BACKEND}/{LOCAL}'
11
+
12
+
13
+ class RouteTableTests(SimpleTestCase):
14
+ def setUp(self):
15
+ self.table = RouteTable()
16
+
17
+ def test_kernel_routes_resolve(self):
18
+ self.assertEqual(self.table.resolve('/').view_name, 'home')
19
+ resolved = self.table.resolve(f'/e/{BACKEND}/{LOCAL}')
20
+ self.assertEqual((resolved.view_name, resolved.entity_id), ('entity', ENTITY))
21
+
22
+ def test_unknown_address_is_not_found(self):
23
+ with self.assertRaises(NotFound):
24
+ self.table.resolve('/nope')
25
+ with self.assertRaises(NotFound):
26
+ self.table.resolve('/e/not-an-id')
27
+
28
+ def test_app_routes_resolve(self):
29
+ with self.table.declaring_app('stacks'):
30
+ self.table.add('/stacks/{entity}/history', 'stacks.history')
31
+ self.table.add('/stacks/review-queue', 'stacks.review_queue')
32
+ resolved = self.table.resolve(f'/stacks/{BACKEND}/{LOCAL}/history')
33
+ self.assertEqual((resolved.view_name, resolved.entity_id), ('stacks.history', ENTITY))
34
+ self.assertIsNone(self.table.resolve('/stacks/review-queue').entity_id)
35
+
36
+ def test_same_path_from_two_apps_names_both(self):
37
+ with self.table.declaring_app('one'):
38
+ self.table.add('/notes', 'one.notes')
39
+ with self.table.declaring_app('two'):
40
+ self.table.add('/notes', 'two.notes')
41
+ with self.assertRaisesRegex(ValueError, r"\(one\).*\(two\)"):
42
+ self.table.finalize()
43
+
44
+ def test_wildcard_and_literal_collide_only_when_widths_match(self):
45
+ with self.table.declaring_app('one'):
46
+ self.table.add('/x/{entity}', 'one.x')
47
+ self.table.add('/x/a', 'one.a')
48
+ self.table.finalize()
49
+
50
+ other = RouteTable()
51
+ with other.declaring_app('one'):
52
+ other.add('/x/{entity}', 'one.x')
53
+ other.add('/x/a/b', 'one.ab')
54
+ with self.assertRaises(ValueError):
55
+ other.finalize()
56
+
57
+ def test_app_route_cannot_shadow_kernel_entity_route(self):
58
+ with self.table.declaring_app('one'):
59
+ self.table.add('/e/a/b', 'one.shadow')
60
+ with self.assertRaisesRegex(ValueError, 'graphcore'):
61
+ self.table.finalize()
62
+
63
+ def test_invalid_patterns(self):
64
+ for pattern in ('no-slash', '/a|b', '/a@b', '/{entity}/{entity}', '/{other}'):
65
+ with self.assertRaises(ValueError, msg=pattern):
66
+ Route(pattern, 'v', 'app')
67
+ with self.assertRaises(ValueError):
68
+ Route('/ok', 'a|b', 'app')
69
+
70
+ def test_add_after_finalize_raises(self):
71
+ self.table.finalize()
72
+ with self.assertRaises(RuntimeError):
73
+ self.table.add('/late', 'v')
74
+
75
+
76
+ class RouteEndpointTests(TestCase):
77
+ async def test_routes_lists_kernel_routes(self):
78
+ response = await views.api_get_routes(RequestFactory().get('/'))
79
+ patterns = {r['pattern'] for r in json.loads(response.content)['routes']}
80
+ self.assertTrue({'/', '/e/{entity}'} <= patterns)
81
+
82
+ async def test_resolve_returns_entity(self):
83
+ response = await views.api_resolve(RequestFactory().get('/', {'address': f'/e/{BACKEND}/{LOCAL}'}))
84
+ self.assertEqual(
85
+ json.loads(response.content), {'view_name': 'entity', 'entity_id': ENTITY},
86
+ )
87
+
88
+ async def test_resolve_unknown_is_404(self):
89
+ response = await views.api_resolve(RequestFactory().get('/', {'address': '/nope'}))
90
+ self.assertEqual(response.status_code, 404)
graphcore/runtime.py ADDED
@@ -0,0 +1,243 @@
1
+ """The kernel's composition root.
2
+
3
+ `build_runtime` wires an already-constructed repository/cache/access/
4
+ broadcaster/scheduler into the service graph - it has no Django dependency
5
+ and is the extension point for any deployment. `django_runtime` is the
6
+ shipped default: the one place a concrete Django/Channels/Beat class is named or
7
+ `django.conf.settings` is read for these components. A deployment that needs
8
+ different components (a different access adapter, a non-Django repository,
9
+ ...) points `GRAPHCORE_RUNTIME_FACTORY` at its own
10
+ `(registry, *, worker) -> Runtime` function instead of patching this module. `get_runtime()` is the lazy,
11
+ process-wide instance; call it only from a process entrypoint (HTTP view, WS
12
+ consumer, Celery task, authn, `dev_reset`), never at import time and never
13
+ from `graph/`, `execution/`, `auth/authz/`, `broadcast/`, `persistence/`, or
14
+ `domain/` - the factory is resolved lazily on first use in each process so
15
+ Celery's prefork model never shares connections across forked workers.
16
+ """
17
+ from __future__ import annotations
18
+
19
+ from dataclasses import dataclass
20
+ from typing import Callable, Mapping, Optional
21
+
22
+ from graphcore.api.graph_api import GraphAPI
23
+ from graphcore.auth.authz.access_port import AccessPort
24
+ from graphcore.broadcast.broadcaster import Broadcaster
25
+ from graphcore.errors import ConfigurationError
26
+ from graphcore.execution.StrategyExecutorService import StrategyExecutorService
27
+ from graphcore.execution.dispatcher import Dispatcher
28
+ from graphcore.execution.run_events import InMemoryRunEvents, RunEvents
29
+ from graphcore.gateway.remote import RemoteGateway
30
+ from graphcore.graph.EntityService import EntityService
31
+ from graphcore.graph.entity_cache import EntityCache
32
+ from graphcore.persistence.repository import EntityRepository
33
+ from graphcore.registry.registry import DEFAULT_REGISTRY, Registry
34
+ from graphcore.scheduling.scheduler import Scheduler
35
+ from graphcore.types.strategy import Workload
36
+
37
+ DispatcherFactory = Callable[[StrategyExecutorService, EntityService, Registry], Mapping[Workload, Dispatcher]]
38
+
39
+
40
+ @dataclass(frozen=True)
41
+ class Runtime:
42
+ """The live service graph for one process (or one test)."""
43
+
44
+ registry: Registry
45
+ repository: EntityRepository
46
+ cache: EntityCache
47
+ access: AccessPort
48
+ broadcaster: Broadcaster
49
+ scheduler: Scheduler
50
+ entity_service: EntityService
51
+ executor: StrategyExecutorService
52
+ graph_api: GraphAPI
53
+ gateway: RemoteGateway
54
+ backend_id: str
55
+ handle: str | None
56
+
57
+ @property
58
+ def deployment_entity_id(self) -> str:
59
+ return self.entity_service.deployment_entity_id
60
+
61
+
62
+ def build_runtime(
63
+ registry: Registry,
64
+ repository: EntityRepository,
65
+ cache: EntityCache,
66
+ access: AccessPort,
67
+ broadcaster: Broadcaster,
68
+ scheduler: Scheduler,
69
+ gateway: RemoteGateway,
70
+ max_batch_size: int,
71
+ handle: str | None = None,
72
+ dispatchers: Optional[DispatcherFactory] = None,
73
+ default_timeout: int = 600,
74
+ max_attempts: int = 6,
75
+ run_events: Optional[RunEvents] = None,
76
+ dispatch_timeout: int = 300,
77
+ ) -> Runtime:
78
+ """Wire already-constructed components into the service graph.
79
+ `dispatchers` builds the workload-to-dispatcher mapping once the executor
80
+ exists, since an in-process environment reports back to it; it defaults
81
+ to `default_dispatchers`. `run_events` defaults to `InMemoryRunEvents`,
82
+ which suffices only when every run finishes in the process waiting on
83
+ it."""
84
+ backend_id = repository.backend_id()
85
+ entity_service = EntityService(
86
+ registry, repository, cache, access, broadcaster, scheduler, backend_id, gateway,
87
+ )
88
+ executor = StrategyExecutorService(
89
+ registry, entity_service, default_timeout=default_timeout, max_attempts=max_attempts,
90
+ run_events=run_events or InMemoryRunEvents(), dispatch_timeout=dispatch_timeout,
91
+ )
92
+ executor.bind_dispatchers((dispatchers or default_dispatchers)(executor, entity_service, registry))
93
+ graph_api = GraphAPI(registry, entity_service, executor, max_batch_size)
94
+ return Runtime(
95
+ registry, repository, cache, access, broadcaster, scheduler, entity_service, executor,
96
+ graph_api, gateway, backend_id, handle,
97
+ )
98
+
99
+
100
+ def default_dispatchers(executor, entity_service, registry) -> dict[Workload, Dispatcher]:
101
+ """IO on this process's event loop, CPU in a Celery worker."""
102
+ from graphcore.execution.environments.celery import CeleryDispatcher
103
+ from graphcore.execution.environments.in_process import InProcessDispatcher
104
+
105
+ return {
106
+ Workload.IO: InProcessDispatcher(executor, entity_service, registry),
107
+ Workload.CPU: CeleryDispatcher(),
108
+ }
109
+
110
+
111
+ def worker_dispatchers(executor, entity_service, registry) -> dict[Workload, Dispatcher]:
112
+ """A Celery worker's: it is the CPU environment, so CPU runs on its own
113
+ loop beside IO, and a CPU strategy's nested CPU request never waits on a
114
+ second worker slot."""
115
+ from graphcore.execution.environments.in_process import InProcessDispatcher
116
+
117
+ in_process = InProcessDispatcher(executor, entity_service, registry)
118
+ return {Workload.IO: in_process, Workload.CPU: in_process}
119
+
120
+
121
+ def _run_events(broker_url: str) -> RunEvents:
122
+ """Runs finish in the Celery process, so waiters elsewhere hear of them
123
+ over the broker Celery already needs. An in-memory broker runs tasks in
124
+ this process."""
125
+ from django.core.exceptions import ImproperlyConfigured
126
+
127
+ if broker_url.startswith('memory://'):
128
+ return InMemoryRunEvents()
129
+ if broker_url.startswith(('redis://', 'rediss://')):
130
+ from graphcore.execution.redis_run_events import RedisRunEvents
131
+
132
+ return RedisRunEvents(broker_url)
133
+ raise ImproperlyConfigured(
134
+ f"No RunEvents adapter for CELERY_BROKER_URL {broker_url.split('://')[0]}://; "
135
+ "use a Redis broker or a runtime factory that passes run_events"
136
+ )
137
+
138
+
139
+ def _channel_layer():
140
+ from channels.layers import get_channel_layer
141
+
142
+ try:
143
+ return get_channel_layer()
144
+ except Exception:
145
+ return None
146
+
147
+
148
+ def django_runtime(registry: Registry, *, worker: bool = False) -> Runtime:
149
+ """The shipped Django/Channels/Redis-backed runtime. Settings read here
150
+ are the only place this default deployment changes without touching call
151
+ sites. A deployment wanting different components entirely - not just
152
+ different settings values - writes its own factory and points
153
+ `GRAPHCORE_RUNTIME_FACTORY` at it instead of changing this function.
154
+ `worker` selects `worker_dispatchers` over `default_dispatchers`."""
155
+ from django.conf import settings
156
+
157
+ from graphcore.persistence.django.cache import DjangoCache
158
+
159
+ cache = DjangoCache(entity_ttl=getattr(settings, 'GRAPHCORE_ENTITY_CACHE_TTL', 900))
160
+ return _django_components(
161
+ registry, cache, _channel_layer(), worker_dispatchers if worker else default_dispatchers,
162
+ )
163
+
164
+
165
+ def bootstrap_runtime() -> Runtime:
166
+ """The Django store alone - no cache server, no channel layer - for
167
+ writes made while migrating, when neither need be reachable (the
168
+ compose `migrate` service has only the database)."""
169
+ from graphcore.graph.entity_cache import NullEntityCache
170
+
171
+ return _django_components(DEFAULT_REGISTRY, NullEntityCache(), None, default_dispatchers)
172
+
173
+
174
+ def _django_components(registry: Registry, cache, channel_layer, dispatchers: DispatcherFactory) -> Runtime:
175
+ from django.conf import settings
176
+
177
+ from graphcore.auth.authz.adapter.default_access_adapter import DefaultAccessAdapter
178
+ from graphcore.broadcast.channels import ChannelsBroadcaster
179
+ from graphcore.gateway.remote import NoRemoteGateway
180
+ from graphcore.persistence.django.repository import DjangoEntityRepository
181
+ from graphcore.scheduling.beat_scheduler import BeatScheduler
182
+
183
+ repository = DjangoEntityRepository(
184
+ registry.entities, db_threads=int(getattr(settings, 'GRAPHCORE_DB_THREADS', 4)),
185
+ )
186
+ access = DefaultAccessAdapter(registry, repository, cache)
187
+ broadcaster = ChannelsBroadcaster(access, registry, channel_layer)
188
+ reconcile_interval = int(getattr(settings, 'GRAPHCORE_RECONCILE_INTERVAL', 300))
189
+ scheduler = BeatScheduler(
190
+ repository, reconcile_interval=reconcile_interval, run_reconcile_interval=reconcile_interval,
191
+ )
192
+ gateway = NoRemoteGateway()
193
+ max_batch_size = getattr(settings, 'GRAPHCORE_MAX_BATCH_SIZE', 100)
194
+ handle = getattr(settings, 'GRAPHCORE_HANDLE', None)
195
+ return build_runtime(
196
+ registry, repository, cache, access, broadcaster, scheduler, gateway, max_batch_size,
197
+ handle,
198
+ dispatchers=dispatchers,
199
+ default_timeout=int(getattr(settings, 'GRAPHCORE_EXECUTION_DEFAULT_TIMEOUT', 600)),
200
+ max_attempts=int(getattr(settings, 'GRAPHCORE_EXECUTION_MAX_ATTEMPTS', 6)),
201
+ dispatch_timeout=int(getattr(settings, 'GRAPHCORE_EXECUTION_DISPATCH_TIMEOUT', 300)),
202
+ run_events=_run_events(getattr(settings, 'CELERY_BROKER_URL', 'memory://')),
203
+ )
204
+
205
+ _runtime: Runtime | None = None
206
+ _worker = False
207
+
208
+
209
+ def serve_as_worker() -> None:
210
+ """Called when a worker process starts, before its runtime is built:
211
+ `get_runtime` then builds it with `worker=True`."""
212
+ global _worker
213
+ if _runtime is not None:
214
+ raise ConfigurationError("serve_as_worker() after this process's runtime was built")
215
+ _worker = True
216
+
217
+
218
+ def configured_runtime(*, worker: bool = False) -> Runtime:
219
+ """A new `Runtime` from `GRAPHCORE_RUNTIME_FACTORY` (default
220
+ `django_runtime`). `get_runtime` keeps one per process; a test harness
221
+ builds one per test."""
222
+ from django.conf import settings
223
+ from django.utils.module_loading import import_string
224
+
225
+ factory = import_string(getattr(
226
+ settings, 'GRAPHCORE_RUNTIME_FACTORY', 'graphcore.runtime.django_runtime',
227
+ ))
228
+ return factory(DEFAULT_REGISTRY, worker=worker)
229
+
230
+
231
+ def get_runtime() -> Runtime:
232
+ """The process-wide `Runtime`, built on first use by
233
+ `configured_runtime`. Entry points only."""
234
+ global _runtime
235
+ if _runtime is None:
236
+ _runtime = configured_runtime(worker=_worker)
237
+ return _runtime
238
+
239
+
240
+ __all__ = [
241
+ 'Runtime', 'bootstrap_runtime', 'build_runtime', 'configured_runtime', 'default_dispatchers',
242
+ 'django_runtime', 'get_runtime', 'serve_as_worker', 'worker_dispatchers',
243
+ ]
File without changes
@@ -0,0 +1,164 @@
1
+ """`Scheduler` over django-celery-beat's database scheduler.
2
+
3
+ With `beat_tasks.py`, the only module that imports `django_celery_beat`. A
4
+ held schedule is the `PeriodicTask` named after its template, so lookup and
5
+ idempotency need no stored row id. Depends on `RequestTemplate`, the kernel
6
+ type that declares schedules.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ import logging
12
+ from typing import Iterable, Optional
13
+
14
+ from asgiref.sync import sync_to_async
15
+
16
+ from graphcore.domain.entity.request_template.RequestTemplate import RequestTemplate
17
+ from graphcore.scheduling.scheduler import Scheduler
18
+ from graphcore.types.query import Equal
19
+ from graphcore.types.schedules import HeldTask, ScheduleInfo, ScheduleSpec
20
+
21
+ logger = logging.getLogger(__name__)
22
+
23
+ TICK_TASK = 'graphcore.tasks.scheduled_tick'
24
+ RECONCILE_SCHEDULES_TASK = 'graphcore.tasks.reconcile_schedules'
25
+ RECONCILE_RUNS_TASK = 'graphcore.tasks.reconcile_runs'
26
+ PREFIX = 'scheduled_strategy_'
27
+
28
+ # Keeps each DELETE under SQLite's bound-parameter limit.
29
+ REMOVE_BATCH = 500
30
+
31
+
32
+ def task_name(target_id: str) -> str:
33
+ return f"{PREFIX}{target_id}"
34
+
35
+
36
+ class BeatScheduler(Scheduler):
37
+ """Reads the repository and never writes through it."""
38
+
39
+ def __init__(self, repository, reconcile_interval: int = 300, run_reconcile_interval: int = 300):
40
+ self.repository = repository
41
+ self.reconcile_interval = reconcile_interval
42
+ self.run_reconcile_interval = run_reconcile_interval
43
+
44
+ async def entity_saved(self, entity, previous) -> None:
45
+ if not isinstance(entity, RequestTemplate):
46
+ return
47
+ spec = entity.schedule_spec()
48
+ if isinstance(previous, RequestTemplate) and previous.schedule_spec() == spec:
49
+ return
50
+ try:
51
+ if spec is None:
52
+ await self._remove([entity.entity_id])
53
+ else:
54
+ await sync_to_async(_write)(spec)
55
+ except Exception:
56
+ logger.exception("Could not apply the schedule of template %s", entity.entity_id)
57
+
58
+ async def entities_deleted(self, entity_ids: list[str]) -> None:
59
+ try:
60
+ await self._remove(entity_ids)
61
+ except Exception:
62
+ logger.exception("Could not remove schedules for %d deleted entities", len(entity_ids))
63
+
64
+ async def info(self, target_id: str) -> Optional[ScheduleInfo]:
65
+ return await sync_to_async(_info)(target_id)
66
+
67
+ async def held(self) -> list[HeldTask]:
68
+ return await sync_to_async(_held)()
69
+
70
+ async def reconcile(self) -> None:
71
+ """The full convergence: write every desired schedule, delete every
72
+ held one whose template is gone or unscheduled, and ensure the
73
+ periodic reconcile tasks themselves."""
74
+ ids = await self.repository.list_ids(Equal('entity_type_id', RequestTemplate.get_entity_type_id()))
75
+ templates = (await self.repository.load_many(ids)).values()
76
+ specs = [spec for spec in (t.schedule_spec() for t in templates) if spec is not None]
77
+ await sync_to_async(_converge)(specs, self.reconcile_interval, self.run_reconcile_interval)
78
+
79
+ async def _remove(self, target_ids: Iterable[str]) -> None:
80
+ from django_celery_beat.models import PeriodicTask
81
+
82
+ names = [task_name(target_id) for target_id in target_ids]
83
+ for start in range(0, len(names), REMOVE_BATCH):
84
+ await PeriodicTask.objects.filter(name__in=names[start:start + REMOVE_BATCH]).adelete()
85
+
86
+
87
+ def _write(spec: ScheduleSpec) -> None:
88
+ from django_celery_beat.models import CrontabSchedule, PeriodicTask
89
+
90
+ minute, hour, day_of_month, month_of_year, day_of_week = spec.cron_fields
91
+ crontab, _ = CrontabSchedule.objects.get_or_create(
92
+ minute=minute, hour=hour, day_of_month=day_of_month,
93
+ month_of_year=month_of_year, day_of_week=day_of_week,
94
+ )
95
+ PeriodicTask.objects.update_or_create(
96
+ name=task_name(spec.target_id),
97
+ defaults={
98
+ 'crontab': crontab,
99
+ 'interval': None,
100
+ 'task': TICK_TASK,
101
+ 'args': json.dumps([spec.target_id, spec.principal_id]),
102
+ 'kwargs': json.dumps({}),
103
+ 'enabled': spec.enabled,
104
+ },
105
+ )
106
+
107
+
108
+ def _ensure_every(name: str, task: str, seconds: int) -> None:
109
+ from django_celery_beat.models import IntervalSchedule, PeriodicTask
110
+
111
+ interval, _ = IntervalSchedule.objects.get_or_create(every=seconds, period=IntervalSchedule.SECONDS)
112
+ PeriodicTask.objects.update_or_create(
113
+ name=name, defaults={'interval': interval, 'crontab': None, 'task': task, 'enabled': True},
114
+ )
115
+
116
+
117
+ def _converge(specs: list[ScheduleSpec], reconcile_interval: int, run_reconcile_interval: int) -> None:
118
+ from django_celery_beat.models import PeriodicTask
119
+
120
+ for spec in specs:
121
+ _write(spec)
122
+ desired = {task_name(spec.target_id) for spec in specs}
123
+ PeriodicTask.objects.filter(name__startswith=PREFIX).exclude(name__in=desired).delete()
124
+ _ensure_every('graphcore_reconcile_schedules', RECONCILE_SCHEDULES_TASK, reconcile_interval)
125
+ _ensure_every('graphcore_reconcile_runs', RECONCILE_RUNS_TASK, run_reconcile_interval)
126
+
127
+
128
+ def _info(target_id: str) -> Optional[ScheduleInfo]:
129
+ from django_celery_beat.models import PeriodicTask
130
+
131
+ task = PeriodicTask.objects.select_related('crontab').filter(name=task_name(target_id)).first()
132
+ if task is None or task.crontab is None:
133
+ return None
134
+ crontab = task.crontab
135
+ return ScheduleInfo(
136
+ target_id=target_id,
137
+ cron=' '.join((crontab.minute, crontab.hour, crontab.day_of_month,
138
+ crontab.month_of_year, crontab.day_of_week)),
139
+ enabled=task.enabled,
140
+ last_run_at=task.last_run_at,
141
+ run_count=task.total_run_count,
142
+ )
143
+
144
+
145
+ def _held() -> list[HeldTask]:
146
+ from django_celery_beat.models import PeriodicTask
147
+
148
+ held = []
149
+ for task in PeriodicTask.objects.select_related('crontab', 'interval').order_by('name'):
150
+ every = None
151
+ if task.crontab is not None:
152
+ c = task.crontab
153
+ schedule = ' '.join((c.minute, c.hour, c.day_of_month, c.month_of_year, c.day_of_week))
154
+ elif task.interval is not None:
155
+ every = int(task.interval.schedule.run_every.total_seconds())
156
+ schedule = f'every {every}s'
157
+ else:
158
+ schedule = '(other)'
159
+ held.append(HeldTask(
160
+ name=task.name, task=task.task, schedule=schedule, enabled=task.enabled,
161
+ args=tuple(json.loads(task.args or '[]')), last_run_at=task.last_run_at,
162
+ every_seconds=every,
163
+ ))
164
+ return held
@@ -0,0 +1,63 @@
1
+ """`BeatScheduler`'s process entry points: the tick and the full reconcile.
2
+ Each resolves the Celery process's runtime."""
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+
7
+ from asgiref.sync import async_to_sync
8
+ from celery import shared_task
9
+ from celery.signals import beat_init, worker_ready
10
+
11
+ from graphcore.domain.entity.request_template.RequestTemplate import RequestTemplate
12
+ from graphcore.errors import KernelError
13
+ from graphcore.execution.environments.celery import settle
14
+ from graphcore.execution.requests import new_strategy_request
15
+ from graphcore.runtime import get_runtime
16
+ from graphcore.types.identity import Principal
17
+
18
+ logger = logging.getLogger(__name__)
19
+
20
+
21
+ async def tick(entity_service, executor, template_id: str, principal_id: str) -> None:
22
+ """The port's tick contract: load the template as its owner, skip it if
23
+ missing, unscheduled, or disabled, else submit a request made from it."""
24
+ try:
25
+ template = await entity_service.get_entity(template_id, Principal(principal_entity_id=principal_id))
26
+ except KernelError as exc:
27
+ logger.info("Skipping tick for template %s: %s", template_id, exc)
28
+ return
29
+ if not isinstance(template, RequestTemplate) or template.schedule_spec() is None or not template.enabled:
30
+ logger.info("Skipping tick for template %s: not scheduled", template_id)
31
+ return
32
+ # TODO: a random request id per tick means two Beats run each slot twice;
33
+ # derive it from (template, slot) so the second tick is refused.
34
+ request = new_strategy_request(entity_service, template.to_payload(), template_id=template.entity_id)
35
+ try:
36
+ await executor.execute_request(request, Principal(principal_entity_id=template.owner), wait=False)
37
+ except KernelError as exc:
38
+ logger.info("Tick for template %s refused: %s", template_id, exc)
39
+
40
+
41
+ @shared_task(name='graphcore.tasks.scheduled_tick')
42
+ def scheduled_tick(template_id: str, principal_id: str) -> None:
43
+ rt = get_runtime()
44
+
45
+ async def run():
46
+ await tick(rt.entity_service, rt.executor, template_id, principal_id)
47
+ await settle(rt.executor)
48
+
49
+ async_to_sync(run)()
50
+
51
+
52
+ @shared_task(name='graphcore.tasks.reconcile_schedules')
53
+ def reconcile_schedules() -> None:
54
+ async_to_sync(get_runtime().scheduler.reconcile)()
55
+
56
+
57
+ @beat_init.connect
58
+ @worker_ready.connect
59
+ def _reconcile_on_start(**kwargs) -> None:
60
+ try:
61
+ reconcile_schedules()
62
+ except Exception:
63
+ logger.exception("Could not reconcile schedules at startup")