graphcore-kernel 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- graphcore/__init__.py +5 -0
- graphcore/agent_frontend/__init__.py +8 -0
- graphcore/agent_frontend/api.py +69 -0
- graphcore/agent_frontend/base/__init__.py +11 -0
- graphcore/agent_frontend/base/views.py +167 -0
- graphcore/agent_frontend/client.py +31 -0
- graphcore/agent_frontend/context.py +157 -0
- graphcore/agent_frontend/errors.py +12 -0
- graphcore/agent_frontend/loading.py +54 -0
- graphcore/agent_frontend/registry.py +42 -0
- graphcore/agent_frontend/render.py +207 -0
- graphcore/agent_frontend/server.py +103 -0
- graphcore/agent_frontend/signatures.py +35 -0
- graphcore/agent_frontend/testing.py +45 -0
- graphcore/agent_frontend/tests/__init__.py +0 -0
- graphcore/agent_frontend/tests/fakes.py +114 -0
- graphcore/agent_frontend/tests/test_frontend.py +447 -0
- graphcore/agent_frontend/tests/test_loading.py +83 -0
- graphcore/agent_frontend/tests/test_server.py +109 -0
- graphcore/agent_frontend/tools.py +137 -0
- graphcore/api/__init__.py +4 -0
- graphcore/api/graph_api.py +378 -0
- graphcore/api/predicate_codec.py +81 -0
- graphcore/api/tests/__init__.py +0 -0
- graphcore/api/tests/test_changed_entities.py +109 -0
- graphcore/api/tests/test_graph_api.py +290 -0
- graphcore/api/tests/test_graph_api_access.py +94 -0
- graphcore/api/tests/test_graph_api_grants.py +90 -0
- graphcore/api/tests/test_predicate_codec.py +83 -0
- graphcore/api/tests/test_subtree_query.py +79 -0
- graphcore/apps.py +71 -0
- graphcore/auth/__init__.py +1 -0
- graphcore/auth/authn/README.md +178 -0
- graphcore/auth/authn/__init__.py +3 -0
- graphcore/auth/authn/authenticator.py +120 -0
- graphcore/auth/authn/onboarding.py +105 -0
- graphcore/auth/authn/principals.py +20 -0
- graphcore/auth/authn/resource_metadata.py +43 -0
- graphcore/auth/authn/service_accounts.py +96 -0
- graphcore/auth/authn/testing.py +96 -0
- graphcore/auth/authn/tests/__init__.py +0 -0
- graphcore/auth/authn/tests/test_authenticator.py +145 -0
- graphcore/auth/authn/tests/test_onboarding.py +145 -0
- graphcore/auth/authn/tests/test_service_accounts.py +180 -0
- graphcore/auth/authn/users.py +59 -0
- graphcore/auth/authz/__init__.py +0 -0
- graphcore/auth/authz/access_port.py +120 -0
- graphcore/auth/authz/adapter/__init__.py +0 -0
- graphcore/auth/authz/adapter/default_access_adapter.py +556 -0
- graphcore/auth/authz/adapter/tests/__init__.py +0 -0
- graphcore/auth/authz/adapter/tests/test_default_adapter.py +543 -0
- graphcore/auth/authz/adapter/tests/test_entity_write_hooks.py +207 -0
- graphcore/auth/authz/adapter/tests/test_group_types.py +70 -0
- graphcore/auth/authz/adapter/tests/test_sync_support.py +209 -0
- graphcore/auth/authz/defaults.py +26 -0
- graphcore/auth/authz/kernel_policy.py +191 -0
- graphcore/auth/authz/kernel_writes.py +31 -0
- graphcore/auth/authz/lookups.py +25 -0
- graphcore/auth/authz/policy.py +403 -0
- graphcore/auth/authz/tests/__init__.py +0 -0
- graphcore/auth/authz/tests/test_can_contain.py +150 -0
- graphcore/auth/authz/tests/test_contained_by.py +167 -0
- graphcore/auth/authz/tests/test_creation.py +391 -0
- graphcore/auth/authz/tests/test_delete_permission.py +82 -0
- graphcore/auth/authz/tests/test_granted_to_me.py +160 -0
- graphcore/auth/authz/tests/test_integration.py +637 -0
- graphcore/auth/authz/tests/test_invites.py +184 -0
- graphcore/auth/authz/tests/test_kernel_policy.py +140 -0
- graphcore/auth/authz/tests/test_lockout.py +132 -0
- graphcore/auth/authz/tests/test_policy.py +329 -0
- graphcore/auth/authz/tests/test_primitives.py +73 -0
- graphcore/auth/authz/tests/test_principal_creation.py +62 -0
- graphcore/bootstrap/__init__.py +0 -0
- graphcore/bootstrap/deployment.py +67 -0
- graphcore/bootstrap/signals.py +16 -0
- graphcore/bootstrap/tests/__init__.py +0 -0
- graphcore/bootstrap/tests/test_deployment_seed.py +154 -0
- graphcore/broadcast/__init__.py +4 -0
- graphcore/broadcast/broadcaster.py +201 -0
- graphcore/broadcast/channels.py +56 -0
- graphcore/broadcast/tests/__init__.py +0 -0
- graphcore/broadcast/tests/test_broadcaster.py +444 -0
- graphcore/celery.py +14 -0
- graphcore/docs/AGENT_FRONTEND_TDD.md +389 -0
- graphcore/docs/ARCHITECTURE_PRINCIPLES.md +152 -0
- graphcore/docs/AUTHORIZATION_TECHNICAL_TDD.md +1379 -0
- graphcore/docs/ENTITY_REFERENCE_TDD.md +561 -0
- graphcore/docs/GRAPHAPI_FRONT_DOOR.md +173 -0
- graphcore/docs/RUNTIME.md +90 -0
- graphcore/docs/SCHEDULING_TDD.md +244 -0
- graphcore/docs/STRATEGY_EXECUTION_TDD.md +1043 -0
- graphcore/docs/VIEW_ENTITY_TDD.md +522 -0
- graphcore/docs/VIEW_SYSTEM_TDD.md +307 -0
- graphcore/docs/__init__.py +0 -0
- graphcore/docs/doc_topics.py +97 -0
- graphcore/docs/docs_discovery.py +64 -0
- graphcore/docs/public_api.py +56 -0
- graphcore/domain/__init__.py +0 -0
- graphcore/domain/entity/Entity.py +204 -0
- graphcore/domain/entity/__init__.py +0 -0
- graphcore/domain/entity/deployment/Deployment.py +25 -0
- graphcore/domain/entity/deployment/__init__.py +0 -0
- graphcore/domain/entity/grant/Grant.py +31 -0
- graphcore/domain/entity/grant/__init__.py +0 -0
- graphcore/domain/entity/group/Group.py +48 -0
- graphcore/domain/entity/group/README.md +88 -0
- graphcore/domain/entity/group/__init__.py +0 -0
- graphcore/domain/entity/invite/Invite.py +31 -0
- graphcore/domain/entity/invite/__init__.py +0 -0
- graphcore/domain/entity/request_template/RequestTemplate.py +48 -0
- graphcore/domain/entity/request_template/__init__.py +0 -0
- graphcore/domain/entity/request_template/tests/__init__.py +0 -0
- graphcore/domain/entity/request_template/tests/test_request_template.py +52 -0
- graphcore/domain/entity/service_account/ServiceAccount.py +41 -0
- graphcore/domain/entity/service_account/__init__.py +0 -0
- graphcore/domain/entity/strategy_request/StrategyRequest.py +61 -0
- graphcore/domain/entity/strategy_request/__init__.py +0 -0
- graphcore/domain/entity/strategy_request/tests/__init__.py +0 -0
- graphcore/domain/entity/strategy_request/tests/test_strategy_request_entity.py +35 -0
- graphcore/domain/entity/tests/__init__.py +0 -0
- graphcore/domain/entity/tests/test_entity.py +227 -0
- graphcore/domain/entity/tests/test_grant_group_entities.py +36 -0
- graphcore/domain/entity/tests/test_write_once_fields.py +98 -0
- graphcore/domain/entity/user/User.py +39 -0
- graphcore/domain/entity/user/__init__.py +0 -0
- graphcore/domain/entity/user/tests/__init__.py +0 -0
- graphcore/domain/entity/user/tests/test_user_entity.py +33 -0
- graphcore/domain/entity/view/View.py +37 -0
- graphcore/domain/entity/view/__init__.py +0 -0
- graphcore/domain/entity/view/position.py +78 -0
- graphcore/domain/entity/view/tests/__init__.py +0 -0
- graphcore/domain/entity/view/tests/test_position.py +71 -0
- graphcore/domain/entity/view/tests/test_view_entity.py +134 -0
- graphcore/domain/strategy/BaseStrategy.py +759 -0
- graphcore/domain/strategy/DeploymentStrategies.py +55 -0
- graphcore/domain/strategy/GrantStrategies.py +189 -0
- graphcore/domain/strategy/GroupStrategies.py +193 -0
- graphcore/domain/strategy/InviteStrategies.py +84 -0
- graphcore/domain/strategy/RequestTemplateStrategies.py +126 -0
- graphcore/domain/strategy/ViewStrategies.py +219 -0
- graphcore/domain/strategy/__init__.py +0 -0
- graphcore/domain/strategy/runner.py +84 -0
- graphcore/domain/strategy/session.py +56 -0
- graphcore/domain/strategy/tests/BaseStrategyTest.py +167 -0
- graphcore/domain/strategy/tests/__init__.py +0 -0
- graphcore/domain/strategy/tests/test_BaseStrategies.py +870 -0
- graphcore/domain/strategy/tests/test_grant_strategies.py +245 -0
- graphcore/domain/strategy/tests/test_group_strategies.py +99 -0
- graphcore/domain/strategy/tests/test_runner.py +220 -0
- graphcore/domain/strategy/tests/test_strategy_locks.py +446 -0
- graphcore/domain/strategy/tests/test_utils.py +95 -0
- graphcore/domain/strategy/tests/test_view_strategies.py +436 -0
- graphcore/errors.py +145 -0
- graphcore/execution/README.md +97 -0
- graphcore/execution/StrategyExecutorService.py +333 -0
- graphcore/execution/__init__.py +0 -0
- graphcore/execution/dispatcher.py +19 -0
- graphcore/execution/environments/__init__.py +0 -0
- graphcore/execution/environments/celery.py +59 -0
- graphcore/execution/environments/in_process.py +55 -0
- graphcore/execution/local_session.py +117 -0
- graphcore/execution/redis_run_events.py +74 -0
- graphcore/execution/requests.py +22 -0
- graphcore/execution/run_events.py +85 -0
- graphcore/execution/strategy_context.py +54 -0
- graphcore/execution/tasks.py +34 -0
- graphcore/execution/tests/__init__.py +5 -0
- graphcore/execution/tests/contract.py +171 -0
- graphcore/execution/tests/support.py +310 -0
- graphcore/execution/tests/test_dispatchers.py +76 -0
- graphcore/execution/tests/test_executor.py +368 -0
- graphcore/execution/tests/test_local_session.py +95 -0
- graphcore/execution/tests/test_reconcile_runs.py +134 -0
- graphcore/execution/tests/test_run_events.py +106 -0
- graphcore/execution/tests/test_strategy_context.py +62 -0
- graphcore/gateway/__init__.py +1 -0
- graphcore/gateway/remote.py +35 -0
- graphcore/gateway/tests/__init__.py +0 -0
- graphcore/gateway/tests/test_remote.py +30 -0
- graphcore/graph/EntityService.py +710 -0
- graphcore/graph/__init__.py +0 -0
- graphcore/graph/entity_cache.py +152 -0
- graphcore/graph/tests/__init__.py +4 -0
- graphcore/graph/tests/test_EntityService.py +908 -0
- graphcore/graph/tests/test_EntityService_list.py +170 -0
- graphcore/graph/tests/test_EntityService_model_resolution.py +339 -0
- graphcore/graph/tests/test_EntityService_referrers.py +236 -0
- graphcore/graph/tests/test_EntityService_routing.py +119 -0
- graphcore/graph/tests/test_dependent_on.py +141 -0
- graphcore/graph/tests/test_engine_is_type_agnostic.py +25 -0
- graphcore/graph/tests/test_engine_resolves_read_permission.py +93 -0
- graphcore/graph/tests/test_entity_cache.py +159 -0
- graphcore/graph/tests/test_null_entity_cache.py +12 -0
- graphcore/management/__init__.py +0 -0
- graphcore/management/commands/__init__.py +0 -0
- graphcore/management/commands/dev_reset.py +18 -0
- graphcore/management/commands/graphcore_createadmin.py +65 -0
- graphcore/management/commands/graphcore_policy.py +59 -0
- graphcore/management/commands/graphcore_status.py +98 -0
- graphcore/management/tests/__init__.py +0 -0
- graphcore/management/tests/test_createadmin.py +78 -0
- graphcore/management/tests/test_policy_command.py +46 -0
- graphcore/management/tests/test_status_command.py +85 -0
- graphcore/migrations/0001_initial.py +186 -0
- graphcore/migrations/0002_invite.py +26 -0
- graphcore/migrations/__init__.py +0 -0
- graphcore/models.py +11 -0
- graphcore/persistence/ENTITY_REPOSITORY_TECHNICAL_TDD.md +396 -0
- graphcore/persistence/__init__.py +6 -0
- graphcore/persistence/django/__init__.py +0 -0
- graphcore/persistence/django/backends.py +43 -0
- graphcore/persistence/django/bindings.py +28 -0
- graphcore/persistence/django/cache.py +38 -0
- graphcore/persistence/django/compiler.py +214 -0
- graphcore/persistence/django/identity.py +18 -0
- graphcore/persistence/django/models.py +252 -0
- graphcore/persistence/django/repository.py +525 -0
- graphcore/persistence/django/tests/__init__.py +0 -0
- graphcore/persistence/django/tests/test_backends.py +68 -0
- graphcore/persistence/django/tests/test_compiler.py +269 -0
- graphcore/persistence/django/tests/test_repository.py +472 -0
- graphcore/persistence/django/tests/test_shared_column.py +120 -0
- graphcore/persistence/django/tests/test_threads.py +50 -0
- graphcore/persistence/django/tests/test_update_shape.py +175 -0
- graphcore/persistence/django/threads.py +38 -0
- graphcore/persistence/repository.py +172 -0
- graphcore/persistence/tests/__init__.py +0 -0
- graphcore/persistence/tests/contract.py +688 -0
- graphcore/persistence/tests/evaluate.py +51 -0
- graphcore/persistence/tests/fixtures.py +32 -0
- graphcore/persistence/tests/memory.py +263 -0
- graphcore/persistence/tests/test_memory_repository.py +10 -0
- graphcore/persistence/tests/test_query_types.py +127 -0
- graphcore/public/__init__.py +9 -0
- graphcore/public/api.py +9 -0
- graphcore/public/apps.py +10 -0
- graphcore/public/auth/__init__.py +1 -0
- graphcore/public/auth/grants.py +7 -0
- graphcore/public/auth/onboarding.py +8 -0
- graphcore/public/auth/perms.py +7 -0
- graphcore/public/auth/policy.py +10 -0
- graphcore/public/auth/roles.py +8 -0
- graphcore/public/auth/subjects.py +7 -0
- graphcore/public/entities/__init__.py +32 -0
- graphcore/public/errors.py +19 -0
- graphcore/public/persistence/__init__.py +44 -0
- graphcore/public/persistence/django.py +24 -0
- graphcore/public/scheduling.py +23 -0
- graphcore/public/strategies/__init__.py +85 -0
- graphcore/public/test_settings.py +16 -0
- graphcore/public/testing.py +47 -0
- graphcore/registry/EntityDirectory.py +246 -0
- graphcore/registry/README.md +147 -0
- graphcore/registry/StrategyDirectory.py +308 -0
- graphcore/registry/TYPE_METADATA_PLACEMENT.md +66 -0
- graphcore/registry/__init__.py +0 -0
- graphcore/registry/app_discovery.py +108 -0
- graphcore/registry/decorators.py +77 -0
- graphcore/registry/discovery.py +111 -0
- graphcore/registry/registry.py +79 -0
- graphcore/registry/tests/__init__.py +0 -0
- graphcore/registry/tests/test_agent_frontend_discovery.py +56 -0
- graphcore/registry/tests/test_app_config.py +36 -0
- graphcore/registry/tests/test_discovery.py +147 -0
- graphcore/registry/tests/test_entity_directory_storage.py +73 -0
- graphcore/registry/tests/test_import_package.py +87 -0
- graphcore/registry/tests/test_principal_flag.py +67 -0
- graphcore/registry/tests/test_strategy_directory.py +29 -0
- graphcore/routes/__init__.py +116 -0
- graphcore/routes/tests/__init__.py +0 -0
- graphcore/routes/tests/test_routes.py +90 -0
- graphcore/runtime.py +243 -0
- graphcore/scheduling/__init__.py +0 -0
- graphcore/scheduling/beat_scheduler.py +164 -0
- graphcore/scheduling/beat_tasks.py +63 -0
- graphcore/scheduling/scheduler.py +37 -0
- graphcore/scheduling/tests/__init__.py +0 -0
- graphcore/scheduling/tests/contract.py +136 -0
- graphcore/scheduling/tests/memory.py +42 -0
- graphcore/scheduling/tests/test_beat_scheduler.py +87 -0
- graphcore/scheduling/tests/test_memory_scheduler.py +15 -0
- graphcore/scheduling/tests/test_ticks.py +143 -0
- graphcore/tasks.py +9 -0
- graphcore/test_settings.py +78 -0
- graphcore/testing/__init__.py +0 -0
- graphcore/testing/case.py +112 -0
- graphcore/testing/runner.py +32 -0
- graphcore/testing/settings.py +26 -0
- graphcore/testing/tests/__init__.py +0 -0
- graphcore/testing/tests/agent_view_fixture.py +19 -0
- graphcore/testing/tests/test_case_and_runner.py +219 -0
- graphcore/testing/tests/test_settings_for_apps.py +50 -0
- graphcore/tests/__init__.py +0 -0
- graphcore/tests/ids.py +14 -0
- graphcore/tests/public_api_manifest.py +102 -0
- graphcore/tests/runs.py +7 -0
- graphcore/tests/test_app_discovery.py +90 -0
- graphcore/tests/test_composition_boundaries.py +231 -0
- graphcore/tests/test_docs_discovery.py +108 -0
- graphcore/tests/test_no_print.py +70 -0
- graphcore/tests/test_packaging.py +64 -0
- graphcore/tests/test_public_api_boundaries.py +229 -0
- graphcore/tests/test_public_api_contract.py +62 -0
- graphcore/tests/test_public_api_docs.py +28 -0
- graphcore/tests/test_runtime.py +42 -0
- graphcore/tests/test_shared_type_boundaries.py +128 -0
- graphcore/tests/test_shared_types.py +144 -0
- graphcore/tests/test_test_settings.py +45 -0
- graphcore/transport/__init__.py +0 -0
- graphcore/transport/agent_asgi.py +68 -0
- graphcore/transport/agent_client.py +85 -0
- graphcore/transport/channel_layer.py +68 -0
- graphcore/transport/consumers.py +507 -0
- graphcore/transport/routing.py +6 -0
- graphcore/transport/tests/__init__.py +0 -0
- graphcore/transport/tests/test_agent_client.py +31 -0
- graphcore/transport/tests/test_api_views.py +733 -0
- graphcore/transport/tests/test_channel_layer.py +96 -0
- graphcore/transport/tests/test_consumers.py +963 -0
- graphcore/transport/tests/test_resource_metadata.py +67 -0
- graphcore/transport/urls.py +29 -0
- graphcore/transport/views.py +358 -0
- graphcore/types/__init__.py +4 -0
- graphcore/types/access.py +114 -0
- graphcore/types/api.py +76 -0
- graphcore/types/entities.py +100 -0
- graphcore/types/entity_id.py +85 -0
- graphcore/types/execution.py +41 -0
- graphcore/types/identity.py +20 -0
- graphcore/types/onboarding.py +20 -0
- graphcore/types/query.py +116 -0
- graphcore/types/schedules.py +91 -0
- graphcore/types/strategy.py +73 -0
- graphcore/types/tests/__init__.py +0 -0
- graphcore/types/tests/test_entity_id.py +50 -0
- graphcore/urls.py +5 -0
- graphcore_kernel-0.1.0.dist-info/METADATA +94 -0
- graphcore_kernel-0.1.0.dist-info/RECORD +343 -0
- graphcore_kernel-0.1.0.dist-info/WHEEL +5 -0
- graphcore_kernel-0.1.0.dist-info/entry_points.txt +2 -0
- graphcore_kernel-0.1.0.dist-info/licenses/LICENSE +202 -0
- graphcore_kernel-0.1.0.dist-info/licenses/NOTICE +2 -0
- graphcore_kernel-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1043 @@
|
|
|
1
|
+
# Strategy Execution TDD
|
|
2
|
+
|
|
3
|
+
## Introduction
|
|
4
|
+
|
|
5
|
+
Strategies are the only writes in GraphCore. This document specifies how a
|
|
6
|
+
strategy request is run: how the kernel decides where it runs, how a running
|
|
7
|
+
strategy reads and writes the graph, and how a run is reported back to its
|
|
8
|
+
caller.
|
|
9
|
+
|
|
10
|
+
The design separates three concerns:
|
|
11
|
+
|
|
12
|
+
- **What a strategy is.** A strategy is domain code. It declares its
|
|
13
|
+
workload and talks to the graph through one interface, the
|
|
14
|
+
`StrategySession`. It carries no knowledge of the process it runs in.
|
|
15
|
+
- **Where a strategy runs.** Infrastructure. A `Dispatcher` port hands one
|
|
16
|
+
attempt to an execution environment, which runs it with the
|
|
17
|
+
`StrategyRunner` and reports the result back.
|
|
18
|
+
- **What a run guarantees.** Kernel policy. `StrategyExecutorService`
|
|
19
|
+
authorizes, records, retries, and reports, in the kernel, for every
|
|
20
|
+
environment.
|
|
21
|
+
|
|
22
|
+
An execution environment either holds a runtime or does not. One with a
|
|
23
|
+
runtime reaches the graph through its own `EntityService`; one without
|
|
24
|
+
reaches it through the kernel's execution transport. The runner, the
|
|
25
|
+
session contract, and the way results are reported are the same in both,
|
|
26
|
+
so moving an environment from one kind to the other changes its entry point
|
|
27
|
+
and nothing else. The kernel ships two environments, the event loop of the
|
|
28
|
+
submitting process and a Celery process, both with a runtime. A project
|
|
29
|
+
adds an environment by supplying a workload, a dispatcher, and an entry
|
|
30
|
+
point, without changing kernel or strategy code.
|
|
31
|
+
|
|
32
|
+
## Definitions
|
|
33
|
+
|
|
34
|
+
- **Strategy**: a subclass of `Strategy` (`domain/strategy/BaseStrategy.py`).
|
|
35
|
+
Its `apply()` reads and writes the graph to fulfil one request. Kernel and
|
|
36
|
+
application strategies are the same kind of object.
|
|
37
|
+
- **Strategy request**: a `StrategyRequest`, an ordinary top-level entity
|
|
38
|
+
naming a strategy, its parameters, its target, and its outcome. It runs as
|
|
39
|
+
its `owner`. A `StrategyRequestPayload` is the client-safe input a caller
|
|
40
|
+
creates one from.
|
|
41
|
+
- **Request template**: a `RequestTemplate`, an ordinary top-level entity
|
|
42
|
+
describing a request to make later, optionally on a schedule
|
|
43
|
+
(`docs/SCHEDULING_TDD.md`). It is never run itself.
|
|
44
|
+
- **Workload**: the `Workload` value a strategy declares, naming the kind of
|
|
45
|
+
environment its `apply()` needs. The set is open: the kernel defines
|
|
46
|
+
`Workload.IO` (the default) and `Workload.CPU`; a project defines more.
|
|
47
|
+
- **Runtime**: the `Runtime` that `build_runtime` constructs
|
|
48
|
+
(`graphcore/runtime.py`): one process's live service graph, holding the
|
|
49
|
+
registry, repository, cache, access port, broadcaster, scheduler,
|
|
50
|
+
`EntityService`, and `StrategyExecutorService`. Building one requires the
|
|
51
|
+
deployment's database, cache, and broker connections.
|
|
52
|
+
- **Kernel process**: a process holding a runtime. The ASGI process is one;
|
|
53
|
+
so is the Celery process.
|
|
54
|
+
- **Front door**: `StrategyExecutorService.execute_request`, the single entry
|
|
55
|
+
point for submitting a `StrategyRequest`.
|
|
56
|
+
- **Run**: everything that happens to one request between submission and
|
|
57
|
+
its terminal status, across all attempts.
|
|
58
|
+
- **Attempt**: one execution of a request's strategy, numbered from 1,
|
|
59
|
+
described by an `ExecutionAttempt` and reported by an `AttemptResult`.
|
|
60
|
+
- **Execution environment**: where an attempt runs. Defined by a dispatcher
|
|
61
|
+
(how attempts reach it) and an entry point (the code that builds a session
|
|
62
|
+
and runs the runner). An environment with a runtime is a kernel process;
|
|
63
|
+
one without reaches the kernel over the execution transport.
|
|
64
|
+
- **Dispatcher**: an implementation of the `Dispatcher` port. It delivers an
|
|
65
|
+
attempt to an execution environment and returns.
|
|
66
|
+
- **Strategy runner**: `StrategyRunner`, the domain-level code that performs
|
|
67
|
+
one attempt. It depends only on `StrategySession` and `ChangeTracker`.
|
|
68
|
+
- **Strategy session**: an implementation of `StrategySession`, bound to one
|
|
69
|
+
attempt. It is the strategy's and the runner's only path to the graph.
|
|
70
|
+
`LocalStrategySession` reaches the graph through a runtime's
|
|
71
|
+
`EntityService`; `RemoteStrategySession` reaches it through a
|
|
72
|
+
`KernelClient`.
|
|
73
|
+
- **Change tracker**: a `ChangeTracker`, built for one attempt, holding the
|
|
74
|
+
last persisted state of every entity the attempt's session returned or
|
|
75
|
+
saved.
|
|
76
|
+
- **Execution transport**: the kernel's HTTP routes that serve environments
|
|
77
|
+
without a runtime: the session operations, `finish_attempt`, and nested
|
|
78
|
+
`execute`. Mounted separately from client transports and never reachable
|
|
79
|
+
by clients.
|
|
80
|
+
- **Environment credential**: what authenticates an environment without a
|
|
81
|
+
runtime to the execution transport. It identifies the environment, not a
|
|
82
|
+
user.
|
|
83
|
+
- **Kernel client**: `KernelClient`, the client for the execution transport.
|
|
84
|
+
- **Fence**: the attempt number a request currently holds. A session write
|
|
85
|
+
or an attempt result carrying any other attempt number is refused.
|
|
86
|
+
- **Claim**: the entry point's conditional write that moves a dispatched
|
|
87
|
+
attempt from `pending` to `running` before running it. Only one delivery
|
|
88
|
+
of an attempt claims it.
|
|
89
|
+
- **Deadline**: `attempt_deadline` on a request. While `pending`, the time
|
|
90
|
+
by which an environment must claim the attempt; while `running`, the time
|
|
91
|
+
by which the claimed attempt must report.
|
|
92
|
+
- **Run reconciler**: the kernel task that sends again, re-dispatches, or
|
|
93
|
+
fails requests whose deadline has passed.
|
|
94
|
+
|
|
95
|
+
## Requirements
|
|
96
|
+
|
|
97
|
+
1. A strategy's code is identical whichever execution environment runs it.
|
|
98
|
+
Placement is decided by declared workload and deployment configuration,
|
|
99
|
+
never by the strategy's shape.
|
|
100
|
+
2. `domain/strategy/` imports no kernel service and no infrastructure. It
|
|
101
|
+
owns the `StrategySession` interface and depends only on that; adapters
|
|
102
|
+
implement the interface over kernel services or a kernel client
|
|
103
|
+
(dependency inversion, as with `AccessPort`).
|
|
104
|
+
3. Every strategy, kernel or application, reaches the graph through the same
|
|
105
|
+
`StrategySession` contract. No strategy holds a capability its principal
|
|
106
|
+
lacks.
|
|
107
|
+
4. GraphAPI exposes no operation that exists to serve strategy execution.
|
|
108
|
+
Clients write only by executing strategies.
|
|
109
|
+
5. Every save made by a strategy is authorized by the persistence chokepoint
|
|
110
|
+
against the running strategy's class and the request's owner, whatever
|
|
111
|
+
environment the strategy runs in.
|
|
112
|
+
6. Authorization, request bookkeeping, retries, and reporting have exactly
|
|
113
|
+
one implementation, in the kernel. An execution environment performs one
|
|
114
|
+
attempt and nothing else.
|
|
115
|
+
7. An environment without a runtime can run attempts holding no database
|
|
116
|
+
credentials and no broker credentials, with the strategy code and
|
|
117
|
+
`graphcore.public` as its only kernel dependencies. Whether an
|
|
118
|
+
environment holds a runtime is its own decision and changes only its
|
|
119
|
+
entry point.
|
|
120
|
+
8. The execution environment for a workload is chosen in the composition
|
|
121
|
+
root and can change without changing kernel or strategy code.
|
|
122
|
+
9. A deployment that maps a declared workload to no dispatcher fails at
|
|
123
|
+
startup, not on the first request.
|
|
124
|
+
10. A bearer token never crosses a process boundary.
|
|
125
|
+
11. No execution environment is trusted. An environment can act only for an
|
|
126
|
+
attempt the kernel dispatched, only while that attempt is current, and
|
|
127
|
+
only as that request's owner.
|
|
128
|
+
12. No strategy runs with kernel-internal authority. Kernel code opens
|
|
129
|
+
`kernel_internal_context()` for its own writes (deployment seeding,
|
|
130
|
+
principal creation, onboarding, the executor's request writes), and
|
|
131
|
+
never while a strategy context is active.
|
|
132
|
+
13. Every submitted request is saved before it is dispatched, and its
|
|
133
|
+
outcome is saved on it.
|
|
134
|
+
14. A run does not depend on the process that submitted it. A submitting
|
|
135
|
+
process can restart while an attempt is in flight without losing the
|
|
136
|
+
run.
|
|
137
|
+
15. A run whose environment fails is detected and either re-dispatched or
|
|
138
|
+
failed, without user action.
|
|
139
|
+
|
|
140
|
+
## Architecture
|
|
141
|
+
|
|
142
|
+
### Components
|
|
143
|
+
|
|
144
|
+
```
|
|
145
|
+
kernel process (ASGI or Celery) environment without a runtime
|
|
146
|
+
┌────────────────────────────────────────────┐ ┌─────────────────────────────────────────┐
|
|
147
|
+
│ caller ─► executor.execute_request │ │ entry point(data) │
|
|
148
|
+
│ │ save (status = pending, │ │ attempt = ExecutionAttempt.from_dict │
|
|
149
|
+
│ │ attempt = 1, deadline) │ │ kernel.claim_attempt(attempt) │
|
|
150
|
+
│ │ or, on a retry, │ │ tracker = ChangeTracker() │
|
|
151
|
+
│ executor.start_attempt │ │ session = RemoteStrategySession( │
|
|
152
|
+
│ │ save (attempt = n, deadline) │ ──────► │ attempt, tracker, kernel) │
|
|
153
|
+
│ ▼ │ │ result = runner.run( │
|
|
154
|
+
│ dispatchers[workload].dispatch │ │ attempt, session, tracker)│
|
|
155
|
+
│ │ │ kernel.finish_attempt(result) │
|
|
156
|
+
│ entry point (InProcessDispatcher._run, │ │ │
|
|
157
|
+
│ celery run_attempt) │ │ RemoteStrategySession │
|
|
158
|
+
│ executor.claim_attempt(attempt) │ │ get / list / save / execute ──────────┼─┐
|
|
159
|
+
│ save (status = running, deadline) │ └─────────────────────────────────────────┘ │
|
|
160
|
+
│ tracker = ChangeTracker() │ │
|
|
161
|
+
│ session = LocalStrategySession( │ execution transport (kernel process) │
|
|
162
|
+
│ attempt, tracker, entity_service, │ ┌─────────────────────────────────────────┐ │
|
|
163
|
+
│ executor, registry) │ │ environment credential ─► │◄┘
|
|
164
|
+
│ result = runner.run( │ │ LocalStrategySession(attempt, …).<op> │
|
|
165
|
+
│ attempt, session, tracker) │ │ claim / finish_attempt ─► executor │
|
|
166
|
+
│ executor.finish_attempt(result) │ └─────────────────────────────────────────┘
|
|
167
|
+
│ │
|
|
168
|
+
│ executor.finish_attempt │
|
|
169
|
+
│ fence check; save (succeeded | failed) │
|
|
170
|
+
│ or start next attempt; callbacks │
|
|
171
|
+
│ │
|
|
172
|
+
│ LocalStrategySession │
|
|
173
|
+
│ fence check ─► strategy_context ─► │
|
|
174
|
+
│ EntityService / front door │
|
|
175
|
+
│ │
|
|
176
|
+
│ reconcile_runs (periodic) │
|
|
177
|
+
└────────────────────────────────────────────┘
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
The runner, the strategy, the tracker, the auto-save, and `finish_attempt`
|
|
181
|
+
are identical in every environment. What differs is the session adapter the
|
|
182
|
+
entry point constructs: the local one holds the runtime's services, the
|
|
183
|
+
remote one holds a `KernelClient`. The fence check and `strategy_context`
|
|
184
|
+
live in `LocalStrategySession`; the execution transport serves a remote
|
|
185
|
+
session by building a local one for the same attempt.
|
|
186
|
+
|
|
187
|
+
| Component | Package | Kind | Responsibility |
|
|
188
|
+
|---|---|---|---|
|
|
189
|
+
| `Strategy` | `domain/strategy/` | domain | Strategy behavior; declares `workload` and `timeout` |
|
|
190
|
+
| `StrategySession` | `domain/strategy/session.py` | port (owned by the domain) | The strategy's view of the graph |
|
|
191
|
+
| `StrategyRunner`, `ChangeTracker` | `domain/strategy/runner.py` | domain | Performs one attempt through a session |
|
|
192
|
+
| `LocalStrategySession` | `execution/local_session.py` | adapter | The session over a runtime's `EntityService` and front door |
|
|
193
|
+
| `RemoteStrategySession`, `KernelClient` | `execution/remote_session.py` | adapter | The session over the execution transport |
|
|
194
|
+
| execution transport | `transport/execution.py` | transport | Serves environments without a runtime |
|
|
195
|
+
| `Workload` | `types/strategy.py` | shared type | Declared environment kind |
|
|
196
|
+
| `StrategyRequestPayload` | `types/strategy.py` | shared type | Client-safe description of a request |
|
|
197
|
+
| `RequestStatus` | `types/strategy.py` | shared type | A request's status |
|
|
198
|
+
| `ExecutionAttempt`, `AttemptResult` | `types/execution.py` | shared types | What crosses the dispatcher port, both ways |
|
|
199
|
+
| `new_strategy_request` | `execution/requests.py` | kernel function | The one way a `StrategyRequest` is built |
|
|
200
|
+
| `StrategyExecutorService` | `execution/StrategyExecutorService.py` | kernel service | Front door, attempts, results, callbacks |
|
|
201
|
+
| `Dispatcher` | `execution/dispatcher.py` | port | Delivery of an attempt to an environment |
|
|
202
|
+
| in-process environment | `execution/environments/in_process.py` | adapter and entry point | `InProcessDispatcher`; runs the attempt on this process's event loop |
|
|
203
|
+
| Celery environment | `execution/environments/celery.py` | adapter and entry point | `CeleryDispatcher` and the `run_attempt` task |
|
|
204
|
+
| `RunEvents` | `execution/run_events.py` | port | Tells a waiting caller its run has ended |
|
|
205
|
+
| `InMemoryRunEvents` | `execution/run_events.py` | adapter | `RunEvents` within one process |
|
|
206
|
+
| `RedisRunEvents` | `execution/redis_run_events.py` | adapter | `RunEvents` across processes over Redis pub/sub |
|
|
207
|
+
| `reconcile_runs` | `execution/tasks.py` | kernel task | Run reconciliation |
|
|
208
|
+
|
|
209
|
+
### Declaring a workload and a timeout
|
|
210
|
+
|
|
211
|
+
```python
|
|
212
|
+
@dataclass(frozen=True)
|
|
213
|
+
class Workload:
|
|
214
|
+
name: str
|
|
215
|
+
|
|
216
|
+
Workload.IO = Workload('io')
|
|
217
|
+
Workload.CPU = Workload('cpu')
|
|
218
|
+
|
|
219
|
+
class Strategy(ABC):
|
|
220
|
+
workload: Workload = Workload.IO
|
|
221
|
+
timeout: int | None = None # seconds per attempt; None means the deployment default
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
A workload describes the strategy's needs, not a place. A project defines
|
|
225
|
+
its own (`SANDBOX = Workload('sandbox')`) next to the strategies that
|
|
226
|
+
declare it. The composition root binds workloads to dispatchers:
|
|
227
|
+
|
|
228
|
+
```python
|
|
229
|
+
executor = StrategyExecutorService(registry, entity_service, default_timeout=600,
|
|
230
|
+
max_attempts=6, dispatch_timeout=300)
|
|
231
|
+
executor.bind_dispatchers({ # default_dispatchers
|
|
232
|
+
Workload.IO: InProcessDispatcher(executor, entity_service, registry),
|
|
233
|
+
Workload.CPU: CeleryDispatcher(),
|
|
234
|
+
})
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
A Celery worker is the CPU environment, so its runtime binds
|
|
238
|
+
`worker_dispatchers` instead: CPU runs on the worker's own loop, beside IO.
|
|
239
|
+
A CPU strategy that waits on a nested CPU request then runs it beside
|
|
240
|
+
itself instead of waiting for a second worker slot, which a busy pool may
|
|
241
|
+
never free before the parent's deadline. The worker says what it is when it
|
|
242
|
+
starts (`serve_as_worker()`, from Celery's `worker_init`), before its
|
|
243
|
+
runtime is built; the runtime factory receives it as `worker=True`.
|
|
244
|
+
|
|
245
|
+
Invariants:
|
|
246
|
+
|
|
247
|
+
- `build_runtime` raises if a registered strategy declares a workload absent
|
|
248
|
+
from `dispatchers`.
|
|
249
|
+
- Adding a workload is defining a value and a mapping entry. The executor
|
|
250
|
+
never branches on a workload value.
|
|
251
|
+
- `timeout` is declarative metadata, like `workload` and
|
|
252
|
+
`read_only_strategy`: read by the executor, never acted on by the
|
|
253
|
+
strategy.
|
|
254
|
+
|
|
255
|
+
### The strategy session
|
|
256
|
+
|
|
257
|
+
```python
|
|
258
|
+
class StrategySession(ABC):
|
|
259
|
+
principal: Principal
|
|
260
|
+
|
|
261
|
+
def create(self, entity_cls: type[E], **fields) -> E: ...
|
|
262
|
+
async def get(self, entity_id: str) -> Entity: ...
|
|
263
|
+
async def get_many(self, entity_ids: list[str]) -> dict[str, Entity]: ...
|
|
264
|
+
async def list(self, entity_filter: EntityFilter, cursor: str | None = None,
|
|
265
|
+
limit: int = 25) -> EntityPage[Entity]: ...
|
|
266
|
+
async def save(self, entity: E) -> E: ...
|
|
267
|
+
async def delete_tree(self, root_id: str) -> list[str]: ...
|
|
268
|
+
async def execute(self, payload: StrategyRequestPayload, *, wait: bool = True,
|
|
269
|
+
callback_channel: str | None = None,
|
|
270
|
+
) -> tuple[StrategyRequest, Entity | None]: ...
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
The port holds only what strategies call. Strategies and the runner are its
|
|
274
|
+
only users.
|
|
275
|
+
|
|
276
|
+
A session is bound to one attempt. Its principal is `Principal(owner_id)`
|
|
277
|
+
from the attempt, identity-only; strategy code never passes a principal or
|
|
278
|
+
a strategy identity to it.
|
|
279
|
+
|
|
280
|
+
Contract:
|
|
281
|
+
|
|
282
|
+
- `create` constructs an entity with an identifier minted for this
|
|
283
|
+
deployment. The k-th `create` of an attempt mints
|
|
284
|
+
`derived_entity_id(attempt.backend_id, f"{request_id}/{k}")`, so every
|
|
285
|
+
attempt of a request mints the same identifiers in the same call order.
|
|
286
|
+
It does not persist.
|
|
287
|
+
- The first `save` in an attempt of an entity that attempt's `create`
|
|
288
|
+
minted replaces the entity an earlier attempt of the request saved under
|
|
289
|
+
that identifier, if one did. A retried `apply()` therefore recreates its
|
|
290
|
+
entities instead of duplicating them or conflicting with them.
|
|
291
|
+
- `get` raises `NotFound` or `Forbidden`. `get_many` returns only the
|
|
292
|
+
entities the principal may read.
|
|
293
|
+
- `list` is principal-scoped and has `EntityService.list_entities`
|
|
294
|
+
semantics.
|
|
295
|
+
- `save` persists through the persistence chokepoint and raises
|
|
296
|
+
`ConcurrentModificationError` on a stale version.
|
|
297
|
+
- `delete_tree` deletes through the deletion chokepoint.
|
|
298
|
+
- `execute` creates a `StrategyRequest` from the payload and submits it
|
|
299
|
+
through a front door as the session's principal. The nested request is
|
|
300
|
+
authorized on its own, and behaves as the front door does for any caller:
|
|
301
|
+
with `wait=True` it returns once the run succeeds or raises the error the
|
|
302
|
+
run failed with. The calling strategy decides whether to catch it; if it
|
|
303
|
+
does not, its own run fails with that error.
|
|
304
|
+
- Every write is fenced: `save` or `delete_tree` takes effect only while
|
|
305
|
+
the attempt's request is `running` with the attempt's number, or the
|
|
306
|
+
write is refused with `StaleAttempt`. The check and the write are atomic.
|
|
307
|
+
- A strategy calls `save` whenever it wants a change persisted mid-run.
|
|
308
|
+
Each save is broadcast, so saves are how a strategy streams progress to
|
|
309
|
+
clients, and each is a checkpoint. Saving is never required: the runner
|
|
310
|
+
persists the returned target for the strategy (see The runner).
|
|
311
|
+
- A session is built with the attempt's `ChangeTracker` and records in it
|
|
312
|
+
the state of every entity it returns from `get`, `get_many`, or `list`,
|
|
313
|
+
or persists with `save`. This is a requirement on session adapters, not
|
|
314
|
+
an operation strategies see.
|
|
315
|
+
- Each `save` commits on return. A session is not a transaction.
|
|
316
|
+
- Every read is scoped to the session's principal. No session operation
|
|
317
|
+
reads, writes, or checks access on behalf of anyone else.
|
|
318
|
+
- A strategy makes no authorization decision of its own. A rule about who
|
|
319
|
+
may set a field (for example, only a deployment administrator may give a
|
|
320
|
+
group a `key`) belongs to the type's access policy, as a
|
|
321
|
+
`deployment_fields` entry naming the permission on the deployment that
|
|
322
|
+
setting the field requires, and is enforced at the persistence
|
|
323
|
+
chokepoint.
|
|
324
|
+
|
|
325
|
+
`Strategy` exposes the session as `self.session`, alongside
|
|
326
|
+
`self.strategy_request`, `self.entity`, `self.principal`, and
|
|
327
|
+
`self.registry` (read-only type metadata). The helpers
|
|
328
|
+
`self.get_entity`, `self.save_entity`, and `self.execute_request` delegate to
|
|
329
|
+
the session. A strategy holds no reference to `EntityService` or
|
|
330
|
+
`StrategyExecutorService`.
|
|
331
|
+
|
|
332
|
+
```python
|
|
333
|
+
class SummarizeNotes(Strategy):
|
|
334
|
+
async def apply(self):
|
|
335
|
+
page = await self.session.list(EntityFilter(type_ids=frozenset({'Note'}),
|
|
336
|
+
parent_id=self.entity.entity_id))
|
|
337
|
+
summary = self.session.create(Summary, parent_id=self.entity.entity_id)
|
|
338
|
+
summary.text = summarize(page.items)
|
|
339
|
+
await self.session.save(summary)
|
|
340
|
+
return self.strategy_request, self.entity
|
|
341
|
+
```
|
|
342
|
+
|
|
343
|
+
### The session adapters
|
|
344
|
+
|
|
345
|
+
```python
|
|
346
|
+
class LocalStrategySession(StrategySession):
|
|
347
|
+
def __init__(self, attempt: ExecutionAttempt, tracker: ChangeTracker,
|
|
348
|
+
entity_service: EntityService, executor: StrategyExecutorService,
|
|
349
|
+
registry: Registry): ...
|
|
350
|
+
|
|
351
|
+
class RemoteStrategySession(StrategySession):
|
|
352
|
+
def __init__(self, attempt: ExecutionAttempt, tracker: ChangeTracker,
|
|
353
|
+
kernel: KernelClient): ...
|
|
354
|
+
```
|
|
355
|
+
|
|
356
|
+
Both take the attempt and the tracker. The rest is how the session reaches
|
|
357
|
+
the graph.
|
|
358
|
+
|
|
359
|
+
`LocalStrategySession` reads and saves through `entity_service` as the
|
|
360
|
+
attempt's principal, submits nested requests through `executor`'s front
|
|
361
|
+
door, and resolves the strategy class from `registry` by
|
|
362
|
+
`attempt.strategy_name`. Each write passes the fence to `EntityService` as
|
|
363
|
+
a repository `Precondition` on the request (`status = running`,
|
|
364
|
+
`attempt = attempt.attempt`), which the repository checks in the write's
|
|
365
|
+
own transaction, and runs inside `strategy_context(strategy_cls)`. A
|
|
366
|
+
`PreconditionFailed` becomes `StaleAttempt`.
|
|
367
|
+
|
|
368
|
+
`RemoteStrategySession` forwards every operation to `kernel`, with the
|
|
369
|
+
attempt attached. It opens no context and checks no fence; the kernel side
|
|
370
|
+
of the execution transport does both, through a `LocalStrategySession`.
|
|
371
|
+
Entities cross the wire as `to_dict()` and are rehydrated with `from_dict`.
|
|
372
|
+
|
|
373
|
+
`KernelClient(base_url, environment_credential)` is the client for the
|
|
374
|
+
execution transport. It is the only thing an environment without a runtime
|
|
375
|
+
is configured with.
|
|
376
|
+
|
|
377
|
+
Invariants:
|
|
378
|
+
|
|
379
|
+
- A session adapter is constructed only by an environment's entry point, or
|
|
380
|
+
by the execution transport on behalf of one.
|
|
381
|
+
- The two adapters pass the same `StrategySessionContract`.
|
|
382
|
+
- Neither adapter holds a bearer token. The remote one holds the
|
|
383
|
+
deployment's environment credential, which authenticates the environment,
|
|
384
|
+
not a user.
|
|
385
|
+
|
|
386
|
+
### The execution transport
|
|
387
|
+
|
|
388
|
+
`transport/execution.py` mounts routes for environments without a runtime,
|
|
389
|
+
separately from the client transports, under a path that client routing
|
|
390
|
+
never exposes. It is an HTTP transport like the client one, doing wire
|
|
391
|
+
format and error mapping; the logic behind every route is
|
|
392
|
+
`LocalStrategySession` or the executor. Every route takes the attempt
|
|
393
|
+
(`request_id`, `attempt`) and is authenticated at two levels:
|
|
394
|
+
|
|
395
|
+
1. **The environment.** The request carries the deployment's environment
|
|
396
|
+
credential. The shipped default is a shared secret
|
|
397
|
+
(`GRAPHCORE_ENVIRONMENT_SECRET`); a deployment substitutes mTLS or a
|
|
398
|
+
network boundary through its runtime factory.
|
|
399
|
+
2. **The attempt.** The named request must exist and be `running` with the
|
|
400
|
+
named attempt number. Otherwise the call is refused with `StaleAttempt`.
|
|
401
|
+
This is the fence, applied to every call, reads included.
|
|
402
|
+
|
|
403
|
+
| Route | Does |
|
|
404
|
+
|---|---|
|
|
405
|
+
| `get`, `get_many`, `list`, `save`, `delete_tree`, `execute` | builds a `LocalStrategySession` for the attempt and calls the same operation on it |
|
|
406
|
+
| `finish_attempt` | passes the `AttemptResult` to this process's executor |
|
|
407
|
+
|
|
408
|
+
Invariants:
|
|
409
|
+
|
|
410
|
+
- The execution transport makes no authorization decision of its own beyond
|
|
411
|
+
the environment credential. The fence, the strategy context, and the
|
|
412
|
+
principal come from `LocalStrategySession`, which takes them from the
|
|
413
|
+
persisted request, never from the caller.
|
|
414
|
+
- GraphAPI and the client transports expose none of these routes.
|
|
415
|
+
- A nested `execute` from a remote session is submitted and dispatched by
|
|
416
|
+
the kernel process serving the transport. The remote environment never
|
|
417
|
+
blocks on anything it submitted.
|
|
418
|
+
|
|
419
|
+
### Strategy requests and request templates
|
|
420
|
+
|
|
421
|
+
A `StrategyRequest` is one run: it is submitted to the front door and runs
|
|
422
|
+
immediately. A `RequestTemplate` describes a request to make later and is
|
|
423
|
+
never run itself; requests are created from it, for example on each tick of
|
|
424
|
+
its schedule (`docs/SCHEDULING_TDD.md`). Both are ordinary top-level
|
|
425
|
+
entities: created with `EntityService.create`, saved through `EntityService`,
|
|
426
|
+
with the deployment as parent whatever their target.
|
|
427
|
+
|
|
428
|
+
| Field | `StrategyRequest` | `RequestTemplate` |
|
|
429
|
+
|---|---|---|
|
|
430
|
+
| `strategy_name`, `param_config`, `target_entity_id` | what it runs | what each request made from it runs |
|
|
431
|
+
| `owner` | the principal it runs as | the principal its requests run as |
|
|
432
|
+
| `callbacks` | payloads submitted, as the same principal, after it succeeds | copied to each request made from it |
|
|
433
|
+
| `callback_channel`, `client_request_id` | where and how to report completion | — |
|
|
434
|
+
| `status` | a `RequestStatus` | — |
|
|
435
|
+
| `attempt`, `attempt_deadline` | the current attempt number and its deadline | — |
|
|
436
|
+
| `ret_val`, `error`, `error_code`, `error_details` | the outcome; a failure keeps its error's message, `code`, and details | — |
|
|
437
|
+
| `template_id` | the template it was made from, if any | — |
|
|
438
|
+
| `cron`, `enabled` | — | its optional schedule |
|
|
439
|
+
|
|
440
|
+
`StrategyRequestPayload` is the client-safe description of a request:
|
|
441
|
+
`strategy_name`, `param_config`, `target_entity_id`, `callbacks`, and
|
|
442
|
+
`client_request_id`. Every `StrategyRequest` is built from one by
|
|
443
|
+
`new_strategy_request(entity_service, payload, template_id=None)`, which
|
|
444
|
+
creates the entity with `EntityService.create` and copies the payload's
|
|
445
|
+
fields onto it. The caller that submits a request builds it:
|
|
446
|
+
|
|
447
|
+
| Caller | Payload | `template_id` |
|
|
448
|
+
|---|---|---|
|
|
449
|
+
| GraphAPI | the client's | none |
|
|
450
|
+
| `StrategySession.execute` | the strategy's (`request_constructor`) | none |
|
|
451
|
+
| `finish_attempt` | each of a succeeded request's `callbacks`, with `source_ret_val` added to its `param_config` | none |
|
|
452
|
+
| a scheduler's tick | `template.to_payload()` | the template's id |
|
|
453
|
+
|
|
454
|
+
`template_id` is never read from a payload, so a client cannot claim a
|
|
455
|
+
request was made from a template.
|
|
456
|
+
|
|
457
|
+
A `RequestTemplate` is created by `CreateRequestTemplate`, an
|
|
458
|
+
ordinary strategy, from its `param_config` (`strategy_name`, `param_config`,
|
|
459
|
+
`target_entity_id`, and optionally `callbacks`, `cron`, `enabled`), and saved
|
|
460
|
+
through its session.
|
|
461
|
+
|
|
462
|
+
```python
|
|
463
|
+
RequestStatus = Literal['pending', 'running', 'succeeded', 'failed']
|
|
464
|
+
```
|
|
465
|
+
|
|
466
|
+
A request is `pending` from the front door's save until an environment
|
|
467
|
+
claims an attempt, and again whenever a next attempt is dispatched;
|
|
468
|
+
`running` while a claimed attempt is current; and `succeeded` or `failed`
|
|
469
|
+
once the run ends. `succeeded` and `failed` are terminal. Like the kernel's
|
|
470
|
+
other closed value sets, `RequestStatus` is a `Literal` alias, not an
|
|
471
|
+
`Enum`; unlike `Workload`, no project adds values to it.
|
|
472
|
+
|
|
473
|
+
`RequestStatus` is the only status vocabulary for a run. GraphAPI's
|
|
474
|
+
`StrategyOutcome.status` is the request's `status` at the time the front
|
|
475
|
+
door returns: `pending` or `running` with `wait=False`, `succeeded` or
|
|
476
|
+
`failed` with `wait=True`. A client reads the same values on the outcome,
|
|
477
|
+
on the request entity, and on the request's broadcasts.
|
|
478
|
+
|
|
479
|
+
Request saves are broadcast like any entity save, so a client can follow a
|
|
480
|
+
request's status by subscribing to it.
|
|
481
|
+
|
|
482
|
+
Invariants:
|
|
483
|
+
|
|
484
|
+
- A request runs as its `owner`, and nothing else names who it runs as. The
|
|
485
|
+
persistence chokepoint sets `owner` to the acting principal when an entity
|
|
486
|
+
is created and restores it from storage on every later save, so neither a
|
|
487
|
+
caller, a client payload, nor a strategy can choose or change it. The same
|
|
488
|
+
holds for a template's `owner`.
|
|
489
|
+
- `RequestTemplate` declares `dependent_on = ('target_entity_id',)`:
|
|
490
|
+
deleting a target deletes the templates that target it. `StrategyRequest`
|
|
491
|
+
declares none, so deleting a target keeps the history of runs on it,
|
|
492
|
+
including the run doing the deleting.
|
|
493
|
+
- No strategy writes a `StrategyRequest`. The executor creates it and writes
|
|
494
|
+
`status`, `attempt`, `attempt_deadline`, `ret_val`, and the error fields, in
|
|
495
|
+
`kernel_internal_context()`, never inside a strategy context. A strategy
|
|
496
|
+
sets `self.strategy_request.ret_val`; it reaches the request through the
|
|
497
|
+
`AttemptResult`.
|
|
498
|
+
- `CreateRequestTemplate` and `ManageRequestTemplate` are
|
|
499
|
+
the only strategies the access policy allows to write a `RequestTemplate`.
|
|
500
|
+
Both reject an unregistered `strategy_name` and an invalid `cron` before
|
|
501
|
+
saving.
|
|
502
|
+
- Requests and templates are readable by their owner only.
|
|
503
|
+
|
|
504
|
+
### Attempts and attempt results
|
|
505
|
+
|
|
506
|
+
```python
|
|
507
|
+
@dataclass(frozen=True)
|
|
508
|
+
class ExecutionAttempt:
|
|
509
|
+
request_id: str
|
|
510
|
+
attempt: int
|
|
511
|
+
strategy_name: str
|
|
512
|
+
target_entity_id: str | None
|
|
513
|
+
owner_id: str
|
|
514
|
+
backend_id: str # for identifier minting in `create`
|
|
515
|
+
|
|
516
|
+
@dataclass(frozen=True)
|
|
517
|
+
class AttemptResult:
|
|
518
|
+
request_id: str
|
|
519
|
+
attempt: int
|
|
520
|
+
outcome: Literal['succeeded', 'conflict', 'failed']
|
|
521
|
+
ret_val: dict | None = None
|
|
522
|
+
error: dict | None = None # the failing KernelError.to_dict()
|
|
523
|
+
```
|
|
524
|
+
|
|
525
|
+
Both have `to_dict()` and `from_dict()`, like every shared type.
|
|
526
|
+
|
|
527
|
+
Invariants:
|
|
528
|
+
|
|
529
|
+
- The `to_dict()` form never carries a token, a live entity, or a pickled object.
|
|
530
|
+
The environment acts as `Principal(owner_id)`, identity-only, so an
|
|
531
|
+
out-of-process attempt cannot perform delegated backend-to-backend calls.
|
|
532
|
+
- An attempt carries the request by reference and the fields the runner
|
|
533
|
+
needs to start without a kernel. Everything else the runner reads comes
|
|
534
|
+
through its session.
|
|
535
|
+
- An `AttemptResult` never carries an entity. The target's final state is
|
|
536
|
+
written by the runner through its session, not returned.
|
|
537
|
+
|
|
538
|
+
### The dispatcher port
|
|
539
|
+
|
|
540
|
+
```python
|
|
541
|
+
class Dispatcher(ABC):
|
|
542
|
+
@abstractmethod
|
|
543
|
+
async def dispatch(self, attempt: ExecutionAttempt) -> None:
|
|
544
|
+
"""Deliver the attempt to the environment and return once it is accepted."""
|
|
545
|
+
|
|
546
|
+
async def drain(self) -> None:
|
|
547
|
+
"""Wait for attempts this process is running on the current loop."""
|
|
548
|
+
```
|
|
549
|
+
|
|
550
|
+
Contract:
|
|
551
|
+
|
|
552
|
+
- `dispatch` returns when the environment has accepted the attempt, not when
|
|
553
|
+
it finishes. It raises only if the attempt could not be delivered.
|
|
554
|
+
- The result of the attempt reaches the kernel through
|
|
555
|
+
`StrategyExecutorService.finish_attempt`, however the environment gets it
|
|
556
|
+
there. `dispatch` never returns a result.
|
|
557
|
+
- A dispatcher performs no persistence, authorization, or reporting.
|
|
558
|
+
- A dispatcher may deliver an attempt more than once. The entry point's
|
|
559
|
+
claim makes every delivery after the first run nothing.
|
|
560
|
+
- `drain` returns once the attempts this process started on the current
|
|
561
|
+
event loop have finished. An entry point that owns its loop, such as a
|
|
562
|
+
Celery task, drains every dispatcher before returning, so an IO child
|
|
563
|
+
submitted by a CPU strategy is not lost when the loop closes.
|
|
564
|
+
|
|
565
|
+
### Execution environments
|
|
566
|
+
|
|
567
|
+
An environment is one module holding its dispatcher and its entry point:
|
|
568
|
+
the two halves of one serialization, `to_dict()` on the kernel side and
|
|
569
|
+
`from_dict()` on the environment's. The entry point builds a
|
|
570
|
+
`ChangeTracker` and a session, runs the runner, and reports the result. Which
|
|
571
|
+
session it builds follows one rule:
|
|
572
|
+
|
|
573
|
+
- An environment with a runtime builds `LocalStrategySession` from that
|
|
574
|
+
runtime's services and calls `finish_attempt` on its own executor.
|
|
575
|
+
- An environment without a runtime builds `RemoteStrategySession` over a
|
|
576
|
+
`KernelClient` and posts its result to the execution transport.
|
|
577
|
+
|
|
578
|
+
Nothing else differs. An environment moves from one kind to the other by
|
|
579
|
+
changing its entry point.
|
|
580
|
+
|
|
581
|
+
#### In-process (`execution/environments/in_process.py`)
|
|
582
|
+
|
|
583
|
+
The kernel process's own event loop, so the dispatcher is also the entry
|
|
584
|
+
point:
|
|
585
|
+
|
|
586
|
+
```python
|
|
587
|
+
class InProcessDispatcher(Dispatcher):
|
|
588
|
+
def __init__(self, executor, entity_service, registry): ...
|
|
589
|
+
|
|
590
|
+
async def dispatch(self, attempt):
|
|
591
|
+
asyncio.create_task(self._run(attempt))
|
|
592
|
+
|
|
593
|
+
async def _run(self, attempt):
|
|
594
|
+
if not await self.executor.claim_attempt(attempt):
|
|
595
|
+
return
|
|
596
|
+
tracker = ChangeTracker()
|
|
597
|
+
session = LocalStrategySession(attempt, tracker, self.entity_service,
|
|
598
|
+
self.executor, self.registry)
|
|
599
|
+
result = await StrategyRunner(self.registry).run(attempt, session, tracker)
|
|
600
|
+
await self.executor.finish_attempt(result)
|
|
601
|
+
```
|
|
602
|
+
|
|
603
|
+
A failure the runner does not catch becomes a `failed` result.
|
|
604
|
+
|
|
605
|
+
#### Celery (`execution/environments/celery.py`)
|
|
606
|
+
|
|
607
|
+
A Celery process that holds a runtime. The dispatcher enqueues the attempt's
|
|
608
|
+
`to_dict()` form from a plain executor thread, so an eager-mode task runs on
|
|
609
|
+
its own event loop, as in a worker, never on the caller's. The task is the
|
|
610
|
+
entry point:
|
|
611
|
+
|
|
612
|
+
```python
|
|
613
|
+
class CeleryDispatcher(Dispatcher):
|
|
614
|
+
async def dispatch(self, attempt):
|
|
615
|
+
await asyncio.get_running_loop().run_in_executor(
|
|
616
|
+
None, run_attempt.delay, attempt.to_dict())
|
|
617
|
+
|
|
618
|
+
@shared_task(name='graphcore.tasks.run_attempt')
|
|
619
|
+
def run_attempt(data):
|
|
620
|
+
rt = get_runtime()
|
|
621
|
+
attempt = ExecutionAttempt.from_dict(data)
|
|
622
|
+
|
|
623
|
+
async def run():
|
|
624
|
+
if not await rt.executor.claim_attempt(attempt):
|
|
625
|
+
return
|
|
626
|
+
tracker = ChangeTracker()
|
|
627
|
+
session = LocalStrategySession(attempt, tracker, rt.entity_service, rt.executor, rt.registry)
|
|
628
|
+
result = await StrategyRunner(rt.registry).run(attempt, session, tracker)
|
|
629
|
+
await rt.executor.finish_attempt(result)
|
|
630
|
+
await settle(rt.executor) # every dispatcher's drain()
|
|
631
|
+
|
|
632
|
+
async_to_sync(run)()
|
|
633
|
+
```
|
|
634
|
+
|
|
635
|
+
Every kernel task has an explicit `graphcore.tasks.*` name, so the name a
|
|
636
|
+
producer enqueues never depends on a module path.
|
|
637
|
+
|
|
638
|
+
The same process consumes the kernel's periodic tasks (`reconcile_runs`,
|
|
639
|
+
and the scheduler's tasks, `docs/SCHEDULING_TDD.md`).
|
|
640
|
+
|
|
641
|
+
#### An environment without a runtime
|
|
642
|
+
|
|
643
|
+
A project's sandbox or GPU host supplies its own dispatcher and an entry
|
|
644
|
+
point of the same shape, built from `graphcore.public`:
|
|
645
|
+
|
|
646
|
+
```python
|
|
647
|
+
kernel = KernelClient(settings.GRAPHCORE_KERNEL_URL, settings.GRAPHCORE_ENVIRONMENT_SECRET)
|
|
648
|
+
runner = StrategyRunner(registry) # the project's registered strategies
|
|
649
|
+
|
|
650
|
+
def run_attempt(data):
|
|
651
|
+
attempt = ExecutionAttempt.from_dict(data)
|
|
652
|
+
if not kernel.claim_attempt(attempt):
|
|
653
|
+
return
|
|
654
|
+
tracker = ChangeTracker()
|
|
655
|
+
session = RemoteStrategySession(attempt, tracker, kernel)
|
|
656
|
+
result = async_to_sync(runner.run)(attempt, session, tracker)
|
|
657
|
+
kernel.finish_attempt(result)
|
|
658
|
+
```
|
|
659
|
+
|
|
660
|
+
Its only configuration is the kernel URL and the environment credential.
|
|
661
|
+
The strategy code must be importable there; nothing else from the kernel is
|
|
662
|
+
needed.
|
|
663
|
+
|
|
664
|
+
Invariants:
|
|
665
|
+
|
|
666
|
+
- Only the composition root constructs a dispatcher.
|
|
667
|
+
- `execution/environments/celery.py` is the only module in `execution/`
|
|
668
|
+
that imports Celery, apart from `execution/tasks.py`.
|
|
669
|
+
- An entry point never writes a `StrategyRequest` and never opens
|
|
670
|
+
`kernel_internal_context()`; it claims through `claim_attempt` and
|
|
671
|
+
reports through `finish_attempt`.
|
|
672
|
+
- An entry point runs an attempt only after claiming it.
|
|
673
|
+
|
|
674
|
+
### The runner
|
|
675
|
+
|
|
676
|
+
`StrategyRunner(registry).run(attempt, session, tracker) -> AttemptResult`
|
|
677
|
+
performs one attempt. It is domain code: it imports `Strategy`, `StrategySession`,
|
|
678
|
+
`ChangeTracker`, and the registry, and nothing from the kernel's services.
|
|
679
|
+
|
|
680
|
+
1. Load the target through the session, if the request has one; a missing
|
|
681
|
+
target is a `failed` result.
|
|
682
|
+
2. Construct the strategy with the session, validate `param_config` against
|
|
683
|
+
its schema, check `verify_executable()`, and await `apply()`. An invalid
|
|
684
|
+
`param_config` fails with the path of the offending parameter.
|
|
685
|
+
3. If the strategy is not `read_only_strategy` and
|
|
686
|
+
`tracker.is_unsaved(target)`, save the returned target through the
|
|
687
|
+
session. This is the auto-save: the same path, chokepoint, and fence as
|
|
688
|
+
any save the strategy made itself, so a target the strategy already saved,
|
|
689
|
+
or never changed, is not saved again.
|
|
690
|
+
4. Run `cleanup()`; a cleanup error is logged, never raised.
|
|
691
|
+
5. Return `succeeded` with `strategy_request.ret_val`, `conflict` on
|
|
692
|
+
`ConcurrentModificationError` from `apply()` or the auto-save, or `failed`
|
|
693
|
+
on any other exception. A failure carries its error as `to_dict()`; an
|
|
694
|
+
exception that is not a `KernelError` is reported as `StrategyFailed`.
|
|
695
|
+
`StaleAttempt` is `failed`: the kernel has already moved on.
|
|
696
|
+
|
|
697
|
+
```python
|
|
698
|
+
class ChangeTracker:
|
|
699
|
+
def record(self, entity: Entity) -> None: ... # called by session adapters
|
|
700
|
+
def is_unsaved(self, entity: Entity) -> bool: ... # called by the runner
|
|
701
|
+
```
|
|
702
|
+
|
|
703
|
+
`is_unsaved(entity)` is true when the entity differs from its recorded state
|
|
704
|
+
or has none.
|
|
705
|
+
|
|
706
|
+
Invariants:
|
|
707
|
+
|
|
708
|
+
- A strategy is constructed only by the runner. The runner opens no
|
|
709
|
+
context: the session opens `strategy_context` around each write, on the
|
|
710
|
+
kernel side for a remote session.
|
|
711
|
+
- The runner writes only through the session. It never writes a
|
|
712
|
+
`StrategyRequest` and never opens `kernel_internal_context()`.
|
|
713
|
+
- `apply()` runs once per attempt. A strategy is safe to run more than once.
|
|
714
|
+
- The runner performs no authorization and no retry. Both belong to the
|
|
715
|
+
kernel.
|
|
716
|
+
|
|
717
|
+
### The executor
|
|
718
|
+
|
|
719
|
+
`StrategyExecutorService` lives in every kernel process. It owns three
|
|
720
|
+
operations.
|
|
721
|
+
|
|
722
|
+
#### The front door
|
|
723
|
+
|
|
724
|
+
`execute_request(request, principal, wait=True, callback_channel=None)
|
|
725
|
+
-> tuple[StrategyRequest, Entity | None]`:
|
|
726
|
+
|
|
727
|
+
1. Refuse a request whose id already exists. The front door only creates
|
|
728
|
+
requests.
|
|
729
|
+
2. Resolve the strategy class; an unregistered name raises `InvalidRequest`.
|
|
730
|
+
3. Authorize (`_can_execute`): allowlist on the target's type, read
|
|
731
|
+
permission and the strategy's required permission on the persisted
|
|
732
|
+
target, or on the deployment for an untargeted strategy.
|
|
733
|
+
4. Record `callback_channel` on the request and save it with
|
|
734
|
+
`status = pending`, `attempt = 1`, and
|
|
735
|
+
`attempt_deadline = now + dispatch_timeout` as `principal`, in
|
|
736
|
+
`kernel_internal_context()`. The chokepoint sets its `owner` to
|
|
737
|
+
`principal`. Should the process stop before step 5, the reconciler sends
|
|
738
|
+
attempt 1 at that deadline.
|
|
739
|
+
5. Dispatch attempt 1 as Starting an attempt's step 3 does, without its
|
|
740
|
+
re-authorization: step 3 above just authorized the request's owner.
|
|
741
|
+
6. With `wait=False`, return `(request, None)`. With `wait=True`, wait for
|
|
742
|
+
the request to reach a terminal status (see Waiting). If it `succeeded`,
|
|
743
|
+
return `(request, target)`, where `target` is the target reloaded as the
|
|
744
|
+
principal, or `None` for an untargeted request. If it `failed`, raise
|
|
745
|
+
`request.failure()`: the `KernelError` of the class whose `code` the
|
|
746
|
+
request recorded, with its message and details and the request's id as
|
|
747
|
+
`strategy_request_id`, or `StrategyFailed` for an unexpected error.
|
|
748
|
+
|
|
749
|
+
The front door does not know who called it. A client transport, a
|
|
750
|
+
strategy's nested `execute`, and a scheduler's tick all get the same
|
|
751
|
+
result, and each decides what a raised error means to it. GraphAPI turns it
|
|
752
|
+
into a `failed` `StrategyOutcome`.
|
|
753
|
+
|
|
754
|
+
A request refused at steps 1 to 3 is not saved.
|
|
755
|
+
|
|
756
|
+
#### Starting an attempt
|
|
757
|
+
|
|
758
|
+
`start_attempt(request)` sends attempt `request.attempt + 1`; the
|
|
759
|
+
reconciler also resends an unclaimed attempt. Sending attempt `n`:
|
|
760
|
+
|
|
761
|
+
1. Re-run `_can_execute` for the request's owner against current persisted
|
|
762
|
+
state. On refusal, finish the run as `failed`.
|
|
763
|
+
2. In `kernel_internal_context()`, save the request with
|
|
764
|
+
`status = pending`, `attempt = n`, and
|
|
765
|
+
`attempt_deadline = now + dispatch_timeout`.
|
|
766
|
+
3. Build the `ExecutionAttempt` and call
|
|
767
|
+
`dispatchers[strategy_cls.workload].dispatch(attempt)`. If dispatch
|
|
768
|
+
raises, finish the run as `failed`.
|
|
769
|
+
|
|
770
|
+
#### Claiming an attempt
|
|
771
|
+
|
|
772
|
+
`claim_attempt(attempt: ExecutionAttempt) -> bool`, called by the entry
|
|
773
|
+
point before it runs the attempt:
|
|
774
|
+
|
|
775
|
+
1. Load the request. If it is not `pending` with `attempt.attempt`, return
|
|
776
|
+
`False`.
|
|
777
|
+
2. In `kernel_internal_context()`, save it with `status = running` and
|
|
778
|
+
`attempt_deadline = now + (strategy.timeout or default_timeout)`. A
|
|
779
|
+
version conflict means another delivery claimed it first: return
|
|
780
|
+
`False`.
|
|
781
|
+
3. Return `True`.
|
|
782
|
+
|
|
783
|
+
The strategy's timeout therefore counts from when an environment starts the
|
|
784
|
+
attempt, not from when it was queued. An entry point given `False` runs
|
|
785
|
+
nothing and reports nothing.
|
|
786
|
+
|
|
787
|
+
#### Finishing an attempt
|
|
788
|
+
|
|
789
|
+
`finish_attempt(result: AttemptResult)`:
|
|
790
|
+
|
|
791
|
+
1. Load the request. If it is not `running` or `result.attempt !=
|
|
792
|
+
request.attempt`, discard the result and return. This is the fence: a
|
|
793
|
+
result from an attempt the reconciler has superseded, or a duplicate
|
|
794
|
+
delivery, has no effect.
|
|
795
|
+
2. On `succeeded`: in `kernel_internal_context()`, save
|
|
796
|
+
`status = succeeded` and `ret_val`; create a request from each of the
|
|
797
|
+
request's `callbacks` and submit it through the front door as the owner
|
|
798
|
+
with `wait=False`; send `strategy_complete` with `success: True` to
|
|
799
|
+
`callback_channel`, echoing `client_request_id`.
|
|
800
|
+
3. On `conflict`: if `request.attempt < max_attempts`, wait
|
|
801
|
+
`0.1 * 2**attempt` seconds plus up to 0.1 seconds of jitter, then
|
|
802
|
+
`start_attempt(request)`. Otherwise finish as `failed`.
|
|
803
|
+
4. On `failed`: in `kernel_internal_context()`, save `status = failed` and
|
|
804
|
+
the error from `KernelError.from_dict(result.error)` as `error`,
|
|
805
|
+
`error_code`, and `error_details`; send `strategy_complete` with
|
|
806
|
+
`success: False`, the error, and its `code`, echoing `client_request_id`.
|
|
807
|
+
|
|
808
|
+
Every step is idempotent for a given `(request_id, attempt)`.
|
|
809
|
+
|
|
810
|
+
#### Waiting
|
|
811
|
+
|
|
812
|
+
Waiting is event-driven through the `RunEvents` port:
|
|
813
|
+
|
|
814
|
+
```python
|
|
815
|
+
class RunEvents(ABC):
|
|
816
|
+
def subscribe(self, request_id) -> AsyncContextManager[RunSubscription]: ...
|
|
817
|
+
async def publish(self, request_id) -> None: ... # never raises
|
|
818
|
+
|
|
819
|
+
class RunSubscription:
|
|
820
|
+
async def wait(self, until: Optional[float]) -> None: ...
|
|
821
|
+
```
|
|
822
|
+
|
|
823
|
+
`wait=True` subscribes to the request before dispatching, then loops:
|
|
824
|
+
read the request; return it if its status is terminal; if it is past its
|
|
825
|
+
`attempt_deadline`, reconcile it (see Run reconciliation) and read
|
|
826
|
+
again; otherwise wait on the subscription until it is notified or the
|
|
827
|
+
clock reaches `attempt_deadline`. Whichever process saves a terminal
|
|
828
|
+
status publishes the request's id after the save.
|
|
829
|
+
|
|
830
|
+
A notification carries only the id, never the outcome: the waiter always
|
|
831
|
+
reads the request, so a duplicate, early, or lost notification changes
|
|
832
|
+
nothing but when it reads. The attempt deadline is the only timer. A lost
|
|
833
|
+
notification delays the waiter to that deadline, at which it reads the
|
|
834
|
+
request, finds it terminal, and returns.
|
|
835
|
+
|
|
836
|
+
`InMemoryRunEvents` notifies subscribers in its own process, from any
|
|
837
|
+
thread or event loop. `RedisRunEvents` also publishes on one Redis channel
|
|
838
|
+
and runs one listener thread per process, started on its first
|
|
839
|
+
subscription. Each time the listener subscribes or resubscribes, it
|
|
840
|
+
notifies every subscriber in its process, so each re-reads its request and
|
|
841
|
+
nothing published while the listener was disconnected is relied on.
|
|
842
|
+
|
|
843
|
+
Invariants:
|
|
844
|
+
|
|
845
|
+
- Every attempt is authorized in the kernel when it starts, so an owner who
|
|
846
|
+
lost access stops running, including on reconciled re-dispatch.
|
|
847
|
+
- `kernel_internal_context()` raises when opened while a strategy context is
|
|
848
|
+
active. The executor opens it only to write requests.
|
|
849
|
+
- The `strategy_complete` message has one shape for every environment,
|
|
850
|
+
including `client_request_id`.
|
|
851
|
+
- Nothing in the submitting process holds the run. The result is handled by
|
|
852
|
+
whichever kernel process receives it.
|
|
853
|
+
|
|
854
|
+
### Run reconciliation
|
|
855
|
+
|
|
856
|
+
Requests are the single source of truth for runs; the executor reconciles
|
|
857
|
+
toward them (`ARCHITECTURE_PRINCIPLES.md`, rule 8). `finish_attempt` only
|
|
858
|
+
runs if the environment lives long enough to report, so the
|
|
859
|
+
`graphcore.tasks.reconcile_runs` task runs in the Celery process when a
|
|
860
|
+
worker starts and every `GRAPHCORE_RECONCILE_INTERVAL` seconds (300 by
|
|
861
|
+
default), through the `graphcore_reconcile_runs` periodic task that
|
|
862
|
+
`BeatScheduler` ensures alongside its own (`docs/SCHEDULING_TDD.md`). For each request with `status` `pending` or `running` and
|
|
863
|
+
`attempt_deadline` in the past:
|
|
864
|
+
|
|
865
|
+
- if `pending`, no environment claimed the attempt, or the front door
|
|
866
|
+
stopped before dispatching: send attempt `attempt` again. The
|
|
867
|
+
same attempt number is sent, so a slow queue costs the run no attempts,
|
|
868
|
+
and whichever delivery claims first runs;
|
|
869
|
+
- if `running` and `attempt < max_attempts`, `start_attempt(request)`;
|
|
870
|
+
- otherwise finish the run as `failed` with error `attempt timed out`.
|
|
871
|
+
|
|
872
|
+
Because `start_attempt` increments `attempt`, any late result or write from
|
|
873
|
+
the superseded attempt is refused by the fence.
|
|
874
|
+
|
|
875
|
+
Invariants:
|
|
876
|
+
|
|
877
|
+
- A request never stays `pending` or `running` past its deadline plus one
|
|
878
|
+
reconcile interval. A request with a waiter is reconciled by that waiter
|
|
879
|
+
at its deadline.
|
|
880
|
+
- A re-dispatched attempt is authorized and fenced exactly like a first one.
|
|
881
|
+
|
|
882
|
+
### Composition
|
|
883
|
+
|
|
884
|
+
`build_runtime` receives the dispatcher mapping and the executor's settings
|
|
885
|
+
and constructs `StrategyExecutorService(registry, entity_service,
|
|
886
|
+
default_timeout, max_attempts, run_events)`, then calls `executor.bind_dispatchers`
|
|
887
|
+
with the mapping a dispatcher factory `(executor, entity_service, registry)`
|
|
888
|
+
returns, since an in-process dispatcher reports back to the executor.
|
|
889
|
+
`bind_dispatchers` is called only there, and it raises if a registered
|
|
890
|
+
strategy's workload has no dispatcher. `InProcessDispatcher` is
|
|
891
|
+
constructed there with the executor, `entity_service`, and the registry.
|
|
892
|
+
`run_events` defaults to `InMemoryRunEvents`. The Django runtime chooses it
|
|
893
|
+
from `CELERY_BROKER_URL`: `memory://` (Celery eager, one process) gives
|
|
894
|
+
`InMemoryRunEvents`, `redis://` or `rediss://` gives `RedisRunEvents` on
|
|
895
|
+
the broker's Redis, and any other broker fails at startup, since a run
|
|
896
|
+
finished by a worker would never wake its waiter. A project's runtime
|
|
897
|
+
factory can pass its own adapter.
|
|
898
|
+
`transport/execution.py` is mounted by the ASGI process's URL
|
|
899
|
+
configuration, behind the environment credential.
|
|
900
|
+
|
|
901
|
+
`Workload`, `Dispatcher`, `ExecutionAttempt`, `AttemptResult`,
|
|
902
|
+
`StrategySession`, `StrategyRunner`, `ChangeTracker`,
|
|
903
|
+
`RemoteStrategySession`, and `KernelClient` are exported from
|
|
904
|
+
`graphcore.public`, so a project's runtime factory
|
|
905
|
+
(`GRAPHCORE_RUNTIME_FACTORY`) can supply its own dispatchers and a project's
|
|
906
|
+
environment can run an entry point of its own.
|
|
907
|
+
|
|
908
|
+
Boundary tests in `tests/test_composition_boundaries.py` hold:
|
|
909
|
+
|
|
910
|
+
- Only the composition root constructs a `Dispatcher` adapter or
|
|
911
|
+
`RedisRunEvents`.
|
|
912
|
+
- Only `execution/environments/celery.py` and `execution/tasks.py` import
|
|
913
|
+
Celery within `execution/`.
|
|
914
|
+
- `domain/strategy/` imports nothing from `graph/`, `execution/`,
|
|
915
|
+
`persistence/`, `broadcast/`, `scheduling/`, or `auth/authz/adapter/`.
|
|
916
|
+
`StrategyRunner` is in `domain/strategy/` and is held to the same rule.
|
|
917
|
+
- `execution/remote_session.py` imports nothing from `graph/`,
|
|
918
|
+
`persistence/`, `broadcast/`, `scheduling/`, or `auth/`, and never calls
|
|
919
|
+
`get_runtime()`.
|
|
920
|
+
|
|
921
|
+
Conformance suites, exported from `graphcore.public.testing`:
|
|
922
|
+
|
|
923
|
+
- `DispatcherContract`: `dispatch` returns before the attempt finishes;
|
|
924
|
+
the attempt's result reaches `finish_attempt`; a failing runner yields a
|
|
925
|
+
`failed` result. Runs against `InProcessDispatcher` and against
|
|
926
|
+
`CeleryDispatcher` with Celery in eager mode.
|
|
927
|
+
- `StrategySessionContract`: the session contract above, including the
|
|
928
|
+
fence, against `LocalStrategySession` and against `RemoteStrategySession`
|
|
929
|
+
over the execution transport.
|
|
930
|
+
|
|
931
|
+
### Worked example: a CPU workload
|
|
932
|
+
|
|
933
|
+
```python
|
|
934
|
+
@register_strategy(Document)
|
|
935
|
+
class EmbedDocument(Strategy):
|
|
936
|
+
workload = Workload.CPU
|
|
937
|
+
timeout = 1800
|
|
938
|
+
|
|
939
|
+
async def apply(self):
|
|
940
|
+
chunks = await self.session.list(EntityFilter(type_ids=frozenset({'Chunk'}),
|
|
941
|
+
parent_id=self.entity.entity_id))
|
|
942
|
+
self.entity.embedding = embed([c.text for c in chunks.items])
|
|
943
|
+
return self.strategy_request, self.entity
|
|
944
|
+
```
|
|
945
|
+
|
|
946
|
+
1. A client executes `EmbedDocument` through GraphAPI. The front door
|
|
947
|
+
authorizes it, saves the request with `status = pending`, `attempt = 1`
|
|
948
|
+
and a dispatch deadline, and `CeleryDispatcher` enqueues the attempt.
|
|
949
|
+
2. The Celery process's `run_attempt` claims attempt 1, which saves it
|
|
950
|
+
`running` with `attempt_deadline` 30 minutes out, then builds a
|
|
951
|
+
`LocalStrategySession` for it from its runtime and runs the runner. `apply()`'s `list` and
|
|
952
|
+
the auto-save go through that session, which checks the fence, opens
|
|
953
|
+
`strategy_context(EmbedDocument)`, and writes through the chokepoint as
|
|
954
|
+
the request's owner.
|
|
955
|
+
3. `run_attempt` passes its `AttemptResult` to its own executor's
|
|
956
|
+
`finish_attempt`, which saves `succeeded` and sends `strategy_complete`
|
|
957
|
+
to the client's channel.
|
|
958
|
+
4. With `wait=True`, the ASGI process's front door sees the request's
|
|
959
|
+
`succeeded` save on its subscription, reloads the target, and returns
|
|
960
|
+
`(request, entity)`.
|
|
961
|
+
|
|
962
|
+
If the Celery process dies during step 2, `reconcile_runs` finds the request
|
|
963
|
+
past its deadline; `start_attempt` saves it with `attempt = 2` and a new
|
|
964
|
+
`attempt_deadline`, and dispatches again. A save from
|
|
965
|
+
the dead attempt, should it ever arrive, is refused by the fence.
|
|
966
|
+
|
|
967
|
+
The strategy's code is the same with `workload = Workload.IO`; only the
|
|
968
|
+
dispatcher differs, and the attempt runs in the ASGI process. It is also the
|
|
969
|
+
same in an environment without a runtime; only the entry point differs.
|
|
970
|
+
|
|
971
|
+
### Trade-offs
|
|
972
|
+
|
|
973
|
+
- **Every remote read and write is a round trip.** An environment without
|
|
974
|
+
a runtime reaches the graph through the execution transport. For CPU
|
|
975
|
+
workloads, which read once, compute, and save once, the cost is noise; a
|
|
976
|
+
strategy that performs many small reads is an IO workload.
|
|
977
|
+
- **`apply()` reruns on conflict and on timeout.** Every attempt repeats the
|
|
978
|
+
whole strategy, including expensive CPU work. Strategies with costly
|
|
979
|
+
`apply()` bodies keep their contended write small and late, and declare a
|
|
980
|
+
`timeout` that covers their worst case.
|
|
981
|
+
- **Sessions are a contract, not a sandbox.** In a kernel process, strategy
|
|
982
|
+
code can import kernel modules. The session is the supported surface for
|
|
983
|
+
strategies; the boundary tests enforce it for the kernel's own code. In
|
|
984
|
+
an environment without a runtime, the session is the only path there is.
|
|
985
|
+
- **Out-of-process attempts are identity-only.** A CPU workload cannot make
|
|
986
|
+
delegated backend-to-backend calls, which follows from requirement 10.
|
|
987
|
+
- **The Celery process holds a runtime.** It has database and broker
|
|
988
|
+
credentials and could reach the graph around its session; the boundary
|
|
989
|
+
tests, not isolation, keep it on the session. Moving it to the remote
|
|
990
|
+
session is an entry-point change, taken when isolation matters more than
|
|
991
|
+
the round trips.
|
|
992
|
+
- **Request writes per run.** A run costs a `pending` save, a claim save per
|
|
993
|
+
attempt, a dispatch save per retry, and a terminal save, each broadcast, in
|
|
994
|
+
exchange for a durable, observable history of every run, a timeout that
|
|
995
|
+
excludes queue time, and a reconciler that needs no other state.
|
|
996
|
+
- **A slow queue is sent duplicates.** An attempt still queued at its
|
|
997
|
+
dispatch deadline is sent again, once per deadline, until a delivery
|
|
998
|
+
claims it; every later delivery is a claim that fails and runs nothing.
|
|
999
|
+
With no worker consuming, a run stays `pending` rather than failing.
|
|
1000
|
+
- **CPU work in a worker stays in that worker.** A CPU request submitted
|
|
1001
|
+
from a Celery worker, nested, scheduled, or reconciled, runs on that
|
|
1002
|
+
worker's loop rather than on another worker, so the task that submitted it
|
|
1003
|
+
lasts until it finishes.
|
|
1004
|
+
- **Created identifiers follow from the request.** Anyone who can read a
|
|
1005
|
+
request can compute the identifiers its strategy will create. Taking one
|
|
1006
|
+
first makes that run fail; it grants nothing, since the run's save is
|
|
1007
|
+
still authorized against the stored entity.
|
|
1008
|
+
- **A fence per write.** Every session write locks the request's row for
|
|
1009
|
+
the length of its transaction, so writes of one attempt serialize on it,
|
|
1010
|
+
and a reconciler's `start_attempt` waits for an in-flight write to commit.
|
|
1011
|
+
Every execution-transport call also checks the fence, reads included.
|
|
1012
|
+
- **Waiting across processes needs a broker.** A waiter in one process
|
|
1013
|
+
learns of a run finished in another only through a cross-process
|
|
1014
|
+
`RunEvents` adapter, so the Django runtime refuses a broker it has no
|
|
1015
|
+
adapter for. Each process holds one listener connection to Redis for as
|
|
1016
|
+
long as it has had a waiter.
|
|
1017
|
+
- **Strategies see only what their principal sees.** A kernel strategy
|
|
1018
|
+
validates against principal-scoped reads like any other. An administrator
|
|
1019
|
+
can name, for example, only groups they can read as new-user defaults.
|
|
1020
|
+
- **A scheduled tick is submitted from the Celery process.** An IO strategy
|
|
1021
|
+
on a schedule therefore runs on that process's event loop through its
|
|
1022
|
+
`InProcessDispatcher`, not in the ASGI process. Workload decides the kind
|
|
1023
|
+
of environment, not a specific process.
|
|
1024
|
+
|
|
1025
|
+
### Open Items
|
|
1026
|
+
|
|
1027
|
+
- **Environment authentication.** The shipped default is a shared secret.
|
|
1028
|
+
Whether the kernel also issues per-attempt tokens so an environment
|
|
1029
|
+
credential alone cannot act for an attempt it was not dispatched, and
|
|
1030
|
+
whether mTLS is the shipped default for deployments with a network
|
|
1031
|
+
boundary.
|
|
1032
|
+
- **Authorization at template creation.** Creating a template checks only
|
|
1033
|
+
that its creator may create a `RequestTemplate`. Whether the owner may
|
|
1034
|
+
execute the named strategy on its target is checked for each request made
|
|
1035
|
+
from it. Whether to also check it when the template is created, and
|
|
1036
|
+
through which session operation.
|
|
1037
|
+
- **Request retention.** Requests accumulate. Whether to prune finished
|
|
1038
|
+
ones, after how long, and whether pruning is a reconciler over requests.
|
|
1039
|
+
- **Reconcile interval and default timeout.** Whether 600 seconds per
|
|
1040
|
+
attempt and the reconcile interval are exposed as settings, and what
|
|
1041
|
+
defaults suit deployments with long-running CPU strategies.
|
|
1042
|
+
- **GPU and other kernel workloads.** The kernel defines further `Workload`
|
|
1043
|
+
values only when it ships a dispatcher for them.
|