graphcore-kernel 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- graphcore/__init__.py +5 -0
- graphcore/agent_frontend/__init__.py +8 -0
- graphcore/agent_frontend/api.py +69 -0
- graphcore/agent_frontend/base/__init__.py +11 -0
- graphcore/agent_frontend/base/views.py +167 -0
- graphcore/agent_frontend/client.py +31 -0
- graphcore/agent_frontend/context.py +157 -0
- graphcore/agent_frontend/errors.py +12 -0
- graphcore/agent_frontend/loading.py +54 -0
- graphcore/agent_frontend/registry.py +42 -0
- graphcore/agent_frontend/render.py +207 -0
- graphcore/agent_frontend/server.py +103 -0
- graphcore/agent_frontend/signatures.py +35 -0
- graphcore/agent_frontend/testing.py +45 -0
- graphcore/agent_frontend/tests/__init__.py +0 -0
- graphcore/agent_frontend/tests/fakes.py +114 -0
- graphcore/agent_frontend/tests/test_frontend.py +447 -0
- graphcore/agent_frontend/tests/test_loading.py +83 -0
- graphcore/agent_frontend/tests/test_server.py +109 -0
- graphcore/agent_frontend/tools.py +137 -0
- graphcore/api/__init__.py +4 -0
- graphcore/api/graph_api.py +378 -0
- graphcore/api/predicate_codec.py +81 -0
- graphcore/api/tests/__init__.py +0 -0
- graphcore/api/tests/test_changed_entities.py +109 -0
- graphcore/api/tests/test_graph_api.py +290 -0
- graphcore/api/tests/test_graph_api_access.py +94 -0
- graphcore/api/tests/test_graph_api_grants.py +90 -0
- graphcore/api/tests/test_predicate_codec.py +83 -0
- graphcore/api/tests/test_subtree_query.py +79 -0
- graphcore/apps.py +71 -0
- graphcore/auth/__init__.py +1 -0
- graphcore/auth/authn/README.md +178 -0
- graphcore/auth/authn/__init__.py +3 -0
- graphcore/auth/authn/authenticator.py +120 -0
- graphcore/auth/authn/onboarding.py +105 -0
- graphcore/auth/authn/principals.py +20 -0
- graphcore/auth/authn/resource_metadata.py +43 -0
- graphcore/auth/authn/service_accounts.py +96 -0
- graphcore/auth/authn/testing.py +96 -0
- graphcore/auth/authn/tests/__init__.py +0 -0
- graphcore/auth/authn/tests/test_authenticator.py +145 -0
- graphcore/auth/authn/tests/test_onboarding.py +145 -0
- graphcore/auth/authn/tests/test_service_accounts.py +180 -0
- graphcore/auth/authn/users.py +59 -0
- graphcore/auth/authz/__init__.py +0 -0
- graphcore/auth/authz/access_port.py +120 -0
- graphcore/auth/authz/adapter/__init__.py +0 -0
- graphcore/auth/authz/adapter/default_access_adapter.py +556 -0
- graphcore/auth/authz/adapter/tests/__init__.py +0 -0
- graphcore/auth/authz/adapter/tests/test_default_adapter.py +543 -0
- graphcore/auth/authz/adapter/tests/test_entity_write_hooks.py +207 -0
- graphcore/auth/authz/adapter/tests/test_group_types.py +70 -0
- graphcore/auth/authz/adapter/tests/test_sync_support.py +209 -0
- graphcore/auth/authz/defaults.py +26 -0
- graphcore/auth/authz/kernel_policy.py +191 -0
- graphcore/auth/authz/kernel_writes.py +31 -0
- graphcore/auth/authz/lookups.py +25 -0
- graphcore/auth/authz/policy.py +403 -0
- graphcore/auth/authz/tests/__init__.py +0 -0
- graphcore/auth/authz/tests/test_can_contain.py +150 -0
- graphcore/auth/authz/tests/test_contained_by.py +167 -0
- graphcore/auth/authz/tests/test_creation.py +391 -0
- graphcore/auth/authz/tests/test_delete_permission.py +82 -0
- graphcore/auth/authz/tests/test_granted_to_me.py +160 -0
- graphcore/auth/authz/tests/test_integration.py +637 -0
- graphcore/auth/authz/tests/test_invites.py +184 -0
- graphcore/auth/authz/tests/test_kernel_policy.py +140 -0
- graphcore/auth/authz/tests/test_lockout.py +132 -0
- graphcore/auth/authz/tests/test_policy.py +329 -0
- graphcore/auth/authz/tests/test_primitives.py +73 -0
- graphcore/auth/authz/tests/test_principal_creation.py +62 -0
- graphcore/bootstrap/__init__.py +0 -0
- graphcore/bootstrap/deployment.py +67 -0
- graphcore/bootstrap/signals.py +16 -0
- graphcore/bootstrap/tests/__init__.py +0 -0
- graphcore/bootstrap/tests/test_deployment_seed.py +154 -0
- graphcore/broadcast/__init__.py +4 -0
- graphcore/broadcast/broadcaster.py +201 -0
- graphcore/broadcast/channels.py +56 -0
- graphcore/broadcast/tests/__init__.py +0 -0
- graphcore/broadcast/tests/test_broadcaster.py +444 -0
- graphcore/celery.py +14 -0
- graphcore/docs/AGENT_FRONTEND_TDD.md +389 -0
- graphcore/docs/ARCHITECTURE_PRINCIPLES.md +152 -0
- graphcore/docs/AUTHORIZATION_TECHNICAL_TDD.md +1379 -0
- graphcore/docs/ENTITY_REFERENCE_TDD.md +561 -0
- graphcore/docs/GRAPHAPI_FRONT_DOOR.md +173 -0
- graphcore/docs/RUNTIME.md +90 -0
- graphcore/docs/SCHEDULING_TDD.md +244 -0
- graphcore/docs/STRATEGY_EXECUTION_TDD.md +1043 -0
- graphcore/docs/VIEW_ENTITY_TDD.md +522 -0
- graphcore/docs/VIEW_SYSTEM_TDD.md +307 -0
- graphcore/docs/__init__.py +0 -0
- graphcore/docs/doc_topics.py +97 -0
- graphcore/docs/docs_discovery.py +64 -0
- graphcore/docs/public_api.py +56 -0
- graphcore/domain/__init__.py +0 -0
- graphcore/domain/entity/Entity.py +204 -0
- graphcore/domain/entity/__init__.py +0 -0
- graphcore/domain/entity/deployment/Deployment.py +25 -0
- graphcore/domain/entity/deployment/__init__.py +0 -0
- graphcore/domain/entity/grant/Grant.py +31 -0
- graphcore/domain/entity/grant/__init__.py +0 -0
- graphcore/domain/entity/group/Group.py +48 -0
- graphcore/domain/entity/group/README.md +88 -0
- graphcore/domain/entity/group/__init__.py +0 -0
- graphcore/domain/entity/invite/Invite.py +31 -0
- graphcore/domain/entity/invite/__init__.py +0 -0
- graphcore/domain/entity/request_template/RequestTemplate.py +48 -0
- graphcore/domain/entity/request_template/__init__.py +0 -0
- graphcore/domain/entity/request_template/tests/__init__.py +0 -0
- graphcore/domain/entity/request_template/tests/test_request_template.py +52 -0
- graphcore/domain/entity/service_account/ServiceAccount.py +41 -0
- graphcore/domain/entity/service_account/__init__.py +0 -0
- graphcore/domain/entity/strategy_request/StrategyRequest.py +61 -0
- graphcore/domain/entity/strategy_request/__init__.py +0 -0
- graphcore/domain/entity/strategy_request/tests/__init__.py +0 -0
- graphcore/domain/entity/strategy_request/tests/test_strategy_request_entity.py +35 -0
- graphcore/domain/entity/tests/__init__.py +0 -0
- graphcore/domain/entity/tests/test_entity.py +227 -0
- graphcore/domain/entity/tests/test_grant_group_entities.py +36 -0
- graphcore/domain/entity/tests/test_write_once_fields.py +98 -0
- graphcore/domain/entity/user/User.py +39 -0
- graphcore/domain/entity/user/__init__.py +0 -0
- graphcore/domain/entity/user/tests/__init__.py +0 -0
- graphcore/domain/entity/user/tests/test_user_entity.py +33 -0
- graphcore/domain/entity/view/View.py +37 -0
- graphcore/domain/entity/view/__init__.py +0 -0
- graphcore/domain/entity/view/position.py +78 -0
- graphcore/domain/entity/view/tests/__init__.py +0 -0
- graphcore/domain/entity/view/tests/test_position.py +71 -0
- graphcore/domain/entity/view/tests/test_view_entity.py +134 -0
- graphcore/domain/strategy/BaseStrategy.py +759 -0
- graphcore/domain/strategy/DeploymentStrategies.py +55 -0
- graphcore/domain/strategy/GrantStrategies.py +189 -0
- graphcore/domain/strategy/GroupStrategies.py +193 -0
- graphcore/domain/strategy/InviteStrategies.py +84 -0
- graphcore/domain/strategy/RequestTemplateStrategies.py +126 -0
- graphcore/domain/strategy/ViewStrategies.py +219 -0
- graphcore/domain/strategy/__init__.py +0 -0
- graphcore/domain/strategy/runner.py +84 -0
- graphcore/domain/strategy/session.py +56 -0
- graphcore/domain/strategy/tests/BaseStrategyTest.py +167 -0
- graphcore/domain/strategy/tests/__init__.py +0 -0
- graphcore/domain/strategy/tests/test_BaseStrategies.py +870 -0
- graphcore/domain/strategy/tests/test_grant_strategies.py +245 -0
- graphcore/domain/strategy/tests/test_group_strategies.py +99 -0
- graphcore/domain/strategy/tests/test_runner.py +220 -0
- graphcore/domain/strategy/tests/test_strategy_locks.py +446 -0
- graphcore/domain/strategy/tests/test_utils.py +95 -0
- graphcore/domain/strategy/tests/test_view_strategies.py +436 -0
- graphcore/errors.py +145 -0
- graphcore/execution/README.md +97 -0
- graphcore/execution/StrategyExecutorService.py +333 -0
- graphcore/execution/__init__.py +0 -0
- graphcore/execution/dispatcher.py +19 -0
- graphcore/execution/environments/__init__.py +0 -0
- graphcore/execution/environments/celery.py +59 -0
- graphcore/execution/environments/in_process.py +55 -0
- graphcore/execution/local_session.py +117 -0
- graphcore/execution/redis_run_events.py +74 -0
- graphcore/execution/requests.py +22 -0
- graphcore/execution/run_events.py +85 -0
- graphcore/execution/strategy_context.py +54 -0
- graphcore/execution/tasks.py +34 -0
- graphcore/execution/tests/__init__.py +5 -0
- graphcore/execution/tests/contract.py +171 -0
- graphcore/execution/tests/support.py +310 -0
- graphcore/execution/tests/test_dispatchers.py +76 -0
- graphcore/execution/tests/test_executor.py +368 -0
- graphcore/execution/tests/test_local_session.py +95 -0
- graphcore/execution/tests/test_reconcile_runs.py +134 -0
- graphcore/execution/tests/test_run_events.py +106 -0
- graphcore/execution/tests/test_strategy_context.py +62 -0
- graphcore/gateway/__init__.py +1 -0
- graphcore/gateway/remote.py +35 -0
- graphcore/gateway/tests/__init__.py +0 -0
- graphcore/gateway/tests/test_remote.py +30 -0
- graphcore/graph/EntityService.py +710 -0
- graphcore/graph/__init__.py +0 -0
- graphcore/graph/entity_cache.py +152 -0
- graphcore/graph/tests/__init__.py +4 -0
- graphcore/graph/tests/test_EntityService.py +908 -0
- graphcore/graph/tests/test_EntityService_list.py +170 -0
- graphcore/graph/tests/test_EntityService_model_resolution.py +339 -0
- graphcore/graph/tests/test_EntityService_referrers.py +236 -0
- graphcore/graph/tests/test_EntityService_routing.py +119 -0
- graphcore/graph/tests/test_dependent_on.py +141 -0
- graphcore/graph/tests/test_engine_is_type_agnostic.py +25 -0
- graphcore/graph/tests/test_engine_resolves_read_permission.py +93 -0
- graphcore/graph/tests/test_entity_cache.py +159 -0
- graphcore/graph/tests/test_null_entity_cache.py +12 -0
- graphcore/management/__init__.py +0 -0
- graphcore/management/commands/__init__.py +0 -0
- graphcore/management/commands/dev_reset.py +18 -0
- graphcore/management/commands/graphcore_createadmin.py +65 -0
- graphcore/management/commands/graphcore_policy.py +59 -0
- graphcore/management/commands/graphcore_status.py +98 -0
- graphcore/management/tests/__init__.py +0 -0
- graphcore/management/tests/test_createadmin.py +78 -0
- graphcore/management/tests/test_policy_command.py +46 -0
- graphcore/management/tests/test_status_command.py +85 -0
- graphcore/migrations/0001_initial.py +186 -0
- graphcore/migrations/0002_invite.py +26 -0
- graphcore/migrations/__init__.py +0 -0
- graphcore/models.py +11 -0
- graphcore/persistence/ENTITY_REPOSITORY_TECHNICAL_TDD.md +396 -0
- graphcore/persistence/__init__.py +6 -0
- graphcore/persistence/django/__init__.py +0 -0
- graphcore/persistence/django/backends.py +43 -0
- graphcore/persistence/django/bindings.py +28 -0
- graphcore/persistence/django/cache.py +38 -0
- graphcore/persistence/django/compiler.py +214 -0
- graphcore/persistence/django/identity.py +18 -0
- graphcore/persistence/django/models.py +252 -0
- graphcore/persistence/django/repository.py +525 -0
- graphcore/persistence/django/tests/__init__.py +0 -0
- graphcore/persistence/django/tests/test_backends.py +68 -0
- graphcore/persistence/django/tests/test_compiler.py +269 -0
- graphcore/persistence/django/tests/test_repository.py +472 -0
- graphcore/persistence/django/tests/test_shared_column.py +120 -0
- graphcore/persistence/django/tests/test_threads.py +50 -0
- graphcore/persistence/django/tests/test_update_shape.py +175 -0
- graphcore/persistence/django/threads.py +38 -0
- graphcore/persistence/repository.py +172 -0
- graphcore/persistence/tests/__init__.py +0 -0
- graphcore/persistence/tests/contract.py +688 -0
- graphcore/persistence/tests/evaluate.py +51 -0
- graphcore/persistence/tests/fixtures.py +32 -0
- graphcore/persistence/tests/memory.py +263 -0
- graphcore/persistence/tests/test_memory_repository.py +10 -0
- graphcore/persistence/tests/test_query_types.py +127 -0
- graphcore/public/__init__.py +9 -0
- graphcore/public/api.py +9 -0
- graphcore/public/apps.py +10 -0
- graphcore/public/auth/__init__.py +1 -0
- graphcore/public/auth/grants.py +7 -0
- graphcore/public/auth/onboarding.py +8 -0
- graphcore/public/auth/perms.py +7 -0
- graphcore/public/auth/policy.py +10 -0
- graphcore/public/auth/roles.py +8 -0
- graphcore/public/auth/subjects.py +7 -0
- graphcore/public/entities/__init__.py +32 -0
- graphcore/public/errors.py +19 -0
- graphcore/public/persistence/__init__.py +44 -0
- graphcore/public/persistence/django.py +24 -0
- graphcore/public/scheduling.py +23 -0
- graphcore/public/strategies/__init__.py +85 -0
- graphcore/public/test_settings.py +16 -0
- graphcore/public/testing.py +47 -0
- graphcore/registry/EntityDirectory.py +246 -0
- graphcore/registry/README.md +147 -0
- graphcore/registry/StrategyDirectory.py +308 -0
- graphcore/registry/TYPE_METADATA_PLACEMENT.md +66 -0
- graphcore/registry/__init__.py +0 -0
- graphcore/registry/app_discovery.py +108 -0
- graphcore/registry/decorators.py +77 -0
- graphcore/registry/discovery.py +111 -0
- graphcore/registry/registry.py +79 -0
- graphcore/registry/tests/__init__.py +0 -0
- graphcore/registry/tests/test_agent_frontend_discovery.py +56 -0
- graphcore/registry/tests/test_app_config.py +36 -0
- graphcore/registry/tests/test_discovery.py +147 -0
- graphcore/registry/tests/test_entity_directory_storage.py +73 -0
- graphcore/registry/tests/test_import_package.py +87 -0
- graphcore/registry/tests/test_principal_flag.py +67 -0
- graphcore/registry/tests/test_strategy_directory.py +29 -0
- graphcore/routes/__init__.py +116 -0
- graphcore/routes/tests/__init__.py +0 -0
- graphcore/routes/tests/test_routes.py +90 -0
- graphcore/runtime.py +243 -0
- graphcore/scheduling/__init__.py +0 -0
- graphcore/scheduling/beat_scheduler.py +164 -0
- graphcore/scheduling/beat_tasks.py +63 -0
- graphcore/scheduling/scheduler.py +37 -0
- graphcore/scheduling/tests/__init__.py +0 -0
- graphcore/scheduling/tests/contract.py +136 -0
- graphcore/scheduling/tests/memory.py +42 -0
- graphcore/scheduling/tests/test_beat_scheduler.py +87 -0
- graphcore/scheduling/tests/test_memory_scheduler.py +15 -0
- graphcore/scheduling/tests/test_ticks.py +143 -0
- graphcore/tasks.py +9 -0
- graphcore/test_settings.py +78 -0
- graphcore/testing/__init__.py +0 -0
- graphcore/testing/case.py +112 -0
- graphcore/testing/runner.py +32 -0
- graphcore/testing/settings.py +26 -0
- graphcore/testing/tests/__init__.py +0 -0
- graphcore/testing/tests/agent_view_fixture.py +19 -0
- graphcore/testing/tests/test_case_and_runner.py +219 -0
- graphcore/testing/tests/test_settings_for_apps.py +50 -0
- graphcore/tests/__init__.py +0 -0
- graphcore/tests/ids.py +14 -0
- graphcore/tests/public_api_manifest.py +102 -0
- graphcore/tests/runs.py +7 -0
- graphcore/tests/test_app_discovery.py +90 -0
- graphcore/tests/test_composition_boundaries.py +231 -0
- graphcore/tests/test_docs_discovery.py +108 -0
- graphcore/tests/test_no_print.py +70 -0
- graphcore/tests/test_packaging.py +64 -0
- graphcore/tests/test_public_api_boundaries.py +229 -0
- graphcore/tests/test_public_api_contract.py +62 -0
- graphcore/tests/test_public_api_docs.py +28 -0
- graphcore/tests/test_runtime.py +42 -0
- graphcore/tests/test_shared_type_boundaries.py +128 -0
- graphcore/tests/test_shared_types.py +144 -0
- graphcore/tests/test_test_settings.py +45 -0
- graphcore/transport/__init__.py +0 -0
- graphcore/transport/agent_asgi.py +68 -0
- graphcore/transport/agent_client.py +85 -0
- graphcore/transport/channel_layer.py +68 -0
- graphcore/transport/consumers.py +507 -0
- graphcore/transport/routing.py +6 -0
- graphcore/transport/tests/__init__.py +0 -0
- graphcore/transport/tests/test_agent_client.py +31 -0
- graphcore/transport/tests/test_api_views.py +733 -0
- graphcore/transport/tests/test_channel_layer.py +96 -0
- graphcore/transport/tests/test_consumers.py +963 -0
- graphcore/transport/tests/test_resource_metadata.py +67 -0
- graphcore/transport/urls.py +29 -0
- graphcore/transport/views.py +358 -0
- graphcore/types/__init__.py +4 -0
- graphcore/types/access.py +114 -0
- graphcore/types/api.py +76 -0
- graphcore/types/entities.py +100 -0
- graphcore/types/entity_id.py +85 -0
- graphcore/types/execution.py +41 -0
- graphcore/types/identity.py +20 -0
- graphcore/types/onboarding.py +20 -0
- graphcore/types/query.py +116 -0
- graphcore/types/schedules.py +91 -0
- graphcore/types/strategy.py +73 -0
- graphcore/types/tests/__init__.py +0 -0
- graphcore/types/tests/test_entity_id.py +50 -0
- graphcore/urls.py +5 -0
- graphcore_kernel-0.1.0.dist-info/METADATA +94 -0
- graphcore_kernel-0.1.0.dist-info/RECORD +343 -0
- graphcore_kernel-0.1.0.dist-info/WHEEL +5 -0
- graphcore_kernel-0.1.0.dist-info/entry_points.txt +2 -0
- graphcore_kernel-0.1.0.dist-info/licenses/LICENSE +202 -0
- graphcore_kernel-0.1.0.dist-info/licenses/NOTICE +2 -0
- graphcore_kernel-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Both kernel dispatchers against the dispatcher contract, and which one
|
|
2
|
+
the composition root maps each workload to."""
|
|
3
|
+
import threading
|
|
4
|
+
from unittest.mock import patch
|
|
5
|
+
|
|
6
|
+
from asgiref.sync import async_to_sync
|
|
7
|
+
from celery.signals import worker_init
|
|
8
|
+
from django.test import SimpleTestCase
|
|
9
|
+
|
|
10
|
+
from graphcore.execution.environments.celery import CeleryDispatcher
|
|
11
|
+
from graphcore.execution.environments.in_process import InProcessDispatcher
|
|
12
|
+
from graphcore.execution.tests.contract import DispatcherContract
|
|
13
|
+
from graphcore.execution.tests.support import TEST_REGISTRY, ExecutionTestCase, Fail, Increment
|
|
14
|
+
from graphcore import runtime
|
|
15
|
+
from graphcore.errors import ConfigurationError
|
|
16
|
+
from graphcore.runtime import default_dispatchers, worker_dispatchers
|
|
17
|
+
from graphcore.types.strategy import Workload
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class _Fixture(ExecutionTestCase):
|
|
21
|
+
def setUp(self):
|
|
22
|
+
super().setUp()
|
|
23
|
+
self.dispatcher = self.make_dispatcher()
|
|
24
|
+
self.executor.bind_dispatchers({Workload.IO: self.dispatcher, Workload.CPU: self.dispatcher})
|
|
25
|
+
|
|
26
|
+
async def arrange(self):
|
|
27
|
+
await self.seed()
|
|
28
|
+
self.owner = self.alice
|
|
29
|
+
self.succeeding = Increment.request_constructor(self.probe.entity_id)
|
|
30
|
+
self.failing = Fail.request_constructor(self.probe.entity_id)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _sync(cls):
|
|
34
|
+
for name in dir(cls):
|
|
35
|
+
if name.startswith('test_'):
|
|
36
|
+
setattr(cls, name, async_to_sync(getattr(cls, name)))
|
|
37
|
+
return cls
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@_sync
|
|
41
|
+
class InProcessDispatcherTests(_Fixture, DispatcherContract):
|
|
42
|
+
def make_dispatcher(self):
|
|
43
|
+
return InProcessDispatcher(self.executor, self.service, TEST_REGISTRY)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@_sync
|
|
47
|
+
class CeleryDispatcherTests(_Fixture, DispatcherContract):
|
|
48
|
+
def make_dispatcher(self):
|
|
49
|
+
return CeleryDispatcher()
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class WorkerPlacementTests(SimpleTestCase):
|
|
53
|
+
def test_a_worker_runs_cpu_beside_io_and_every_other_process_queues_it(self):
|
|
54
|
+
self.assertIsInstance(default_dispatchers(None, None, TEST_REGISTRY)[Workload.CPU], CeleryDispatcher)
|
|
55
|
+
dispatchers = worker_dispatchers(None, None, TEST_REGISTRY)
|
|
56
|
+
self.assertIsInstance(dispatchers[Workload.CPU], InProcessDispatcher)
|
|
57
|
+
self.assertIs(dispatchers[Workload.CPU], dispatchers[Workload.IO])
|
|
58
|
+
|
|
59
|
+
def test_a_starting_worker_builds_its_runtime_as_a_worker(self):
|
|
60
|
+
with patch.object(runtime, '_worker', False), patch.object(runtime, '_runtime', None), \
|
|
61
|
+
patch.object(runtime, 'configured_runtime') as configured:
|
|
62
|
+
worker_init.send(sender=None)
|
|
63
|
+
runtime.get_runtime()
|
|
64
|
+
configured.assert_called_once_with(worker=True)
|
|
65
|
+
|
|
66
|
+
def test_a_process_becomes_a_worker_only_before_building_its_runtime(self):
|
|
67
|
+
with patch.object(runtime, '_worker', False), patch.object(runtime, '_runtime', object()):
|
|
68
|
+
with self.assertRaises(ConfigurationError):
|
|
69
|
+
runtime.serve_as_worker()
|
|
70
|
+
|
|
71
|
+
def test_a_worker_sharing_its_process_with_a_web_server_leaves_it_unmarked(self):
|
|
72
|
+
with patch.object(runtime, '_worker', False), patch.object(runtime, '_runtime', None):
|
|
73
|
+
thread = threading.Thread(target=lambda: worker_init.send(sender=None))
|
|
74
|
+
thread.start()
|
|
75
|
+
thread.join()
|
|
76
|
+
self.assertFalse(runtime._worker)
|
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
"""The executor's front door, attempts, retries, and reporting
|
|
2
|
+
(docs/STRATEGY_EXECUTION_TDD.md, The executor)."""
|
|
3
|
+
import asyncio
|
|
4
|
+
from unittest.mock import patch
|
|
5
|
+
|
|
6
|
+
from asgiref.sync import async_to_sync
|
|
7
|
+
|
|
8
|
+
from graphcore.errors import ConfigurationError, Conflict, Forbidden, InvalidRequest, NotFound, StrategyFailed
|
|
9
|
+
from graphcore.execution.requests import new_strategy_request
|
|
10
|
+
from graphcore.execution.tests.support import (
|
|
11
|
+
AlwaysConflict, ConflictOnce, CpuFail, CpuIncrement, CpuNested, CpuRefuse, ExecutionTestCase, Fail, Increment,
|
|
12
|
+
Claimed, Nested, NoChange, NotExecutable, ReadOnly, Record, Recording, Refuse, Untargeted, Validated, WriteOther,
|
|
13
|
+
)
|
|
14
|
+
from graphcore.types.execution import AttemptResult
|
|
15
|
+
from graphcore.types.strategy import StrategyRequestPayload, Workload
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class FrontDoorTests(ExecutionTestCase):
|
|
19
|
+
def _record_statuses(self):
|
|
20
|
+
statuses = []
|
|
21
|
+
real = self.service.save_entity
|
|
22
|
+
|
|
23
|
+
async def save(entity, principal, **kwargs):
|
|
24
|
+
saved = await real(entity, principal, **kwargs)
|
|
25
|
+
if hasattr(entity, 'status') and hasattr(entity, 'attempt'):
|
|
26
|
+
statuses.append((saved.status, saved.attempt))
|
|
27
|
+
return saved
|
|
28
|
+
|
|
29
|
+
patcher = patch.object(self.service, 'save_entity', side_effect=save)
|
|
30
|
+
patcher.start()
|
|
31
|
+
self.addCleanup(patcher.stop)
|
|
32
|
+
return statuses
|
|
33
|
+
|
|
34
|
+
@async_to_sync
|
|
35
|
+
async def test_an_io_run_goes_pending_running_succeeded(self):
|
|
36
|
+
await self.seed()
|
|
37
|
+
statuses = self._record_statuses()
|
|
38
|
+
request, target = await self.submit(Increment)
|
|
39
|
+
self.assertEqual(statuses, [('pending', 1), ('running', 1), ('succeeded', 1)])
|
|
40
|
+
self.assertEqual(request.status, 'succeeded')
|
|
41
|
+
self.assertEqual(request.ret_val, {'counter': 1})
|
|
42
|
+
self.assertEqual(request.owner, self.alice.principal_entity_id)
|
|
43
|
+
self.assertEqual(target.get_attribute('counter'), 1)
|
|
44
|
+
self.assertEqual(len(self.io.attempts), 1)
|
|
45
|
+
self.assertEqual(self.cpu.attempts, [])
|
|
46
|
+
|
|
47
|
+
@async_to_sync
|
|
48
|
+
async def test_a_cpu_run_is_dispatched_to_its_workload_and_awaited(self):
|
|
49
|
+
await self.seed()
|
|
50
|
+
request, target = await self.submit(CpuIncrement)
|
|
51
|
+
self.assertEqual(request.status, 'succeeded')
|
|
52
|
+
self.assertEqual(target.get_attribute('counter'), 1)
|
|
53
|
+
self.assertEqual([a.request_id for a in self.cpu.attempts], [request.entity_id])
|
|
54
|
+
self.assertEqual(self.io.attempts, [])
|
|
55
|
+
|
|
56
|
+
@async_to_sync
|
|
57
|
+
async def test_without_wait_the_request_is_returned_before_it_finishes(self):
|
|
58
|
+
await self.seed()
|
|
59
|
+
request, target = await self.submit(Increment, wait=False)
|
|
60
|
+
self.assertIsNone(target)
|
|
61
|
+
self.assertIn(request.status, ('pending', 'running'))
|
|
62
|
+
await self.settle()
|
|
63
|
+
self.assertEqual((await self.load(request.entity_id)).status, 'succeeded')
|
|
64
|
+
self.assertEqual(await self.counter(), 1)
|
|
65
|
+
|
|
66
|
+
@async_to_sync
|
|
67
|
+
async def test_a_failed_run_records_its_error_and_writes_nothing(self):
|
|
68
|
+
await self.seed()
|
|
69
|
+
Fail.cleaned = CpuFail.cleaned = 0
|
|
70
|
+
for strategy in (Fail, CpuFail):
|
|
71
|
+
request = await self.failure(StrategyFailed, strategy)
|
|
72
|
+
self.assertEqual((request.status, request.error, request.error_code), ('failed', 'boom', 'strategy_failed'))
|
|
73
|
+
self.assertIsNone(await self.counter())
|
|
74
|
+
self.assertEqual((Fail.cleaned, CpuFail.cleaned), (1, 1))
|
|
75
|
+
|
|
76
|
+
@async_to_sync
|
|
77
|
+
async def test_an_apps_refusal_records_its_code_and_details_without_a_retry(self):
|
|
78
|
+
await self.seed()
|
|
79
|
+
for strategy in (Refuse, CpuRefuse):
|
|
80
|
+
request = await self.failure(Conflict, strategy)
|
|
81
|
+
self.assertEqual((request.status, request.error, request.error_code, request.attempt),
|
|
82
|
+
('failed', 'stale', 'conflict', 1))
|
|
83
|
+
self.assertEqual(request.error_details, {'invitation': {'answers': {'dinner': 'accepted'}}})
|
|
84
|
+
|
|
85
|
+
@async_to_sync
|
|
86
|
+
async def test_a_request_id_runs_once(self):
|
|
87
|
+
await self.seed()
|
|
88
|
+
request = new_strategy_request(self.service, self.payload(Increment))
|
|
89
|
+
await self.executor.execute_request(request, self.alice)
|
|
90
|
+
with self.assertRaisesRegex(InvalidRequest, 'already exists'):
|
|
91
|
+
await self.executor.execute_request(request, self.alice)
|
|
92
|
+
self.assertEqual(await self.counter(), 1)
|
|
93
|
+
|
|
94
|
+
@async_to_sync
|
|
95
|
+
async def test_refused_requests_are_never_saved(self):
|
|
96
|
+
await self.seed()
|
|
97
|
+
cases = [
|
|
98
|
+
(Forbidden, lambda: self.submit(Increment, principal=self.bob)),
|
|
99
|
+
(NotFound, lambda: self.submit(Increment, target=self.service.create(type(self.probe)).entity_id)),
|
|
100
|
+
(InvalidRequest, lambda: self.executor.submit(
|
|
101
|
+
StrategyRequestPayload(strategy_name='Unregistered', target_entity_id=self.probe.entity_id),
|
|
102
|
+
self.alice)),
|
|
103
|
+
(InvalidRequest, lambda: self.executor.submit(
|
|
104
|
+
StrategyRequestPayload(strategy_name='Increment', target_entity_id=None), self.alice)),
|
|
105
|
+
]
|
|
106
|
+
for error, attempt in cases:
|
|
107
|
+
with self.assertRaises(error):
|
|
108
|
+
await attempt()
|
|
109
|
+
self.assertEqual(await self.requests(), [])
|
|
110
|
+
|
|
111
|
+
@async_to_sync
|
|
112
|
+
async def test_an_untargeted_strategy_runs_on_the_deployment(self):
|
|
113
|
+
await self.seed()
|
|
114
|
+
request, target = await self.executor.submit(Untargeted.request_constructor(), self.alice)
|
|
115
|
+
self.assertEqual(request.status, 'succeeded')
|
|
116
|
+
self.assertEqual(request.ret_val, {'untargeted': True})
|
|
117
|
+
self.assertIsNone(target)
|
|
118
|
+
|
|
119
|
+
@async_to_sync
|
|
120
|
+
async def test_unchanged_and_read_only_targets_are_not_saved(self):
|
|
121
|
+
await self.seed()
|
|
122
|
+
for strategy in (NoChange, ReadOnly):
|
|
123
|
+
request, _ = await self.submit(strategy)
|
|
124
|
+
self.assertEqual(request.status, 'succeeded')
|
|
125
|
+
self.assertEqual((await self.load(self.probe.entity_id)).version, self.probe.version)
|
|
126
|
+
|
|
127
|
+
@async_to_sync
|
|
128
|
+
async def test_boundary_checks_fail_the_run_before_apply(self):
|
|
129
|
+
await self.seed()
|
|
130
|
+
request = await self.failure(InvalidRequest, Validated, n='three')
|
|
131
|
+
self.assertEqual(request.error, "Invalid param_config for Validated: n: 'three' is not of type 'integer'")
|
|
132
|
+
request = await self.failure(StrategyFailed, NotExecutable)
|
|
133
|
+
self.assertEqual((request.status, request.error), ('failed', 'probe is not ready'))
|
|
134
|
+
request, _ = await self.submit(Validated, n=3)
|
|
135
|
+
self.assertEqual(request.status, 'succeeded')
|
|
136
|
+
|
|
137
|
+
@async_to_sync
|
|
138
|
+
async def test_a_strategy_writes_only_what_its_owner_may(self):
|
|
139
|
+
await self.seed()
|
|
140
|
+
theirs = await self.make_probe(self.bob)
|
|
141
|
+
for claim_kernel in (False, True):
|
|
142
|
+
await self.failure(Forbidden, WriteOther, other=theirs.entity_id, claim_kernel=claim_kernel)
|
|
143
|
+
self.assertIsNone(await self.counter(theirs.entity_id))
|
|
144
|
+
|
|
145
|
+
@async_to_sync
|
|
146
|
+
async def test_many_concurrent_runs_all_finish(self):
|
|
147
|
+
await self.seed()
|
|
148
|
+
probes = [await self.make_probe(self.alice) for _ in range(20)]
|
|
149
|
+
results = await asyncio.gather(*(self.submit(Increment, target=p.entity_id) for p in probes))
|
|
150
|
+
self.assertEqual({r.status for r, _ in results}, {'succeeded'})
|
|
151
|
+
for probe in probes:
|
|
152
|
+
self.assertEqual(await self.counter(probe.entity_id), 1)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class AttemptTests(ExecutionTestCase):
|
|
156
|
+
@async_to_sync
|
|
157
|
+
async def test_a_conflict_is_retried_with_backoff(self):
|
|
158
|
+
await self.seed()
|
|
159
|
+
request, target = await self.submit(ConflictOnce)
|
|
160
|
+
self.assertEqual((request.status, request.attempt), ('succeeded', 2))
|
|
161
|
+
self.assertEqual(target.get_attribute('counter'), 1)
|
|
162
|
+
self.assertEqual(self.backoffs, [1])
|
|
163
|
+
|
|
164
|
+
@async_to_sync
|
|
165
|
+
async def test_conflicts_fail_the_run_after_max_attempts(self):
|
|
166
|
+
await self.seed()
|
|
167
|
+
self.executor.max_attempts = 3
|
|
168
|
+
request = await self.failure(Conflict, AlwaysConflict)
|
|
169
|
+
self.assertEqual((request.status, request.attempt), ('failed', 3))
|
|
170
|
+
self.assertEqual(request.error, 'always loses')
|
|
171
|
+
self.assertEqual(self.backoffs, [1, 2])
|
|
172
|
+
|
|
173
|
+
@async_to_sync
|
|
174
|
+
async def test_access_is_checked_again_at_each_attempt(self):
|
|
175
|
+
await self.seed()
|
|
176
|
+
shared = await self.make_probe(self.alice)
|
|
177
|
+
await self.runtime.graph_api.grant(self.alice, shared.entity_id, self.bob.principal_entity_id, 'editor')
|
|
178
|
+
self.executor.bind_dispatchers({Workload.IO: Recording(), Workload.CPU: Recording()})
|
|
179
|
+
request, _ = await self.submit(Increment, principal=self.bob, target=shared.entity_id, wait=False)
|
|
180
|
+
self.executor.bind_dispatchers({Workload.IO: self.io, Workload.CPU: self.cpu})
|
|
181
|
+
await self.runtime.graph_api.revoke(self.alice, shared.entity_id, self.bob.principal_entity_id)
|
|
182
|
+
|
|
183
|
+
await self.executor.start_attempt(await self.load(request.entity_id))
|
|
184
|
+
|
|
185
|
+
request = await self.load(request.entity_id)
|
|
186
|
+
self.assertEqual(request.status, 'failed')
|
|
187
|
+
self.assertIn('not allowed', request.error)
|
|
188
|
+
self.assertIsNone(await self.counter(shared.entity_id))
|
|
189
|
+
|
|
190
|
+
@async_to_sync
|
|
191
|
+
async def test_a_stale_or_repeated_result_has_no_effect(self):
|
|
192
|
+
await self.seed()
|
|
193
|
+
held = Claimed(self.executor)
|
|
194
|
+
self.executor.bind_dispatchers({Workload.IO: held, Workload.CPU: held})
|
|
195
|
+
request, _ = await self.submit(Increment, wait=False)
|
|
196
|
+
await self.executor.finish_attempt(AttemptResult(request.entity_id, 0, 'succeeded', ret_val={'x': 1}))
|
|
197
|
+
self.assertEqual((await self.load(request.entity_id)).status, 'running')
|
|
198
|
+
|
|
199
|
+
done = AttemptResult(request.entity_id, 1, 'succeeded', ret_val={'x': 2})
|
|
200
|
+
await self.executor.finish_attempt(done)
|
|
201
|
+
finished = await self.load(request.entity_id)
|
|
202
|
+
await self.executor.finish_attempt(AttemptResult(request.entity_id, 1, 'failed', error='late'))
|
|
203
|
+
again = await self.load(request.entity_id)
|
|
204
|
+
self.assertEqual((again.status, again.ret_val, again.version), ('succeeded', {'x': 2}, finished.version))
|
|
205
|
+
|
|
206
|
+
@async_to_sync
|
|
207
|
+
async def test_a_dispatch_failure_fails_the_run(self):
|
|
208
|
+
await self.seed()
|
|
209
|
+
|
|
210
|
+
class Down(Recording):
|
|
211
|
+
async def dispatch(self, attempt):
|
|
212
|
+
raise ConnectionError('broker down')
|
|
213
|
+
|
|
214
|
+
self.executor.bind_dispatchers({Workload.IO: self.io, Workload.CPU: Down()})
|
|
215
|
+
request = await self.failure(StrategyFailed, CpuIncrement)
|
|
216
|
+
self.assertIn('broker down', request.error)
|
|
217
|
+
|
|
218
|
+
def test_every_registered_workload_needs_a_dispatcher(self):
|
|
219
|
+
with self.assertRaises(ConfigurationError):
|
|
220
|
+
self.executor.bind_dispatchers({Workload.IO: Recording()})
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
class WaitingTests(ExecutionTestCase):
|
|
224
|
+
@async_to_sync
|
|
225
|
+
async def test_a_waiter_returns_when_its_run_finishes_elsewhere(self):
|
|
226
|
+
await self.seed()
|
|
227
|
+
held = Claimed(self.executor)
|
|
228
|
+
self.executor.bind_dispatchers({Workload.IO: held, Workload.CPU: held})
|
|
229
|
+
waiter = asyncio.ensure_future(self.submit(Increment))
|
|
230
|
+
while not held.attempts:
|
|
231
|
+
await asyncio.sleep(0)
|
|
232
|
+
await self.executor.finish_attempt(AttemptResult(held.attempts[0].request_id, 1, 'succeeded', ret_val={'x': 1}))
|
|
233
|
+
request, _ = await asyncio.wait_for(waiter, 5)
|
|
234
|
+
self.assertEqual((request.status, request.ret_val), ('succeeded', {'x': 1}))
|
|
235
|
+
|
|
236
|
+
@async_to_sync
|
|
237
|
+
async def test_a_waiter_reconciles_its_own_overdue_run(self):
|
|
238
|
+
await self.seed()
|
|
239
|
+
held = Claimed(self.executor)
|
|
240
|
+
self.executor.bind_dispatchers({Workload.IO: held, Workload.CPU: held})
|
|
241
|
+
self.executor.default_timeout = 0.1
|
|
242
|
+
self.executor.max_attempts = 2
|
|
243
|
+
request = await asyncio.wait_for(self.failure(StrategyFailed, Increment), 5)
|
|
244
|
+
self.assertEqual((request.attempt, request.error), (2, 'attempt timed out'))
|
|
245
|
+
self.assertEqual(len(held.attempts), 2)
|
|
246
|
+
|
|
247
|
+
@async_to_sync
|
|
248
|
+
async def test_a_waiter_sends_an_unclaimed_attempt_again_as_the_same_attempt(self):
|
|
249
|
+
await self.seed()
|
|
250
|
+
lost = Recording()
|
|
251
|
+
self.executor.bind_dispatchers({Workload.IO: lost, Workload.CPU: lost})
|
|
252
|
+
self.executor.dispatch_timeout = 0.1
|
|
253
|
+
waiter = asyncio.ensure_future(self.submit(Increment))
|
|
254
|
+
while len(lost.attempts) < 2:
|
|
255
|
+
await asyncio.sleep(0.01)
|
|
256
|
+
self.executor.bind_dispatchers({Workload.IO: self.io, Workload.CPU: self.cpu})
|
|
257
|
+
self.assertEqual([a.attempt for a in lost.attempts[:2]], [1, 1])
|
|
258
|
+
await self.io.dispatch(lost.attempts[0])
|
|
259
|
+
request, target = await asyncio.wait_for(waiter, 5)
|
|
260
|
+
self.assertEqual((request.status, request.attempt), ('succeeded', 1))
|
|
261
|
+
self.assertEqual(target.get_attribute('counter'), 1)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
class ReportingTests(ExecutionTestCase):
|
|
265
|
+
def _capture(self):
|
|
266
|
+
sent = []
|
|
267
|
+
|
|
268
|
+
async def send(channel, message):
|
|
269
|
+
sent.append((channel, message))
|
|
270
|
+
|
|
271
|
+
patcher = patch.object(self.service.broadcaster, 'send', side_effect=send)
|
|
272
|
+
patcher.start()
|
|
273
|
+
self.addCleanup(patcher.stop)
|
|
274
|
+
return sent
|
|
275
|
+
|
|
276
|
+
@async_to_sync
|
|
277
|
+
async def test_strategy_complete_carries_the_client_request_id_in_every_environment(self):
|
|
278
|
+
await self.seed()
|
|
279
|
+
sent = self._capture()
|
|
280
|
+
for strategy in (Increment, CpuIncrement):
|
|
281
|
+
payload = self.payload(strategy)
|
|
282
|
+
payload.client_request_id = f'client-{strategy.__name__}'
|
|
283
|
+
request, _ = await self.executor.submit(payload, self.alice, callback_channel='chan')
|
|
284
|
+
channel, message = sent[-1]
|
|
285
|
+
self.assertEqual(channel, 'chan')
|
|
286
|
+
self.assertEqual(message, {
|
|
287
|
+
'type': 'strategy_complete', 'request_id': request.entity_id,
|
|
288
|
+
'target_entity_id': self.probe.entity_id, 'client_request_id': f'client-{strategy.__name__}',
|
|
289
|
+
'status': 'succeeded', 'success': True, 'ret_val': request.ret_val,
|
|
290
|
+
})
|
|
291
|
+
|
|
292
|
+
@async_to_sync
|
|
293
|
+
async def test_a_failure_is_reported_with_its_error(self):
|
|
294
|
+
await self.seed()
|
|
295
|
+
sent = self._capture()
|
|
296
|
+
for strategy in (Fail, CpuFail):
|
|
297
|
+
await self.failure(StrategyFailed, strategy, callback_channel='chan')
|
|
298
|
+
message = sent[-1][1]
|
|
299
|
+
self.assertEqual(
|
|
300
|
+
(message['status'], message['success'], message['error'], message['code']),
|
|
301
|
+
('failed', False, 'boom', 'strategy_failed'),
|
|
302
|
+
)
|
|
303
|
+
self.assertNotIn('ret_val', message)
|
|
304
|
+
self.assertEqual(message['details'], {})
|
|
305
|
+
|
|
306
|
+
@async_to_sync
|
|
307
|
+
async def test_a_refusal_is_reported_with_its_details(self):
|
|
308
|
+
await self.seed()
|
|
309
|
+
sent = self._capture()
|
|
310
|
+
await self.failure(Conflict, Refuse, callback_channel='chan')
|
|
311
|
+
message = sent[-1][1]
|
|
312
|
+
self.assertEqual((message['code'], message['details']),
|
|
313
|
+
('conflict', {'invitation': {'answers': {'dinner': 'accepted'}}}))
|
|
314
|
+
|
|
315
|
+
@async_to_sync
|
|
316
|
+
async def test_without_a_channel_nothing_is_sent(self):
|
|
317
|
+
await self.seed()
|
|
318
|
+
sent = self._capture()
|
|
319
|
+
await self.submit(Increment)
|
|
320
|
+
self.assertFalse([m for _, m in sent if m.get('type') == 'strategy_complete'])
|
|
321
|
+
|
|
322
|
+
@async_to_sync
|
|
323
|
+
async def test_callbacks_run_as_the_owner_with_the_source_result(self):
|
|
324
|
+
await self.seed()
|
|
325
|
+
other = await self.make_probe(self.alice)
|
|
326
|
+
payload = self.payload(Increment)
|
|
327
|
+
payload.callbacks = [Record.request_constructor(other.entity_id).to_dict()]
|
|
328
|
+
payload.client_request_id = 'not-copied'
|
|
329
|
+
request, _ = await self.executor.submit(payload, self.alice)
|
|
330
|
+
await self.settle()
|
|
331
|
+
|
|
332
|
+
callback = next(r for r in await self.requests() if r.strategy_name == 'Record')
|
|
333
|
+
self.assertEqual(callback.status, 'succeeded')
|
|
334
|
+
self.assertEqual(callback.owner, self.alice.principal_entity_id)
|
|
335
|
+
self.assertEqual(callback.param_config['source_ret_val'], request.ret_val)
|
|
336
|
+
self.assertIsNone(callback.client_request_id)
|
|
337
|
+
self.assertEqual((await self.load(other.entity_id)).get_attribute('received'), {'counter': 1})
|
|
338
|
+
|
|
339
|
+
@async_to_sync
|
|
340
|
+
async def test_a_failed_run_submits_no_callbacks(self):
|
|
341
|
+
await self.seed()
|
|
342
|
+
payload = self.payload(Fail)
|
|
343
|
+
payload.callbacks = [Record.request_constructor(self.probe.entity_id).to_dict()]
|
|
344
|
+
with self.assertRaises(StrategyFailed):
|
|
345
|
+
await self.executor.submit(payload, self.alice)
|
|
346
|
+
await self.settle()
|
|
347
|
+
self.assertEqual([r.strategy_name for r in await self.requests()], ['Fail'])
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
class NestedRequestTests(ExecutionTestCase):
|
|
351
|
+
@async_to_sync
|
|
352
|
+
async def test_a_nested_request_runs_as_the_owner_in_any_environment(self):
|
|
353
|
+
await self.seed()
|
|
354
|
+
for parent, child in ((Nested, 'CpuIncrement'), (CpuNested, 'Increment'), (Nested, 'Increment')):
|
|
355
|
+
request, _ = await self.submit(parent, child=child)
|
|
356
|
+
self.assertEqual(request.status, 'succeeded', request.error)
|
|
357
|
+
self.assertEqual(request.ret_val['child_status'], 'succeeded')
|
|
358
|
+
nested = await self.load(request.ret_val['child_id'])
|
|
359
|
+
self.assertEqual((nested.status, nested.owner), ('succeeded', self.alice.principal_entity_id))
|
|
360
|
+
self.assertEqual(await self.counter(), 3)
|
|
361
|
+
|
|
362
|
+
@async_to_sync
|
|
363
|
+
async def test_a_nested_request_without_wait_still_finishes(self):
|
|
364
|
+
await self.seed()
|
|
365
|
+
request, _ = await self.submit(Nested, child='Increment', child_wait=False)
|
|
366
|
+
await self.settle()
|
|
367
|
+
self.assertEqual((await self.load(request.ret_val['child_id'])).status, 'succeeded')
|
|
368
|
+
self.assertEqual(await self.counter(), 1)
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""`LocalStrategySession` against the session contract, and what only the
|
|
2
|
+
local adapter promises: the change tracker and the strategy context."""
|
|
3
|
+
from unittest.mock import patch
|
|
4
|
+
|
|
5
|
+
from asgiref.sync import async_to_sync
|
|
6
|
+
|
|
7
|
+
from graphcore.domain.strategy.runner import ChangeTracker
|
|
8
|
+
from graphcore.execution.local_session import LocalStrategySession
|
|
9
|
+
from graphcore.execution.strategy_context import current_strategy, kernel_internal_context
|
|
10
|
+
from graphcore.execution.tests.contract import StrategySessionContract
|
|
11
|
+
from graphcore.execution.tests.support import (
|
|
12
|
+
Claimed, ExecutionTestCase, Increment, SeeContext, TEST_REGISTRY, _Probe,
|
|
13
|
+
)
|
|
14
|
+
from graphcore.types.strategy import Workload
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class LocalSessionFixture(ExecutionTestCase):
|
|
18
|
+
async def arrange(self):
|
|
19
|
+
await self.seed()
|
|
20
|
+
self.owner, self.stranger = self.alice, self.bob
|
|
21
|
+
self.entity_cls = _Probe
|
|
22
|
+
self.writer, self.reader = 'Increment', 'ReadOnly'
|
|
23
|
+
self.nested = Increment.request_constructor(self.probe.entity_id)
|
|
24
|
+
|
|
25
|
+
async def seed_entity(self, owner):
|
|
26
|
+
return await self.make_probe(owner)
|
|
27
|
+
|
|
28
|
+
async def open_session(self, strategy_name, tracker=None):
|
|
29
|
+
held = Claimed(self.executor)
|
|
30
|
+
self.executor.bind_dispatchers({Workload.IO: held, Workload.CPU: held})
|
|
31
|
+
strategy = TEST_REGISTRY.strategies.by_name(strategy_name)
|
|
32
|
+
await self.executor.submit(strategy.request_constructor(self.probe.entity_id), self.owner, wait=False)
|
|
33
|
+
self.executor.bind_dispatchers({Workload.IO: self.io, Workload.CPU: self.cpu})
|
|
34
|
+
return LocalStrategySession(held.attempts[-1], tracker or ChangeTracker(), self.service,
|
|
35
|
+
self.executor, TEST_REGISTRY)
|
|
36
|
+
|
|
37
|
+
async def retry(self, session):
|
|
38
|
+
held = Claimed(self.executor)
|
|
39
|
+
self.executor.bind_dispatchers({Workload.IO: held, Workload.CPU: held})
|
|
40
|
+
await self.executor.start_attempt(await self.load(session.attempt.request_id))
|
|
41
|
+
self.executor.bind_dispatchers({Workload.IO: self.io, Workload.CPU: self.cpu})
|
|
42
|
+
return LocalStrategySession(held.attempts[-1], ChangeTracker(), self.service,
|
|
43
|
+
self.executor, TEST_REGISTRY)
|
|
44
|
+
|
|
45
|
+
async def supersede(self, session):
|
|
46
|
+
request = await self.load(session.attempt.request_id)
|
|
47
|
+
request.attempt += 1
|
|
48
|
+
with kernel_internal_context():
|
|
49
|
+
await self.service.save_entity(request, session.principal)
|
|
50
|
+
|
|
51
|
+
def backend_id(self):
|
|
52
|
+
return self.service.backend_id
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _sync(cls):
|
|
56
|
+
for name in dir(cls):
|
|
57
|
+
if name.startswith('test_'):
|
|
58
|
+
setattr(cls, name, async_to_sync(getattr(cls, name)))
|
|
59
|
+
return cls
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@_sync
|
|
63
|
+
class LocalStrategySessionContractTests(LocalSessionFixture, StrategySessionContract):
|
|
64
|
+
pass
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@_sync
|
|
68
|
+
class LocalStrategySessionTests(LocalSessionFixture):
|
|
69
|
+
async def test_the_tracker_holds_what_was_read_and_saved(self):
|
|
70
|
+
await self.arrange()
|
|
71
|
+
tracker = ChangeTracker()
|
|
72
|
+
session = await self.open_session('Increment', tracker)
|
|
73
|
+
probe = await session.get(self.probe.entity_id)
|
|
74
|
+
self.assertFalse(tracker.is_unsaved(probe))
|
|
75
|
+
probe.set_attribute('counter', 7)
|
|
76
|
+
self.assertTrue(tracker.is_unsaved(probe))
|
|
77
|
+
saved = await session.save(probe)
|
|
78
|
+
self.assertFalse(tracker.is_unsaved(saved))
|
|
79
|
+
|
|
80
|
+
async def test_each_write_and_only_writes_run_in_the_strategy_context(self):
|
|
81
|
+
await self.arrange()
|
|
82
|
+
SeeContext.seen = []
|
|
83
|
+
during_save = []
|
|
84
|
+
real = self.service.save_entity
|
|
85
|
+
|
|
86
|
+
async def save(entity, principal, **kwargs):
|
|
87
|
+
if isinstance(entity, _Probe):
|
|
88
|
+
during_save.append(current_strategy())
|
|
89
|
+
return await real(entity, principal, **kwargs)
|
|
90
|
+
|
|
91
|
+
with patch.object(self.service, 'save_entity', side_effect=save):
|
|
92
|
+
request, _ = await self.submit(SeeContext)
|
|
93
|
+
self.assertEqual(request.status, 'succeeded')
|
|
94
|
+
self.assertEqual(SeeContext.seen, [None, None])
|
|
95
|
+
self.assertEqual(during_save, [SeeContext])
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Run reconciliation: an attempt past its deadline is re-dispatched, and
|
|
2
|
+
its late writes and results are refused (docs/STRATEGY_EXECUTION_TDD.md,
|
|
3
|
+
Run reconciliation)."""
|
|
4
|
+
import time
|
|
5
|
+
from unittest.mock import patch
|
|
6
|
+
|
|
7
|
+
from asgiref.sync import async_to_sync
|
|
8
|
+
|
|
9
|
+
from graphcore.domain.strategy.runner import ChangeTracker
|
|
10
|
+
from graphcore.errors import StaleAttempt
|
|
11
|
+
from graphcore.execution.local_session import LocalStrategySession
|
|
12
|
+
from graphcore.execution.strategy_context import kernel_internal_context
|
|
13
|
+
from graphcore.execution.tests.support import (
|
|
14
|
+
TEST_REGISTRY, Claimed, CpuIncrement, ExecutionTestCase, Increment, Recording,
|
|
15
|
+
)
|
|
16
|
+
from graphcore.types.execution import AttemptResult
|
|
17
|
+
from graphcore.types.strategy import Workload
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class ReconcileRunsTests(ExecutionTestCase):
|
|
21
|
+
async def _lost(self, strategy=Increment):
|
|
22
|
+
"""A running request whose attempt never reports, past its deadline."""
|
|
23
|
+
lost = Claimed(self.executor)
|
|
24
|
+
self.executor.bind_dispatchers({Workload.IO: lost, Workload.CPU: lost})
|
|
25
|
+
request, _ = await self.submit(strategy, wait=False)
|
|
26
|
+
self.executor.bind_dispatchers({Workload.IO: self.io, Workload.CPU: self.cpu})
|
|
27
|
+
await self._expire(request.entity_id)
|
|
28
|
+
return lost.attempts[0]
|
|
29
|
+
|
|
30
|
+
async def _expire(self, request_id):
|
|
31
|
+
request = await self.load(request_id)
|
|
32
|
+
request.attempt_deadline = time.time() - 1
|
|
33
|
+
with kernel_internal_context():
|
|
34
|
+
await self.service.save_entity(request, self.alice)
|
|
35
|
+
|
|
36
|
+
@async_to_sync
|
|
37
|
+
async def test_an_expired_attempt_is_dispatched_again(self):
|
|
38
|
+
await self.seed()
|
|
39
|
+
for strategy in (Increment, CpuIncrement):
|
|
40
|
+
first = await self._lost(strategy)
|
|
41
|
+
self.assertEqual(await self.executor.reconcile_runs(), 1)
|
|
42
|
+
await self.settle()
|
|
43
|
+
request = await self.load(first.request_id)
|
|
44
|
+
self.assertEqual((request.status, request.attempt), ('succeeded', 2))
|
|
45
|
+
self.assertEqual(await self.counter(), 2)
|
|
46
|
+
|
|
47
|
+
@async_to_sync
|
|
48
|
+
async def test_the_expired_attempt_writes_nothing_and_its_result_is_discarded(self):
|
|
49
|
+
await self.seed()
|
|
50
|
+
first = await self._lost()
|
|
51
|
+
late = LocalStrategySession(first, ChangeTracker(), self.service, self.executor, TEST_REGISTRY)
|
|
52
|
+
probe = await late.get(self.probe.entity_id)
|
|
53
|
+
await self.executor.reconcile_runs()
|
|
54
|
+
await self.settle()
|
|
55
|
+
|
|
56
|
+
probe.set_attribute('counter', 100)
|
|
57
|
+
with self.assertRaises(StaleAttempt):
|
|
58
|
+
await late.save(probe)
|
|
59
|
+
await self.executor.finish_attempt(AttemptResult(first.request_id, 1, 'failed', error='late'))
|
|
60
|
+
request = await self.load(first.request_id)
|
|
61
|
+
self.assertEqual((request.status, request.attempt, request.error), ('succeeded', 2, None))
|
|
62
|
+
self.assertEqual(await self.counter(), 1)
|
|
63
|
+
|
|
64
|
+
@async_to_sync
|
|
65
|
+
async def test_an_expired_last_attempt_fails_the_run(self):
|
|
66
|
+
await self.seed()
|
|
67
|
+
self.executor.max_attempts = 1
|
|
68
|
+
first = await self._lost()
|
|
69
|
+
self.assertEqual(await self.executor.reconcile_runs(), 1)
|
|
70
|
+
request = await self.load(first.request_id)
|
|
71
|
+
self.assertEqual((request.status, request.error, request.attempt_deadline), ('failed', 'attempt timed out', None))
|
|
72
|
+
|
|
73
|
+
@async_to_sync
|
|
74
|
+
async def test_attempts_within_their_deadline_and_finished_runs_are_left_alone(self):
|
|
75
|
+
await self.seed()
|
|
76
|
+
held = Recording()
|
|
77
|
+
self.executor.bind_dispatchers({Workload.IO: held, Workload.CPU: held})
|
|
78
|
+
running, _ = await self.submit(Increment, wait=False)
|
|
79
|
+
self.executor.bind_dispatchers({Workload.IO: self.io, Workload.CPU: self.cpu})
|
|
80
|
+
done, _ = await self.submit(Increment)
|
|
81
|
+
self.assertEqual(await self.executor.reconcile_runs(), 0)
|
|
82
|
+
self.assertEqual((await self.load(running.entity_id)).attempt, 1)
|
|
83
|
+
self.assertEqual((await self.load(done.entity_id)).status, 'succeeded')
|
|
84
|
+
|
|
85
|
+
@async_to_sync
|
|
86
|
+
async def test_a_deadline_follows_the_strategy_timeout(self):
|
|
87
|
+
await self.seed()
|
|
88
|
+
held = Claimed(self.executor)
|
|
89
|
+
self.executor.bind_dispatchers({Workload.IO: held, Workload.CPU: held})
|
|
90
|
+
self.executor.default_timeout = 1000
|
|
91
|
+
before = time.time()
|
|
92
|
+
request, _ = await self.submit(Increment, wait=False)
|
|
93
|
+
deadline = (await self.load(request.entity_id)).attempt_deadline
|
|
94
|
+
self.assertGreaterEqual(deadline, before + 1000)
|
|
95
|
+
Increment.timeout = 5
|
|
96
|
+
try:
|
|
97
|
+
request, _ = await self.submit(Increment, wait=False)
|
|
98
|
+
finally:
|
|
99
|
+
Increment.timeout = None
|
|
100
|
+
self.assertLess((await self.load(request.entity_id)).attempt_deadline, before + 100)
|
|
101
|
+
|
|
102
|
+
@async_to_sync
|
|
103
|
+
async def test_a_request_saved_but_never_started_is_started(self):
|
|
104
|
+
await self.seed()
|
|
105
|
+
with patch.object(self.executor, '_dispatch'):
|
|
106
|
+
request, _ = await self.submit(Increment, wait=False)
|
|
107
|
+
await self._expire(request.entity_id)
|
|
108
|
+
self.assertEqual(await self.executor.reconcile_runs(), 1)
|
|
109
|
+
await self.settle()
|
|
110
|
+
request = await self.load(request.entity_id)
|
|
111
|
+
self.assertEqual((request.status, request.attempt), ('succeeded', 1))
|
|
112
|
+
|
|
113
|
+
@async_to_sync
|
|
114
|
+
async def test_an_unclaimed_attempt_is_sent_again_and_runs_once(self):
|
|
115
|
+
await self.seed()
|
|
116
|
+
lost = Recording()
|
|
117
|
+
self.executor.bind_dispatchers({Workload.IO: lost, Workload.CPU: lost})
|
|
118
|
+
request, _ = await self.submit(Increment, wait=False)
|
|
119
|
+
await self._expire(request.entity_id)
|
|
120
|
+
self.assertEqual(await self.executor.reconcile_runs(), 1)
|
|
121
|
+
self.assertEqual([a.attempt for a in lost.attempts], [1, 1])
|
|
122
|
+
for attempt in lost.attempts:
|
|
123
|
+
await self.io.dispatch(attempt)
|
|
124
|
+
await self.settle()
|
|
125
|
+
request = await self.load(request.entity_id)
|
|
126
|
+
self.assertEqual((request.status, request.attempt), ('succeeded', 1))
|
|
127
|
+
self.assertEqual(await self.counter(), 1)
|
|
128
|
+
|
|
129
|
+
def test_the_reconcile_task_runs_the_executor(self):
|
|
130
|
+
from graphcore.execution.tasks import reconcile_runs
|
|
131
|
+
async_to_sync(self.seed)()
|
|
132
|
+
first = async_to_sync(self._lost)()
|
|
133
|
+
self.assertEqual(reconcile_runs.delay().get(), 1)
|
|
134
|
+
self.assertEqual(async_to_sync(self.load)(first.request_id).status, 'succeeded')
|