graphcore-kernel 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- graphcore/__init__.py +5 -0
- graphcore/agent_frontend/__init__.py +8 -0
- graphcore/agent_frontend/api.py +69 -0
- graphcore/agent_frontend/base/__init__.py +11 -0
- graphcore/agent_frontend/base/views.py +167 -0
- graphcore/agent_frontend/client.py +31 -0
- graphcore/agent_frontend/context.py +157 -0
- graphcore/agent_frontend/errors.py +12 -0
- graphcore/agent_frontend/loading.py +54 -0
- graphcore/agent_frontend/registry.py +42 -0
- graphcore/agent_frontend/render.py +207 -0
- graphcore/agent_frontend/server.py +103 -0
- graphcore/agent_frontend/signatures.py +35 -0
- graphcore/agent_frontend/testing.py +45 -0
- graphcore/agent_frontend/tests/__init__.py +0 -0
- graphcore/agent_frontend/tests/fakes.py +114 -0
- graphcore/agent_frontend/tests/test_frontend.py +447 -0
- graphcore/agent_frontend/tests/test_loading.py +83 -0
- graphcore/agent_frontend/tests/test_server.py +109 -0
- graphcore/agent_frontend/tools.py +137 -0
- graphcore/api/__init__.py +4 -0
- graphcore/api/graph_api.py +378 -0
- graphcore/api/predicate_codec.py +81 -0
- graphcore/api/tests/__init__.py +0 -0
- graphcore/api/tests/test_changed_entities.py +109 -0
- graphcore/api/tests/test_graph_api.py +290 -0
- graphcore/api/tests/test_graph_api_access.py +94 -0
- graphcore/api/tests/test_graph_api_grants.py +90 -0
- graphcore/api/tests/test_predicate_codec.py +83 -0
- graphcore/api/tests/test_subtree_query.py +79 -0
- graphcore/apps.py +71 -0
- graphcore/auth/__init__.py +1 -0
- graphcore/auth/authn/README.md +178 -0
- graphcore/auth/authn/__init__.py +3 -0
- graphcore/auth/authn/authenticator.py +120 -0
- graphcore/auth/authn/onboarding.py +105 -0
- graphcore/auth/authn/principals.py +20 -0
- graphcore/auth/authn/resource_metadata.py +43 -0
- graphcore/auth/authn/service_accounts.py +96 -0
- graphcore/auth/authn/testing.py +96 -0
- graphcore/auth/authn/tests/__init__.py +0 -0
- graphcore/auth/authn/tests/test_authenticator.py +145 -0
- graphcore/auth/authn/tests/test_onboarding.py +145 -0
- graphcore/auth/authn/tests/test_service_accounts.py +180 -0
- graphcore/auth/authn/users.py +59 -0
- graphcore/auth/authz/__init__.py +0 -0
- graphcore/auth/authz/access_port.py +120 -0
- graphcore/auth/authz/adapter/__init__.py +0 -0
- graphcore/auth/authz/adapter/default_access_adapter.py +556 -0
- graphcore/auth/authz/adapter/tests/__init__.py +0 -0
- graphcore/auth/authz/adapter/tests/test_default_adapter.py +543 -0
- graphcore/auth/authz/adapter/tests/test_entity_write_hooks.py +207 -0
- graphcore/auth/authz/adapter/tests/test_group_types.py +70 -0
- graphcore/auth/authz/adapter/tests/test_sync_support.py +209 -0
- graphcore/auth/authz/defaults.py +26 -0
- graphcore/auth/authz/kernel_policy.py +191 -0
- graphcore/auth/authz/kernel_writes.py +31 -0
- graphcore/auth/authz/lookups.py +25 -0
- graphcore/auth/authz/policy.py +403 -0
- graphcore/auth/authz/tests/__init__.py +0 -0
- graphcore/auth/authz/tests/test_can_contain.py +150 -0
- graphcore/auth/authz/tests/test_contained_by.py +167 -0
- graphcore/auth/authz/tests/test_creation.py +391 -0
- graphcore/auth/authz/tests/test_delete_permission.py +82 -0
- graphcore/auth/authz/tests/test_granted_to_me.py +160 -0
- graphcore/auth/authz/tests/test_integration.py +637 -0
- graphcore/auth/authz/tests/test_invites.py +184 -0
- graphcore/auth/authz/tests/test_kernel_policy.py +140 -0
- graphcore/auth/authz/tests/test_lockout.py +132 -0
- graphcore/auth/authz/tests/test_policy.py +329 -0
- graphcore/auth/authz/tests/test_primitives.py +73 -0
- graphcore/auth/authz/tests/test_principal_creation.py +62 -0
- graphcore/bootstrap/__init__.py +0 -0
- graphcore/bootstrap/deployment.py +67 -0
- graphcore/bootstrap/signals.py +16 -0
- graphcore/bootstrap/tests/__init__.py +0 -0
- graphcore/bootstrap/tests/test_deployment_seed.py +154 -0
- graphcore/broadcast/__init__.py +4 -0
- graphcore/broadcast/broadcaster.py +201 -0
- graphcore/broadcast/channels.py +56 -0
- graphcore/broadcast/tests/__init__.py +0 -0
- graphcore/broadcast/tests/test_broadcaster.py +444 -0
- graphcore/celery.py +14 -0
- graphcore/docs/AGENT_FRONTEND_TDD.md +389 -0
- graphcore/docs/ARCHITECTURE_PRINCIPLES.md +152 -0
- graphcore/docs/AUTHORIZATION_TECHNICAL_TDD.md +1379 -0
- graphcore/docs/ENTITY_REFERENCE_TDD.md +561 -0
- graphcore/docs/GRAPHAPI_FRONT_DOOR.md +173 -0
- graphcore/docs/RUNTIME.md +90 -0
- graphcore/docs/SCHEDULING_TDD.md +244 -0
- graphcore/docs/STRATEGY_EXECUTION_TDD.md +1043 -0
- graphcore/docs/VIEW_ENTITY_TDD.md +522 -0
- graphcore/docs/VIEW_SYSTEM_TDD.md +307 -0
- graphcore/docs/__init__.py +0 -0
- graphcore/docs/doc_topics.py +97 -0
- graphcore/docs/docs_discovery.py +64 -0
- graphcore/docs/public_api.py +56 -0
- graphcore/domain/__init__.py +0 -0
- graphcore/domain/entity/Entity.py +204 -0
- graphcore/domain/entity/__init__.py +0 -0
- graphcore/domain/entity/deployment/Deployment.py +25 -0
- graphcore/domain/entity/deployment/__init__.py +0 -0
- graphcore/domain/entity/grant/Grant.py +31 -0
- graphcore/domain/entity/grant/__init__.py +0 -0
- graphcore/domain/entity/group/Group.py +48 -0
- graphcore/domain/entity/group/README.md +88 -0
- graphcore/domain/entity/group/__init__.py +0 -0
- graphcore/domain/entity/invite/Invite.py +31 -0
- graphcore/domain/entity/invite/__init__.py +0 -0
- graphcore/domain/entity/request_template/RequestTemplate.py +48 -0
- graphcore/domain/entity/request_template/__init__.py +0 -0
- graphcore/domain/entity/request_template/tests/__init__.py +0 -0
- graphcore/domain/entity/request_template/tests/test_request_template.py +52 -0
- graphcore/domain/entity/service_account/ServiceAccount.py +41 -0
- graphcore/domain/entity/service_account/__init__.py +0 -0
- graphcore/domain/entity/strategy_request/StrategyRequest.py +61 -0
- graphcore/domain/entity/strategy_request/__init__.py +0 -0
- graphcore/domain/entity/strategy_request/tests/__init__.py +0 -0
- graphcore/domain/entity/strategy_request/tests/test_strategy_request_entity.py +35 -0
- graphcore/domain/entity/tests/__init__.py +0 -0
- graphcore/domain/entity/tests/test_entity.py +227 -0
- graphcore/domain/entity/tests/test_grant_group_entities.py +36 -0
- graphcore/domain/entity/tests/test_write_once_fields.py +98 -0
- graphcore/domain/entity/user/User.py +39 -0
- graphcore/domain/entity/user/__init__.py +0 -0
- graphcore/domain/entity/user/tests/__init__.py +0 -0
- graphcore/domain/entity/user/tests/test_user_entity.py +33 -0
- graphcore/domain/entity/view/View.py +37 -0
- graphcore/domain/entity/view/__init__.py +0 -0
- graphcore/domain/entity/view/position.py +78 -0
- graphcore/domain/entity/view/tests/__init__.py +0 -0
- graphcore/domain/entity/view/tests/test_position.py +71 -0
- graphcore/domain/entity/view/tests/test_view_entity.py +134 -0
- graphcore/domain/strategy/BaseStrategy.py +759 -0
- graphcore/domain/strategy/DeploymentStrategies.py +55 -0
- graphcore/domain/strategy/GrantStrategies.py +189 -0
- graphcore/domain/strategy/GroupStrategies.py +193 -0
- graphcore/domain/strategy/InviteStrategies.py +84 -0
- graphcore/domain/strategy/RequestTemplateStrategies.py +126 -0
- graphcore/domain/strategy/ViewStrategies.py +219 -0
- graphcore/domain/strategy/__init__.py +0 -0
- graphcore/domain/strategy/runner.py +84 -0
- graphcore/domain/strategy/session.py +56 -0
- graphcore/domain/strategy/tests/BaseStrategyTest.py +167 -0
- graphcore/domain/strategy/tests/__init__.py +0 -0
- graphcore/domain/strategy/tests/test_BaseStrategies.py +870 -0
- graphcore/domain/strategy/tests/test_grant_strategies.py +245 -0
- graphcore/domain/strategy/tests/test_group_strategies.py +99 -0
- graphcore/domain/strategy/tests/test_runner.py +220 -0
- graphcore/domain/strategy/tests/test_strategy_locks.py +446 -0
- graphcore/domain/strategy/tests/test_utils.py +95 -0
- graphcore/domain/strategy/tests/test_view_strategies.py +436 -0
- graphcore/errors.py +145 -0
- graphcore/execution/README.md +97 -0
- graphcore/execution/StrategyExecutorService.py +333 -0
- graphcore/execution/__init__.py +0 -0
- graphcore/execution/dispatcher.py +19 -0
- graphcore/execution/environments/__init__.py +0 -0
- graphcore/execution/environments/celery.py +59 -0
- graphcore/execution/environments/in_process.py +55 -0
- graphcore/execution/local_session.py +117 -0
- graphcore/execution/redis_run_events.py +74 -0
- graphcore/execution/requests.py +22 -0
- graphcore/execution/run_events.py +85 -0
- graphcore/execution/strategy_context.py +54 -0
- graphcore/execution/tasks.py +34 -0
- graphcore/execution/tests/__init__.py +5 -0
- graphcore/execution/tests/contract.py +171 -0
- graphcore/execution/tests/support.py +310 -0
- graphcore/execution/tests/test_dispatchers.py +76 -0
- graphcore/execution/tests/test_executor.py +368 -0
- graphcore/execution/tests/test_local_session.py +95 -0
- graphcore/execution/tests/test_reconcile_runs.py +134 -0
- graphcore/execution/tests/test_run_events.py +106 -0
- graphcore/execution/tests/test_strategy_context.py +62 -0
- graphcore/gateway/__init__.py +1 -0
- graphcore/gateway/remote.py +35 -0
- graphcore/gateway/tests/__init__.py +0 -0
- graphcore/gateway/tests/test_remote.py +30 -0
- graphcore/graph/EntityService.py +710 -0
- graphcore/graph/__init__.py +0 -0
- graphcore/graph/entity_cache.py +152 -0
- graphcore/graph/tests/__init__.py +4 -0
- graphcore/graph/tests/test_EntityService.py +908 -0
- graphcore/graph/tests/test_EntityService_list.py +170 -0
- graphcore/graph/tests/test_EntityService_model_resolution.py +339 -0
- graphcore/graph/tests/test_EntityService_referrers.py +236 -0
- graphcore/graph/tests/test_EntityService_routing.py +119 -0
- graphcore/graph/tests/test_dependent_on.py +141 -0
- graphcore/graph/tests/test_engine_is_type_agnostic.py +25 -0
- graphcore/graph/tests/test_engine_resolves_read_permission.py +93 -0
- graphcore/graph/tests/test_entity_cache.py +159 -0
- graphcore/graph/tests/test_null_entity_cache.py +12 -0
- graphcore/management/__init__.py +0 -0
- graphcore/management/commands/__init__.py +0 -0
- graphcore/management/commands/dev_reset.py +18 -0
- graphcore/management/commands/graphcore_createadmin.py +65 -0
- graphcore/management/commands/graphcore_policy.py +59 -0
- graphcore/management/commands/graphcore_status.py +98 -0
- graphcore/management/tests/__init__.py +0 -0
- graphcore/management/tests/test_createadmin.py +78 -0
- graphcore/management/tests/test_policy_command.py +46 -0
- graphcore/management/tests/test_status_command.py +85 -0
- graphcore/migrations/0001_initial.py +186 -0
- graphcore/migrations/0002_invite.py +26 -0
- graphcore/migrations/__init__.py +0 -0
- graphcore/models.py +11 -0
- graphcore/persistence/ENTITY_REPOSITORY_TECHNICAL_TDD.md +396 -0
- graphcore/persistence/__init__.py +6 -0
- graphcore/persistence/django/__init__.py +0 -0
- graphcore/persistence/django/backends.py +43 -0
- graphcore/persistence/django/bindings.py +28 -0
- graphcore/persistence/django/cache.py +38 -0
- graphcore/persistence/django/compiler.py +214 -0
- graphcore/persistence/django/identity.py +18 -0
- graphcore/persistence/django/models.py +252 -0
- graphcore/persistence/django/repository.py +525 -0
- graphcore/persistence/django/tests/__init__.py +0 -0
- graphcore/persistence/django/tests/test_backends.py +68 -0
- graphcore/persistence/django/tests/test_compiler.py +269 -0
- graphcore/persistence/django/tests/test_repository.py +472 -0
- graphcore/persistence/django/tests/test_shared_column.py +120 -0
- graphcore/persistence/django/tests/test_threads.py +50 -0
- graphcore/persistence/django/tests/test_update_shape.py +175 -0
- graphcore/persistence/django/threads.py +38 -0
- graphcore/persistence/repository.py +172 -0
- graphcore/persistence/tests/__init__.py +0 -0
- graphcore/persistence/tests/contract.py +688 -0
- graphcore/persistence/tests/evaluate.py +51 -0
- graphcore/persistence/tests/fixtures.py +32 -0
- graphcore/persistence/tests/memory.py +263 -0
- graphcore/persistence/tests/test_memory_repository.py +10 -0
- graphcore/persistence/tests/test_query_types.py +127 -0
- graphcore/public/__init__.py +9 -0
- graphcore/public/api.py +9 -0
- graphcore/public/apps.py +10 -0
- graphcore/public/auth/__init__.py +1 -0
- graphcore/public/auth/grants.py +7 -0
- graphcore/public/auth/onboarding.py +8 -0
- graphcore/public/auth/perms.py +7 -0
- graphcore/public/auth/policy.py +10 -0
- graphcore/public/auth/roles.py +8 -0
- graphcore/public/auth/subjects.py +7 -0
- graphcore/public/entities/__init__.py +32 -0
- graphcore/public/errors.py +19 -0
- graphcore/public/persistence/__init__.py +44 -0
- graphcore/public/persistence/django.py +24 -0
- graphcore/public/scheduling.py +23 -0
- graphcore/public/strategies/__init__.py +85 -0
- graphcore/public/test_settings.py +16 -0
- graphcore/public/testing.py +47 -0
- graphcore/registry/EntityDirectory.py +246 -0
- graphcore/registry/README.md +147 -0
- graphcore/registry/StrategyDirectory.py +308 -0
- graphcore/registry/TYPE_METADATA_PLACEMENT.md +66 -0
- graphcore/registry/__init__.py +0 -0
- graphcore/registry/app_discovery.py +108 -0
- graphcore/registry/decorators.py +77 -0
- graphcore/registry/discovery.py +111 -0
- graphcore/registry/registry.py +79 -0
- graphcore/registry/tests/__init__.py +0 -0
- graphcore/registry/tests/test_agent_frontend_discovery.py +56 -0
- graphcore/registry/tests/test_app_config.py +36 -0
- graphcore/registry/tests/test_discovery.py +147 -0
- graphcore/registry/tests/test_entity_directory_storage.py +73 -0
- graphcore/registry/tests/test_import_package.py +87 -0
- graphcore/registry/tests/test_principal_flag.py +67 -0
- graphcore/registry/tests/test_strategy_directory.py +29 -0
- graphcore/routes/__init__.py +116 -0
- graphcore/routes/tests/__init__.py +0 -0
- graphcore/routes/tests/test_routes.py +90 -0
- graphcore/runtime.py +243 -0
- graphcore/scheduling/__init__.py +0 -0
- graphcore/scheduling/beat_scheduler.py +164 -0
- graphcore/scheduling/beat_tasks.py +63 -0
- graphcore/scheduling/scheduler.py +37 -0
- graphcore/scheduling/tests/__init__.py +0 -0
- graphcore/scheduling/tests/contract.py +136 -0
- graphcore/scheduling/tests/memory.py +42 -0
- graphcore/scheduling/tests/test_beat_scheduler.py +87 -0
- graphcore/scheduling/tests/test_memory_scheduler.py +15 -0
- graphcore/scheduling/tests/test_ticks.py +143 -0
- graphcore/tasks.py +9 -0
- graphcore/test_settings.py +78 -0
- graphcore/testing/__init__.py +0 -0
- graphcore/testing/case.py +112 -0
- graphcore/testing/runner.py +32 -0
- graphcore/testing/settings.py +26 -0
- graphcore/testing/tests/__init__.py +0 -0
- graphcore/testing/tests/agent_view_fixture.py +19 -0
- graphcore/testing/tests/test_case_and_runner.py +219 -0
- graphcore/testing/tests/test_settings_for_apps.py +50 -0
- graphcore/tests/__init__.py +0 -0
- graphcore/tests/ids.py +14 -0
- graphcore/tests/public_api_manifest.py +102 -0
- graphcore/tests/runs.py +7 -0
- graphcore/tests/test_app_discovery.py +90 -0
- graphcore/tests/test_composition_boundaries.py +231 -0
- graphcore/tests/test_docs_discovery.py +108 -0
- graphcore/tests/test_no_print.py +70 -0
- graphcore/tests/test_packaging.py +64 -0
- graphcore/tests/test_public_api_boundaries.py +229 -0
- graphcore/tests/test_public_api_contract.py +62 -0
- graphcore/tests/test_public_api_docs.py +28 -0
- graphcore/tests/test_runtime.py +42 -0
- graphcore/tests/test_shared_type_boundaries.py +128 -0
- graphcore/tests/test_shared_types.py +144 -0
- graphcore/tests/test_test_settings.py +45 -0
- graphcore/transport/__init__.py +0 -0
- graphcore/transport/agent_asgi.py +68 -0
- graphcore/transport/agent_client.py +85 -0
- graphcore/transport/channel_layer.py +68 -0
- graphcore/transport/consumers.py +507 -0
- graphcore/transport/routing.py +6 -0
- graphcore/transport/tests/__init__.py +0 -0
- graphcore/transport/tests/test_agent_client.py +31 -0
- graphcore/transport/tests/test_api_views.py +733 -0
- graphcore/transport/tests/test_channel_layer.py +96 -0
- graphcore/transport/tests/test_consumers.py +963 -0
- graphcore/transport/tests/test_resource_metadata.py +67 -0
- graphcore/transport/urls.py +29 -0
- graphcore/transport/views.py +358 -0
- graphcore/types/__init__.py +4 -0
- graphcore/types/access.py +114 -0
- graphcore/types/api.py +76 -0
- graphcore/types/entities.py +100 -0
- graphcore/types/entity_id.py +85 -0
- graphcore/types/execution.py +41 -0
- graphcore/types/identity.py +20 -0
- graphcore/types/onboarding.py +20 -0
- graphcore/types/query.py +116 -0
- graphcore/types/schedules.py +91 -0
- graphcore/types/strategy.py +73 -0
- graphcore/types/tests/__init__.py +0 -0
- graphcore/types/tests/test_entity_id.py +50 -0
- graphcore/urls.py +5 -0
- graphcore_kernel-0.1.0.dist-info/METADATA +94 -0
- graphcore_kernel-0.1.0.dist-info/RECORD +343 -0
- graphcore_kernel-0.1.0.dist-info/WHEEL +5 -0
- graphcore_kernel-0.1.0.dist-info/entry_points.txt +2 -0
- graphcore_kernel-0.1.0.dist-info/licenses/LICENSE +202 -0
- graphcore_kernel-0.1.0.dist-info/licenses/NOTICE +2 -0
- graphcore_kernel-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""The one way a `StrategyRequest` is built (docs/STRATEGY_EXECUTION_TDD.md,
|
|
2
|
+
Strategy requests and request templates)."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
from graphcore.domain.entity.strategy_request.StrategyRequest import StrategyRequest
|
|
8
|
+
from graphcore.types.strategy import StrategyRequestPayload
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def new_strategy_request(entity_service, payload: StrategyRequestPayload,
|
|
12
|
+
template_id: Optional[str] = None) -> StrategyRequest:
|
|
13
|
+
"""`template_id` is never read from the payload, so a client cannot claim
|
|
14
|
+
a request was made from a template."""
|
|
15
|
+
request = entity_service.create(StrategyRequest)
|
|
16
|
+
request.strategy_name = payload.strategy_name
|
|
17
|
+
request.param_config = dict(payload.param_config or {})
|
|
18
|
+
request.target_entity_id = payload.target_entity_id
|
|
19
|
+
request.callbacks = list(payload.callbacks or [])
|
|
20
|
+
request.client_request_id = payload.client_request_id
|
|
21
|
+
request.template_id = template_id
|
|
22
|
+
return request
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""The port that tells a waiting caller its run has ended
|
|
2
|
+
(docs/STRATEGY_EXECUTION_TDD.md, Waiting)."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import threading
|
|
7
|
+
import time
|
|
8
|
+
from abc import ABC, abstractmethod
|
|
9
|
+
from contextlib import asynccontextmanager
|
|
10
|
+
from typing import AsyncIterator, Optional
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class RunSubscription:
|
|
14
|
+
"""One caller's interest in one request."""
|
|
15
|
+
|
|
16
|
+
def __init__(self) -> None:
|
|
17
|
+
self._loop = asyncio.get_running_loop()
|
|
18
|
+
self._event = asyncio.Event()
|
|
19
|
+
|
|
20
|
+
def notify(self) -> None:
|
|
21
|
+
"""Safe from any thread or event loop."""
|
|
22
|
+
self._loop.call_soon_threadsafe(self._event.set)
|
|
23
|
+
|
|
24
|
+
async def wait(self, until: Optional[float]) -> None:
|
|
25
|
+
"""Return when notified, or once the clock reaches `until`."""
|
|
26
|
+
try:
|
|
27
|
+
if until is None:
|
|
28
|
+
await self._event.wait()
|
|
29
|
+
else:
|
|
30
|
+
await asyncio.wait_for(self._event.wait(), max(until - time.time(), 0))
|
|
31
|
+
except asyncio.TimeoutError:
|
|
32
|
+
pass
|
|
33
|
+
self._event.clear()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class RunEvents(ABC):
|
|
37
|
+
"""`publish` is called once a run's terminal status is saved, by
|
|
38
|
+
whichever process saved it; `subscribe` wakes the callers waiting on that
|
|
39
|
+
run, in any process. A subscriber reads the request after subscribing,
|
|
40
|
+
so an event published before it subscribed is never needed."""
|
|
41
|
+
|
|
42
|
+
def __init__(self) -> None:
|
|
43
|
+
self._subscriptions: dict[str, set[RunSubscription]] = {}
|
|
44
|
+
self._lock = threading.Lock()
|
|
45
|
+
|
|
46
|
+
@asynccontextmanager
|
|
47
|
+
async def subscribe(self, request_id: str) -> AsyncIterator[RunSubscription]:
|
|
48
|
+
subscription = RunSubscription()
|
|
49
|
+
with self._lock:
|
|
50
|
+
self._subscriptions.setdefault(request_id, set()).add(subscription)
|
|
51
|
+
self._subscribed()
|
|
52
|
+
try:
|
|
53
|
+
yield subscription
|
|
54
|
+
finally:
|
|
55
|
+
with self._lock:
|
|
56
|
+
held = self._subscriptions.get(request_id)
|
|
57
|
+
if held is not None:
|
|
58
|
+
held.discard(subscription)
|
|
59
|
+
if not held:
|
|
60
|
+
del self._subscriptions[request_id]
|
|
61
|
+
|
|
62
|
+
def notify_local(self, request_id: Optional[str] = None) -> None:
|
|
63
|
+
"""Wake this process's subscribers to `request_id`, or all of them."""
|
|
64
|
+
with self._lock:
|
|
65
|
+
if request_id is None:
|
|
66
|
+
held = [s for subs in self._subscriptions.values() for s in subs]
|
|
67
|
+
else:
|
|
68
|
+
held = list(self._subscriptions.get(request_id, ()))
|
|
69
|
+
for subscription in held:
|
|
70
|
+
subscription.notify()
|
|
71
|
+
|
|
72
|
+
def _subscribed(self) -> None:
|
|
73
|
+
"""Hook for an adapter that listens across processes."""
|
|
74
|
+
|
|
75
|
+
@abstractmethod
|
|
76
|
+
async def publish(self, request_id: str) -> None:
|
|
77
|
+
"""Never raises: a lost event delays a waiter until its deadline."""
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class InMemoryRunEvents(RunEvents):
|
|
81
|
+
"""Subscribers in this process only: every run's terminal save happens
|
|
82
|
+
in the process its waiter runs in."""
|
|
83
|
+
|
|
84
|
+
async def publish(self, request_id: str) -> None:
|
|
85
|
+
self.notify_local(request_id)
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""The executor's record of which strategy is currently running.
|
|
2
|
+
|
|
3
|
+
The persistence check (Step 8) trusts this to know, for every save, which
|
|
4
|
+
strategy's policy governs it - not the strategy's declared target, which
|
|
5
|
+
strategy code could launder around. Owned exclusively by the execution
|
|
6
|
+
layer: no kernel API accepts a strategy identity as an argument, so there
|
|
7
|
+
is nothing for strategy code to pass or misstate. See
|
|
8
|
+
docs/AUTHORIZATION_TECHNICAL_TDD.md (Enforcement).
|
|
9
|
+
|
|
10
|
+
`ContextVar` is asyncio- and thread-scoped, so concurrent executions never
|
|
11
|
+
observe each other's value. A strategy session opens the strategy's context
|
|
12
|
+
around each write it makes, and only there.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from contextlib import contextmanager
|
|
17
|
+
from contextvars import ContextVar
|
|
18
|
+
from typing import Optional, Type
|
|
19
|
+
|
|
20
|
+
KERNEL_INTERNAL = object()
|
|
21
|
+
"""Sentinel identifying a documented kernel-internal write with no
|
|
22
|
+
strategy behind it (e.g. identity bootstrap). Not a strategy class."""
|
|
23
|
+
|
|
24
|
+
_current_strategy: ContextVar[Optional[object]] = ContextVar('current_strategy', default=None)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def current_strategy() -> Optional[object]:
|
|
28
|
+
"""The strategy class currently executing in this task/thread, or
|
|
29
|
+
`KERNEL_INTERNAL`, or `None` if no strategy context is open."""
|
|
30
|
+
return _current_strategy.get()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@contextmanager
|
|
34
|
+
def strategy_context(strategy_cls: Type):
|
|
35
|
+
token = _current_strategy.set(strategy_cls)
|
|
36
|
+
try:
|
|
37
|
+
yield
|
|
38
|
+
finally:
|
|
39
|
+
_current_strategy.reset(token)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@contextmanager
|
|
43
|
+
def kernel_internal_context():
|
|
44
|
+
"""Open the named kernel-internal context for a documented no-strategy
|
|
45
|
+
write (identity bootstrap, executor bookkeeping). Never inside a
|
|
46
|
+
strategy context: no strategy runs with kernel authority."""
|
|
47
|
+
if _current_strategy.get() not in (None, KERNEL_INTERNAL):
|
|
48
|
+
raise RuntimeError("kernel_internal_context() opened inside a strategy context")
|
|
49
|
+
token = _current_strategy.set(KERNEL_INTERNAL)
|
|
50
|
+
try:
|
|
51
|
+
yield
|
|
52
|
+
finally:
|
|
53
|
+
_current_strategy.reset(token)
|
|
54
|
+
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Kernel tasks run by the Celery process (docs/STRATEGY_EXECUTION_TDD.md,
|
|
2
|
+
Run reconciliation)."""
|
|
3
|
+
import logging
|
|
4
|
+
|
|
5
|
+
from asgiref.sync import async_to_sync
|
|
6
|
+
from celery import shared_task
|
|
7
|
+
from celery.signals import worker_ready
|
|
8
|
+
|
|
9
|
+
from graphcore.execution.environments.celery import settle
|
|
10
|
+
from graphcore.runtime import get_runtime
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
# TODO: a periodic sweep is polling, and attempts it re-sends run inside this
|
|
16
|
+
# task in the worker. Replace with one leased, deadline-driven reconciler.
|
|
17
|
+
@shared_task(name='graphcore.tasks.reconcile_runs')
|
|
18
|
+
def reconcile_runs() -> int:
|
|
19
|
+
executor = get_runtime().executor
|
|
20
|
+
|
|
21
|
+
async def run():
|
|
22
|
+
acted = await executor.reconcile_runs()
|
|
23
|
+
await settle(executor)
|
|
24
|
+
return acted
|
|
25
|
+
|
|
26
|
+
return async_to_sync(run)()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@worker_ready.connect
|
|
30
|
+
def _reconcile_on_start(**kwargs) -> None:
|
|
31
|
+
try:
|
|
32
|
+
reconcile_runs.delay()
|
|
33
|
+
except Exception:
|
|
34
|
+
logger.exception("Could not queue run reconciliation at worker start")
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""The `StrategySession` and `Dispatcher` contracts, run against every
|
|
2
|
+
implementation. Published from `graphcore.public.testing`.
|
|
3
|
+
|
|
4
|
+
Both are mixins listed after the test case that supplies their hooks; their
|
|
5
|
+
test methods are `async def`, to be wrapped with `async_to_sync`.
|
|
6
|
+
|
|
7
|
+
`StrategySessionContract`: an implementation supplies `arrange()`, which
|
|
8
|
+
sets `self.owner` and `self.stranger` (principals), `self.entity_cls` (a
|
|
9
|
+
type the owner may create at the top level), `self.writer` and
|
|
10
|
+
`self.reader` (the names of a writing and a read-only strategy on that
|
|
11
|
+
type), and `self.nested` (a payload the owner may run); `seed_entity(owner)`, a
|
|
12
|
+
saved top-level `entity_cls` owned by `owner`; `open_session(strategy_name)`,
|
|
13
|
+
a session for the current attempt of a running request owned by
|
|
14
|
+
`self.owner`; `retry(session)`, a session for the next attempt of that
|
|
15
|
+
session's request; `supersede(session)`, which makes that attempt stale;
|
|
16
|
+
and `backend_id()`, this deployment's backend id.
|
|
17
|
+
|
|
18
|
+
`DispatcherContract`: an implementation supplies `make_dispatcher()`, an
|
|
19
|
+
`executor` whose every workload is dispatched to it, and `arrange()`, which
|
|
20
|
+
sets `self.owner` and `self.succeeding` and `self.failing` (payloads the
|
|
21
|
+
owner may run, whose runs succeed and fail).
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from graphcore.errors import Forbidden, KernelError, NotFound, StaleAttempt
|
|
26
|
+
from graphcore.types.entities import EntityFilter
|
|
27
|
+
from graphcore.types.entity_id import backend_of
|
|
28
|
+
from graphcore.types.execution import ExecutionAttempt
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class StrategySessionContract:
|
|
32
|
+
async def arrange(self) -> None:
|
|
33
|
+
raise NotImplementedError
|
|
34
|
+
|
|
35
|
+
async def seed_entity(self, owner):
|
|
36
|
+
raise NotImplementedError
|
|
37
|
+
|
|
38
|
+
async def open_session(self, strategy_name: str):
|
|
39
|
+
raise NotImplementedError
|
|
40
|
+
|
|
41
|
+
async def retry(self, session):
|
|
42
|
+
raise NotImplementedError
|
|
43
|
+
|
|
44
|
+
async def supersede(self, session) -> None:
|
|
45
|
+
raise NotImplementedError
|
|
46
|
+
|
|
47
|
+
def backend_id(self) -> str:
|
|
48
|
+
raise NotImplementedError
|
|
49
|
+
|
|
50
|
+
async def test_get_reads_as_the_owner(self):
|
|
51
|
+
await self.arrange()
|
|
52
|
+
mine, theirs = await self.seed_entity(self.owner), await self.seed_entity(self.stranger)
|
|
53
|
+
session = await self.open_session(self.writer)
|
|
54
|
+
self.assertEqual((await session.get(mine.entity_id)).entity_id, mine.entity_id)
|
|
55
|
+
with self.assertRaises(Forbidden):
|
|
56
|
+
await session.get(theirs.entity_id)
|
|
57
|
+
with self.assertRaises(NotFound):
|
|
58
|
+
await session.get(session.create(self.entity_cls).entity_id)
|
|
59
|
+
|
|
60
|
+
async def test_get_many_and_list_return_only_what_the_owner_may_read(self):
|
|
61
|
+
await self.arrange()
|
|
62
|
+
mine, theirs = await self.seed_entity(self.owner), await self.seed_entity(self.stranger)
|
|
63
|
+
session = await self.open_session(self.writer)
|
|
64
|
+
self.assertEqual(set(await session.get_many([mine.entity_id, theirs.entity_id])), {mine.entity_id})
|
|
65
|
+
page = await session.list(EntityFilter(type_ids=frozenset({self.entity_cls.get_entity_type_id()})))
|
|
66
|
+
listed = {e.entity_id for e in page.entities}
|
|
67
|
+
self.assertIn(mine.entity_id, listed)
|
|
68
|
+
self.assertNotIn(theirs.entity_id, listed)
|
|
69
|
+
|
|
70
|
+
async def test_create_mints_a_local_id_and_saves_nothing(self):
|
|
71
|
+
await self.arrange()
|
|
72
|
+
session = await self.open_session(self.writer)
|
|
73
|
+
entity = session.create(self.entity_cls)
|
|
74
|
+
self.assertEqual(backend_of(entity.entity_id), self.backend_id())
|
|
75
|
+
with self.assertRaises(NotFound):
|
|
76
|
+
await session.get(entity.entity_id)
|
|
77
|
+
|
|
78
|
+
async def test_a_retried_attempt_creates_and_replaces_what_the_earlier_one_did(self):
|
|
79
|
+
await self.arrange()
|
|
80
|
+
first = await self.open_session(self.writer)
|
|
81
|
+
earlier = await first.save(first.create(self.entity_cls))
|
|
82
|
+
second = await self.retry(first)
|
|
83
|
+
again = second.create(self.entity_cls)
|
|
84
|
+
self.assertEqual(again.entity_id, earlier.entity_id)
|
|
85
|
+
self.assertEqual((await second.save(again)).version, earlier.version + 1)
|
|
86
|
+
self.assertNotEqual(second.create(self.entity_cls).entity_id, earlier.entity_id)
|
|
87
|
+
other = await self.open_session(self.writer)
|
|
88
|
+
self.assertNotEqual(other.create(self.entity_cls).entity_id, earlier.entity_id)
|
|
89
|
+
|
|
90
|
+
async def test_save_and_delete_write_as_the_owner(self):
|
|
91
|
+
await self.arrange()
|
|
92
|
+
session = await self.open_session(self.writer)
|
|
93
|
+
entity = session.create(self.entity_cls)
|
|
94
|
+
saved = await session.save(entity)
|
|
95
|
+
self.assertEqual((await session.get(entity.entity_id)).version, saved.version)
|
|
96
|
+
self.assertEqual(saved.owner, self.owner.principal_entity_id)
|
|
97
|
+
self.assertIn(entity.entity_id, await session.delete_tree(entity.entity_id))
|
|
98
|
+
with self.assertRaises(NotFound):
|
|
99
|
+
await session.get(entity.entity_id)
|
|
100
|
+
|
|
101
|
+
async def test_writes_are_authorized(self):
|
|
102
|
+
await self.arrange()
|
|
103
|
+
theirs = await self.seed_entity(self.stranger)
|
|
104
|
+
writer = await self.open_session(self.writer)
|
|
105
|
+
with self.assertRaises(Forbidden):
|
|
106
|
+
await writer.save(self.entity_cls(entity_id=theirs.entity_id))
|
|
107
|
+
mine = await self.seed_entity(self.owner)
|
|
108
|
+
reader = await self.open_session(self.reader)
|
|
109
|
+
with self.assertRaises(Forbidden):
|
|
110
|
+
await reader.save(await reader.get(mine.entity_id))
|
|
111
|
+
|
|
112
|
+
async def test_a_superseded_attempt_writes_nothing(self):
|
|
113
|
+
await self.arrange()
|
|
114
|
+
mine = await self.seed_entity(self.owner)
|
|
115
|
+
session = await self.open_session(self.writer)
|
|
116
|
+
entity = await session.get(mine.entity_id)
|
|
117
|
+
await self.supersede(session)
|
|
118
|
+
with self.assertRaises(StaleAttempt):
|
|
119
|
+
await session.save(entity)
|
|
120
|
+
with self.assertRaises(StaleAttempt):
|
|
121
|
+
await session.delete_tree(mine.entity_id)
|
|
122
|
+
with self.assertRaises(StaleAttempt):
|
|
123
|
+
await session.save(session.create(self.entity_cls))
|
|
124
|
+
self.assertEqual((await self.seed_version(mine)), mine.version)
|
|
125
|
+
|
|
126
|
+
async def seed_version(self, entity) -> int:
|
|
127
|
+
session = await self.open_session(self.writer)
|
|
128
|
+
return (await session.get(entity.entity_id)).version
|
|
129
|
+
|
|
130
|
+
async def test_execute_runs_a_nested_request_as_the_owner(self):
|
|
131
|
+
await self.arrange()
|
|
132
|
+
session = await self.open_session(self.writer)
|
|
133
|
+
request, _ = await session.execute(self.nested)
|
|
134
|
+
self.assertEqual((request.status, request.owner), ('succeeded', self.owner.principal_entity_id))
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class DispatcherContract:
|
|
138
|
+
def make_dispatcher(self):
|
|
139
|
+
raise NotImplementedError
|
|
140
|
+
|
|
141
|
+
async def arrange(self) -> None:
|
|
142
|
+
raise NotImplementedError
|
|
143
|
+
|
|
144
|
+
async def test_a_dispatched_attempt_reaches_its_result(self):
|
|
145
|
+
await self.arrange()
|
|
146
|
+
request, _ = await self.executor.submit(self.succeeding, self.owner)
|
|
147
|
+
self.assertEqual(request.status, 'succeeded')
|
|
148
|
+
with self.assertRaises(KernelError) as raised:
|
|
149
|
+
await self.executor.submit(self.failing, self.owner)
|
|
150
|
+
self.assertTrue(raised.exception.details['strategy_request_id'])
|
|
151
|
+
|
|
152
|
+
async def test_an_attempt_dispatched_without_waiting_finishes_by_drain(self):
|
|
153
|
+
await self.arrange()
|
|
154
|
+
request, _ = await self.executor.submit(self.succeeding, self.owner, wait=False)
|
|
155
|
+
await self.dispatcher.drain()
|
|
156
|
+
loaded = await self.executor.entity_service.repository.load(request.entity_id)
|
|
157
|
+
self.assertEqual(loaded.status, 'succeeded')
|
|
158
|
+
|
|
159
|
+
async def test_an_attempt_that_cannot_run_is_reported_not_raised(self):
|
|
160
|
+
await self.arrange()
|
|
161
|
+
request, _ = await self.executor.submit(self.succeeding, self.owner, wait=False)
|
|
162
|
+
await self.dispatcher.drain()
|
|
163
|
+
stale = ExecutionAttempt(
|
|
164
|
+
request_id=request.entity_id, attempt=99, strategy_name='NoSuchStrategy',
|
|
165
|
+
target_entity_id=request.target_entity_id, owner_id=self.owner.principal_entity_id,
|
|
166
|
+
backend_id=self.executor.entity_service.backend_id,
|
|
167
|
+
)
|
|
168
|
+
await self.dispatcher.dispatch(stale)
|
|
169
|
+
await self.dispatcher.drain()
|
|
170
|
+
loaded = await self.executor.entity_service.repository.load(request.entity_id)
|
|
171
|
+
self.assertEqual((loaded.status, loaded.attempt), ('succeeded', 1))
|
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
"""Strategies, a runtime, and fixtures for the execution tests."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import asyncio
|
|
5
|
+
from unittest.mock import patch
|
|
6
|
+
|
|
7
|
+
from django.test import TransactionTestCase
|
|
8
|
+
|
|
9
|
+
from graphcore.bootstrap.deployment import ensure_deployment
|
|
10
|
+
from graphcore.domain.entity.deployment.Deployment import Deployment
|
|
11
|
+
from graphcore.domain.entity.Entity import Entity
|
|
12
|
+
from graphcore.domain.entity.strategy_request.StrategyRequest import StrategyRequest
|
|
13
|
+
from graphcore.domain.entity.user.User import User
|
|
14
|
+
from graphcore.domain.strategy.BaseStrategy import Strategy
|
|
15
|
+
from graphcore.errors import ConcurrentModificationError
|
|
16
|
+
from graphcore.execution.dispatcher import Dispatcher
|
|
17
|
+
from graphcore.execution.strategy_context import current_strategy, kernel_internal_context
|
|
18
|
+
from graphcore.registry.registry import DEFAULT_REGISTRY
|
|
19
|
+
from graphcore.public.errors import Conflict
|
|
20
|
+
import graphcore.public.testing # noqa: F401 closes eager tasks' connections
|
|
21
|
+
from graphcore.runtime import django_runtime
|
|
22
|
+
from graphcore.types.access import Create
|
|
23
|
+
from graphcore.types.identity import Principal
|
|
24
|
+
from graphcore.types.query import Equal
|
|
25
|
+
from graphcore.types.strategy import StrategyRequestPayload, Workload
|
|
26
|
+
|
|
27
|
+
TEST_REGISTRY = DEFAULT_REGISTRY.fork_open()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class _Probe(Entity):
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def attribute(entity, name):
|
|
35
|
+
try:
|
|
36
|
+
return entity.get_attribute(name)
|
|
37
|
+
except KeyError:
|
|
38
|
+
return None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _bump(entity) -> int:
|
|
42
|
+
counter = (attribute(entity, 'counter') or 0) + 1
|
|
43
|
+
entity.set_attribute('counter', counter)
|
|
44
|
+
return counter
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class _ProbeStrategy(Strategy):
|
|
48
|
+
@classmethod
|
|
49
|
+
def request_constructor(cls, target_entity_id, **params):
|
|
50
|
+
return cls._build_request(target_entity_id=target_entity_id, param_config=params)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class Increment(_ProbeStrategy):
|
|
54
|
+
async def apply(self):
|
|
55
|
+
self.strategy_request.ret_val = {'counter': _bump(self.entity)}
|
|
56
|
+
return self.strategy_request, self.entity
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class CpuIncrement(Increment):
|
|
60
|
+
workload = Workload.CPU
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class Fail(_ProbeStrategy):
|
|
64
|
+
cleaned = 0
|
|
65
|
+
|
|
66
|
+
async def apply(self):
|
|
67
|
+
_bump(self.entity)
|
|
68
|
+
raise RuntimeError('boom')
|
|
69
|
+
|
|
70
|
+
async def cleanup(self):
|
|
71
|
+
type(self).cleaned += 1
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class CpuFail(Fail):
|
|
75
|
+
workload = Workload.CPU
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class NoChange(_ProbeStrategy):
|
|
79
|
+
async def apply(self):
|
|
80
|
+
return self.strategy_request, self.entity
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class ReadOnly(_ProbeStrategy):
|
|
84
|
+
read_only_strategy = True
|
|
85
|
+
|
|
86
|
+
async def apply(self):
|
|
87
|
+
_bump(self.entity)
|
|
88
|
+
return self.strategy_request, self.entity
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class ConflictOnce(_ProbeStrategy):
|
|
92
|
+
async def apply(self):
|
|
93
|
+
if self.strategy_request.attempt == 1:
|
|
94
|
+
raise ConcurrentModificationError('first attempt loses')
|
|
95
|
+
self.strategy_request.ret_val = {'counter': _bump(self.entity)}
|
|
96
|
+
return self.strategy_request, self.entity
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class AlwaysConflict(_ProbeStrategy):
|
|
100
|
+
async def apply(self):
|
|
101
|
+
raise ConcurrentModificationError('always loses')
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class Refuse(_ProbeStrategy):
|
|
105
|
+
"""Refuses as an app does: a public error with details."""
|
|
106
|
+
|
|
107
|
+
async def apply(self):
|
|
108
|
+
raise Conflict('stale', invitation={'answers': {'dinner': 'accepted'}})
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class CpuRefuse(Refuse):
|
|
112
|
+
workload = Workload.CPU
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class Validated(_ProbeStrategy):
|
|
116
|
+
param_schema = {'type': 'object', 'properties': {'n': {'type': 'integer'}}, 'required': ['n']}
|
|
117
|
+
|
|
118
|
+
async def apply(self):
|
|
119
|
+
return self.strategy_request, self.entity
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class NotExecutable(_ProbeStrategy):
|
|
123
|
+
def verify_executable(self):
|
|
124
|
+
raise ValueError('probe is not ready')
|
|
125
|
+
|
|
126
|
+
async def apply(self):
|
|
127
|
+
return self.strategy_request, self.entity
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class Record(_ProbeStrategy):
|
|
131
|
+
"""A callback: keeps what its source returned."""
|
|
132
|
+
|
|
133
|
+
async def apply(self):
|
|
134
|
+
self.entity.set_attribute('received', self.strategy_request.param_config.get('source_ret_val'))
|
|
135
|
+
return self.strategy_request, self.entity
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class Nested(_ProbeStrategy):
|
|
139
|
+
"""Runs `child` on its own target, waiting or not."""
|
|
140
|
+
|
|
141
|
+
async def apply(self):
|
|
142
|
+
params = self.strategy_request.param_config
|
|
143
|
+
child = TEST_REGISTRY.strategies.by_name(params['child'])
|
|
144
|
+
request, _ = await self.session.execute(
|
|
145
|
+
child.request_constructor(self.entity.entity_id), wait=params.get('child_wait', True),
|
|
146
|
+
)
|
|
147
|
+
self.strategy_request.ret_val = {'child_id': request.entity_id, 'child_status': request.status}
|
|
148
|
+
return self.strategy_request, None
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class CpuNested(Nested):
|
|
152
|
+
workload = Workload.CPU
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class WriteOther(_ProbeStrategy):
|
|
156
|
+
"""Saves an entity named in its params, optionally claiming kernel
|
|
157
|
+
authority first."""
|
|
158
|
+
|
|
159
|
+
async def apply(self):
|
|
160
|
+
params = self.strategy_request.param_config
|
|
161
|
+
other = await self.session.get(params['other']) if params.get('readable') else None
|
|
162
|
+
if other is None:
|
|
163
|
+
other = _Probe(entity_id=params['other'])
|
|
164
|
+
_bump(other)
|
|
165
|
+
if params.get('claim_kernel'):
|
|
166
|
+
with kernel_internal_context():
|
|
167
|
+
await self.session.save(other)
|
|
168
|
+
else:
|
|
169
|
+
await self.session.save(other)
|
|
170
|
+
return self.strategy_request, None
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
class SeeContext(_ProbeStrategy):
|
|
174
|
+
seen: list = []
|
|
175
|
+
|
|
176
|
+
async def apply(self):
|
|
177
|
+
SeeContext.seen.append(current_strategy())
|
|
178
|
+
_bump(self.entity)
|
|
179
|
+
await self.session.save(self.entity)
|
|
180
|
+
SeeContext.seen.append(current_strategy())
|
|
181
|
+
return self.strategy_request, self.entity
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
class Untargeted(Strategy):
|
|
185
|
+
async def apply(self):
|
|
186
|
+
self.strategy_request.ret_val = {'untargeted': True}
|
|
187
|
+
return self.strategy_request, None
|
|
188
|
+
|
|
189
|
+
@classmethod
|
|
190
|
+
def request_constructor(cls, target_entity_id=None):
|
|
191
|
+
return cls._build_request(target_entity_id=None)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
PROBE_STRATEGIES = (
|
|
195
|
+
Increment, CpuIncrement, Fail, CpuFail, Refuse, CpuRefuse, NoChange, ReadOnly, ConflictOnce, AlwaysConflict,
|
|
196
|
+
Validated, NotExecutable, Record, Nested, CpuNested, WriteOther, SeeContext,
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
TEST_REGISTRY.entities.register_entity(_Probe, app_label='graphcore')
|
|
200
|
+
TEST_REGISTRY.policy.define(_Probe, shareable=True)
|
|
201
|
+
for _strategy in PROBE_STRATEGIES:
|
|
202
|
+
TEST_REGISTRY.strategies.register_strategy(_Probe.__name__, _strategy)
|
|
203
|
+
TEST_REGISTRY.strategies.register_strategy('Deployment', Untargeted)
|
|
204
|
+
TEST_REGISTRY.policy.extend(Deployment, strategies={Untargeted: Create()})
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
class Recording(Dispatcher):
|
|
208
|
+
"""Delegates to `inner`, or holds every attempt when `inner` is None."""
|
|
209
|
+
|
|
210
|
+
def __init__(self, inner=None):
|
|
211
|
+
self.inner = inner
|
|
212
|
+
self.attempts = []
|
|
213
|
+
|
|
214
|
+
async def dispatch(self, attempt):
|
|
215
|
+
self.attempts.append(attempt)
|
|
216
|
+
if self.inner is not None:
|
|
217
|
+
await self.inner.dispatch(attempt)
|
|
218
|
+
|
|
219
|
+
async def drain(self):
|
|
220
|
+
if self.inner is not None:
|
|
221
|
+
await self.inner.drain()
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
class Claimed(Recording):
|
|
225
|
+
"""Claims every attempt and holds it, as an environment still running it would."""
|
|
226
|
+
|
|
227
|
+
def __init__(self, executor):
|
|
228
|
+
super().__init__()
|
|
229
|
+
self.executor = executor
|
|
230
|
+
|
|
231
|
+
async def dispatch(self, attempt):
|
|
232
|
+
await self.executor.claim_attempt(attempt)
|
|
233
|
+
await super().dispatch(attempt)
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
class ExecutionTestCase(TransactionTestCase):
|
|
237
|
+
"""A runtime over `TEST_REGISTRY` whose Celery tasks run eagerly against
|
|
238
|
+
it, with `alice` owning `probe` and `bob` holding nothing."""
|
|
239
|
+
|
|
240
|
+
def setUp(self):
|
|
241
|
+
super().setUp()
|
|
242
|
+
self.runtime = django_runtime(TEST_REGISTRY)
|
|
243
|
+
self.service = self.runtime.entity_service
|
|
244
|
+
self.executor = self.runtime.executor
|
|
245
|
+
self.io = Recording(self.executor.dispatchers[Workload.IO])
|
|
246
|
+
self.cpu = Recording(self.executor.dispatchers[Workload.CPU])
|
|
247
|
+
self.executor.bind_dispatchers({Workload.IO: self.io, Workload.CPU: self.cpu})
|
|
248
|
+
for target in ('graphcore.execution.environments.celery.get_runtime',
|
|
249
|
+
'graphcore.execution.tasks.get_runtime',
|
|
250
|
+
'graphcore.scheduling.beat_tasks.get_runtime'):
|
|
251
|
+
patcher = patch(target, return_value=self.runtime)
|
|
252
|
+
patcher.start()
|
|
253
|
+
self.addCleanup(patcher.stop)
|
|
254
|
+
self.backoffs = []
|
|
255
|
+
self.executor._backoff = self._backoff
|
|
256
|
+
|
|
257
|
+
async def _backoff(self, attempt):
|
|
258
|
+
self.backoffs.append(attempt)
|
|
259
|
+
|
|
260
|
+
async def seed(self):
|
|
261
|
+
await ensure_deployment(self.service, TEST_REGISTRY)
|
|
262
|
+
self.alice = await self.user('alice')
|
|
263
|
+
self.bob = await self.user('bob')
|
|
264
|
+
self.probe = await self.make_probe(self.alice)
|
|
265
|
+
|
|
266
|
+
async def user(self, username) -> Principal:
|
|
267
|
+
user = self.service.create(User, username=username)
|
|
268
|
+
with kernel_internal_context():
|
|
269
|
+
await self.service.save_entity(user, Principal(principal_entity_id=user.entity_id))
|
|
270
|
+
return Principal(principal_entity_id=user.entity_id)
|
|
271
|
+
|
|
272
|
+
async def make_probe(self, owner: Principal):
|
|
273
|
+
probe = self.service.create(_Probe)
|
|
274
|
+
probe.owner = owner.principal_entity_id
|
|
275
|
+
with kernel_internal_context():
|
|
276
|
+
return await self.service.save_entity(probe, owner)
|
|
277
|
+
|
|
278
|
+
async def submit(self, strategy_cls, principal=None, target=None, wait=True, callback_channel=None,
|
|
279
|
+
**params):
|
|
280
|
+
target_id = self.probe.entity_id if target is None else target
|
|
281
|
+
payload = strategy_cls.request_constructor(target_id, **params)
|
|
282
|
+
return await self.executor.submit(payload, principal or self.alice, wait=wait,
|
|
283
|
+
callback_channel=callback_channel)
|
|
284
|
+
|
|
285
|
+
async def failure(self, error, strategy_cls, **kwargs):
|
|
286
|
+
"""The stored request of a run expected to raise `error`."""
|
|
287
|
+
with self.assertRaises(error) as raised:
|
|
288
|
+
await self.submit(strategy_cls, **kwargs)
|
|
289
|
+
return await self.load(raised.exception.details['strategy_request_id'])
|
|
290
|
+
|
|
291
|
+
async def settle(self):
|
|
292
|
+
"""Let every in-process attempt, and those it starts, finish."""
|
|
293
|
+
for _ in range(5):
|
|
294
|
+
await self.io.drain()
|
|
295
|
+
await asyncio.sleep(0)
|
|
296
|
+
|
|
297
|
+
async def load(self, entity_id):
|
|
298
|
+
return await self.service.repository.load(entity_id)
|
|
299
|
+
|
|
300
|
+
async def counter(self, entity_id=None):
|
|
301
|
+
return attribute(await self.load(entity_id or self.probe.entity_id), 'counter')
|
|
302
|
+
|
|
303
|
+
async def requests(self) -> list[StrategyRequest]:
|
|
304
|
+
ids = await self.service.repository.list_ids(
|
|
305
|
+
Equal('entity_type_id', StrategyRequest.get_entity_type_id()),
|
|
306
|
+
)
|
|
307
|
+
return list((await self.service.repository.load_many(ids)).values())
|
|
308
|
+
|
|
309
|
+
def payload(self, strategy_cls, target=None, **params) -> StrategyRequestPayload:
|
|
310
|
+
return strategy_cls.request_constructor(target or self.probe.entity_id, **params)
|