graphcore-kernel 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- graphcore/__init__.py +5 -0
- graphcore/agent_frontend/__init__.py +8 -0
- graphcore/agent_frontend/api.py +69 -0
- graphcore/agent_frontend/base/__init__.py +11 -0
- graphcore/agent_frontend/base/views.py +167 -0
- graphcore/agent_frontend/client.py +31 -0
- graphcore/agent_frontend/context.py +157 -0
- graphcore/agent_frontend/errors.py +12 -0
- graphcore/agent_frontend/loading.py +54 -0
- graphcore/agent_frontend/registry.py +42 -0
- graphcore/agent_frontend/render.py +207 -0
- graphcore/agent_frontend/server.py +103 -0
- graphcore/agent_frontend/signatures.py +35 -0
- graphcore/agent_frontend/testing.py +45 -0
- graphcore/agent_frontend/tests/__init__.py +0 -0
- graphcore/agent_frontend/tests/fakes.py +114 -0
- graphcore/agent_frontend/tests/test_frontend.py +447 -0
- graphcore/agent_frontend/tests/test_loading.py +83 -0
- graphcore/agent_frontend/tests/test_server.py +109 -0
- graphcore/agent_frontend/tools.py +137 -0
- graphcore/api/__init__.py +4 -0
- graphcore/api/graph_api.py +378 -0
- graphcore/api/predicate_codec.py +81 -0
- graphcore/api/tests/__init__.py +0 -0
- graphcore/api/tests/test_changed_entities.py +109 -0
- graphcore/api/tests/test_graph_api.py +290 -0
- graphcore/api/tests/test_graph_api_access.py +94 -0
- graphcore/api/tests/test_graph_api_grants.py +90 -0
- graphcore/api/tests/test_predicate_codec.py +83 -0
- graphcore/api/tests/test_subtree_query.py +79 -0
- graphcore/apps.py +71 -0
- graphcore/auth/__init__.py +1 -0
- graphcore/auth/authn/README.md +178 -0
- graphcore/auth/authn/__init__.py +3 -0
- graphcore/auth/authn/authenticator.py +120 -0
- graphcore/auth/authn/onboarding.py +105 -0
- graphcore/auth/authn/principals.py +20 -0
- graphcore/auth/authn/resource_metadata.py +43 -0
- graphcore/auth/authn/service_accounts.py +96 -0
- graphcore/auth/authn/testing.py +96 -0
- graphcore/auth/authn/tests/__init__.py +0 -0
- graphcore/auth/authn/tests/test_authenticator.py +145 -0
- graphcore/auth/authn/tests/test_onboarding.py +145 -0
- graphcore/auth/authn/tests/test_service_accounts.py +180 -0
- graphcore/auth/authn/users.py +59 -0
- graphcore/auth/authz/__init__.py +0 -0
- graphcore/auth/authz/access_port.py +120 -0
- graphcore/auth/authz/adapter/__init__.py +0 -0
- graphcore/auth/authz/adapter/default_access_adapter.py +556 -0
- graphcore/auth/authz/adapter/tests/__init__.py +0 -0
- graphcore/auth/authz/adapter/tests/test_default_adapter.py +543 -0
- graphcore/auth/authz/adapter/tests/test_entity_write_hooks.py +207 -0
- graphcore/auth/authz/adapter/tests/test_group_types.py +70 -0
- graphcore/auth/authz/adapter/tests/test_sync_support.py +209 -0
- graphcore/auth/authz/defaults.py +26 -0
- graphcore/auth/authz/kernel_policy.py +191 -0
- graphcore/auth/authz/kernel_writes.py +31 -0
- graphcore/auth/authz/lookups.py +25 -0
- graphcore/auth/authz/policy.py +403 -0
- graphcore/auth/authz/tests/__init__.py +0 -0
- graphcore/auth/authz/tests/test_can_contain.py +150 -0
- graphcore/auth/authz/tests/test_contained_by.py +167 -0
- graphcore/auth/authz/tests/test_creation.py +391 -0
- graphcore/auth/authz/tests/test_delete_permission.py +82 -0
- graphcore/auth/authz/tests/test_granted_to_me.py +160 -0
- graphcore/auth/authz/tests/test_integration.py +637 -0
- graphcore/auth/authz/tests/test_invites.py +184 -0
- graphcore/auth/authz/tests/test_kernel_policy.py +140 -0
- graphcore/auth/authz/tests/test_lockout.py +132 -0
- graphcore/auth/authz/tests/test_policy.py +329 -0
- graphcore/auth/authz/tests/test_primitives.py +73 -0
- graphcore/auth/authz/tests/test_principal_creation.py +62 -0
- graphcore/bootstrap/__init__.py +0 -0
- graphcore/bootstrap/deployment.py +67 -0
- graphcore/bootstrap/signals.py +16 -0
- graphcore/bootstrap/tests/__init__.py +0 -0
- graphcore/bootstrap/tests/test_deployment_seed.py +154 -0
- graphcore/broadcast/__init__.py +4 -0
- graphcore/broadcast/broadcaster.py +201 -0
- graphcore/broadcast/channels.py +56 -0
- graphcore/broadcast/tests/__init__.py +0 -0
- graphcore/broadcast/tests/test_broadcaster.py +444 -0
- graphcore/celery.py +14 -0
- graphcore/docs/AGENT_FRONTEND_TDD.md +389 -0
- graphcore/docs/ARCHITECTURE_PRINCIPLES.md +152 -0
- graphcore/docs/AUTHORIZATION_TECHNICAL_TDD.md +1379 -0
- graphcore/docs/ENTITY_REFERENCE_TDD.md +561 -0
- graphcore/docs/GRAPHAPI_FRONT_DOOR.md +173 -0
- graphcore/docs/RUNTIME.md +90 -0
- graphcore/docs/SCHEDULING_TDD.md +244 -0
- graphcore/docs/STRATEGY_EXECUTION_TDD.md +1043 -0
- graphcore/docs/VIEW_ENTITY_TDD.md +522 -0
- graphcore/docs/VIEW_SYSTEM_TDD.md +307 -0
- graphcore/docs/__init__.py +0 -0
- graphcore/docs/doc_topics.py +97 -0
- graphcore/docs/docs_discovery.py +64 -0
- graphcore/docs/public_api.py +56 -0
- graphcore/domain/__init__.py +0 -0
- graphcore/domain/entity/Entity.py +204 -0
- graphcore/domain/entity/__init__.py +0 -0
- graphcore/domain/entity/deployment/Deployment.py +25 -0
- graphcore/domain/entity/deployment/__init__.py +0 -0
- graphcore/domain/entity/grant/Grant.py +31 -0
- graphcore/domain/entity/grant/__init__.py +0 -0
- graphcore/domain/entity/group/Group.py +48 -0
- graphcore/domain/entity/group/README.md +88 -0
- graphcore/domain/entity/group/__init__.py +0 -0
- graphcore/domain/entity/invite/Invite.py +31 -0
- graphcore/domain/entity/invite/__init__.py +0 -0
- graphcore/domain/entity/request_template/RequestTemplate.py +48 -0
- graphcore/domain/entity/request_template/__init__.py +0 -0
- graphcore/domain/entity/request_template/tests/__init__.py +0 -0
- graphcore/domain/entity/request_template/tests/test_request_template.py +52 -0
- graphcore/domain/entity/service_account/ServiceAccount.py +41 -0
- graphcore/domain/entity/service_account/__init__.py +0 -0
- graphcore/domain/entity/strategy_request/StrategyRequest.py +61 -0
- graphcore/domain/entity/strategy_request/__init__.py +0 -0
- graphcore/domain/entity/strategy_request/tests/__init__.py +0 -0
- graphcore/domain/entity/strategy_request/tests/test_strategy_request_entity.py +35 -0
- graphcore/domain/entity/tests/__init__.py +0 -0
- graphcore/domain/entity/tests/test_entity.py +227 -0
- graphcore/domain/entity/tests/test_grant_group_entities.py +36 -0
- graphcore/domain/entity/tests/test_write_once_fields.py +98 -0
- graphcore/domain/entity/user/User.py +39 -0
- graphcore/domain/entity/user/__init__.py +0 -0
- graphcore/domain/entity/user/tests/__init__.py +0 -0
- graphcore/domain/entity/user/tests/test_user_entity.py +33 -0
- graphcore/domain/entity/view/View.py +37 -0
- graphcore/domain/entity/view/__init__.py +0 -0
- graphcore/domain/entity/view/position.py +78 -0
- graphcore/domain/entity/view/tests/__init__.py +0 -0
- graphcore/domain/entity/view/tests/test_position.py +71 -0
- graphcore/domain/entity/view/tests/test_view_entity.py +134 -0
- graphcore/domain/strategy/BaseStrategy.py +759 -0
- graphcore/domain/strategy/DeploymentStrategies.py +55 -0
- graphcore/domain/strategy/GrantStrategies.py +189 -0
- graphcore/domain/strategy/GroupStrategies.py +193 -0
- graphcore/domain/strategy/InviteStrategies.py +84 -0
- graphcore/domain/strategy/RequestTemplateStrategies.py +126 -0
- graphcore/domain/strategy/ViewStrategies.py +219 -0
- graphcore/domain/strategy/__init__.py +0 -0
- graphcore/domain/strategy/runner.py +84 -0
- graphcore/domain/strategy/session.py +56 -0
- graphcore/domain/strategy/tests/BaseStrategyTest.py +167 -0
- graphcore/domain/strategy/tests/__init__.py +0 -0
- graphcore/domain/strategy/tests/test_BaseStrategies.py +870 -0
- graphcore/domain/strategy/tests/test_grant_strategies.py +245 -0
- graphcore/domain/strategy/tests/test_group_strategies.py +99 -0
- graphcore/domain/strategy/tests/test_runner.py +220 -0
- graphcore/domain/strategy/tests/test_strategy_locks.py +446 -0
- graphcore/domain/strategy/tests/test_utils.py +95 -0
- graphcore/domain/strategy/tests/test_view_strategies.py +436 -0
- graphcore/errors.py +145 -0
- graphcore/execution/README.md +97 -0
- graphcore/execution/StrategyExecutorService.py +333 -0
- graphcore/execution/__init__.py +0 -0
- graphcore/execution/dispatcher.py +19 -0
- graphcore/execution/environments/__init__.py +0 -0
- graphcore/execution/environments/celery.py +59 -0
- graphcore/execution/environments/in_process.py +55 -0
- graphcore/execution/local_session.py +117 -0
- graphcore/execution/redis_run_events.py +74 -0
- graphcore/execution/requests.py +22 -0
- graphcore/execution/run_events.py +85 -0
- graphcore/execution/strategy_context.py +54 -0
- graphcore/execution/tasks.py +34 -0
- graphcore/execution/tests/__init__.py +5 -0
- graphcore/execution/tests/contract.py +171 -0
- graphcore/execution/tests/support.py +310 -0
- graphcore/execution/tests/test_dispatchers.py +76 -0
- graphcore/execution/tests/test_executor.py +368 -0
- graphcore/execution/tests/test_local_session.py +95 -0
- graphcore/execution/tests/test_reconcile_runs.py +134 -0
- graphcore/execution/tests/test_run_events.py +106 -0
- graphcore/execution/tests/test_strategy_context.py +62 -0
- graphcore/gateway/__init__.py +1 -0
- graphcore/gateway/remote.py +35 -0
- graphcore/gateway/tests/__init__.py +0 -0
- graphcore/gateway/tests/test_remote.py +30 -0
- graphcore/graph/EntityService.py +710 -0
- graphcore/graph/__init__.py +0 -0
- graphcore/graph/entity_cache.py +152 -0
- graphcore/graph/tests/__init__.py +4 -0
- graphcore/graph/tests/test_EntityService.py +908 -0
- graphcore/graph/tests/test_EntityService_list.py +170 -0
- graphcore/graph/tests/test_EntityService_model_resolution.py +339 -0
- graphcore/graph/tests/test_EntityService_referrers.py +236 -0
- graphcore/graph/tests/test_EntityService_routing.py +119 -0
- graphcore/graph/tests/test_dependent_on.py +141 -0
- graphcore/graph/tests/test_engine_is_type_agnostic.py +25 -0
- graphcore/graph/tests/test_engine_resolves_read_permission.py +93 -0
- graphcore/graph/tests/test_entity_cache.py +159 -0
- graphcore/graph/tests/test_null_entity_cache.py +12 -0
- graphcore/management/__init__.py +0 -0
- graphcore/management/commands/__init__.py +0 -0
- graphcore/management/commands/dev_reset.py +18 -0
- graphcore/management/commands/graphcore_createadmin.py +65 -0
- graphcore/management/commands/graphcore_policy.py +59 -0
- graphcore/management/commands/graphcore_status.py +98 -0
- graphcore/management/tests/__init__.py +0 -0
- graphcore/management/tests/test_createadmin.py +78 -0
- graphcore/management/tests/test_policy_command.py +46 -0
- graphcore/management/tests/test_status_command.py +85 -0
- graphcore/migrations/0001_initial.py +186 -0
- graphcore/migrations/0002_invite.py +26 -0
- graphcore/migrations/__init__.py +0 -0
- graphcore/models.py +11 -0
- graphcore/persistence/ENTITY_REPOSITORY_TECHNICAL_TDD.md +396 -0
- graphcore/persistence/__init__.py +6 -0
- graphcore/persistence/django/__init__.py +0 -0
- graphcore/persistence/django/backends.py +43 -0
- graphcore/persistence/django/bindings.py +28 -0
- graphcore/persistence/django/cache.py +38 -0
- graphcore/persistence/django/compiler.py +214 -0
- graphcore/persistence/django/identity.py +18 -0
- graphcore/persistence/django/models.py +252 -0
- graphcore/persistence/django/repository.py +525 -0
- graphcore/persistence/django/tests/__init__.py +0 -0
- graphcore/persistence/django/tests/test_backends.py +68 -0
- graphcore/persistence/django/tests/test_compiler.py +269 -0
- graphcore/persistence/django/tests/test_repository.py +472 -0
- graphcore/persistence/django/tests/test_shared_column.py +120 -0
- graphcore/persistence/django/tests/test_threads.py +50 -0
- graphcore/persistence/django/tests/test_update_shape.py +175 -0
- graphcore/persistence/django/threads.py +38 -0
- graphcore/persistence/repository.py +172 -0
- graphcore/persistence/tests/__init__.py +0 -0
- graphcore/persistence/tests/contract.py +688 -0
- graphcore/persistence/tests/evaluate.py +51 -0
- graphcore/persistence/tests/fixtures.py +32 -0
- graphcore/persistence/tests/memory.py +263 -0
- graphcore/persistence/tests/test_memory_repository.py +10 -0
- graphcore/persistence/tests/test_query_types.py +127 -0
- graphcore/public/__init__.py +9 -0
- graphcore/public/api.py +9 -0
- graphcore/public/apps.py +10 -0
- graphcore/public/auth/__init__.py +1 -0
- graphcore/public/auth/grants.py +7 -0
- graphcore/public/auth/onboarding.py +8 -0
- graphcore/public/auth/perms.py +7 -0
- graphcore/public/auth/policy.py +10 -0
- graphcore/public/auth/roles.py +8 -0
- graphcore/public/auth/subjects.py +7 -0
- graphcore/public/entities/__init__.py +32 -0
- graphcore/public/errors.py +19 -0
- graphcore/public/persistence/__init__.py +44 -0
- graphcore/public/persistence/django.py +24 -0
- graphcore/public/scheduling.py +23 -0
- graphcore/public/strategies/__init__.py +85 -0
- graphcore/public/test_settings.py +16 -0
- graphcore/public/testing.py +47 -0
- graphcore/registry/EntityDirectory.py +246 -0
- graphcore/registry/README.md +147 -0
- graphcore/registry/StrategyDirectory.py +308 -0
- graphcore/registry/TYPE_METADATA_PLACEMENT.md +66 -0
- graphcore/registry/__init__.py +0 -0
- graphcore/registry/app_discovery.py +108 -0
- graphcore/registry/decorators.py +77 -0
- graphcore/registry/discovery.py +111 -0
- graphcore/registry/registry.py +79 -0
- graphcore/registry/tests/__init__.py +0 -0
- graphcore/registry/tests/test_agent_frontend_discovery.py +56 -0
- graphcore/registry/tests/test_app_config.py +36 -0
- graphcore/registry/tests/test_discovery.py +147 -0
- graphcore/registry/tests/test_entity_directory_storage.py +73 -0
- graphcore/registry/tests/test_import_package.py +87 -0
- graphcore/registry/tests/test_principal_flag.py +67 -0
- graphcore/registry/tests/test_strategy_directory.py +29 -0
- graphcore/routes/__init__.py +116 -0
- graphcore/routes/tests/__init__.py +0 -0
- graphcore/routes/tests/test_routes.py +90 -0
- graphcore/runtime.py +243 -0
- graphcore/scheduling/__init__.py +0 -0
- graphcore/scheduling/beat_scheduler.py +164 -0
- graphcore/scheduling/beat_tasks.py +63 -0
- graphcore/scheduling/scheduler.py +37 -0
- graphcore/scheduling/tests/__init__.py +0 -0
- graphcore/scheduling/tests/contract.py +136 -0
- graphcore/scheduling/tests/memory.py +42 -0
- graphcore/scheduling/tests/test_beat_scheduler.py +87 -0
- graphcore/scheduling/tests/test_memory_scheduler.py +15 -0
- graphcore/scheduling/tests/test_ticks.py +143 -0
- graphcore/tasks.py +9 -0
- graphcore/test_settings.py +78 -0
- graphcore/testing/__init__.py +0 -0
- graphcore/testing/case.py +112 -0
- graphcore/testing/runner.py +32 -0
- graphcore/testing/settings.py +26 -0
- graphcore/testing/tests/__init__.py +0 -0
- graphcore/testing/tests/agent_view_fixture.py +19 -0
- graphcore/testing/tests/test_case_and_runner.py +219 -0
- graphcore/testing/tests/test_settings_for_apps.py +50 -0
- graphcore/tests/__init__.py +0 -0
- graphcore/tests/ids.py +14 -0
- graphcore/tests/public_api_manifest.py +102 -0
- graphcore/tests/runs.py +7 -0
- graphcore/tests/test_app_discovery.py +90 -0
- graphcore/tests/test_composition_boundaries.py +231 -0
- graphcore/tests/test_docs_discovery.py +108 -0
- graphcore/tests/test_no_print.py +70 -0
- graphcore/tests/test_packaging.py +64 -0
- graphcore/tests/test_public_api_boundaries.py +229 -0
- graphcore/tests/test_public_api_contract.py +62 -0
- graphcore/tests/test_public_api_docs.py +28 -0
- graphcore/tests/test_runtime.py +42 -0
- graphcore/tests/test_shared_type_boundaries.py +128 -0
- graphcore/tests/test_shared_types.py +144 -0
- graphcore/tests/test_test_settings.py +45 -0
- graphcore/transport/__init__.py +0 -0
- graphcore/transport/agent_asgi.py +68 -0
- graphcore/transport/agent_client.py +85 -0
- graphcore/transport/channel_layer.py +68 -0
- graphcore/transport/consumers.py +507 -0
- graphcore/transport/routing.py +6 -0
- graphcore/transport/tests/__init__.py +0 -0
- graphcore/transport/tests/test_agent_client.py +31 -0
- graphcore/transport/tests/test_api_views.py +733 -0
- graphcore/transport/tests/test_channel_layer.py +96 -0
- graphcore/transport/tests/test_consumers.py +963 -0
- graphcore/transport/tests/test_resource_metadata.py +67 -0
- graphcore/transport/urls.py +29 -0
- graphcore/transport/views.py +358 -0
- graphcore/types/__init__.py +4 -0
- graphcore/types/access.py +114 -0
- graphcore/types/api.py +76 -0
- graphcore/types/entities.py +100 -0
- graphcore/types/entity_id.py +85 -0
- graphcore/types/execution.py +41 -0
- graphcore/types/identity.py +20 -0
- graphcore/types/onboarding.py +20 -0
- graphcore/types/query.py +116 -0
- graphcore/types/schedules.py +91 -0
- graphcore/types/strategy.py +73 -0
- graphcore/types/tests/__init__.py +0 -0
- graphcore/types/tests/test_entity_id.py +50 -0
- graphcore/urls.py +5 -0
- graphcore_kernel-0.1.0.dist-info/METADATA +94 -0
- graphcore_kernel-0.1.0.dist-info/RECORD +343 -0
- graphcore_kernel-0.1.0.dist-info/WHEEL +5 -0
- graphcore_kernel-0.1.0.dist-info/entry_points.txt +2 -0
- graphcore_kernel-0.1.0.dist-info/licenses/LICENSE +202 -0
- graphcore_kernel-0.1.0.dist-info/licenses/NOTICE +2 -0
- graphcore_kernel-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
"""The kernel's single implementation of authorization, request
|
|
2
|
+
bookkeeping, retries, and reporting for strategy runs
|
|
3
|
+
(docs/STRATEGY_EXECUTION_TDD.md, The executor)."""
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import asyncio
|
|
7
|
+
import copy
|
|
8
|
+
import logging
|
|
9
|
+
import random
|
|
10
|
+
import time
|
|
11
|
+
from typing import Mapping, Optional
|
|
12
|
+
|
|
13
|
+
from graphcore.domain.entity.Entity import Entity
|
|
14
|
+
from graphcore.domain.entity.deployment.Deployment import Deployment
|
|
15
|
+
from graphcore.domain.entity.strategy_request.StrategyRequest import StrategyRequest
|
|
16
|
+
from graphcore.errors import (
|
|
17
|
+
ConcurrentModificationError, ConfigurationError, Forbidden, InvalidRequest, KernelError, NotFound,
|
|
18
|
+
StrategyFailed,
|
|
19
|
+
)
|
|
20
|
+
from graphcore.execution.dispatcher import Dispatcher
|
|
21
|
+
from graphcore.execution.requests import new_strategy_request
|
|
22
|
+
from graphcore.execution.run_events import InMemoryRunEvents, RunEvents, RunSubscription
|
|
23
|
+
from graphcore.execution.strategy_context import kernel_internal_context
|
|
24
|
+
from graphcore.graph.EntityService import EntityService
|
|
25
|
+
from graphcore.registry.registry import Registry
|
|
26
|
+
from graphcore.types.execution import AttemptResult, ExecutionAttempt
|
|
27
|
+
from graphcore.types.identity import Principal
|
|
28
|
+
from graphcore.types.query import AllOf, Equal, InValues
|
|
29
|
+
from graphcore.types.strategy import TERMINAL_STATUSES, StrategyRequestPayload, Workload
|
|
30
|
+
|
|
31
|
+
logger = logging.getLogger(__name__)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class StrategyExecutorService:
|
|
35
|
+
"""Front door (`execute_request`), `start_attempt`, `claim_attempt`,
|
|
36
|
+
and `finish_attempt`. Nothing here holds a run: every step reads the
|
|
37
|
+
persisted request, so any kernel process can take the next one."""
|
|
38
|
+
|
|
39
|
+
def __init__(self, registry: Registry, entity_service: EntityService,
|
|
40
|
+
default_timeout: int = 600, max_attempts: int = 6,
|
|
41
|
+
run_events: Optional[RunEvents] = None, dispatch_timeout: int = 300):
|
|
42
|
+
self.registry = registry
|
|
43
|
+
self.entity_service = entity_service
|
|
44
|
+
self.default_timeout = default_timeout
|
|
45
|
+
self.max_attempts = max_attempts
|
|
46
|
+
self.dispatch_timeout = dispatch_timeout
|
|
47
|
+
self.run_events = run_events or InMemoryRunEvents()
|
|
48
|
+
self.dispatchers: dict[Workload, Dispatcher] = {}
|
|
49
|
+
|
|
50
|
+
def bind_dispatchers(self, dispatchers: Mapping[Workload, Dispatcher]) -> None:
|
|
51
|
+
"""Called once by the composition root. A registered strategy whose
|
|
52
|
+
workload has no dispatcher fails here, not on its first request."""
|
|
53
|
+
missing = {
|
|
54
|
+
strategy_cls.workload
|
|
55
|
+
for classes in self.registry.strategies.get_all_strategies().values()
|
|
56
|
+
for strategy_cls in classes
|
|
57
|
+
} - set(dispatchers)
|
|
58
|
+
if missing:
|
|
59
|
+
raise ConfigurationError(
|
|
60
|
+
f"No dispatcher for workload(s): {', '.join(sorted(w.name for w in missing))}"
|
|
61
|
+
)
|
|
62
|
+
self.dispatchers = dict(dispatchers)
|
|
63
|
+
|
|
64
|
+
# --- the front door ---------------------------------------------------
|
|
65
|
+
|
|
66
|
+
async def execute_request(
|
|
67
|
+
self,
|
|
68
|
+
request: StrategyRequest,
|
|
69
|
+
principal: Principal,
|
|
70
|
+
wait: bool = True,
|
|
71
|
+
callback_channel: Optional[str] = None,
|
|
72
|
+
) -> tuple[StrategyRequest, Optional[Entity]]:
|
|
73
|
+
"""Authorize, save `pending` as attempt 1, and dispatch it: the first
|
|
74
|
+
attempt is authorized here, not again by `_send`. A request
|
|
75
|
+
refused before the save is never saved. With `wait`, return once the
|
|
76
|
+
run has succeeded, with the target reloaded as `principal`, or raise
|
|
77
|
+
the error the run failed with."""
|
|
78
|
+
if await self.entity_service.entity_exists_in_db(request.entity_id):
|
|
79
|
+
raise InvalidRequest(f"Strategy request {request.entity_id} already exists")
|
|
80
|
+
strategy_cls = self.registry.strategies.require(request.strategy_name)
|
|
81
|
+
if not request.target_entity_id and not self._runs_untargeted(strategy_cls):
|
|
82
|
+
raise InvalidRequest("target_entity_id is required")
|
|
83
|
+
await self._can_execute(strategy_cls, self._authorized_on(request), principal)
|
|
84
|
+
|
|
85
|
+
request.callback_channel = callback_channel
|
|
86
|
+
request.status = 'pending'
|
|
87
|
+
request.attempt = 1
|
|
88
|
+
request.attempt_deadline = time.time() + self.dispatch_timeout
|
|
89
|
+
with kernel_internal_context():
|
|
90
|
+
request = await self.entity_service.save_entity(request, principal, existing=None)
|
|
91
|
+
|
|
92
|
+
if not wait:
|
|
93
|
+
await self._dispatch(request, strategy_cls)
|
|
94
|
+
return request, None
|
|
95
|
+
async with self.run_events.subscribe(request.entity_id) as subscription:
|
|
96
|
+
await self._dispatch(request, strategy_cls)
|
|
97
|
+
request = await self._until_terminal(request.entity_id, subscription)
|
|
98
|
+
|
|
99
|
+
if request.status == 'failed':
|
|
100
|
+
raise request.failure()
|
|
101
|
+
target = None
|
|
102
|
+
if request.target_entity_id:
|
|
103
|
+
try:
|
|
104
|
+
target = await self.entity_service.get_entity(request.target_entity_id, principal)
|
|
105
|
+
except (NotFound, Forbidden):
|
|
106
|
+
target = None
|
|
107
|
+
return request, target
|
|
108
|
+
|
|
109
|
+
async def submit(self, payload: StrategyRequestPayload, principal: Principal, wait: bool = True,
|
|
110
|
+
callback_channel: Optional[str] = None):
|
|
111
|
+
"""Build a request from `payload` and execute it."""
|
|
112
|
+
return await self.execute_request(
|
|
113
|
+
new_strategy_request(self.entity_service, payload), principal,
|
|
114
|
+
wait=wait, callback_channel=callback_channel,
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
def _authorized_on(self, request: StrategyRequest) -> str:
|
|
118
|
+
"""An untargeted request is authorized on the deployment, as an
|
|
119
|
+
entity with no parent is at the persistence chokepoint."""
|
|
120
|
+
return request.target_entity_id or self.entity_service.deployment_entity_id
|
|
121
|
+
|
|
122
|
+
def _runs_untargeted(self, strategy_cls) -> bool:
|
|
123
|
+
allowed = self.registry.policy.allowed_strategies(Deployment)
|
|
124
|
+
return allowed is not None and strategy_cls in allowed
|
|
125
|
+
|
|
126
|
+
async def _can_execute(self, strategy_cls, target_entity_id: str, principal: Principal) -> None:
|
|
127
|
+
"""Strategy in the target type's allowlist, and the principal holds
|
|
128
|
+
the type's read permission and the strategy's required permission on
|
|
129
|
+
the persisted target."""
|
|
130
|
+
target = await self.entity_service.repository.load(target_entity_id)
|
|
131
|
+
if target is None:
|
|
132
|
+
raise NotFound(f"Entity with ID {target_entity_id} not found")
|
|
133
|
+
|
|
134
|
+
entity_type = self.registry.entities.get_entity_class(target.entity_type_id)
|
|
135
|
+
if entity_type is None:
|
|
136
|
+
raise InvalidRequest(f"Entity type {target.entity_type_id} is not registered.")
|
|
137
|
+
|
|
138
|
+
self.registry.policy.require_listed(entity_type, strategy_cls, 'target')
|
|
139
|
+
|
|
140
|
+
read_perm = self.registry.policy.read_permission(entity_type)
|
|
141
|
+
perm = self.registry.policy.required_permission(entity_type, strategy_cls)
|
|
142
|
+
for needed in dict.fromkeys((read_perm, perm)):
|
|
143
|
+
if not await self.entity_service.access_adapter.allows(principal, needed, target_entity_id):
|
|
144
|
+
raise Forbidden(
|
|
145
|
+
f"Strategy {strategy_cls.__name__} is not allowed on entity {target_entity_id}: "
|
|
146
|
+
f"user {principal.principal_entity_id} does not have {type(needed).__name__}"
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
# --- attempts ---------------------------------------------------------
|
|
150
|
+
|
|
151
|
+
async def start_attempt(self, request: StrategyRequest) -> None:
|
|
152
|
+
"""Dispatch the request's next attempt."""
|
|
153
|
+
await self._send(request, request.attempt + 1)
|
|
154
|
+
|
|
155
|
+
async def _send(self, request: StrategyRequest, attempt_number: int) -> None:
|
|
156
|
+
"""Re-authorize the owner against current state, save `pending` with
|
|
157
|
+
`attempt_number` and a dispatch deadline, and dispatch by workload."""
|
|
158
|
+
owner = Principal(principal_entity_id=request.owner)
|
|
159
|
+
try:
|
|
160
|
+
strategy_cls = self.registry.strategies.require(request.strategy_name)
|
|
161
|
+
await self._can_execute(strategy_cls, self._authorized_on(request), owner)
|
|
162
|
+
except Exception as exc:
|
|
163
|
+
await self._finish(request, 'failed', error=KernelError.of(exc))
|
|
164
|
+
return
|
|
165
|
+
|
|
166
|
+
stored = copy.copy(request)
|
|
167
|
+
request.status = 'pending'
|
|
168
|
+
request.attempt = attempt_number
|
|
169
|
+
request.attempt_deadline = time.time() + self.dispatch_timeout
|
|
170
|
+
with kernel_internal_context():
|
|
171
|
+
request = await self.entity_service.save_entity(request, owner, existing=stored)
|
|
172
|
+
await self._dispatch(request, strategy_cls)
|
|
173
|
+
|
|
174
|
+
async def _dispatch(self, request: StrategyRequest, strategy_cls) -> None:
|
|
175
|
+
"""Hand the request's saved `pending` attempt to its workload's environment."""
|
|
176
|
+
attempt = ExecutionAttempt(
|
|
177
|
+
request_id=request.entity_id, attempt=request.attempt,
|
|
178
|
+
strategy_name=request.strategy_name, target_entity_id=request.target_entity_id,
|
|
179
|
+
owner_id=request.owner, backend_id=self.entity_service.backend_id,
|
|
180
|
+
)
|
|
181
|
+
try:
|
|
182
|
+
await self.dispatchers[strategy_cls.workload].dispatch(attempt)
|
|
183
|
+
except Exception as exc:
|
|
184
|
+
logger.exception("Could not dispatch attempt %s of %s", attempt.attempt, attempt.request_id)
|
|
185
|
+
await self._finish(request, 'failed', error=StrategyFailed(f"dispatch failed: {exc}"))
|
|
186
|
+
|
|
187
|
+
async def claim_attempt(self, attempt: ExecutionAttempt) -> bool:
|
|
188
|
+
"""Called by an entry point before it runs `attempt`: save `running`
|
|
189
|
+
with the strategy's deadline, counted from now. False when the
|
|
190
|
+
request does not hold this attempt as `pending`, including when
|
|
191
|
+
another delivery of it claimed it first; the entry point then runs
|
|
192
|
+
nothing."""
|
|
193
|
+
request = await self.entity_service.repository.load(attempt.request_id)
|
|
194
|
+
if request is None or request.status != 'pending' or request.attempt != attempt.attempt:
|
|
195
|
+
return False
|
|
196
|
+
strategy_cls = self.registry.strategies.by_name(request.strategy_name)
|
|
197
|
+
stored = copy.copy(request)
|
|
198
|
+
request.status = 'running'
|
|
199
|
+
request.attempt_deadline = time.time() + (
|
|
200
|
+
getattr(strategy_cls, 'timeout', None) or self.default_timeout
|
|
201
|
+
)
|
|
202
|
+
try:
|
|
203
|
+
with kernel_internal_context():
|
|
204
|
+
await self.entity_service.save_entity(
|
|
205
|
+
request, Principal(principal_entity_id=request.owner), existing=stored,
|
|
206
|
+
)
|
|
207
|
+
except ConcurrentModificationError:
|
|
208
|
+
return False
|
|
209
|
+
return True
|
|
210
|
+
|
|
211
|
+
async def finish_attempt(self, result: AttemptResult) -> None:
|
|
212
|
+
"""Apply one attempt's result. A result for an attempt the request no
|
|
213
|
+
longer holds, or a repeat delivery, has no effect."""
|
|
214
|
+
request = await self.entity_service.repository.load(result.request_id)
|
|
215
|
+
if request is None or request.status != 'running' or request.attempt != result.attempt:
|
|
216
|
+
logger.info("Discarding stale result for attempt %s of %s", result.attempt, result.request_id)
|
|
217
|
+
return
|
|
218
|
+
|
|
219
|
+
if result.outcome == 'succeeded':
|
|
220
|
+
await self._finish(request, 'succeeded', ret_val=result.ret_val or {})
|
|
221
|
+
elif result.outcome == 'conflict' and request.attempt < self.max_attempts:
|
|
222
|
+
await self._backoff(request.attempt)
|
|
223
|
+
await self.start_attempt(request)
|
|
224
|
+
else:
|
|
225
|
+
await self._finish(request, 'failed', error=KernelError.from_dict(result.error or {}))
|
|
226
|
+
|
|
227
|
+
async def _backoff(self, attempt: int) -> None:
|
|
228
|
+
await asyncio.sleep(0.1 * 2 ** attempt + random.uniform(0, 0.1))
|
|
229
|
+
|
|
230
|
+
async def reconcile_runs(self) -> int:
|
|
231
|
+
"""Reconcile every unfinished request past its deadline. Returns how
|
|
232
|
+
many it acted on."""
|
|
233
|
+
repository = self.entity_service.repository
|
|
234
|
+
unfinished = await repository.list_ids(AllOf((
|
|
235
|
+
Equal('entity_type_id', StrategyRequest.get_entity_type_id()),
|
|
236
|
+
InValues('status', frozenset(UNFINISHED_STATUSES)),
|
|
237
|
+
)))
|
|
238
|
+
acted = 0
|
|
239
|
+
for request in (await repository.load_many(unfinished)).values():
|
|
240
|
+
if _overdue(request):
|
|
241
|
+
acted += await self._reconcile(request)
|
|
242
|
+
return acted
|
|
243
|
+
|
|
244
|
+
async def _reconcile(self, request: StrategyRequest) -> int:
|
|
245
|
+
"""Send an unclaimed attempt again, as the same attempt; start the
|
|
246
|
+
next attempt of an overdue running one, or fail the run once out of
|
|
247
|
+
attempts. 0 when another writer changed the request first."""
|
|
248
|
+
try:
|
|
249
|
+
if request.status == 'pending':
|
|
250
|
+
await self._send(request, request.attempt)
|
|
251
|
+
elif request.attempt < self.max_attempts:
|
|
252
|
+
await self.start_attempt(request)
|
|
253
|
+
else:
|
|
254
|
+
await self._finish(request, 'failed', error=StrategyFailed('attempt timed out'))
|
|
255
|
+
return 1
|
|
256
|
+
except ConcurrentModificationError:
|
|
257
|
+
logger.info("Request %s changed while reconciling; left to its writer", request.entity_id)
|
|
258
|
+
return 0
|
|
259
|
+
|
|
260
|
+
async def _finish(self, request: StrategyRequest, status: str, ret_val: Optional[dict] = None,
|
|
261
|
+
error: Optional[KernelError] = None) -> None:
|
|
262
|
+
owner = Principal(principal_entity_id=request.owner)
|
|
263
|
+
stored = copy.copy(request)
|
|
264
|
+
request.status = status
|
|
265
|
+
request.ret_val = ret_val or {}
|
|
266
|
+
request.error = error.message if error else None
|
|
267
|
+
request.error_code = error.code if error else None
|
|
268
|
+
request.error_details = dict(error.details) if error else {}
|
|
269
|
+
request.attempt_deadline = None
|
|
270
|
+
with kernel_internal_context():
|
|
271
|
+
request = await self.entity_service.save_entity(request, owner, existing=stored)
|
|
272
|
+
await self.run_events.publish(request.entity_id)
|
|
273
|
+
|
|
274
|
+
if status == 'succeeded':
|
|
275
|
+
for callback in request.callbacks or []:
|
|
276
|
+
await self._submit_callback(request, callback, owner)
|
|
277
|
+
if request.callback_channel:
|
|
278
|
+
await self._report(request)
|
|
279
|
+
|
|
280
|
+
async def _submit_callback(self, request: StrategyRequest, callback: dict, owner: Principal) -> None:
|
|
281
|
+
try:
|
|
282
|
+
payload = StrategyRequestPayload.from_json(callback)
|
|
283
|
+
payload.param_config = {**payload.param_config, 'source_ret_val': request.ret_val}
|
|
284
|
+
payload.client_request_id = None
|
|
285
|
+
await self.submit(payload, owner, wait=False)
|
|
286
|
+
except Exception:
|
|
287
|
+
logger.exception("Callback %r of request %s was not submitted", callback, request.entity_id)
|
|
288
|
+
|
|
289
|
+
async def _report(self, request: StrategyRequest) -> None:
|
|
290
|
+
"""One `strategy_complete` shape for every environment."""
|
|
291
|
+
message = {
|
|
292
|
+
'type': 'strategy_complete',
|
|
293
|
+
'request_id': request.entity_id,
|
|
294
|
+
'target_entity_id': request.target_entity_id,
|
|
295
|
+
'client_request_id': request.client_request_id,
|
|
296
|
+
'status': request.status,
|
|
297
|
+
'success': request.status == 'succeeded',
|
|
298
|
+
}
|
|
299
|
+
if request.status == 'succeeded':
|
|
300
|
+
message['ret_val'] = request.ret_val
|
|
301
|
+
else:
|
|
302
|
+
message['error'] = request.error
|
|
303
|
+
message['code'] = request.error_code
|
|
304
|
+
message['details'] = request.error_details or {}
|
|
305
|
+
try:
|
|
306
|
+
await self.entity_service.broadcaster.send(request.callback_channel, message)
|
|
307
|
+
except Exception:
|
|
308
|
+
logger.exception("strategy_complete for %s was not delivered", request.entity_id)
|
|
309
|
+
|
|
310
|
+
# --- waiting ----------------------------------------------------------
|
|
311
|
+
|
|
312
|
+
async def _until_terminal(self, request_id: str, subscription: RunSubscription) -> StrategyRequest:
|
|
313
|
+
"""Read the request whenever its run's event arrives or its attempt's
|
|
314
|
+
deadline passes. A waiter past the deadline reconciles the run
|
|
315
|
+
itself, so it never depends on the periodic reconciler."""
|
|
316
|
+
while True:
|
|
317
|
+
request = await self.entity_service.repository.load(request_id)
|
|
318
|
+
if request is None:
|
|
319
|
+
raise NotFound(f"Strategy request {request_id} no longer exists")
|
|
320
|
+
if request.status in TERMINAL_STATUSES:
|
|
321
|
+
return request
|
|
322
|
+
if _overdue(request):
|
|
323
|
+
await self._reconcile(request)
|
|
324
|
+
continue
|
|
325
|
+
await subscription.wait(until=request.attempt_deadline)
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
UNFINISHED_STATUSES = ('pending', 'running')
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _overdue(request: StrategyRequest) -> bool:
|
|
332
|
+
return (request.status in UNFINISHED_STATUSES and request.attempt_deadline is not None
|
|
333
|
+
and request.attempt_deadline <= time.time())
|
|
File without changes
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""The port that hands one attempt to an execution environment
|
|
2
|
+
(docs/STRATEGY_EXECUTION_TDD.md, The dispatcher port)."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
|
|
7
|
+
from graphcore.types.execution import ExecutionAttempt
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Dispatcher(ABC):
|
|
11
|
+
@abstractmethod
|
|
12
|
+
async def dispatch(self, attempt: ExecutionAttempt) -> None:
|
|
13
|
+
"""Deliver the attempt to the environment and return once it is
|
|
14
|
+
accepted. Raises only if it could not be delivered. The result
|
|
15
|
+
reaches `StrategyExecutorService.finish_attempt`, never this call."""
|
|
16
|
+
|
|
17
|
+
async def drain(self) -> None:
|
|
18
|
+
"""Wait for attempts this process is running on the current event
|
|
19
|
+
loop. An entry point that owns its loop calls it before returning."""
|
|
File without changes
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""A Celery process holding a runtime (docs/STRATEGY_EXECUTION_TDD.md,
|
|
2
|
+
Celery). The dispatcher enqueues the attempt's `to_dict()` form; the
|
|
3
|
+
`run_attempt` task is the entry point."""
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import asyncio
|
|
7
|
+
import threading
|
|
8
|
+
|
|
9
|
+
from asgiref.sync import async_to_sync
|
|
10
|
+
from celery import shared_task
|
|
11
|
+
from celery.signals import worker_init
|
|
12
|
+
|
|
13
|
+
from graphcore.domain.strategy.runner import ChangeTracker, StrategyRunner
|
|
14
|
+
from graphcore.errors import KernelError
|
|
15
|
+
from graphcore.execution.dispatcher import Dispatcher
|
|
16
|
+
from graphcore.execution.local_session import LocalStrategySession
|
|
17
|
+
from graphcore.runtime import get_runtime, serve_as_worker
|
|
18
|
+
from graphcore.types.execution import AttemptResult, ExecutionAttempt
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@worker_init.connect
|
|
22
|
+
def _serve_as_worker(**kwargs) -> None:
|
|
23
|
+
# A worker on a thread shares its process with a web server (`graphcore run --debug`).
|
|
24
|
+
if threading.current_thread() is threading.main_thread():
|
|
25
|
+
serve_as_worker()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class CeleryDispatcher(Dispatcher):
|
|
29
|
+
async def dispatch(self, attempt: ExecutionAttempt) -> None:
|
|
30
|
+
# A plain executor thread, not sync_to_async: in eager mode the task
|
|
31
|
+
# then runs on its own loop, as in a worker, not on the caller's.
|
|
32
|
+
await asyncio.get_running_loop().run_in_executor(None, run_attempt.delay, attempt.to_dict())
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
async def settle(executor) -> None:
|
|
36
|
+
"""Let in-process attempts this loop started finish before the task's
|
|
37
|
+
loop closes, e.g. an IO child submitted by a CPU strategy."""
|
|
38
|
+
for dispatcher in executor.dispatchers.values():
|
|
39
|
+
await dispatcher.drain()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@shared_task(name='graphcore.tasks.run_attempt')
|
|
43
|
+
def run_attempt(data: dict) -> None:
|
|
44
|
+
rt = get_runtime()
|
|
45
|
+
attempt = ExecutionAttempt.from_dict(data)
|
|
46
|
+
|
|
47
|
+
async def run():
|
|
48
|
+
if not await rt.executor.claim_attempt(attempt):
|
|
49
|
+
return
|
|
50
|
+
tracker = ChangeTracker()
|
|
51
|
+
session = LocalStrategySession(attempt, tracker, rt.entity_service, rt.executor, rt.registry)
|
|
52
|
+
try:
|
|
53
|
+
result = await StrategyRunner(rt.registry).run(attempt, session, tracker)
|
|
54
|
+
except Exception as exc:
|
|
55
|
+
result = AttemptResult(attempt.request_id, attempt.attempt, 'failed', error=KernelError.of(exc).to_dict())
|
|
56
|
+
await rt.executor.finish_attempt(result)
|
|
57
|
+
await settle(rt.executor)
|
|
58
|
+
|
|
59
|
+
async_to_sync(run)()
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""The submitting process's own event loop: the dispatcher is also the
|
|
2
|
+
entry point (docs/STRATEGY_EXECUTION_TDD.md, In-process)."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import logging
|
|
7
|
+
|
|
8
|
+
from graphcore.domain.strategy.runner import ChangeTracker, StrategyRunner
|
|
9
|
+
from graphcore.errors import KernelError
|
|
10
|
+
from graphcore.execution.dispatcher import Dispatcher
|
|
11
|
+
from graphcore.execution.local_session import LocalStrategySession
|
|
12
|
+
from graphcore.types.execution import AttemptResult, ExecutionAttempt
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class InProcessDispatcher(Dispatcher):
|
|
18
|
+
def __init__(self, executor, entity_service, registry):
|
|
19
|
+
self.executor = executor
|
|
20
|
+
self.entity_service = entity_service
|
|
21
|
+
self.registry = registry
|
|
22
|
+
self._tasks: set[asyncio.Task] = set()
|
|
23
|
+
|
|
24
|
+
async def dispatch(self, attempt: ExecutionAttempt) -> None:
|
|
25
|
+
task = asyncio.create_task(self._run(attempt))
|
|
26
|
+
self._tasks.add(task)
|
|
27
|
+
task.add_done_callback(self._tasks.discard)
|
|
28
|
+
|
|
29
|
+
async def _run(self, attempt: ExecutionAttempt) -> None:
|
|
30
|
+
self.entity_service.repository.use_shared_threads()
|
|
31
|
+
try:
|
|
32
|
+
if not await self.executor.claim_attempt(attempt):
|
|
33
|
+
return
|
|
34
|
+
except Exception:
|
|
35
|
+
logger.exception("Could not claim attempt %s of %s", attempt.attempt, attempt.request_id)
|
|
36
|
+
return
|
|
37
|
+
try:
|
|
38
|
+
tracker = ChangeTracker()
|
|
39
|
+
session = LocalStrategySession(attempt, tracker, self.entity_service, self.executor, self.registry)
|
|
40
|
+
result = await StrategyRunner(self.registry).run(attempt, session, tracker)
|
|
41
|
+
except Exception as exc:
|
|
42
|
+
logger.exception("In-process attempt %s of %s failed", attempt.attempt, attempt.request_id)
|
|
43
|
+
result = AttemptResult(attempt.request_id, attempt.attempt, 'failed', error=KernelError.of(exc).to_dict())
|
|
44
|
+
try:
|
|
45
|
+
await self.executor.finish_attempt(result)
|
|
46
|
+
except Exception:
|
|
47
|
+
logger.exception("Could not finish attempt %s of %s", attempt.attempt, attempt.request_id)
|
|
48
|
+
|
|
49
|
+
async def drain(self) -> None:
|
|
50
|
+
loop = asyncio.get_running_loop()
|
|
51
|
+
while True:
|
|
52
|
+
pending = [t for t in self._tasks if not t.done() and t.get_loop() is loop]
|
|
53
|
+
if not pending:
|
|
54
|
+
return
|
|
55
|
+
await asyncio.gather(*pending, return_exceptions=True)
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""`StrategySession` over a runtime's `EntityService` and the executor's
|
|
2
|
+
front door (docs/STRATEGY_EXECUTION_TDD.md, The session adapters)."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
from graphcore.domain.entity.strategy_request.StrategyRequest import StrategyRequest
|
|
8
|
+
from graphcore.domain.strategy.runner import ChangeTracker
|
|
9
|
+
from graphcore.domain.strategy.session import StrategySession
|
|
10
|
+
from graphcore.errors import PreconditionFailed, StaleAttempt
|
|
11
|
+
from graphcore.execution.requests import new_strategy_request
|
|
12
|
+
from graphcore.execution.strategy_context import strategy_context
|
|
13
|
+
from graphcore.persistence.repository import Precondition
|
|
14
|
+
from graphcore.types.entities import EntityFilter, Tombstone
|
|
15
|
+
from graphcore.types.entity_id import derived_entity_id
|
|
16
|
+
from graphcore.types.execution import ExecutionAttempt
|
|
17
|
+
from graphcore.types.identity import Principal
|
|
18
|
+
from graphcore.types.query import AllOf, Equal
|
|
19
|
+
from graphcore.types.strategy import StrategyRequestPayload
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class LocalStrategySession(StrategySession):
|
|
23
|
+
"""Reads and writes as the attempt's owner. Every write is fenced and
|
|
24
|
+
runs inside the strategy's context, so the persistence chokepoint
|
|
25
|
+
authorizes it against the strategy's class."""
|
|
26
|
+
|
|
27
|
+
def __init__(self, attempt: ExecutionAttempt, tracker: ChangeTracker, entity_service,
|
|
28
|
+
executor, registry):
|
|
29
|
+
self.attempt = attempt
|
|
30
|
+
self.tracker = tracker
|
|
31
|
+
self.entity_service = entity_service
|
|
32
|
+
self.executor = executor
|
|
33
|
+
self.registry = registry
|
|
34
|
+
self.principal = Principal(principal_entity_id=attempt.owner_id)
|
|
35
|
+
self._strategy_cls = None
|
|
36
|
+
self._created = 0
|
|
37
|
+
self._unsaved_creations: set[str] = set()
|
|
38
|
+
|
|
39
|
+
def _strategy(self):
|
|
40
|
+
if self._strategy_cls is None:
|
|
41
|
+
self._strategy_cls = self.registry.strategies.require(self.attempt.strategy_name)
|
|
42
|
+
return self._strategy_cls
|
|
43
|
+
|
|
44
|
+
def _fence(self) -> Precondition:
|
|
45
|
+
return Precondition(self.attempt.request_id, AllOf((
|
|
46
|
+
Equal('entity_type_id', StrategyRequest.get_entity_type_id()),
|
|
47
|
+
Equal('status', 'running'),
|
|
48
|
+
Equal('attempt', self.attempt.attempt),
|
|
49
|
+
)))
|
|
50
|
+
|
|
51
|
+
def _stale(self) -> StaleAttempt:
|
|
52
|
+
return StaleAttempt(
|
|
53
|
+
f"Attempt {self.attempt.attempt} of request {self.attempt.request_id} is no longer current"
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
def create(self, entity_cls, **fields):
|
|
57
|
+
entity_id = derived_entity_id(self.attempt.backend_id, f"{self.attempt.request_id}/{self._created}")
|
|
58
|
+
self._created += 1
|
|
59
|
+
self._unsaved_creations.add(entity_id)
|
|
60
|
+
entity = entity_cls(entity_id=entity_id)
|
|
61
|
+
parent_id = fields.pop('parent_id', None)
|
|
62
|
+
if parent_id is not None:
|
|
63
|
+
entity.set_parent(parent_id)
|
|
64
|
+
for name, value in fields.items():
|
|
65
|
+
setattr(entity, name, value)
|
|
66
|
+
return entity
|
|
67
|
+
|
|
68
|
+
async def get(self, entity_id: str):
|
|
69
|
+
entity = await self.entity_service.get_entity(entity_id, self.principal)
|
|
70
|
+
self.tracker.record(entity)
|
|
71
|
+
return entity
|
|
72
|
+
|
|
73
|
+
async def get_many(self, entity_ids: list[str]) -> dict:
|
|
74
|
+
found = await self.entity_service.get_entities(entity_ids, self.principal)
|
|
75
|
+
entities = {eid: e for eid, e in found.items() if not isinstance(e, Tombstone)}
|
|
76
|
+
for entity in entities.values():
|
|
77
|
+
self.tracker.record(entity)
|
|
78
|
+
return entities
|
|
79
|
+
|
|
80
|
+
async def list(self, entity_filter: EntityFilter, cursor: Optional[str] = None, limit: int = 25):
|
|
81
|
+
page = await self.entity_service.list_entities(self.principal, entity_filter, cursor=cursor, limit=limit)
|
|
82
|
+
for entity in page.entities:
|
|
83
|
+
self.tracker.record(entity)
|
|
84
|
+
return page
|
|
85
|
+
|
|
86
|
+
async def save(self, entity):
|
|
87
|
+
if entity.entity_id in self._unsaved_creations and entity.version == 0:
|
|
88
|
+
await self._replace_earlier_attempts(entity)
|
|
89
|
+
try:
|
|
90
|
+
with strategy_context(self._strategy()):
|
|
91
|
+
saved = await self.entity_service.save_entity(entity, self.principal, precondition=self._fence())
|
|
92
|
+
except PreconditionFailed as exc:
|
|
93
|
+
raise self._stale() from exc
|
|
94
|
+
self._unsaved_creations.discard(entity.entity_id)
|
|
95
|
+
self.tracker.record(saved)
|
|
96
|
+
return saved
|
|
97
|
+
|
|
98
|
+
async def _replace_earlier_attempts(self, entity) -> None:
|
|
99
|
+
"""An earlier attempt of this request may have saved the entity
|
|
100
|
+
`create` minted again; this attempt's first save replaces it."""
|
|
101
|
+
stored = await self.entity_service.repository.load(entity.entity_id)
|
|
102
|
+
if stored is not None and type(stored) is type(entity):
|
|
103
|
+
entity.version = stored.version
|
|
104
|
+
|
|
105
|
+
async def delete_tree(self, root_id: str) -> list[str]:
|
|
106
|
+
try:
|
|
107
|
+
with strategy_context(self._strategy()):
|
|
108
|
+
return await self.entity_service.delete_entity_tree(root_id, self.principal, precondition=self._fence())
|
|
109
|
+
except PreconditionFailed as exc:
|
|
110
|
+
raise self._stale() from exc
|
|
111
|
+
|
|
112
|
+
async def execute(self, payload: StrategyRequestPayload, *, wait: bool = True,
|
|
113
|
+
callback_channel: Optional[str] = None):
|
|
114
|
+
request = new_strategy_request(self.entity_service, payload)
|
|
115
|
+
return await self.executor.execute_request(
|
|
116
|
+
request, self.principal, wait=wait, callback_channel=callback_channel,
|
|
117
|
+
)
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""`RunEvents` across processes over Redis pub/sub
|
|
2
|
+
(docs/STRATEGY_EXECUTION_TDD.md, Waiting)."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import logging
|
|
7
|
+
import os
|
|
8
|
+
import threading
|
|
9
|
+
import time
|
|
10
|
+
|
|
11
|
+
from graphcore.execution.run_events import RunEvents
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
CHANNEL = 'graphcore.run_finished'
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class RedisRunEvents(RunEvents):
|
|
19
|
+
"""Each process publishes finished request ids on one channel and runs
|
|
20
|
+
one listener thread, started on its first subscription, that wakes the
|
|
21
|
+
process's subscribers. Whenever the listener (re)subscribes, it wakes
|
|
22
|
+
every subscriber so each re-reads its request: nothing published while
|
|
23
|
+
it was not listening is relied on."""
|
|
24
|
+
|
|
25
|
+
def __init__(self, url: str, channel: str = CHANNEL, reconnect_delay: float = 1.0):
|
|
26
|
+
super().__init__()
|
|
27
|
+
self.url = url
|
|
28
|
+
self.channel = channel
|
|
29
|
+
self.reconnect_delay = reconnect_delay
|
|
30
|
+
self._pid = None
|
|
31
|
+
self._client = None
|
|
32
|
+
self._listener = None
|
|
33
|
+
|
|
34
|
+
def _redis(self):
|
|
35
|
+
import redis
|
|
36
|
+
|
|
37
|
+
# Per process: a forked worker must not share its parent's sockets.
|
|
38
|
+
if self._pid != os.getpid():
|
|
39
|
+
self._pid = os.getpid()
|
|
40
|
+
self._client = redis.Redis.from_url(self.url, decode_responses=True)
|
|
41
|
+
self._listener = None
|
|
42
|
+
return self._client
|
|
43
|
+
|
|
44
|
+
def _subscribed(self) -> None:
|
|
45
|
+
client = self._redis()
|
|
46
|
+
with self._lock:
|
|
47
|
+
if self._listener is None or not self._listener.is_alive():
|
|
48
|
+
self._listener = threading.Thread(
|
|
49
|
+
target=self._listen, args=(client,), name='graphcore-run-events', daemon=True,
|
|
50
|
+
)
|
|
51
|
+
self._listener.start()
|
|
52
|
+
|
|
53
|
+
def _listen(self, client) -> None:
|
|
54
|
+
while True:
|
|
55
|
+
try:
|
|
56
|
+
pubsub = client.pubsub()
|
|
57
|
+
pubsub.subscribe(self.channel)
|
|
58
|
+
for message in pubsub.listen():
|
|
59
|
+
if message['type'] == 'subscribe':
|
|
60
|
+
self.notify_local()
|
|
61
|
+
elif message['type'] == 'message':
|
|
62
|
+
self.notify_local(message['data'])
|
|
63
|
+
except Exception:
|
|
64
|
+
logger.warning("Run events listener lost %s; reconnecting", self.channel, exc_info=True)
|
|
65
|
+
time.sleep(self.reconnect_delay)
|
|
66
|
+
|
|
67
|
+
async def publish(self, request_id: str) -> None:
|
|
68
|
+
self.notify_local(request_id)
|
|
69
|
+
try:
|
|
70
|
+
await asyncio.get_running_loop().run_in_executor(
|
|
71
|
+
None, self._redis().publish, self.channel, request_id,
|
|
72
|
+
)
|
|
73
|
+
except Exception:
|
|
74
|
+
logger.exception("Run event for %s was not published", request_id)
|