graphcore-kernel 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- graphcore/__init__.py +5 -0
- graphcore/agent_frontend/__init__.py +8 -0
- graphcore/agent_frontend/api.py +69 -0
- graphcore/agent_frontend/base/__init__.py +11 -0
- graphcore/agent_frontend/base/views.py +167 -0
- graphcore/agent_frontend/client.py +31 -0
- graphcore/agent_frontend/context.py +157 -0
- graphcore/agent_frontend/errors.py +12 -0
- graphcore/agent_frontend/loading.py +54 -0
- graphcore/agent_frontend/registry.py +42 -0
- graphcore/agent_frontend/render.py +207 -0
- graphcore/agent_frontend/server.py +103 -0
- graphcore/agent_frontend/signatures.py +35 -0
- graphcore/agent_frontend/testing.py +45 -0
- graphcore/agent_frontend/tests/__init__.py +0 -0
- graphcore/agent_frontend/tests/fakes.py +114 -0
- graphcore/agent_frontend/tests/test_frontend.py +447 -0
- graphcore/agent_frontend/tests/test_loading.py +83 -0
- graphcore/agent_frontend/tests/test_server.py +109 -0
- graphcore/agent_frontend/tools.py +137 -0
- graphcore/api/__init__.py +4 -0
- graphcore/api/graph_api.py +378 -0
- graphcore/api/predicate_codec.py +81 -0
- graphcore/api/tests/__init__.py +0 -0
- graphcore/api/tests/test_changed_entities.py +109 -0
- graphcore/api/tests/test_graph_api.py +290 -0
- graphcore/api/tests/test_graph_api_access.py +94 -0
- graphcore/api/tests/test_graph_api_grants.py +90 -0
- graphcore/api/tests/test_predicate_codec.py +83 -0
- graphcore/api/tests/test_subtree_query.py +79 -0
- graphcore/apps.py +71 -0
- graphcore/auth/__init__.py +1 -0
- graphcore/auth/authn/README.md +178 -0
- graphcore/auth/authn/__init__.py +3 -0
- graphcore/auth/authn/authenticator.py +120 -0
- graphcore/auth/authn/onboarding.py +105 -0
- graphcore/auth/authn/principals.py +20 -0
- graphcore/auth/authn/resource_metadata.py +43 -0
- graphcore/auth/authn/service_accounts.py +96 -0
- graphcore/auth/authn/testing.py +96 -0
- graphcore/auth/authn/tests/__init__.py +0 -0
- graphcore/auth/authn/tests/test_authenticator.py +145 -0
- graphcore/auth/authn/tests/test_onboarding.py +145 -0
- graphcore/auth/authn/tests/test_service_accounts.py +180 -0
- graphcore/auth/authn/users.py +59 -0
- graphcore/auth/authz/__init__.py +0 -0
- graphcore/auth/authz/access_port.py +120 -0
- graphcore/auth/authz/adapter/__init__.py +0 -0
- graphcore/auth/authz/adapter/default_access_adapter.py +556 -0
- graphcore/auth/authz/adapter/tests/__init__.py +0 -0
- graphcore/auth/authz/adapter/tests/test_default_adapter.py +543 -0
- graphcore/auth/authz/adapter/tests/test_entity_write_hooks.py +207 -0
- graphcore/auth/authz/adapter/tests/test_group_types.py +70 -0
- graphcore/auth/authz/adapter/tests/test_sync_support.py +209 -0
- graphcore/auth/authz/defaults.py +26 -0
- graphcore/auth/authz/kernel_policy.py +191 -0
- graphcore/auth/authz/kernel_writes.py +31 -0
- graphcore/auth/authz/lookups.py +25 -0
- graphcore/auth/authz/policy.py +403 -0
- graphcore/auth/authz/tests/__init__.py +0 -0
- graphcore/auth/authz/tests/test_can_contain.py +150 -0
- graphcore/auth/authz/tests/test_contained_by.py +167 -0
- graphcore/auth/authz/tests/test_creation.py +391 -0
- graphcore/auth/authz/tests/test_delete_permission.py +82 -0
- graphcore/auth/authz/tests/test_granted_to_me.py +160 -0
- graphcore/auth/authz/tests/test_integration.py +637 -0
- graphcore/auth/authz/tests/test_invites.py +184 -0
- graphcore/auth/authz/tests/test_kernel_policy.py +140 -0
- graphcore/auth/authz/tests/test_lockout.py +132 -0
- graphcore/auth/authz/tests/test_policy.py +329 -0
- graphcore/auth/authz/tests/test_primitives.py +73 -0
- graphcore/auth/authz/tests/test_principal_creation.py +62 -0
- graphcore/bootstrap/__init__.py +0 -0
- graphcore/bootstrap/deployment.py +67 -0
- graphcore/bootstrap/signals.py +16 -0
- graphcore/bootstrap/tests/__init__.py +0 -0
- graphcore/bootstrap/tests/test_deployment_seed.py +154 -0
- graphcore/broadcast/__init__.py +4 -0
- graphcore/broadcast/broadcaster.py +201 -0
- graphcore/broadcast/channels.py +56 -0
- graphcore/broadcast/tests/__init__.py +0 -0
- graphcore/broadcast/tests/test_broadcaster.py +444 -0
- graphcore/celery.py +14 -0
- graphcore/docs/AGENT_FRONTEND_TDD.md +389 -0
- graphcore/docs/ARCHITECTURE_PRINCIPLES.md +152 -0
- graphcore/docs/AUTHORIZATION_TECHNICAL_TDD.md +1379 -0
- graphcore/docs/ENTITY_REFERENCE_TDD.md +561 -0
- graphcore/docs/GRAPHAPI_FRONT_DOOR.md +173 -0
- graphcore/docs/RUNTIME.md +90 -0
- graphcore/docs/SCHEDULING_TDD.md +244 -0
- graphcore/docs/STRATEGY_EXECUTION_TDD.md +1043 -0
- graphcore/docs/VIEW_ENTITY_TDD.md +522 -0
- graphcore/docs/VIEW_SYSTEM_TDD.md +307 -0
- graphcore/docs/__init__.py +0 -0
- graphcore/docs/doc_topics.py +97 -0
- graphcore/docs/docs_discovery.py +64 -0
- graphcore/docs/public_api.py +56 -0
- graphcore/domain/__init__.py +0 -0
- graphcore/domain/entity/Entity.py +204 -0
- graphcore/domain/entity/__init__.py +0 -0
- graphcore/domain/entity/deployment/Deployment.py +25 -0
- graphcore/domain/entity/deployment/__init__.py +0 -0
- graphcore/domain/entity/grant/Grant.py +31 -0
- graphcore/domain/entity/grant/__init__.py +0 -0
- graphcore/domain/entity/group/Group.py +48 -0
- graphcore/domain/entity/group/README.md +88 -0
- graphcore/domain/entity/group/__init__.py +0 -0
- graphcore/domain/entity/invite/Invite.py +31 -0
- graphcore/domain/entity/invite/__init__.py +0 -0
- graphcore/domain/entity/request_template/RequestTemplate.py +48 -0
- graphcore/domain/entity/request_template/__init__.py +0 -0
- graphcore/domain/entity/request_template/tests/__init__.py +0 -0
- graphcore/domain/entity/request_template/tests/test_request_template.py +52 -0
- graphcore/domain/entity/service_account/ServiceAccount.py +41 -0
- graphcore/domain/entity/service_account/__init__.py +0 -0
- graphcore/domain/entity/strategy_request/StrategyRequest.py +61 -0
- graphcore/domain/entity/strategy_request/__init__.py +0 -0
- graphcore/domain/entity/strategy_request/tests/__init__.py +0 -0
- graphcore/domain/entity/strategy_request/tests/test_strategy_request_entity.py +35 -0
- graphcore/domain/entity/tests/__init__.py +0 -0
- graphcore/domain/entity/tests/test_entity.py +227 -0
- graphcore/domain/entity/tests/test_grant_group_entities.py +36 -0
- graphcore/domain/entity/tests/test_write_once_fields.py +98 -0
- graphcore/domain/entity/user/User.py +39 -0
- graphcore/domain/entity/user/__init__.py +0 -0
- graphcore/domain/entity/user/tests/__init__.py +0 -0
- graphcore/domain/entity/user/tests/test_user_entity.py +33 -0
- graphcore/domain/entity/view/View.py +37 -0
- graphcore/domain/entity/view/__init__.py +0 -0
- graphcore/domain/entity/view/position.py +78 -0
- graphcore/domain/entity/view/tests/__init__.py +0 -0
- graphcore/domain/entity/view/tests/test_position.py +71 -0
- graphcore/domain/entity/view/tests/test_view_entity.py +134 -0
- graphcore/domain/strategy/BaseStrategy.py +759 -0
- graphcore/domain/strategy/DeploymentStrategies.py +55 -0
- graphcore/domain/strategy/GrantStrategies.py +189 -0
- graphcore/domain/strategy/GroupStrategies.py +193 -0
- graphcore/domain/strategy/InviteStrategies.py +84 -0
- graphcore/domain/strategy/RequestTemplateStrategies.py +126 -0
- graphcore/domain/strategy/ViewStrategies.py +219 -0
- graphcore/domain/strategy/__init__.py +0 -0
- graphcore/domain/strategy/runner.py +84 -0
- graphcore/domain/strategy/session.py +56 -0
- graphcore/domain/strategy/tests/BaseStrategyTest.py +167 -0
- graphcore/domain/strategy/tests/__init__.py +0 -0
- graphcore/domain/strategy/tests/test_BaseStrategies.py +870 -0
- graphcore/domain/strategy/tests/test_grant_strategies.py +245 -0
- graphcore/domain/strategy/tests/test_group_strategies.py +99 -0
- graphcore/domain/strategy/tests/test_runner.py +220 -0
- graphcore/domain/strategy/tests/test_strategy_locks.py +446 -0
- graphcore/domain/strategy/tests/test_utils.py +95 -0
- graphcore/domain/strategy/tests/test_view_strategies.py +436 -0
- graphcore/errors.py +145 -0
- graphcore/execution/README.md +97 -0
- graphcore/execution/StrategyExecutorService.py +333 -0
- graphcore/execution/__init__.py +0 -0
- graphcore/execution/dispatcher.py +19 -0
- graphcore/execution/environments/__init__.py +0 -0
- graphcore/execution/environments/celery.py +59 -0
- graphcore/execution/environments/in_process.py +55 -0
- graphcore/execution/local_session.py +117 -0
- graphcore/execution/redis_run_events.py +74 -0
- graphcore/execution/requests.py +22 -0
- graphcore/execution/run_events.py +85 -0
- graphcore/execution/strategy_context.py +54 -0
- graphcore/execution/tasks.py +34 -0
- graphcore/execution/tests/__init__.py +5 -0
- graphcore/execution/tests/contract.py +171 -0
- graphcore/execution/tests/support.py +310 -0
- graphcore/execution/tests/test_dispatchers.py +76 -0
- graphcore/execution/tests/test_executor.py +368 -0
- graphcore/execution/tests/test_local_session.py +95 -0
- graphcore/execution/tests/test_reconcile_runs.py +134 -0
- graphcore/execution/tests/test_run_events.py +106 -0
- graphcore/execution/tests/test_strategy_context.py +62 -0
- graphcore/gateway/__init__.py +1 -0
- graphcore/gateway/remote.py +35 -0
- graphcore/gateway/tests/__init__.py +0 -0
- graphcore/gateway/tests/test_remote.py +30 -0
- graphcore/graph/EntityService.py +710 -0
- graphcore/graph/__init__.py +0 -0
- graphcore/graph/entity_cache.py +152 -0
- graphcore/graph/tests/__init__.py +4 -0
- graphcore/graph/tests/test_EntityService.py +908 -0
- graphcore/graph/tests/test_EntityService_list.py +170 -0
- graphcore/graph/tests/test_EntityService_model_resolution.py +339 -0
- graphcore/graph/tests/test_EntityService_referrers.py +236 -0
- graphcore/graph/tests/test_EntityService_routing.py +119 -0
- graphcore/graph/tests/test_dependent_on.py +141 -0
- graphcore/graph/tests/test_engine_is_type_agnostic.py +25 -0
- graphcore/graph/tests/test_engine_resolves_read_permission.py +93 -0
- graphcore/graph/tests/test_entity_cache.py +159 -0
- graphcore/graph/tests/test_null_entity_cache.py +12 -0
- graphcore/management/__init__.py +0 -0
- graphcore/management/commands/__init__.py +0 -0
- graphcore/management/commands/dev_reset.py +18 -0
- graphcore/management/commands/graphcore_createadmin.py +65 -0
- graphcore/management/commands/graphcore_policy.py +59 -0
- graphcore/management/commands/graphcore_status.py +98 -0
- graphcore/management/tests/__init__.py +0 -0
- graphcore/management/tests/test_createadmin.py +78 -0
- graphcore/management/tests/test_policy_command.py +46 -0
- graphcore/management/tests/test_status_command.py +85 -0
- graphcore/migrations/0001_initial.py +186 -0
- graphcore/migrations/0002_invite.py +26 -0
- graphcore/migrations/__init__.py +0 -0
- graphcore/models.py +11 -0
- graphcore/persistence/ENTITY_REPOSITORY_TECHNICAL_TDD.md +396 -0
- graphcore/persistence/__init__.py +6 -0
- graphcore/persistence/django/__init__.py +0 -0
- graphcore/persistence/django/backends.py +43 -0
- graphcore/persistence/django/bindings.py +28 -0
- graphcore/persistence/django/cache.py +38 -0
- graphcore/persistence/django/compiler.py +214 -0
- graphcore/persistence/django/identity.py +18 -0
- graphcore/persistence/django/models.py +252 -0
- graphcore/persistence/django/repository.py +525 -0
- graphcore/persistence/django/tests/__init__.py +0 -0
- graphcore/persistence/django/tests/test_backends.py +68 -0
- graphcore/persistence/django/tests/test_compiler.py +269 -0
- graphcore/persistence/django/tests/test_repository.py +472 -0
- graphcore/persistence/django/tests/test_shared_column.py +120 -0
- graphcore/persistence/django/tests/test_threads.py +50 -0
- graphcore/persistence/django/tests/test_update_shape.py +175 -0
- graphcore/persistence/django/threads.py +38 -0
- graphcore/persistence/repository.py +172 -0
- graphcore/persistence/tests/__init__.py +0 -0
- graphcore/persistence/tests/contract.py +688 -0
- graphcore/persistence/tests/evaluate.py +51 -0
- graphcore/persistence/tests/fixtures.py +32 -0
- graphcore/persistence/tests/memory.py +263 -0
- graphcore/persistence/tests/test_memory_repository.py +10 -0
- graphcore/persistence/tests/test_query_types.py +127 -0
- graphcore/public/__init__.py +9 -0
- graphcore/public/api.py +9 -0
- graphcore/public/apps.py +10 -0
- graphcore/public/auth/__init__.py +1 -0
- graphcore/public/auth/grants.py +7 -0
- graphcore/public/auth/onboarding.py +8 -0
- graphcore/public/auth/perms.py +7 -0
- graphcore/public/auth/policy.py +10 -0
- graphcore/public/auth/roles.py +8 -0
- graphcore/public/auth/subjects.py +7 -0
- graphcore/public/entities/__init__.py +32 -0
- graphcore/public/errors.py +19 -0
- graphcore/public/persistence/__init__.py +44 -0
- graphcore/public/persistence/django.py +24 -0
- graphcore/public/scheduling.py +23 -0
- graphcore/public/strategies/__init__.py +85 -0
- graphcore/public/test_settings.py +16 -0
- graphcore/public/testing.py +47 -0
- graphcore/registry/EntityDirectory.py +246 -0
- graphcore/registry/README.md +147 -0
- graphcore/registry/StrategyDirectory.py +308 -0
- graphcore/registry/TYPE_METADATA_PLACEMENT.md +66 -0
- graphcore/registry/__init__.py +0 -0
- graphcore/registry/app_discovery.py +108 -0
- graphcore/registry/decorators.py +77 -0
- graphcore/registry/discovery.py +111 -0
- graphcore/registry/registry.py +79 -0
- graphcore/registry/tests/__init__.py +0 -0
- graphcore/registry/tests/test_agent_frontend_discovery.py +56 -0
- graphcore/registry/tests/test_app_config.py +36 -0
- graphcore/registry/tests/test_discovery.py +147 -0
- graphcore/registry/tests/test_entity_directory_storage.py +73 -0
- graphcore/registry/tests/test_import_package.py +87 -0
- graphcore/registry/tests/test_principal_flag.py +67 -0
- graphcore/registry/tests/test_strategy_directory.py +29 -0
- graphcore/routes/__init__.py +116 -0
- graphcore/routes/tests/__init__.py +0 -0
- graphcore/routes/tests/test_routes.py +90 -0
- graphcore/runtime.py +243 -0
- graphcore/scheduling/__init__.py +0 -0
- graphcore/scheduling/beat_scheduler.py +164 -0
- graphcore/scheduling/beat_tasks.py +63 -0
- graphcore/scheduling/scheduler.py +37 -0
- graphcore/scheduling/tests/__init__.py +0 -0
- graphcore/scheduling/tests/contract.py +136 -0
- graphcore/scheduling/tests/memory.py +42 -0
- graphcore/scheduling/tests/test_beat_scheduler.py +87 -0
- graphcore/scheduling/tests/test_memory_scheduler.py +15 -0
- graphcore/scheduling/tests/test_ticks.py +143 -0
- graphcore/tasks.py +9 -0
- graphcore/test_settings.py +78 -0
- graphcore/testing/__init__.py +0 -0
- graphcore/testing/case.py +112 -0
- graphcore/testing/runner.py +32 -0
- graphcore/testing/settings.py +26 -0
- graphcore/testing/tests/__init__.py +0 -0
- graphcore/testing/tests/agent_view_fixture.py +19 -0
- graphcore/testing/tests/test_case_and_runner.py +219 -0
- graphcore/testing/tests/test_settings_for_apps.py +50 -0
- graphcore/tests/__init__.py +0 -0
- graphcore/tests/ids.py +14 -0
- graphcore/tests/public_api_manifest.py +102 -0
- graphcore/tests/runs.py +7 -0
- graphcore/tests/test_app_discovery.py +90 -0
- graphcore/tests/test_composition_boundaries.py +231 -0
- graphcore/tests/test_docs_discovery.py +108 -0
- graphcore/tests/test_no_print.py +70 -0
- graphcore/tests/test_packaging.py +64 -0
- graphcore/tests/test_public_api_boundaries.py +229 -0
- graphcore/tests/test_public_api_contract.py +62 -0
- graphcore/tests/test_public_api_docs.py +28 -0
- graphcore/tests/test_runtime.py +42 -0
- graphcore/tests/test_shared_type_boundaries.py +128 -0
- graphcore/tests/test_shared_types.py +144 -0
- graphcore/tests/test_test_settings.py +45 -0
- graphcore/transport/__init__.py +0 -0
- graphcore/transport/agent_asgi.py +68 -0
- graphcore/transport/agent_client.py +85 -0
- graphcore/transport/channel_layer.py +68 -0
- graphcore/transport/consumers.py +507 -0
- graphcore/transport/routing.py +6 -0
- graphcore/transport/tests/__init__.py +0 -0
- graphcore/transport/tests/test_agent_client.py +31 -0
- graphcore/transport/tests/test_api_views.py +733 -0
- graphcore/transport/tests/test_channel_layer.py +96 -0
- graphcore/transport/tests/test_consumers.py +963 -0
- graphcore/transport/tests/test_resource_metadata.py +67 -0
- graphcore/transport/urls.py +29 -0
- graphcore/transport/views.py +358 -0
- graphcore/types/__init__.py +4 -0
- graphcore/types/access.py +114 -0
- graphcore/types/api.py +76 -0
- graphcore/types/entities.py +100 -0
- graphcore/types/entity_id.py +85 -0
- graphcore/types/execution.py +41 -0
- graphcore/types/identity.py +20 -0
- graphcore/types/onboarding.py +20 -0
- graphcore/types/query.py +116 -0
- graphcore/types/schedules.py +91 -0
- graphcore/types/strategy.py +73 -0
- graphcore/types/tests/__init__.py +0 -0
- graphcore/types/tests/test_entity_id.py +50 -0
- graphcore/urls.py +5 -0
- graphcore_kernel-0.1.0.dist-info/METADATA +94 -0
- graphcore_kernel-0.1.0.dist-info/RECORD +343 -0
- graphcore_kernel-0.1.0.dist-info/WHEEL +5 -0
- graphcore_kernel-0.1.0.dist-info/entry_points.txt +2 -0
- graphcore_kernel-0.1.0.dist-info/licenses/LICENSE +202 -0
- graphcore_kernel-0.1.0.dist-info/licenses/NOTICE +2 -0
- graphcore_kernel-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""The `Scheduler` port (docs/SCHEDULING_TDD.md, The port).
|
|
2
|
+
|
|
3
|
+
A schedule is state on a `RequestTemplate`. `EntityService` tells the
|
|
4
|
+
scheduler about every committed write and deletion; the scheduler keeps its
|
|
5
|
+
infrastructure in step with the templates and converges on them by itself.
|
|
6
|
+
Nothing an implementation stores for itself crosses this port.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from abc import ABC, abstractmethod
|
|
11
|
+
from typing import Optional
|
|
12
|
+
|
|
13
|
+
from graphcore.domain.entity.Entity import Entity
|
|
14
|
+
from graphcore.types.schedules import HeldTask, ScheduleInfo
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Scheduler(ABC):
|
|
18
|
+
@abstractmethod
|
|
19
|
+
async def entity_saved(self, entity: Entity, previous: Optional[Entity]) -> None:
|
|
20
|
+
"""Post-commit, for every save. Ignores anything not a request
|
|
21
|
+
template. Never raises."""
|
|
22
|
+
|
|
23
|
+
@abstractmethod
|
|
24
|
+
async def entities_deleted(self, entity_ids: list[str]) -> None:
|
|
25
|
+
"""Post-commit, with every deleted id; most have no schedule, so cost
|
|
26
|
+
scales with the ids. Never raises."""
|
|
27
|
+
|
|
28
|
+
@abstractmethod
|
|
29
|
+
async def info(self, target_id: str) -> Optional[ScheduleInfo]:
|
|
30
|
+
"""The schedule held for template `target_id`, or None."""
|
|
31
|
+
|
|
32
|
+
@abstractmethod
|
|
33
|
+
async def held(self) -> list[HeldTask]:
|
|
34
|
+
"""Every task the scheduler holds, its own included; for operators."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
__all__ = ['Scheduler']
|
|
File without changes
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""The `Scheduler` contract, run against every implementation.
|
|
2
|
+
|
|
3
|
+
A mixin for a `unittest.TestCase` (or Django's). An implementation supplies
|
|
4
|
+
`make_scheduler()`, `converge(templates)` - its full convergence, with
|
|
5
|
+
`templates` as every persisted template - and `forget(target_id)`, which
|
|
6
|
+
drops a held schedule behind the scheduler's back. Published as
|
|
7
|
+
`graphcore.public.testing.SchedulerContract`.
|
|
8
|
+
"""
|
|
9
|
+
from asgiref.sync import async_to_sync
|
|
10
|
+
|
|
11
|
+
from graphcore.domain.entity.request_template.RequestTemplate import RequestTemplate
|
|
12
|
+
from graphcore.tests.ids import fixture_id
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class SchedulerContract:
|
|
16
|
+
def make_scheduler(self):
|
|
17
|
+
raise NotImplementedError
|
|
18
|
+
|
|
19
|
+
def converge(self, templates: list) -> None:
|
|
20
|
+
raise NotImplementedError
|
|
21
|
+
|
|
22
|
+
def forget(self, target_id: str) -> None:
|
|
23
|
+
raise NotImplementedError
|
|
24
|
+
|
|
25
|
+
def setUp(self):
|
|
26
|
+
super().setUp()
|
|
27
|
+
self.scheduler = self.make_scheduler()
|
|
28
|
+
|
|
29
|
+
def template(self, cron='0 2 * * *', enabled=True, owner=None) -> RequestTemplate:
|
|
30
|
+
template = RequestTemplate(entity_id=fixture_id())
|
|
31
|
+
template.strategy_name = 'Probe'
|
|
32
|
+
template.cron = cron
|
|
33
|
+
template.enabled = enabled
|
|
34
|
+
template.owner = owner or fixture_id()
|
|
35
|
+
return template
|
|
36
|
+
|
|
37
|
+
def changed(self, template: RequestTemplate, **fields) -> RequestTemplate:
|
|
38
|
+
new = RequestTemplate.from_dict(template.to_dict())
|
|
39
|
+
for name, value in fields.items():
|
|
40
|
+
setattr(new, name, value)
|
|
41
|
+
return new
|
|
42
|
+
|
|
43
|
+
def saved(self, entity, previous=None):
|
|
44
|
+
async_to_sync(self.scheduler.entity_saved)(entity, previous)
|
|
45
|
+
|
|
46
|
+
def deleted(self, *entity_ids):
|
|
47
|
+
async_to_sync(self.scheduler.entities_deleted)(list(entity_ids))
|
|
48
|
+
|
|
49
|
+
def info(self, target_id):
|
|
50
|
+
return async_to_sync(self.scheduler.info)(target_id)
|
|
51
|
+
|
|
52
|
+
def test_an_unknown_template_has_no_schedule(self):
|
|
53
|
+
self.assertIsNone(self.info(fixture_id()))
|
|
54
|
+
|
|
55
|
+
def test_a_save_applies_the_desired_schedule(self):
|
|
56
|
+
template = self.template(cron='0 2 * * *')
|
|
57
|
+
|
|
58
|
+
self.saved(template)
|
|
59
|
+
|
|
60
|
+
info = self.info(template.entity_id)
|
|
61
|
+
self.assertEqual((info.target_id, info.cron, info.enabled), (template.entity_id, '0 2 * * *', True))
|
|
62
|
+
|
|
63
|
+
def test_held_lists_a_template_schedule_with_its_tick_arguments(self):
|
|
64
|
+
template = self.template(cron='0 2 * * *')
|
|
65
|
+
|
|
66
|
+
self.saved(template)
|
|
67
|
+
|
|
68
|
+
held = [t for t in async_to_sync(self.scheduler.held)() if template.entity_id in t.args]
|
|
69
|
+
self.assertEqual(len(held), 1)
|
|
70
|
+
self.assertEqual((held[0].schedule, held[0].enabled), ('0 2 * * *', True))
|
|
71
|
+
self.assertEqual(held[0].args, (template.entity_id, template.owner))
|
|
72
|
+
|
|
73
|
+
def test_a_disabled_template_is_held_disabled(self):
|
|
74
|
+
template = self.template(enabled=False)
|
|
75
|
+
|
|
76
|
+
self.saved(template)
|
|
77
|
+
|
|
78
|
+
self.assertFalse(self.info(template.entity_id).enabled)
|
|
79
|
+
|
|
80
|
+
def test_changing_cron_or_enabled_updates_the_schedule(self):
|
|
81
|
+
template = self.template(cron='0 2 * * *')
|
|
82
|
+
self.saved(template)
|
|
83
|
+
|
|
84
|
+
paused = self.changed(template, enabled=False)
|
|
85
|
+
self.saved(paused, template)
|
|
86
|
+
moved = self.changed(paused, cron='30 4 * * 1')
|
|
87
|
+
self.saved(moved, paused)
|
|
88
|
+
|
|
89
|
+
info = self.info(template.entity_id)
|
|
90
|
+
self.assertEqual((info.cron, info.enabled), ('30 4 * * 1', False))
|
|
91
|
+
|
|
92
|
+
def test_clearing_cron_removes_the_schedule(self):
|
|
93
|
+
template = self.template()
|
|
94
|
+
self.saved(template)
|
|
95
|
+
|
|
96
|
+
self.saved(self.changed(template, cron=None), template)
|
|
97
|
+
|
|
98
|
+
self.assertIsNone(self.info(template.entity_id))
|
|
99
|
+
|
|
100
|
+
def test_deleting_the_template_removes_the_schedule(self):
|
|
101
|
+
kept, gone = self.template(), self.template()
|
|
102
|
+
self.saved(kept)
|
|
103
|
+
self.saved(gone)
|
|
104
|
+
|
|
105
|
+
self.deleted(gone.entity_id, fixture_id())
|
|
106
|
+
|
|
107
|
+
self.assertIsNone(self.info(gone.entity_id))
|
|
108
|
+
self.assertIsNotNone(self.info(kept.entity_id))
|
|
109
|
+
|
|
110
|
+
def test_other_entities_are_ignored(self):
|
|
111
|
+
from graphcore.domain.entity.Entity import Entity
|
|
112
|
+
|
|
113
|
+
self.saved(Entity(entity_id=fixture_id()))
|
|
114
|
+
self.deleted()
|
|
115
|
+
|
|
116
|
+
def test_convergence_restores_a_lost_schedule_and_removes_an_orphan(self):
|
|
117
|
+
kept, orphan = self.template(), self.template()
|
|
118
|
+
self.saved(kept)
|
|
119
|
+
self.saved(orphan)
|
|
120
|
+
self.forget(kept.entity_id)
|
|
121
|
+
|
|
122
|
+
self.converge([kept])
|
|
123
|
+
|
|
124
|
+
self.assertEqual(self.info(kept.entity_id).cron, kept.cron)
|
|
125
|
+
self.assertIsNone(self.info(orphan.entity_id))
|
|
126
|
+
|
|
127
|
+
def test_convergence_holds_nothing_for_an_unscheduled_template(self):
|
|
128
|
+
template = self.template()
|
|
129
|
+
self.saved(template)
|
|
130
|
+
|
|
131
|
+
self.converge([self.changed(template, cron=None)])
|
|
132
|
+
|
|
133
|
+
self.assertIsNone(self.info(template.entity_id))
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
__all__ = ['SchedulerContract']
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""A `Scheduler` over a dict: the fake for tests that need no Beat."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from typing import Optional
|
|
5
|
+
|
|
6
|
+
from graphcore.domain.entity.request_template.RequestTemplate import RequestTemplate
|
|
7
|
+
from graphcore.scheduling.scheduler import Scheduler
|
|
8
|
+
from graphcore.types.schedules import HeldTask, ScheduleInfo, ScheduleSpec
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class InMemoryScheduler(Scheduler):
|
|
12
|
+
def __init__(self):
|
|
13
|
+
self.specs: dict[str, ScheduleSpec] = {}
|
|
14
|
+
|
|
15
|
+
async def entity_saved(self, entity, previous) -> None:
|
|
16
|
+
if not isinstance(entity, RequestTemplate):
|
|
17
|
+
return
|
|
18
|
+
spec = entity.schedule_spec()
|
|
19
|
+
if spec is None:
|
|
20
|
+
self.specs.pop(entity.entity_id, None)
|
|
21
|
+
else:
|
|
22
|
+
self.specs[entity.entity_id] = spec
|
|
23
|
+
|
|
24
|
+
async def entities_deleted(self, entity_ids: list[str]) -> None:
|
|
25
|
+
for entity_id in entity_ids:
|
|
26
|
+
self.specs.pop(entity_id, None)
|
|
27
|
+
|
|
28
|
+
async def info(self, target_id: str) -> Optional[ScheduleInfo]:
|
|
29
|
+
spec = self.specs.get(target_id)
|
|
30
|
+
if spec is None:
|
|
31
|
+
return None
|
|
32
|
+
return ScheduleInfo(target_id=target_id, cron=spec.cron, enabled=spec.enabled)
|
|
33
|
+
|
|
34
|
+
async def held(self) -> list[HeldTask]:
|
|
35
|
+
return [
|
|
36
|
+
HeldTask(name=spec.target_id, task='tick', schedule=spec.cron, enabled=spec.enabled,
|
|
37
|
+
args=(spec.target_id, spec.principal_id))
|
|
38
|
+
for spec in self.specs.values()
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
def converge(self, templates) -> None:
|
|
42
|
+
self.specs = {spec.target_id: spec for spec in (t.schedule_spec() for t in templates) if spec}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
from asgiref.sync import async_to_sync
|
|
4
|
+
from django.test import TestCase
|
|
5
|
+
from django_celery_beat.models import PeriodicTask
|
|
6
|
+
|
|
7
|
+
from graphcore.scheduling.beat_scheduler import (
|
|
8
|
+
REMOVE_BATCH, RECONCILE_RUNS_TASK, RECONCILE_SCHEDULES_TASK, TICK_TASK, BeatScheduler, task_name,
|
|
9
|
+
)
|
|
10
|
+
from graphcore.scheduling.tests.contract import SchedulerContract
|
|
11
|
+
from graphcore.tests.ids import fixture_id
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class _Templates:
|
|
15
|
+
"""The repository surface `BeatScheduler.reconcile` reads."""
|
|
16
|
+
|
|
17
|
+
def __init__(self):
|
|
18
|
+
self.templates = {}
|
|
19
|
+
|
|
20
|
+
async def list_ids(self, predicate):
|
|
21
|
+
return set(self.templates)
|
|
22
|
+
|
|
23
|
+
async def load_many(self, ids):
|
|
24
|
+
return {i: self.templates[i] for i in ids if i in self.templates}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class BeatSchedulerTests(SchedulerContract, TestCase):
|
|
28
|
+
def make_scheduler(self):
|
|
29
|
+
self.repository = _Templates()
|
|
30
|
+
return BeatScheduler(self.repository, reconcile_interval=60, run_reconcile_interval=30)
|
|
31
|
+
|
|
32
|
+
def converge(self, templates):
|
|
33
|
+
self.repository.templates = {t.entity_id: t for t in templates}
|
|
34
|
+
async_to_sync(self.scheduler.reconcile)()
|
|
35
|
+
|
|
36
|
+
def forget(self, target_id):
|
|
37
|
+
PeriodicTask.objects.filter(name=task_name(target_id)).delete()
|
|
38
|
+
|
|
39
|
+
def test_one_periodic_task_per_template(self):
|
|
40
|
+
template = self.template()
|
|
41
|
+
self.saved(template)
|
|
42
|
+
self.saved(self.changed(template, cron='0 3 * * *'), template)
|
|
43
|
+
|
|
44
|
+
self.assertEqual(PeriodicTask.objects.filter(name__startswith='scheduled_strategy_').count(), 1)
|
|
45
|
+
|
|
46
|
+
def test_a_tick_names_the_template_and_its_owner(self):
|
|
47
|
+
template = self.template()
|
|
48
|
+
|
|
49
|
+
self.saved(template)
|
|
50
|
+
|
|
51
|
+
task = PeriodicTask.objects.get(name=task_name(template.entity_id))
|
|
52
|
+
self.assertEqual(task.task, TICK_TASK)
|
|
53
|
+
self.assertEqual(json.loads(task.args), [template.entity_id, template.owner])
|
|
54
|
+
|
|
55
|
+
def test_deletion_spans_batches(self):
|
|
56
|
+
first, last, kept = self.template(), self.template(), self.template()
|
|
57
|
+
for template in (first, last, kept):
|
|
58
|
+
self.saved(template)
|
|
59
|
+
|
|
60
|
+
self.deleted(first.entity_id, *[fixture_id() for _ in range(REMOVE_BATCH * 2)], last.entity_id)
|
|
61
|
+
|
|
62
|
+
self.assertEqual(list(PeriodicTask.objects.values_list('name', flat=True)), [task_name(kept.entity_id)])
|
|
63
|
+
|
|
64
|
+
def test_convergence_ensures_the_periodic_reconcilers(self):
|
|
65
|
+
self.converge([])
|
|
66
|
+
|
|
67
|
+
tasks = dict(PeriodicTask.objects.values_list('name', 'task'))
|
|
68
|
+
self.assertEqual(tasks['graphcore_reconcile_schedules'], RECONCILE_SCHEDULES_TASK)
|
|
69
|
+
self.assertEqual(tasks['graphcore_reconcile_runs'], RECONCILE_RUNS_TASK)
|
|
70
|
+
self.assertEqual(PeriodicTask.objects.get(name='graphcore_reconcile_runs').interval.every, 30)
|
|
71
|
+
|
|
72
|
+
def test_a_failing_hook_never_raises(self):
|
|
73
|
+
from unittest import mock
|
|
74
|
+
|
|
75
|
+
with mock.patch('graphcore.scheduling.beat_scheduler._write', side_effect=RuntimeError('beat down')):
|
|
76
|
+
self.saved(self.template())
|
|
77
|
+
|
|
78
|
+
def test_info_reports_run_status(self):
|
|
79
|
+
from django.utils import timezone
|
|
80
|
+
|
|
81
|
+
template = self.template()
|
|
82
|
+
self.saved(template)
|
|
83
|
+
ran_at = timezone.now()
|
|
84
|
+
PeriodicTask.objects.filter(name=task_name(template.entity_id)).update(last_run_at=ran_at, total_run_count=3)
|
|
85
|
+
|
|
86
|
+
info = self.info(template.entity_id)
|
|
87
|
+
self.assertEqual((info.last_run_at, info.run_count), (ran_at, 3))
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from django.test import SimpleTestCase
|
|
2
|
+
|
|
3
|
+
from graphcore.scheduling.tests.contract import SchedulerContract
|
|
4
|
+
from graphcore.scheduling.tests.memory import InMemoryScheduler
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class InMemorySchedulerTests(SchedulerContract, SimpleTestCase):
|
|
8
|
+
def make_scheduler(self):
|
|
9
|
+
return InMemoryScheduler()
|
|
10
|
+
|
|
11
|
+
def converge(self, templates):
|
|
12
|
+
self.scheduler.converge(templates)
|
|
13
|
+
|
|
14
|
+
def forget(self, target_id):
|
|
15
|
+
self.scheduler.specs.pop(target_id, None)
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""Templates through the kernel: each schedule change is one write, a tick
|
|
2
|
+
runs a request made from the template as its owner, and deleting the
|
|
3
|
+
target ends the schedule (docs/SCHEDULING_TDD.md)."""
|
|
4
|
+
from asgiref.sync import async_to_sync
|
|
5
|
+
from celery import current_app
|
|
6
|
+
from django_celery_beat.models import PeriodicTask
|
|
7
|
+
|
|
8
|
+
import graphcore.tasks # noqa: F401 registers the kernel's task names
|
|
9
|
+
from graphcore.domain.strategy.BaseStrategy import DeleteEntity
|
|
10
|
+
from graphcore.domain.strategy.RequestTemplateStrategies import (
|
|
11
|
+
CreateRequestTemplate, ManageRequestTemplate,
|
|
12
|
+
)
|
|
13
|
+
from graphcore.errors import InvalidRequest
|
|
14
|
+
from graphcore.execution.tests.support import CpuIncrement, ExecutionTestCase, Increment
|
|
15
|
+
from graphcore.scheduling.beat_scheduler import (
|
|
16
|
+
RECONCILE_RUNS_TASK, RECONCILE_SCHEDULES_TASK, TICK_TASK, task_name,
|
|
17
|
+
)
|
|
18
|
+
from graphcore.scheduling.beat_tasks import scheduled_tick, tick
|
|
19
|
+
from graphcore.tests.runs import run
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class TickTests(ExecutionTestCase):
|
|
23
|
+
async def template(self, strategy=Increment, cron='* * * * *', principal=None, **params):
|
|
24
|
+
request, _ = await run(self.executor, CreateRequestTemplate.request_constructor(
|
|
25
|
+
strategy_name=strategy.__name__, template_target_id=self.probe.entity_id,
|
|
26
|
+
param_config=params, cron=cron,
|
|
27
|
+
), principal or self.alice)
|
|
28
|
+
return request.ret_val['template_id']
|
|
29
|
+
|
|
30
|
+
async def manage(self, template_id, action, principal=None, **changes):
|
|
31
|
+
await run(self.executor, ManageRequestTemplate.request_constructor(template_id, action, **changes),
|
|
32
|
+
principal or self.alice)
|
|
33
|
+
|
|
34
|
+
async def tick(self, template_id):
|
|
35
|
+
template = await self.load(template_id)
|
|
36
|
+
await tick(self.service, self.executor, template_id, template.owner if template else 'nobody')
|
|
37
|
+
await self.settle()
|
|
38
|
+
|
|
39
|
+
async def made_from(self, template_id):
|
|
40
|
+
return [r for r in await self.requests() if r.template_id == template_id]
|
|
41
|
+
|
|
42
|
+
@async_to_sync
|
|
43
|
+
async def test_a_tick_runs_a_request_made_from_the_template_as_its_owner(self):
|
|
44
|
+
await self.seed()
|
|
45
|
+
template_id = await self.template()
|
|
46
|
+
await self.manage(template_id, 'update', param_config={'by': 'update'})
|
|
47
|
+
await self.tick(template_id)
|
|
48
|
+
|
|
49
|
+
[request] = await self.made_from(template_id)
|
|
50
|
+
self.assertEqual((request.status, request.owner), ('succeeded', self.alice.principal_entity_id))
|
|
51
|
+
self.assertEqual(request.param_config, {'by': 'update'})
|
|
52
|
+
self.assertEqual((await self.load(template_id)).owner, self.alice.principal_entity_id)
|
|
53
|
+
self.assertEqual(await self.counter(), 1)
|
|
54
|
+
|
|
55
|
+
@async_to_sync
|
|
56
|
+
async def test_a_tick_is_routed_by_the_strategy_workload(self):
|
|
57
|
+
await self.seed()
|
|
58
|
+
template_id = await self.template(CpuIncrement)
|
|
59
|
+
await self.tick(template_id)
|
|
60
|
+
[request] = await self.made_from(template_id)
|
|
61
|
+
self.assertEqual(request.status, 'succeeded')
|
|
62
|
+
self.assertEqual([a.request_id for a in self.cpu.attempts], [request.entity_id])
|
|
63
|
+
|
|
64
|
+
@async_to_sync
|
|
65
|
+
async def test_a_paused_unscheduled_or_deleted_template_makes_no_request(self):
|
|
66
|
+
await self.seed()
|
|
67
|
+
paused, unscheduled, deleted = await self.template(), await self.template(), await self.template()
|
|
68
|
+
await self.manage(paused, 'pause')
|
|
69
|
+
await self.manage(unscheduled, 'unschedule')
|
|
70
|
+
await run(self.executor, DeleteEntity.request_constructor(target_entity_id=deleted), self.alice)
|
|
71
|
+
for template_id in (paused, unscheduled):
|
|
72
|
+
await self.tick(template_id)
|
|
73
|
+
await tick(self.service, self.executor, deleted, self.alice.principal_entity_id)
|
|
74
|
+
for template_id in (paused, unscheduled, deleted):
|
|
75
|
+
self.assertEqual(await self.made_from(template_id), [])
|
|
76
|
+
|
|
77
|
+
await self.manage(paused, 'resume')
|
|
78
|
+
await self.tick(paused)
|
|
79
|
+
self.assertEqual(len(await self.made_from(paused)), 1)
|
|
80
|
+
|
|
81
|
+
@async_to_sync
|
|
82
|
+
async def test_a_tick_whose_owner_lost_access_fails_and_writes_nothing(self):
|
|
83
|
+
await self.seed()
|
|
84
|
+
shared = await self.make_probe(self.alice)
|
|
85
|
+
await self.runtime.graph_api.grant(self.alice, shared.entity_id, self.bob.principal_entity_id, 'editor')
|
|
86
|
+
self.probe = shared
|
|
87
|
+
template_id = await self.template(principal=self.bob)
|
|
88
|
+
await self.runtime.graph_api.grant(self.alice, shared.entity_id, self.bob.principal_entity_id, 'viewer')
|
|
89
|
+
await self.tick(template_id)
|
|
90
|
+
|
|
91
|
+
self.assertEqual(await self.made_from(template_id), [])
|
|
92
|
+
self.assertIsNone(await self.counter(shared.entity_id))
|
|
93
|
+
|
|
94
|
+
@async_to_sync
|
|
95
|
+
async def test_each_schedule_change_is_one_write_the_scheduler_follows(self):
|
|
96
|
+
await self.seed()
|
|
97
|
+
template_id = await self.template(cron='0 2 * * *')
|
|
98
|
+
held = await self.runtime.scheduler.info(template_id)
|
|
99
|
+
self.assertEqual((held.cron, held.enabled), ('0 2 * * *', True))
|
|
100
|
+
version = (await self.load(template_id)).version
|
|
101
|
+
|
|
102
|
+
await self.manage(template_id, 'update', cron='*/5 * * * *')
|
|
103
|
+
await self.manage(template_id, 'pause')
|
|
104
|
+
self.assertEqual((await self.load(template_id)).version, version + 2)
|
|
105
|
+
held = await self.runtime.scheduler.info(template_id)
|
|
106
|
+
self.assertEqual((held.cron, held.enabled), ('*/5 * * * *', False))
|
|
107
|
+
|
|
108
|
+
await self.manage(template_id, 'unschedule')
|
|
109
|
+
self.assertIsNone(await self.runtime.scheduler.info(template_id))
|
|
110
|
+
self.assertIsNotNone(await self.load(template_id))
|
|
111
|
+
|
|
112
|
+
@async_to_sync
|
|
113
|
+
async def test_an_invalid_schedule_is_refused_before_any_write(self):
|
|
114
|
+
await self.seed()
|
|
115
|
+
for kwargs in ({'cron': 'every minute'}, {'strategy': type('Unregistered', (Increment,), {})}):
|
|
116
|
+
with self.assertRaises(InvalidRequest):
|
|
117
|
+
await self.template(**kwargs)
|
|
118
|
+
template_id = await self.template()
|
|
119
|
+
with self.assertRaises(InvalidRequest):
|
|
120
|
+
await self.manage(template_id, 'update', cron='61 * * * *')
|
|
121
|
+
self.assertEqual((await self.load(template_id)).cron, '* * * * *')
|
|
122
|
+
|
|
123
|
+
@async_to_sync
|
|
124
|
+
async def test_deleting_the_target_deletes_the_template_and_its_schedule(self):
|
|
125
|
+
await self.seed()
|
|
126
|
+
template_id = await self.template()
|
|
127
|
+
self.assertTrue(await PeriodicTask.objects.filter(name=task_name(template_id)).aexists())
|
|
128
|
+
await run(self.executor, DeleteEntity.request_constructor(target_entity_id=self.probe.entity_id),
|
|
129
|
+
self.alice)
|
|
130
|
+
self.assertIsNone(await self.load(template_id))
|
|
131
|
+
self.assertFalse(await PeriodicTask.objects.filter(name=task_name(template_id)).aexists())
|
|
132
|
+
|
|
133
|
+
def test_the_tick_task_runs_through_the_process_runtime(self):
|
|
134
|
+
async_to_sync(self.seed)()
|
|
135
|
+
template_id = async_to_sync(self.template)()
|
|
136
|
+
scheduled_tick.delay(template_id, self.alice.principal_entity_id)
|
|
137
|
+
[request] = async_to_sync(self.made_from)(template_id)
|
|
138
|
+
self.assertEqual(request.status, 'succeeded')
|
|
139
|
+
|
|
140
|
+
def test_beat_enqueues_only_registered_task_names(self):
|
|
141
|
+
registered = set(current_app.tasks)
|
|
142
|
+
for name in (TICK_TASK, RECONCILE_SCHEDULES_TASK, RECONCILE_RUNS_TASK, 'graphcore.tasks.run_attempt'):
|
|
143
|
+
self.assertIn(name, registered)
|
graphcore/tasks.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""Celery discovers the kernel's tasks through this module. Every task is
|
|
2
|
+
named explicitly `graphcore.tasks.*`, so the name Beat or a dispatcher
|
|
3
|
+
enqueues never depends on where the task is defined."""
|
|
4
|
+
|
|
5
|
+
from graphcore.execution.environments.celery import run_attempt # noqa: F401
|
|
6
|
+
from graphcore.execution.tasks import reconcile_runs # noqa: F401
|
|
7
|
+
from graphcore.scheduling.beat_tasks import reconcile_schedules, scheduled_tick # noqa: F401
|
|
8
|
+
|
|
9
|
+
__all__ = ['reconcile_runs', 'reconcile_schedules', 'run_attempt', 'scheduled_tick']
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Standalone settings for running the framework's own test suite.
|
|
2
|
+
|
|
3
|
+
`GRAPHCORE_TEST_DB` picks the database: `postgresql` (the default and the
|
|
4
|
+
reference backend) or `sqlite`. Postgres connects with the same `POSTGRES_*`
|
|
5
|
+
variables and defaults as a generated project, so `graphcore infra up` is
|
|
6
|
+
all it needs. Contributors run the suite through `scripts/test`.
|
|
7
|
+
"""
|
|
8
|
+
import os
|
|
9
|
+
|
|
10
|
+
from django.core.exceptions import ImproperlyConfigured
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def database_settings(name: str) -> dict:
|
|
14
|
+
if name == 'postgresql':
|
|
15
|
+
return {
|
|
16
|
+
'ENGINE': 'django.db.backends.postgresql',
|
|
17
|
+
'NAME': 'graphcore_framework',
|
|
18
|
+
'USER': os.environ.get('POSTGRES_USER', 'postgres'),
|
|
19
|
+
'PASSWORD': os.environ.get('POSTGRES_PASSWORD', 'postgres'),
|
|
20
|
+
'HOST': os.environ.get('POSTGRES_HOST', 'localhost'),
|
|
21
|
+
'PORT': os.environ.get('POSTGRES_PORT', '5432'),
|
|
22
|
+
'TEST': {'NAME': 'test_graphcore_framework'},
|
|
23
|
+
}
|
|
24
|
+
if name == 'sqlite':
|
|
25
|
+
# In memory: Django's --parallel can't reopen a file-based SQLite
|
|
26
|
+
# test database under the spawn start method (the macOS default).
|
|
27
|
+
return {'ENGINE': 'django.db.backends.sqlite3', 'NAME': ':memory:'}
|
|
28
|
+
raise ImproperlyConfigured(
|
|
29
|
+
f"GRAPHCORE_TEST_DB={name!r} is not a test database; use 'postgresql' or 'sqlite'."
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
SECRET_KEY = 'graphcore-test-secret-key'
|
|
34
|
+
DEBUG = True
|
|
35
|
+
USE_TZ = True
|
|
36
|
+
|
|
37
|
+
INSTALLED_APPS = [
|
|
38
|
+
'django.contrib.contenttypes',
|
|
39
|
+
'django.contrib.auth',
|
|
40
|
+
# The kernel's models use GinIndex.
|
|
41
|
+
'django.contrib.postgres',
|
|
42
|
+
'django_celery_beat',
|
|
43
|
+
'graphcore',
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
DATABASES = {'default': database_settings(os.environ.get('GRAPHCORE_TEST_DB', 'postgresql'))}
|
|
47
|
+
|
|
48
|
+
CACHES = {
|
|
49
|
+
'default': {
|
|
50
|
+
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
GRAPHCORE_ENTITY_CACHE_TTL = 900
|
|
55
|
+
GRAPHCORE_MAX_BATCH_SIZE = 100
|
|
56
|
+
|
|
57
|
+
CHANNEL_LAYERS = {
|
|
58
|
+
'default': {
|
|
59
|
+
'BACKEND': 'channels.layers.InMemoryChannelLayer',
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
|
|
64
|
+
|
|
65
|
+
AUTH_PASSWORD_VALIDATORS = [
|
|
66
|
+
{'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator'},
|
|
67
|
+
{'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator'},
|
|
68
|
+
{'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator'},
|
|
69
|
+
{'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator'},
|
|
70
|
+
]
|
|
71
|
+
|
|
72
|
+
# Run tasks inline against an in-memory broker so the suite needs no Redis.
|
|
73
|
+
CELERY_BROKER_URL = 'memory://'
|
|
74
|
+
CELERY_TASK_ALWAYS_EAGER = True
|
|
75
|
+
CELERY_TASK_EAGER_PROPAGATES = True
|
|
76
|
+
CELERY_ACCEPT_CONTENT = ['json']
|
|
77
|
+
CELERY_TASK_SERIALIZER = 'json'
|
|
78
|
+
CELERY_RESULT_SERIALIZER = 'json'
|
|
File without changes
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""`GraphCoreTestCase`: a live runtime per test, for app authors (exported
|
|
2
|
+
by `graphcore.public.testing`)."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Optional
|
|
7
|
+
|
|
8
|
+
from celery.signals import task_postrun
|
|
9
|
+
from django.conf import settings
|
|
10
|
+
from django.db import close_old_connections
|
|
11
|
+
from django.test import TransactionTestCase
|
|
12
|
+
|
|
13
|
+
from graphcore.agent_frontend import testing as agent_testing
|
|
14
|
+
from graphcore.agent_frontend.render import Frontend
|
|
15
|
+
from graphcore.agent_frontend.testing import Place
|
|
16
|
+
from graphcore.auth.authn.principals import create_principal
|
|
17
|
+
from graphcore.auth.authz.lookups import find_group_by_key
|
|
18
|
+
from graphcore.auth.authz.kernel_writes import add_member
|
|
19
|
+
from graphcore.auth.authz.defaults import ADMINS_GROUP_KEY
|
|
20
|
+
from graphcore.execution.strategy_context import kernel_internal_context
|
|
21
|
+
from graphcore.registry.app_discovery import discover_agent_frontends
|
|
22
|
+
from graphcore.runtime import configured_runtime
|
|
23
|
+
from graphcore.transport.agent_client import InProcessGraphClient
|
|
24
|
+
from graphcore.types.identity import Principal
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@task_postrun.connect(weak=False, dispatch_uid='graphcore.testing.close_task_connections')
|
|
28
|
+
def _close_task_connections(**kwargs):
|
|
29
|
+
# A worker's Django fixup does this; an eager task's pool thread would
|
|
30
|
+
# otherwise hold its connection past the test database's teardown.
|
|
31
|
+
close_old_connections()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class GraphCoreTestCase(TransactionTestCase):
|
|
35
|
+
"""A test case with a live GraphCore runtime (`self.api`), built fresh
|
|
36
|
+
for each test from the project's `GRAPHCORE_RUNTIME_FACTORY`; test
|
|
37
|
+
methods may be `async def`. `create_user` and `create_entity` are
|
|
38
|
+
fixtures and skip authorization; `run_strategy` goes through the same
|
|
39
|
+
checks as a client request."""
|
|
40
|
+
|
|
41
|
+
def setUp(self):
|
|
42
|
+
super().setUp()
|
|
43
|
+
self.runtime = configured_runtime()
|
|
44
|
+
self.api = self.runtime.graph_api
|
|
45
|
+
|
|
46
|
+
async def create_user(self, username: str) -> Principal:
|
|
47
|
+
"""A new user, returned as the principal to act as."""
|
|
48
|
+
user = await create_principal(self.runtime.entity_service, self.runtime.executor, username=username)
|
|
49
|
+
return Principal(principal_entity_id=user.entity_id)
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def deployment_entity_id(self) -> str:
|
|
53
|
+
"""The target for creating top-level entities."""
|
|
54
|
+
return self.runtime.deployment_entity_id
|
|
55
|
+
|
|
56
|
+
async def make_admin(self, principal: Principal) -> None:
|
|
57
|
+
"""Add `principal` to `graphcore.admins`, as `graphcore createadmin` does."""
|
|
58
|
+
admins = await find_group_by_key(self.runtime.entity_service, ADMINS_GROUP_KEY)
|
|
59
|
+
with kernel_internal_context():
|
|
60
|
+
await add_member(self.runtime.entity_service, admins, principal.principal_entity_id, None)
|
|
61
|
+
|
|
62
|
+
async def create_entity(self, entity_cls, owner: Principal):
|
|
63
|
+
"""A top-level `entity_cls` owned by `owner`: the starting point of
|
|
64
|
+
a test."""
|
|
65
|
+
entity = self.runtime.entity_service.create(entity_cls)
|
|
66
|
+
entity.owner = owner.principal_entity_id
|
|
67
|
+
with kernel_internal_context():
|
|
68
|
+
return await self.runtime.entity_service.save_entity(entity, owner)
|
|
69
|
+
|
|
70
|
+
async def run_strategy(self, principal: Principal, request):
|
|
71
|
+
"""Run the `StrategyRequestPayload` as `principal` and return its
|
|
72
|
+
succeeded request (results are in its `ret_val`). Raises `Forbidden`
|
|
73
|
+
when the principal may not run it, and the run's own error when it
|
|
74
|
+
fails (`StrategyFailed` for an unexpected one)."""
|
|
75
|
+
completed, _ = await self.runtime.executor.submit(request, principal)
|
|
76
|
+
return completed
|
|
77
|
+
|
|
78
|
+
async def get_entity(self, principal: Principal, entity_id: str):
|
|
79
|
+
"""`entity_id` as `principal` sees it: the entity, or a tombstone
|
|
80
|
+
when they can't read it."""
|
|
81
|
+
return await self.api.get_entity(principal, entity_id)
|
|
82
|
+
|
|
83
|
+
async def grant(self, granter: Principal, resource_id: str, grantee: Principal, role_name: str) -> None:
|
|
84
|
+
"""Grant `role_name` on `resource_id` to `grantee`, as `granter`."""
|
|
85
|
+
await self.api.grant(granter, resource_id, grantee.principal_entity_id, role_name)
|
|
86
|
+
|
|
87
|
+
def agent_frontends(self) -> list[str]:
|
|
88
|
+
"""Module names of the agent frontends the backend serves: installed
|
|
89
|
+
apps' entry points and the project's
|
|
90
|
+
`apps/<name>/backend/<name>_agent_frontend` packages."""
|
|
91
|
+
return discover_agent_frontends(Path(settings.BASE_DIR).resolve().parent / 'apps')
|
|
92
|
+
|
|
93
|
+
def _agent_frontend(self) -> Frontend:
|
|
94
|
+
return Frontend(
|
|
95
|
+
InProcessGraphClient(self.runtime.graph_api),
|
|
96
|
+
agent_testing.cached_registry(tuple(self.agent_frontends())),
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
async def open_place(self, principal: Principal, at: str) -> Place:
|
|
100
|
+
"""The place at address or entity id `at`, rendered for `principal`.
|
|
101
|
+
Raises `AgentError` as `/mcp` reports it: `NotFound` when it is unreadable."""
|
|
102
|
+
return await agent_testing.open_place(self._agent_frontend(), principal, at)
|
|
103
|
+
|
|
104
|
+
async def call_tool(
|
|
105
|
+
self, principal: Principal, at: str, tool: str, params: Optional[dict] = None,
|
|
106
|
+
seen_version: Optional[str] = None,
|
|
107
|
+
) -> dict:
|
|
108
|
+
"""Call `tool` at `at` as `principal`; the result holds `ok`, `tool`,
|
|
109
|
+
`result`, `address`, `version` and `note`. Raises `AgentError` as `/mcp`
|
|
110
|
+
reports it: `Refused` for a refused run, `NotFound`, `Stale` when
|
|
111
|
+
`seen_version` is outdated, `UnknownTool`, `InvalidRequest` for bad params."""
|
|
112
|
+
return await agent_testing.call_tool(self._agent_frontend(), principal, at, tool, params, seen_version)
|