intersect-sdk 0.7.0__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/PKG-INFO +5 -1
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/pyproject.toml +12 -2
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/__init__.py +4 -2
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/control_plane/control_plane_manager.py +1 -1
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/interfaces.py +2 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/schema.py +57 -39
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/capability/base.py +19 -17
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/client.py +0 -45
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/client_callback_definitions.py +3 -19
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/config/shared.py +1 -0
- intersect_sdk-0.8.0/src/intersect_sdk/schema.py +70 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/service.py +119 -68
- intersect_sdk-0.8.0/src/intersect_sdk/service_callback_definitions.py +24 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/version.py +1 -1
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/tests/e2e/test_examples.py +7 -3
- intersect_sdk-0.8.0/tests/fixtures/example_schema.json +1067 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/tests/fixtures/example_schema.py +2 -1
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/tests/integration/test_return_type_mismatch.py +2 -1
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/tests/unit/test_base_capability_implementation.py +10 -2
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/tests/unit/test_invalid_schema_runtime.py +3 -1
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/tests/unit/test_schema_invalids.py +235 -52
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/tests/unit/test_schema_valid.py +8 -10
- intersect_sdk-0.7.0/src/intersect_sdk/schema.py +0 -78
- intersect_sdk-0.7.0/src/intersect_sdk/service_callback_definitions.py +0 -16
- intersect_sdk-0.7.0/tests/fixtures/example_schema.json +0 -1062
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/LICENSE +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/README.md +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/__init__.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/constants.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/control_plane/__init__.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/control_plane/brokers/__init__.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/control_plane/brokers/amqp_client.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/control_plane/brokers/broker_client.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/control_plane/brokers/mqtt_client.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/control_plane/discovery_service.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/control_plane/topic_handler.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/data_plane/__init__.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/data_plane/data_plane_manager.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/data_plane/minio_utils.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/event_metadata.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/exceptions.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/function_metadata.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/logger.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/messages/__init__.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/messages/event.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/messages/lifecycle.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/messages/userspace.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/multi_flag_thread_event.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/pydantic_schema_generator.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/stoppable_thread.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/utils.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/version.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/_internal/version_resolver.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/app_lifecycle.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/capability/__init__.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/config/__init__.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/config/client.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/config/service.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/constants.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/core_definitions.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/py.typed +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/service_definitions.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/shared_callback_definitions.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/tests/__init__.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/tests/conftest.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/tests/e2e/__init__.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/tests/fixtures/__init__.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/tests/fixtures/return_type_mismatch.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/tests/integration/__init__.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/tests/integration/test_service.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/tests/unit/__init__.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/tests/unit/test_annotations.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/tests/unit/test_config.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/tests/unit/test_lifecycle_message.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/tests/unit/test_userspace_message.py +0 -0
- {intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/tests/unit/test_version_resolver.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: intersect-sdk
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: Python SDK to interact with INTERSECT
|
|
5
5
|
Keywords: intersect
|
|
6
6
|
Author-Email: Lance Drane <dranelt@ornl.gov>, Marshall McDonnell <mcdonnellmt@ornl.gov>, Seth Hitefield <hitefieldsd@ornl.gov>, Andrew Ayres <ayresaf@ornl.gov>, Gregory Cage <cagege@ornl.gov>, Jesse McGaha <mcgahajr@ornl.gov>, Robert Smith <smithrw@ornl.gov>, Gavin Wiggins <wigginsg@ornl.gov>, Michael Brim <brimmj@ornl.gov>, Rick Archibald <archibaldrk@ornl.gov>, Addi Malviya Thakur <malviyaa@ornl.gov>
|
|
@@ -10,6 +10,10 @@ Classifier: Programming Language :: Python :: 3.8
|
|
|
10
10
|
Classifier: Programming Language :: Python :: 3.9
|
|
11
11
|
Classifier: Programming Language :: Python :: 3.10
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Project-URL: Homepage, https://github.com/INTERSECT-SDK/python-sdk/
|
|
14
|
+
Project-URL: Changelog, https://github.com/INTERSECT-SDK/python-sdk/blob/main/CHANGELOG.md
|
|
15
|
+
Project-URL: Documentation, https://intersect-python-sdk.readthedocs.io/en/latest/
|
|
16
|
+
Project-URL: Issues, https://github.com/INTERSECT-SDK/python-sdk/issues
|
|
13
17
|
Requires-Python: <4.0,>=3.8.10
|
|
14
18
|
Requires-Dist: pydantic>=2.7.0
|
|
15
19
|
Requires-Dist: retrying<2.0.0,>=1.3.4
|
|
@@ -35,11 +35,17 @@ dependencies = [
|
|
|
35
35
|
"jsonschema[format-nongpl]>=4.21.1",
|
|
36
36
|
"eval-type-backport>=0.1.3;python_version<'3.10'",
|
|
37
37
|
]
|
|
38
|
-
version = "0.
|
|
38
|
+
version = "0.8.0"
|
|
39
39
|
|
|
40
40
|
[project.license]
|
|
41
41
|
text = "BSD-3-Clause"
|
|
42
42
|
|
|
43
|
+
[project.urls]
|
|
44
|
+
Homepage = "https://github.com/INTERSECT-SDK/python-sdk/"
|
|
45
|
+
Changelog = "https://github.com/INTERSECT-SDK/python-sdk/blob/main/CHANGELOG.md"
|
|
46
|
+
Documentation = "https://intersect-python-sdk.readthedocs.io/en/latest/"
|
|
47
|
+
Issues = "https://github.com/INTERSECT-SDK/python-sdk/issues"
|
|
48
|
+
|
|
43
49
|
[project.optional-dependencies]
|
|
44
50
|
amqp = [
|
|
45
51
|
"pika>=1.3.2,<2.0.0",
|
|
@@ -52,7 +58,7 @@ docs = [
|
|
|
52
58
|
[tool.pdm.dev-dependencies]
|
|
53
59
|
lint = [
|
|
54
60
|
"pre-commit>=3.3.1",
|
|
55
|
-
"ruff
|
|
61
|
+
"ruff==0.5.7",
|
|
56
62
|
"mypy>=1.10.0",
|
|
57
63
|
"types-paho-mqtt>=1.6.0.20240106",
|
|
58
64
|
]
|
|
@@ -214,11 +220,15 @@ addopts = "-ra"
|
|
|
214
220
|
[tool.coverage.report]
|
|
215
221
|
omit = [
|
|
216
222
|
"*__init__*",
|
|
223
|
+
"*/discovery_service.py",
|
|
217
224
|
]
|
|
218
225
|
exclude_also = [
|
|
219
226
|
"pragma: no-cover",
|
|
220
227
|
"if (typing\\\\.)?TYPE_CHECKING:",
|
|
221
228
|
"@(abc\\\\.)?abstractmethod",
|
|
229
|
+
"class .*\\bProtocol\\):",
|
|
230
|
+
"raise NotImplementedError",
|
|
231
|
+
"except.* ImportError",
|
|
222
232
|
]
|
|
223
233
|
|
|
224
234
|
[build-system]
|
|
@@ -19,12 +19,13 @@ from .config.client import IntersectClientConfig
|
|
|
19
19
|
from .config.service import IntersectServiceConfig
|
|
20
20
|
from .config.shared import (
|
|
21
21
|
ControlPlaneConfig,
|
|
22
|
+
ControlProvider,
|
|
22
23
|
DataStoreConfig,
|
|
23
24
|
DataStoreConfigMap,
|
|
24
25
|
HierarchyConfig,
|
|
25
26
|
)
|
|
26
27
|
from .core_definitions import IntersectDataHandler, IntersectMimeType
|
|
27
|
-
from .schema import
|
|
28
|
+
from .schema import get_schema_from_capability_implementations
|
|
28
29
|
from .service import IntersectService
|
|
29
30
|
from .service_callback_definitions import (
|
|
30
31
|
INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE,
|
|
@@ -48,7 +49,7 @@ __all__ = [
|
|
|
48
49
|
'intersect_event',
|
|
49
50
|
'intersect_message',
|
|
50
51
|
'intersect_status',
|
|
51
|
-
'
|
|
52
|
+
'get_schema_from_capability_implementations',
|
|
52
53
|
'IntersectService',
|
|
53
54
|
'IntersectClient',
|
|
54
55
|
'IntersectClientCallback',
|
|
@@ -63,6 +64,7 @@ __all__ = [
|
|
|
63
64
|
'IntersectServiceConfig',
|
|
64
65
|
'HierarchyConfig',
|
|
65
66
|
'ControlPlaneConfig',
|
|
67
|
+
'ControlProvider',
|
|
66
68
|
'DataStoreConfig',
|
|
67
69
|
'DataStoreConfigMap',
|
|
68
70
|
'__version__',
|
|
@@ -66,7 +66,7 @@ class ControlPlaneManager:
|
|
|
66
66
|
"""
|
|
67
67
|
if control_configs == 'discovery':
|
|
68
68
|
msg = 'Discovery service not implemented yet'
|
|
69
|
-
raise
|
|
69
|
+
raise NotImplementedError(msg)
|
|
70
70
|
self._control_providers = [
|
|
71
71
|
create_control_provider(config, self.get_subscription_channels)
|
|
72
72
|
for config in control_configs
|
|
@@ -36,12 +36,14 @@ class IntersectEventObserver(ABC):
|
|
|
36
36
|
self,
|
|
37
37
|
request: IntersectDirectMessageParams,
|
|
38
38
|
response_handler: INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None = None,
|
|
39
|
+
timeout: float = 300.0,
|
|
39
40
|
) -> UUID:
|
|
40
41
|
"""Observed entity (capabilitiy) tells observer (i.e. service) to send an external request.
|
|
41
42
|
|
|
42
43
|
Params:
|
|
43
44
|
- request: the request we want to send out, encapsulated as an IntersectClientMessageParams object
|
|
44
45
|
- response_handler: optional callback for how we want to handle the response from this request.
|
|
46
|
+
- timeout: optional value for how long we should wait on the request, in seconds (default: 300 seconds)
|
|
45
47
|
|
|
46
48
|
Returns:
|
|
47
49
|
- generated RequestID associated with your request
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import inspect
|
|
6
|
+
import re
|
|
6
7
|
from enum import Enum
|
|
7
8
|
from typing import (
|
|
8
9
|
TYPE_CHECKING,
|
|
@@ -55,6 +56,9 @@ For networking types, https://docs.pydantic.dev/latest/api/networks/
|
|
|
55
56
|
For a complete reference, https://docs.pydantic.dev/latest/concepts/conversion_table
|
|
56
57
|
"""
|
|
57
58
|
|
|
59
|
+
CAPABILITY_NAME_PATTERN = r'[\w-]+'
|
|
60
|
+
"""Regular expression we use to check valid capability names. Since capability namespacing only occurs in services, we can be more lax than for how we name services/systems/etc. """
|
|
61
|
+
|
|
58
62
|
|
|
59
63
|
class _FunctionAnalysisResult(NamedTuple):
|
|
60
64
|
"""private class generated from static analysis of function."""
|
|
@@ -302,7 +306,7 @@ def _add_events(
|
|
|
302
306
|
|
|
303
307
|
|
|
304
308
|
def _introspection_baseline(
|
|
305
|
-
capability: IntersectBaseCapabilityImplementation,
|
|
309
|
+
capability: type[IntersectBaseCapabilityImplementation],
|
|
306
310
|
excluded_data_handlers: set[IntersectDataHandler],
|
|
307
311
|
) -> tuple[
|
|
308
312
|
dict[Any, Any], # $defs for schemas (common)
|
|
@@ -332,8 +336,9 @@ def _introspection_baseline(
|
|
|
332
336
|
function_map = {}
|
|
333
337
|
event_metadatas: dict[str, EventMetadata] = {}
|
|
334
338
|
|
|
335
|
-
|
|
336
|
-
|
|
339
|
+
# capability_name should have already been checked before calling this function
|
|
340
|
+
cap_name = capability.intersect_sdk_capability_name
|
|
341
|
+
status_func, response_funcs, event_funcs = _get_functions(capability)
|
|
337
342
|
|
|
338
343
|
# parse functions
|
|
339
344
|
for class_name, name, method, min_params in response_funcs:
|
|
@@ -439,7 +444,7 @@ def _introspection_baseline(
|
|
|
439
444
|
|
|
440
445
|
# final function mapping
|
|
441
446
|
function_map[public_name] = FunctionMetadata(
|
|
442
|
-
|
|
447
|
+
capability,
|
|
443
448
|
method,
|
|
444
449
|
function_cache_request_adapter,
|
|
445
450
|
function_cache_response_adapter,
|
|
@@ -477,7 +482,7 @@ def _introspection_baseline(
|
|
|
477
482
|
if status_fn_type_adapter and status_fn and status_fn_name:
|
|
478
483
|
public_status_name = f'{cap_name}.{status_fn_name}'
|
|
479
484
|
function_map[public_status_name] = FunctionMetadata(
|
|
480
|
-
|
|
485
|
+
capability,
|
|
481
486
|
status_fn,
|
|
482
487
|
None,
|
|
483
488
|
status_fn_type_adapter,
|
|
@@ -494,52 +499,78 @@ def _introspection_baseline(
|
|
|
494
499
|
|
|
495
500
|
|
|
496
501
|
def get_schema_and_functions_from_capability_implementations(
|
|
497
|
-
capabilities: list[IntersectBaseCapabilityImplementation],
|
|
502
|
+
capabilities: list[type[IntersectBaseCapabilityImplementation]],
|
|
498
503
|
service_name: HierarchyConfig,
|
|
499
504
|
excluded_data_handlers: set[IntersectDataHandler],
|
|
500
505
|
) -> tuple[
|
|
501
506
|
dict[str, Any],
|
|
502
507
|
dict[str, FunctionMetadata],
|
|
503
508
|
dict[str, EventMetadata],
|
|
504
|
-
IntersectBaseCapabilityImplementation | None,
|
|
509
|
+
type[IntersectBaseCapabilityImplementation] | None,
|
|
505
510
|
str | None,
|
|
506
511
|
TypeAdapter[Any] | None,
|
|
507
512
|
]:
|
|
508
513
|
"""This function generates the core AsyncAPI schema, and the core mappings which are derived from the schema.
|
|
509
514
|
|
|
515
|
+
Importantly, this function needs to be able to work with static classes, and not instances. This is because users
|
|
516
|
+
should be free to define their constructor as they wish, with any arbitrary parameters. Users should also be allowed
|
|
517
|
+
to execute whatever code they'd like in their constructor, such as establishing remote connections. At the same time,
|
|
518
|
+
we want to allow users to quickly generate an INTERSECT schema, without having to worry about any dependencies from their constructor code.
|
|
519
|
+
|
|
510
520
|
In-depth introspection is handled later on.
|
|
511
521
|
"""
|
|
512
|
-
|
|
513
|
-
status_function_cap: IntersectBaseCapabilityImplementation | None = None
|
|
522
|
+
status_function_cap: type[IntersectBaseCapabilityImplementation] | None = None
|
|
514
523
|
status_function_name: str | None = None
|
|
515
524
|
status_function_schema: dict[str, Any] | None = None
|
|
516
525
|
status_function_adapter: TypeAdapter[Any] | None = None
|
|
517
|
-
|
|
518
|
-
|
|
526
|
+
shared_schemas: dict[Any, Any] = {} # "shared" schemas which get put in $defs
|
|
527
|
+
capability_schemas: dict[str, Any] = {} # endpoint schemas
|
|
519
528
|
function_map: dict[str, FunctionMetadata] = {} # endpoint functionality
|
|
520
|
-
events: dict[
|
|
529
|
+
events: dict[
|
|
530
|
+
str, Any
|
|
531
|
+
] = {} # event schemas - TODO event names are currently "global" across capabilities, may want to change this?
|
|
521
532
|
event_map: dict[str, EventMetadata] = {} # event functionality
|
|
522
|
-
for
|
|
523
|
-
|
|
524
|
-
if
|
|
525
|
-
|
|
533
|
+
for capability_type in capabilities:
|
|
534
|
+
cap_name = capability_type.intersect_sdk_capability_name
|
|
535
|
+
if (
|
|
536
|
+
not cap_name
|
|
537
|
+
or not isinstance(cap_name, str)
|
|
538
|
+
or not re.fullmatch(CAPABILITY_NAME_PATTERN, cap_name)
|
|
539
|
+
):
|
|
540
|
+
die(
|
|
541
|
+
f'Invalid intersect_sdk_capability_name on capability {capability_type.__name__} - must be a non-empty string with only alphanumeric characters and hyphens (you must explicitly set this, and do so on the class and not an instance).'
|
|
542
|
+
)
|
|
543
|
+
if cap_name in capability_schemas:
|
|
544
|
+
die(
|
|
545
|
+
f'Invalid intersect_sdk_capability_name on capability {capability_type.__name__} - value "{cap_name}" is a duplicate and has already appeared in another capability.'
|
|
546
|
+
)
|
|
547
|
+
|
|
526
548
|
(
|
|
527
|
-
|
|
549
|
+
subschemas,
|
|
528
550
|
(cap_status_fn_name, cap_status_schema, cap_status_type_adapter),
|
|
529
|
-
|
|
551
|
+
cap_functions,
|
|
530
552
|
cap_function_map,
|
|
531
553
|
cap_events,
|
|
532
554
|
cap_event_map,
|
|
533
|
-
) = _introspection_baseline(
|
|
555
|
+
) = _introspection_baseline(capability_type, excluded_data_handlers)
|
|
534
556
|
|
|
535
557
|
if cap_status_fn_name and cap_status_schema and cap_status_type_adapter:
|
|
536
|
-
|
|
558
|
+
if status_function_name is not None:
|
|
559
|
+
# TODO may want to change this later
|
|
560
|
+
die('Only one capabilitiy may have an @intersect_status function')
|
|
561
|
+
status_function_cap = capability_type
|
|
537
562
|
status_function_name = cap_status_fn_name
|
|
538
563
|
status_function_schema = cap_status_schema
|
|
539
564
|
status_function_adapter = cap_status_type_adapter
|
|
540
565
|
|
|
541
|
-
|
|
542
|
-
|
|
566
|
+
shared_schemas.update(subschemas)
|
|
567
|
+
# NOTE: we will still add the capability to the schema, even if there are no @intersect_message annotations
|
|
568
|
+
capability_schemas[cap_name] = {
|
|
569
|
+
'channels': cap_functions,
|
|
570
|
+
}
|
|
571
|
+
# add documentation for the capabilities
|
|
572
|
+
if capability_type.__doc__:
|
|
573
|
+
capability_schemas[cap_name]['description'] = inspect.cleandoc(capability_type.__doc__)
|
|
543
574
|
function_map.update(cap_function_map)
|
|
544
575
|
events.update(cap_events)
|
|
545
576
|
event_map.update(cap_event_map)
|
|
@@ -549,15 +580,17 @@ def get_schema_and_functions_from_capability_implementations(
|
|
|
549
580
|
'x-intersect-version': version_string,
|
|
550
581
|
'info': {
|
|
551
582
|
'title': service_name.hierarchy_string('.'),
|
|
583
|
+
'description': 'INTERSECT schema',
|
|
552
584
|
'version': '0.0.0', # NOTE: this will be modified by INTERSECT CORE, users do not manage their schema versions
|
|
553
585
|
},
|
|
554
586
|
# applies to how an incoming message payload will be parsed.
|
|
555
587
|
# can be changed per channel
|
|
556
588
|
'defaultContentType': 'application/json',
|
|
557
|
-
'
|
|
589
|
+
'capabilities': capability_schemas,
|
|
558
590
|
'events': events,
|
|
591
|
+
'status': status_function_schema if status_function_schema else {'type': 'null'},
|
|
559
592
|
'components': {
|
|
560
|
-
'schemas':
|
|
593
|
+
'schemas': shared_schemas,
|
|
561
594
|
'messageTraits': {
|
|
562
595
|
# this is where we can define our message headers
|
|
563
596
|
'commonHeaders': {
|
|
@@ -574,21 +607,6 @@ def get_schema_and_functions_from_capability_implementations(
|
|
|
574
607
|
},
|
|
575
608
|
}
|
|
576
609
|
|
|
577
|
-
if capability_type_docs != '':
|
|
578
|
-
asyncapi_spec['info']['description'] = capability_type_docs # type: ignore[index]
|
|
579
|
-
|
|
580
|
-
if status_function_schema:
|
|
581
|
-
asyncapi_spec['status'] = status_function_schema
|
|
582
|
-
|
|
583
|
-
"""
|
|
584
|
-
TODO - might want to include these fields
|
|
585
|
-
"securitySchemes": {},
|
|
586
|
-
"operationTraits": {},
|
|
587
|
-
"externalDocumentation": {
|
|
588
|
-
"url": "https://example.com", # REQUIRED
|
|
589
|
-
},
|
|
590
|
-
"""
|
|
591
|
-
|
|
592
610
|
return (
|
|
593
611
|
asyncapi_spec,
|
|
594
612
|
function_map,
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import inspect
|
|
6
|
-
from typing import TYPE_CHECKING, Any
|
|
6
|
+
from typing import TYPE_CHECKING, Any, ClassVar
|
|
7
7
|
|
|
8
8
|
from typing_extensions import final
|
|
9
9
|
|
|
@@ -29,16 +29,23 @@ class IntersectBaseCapabilityImplementation:
|
|
|
29
29
|
you MUST call `super.__init__()` .
|
|
30
30
|
"""
|
|
31
31
|
|
|
32
|
+
intersect_sdk_capability_name: ClassVar[str] = ''
|
|
33
|
+
"""The advertised name of your capability, as provided by the extension of this class.
|
|
34
|
+
|
|
35
|
+
You MUST override this value per class and set it as a string - it's ideal to do so on the class itself for static analysis purposes,
|
|
36
|
+
though as long as this variable has been set before the capability is added to the Service,
|
|
37
|
+
everything should work fine.
|
|
38
|
+
|
|
39
|
+
Each capability within a Service MUST have a unique capability name.
|
|
40
|
+
This value should not be modified once the capability has been added to the Service.
|
|
41
|
+
This value should ONLY contain alphanumeric characters, hyphens, and underscores.
|
|
42
|
+
"""
|
|
43
|
+
|
|
32
44
|
def __init__(self) -> None:
|
|
33
45
|
"""This constructor just sets up observers.
|
|
34
46
|
|
|
35
47
|
NOTE: If you write your own constructor, you MUST call `super.__init__()` inside of it. The Service will throw an error if you don't.
|
|
36
48
|
"""
|
|
37
|
-
self._capability_name: str = 'InvalidCapability'
|
|
38
|
-
"""
|
|
39
|
-
The advertised name for the capability, as opposed to the implementation class name
|
|
40
|
-
"""
|
|
41
|
-
|
|
42
49
|
self.__intersect_sdk_observers__: list[IntersectEventObserver] = []
|
|
43
50
|
"""
|
|
44
51
|
INTERNAL USE ONLY.
|
|
@@ -63,15 +70,6 @@ class IntersectBaseCapabilityImplementation:
|
|
|
63
70
|
msg = f"{cls.__name__}: Attempted to override a reserved INTERSECT-SDK function (don't start your function names with '_intersect_sdk_' or 'intersect_sdk_')"
|
|
64
71
|
raise RuntimeError(msg)
|
|
65
72
|
|
|
66
|
-
@property
|
|
67
|
-
def capability_name(self) -> str:
|
|
68
|
-
"""The advertised name for the capability provided by this implementation."""
|
|
69
|
-
return self._capability_name
|
|
70
|
-
|
|
71
|
-
@capability_name.setter
|
|
72
|
-
def capability_name(self, cname: str) -> None:
|
|
73
|
-
self._capability_name = cname
|
|
74
|
-
|
|
75
73
|
@final
|
|
76
74
|
def _intersect_sdk_register_observer(self, observer: IntersectEventObserver) -> None:
|
|
77
75
|
"""INTERNAL USE ONLY."""
|
|
@@ -91,7 +89,9 @@ class IntersectBaseCapabilityImplementation:
|
|
|
91
89
|
register the event on the @intersect_event decorator. The @intersect_event annotation will be IGNORED if you place it
|
|
92
90
|
after an @intersect_message annotation; its intended use is for threaded functions you start from the capability.
|
|
93
91
|
|
|
94
|
-
|
|
92
|
+
Do NOT call this function from:
|
|
93
|
+
- any function called from an @intersect_status decorated function
|
|
94
|
+
- outside of the capability class (for example: capability_instance.intersect_sdk_emit_event(...) will not work). Create a function in the capability, decorate it with @intersect_event, and call that function.
|
|
95
95
|
|
|
96
96
|
params:
|
|
97
97
|
event_name: the type of event you are emitting. Note that you must advertise the event in your "entrypoint" function
|
|
@@ -133,12 +133,14 @@ class IntersectBaseCapabilityImplementation:
|
|
|
133
133
|
self,
|
|
134
134
|
request: IntersectDirectMessageParams,
|
|
135
135
|
response_handler: INTERSECT_SERVICE_RESPONSE_CALLBACK_TYPE | None = None,
|
|
136
|
+
timeout: float = 300.0,
|
|
136
137
|
) -> list[UUID]:
|
|
137
138
|
"""Create an external request that we'll send to a different Service.
|
|
138
139
|
|
|
139
140
|
Params:
|
|
140
141
|
- request: the request we want to send out, encapsulated as an IntersectClientMessageParams object
|
|
141
142
|
- response_handler: optional callback for how we want to handle the response from this request.
|
|
143
|
+
- timeout: optional value for how long we should wait on the request, in seconds (default: 300 seconds)
|
|
142
144
|
|
|
143
145
|
Returns:
|
|
144
146
|
- list of generated RequestIDs associated with your request. Note that for almost all use cases,
|
|
@@ -148,6 +150,6 @@ class IntersectBaseCapabilityImplementation:
|
|
|
148
150
|
- pydantic.ValidationError - if the request parameter isn't valid
|
|
149
151
|
"""
|
|
150
152
|
return [
|
|
151
|
-
observer.create_external_request(request, response_handler)
|
|
153
|
+
observer.create_external_request(request, response_handler, timeout)
|
|
152
154
|
for observer in self.__intersect_sdk_observers__
|
|
153
155
|
]
|
|
@@ -35,7 +35,6 @@ from ._internal.messages.userspace import (
|
|
|
35
35
|
create_userspace_message,
|
|
36
36
|
deserialize_and_validate_userspace_message,
|
|
37
37
|
)
|
|
38
|
-
from ._internal.stoppable_thread import StoppableThread
|
|
39
38
|
from ._internal.utils import die, send_os_signal
|
|
40
39
|
from ._internal.version_resolver import resolve_user_version
|
|
41
40
|
from .client_callback_definitions import (
|
|
@@ -121,9 +120,6 @@ class IntersectClient:
|
|
|
121
120
|
organization=config.organization,
|
|
122
121
|
)
|
|
123
122
|
|
|
124
|
-
self._heartbeat_thread: StoppableThread | None = None
|
|
125
|
-
self._heartbeat = 0.0
|
|
126
|
-
|
|
127
123
|
self._data_plane_manager = DataPlaneManager(self._hierarchy, config.data_stores)
|
|
128
124
|
self._control_plane_manager = ControlPlaneManager(
|
|
129
125
|
control_configs=config.brokers,
|
|
@@ -176,15 +172,6 @@ class IntersectClient:
|
|
|
176
172
|
# and has nothing to do with the Service at all.
|
|
177
173
|
time.sleep(1.0)
|
|
178
174
|
|
|
179
|
-
# start the heartbeat thread
|
|
180
|
-
if self._heartbeat_thread is None:
|
|
181
|
-
self._heartbeat = time.time()
|
|
182
|
-
self._heartbeat_thread = StoppableThread(
|
|
183
|
-
target=self._heartbeat_ticker,
|
|
184
|
-
name=f'IntersectClient_{uuid4()!s}_heartbeat_thread',
|
|
185
|
-
)
|
|
186
|
-
self._heartbeat_thread.start()
|
|
187
|
-
|
|
188
175
|
if self._resend_initial_messages or not self._sent_initial_messages:
|
|
189
176
|
for message in self._initial_messages:
|
|
190
177
|
self._send_userspace_message(message)
|
|
@@ -213,13 +200,6 @@ class IntersectClient:
|
|
|
213
200
|
"""
|
|
214
201
|
logger.info(f'Client is shutting down (reason: {reason})')
|
|
215
202
|
|
|
216
|
-
# Stop listening to the heartbeat
|
|
217
|
-
if self._heartbeat_thread is not None:
|
|
218
|
-
self._heartbeat_thread.stop()
|
|
219
|
-
self._heartbeat_thread.join()
|
|
220
|
-
self._heartbeat_thread = None
|
|
221
|
-
self._heartbeat = 0.0
|
|
222
|
-
|
|
223
203
|
self._control_plane_manager.disconnect()
|
|
224
204
|
|
|
225
205
|
logger.info('Client shutdown complete')
|
|
@@ -249,7 +229,6 @@ class IntersectClient:
|
|
|
249
229
|
if self._terminate_after_initial_messages:
|
|
250
230
|
return
|
|
251
231
|
|
|
252
|
-
self._heartbeat = time.time()
|
|
253
232
|
try:
|
|
254
233
|
message = deserialize_and_validate_userspace_message(raw)
|
|
255
234
|
logger.debug(f'Received userspace message:\n{message}')
|
|
@@ -330,7 +309,6 @@ class IntersectClient:
|
|
|
330
309
|
# safety check in case we get messages back faster than we can send them
|
|
331
310
|
return
|
|
332
311
|
|
|
333
|
-
self._heartbeat = time.time()
|
|
334
312
|
try:
|
|
335
313
|
message = deserialize_and_validate_event_message(raw)
|
|
336
314
|
logger.debug(f'Received userspace message:\n{message}')
|
|
@@ -458,26 +436,3 @@ class IntersectClient:
|
|
|
458
436
|
# but cannot communicate the response to the Client.
|
|
459
437
|
# in experiment controllers or production, you'll want to set persist to True
|
|
460
438
|
self._control_plane_manager.publish_message(channel, msg, persist=False)
|
|
461
|
-
|
|
462
|
-
# TODO - consider removing this entire concept
|
|
463
|
-
def _heartbeat_ticker(self) -> None:
|
|
464
|
-
"""Separate thread which checks to see how long it has been since a broker message was received.
|
|
465
|
-
|
|
466
|
-
If a broker has been connected for 5 minutes without sending a message, prepare to terminate the application.
|
|
467
|
-
"""
|
|
468
|
-
if self._heartbeat_thread:
|
|
469
|
-
self._heartbeat_thread.wait(300.0)
|
|
470
|
-
while not self._heartbeat_thread.stopped():
|
|
471
|
-
elapsed = time.time() - self._heartbeat
|
|
472
|
-
if elapsed > 300.0:
|
|
473
|
-
# NOTE
|
|
474
|
-
# This is by design. We explicitly don't want dangling clients
|
|
475
|
-
# sucking up bandwidth on brokers. It could even be argued that we should
|
|
476
|
-
# just call os.abort() here (this way so the Python application can't catch the SIGABRT),
|
|
477
|
-
# but SIGTERM is the soundest to ensure graceful application shutdown.
|
|
478
|
-
# However, graceful application shutdown is not as important for clients as it is for services...
|
|
479
|
-
logger.warning(
|
|
480
|
-
'Client has sat 5 minutes without sending or receiving any messages, exiting'
|
|
481
|
-
)
|
|
482
|
-
send_os_signal()
|
|
483
|
-
self._heartbeat_thread.wait(300.0)
|
{intersect_sdk-0.7.0 → intersect_sdk-0.8.0}/src/intersect_sdk/client_callback_definitions.py
RENAMED
|
@@ -3,13 +3,13 @@
|
|
|
3
3
|
See shared_callback_definitions for additional typings which are also shared by service authors.
|
|
4
4
|
"""
|
|
5
5
|
|
|
6
|
-
from typing import Callable,
|
|
6
|
+
from typing import Callable, List, Optional
|
|
7
7
|
|
|
8
8
|
from pydantic import BaseModel, ConfigDict, Field
|
|
9
|
-
from typing_extensions import Annotated,
|
|
9
|
+
from typing_extensions import Annotated, final
|
|
10
10
|
|
|
11
11
|
from .constants import SYSTEM_OF_SYSTEM_REGEX
|
|
12
|
-
from .shared_callback_definitions import IntersectDirectMessageParams
|
|
12
|
+
from .shared_callback_definitions import INTERSECT_JSON_VALUE, IntersectDirectMessageParams
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
@final
|
|
@@ -44,22 +44,6 @@ class IntersectClientCallback(BaseModel):
|
|
|
44
44
|
model_config = ConfigDict(revalidate_instances='always')
|
|
45
45
|
|
|
46
46
|
|
|
47
|
-
INTERSECT_JSON_VALUE: TypeAlias = Union[
|
|
48
|
-
List['INTERSECT_JSON_VALUE'],
|
|
49
|
-
Dict[str, 'INTERSECT_JSON_VALUE'],
|
|
50
|
-
str,
|
|
51
|
-
bool,
|
|
52
|
-
int,
|
|
53
|
-
float,
|
|
54
|
-
None,
|
|
55
|
-
]
|
|
56
|
-
"""
|
|
57
|
-
This is a simple type representation of JSON as a Python object. INTERSECT will automatically deserialize service payloads into one of these types.
|
|
58
|
-
|
|
59
|
-
(Pydantic has a similar type, "JsonValue", which should be used if you desire functionality beyond type hinting. This is strictly a type hint.)
|
|
60
|
-
"""
|
|
61
|
-
|
|
62
|
-
|
|
63
47
|
INTERSECT_CLIENT_RESPONSE_CALLBACK_TYPE = Callable[
|
|
64
48
|
[str, str, bool, INTERSECT_JSON_VALUE],
|
|
65
49
|
Optional[IntersectClientCallback],
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""This module is responsible for generating the core interface definitions of any Service.
|
|
2
|
+
|
|
3
|
+
There are two things to generate:
|
|
4
|
+
|
|
5
|
+
###
|
|
6
|
+
Schema
|
|
7
|
+
###
|
|
8
|
+
|
|
9
|
+
JSON schema advertises the interfaces to other systems. This is necessary for creating scientific campaigns, and the schema
|
|
10
|
+
is extensible to other clients.
|
|
11
|
+
|
|
12
|
+
Parts of the schema will be generated from users' own definitions. Functions are represented under "channels",
|
|
13
|
+
while Pydantic models defined by users and used as request or response types in their functions will have their schemas generated here.
|
|
14
|
+
|
|
15
|
+
There are also several parameters mainly for use by the central INTERSECT microservices, largely encapsulated from users.
|
|
16
|
+
|
|
17
|
+
###
|
|
18
|
+
Code Interface
|
|
19
|
+
###
|
|
20
|
+
|
|
21
|
+
When external systems send a message to this system, there are two important things in determining what executes: an operation id string,
|
|
22
|
+
and a payload which can be serialized into an auto-validated object. The operation id string is what can be mapped into various function definitions.
|
|
23
|
+
|
|
24
|
+
Users are able to define their response and request content-types, as well as their response data platform, for each exposed function (on the annotation).
|
|
25
|
+
|
|
26
|
+
The SDK needs to be able to dynamically look up functions, validate the request structure and request parameters, and handle responses.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
from typing import (
|
|
32
|
+
TYPE_CHECKING,
|
|
33
|
+
Any,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
from ._internal.schema import get_schema_and_functions_from_capability_implementations
|
|
37
|
+
from .capability.base import IntersectBaseCapabilityImplementation
|
|
38
|
+
|
|
39
|
+
if TYPE_CHECKING:
|
|
40
|
+
from .config.shared import HierarchyConfig
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def get_schema_from_capability_implementations(
|
|
44
|
+
capability_types: list[type[IntersectBaseCapabilityImplementation]],
|
|
45
|
+
hierarchy: HierarchyConfig,
|
|
46
|
+
) -> dict[str, Any]:
|
|
47
|
+
"""The goal of this function is to be able to generate a complete schema somewhat resembling the AsyncAPI spec 2.6.0 from a BaseModel class.
|
|
48
|
+
|
|
49
|
+
The generated schema is currently not a complete replica of the AsyncAPI spec. See https://www.asyncapi.com/docs/reference/specification/v2.6.0 for the complete specification.
|
|
50
|
+
Some key differences:
|
|
51
|
+
- We utilize three custom fields: "capabilities", "events", and "status".
|
|
52
|
+
- "capabilities" contains a dictionary: the keys of this dictionary are capability names. The values are dictionaries with the "description" property being a string which describes the capability,
|
|
53
|
+
and a "channels" property which more closely follows the AsyncAPI specification of the top-level value "channels".
|
|
54
|
+
- "events" is a key-value dictionary: the keys represent the event name, the values represent the associated schema of the event type. Events are currently shared across all capabilities.
|
|
55
|
+
- "status" will have a value of the status schema - if no status has been defined, a null schema is used.
|
|
56
|
+
|
|
57
|
+
Params:
|
|
58
|
+
- capability_types - list of classes (not objects) of capabilities. We do not require capabilities to be instantiated here, in case the instantiation of a capability has external dependencies.
|
|
59
|
+
- hierarchy - the hierarchy configuration. This is currently only reflected in the title of the global schema.
|
|
60
|
+
"""
|
|
61
|
+
if not all(issubclass(cap, IntersectBaseCapabilityImplementation) for cap in capability_types):
|
|
62
|
+
msg = 'get_schema_from_capability_implementations - not all provided values are valid capabilities (class must extend IntersectBaseCapabilityImplementation)'
|
|
63
|
+
raise RuntimeError(msg)
|
|
64
|
+
|
|
65
|
+
schemas, _, _, _, _, _ = get_schema_and_functions_from_capability_implementations(
|
|
66
|
+
capability_types,
|
|
67
|
+
hierarchy,
|
|
68
|
+
set(), # assume all data handlers are configured if user is just checking their schema
|
|
69
|
+
)
|
|
70
|
+
return schemas
|