intersect-sdk-common 0.9.3__tar.gz → 0.9.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/PKG-INFO +2 -2
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/README.md +1 -1
- intersect_sdk_common-0.9.5/pyproject.toml +188 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/control_plane/brokers/amqp_client.py +7 -2
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/control_plane/brokers/broker_client.py +2 -1
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/control_plane/brokers/mqtt_client.py +3 -3
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/control_plane/control_plane_manager.py +13 -4
- intersect_sdk_common-0.9.5/src/intersect_sdk_common/control_plane/custom_binary_protocol.py +77 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/control_plane/topic_handler.py +7 -3
- intersect_sdk_common-0.9.3/pyproject.toml +0 -183
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/__init__.py +0 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/config.py +0 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/constants.py +0 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/control_plane/__init__.py +0 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/control_plane/brokers/__init__.py +0 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/control_plane/definitions.py +0 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/control_plane/messages/__init__.py +0 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/control_plane/messages/event.py +0 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/control_plane/messages/lifecycle.py +0 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/control_plane/messages/userspace.py +0 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/core_definitions.py +0 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/data_plane/__init__.py +0 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/data_plane/data_plane_manager.py +0 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/data_plane/minio_utils.py +0 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/exceptions.py +0 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/logger.py +0 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/py.typed +0 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/utils/__init__.py +0 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/utils/multi_flag_thread_event.py +0 -0
- {intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: intersect-sdk-common
|
|
3
|
-
Version: 0.9.
|
|
3
|
+
Version: 0.9.5
|
|
4
4
|
Summary: Python SDK to interact with INTERSECT
|
|
5
5
|
Keywords: intersect
|
|
6
6
|
Author: Lance Drane, Marshall McDonnell, Seth Hitefield, Andrew Ayres, Gregory Cage, Jesse McGaha, Robert Smith, Gavin Wiggins, Michael Brim, Rick Archibald, Addi Malviya Thakur
|
|
@@ -48,5 +48,5 @@ The INTERSECT-SDK MUST follow semantic versioning, and MUST update its semantic
|
|
|
48
48
|
```bash
|
|
49
49
|
uv venv .venv
|
|
50
50
|
source .venv/bin/activate
|
|
51
|
-
uv sync --
|
|
51
|
+
uv sync --all-extras --all-groups
|
|
52
52
|
```
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "intersect-sdk-common"
|
|
3
|
+
description = "Python SDK to interact with INTERSECT"
|
|
4
|
+
authors = [
|
|
5
|
+
{ name = "Lance Drane", email = "dranelt@ornl.gov" },
|
|
6
|
+
{ name = "Marshall McDonnell", email = "mcdonnellmt@ornl.gov" },
|
|
7
|
+
{ name = "Seth Hitefield", email = "hitefieldsd@ornl.gov" },
|
|
8
|
+
{ name = "Andrew Ayres", email = "ayresaf@ornl.gov" },
|
|
9
|
+
{ name = "Gregory Cage", email = "cagege@ornl.gov" },
|
|
10
|
+
{ name = "Jesse McGaha", email = "mcgahajr@ornl.gov" },
|
|
11
|
+
{ name = "Robert Smith", email = "smithrw@ornl.gov" },
|
|
12
|
+
{ name = "Gavin Wiggins", email = "wigginsg@ornl.gov" },
|
|
13
|
+
{ name = "Michael Brim", email = "brimmj@ornl.gov" },
|
|
14
|
+
{ name = "Rick Archibald", email = "archibaldrk@ornl.gov" },
|
|
15
|
+
{ name = "Addi Malviya Thakur", email = "malviyaa@ornl.gov" },
|
|
16
|
+
]
|
|
17
|
+
readme = "README.md"
|
|
18
|
+
license = { text = "BSD-3-Clause" }
|
|
19
|
+
requires-python = ">=3.10,<4.0"
|
|
20
|
+
keywords = ["intersect"]
|
|
21
|
+
version = "0.9.5"
|
|
22
|
+
classifiers = [
|
|
23
|
+
"Programming Language :: Python :: 3",
|
|
24
|
+
"Programming Language :: Python :: 3.10",
|
|
25
|
+
"Programming Language :: Python :: 3.11",
|
|
26
|
+
"Programming Language :: Python :: 3.12",
|
|
27
|
+
"Programming Language :: Python :: 3.13",
|
|
28
|
+
]
|
|
29
|
+
dependencies = [
|
|
30
|
+
"pydantic>=2.7.0",
|
|
31
|
+
"paho-mqtt>=2.1.0,<3.0.0",
|
|
32
|
+
"pika>=1.3.2,<2.0.0",
|
|
33
|
+
"minio>=7.2.3",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
[dependency-groups]
|
|
37
|
+
dev = [
|
|
38
|
+
"pre-commit>=3.3.1",
|
|
39
|
+
"ruff==0.12.7",
|
|
40
|
+
"mypy>=1.10.0",
|
|
41
|
+
"codespell>=2.3.0",
|
|
42
|
+
"pytest>=7.3.2",
|
|
43
|
+
"pytest-cov>=4.1.0",
|
|
44
|
+
"httpretty>=1.1.4",
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
[build-system]
|
|
48
|
+
requires = ["uv_build>=0.10.2,<0.11.0"]
|
|
49
|
+
build-backend = "uv_build"
|
|
50
|
+
|
|
51
|
+
[tool.ruff]
|
|
52
|
+
line-length = 100
|
|
53
|
+
format = { quote-style = 'single' }
|
|
54
|
+
|
|
55
|
+
[tool.ruff.lint]
|
|
56
|
+
isort = { known-first-party = ['src'] }
|
|
57
|
+
pydocstyle = { convention = 'google' }
|
|
58
|
+
flake8-quotes = { inline-quotes = 'single', multiline-quotes = 'double' }
|
|
59
|
+
mccabe = { max-complexity = 20 }
|
|
60
|
+
pylint = { max-args = 10, max-branches = 20, max-returns = 15, max-statements = 75 }
|
|
61
|
+
# pyflakes and the relevant pycodestyle rules are already configured
|
|
62
|
+
extend-select = [
|
|
63
|
+
'C90', # mccabe complexity
|
|
64
|
+
'I', # isort
|
|
65
|
+
'N', # pep8-naming
|
|
66
|
+
'D', # pydocstyle
|
|
67
|
+
'UP', # pyupgrade
|
|
68
|
+
'YTT', # flake8-2020
|
|
69
|
+
'ANN', # flake8-annotations
|
|
70
|
+
'ASYNC', # flake8-async
|
|
71
|
+
'S', # flake8-bandit
|
|
72
|
+
'BLE', # flake8-blind-except
|
|
73
|
+
'B', # flake8-bugbear
|
|
74
|
+
'A', # flake8-builtins
|
|
75
|
+
'COM', # flake8-commas
|
|
76
|
+
'C4', # flake8-comprehensions
|
|
77
|
+
'DTZ', # flake8-datetimez
|
|
78
|
+
'T10', # flake8-debugger
|
|
79
|
+
'EM', # flake8-error-message
|
|
80
|
+
'FA', # flake8-future-annotations
|
|
81
|
+
'ISC', # flake8-implicit-string-concat
|
|
82
|
+
'ICN', # flake8-import-conventions
|
|
83
|
+
'G', # flake8-logging-format
|
|
84
|
+
'INP', # flake8-no-pep420
|
|
85
|
+
'PIE', # flake8-PIE
|
|
86
|
+
'T20', # flake8-T20
|
|
87
|
+
'PYI', # flake8-pyi
|
|
88
|
+
'PT', # flake8-pytest-style
|
|
89
|
+
'Q', # flake8-quotes
|
|
90
|
+
'RSE', # flake8-raise
|
|
91
|
+
'RET', # flake8-return
|
|
92
|
+
'SLF', # flake8-self
|
|
93
|
+
'SLOT', # flake8-slots
|
|
94
|
+
'SIM', # flake8-simplify
|
|
95
|
+
'TC', # flake8-type-checking
|
|
96
|
+
'ARG', # flake8-unused-arguments
|
|
97
|
+
'PTH', # flake8-use-pathlib
|
|
98
|
+
'PGH', # pygrep-hooks
|
|
99
|
+
'PL', # pylint
|
|
100
|
+
'TRY', # tryceratops
|
|
101
|
+
'FLY', # flynt
|
|
102
|
+
'RUF', # RUFF additional rules
|
|
103
|
+
'INT', # flake8-gettext
|
|
104
|
+
]
|
|
105
|
+
# If you're seeking to disable a rule, first consider whether the rule is overbearing, or if it should only be turned off for your usecase.
|
|
106
|
+
ignore = [
|
|
107
|
+
'COM812', # formatter, handled by Ruff format
|
|
108
|
+
'ISC001', # formatter, handled by Ruff format
|
|
109
|
+
'SIM105', # "with contextlib.suppress():" is slower than try-except-pass
|
|
110
|
+
'ANN401', # allow explicit "Any" typing, use with care
|
|
111
|
+
'PLR2004', # allow "magic numbers"
|
|
112
|
+
]
|
|
113
|
+
|
|
114
|
+
[tool.ruff.lint.flake8-type-checking]
|
|
115
|
+
runtime-evaluated-base-classes = ["pydantic.BaseModel"]
|
|
116
|
+
runtime-evaluated-decorators = [
|
|
117
|
+
"pydantic.dataclasses.dataclass",
|
|
118
|
+
"pydantic.validate_call",
|
|
119
|
+
]
|
|
120
|
+
|
|
121
|
+
[tool.ruff.lint.extend-per-file-ignores]
|
|
122
|
+
'__init__.py' = [
|
|
123
|
+
'F401', # __init__.py commonly has unused imports
|
|
124
|
+
'TC004', # do lazy imports when importing from the base module
|
|
125
|
+
]
|
|
126
|
+
'docs/*' = [
|
|
127
|
+
'D', # the documentation folder does not need documentation
|
|
128
|
+
'INP001', # docs are not a namespace package
|
|
129
|
+
]
|
|
130
|
+
'tests/*' = [
|
|
131
|
+
'S101', # allow assert statements in tests
|
|
132
|
+
'S106', # don't care about credentials in tests
|
|
133
|
+
'S311', # don't care about cryptographic security in tests
|
|
134
|
+
'SLF001', # allow private member access in tests
|
|
135
|
+
'ANN', # tests in general don't need types, unless they are runtime types.
|
|
136
|
+
'ARG', # allow unused parameters in tests
|
|
137
|
+
'D', # ignore documentation in tests
|
|
138
|
+
'RUF012', # permit "mutable" class attributes to not be annotated with typing.ClassVar (these shouldn't be mutated anyways...)
|
|
139
|
+
]
|
|
140
|
+
|
|
141
|
+
# see https://mypy.readthedocs.io/en/stable/config_file.html for a complete reference
|
|
142
|
+
[tool.mypy]
|
|
143
|
+
strict = true
|
|
144
|
+
ignore_missing_imports = true # don't require typing for library stubs if they don't exist
|
|
145
|
+
disallow_untyped_decorators = false # this is needed for library decorator compatibility, i.e. "retrying"
|
|
146
|
+
plugins = ["pydantic.mypy"]
|
|
147
|
+
|
|
148
|
+
[tool.pydantic-mypy]
|
|
149
|
+
init_forbid_extra = true
|
|
150
|
+
init_typed = true
|
|
151
|
+
warn_required_dynamic_aliases = true
|
|
152
|
+
warn_untyped_fields = true
|
|
153
|
+
|
|
154
|
+
[tool.pytest.ini_options]
|
|
155
|
+
log_cli = true
|
|
156
|
+
addopts = "-ra"
|
|
157
|
+
|
|
158
|
+
[tool.coverage.report]
|
|
159
|
+
omit = [
|
|
160
|
+
'*__init__*', # __init__ files should just re-export other classes and functions
|
|
161
|
+
]
|
|
162
|
+
exclude_also = [
|
|
163
|
+
'pragma: no-cover', # standard
|
|
164
|
+
'if (typing\\.)?TYPE_CHECKING:', # type checking blocks are not executed in coverage, but we don't care
|
|
165
|
+
'@(abc\\.)?abstractmethod', # don't try to cover abstract methods
|
|
166
|
+
"class .*\\bProtocol\\):", # don't cover protocol classes (similar to abstract classes)
|
|
167
|
+
'raise NotImplementedError', # it's not implemented so shouldn't be covered
|
|
168
|
+
'except.* ImportError', # these are usually used to throw a "friendlier" error and are not really worth testing
|
|
169
|
+
]
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
#[project.scripts]
|
|
173
|
+
#test-all = "pytest tests/ --cov=src/intersect_sdk_common/ --cov-fail-under=80 --cov-report=html:reports/htmlcov/ --cov-report=xml:reports/coverage_report.xml --junitxml=reports/junit.xml"
|
|
174
|
+
#test-all-debug = "pytest tests/ --cov=src/intersect_sdk_common/ --cov-fail-under=80 --cov-report=html:reports/htmlcov/ --cov-report=xml:reports/coverage_report.xml --junitxml=reports/junit.xml -s"
|
|
175
|
+
#test-unit = "pytest tests/unit --cov=src/intersect_sdk_common/"
|
|
176
|
+
#test-e2e = "pytest tests/e2e --cov=src/intersect_sdk_common/"
|
|
177
|
+
#lint-docs = "sphinx-build -W --keep-going docs docs/_build"
|
|
178
|
+
#lint-format = "ruff format"
|
|
179
|
+
#lint-ruff = "ruff check --fix"
|
|
180
|
+
#lint-mypy = "mypy src/intersect_sdk_common/"
|
|
181
|
+
#lint-spelling = "codespell -w docs examples src tests *.md -S docs/_build"
|
|
182
|
+
|
|
183
|
+
[tool.codespell]
|
|
184
|
+
# Ref: https://github.com/codespell-project/codespell#using-a-config-file
|
|
185
|
+
skip = '.git*,*.lock,.venv,.*cache/*,./docs/_build/*'
|
|
186
|
+
check-hidden = true
|
|
187
|
+
# ignore-regex = ''
|
|
188
|
+
# ignore-words-list = ''
|
|
@@ -223,17 +223,22 @@ class AMQPClient(BrokerClient):
|
|
|
223
223
|
else:
|
|
224
224
|
logger.error('Unable to publish message %s on topic %s', payload, topic)
|
|
225
225
|
|
|
226
|
-
def subscribe(self, topic: str, persist: bool) -> None:
|
|
226
|
+
def subscribe(self, topic: str, persist: bool, queue_name: str) -> None:
|
|
227
227
|
"""Subscribe to a topic.
|
|
228
228
|
|
|
229
229
|
topic: system-of-system hierarchy. In AMQP parlance this gets translated to the routing key.
|
|
230
230
|
persist: If True, we will create an idempotent queue name which should persist
|
|
231
231
|
even on broker or application shutdown. If False, we will allow the server to create a unique
|
|
232
232
|
queue name, and the queue will be destroyed once the associated channel is closed.
|
|
233
|
+
queue_name: The name of the queue to subscribe to.
|
|
233
234
|
"""
|
|
234
235
|
topic = _hierarchy_2_amqp(topic)
|
|
235
236
|
cb = functools.partial(
|
|
236
|
-
self._create_queue,
|
|
237
|
+
self._create_queue,
|
|
238
|
+
channel=self._channel_in,
|
|
239
|
+
topic=topic,
|
|
240
|
+
persist=persist,
|
|
241
|
+
queue_name=queue_name,
|
|
237
242
|
)
|
|
238
243
|
self._connection.ioloop.add_callback_threadsafe(cb)
|
|
239
244
|
|
|
@@ -55,7 +55,7 @@ class BrokerClient(Protocol):
|
|
|
55
55
|
"""
|
|
56
56
|
...
|
|
57
57
|
|
|
58
|
-
def subscribe(self, topic: str, persist: bool) -> None:
|
|
58
|
+
def subscribe(self, topic: str, persist: bool, queue_name: str) -> None:
|
|
59
59
|
"""Subscribe to a topic over the pre-existing connection (via connect()).
|
|
60
60
|
|
|
61
61
|
This function should ALSO be called by reconnect handlers, and not just directly.
|
|
@@ -64,6 +64,7 @@ class BrokerClient(Protocol):
|
|
|
64
64
|
Args:
|
|
65
65
|
topic: Topic to subscribe to.
|
|
66
66
|
persist: Whether or not the queue subscribed to is intended to be long-lived.
|
|
67
|
+
queue_name: The name of the queue to subscribe to. Used or ignored depending on the protocol.
|
|
67
68
|
"""
|
|
68
69
|
...
|
|
69
70
|
|
|
@@ -35,7 +35,6 @@ _MQTT_MAX_RETRIES = 10
|
|
|
35
35
|
# this will be a breaking change, so only add it when ready to break
|
|
36
36
|
def _hierarchy_2_mqtt(hierarchy: str) -> str:
|
|
37
37
|
"""Take the hierarchy string format saved in the Service and map it to the MQTT topic format. Currently just covers wildcards."""
|
|
38
|
-
# return hierarchy.replace('#', '+')
|
|
39
38
|
return hierarchy.replace('*', '+')
|
|
40
39
|
|
|
41
40
|
|
|
@@ -166,12 +165,13 @@ class MQTTClient(BrokerClient):
|
|
|
166
165
|
resolved_topic, payload, qos=self._max_supported_qos if persist else 0, properties=props
|
|
167
166
|
)
|
|
168
167
|
|
|
169
|
-
def subscribe(self, topic: str, persist: bool) -> None:
|
|
168
|
+
def subscribe(self, topic: str, persist: bool, queue_name: str) -> None: # noqa: ARG002
|
|
170
169
|
"""Subscribe to a topic over the pre-existing connection (via connect()).
|
|
171
170
|
|
|
172
171
|
Args:
|
|
173
172
|
topic: Topic to subscribe to.
|
|
174
173
|
persist: Determine if the associated message queue of the topic is long-lived (True) or not (False)
|
|
174
|
+
queue_name: The name of the queue to subscribe to. Ignored with current MQTT implementation.
|
|
175
175
|
"""
|
|
176
176
|
resolved_topic = _hierarchy_2_mqtt(topic)
|
|
177
177
|
# NOTE: RabbitMQ only works with QOS of 1 and 0, and seems to convert QOS2 to QOS1
|
|
@@ -302,7 +302,7 @@ class MQTTClient(BrokerClient):
|
|
|
302
302
|
|
|
303
303
|
self._connected_flag.set()
|
|
304
304
|
for topic, topic_handler in self._control_plane_manager.get_all_subscription_channels():
|
|
305
|
-
self.subscribe(topic, topic_handler.topic_persist)
|
|
305
|
+
self.subscribe(topic, topic_handler.topic_persist, topic_handler.queue_name)
|
|
306
306
|
else:
|
|
307
307
|
# This will generally suggest a misconfiguration
|
|
308
308
|
self._connected = False
|
|
@@ -10,14 +10,17 @@ from .brokers.broker_client import BrokerClient
|
|
|
10
10
|
from .definitions import MessageCallback
|
|
11
11
|
from .topic_handler import TopicHandler
|
|
12
12
|
|
|
13
|
-
_CHANNEL_REGEX = re.compile(r'^[a-zA-Z0-
|
|
13
|
+
_CHANNEL_REGEX = re.compile(r'^[a-zA-Z0-9_*/-]+[a-zA-Z0-9_*#/-]$')
|
|
14
14
|
"""Allow for these patterns:
|
|
15
15
|
|
|
16
16
|
- alphanumeric characters
|
|
17
17
|
- hyphens
|
|
18
|
+
- underscores
|
|
18
19
|
- `/` (topic separator; note that this is '.' on the broker)
|
|
19
20
|
- `*` (wildcard for exactly one word)
|
|
20
21
|
- `#` (wildcard for any number of words, but can ONLY appear at the end of the topic string, see section 4.7.1.2 at https://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html#_Toc398718107)
|
|
22
|
+
|
|
23
|
+
This should be permissive relative to other patterns. You must cover system/service/hierarchy regexes, capability regexes, and event name regexes.
|
|
21
24
|
"""
|
|
22
25
|
|
|
23
26
|
|
|
@@ -75,7 +78,11 @@ class ControlPlaneManager:
|
|
|
75
78
|
self._wildcards: dict[str, TopicHandler] = {}
|
|
76
79
|
|
|
77
80
|
def add_subscription_channel(
|
|
78
|
-
self,
|
|
81
|
+
self,
|
|
82
|
+
channel: str,
|
|
83
|
+
callbacks: set[MessageCallback],
|
|
84
|
+
persist: bool,
|
|
85
|
+
queue_name: str,
|
|
79
86
|
) -> None:
|
|
80
87
|
"""Start listening for messages on a channel on all configured brokers.
|
|
81
88
|
|
|
@@ -115,7 +122,7 @@ class ControlPlaneManager:
|
|
|
115
122
|
topic_handler.queue_name = queue_name
|
|
116
123
|
if self.is_connected():
|
|
117
124
|
for provider in self._control_providers:
|
|
118
|
-
provider.subscribe(channel, persist)
|
|
125
|
+
provider.subscribe(channel, persist, queue_name)
|
|
119
126
|
|
|
120
127
|
def remove_subscription_channel(self, channel: str) -> bool:
|
|
121
128
|
"""Stop subscribing to a channel on all configured brokers.
|
|
@@ -167,7 +174,9 @@ class ControlPlaneManager:
|
|
|
167
174
|
return wildcard, topic_handler
|
|
168
175
|
return None
|
|
169
176
|
|
|
170
|
-
def get_all_subscription_channels(
|
|
177
|
+
def get_all_subscription_channels(
|
|
178
|
+
self,
|
|
179
|
+
) -> itertools.chain[tuple[str, TopicHandler]]:
|
|
171
180
|
"""Get all subscription channels, including wildcard channels.
|
|
172
181
|
|
|
173
182
|
This function is safe to call from the broker clients, as it does not mutate state.
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""This module contains the binary specification for an INTERSECT message IF the protocol does not provide any built-in header support. If it does, use the built-in functionality of the protocol.
|
|
2
|
+
|
|
3
|
+
Some protocols do not have any built-in header support, leaving it up to us to define the binary structure of the message. We want to avoid "chunking" messages into multiple parts, so a message should be guaranteed to include all metadata.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from intersect_sdk_common import IntersectApplicationError
|
|
7
|
+
|
|
8
|
+
_HEADER_KV_SEPARATOR = b'\x02'
|
|
9
|
+
"""Indicates end of a header key and start of a header value.
|
|
10
|
+
|
|
11
|
+
Used because applications should generally assign no special meaning to this byte, and this byte has no reason to appear in header keys or values.
|
|
12
|
+
"""
|
|
13
|
+
_HEADER_VALUE_SEPARATOR = b'\x03'
|
|
14
|
+
"""Indicates end of a header value and start of the next header key.
|
|
15
|
+
|
|
16
|
+
Used because applications should generally assign no special meaning to this byte, and this byte has no reason to appear in header keys or values.
|
|
17
|
+
"""
|
|
18
|
+
_PAYLOAD_SEPARATOR = b'\x01'
|
|
19
|
+
"""Indicates end of headers and start of the payload.
|
|
20
|
+
|
|
21
|
+
Used because applications should generally assign no special meaning to this byte, and this byte has no reason to appear in header keys or values.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
_TOTAL_REASONABLE_HEADER_BYTES = 131072
|
|
25
|
+
"""The total number of bytes that should be reasonably expected to be used for header keys, values, and header separators combined. This provides applications with some level of DOS protection.
|
|
26
|
+
|
|
27
|
+
It is rare in practice for ANY application to use this number of bytes for the total amount of headers, for example http2_max_header_size in NGINX is rarely set above 128KB
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
CONTENT_TYPE_HEADER_KEY = 'content_type'
|
|
31
|
+
CONTENT_TYPE_HEADER_KEY_BYTES = CONTENT_TYPE_HEADER_KEY.encode()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def create_binary_message(body: bytes, content_type: str, headers: dict[str, str]) -> bytes:
|
|
35
|
+
"""Create a binary message from the body, headers, and content type."""
|
|
36
|
+
return b''.join(
|
|
37
|
+
[
|
|
38
|
+
# content-type 'header' first (this is generally handled separately from other headers in many protocols)
|
|
39
|
+
CONTENT_TYPE_HEADER_KEY_BYTES,
|
|
40
|
+
_HEADER_KV_SEPARATOR,
|
|
41
|
+
content_type.encode(),
|
|
42
|
+
_HEADER_VALUE_SEPARATOR if len(headers) else b'',
|
|
43
|
+
# headers
|
|
44
|
+
_HEADER_VALUE_SEPARATOR.join(
|
|
45
|
+
_HEADER_KV_SEPARATOR.join([key.encode(), value.encode()])
|
|
46
|
+
for key, value in headers.items()
|
|
47
|
+
),
|
|
48
|
+
# end of headers
|
|
49
|
+
_PAYLOAD_SEPARATOR,
|
|
50
|
+
# body
|
|
51
|
+
body,
|
|
52
|
+
]
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def parse_binary_message(message: bytes) -> tuple[bytes, str, dict[str, str]]:
|
|
57
|
+
"""Parse a binary message into its body, content type, and headers."""
|
|
58
|
+
# IMPORTANT!!! ----- Total length of header keys and values combined should be limited to first several bytes, terminate header search early if headers aren't a reasonable length.
|
|
59
|
+
payload_sep_location = message.find(_PAYLOAD_SEPARATOR, 0, _TOTAL_REASONABLE_HEADER_BYTES)
|
|
60
|
+
if payload_sep_location == -1:
|
|
61
|
+
msg = 'Probable malformed message: no payload separator found in first expected bytes, .'
|
|
62
|
+
raise IntersectApplicationError(msg)
|
|
63
|
+
header_string = message[:payload_sep_location]
|
|
64
|
+
headers = {
|
|
65
|
+
key.decode(): value.decode()
|
|
66
|
+
for key, value in (
|
|
67
|
+
header.split(_HEADER_KV_SEPARATOR)
|
|
68
|
+
for header in header_string.split(_HEADER_VALUE_SEPARATOR)
|
|
69
|
+
)
|
|
70
|
+
}
|
|
71
|
+
try:
|
|
72
|
+
content_type = headers.pop(CONTENT_TYPE_HEADER_KEY)
|
|
73
|
+
except KeyError as e:
|
|
74
|
+
msg = 'Probable malformed message: no content_type header found in message, discarding it.'
|
|
75
|
+
raise IntersectApplicationError(msg) from e
|
|
76
|
+
|
|
77
|
+
return message[payload_sep_location + 1 :], content_type, headers
|
|
@@ -23,7 +23,10 @@ class TopicHandler:
|
|
|
23
23
|
"""If this is a wildcard topic, the regex pattern to match incoming topics against (if no wildcards, this will be None)."""
|
|
24
24
|
|
|
25
25
|
def __init__(
|
|
26
|
-
self,
|
|
26
|
+
self,
|
|
27
|
+
topic_persist: bool,
|
|
28
|
+
queue_name: str,
|
|
29
|
+
regex_pattern: str | None = None,
|
|
27
30
|
) -> None:
|
|
28
31
|
"""Initialize a TopicHandler instance.
|
|
29
32
|
|
|
@@ -37,14 +40,15 @@ class TopicHandler:
|
|
|
37
40
|
self.queue_name = queue_name
|
|
38
41
|
|
|
39
42
|
if regex_pattern:
|
|
43
|
+
# the generated regex must be permissive and allow any non-special characters _CHANNEL_REGEX (ControlPlaneManager) allows
|
|
40
44
|
regex_builder = ['^']
|
|
41
45
|
for char in regex_pattern:
|
|
42
46
|
if char == '*':
|
|
43
47
|
# match a single word (do not include the separator)
|
|
44
|
-
regex_builder.append('[a-zA-Z0-
|
|
48
|
+
regex_builder.append('[a-zA-Z0-9_-]+')
|
|
45
49
|
elif char == '#':
|
|
46
50
|
# match any sequence of 0 or more words (make sure to allow for the separator)
|
|
47
|
-
regex_builder.append('[a-zA-Z0-9
|
|
51
|
+
regex_builder.append('[a-zA-Z0-9/_-]*')
|
|
48
52
|
elif char == '/':
|
|
49
53
|
# in-memory topics use '/' as the separator, it is the protocol handler's responsibility to convert topics to use '/' as the separator
|
|
50
54
|
regex_builder.append('/')
|
|
@@ -1,183 +0,0 @@
|
|
|
1
|
-
[project]
|
|
2
|
-
name = "intersect-sdk-common"
|
|
3
|
-
description = "Python SDK to interact with INTERSECT"
|
|
4
|
-
authors = [
|
|
5
|
-
{ name = "Lance Drane", email = "dranelt@ornl.gov" },
|
|
6
|
-
{ name = "Marshall McDonnell", email = "mcdonnellmt@ornl.gov" },
|
|
7
|
-
{ name = "Seth Hitefield", email = "hitefieldsd@ornl.gov" },
|
|
8
|
-
{ name = "Andrew Ayres", email = "ayresaf@ornl.gov" },
|
|
9
|
-
{ name = "Gregory Cage", email = "cagege@ornl.gov" },
|
|
10
|
-
{ name = "Jesse McGaha", email = "mcgahajr@ornl.gov" },
|
|
11
|
-
{ name = "Robert Smith", email = "smithrw@ornl.gov" },
|
|
12
|
-
{ name = "Gavin Wiggins", email = "wigginsg@ornl.gov" },
|
|
13
|
-
{ name = "Michael Brim", email = "brimmj@ornl.gov" },
|
|
14
|
-
{ name = "Rick Archibald", email = "archibaldrk@ornl.gov" },
|
|
15
|
-
{ name = "Addi Malviya Thakur", email = "malviyaa@ornl.gov" },
|
|
16
|
-
]
|
|
17
|
-
readme = "README.md"
|
|
18
|
-
license = { text = "BSD-3-Clause" }
|
|
19
|
-
requires-python = ">=3.10,<4.0"
|
|
20
|
-
keywords = ["intersect"]
|
|
21
|
-
version = "0.9.3"
|
|
22
|
-
classifiers = [
|
|
23
|
-
"Programming Language :: Python :: 3",
|
|
24
|
-
"Programming Language :: Python :: 3.10",
|
|
25
|
-
"Programming Language :: Python :: 3.11",
|
|
26
|
-
"Programming Language :: Python :: 3.12",
|
|
27
|
-
"Programming Language :: Python :: 3.13",
|
|
28
|
-
]
|
|
29
|
-
dependencies = [
|
|
30
|
-
"pydantic>=2.7.0",
|
|
31
|
-
"paho-mqtt>=2.1.0,<3.0.0",
|
|
32
|
-
"pika>=1.3.2,<2.0.0",
|
|
33
|
-
"minio>=7.2.3",
|
|
34
|
-
]
|
|
35
|
-
|
|
36
|
-
[dependency-groups]
|
|
37
|
-
lint = [
|
|
38
|
-
"pre-commit>=3.3.1",
|
|
39
|
-
"ruff==0.12.7",
|
|
40
|
-
"mypy>=1.10.0",
|
|
41
|
-
"codespell>=2.3.0",
|
|
42
|
-
]
|
|
43
|
-
test = ["pytest>=7.3.2", "pytest-cov>=4.1.0", "httpretty>=1.1.4"]
|
|
44
|
-
|
|
45
|
-
[build-system]
|
|
46
|
-
requires = ["uv_build>=0.10.2,<0.11.0"]
|
|
47
|
-
build-backend = "uv_build"
|
|
48
|
-
|
|
49
|
-
[tool.ruff]
|
|
50
|
-
line-length = 100
|
|
51
|
-
format = { quote-style = 'single' }
|
|
52
|
-
|
|
53
|
-
[tool.ruff.lint]
|
|
54
|
-
isort = { known-first-party = ['src'] }
|
|
55
|
-
pydocstyle = { convention = 'google' }
|
|
56
|
-
flake8-quotes = { inline-quotes = 'single', multiline-quotes = 'double' }
|
|
57
|
-
mccabe = { max-complexity = 20 }
|
|
58
|
-
pylint = { max-args = 10, max-branches = 20, max-returns = 15, max-statements = 75 }
|
|
59
|
-
# pyflakes and the relevant pycodestyle rules are already configured
|
|
60
|
-
extend-select = [
|
|
61
|
-
'C90', # mccabe complexity
|
|
62
|
-
'I', # isort
|
|
63
|
-
'N', # pep8-naming
|
|
64
|
-
'D', # pydocstyle
|
|
65
|
-
'UP', # pyupgrade
|
|
66
|
-
'YTT', # flake8-2020
|
|
67
|
-
'ANN', # flake8-annotations
|
|
68
|
-
'ASYNC', # flake8-async
|
|
69
|
-
'S', # flake8-bandit
|
|
70
|
-
'BLE', # flake8-blind-except
|
|
71
|
-
'B', # flake8-bugbear
|
|
72
|
-
'A', # flake8-builtins
|
|
73
|
-
'COM', # flake8-commas
|
|
74
|
-
'C4', # flake8-comprehensions
|
|
75
|
-
'DTZ', # flake8-datetimez
|
|
76
|
-
'T10', # flake8-debugger
|
|
77
|
-
'EM', # flake8-error-message
|
|
78
|
-
'FA', # flake8-future-annotations
|
|
79
|
-
'ISC', # flake8-implicit-string-concat
|
|
80
|
-
'ICN', # flake8-import-conventions
|
|
81
|
-
'G', # flake8-logging-format
|
|
82
|
-
'INP', # flake8-no-pep420
|
|
83
|
-
'PIE', # flake8-PIE
|
|
84
|
-
'T20', # flake8-T20
|
|
85
|
-
'PYI', # flake8-pyi
|
|
86
|
-
'PT', # flake8-pytest-style
|
|
87
|
-
'Q', # flake8-quotes
|
|
88
|
-
'RSE', # flake8-raise
|
|
89
|
-
'RET', # flake8-return
|
|
90
|
-
'SLF', # flake8-self
|
|
91
|
-
'SLOT', # flake8-slots
|
|
92
|
-
'SIM', # flake8-simplify
|
|
93
|
-
'TC', # flake8-type-checking
|
|
94
|
-
'ARG', # flake8-unused-arguments
|
|
95
|
-
'PTH', # flake8-use-pathlib
|
|
96
|
-
'PGH', # pygrep-hooks
|
|
97
|
-
'PL', # pylint
|
|
98
|
-
'TRY', # tryceratops
|
|
99
|
-
'FLY', # flynt
|
|
100
|
-
'RUF', # RUFF additional rules
|
|
101
|
-
'INT', # flake8-gettext
|
|
102
|
-
]
|
|
103
|
-
# If you're seeking to disable a rule, first consider whether the rule is overbearing, or if it should only be turned off for your usecase.
|
|
104
|
-
ignore = [
|
|
105
|
-
'COM812', # formatter, handled by Ruff format
|
|
106
|
-
'ISC001', # formatter, handled by Ruff format
|
|
107
|
-
'SIM105', # "with contextlib.suppress():" is slower than try-except-pass
|
|
108
|
-
'ANN401', # allow explicit "Any" typing, use with care
|
|
109
|
-
'PLR2004', # allow "magic numbers"
|
|
110
|
-
]
|
|
111
|
-
|
|
112
|
-
[tool.ruff.lint.flake8-type-checking]
|
|
113
|
-
runtime-evaluated-base-classes = ["pydantic.BaseModel"]
|
|
114
|
-
runtime-evaluated-decorators = ["pydantic.dataclasses.dataclass","pydantic.validate_call"]
|
|
115
|
-
|
|
116
|
-
[tool.ruff.lint.extend-per-file-ignores]
|
|
117
|
-
'__init__.py' = [
|
|
118
|
-
'F401', # __init__.py commonly has unused imports
|
|
119
|
-
'TC004', # do lazy imports when importing from the base module
|
|
120
|
-
]
|
|
121
|
-
'docs/*' = [
|
|
122
|
-
'D', # the documentation folder does not need documentation
|
|
123
|
-
'INP001', # docs are not a namespace package
|
|
124
|
-
]
|
|
125
|
-
'tests/*' = [
|
|
126
|
-
'S101', # allow assert statements in tests
|
|
127
|
-
'S106', # don't care about credentials in tests
|
|
128
|
-
'S311', # don't care about cryptographic security in tests
|
|
129
|
-
'SLF001', # allow private member access in tests
|
|
130
|
-
'ANN', # tests in general don't need types, unless they are runtime types.
|
|
131
|
-
'ARG', # allow unused parameters in tests
|
|
132
|
-
'D', # ignore documentation in tests
|
|
133
|
-
'RUF012', # permit "mutable" class attributes to not be annotated with typing.ClassVar (these shouldn't be mutated anyways...)
|
|
134
|
-
]
|
|
135
|
-
|
|
136
|
-
# see https://mypy.readthedocs.io/en/stable/config_file.html for a complete reference
|
|
137
|
-
[tool.mypy]
|
|
138
|
-
strict = true
|
|
139
|
-
ignore_missing_imports = true # don't require typing for library stubs if they don't exist
|
|
140
|
-
disallow_untyped_decorators = false # this is needed for library decorator compatibility, i.e. "retrying"
|
|
141
|
-
plugins = ["pydantic.mypy"]
|
|
142
|
-
|
|
143
|
-
[tool.pydantic-mypy]
|
|
144
|
-
init_forbid_extra = true
|
|
145
|
-
init_typed = true
|
|
146
|
-
warn_required_dynamic_aliases = true
|
|
147
|
-
warn_untyped_fields = true
|
|
148
|
-
|
|
149
|
-
[tool.pytest.ini_options]
|
|
150
|
-
log_cli = true
|
|
151
|
-
addopts = "-ra"
|
|
152
|
-
|
|
153
|
-
[tool.coverage.report]
|
|
154
|
-
omit = [
|
|
155
|
-
'*__init__*', # __init__ files should just re-export other classes and functions
|
|
156
|
-
]
|
|
157
|
-
exclude_also = [
|
|
158
|
-
'pragma: no-cover', # standard
|
|
159
|
-
'if (typing\\.)?TYPE_CHECKING:', # type checking blocks are not executed in coverage, but we don't care
|
|
160
|
-
'@(abc\\.)?abstractmethod', # don't try to cover abstract methods
|
|
161
|
-
"class .*\\bProtocol\\):", # don't cover protocol classes (similar to abstract classes)
|
|
162
|
-
'raise NotImplementedError', # it's not implemented so shouldn't be covered
|
|
163
|
-
'except.* ImportError', # these are usually used to throw a "friendlier" error and are not really worth testing
|
|
164
|
-
]
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
#[project.scripts]
|
|
168
|
-
#test-all = "pytest tests/ --cov=src/intersect_sdk_common/ --cov-fail-under=80 --cov-report=html:reports/htmlcov/ --cov-report=xml:reports/coverage_report.xml --junitxml=reports/junit.xml"
|
|
169
|
-
#test-all-debug = "pytest tests/ --cov=src/intersect_sdk_common/ --cov-fail-under=80 --cov-report=html:reports/htmlcov/ --cov-report=xml:reports/coverage_report.xml --junitxml=reports/junit.xml -s"
|
|
170
|
-
#test-unit = "pytest tests/unit --cov=src/intersect_sdk_common/"
|
|
171
|
-
#test-e2e = "pytest tests/e2e --cov=src/intersect_sdk_common/"
|
|
172
|
-
#lint-docs = "sphinx-build -W --keep-going docs docs/_build"
|
|
173
|
-
#lint-format = "ruff format"
|
|
174
|
-
#lint-ruff = "ruff check --fix"
|
|
175
|
-
#lint-mypy = "mypy src/intersect_sdk_common/"
|
|
176
|
-
#lint-spelling = "codespell -w docs examples src tests *.md -S docs/_build"
|
|
177
|
-
|
|
178
|
-
[tool.codespell]
|
|
179
|
-
# Ref: https://github.com/codespell-project/codespell#using-a-config-file
|
|
180
|
-
skip = '.git*,*.lock,.venv,.*cache/*,./docs/_build/*'
|
|
181
|
-
check-hidden = true
|
|
182
|
-
# ignore-regex = ''
|
|
183
|
-
# ignore-words-list = ''
|
{intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/__init__.py
RENAMED
|
File without changes
|
{intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/config.py
RENAMED
|
File without changes
|
{intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/constants.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/exceptions.py
RENAMED
|
File without changes
|
{intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/logger.py
RENAMED
|
File without changes
|
|
File without changes
|
{intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/utils/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{intersect_sdk_common-0.9.3 → intersect_sdk_common-0.9.5}/src/intersect_sdk_common/version.py
RENAMED
|
File without changes
|