ai-box-lib 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ai_box_lib-0.1.0/LICENSE +27 -0
- ai_box_lib-0.1.0/PKG-INFO +49 -0
- ai_box_lib-0.1.0/README.md +9 -0
- ai_box_lib-0.1.0/pyproject.toml +21 -0
- ai_box_lib-0.1.0/setup.cfg +4 -0
- ai_box_lib-0.1.0/src/ai_box_lib/__init__.py +0 -0
- ai_box_lib-0.1.0/src/ai_box_lib/component_io.py +306 -0
- ai_box_lib-0.1.0/src/ai_box_lib/context_engine_client.py +72 -0
- ai_box_lib-0.1.0/src/ai_box_lib/data_collector_client.py +75 -0
- ai_box_lib-0.1.0/src/ai_box_lib/pre_processor_client.py +136 -0
- ai_box_lib-0.1.0/src/ai_box_lib/typing.py +6 -0
- ai_box_lib-0.1.0/src/ai_box_lib.egg-info/PKG-INFO +49 -0
- ai_box_lib-0.1.0/src/ai_box_lib.egg-info/SOURCES.txt +14 -0
- ai_box_lib-0.1.0/src/ai_box_lib.egg-info/dependency_links.txt +1 -0
- ai_box_lib-0.1.0/src/ai_box_lib.egg-info/requires.txt +2 -0
- ai_box_lib-0.1.0/src/ai_box_lib.egg-info/top_level.txt +1 -0
ai_box_lib-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
SOFTWARE LICENSE AGREEMENT
|
|
2
|
+
|
|
3
|
+
This Software Library ("Library") is licensed, not sold, to you by Synadia ("Licensor"). By using, copying, or distributing this Library, you agree to the following terms:
|
|
4
|
+
|
|
5
|
+
1. Authorized Use
|
|
6
|
+
- You may use this Library only if you have purchased a compatible device from Synadia.
|
|
7
|
+
- You may use the Library solely to develop software that runs on the purchased device.
|
|
8
|
+
- Any other use, including use on non-Synadia devices, is strictly prohibited.
|
|
9
|
+
|
|
10
|
+
2. Restrictions
|
|
11
|
+
- You may not distribute, sublicense, or otherwise make the Library available to any third party except as part of software running exclusively on the purchased device.
|
|
12
|
+
- You may not use the Library for any commercial purpose other than developing software for the purchased device.
|
|
13
|
+
- Reverse engineering, modification, or derivative works are permitted only for the purpose of developing software for the purchased device.
|
|
14
|
+
|
|
15
|
+
3. Ownership
|
|
16
|
+
- The Library remains the property of Synadia. No ownership rights are transferred.
|
|
17
|
+
|
|
18
|
+
4. Termination
|
|
19
|
+
- This license is automatically terminated if you breach any of its terms. Upon termination, you must destroy all copies of the Library.
|
|
20
|
+
|
|
21
|
+
5. Disclaimer
|
|
22
|
+
- The Library is provided "AS IS" without warranty of any kind. Synadia disclaims all warranties, express or implied, including but not limited to merchantability and fitness for a particular purpose.
|
|
23
|
+
|
|
24
|
+
6. Limitation of Liability
|
|
25
|
+
- In no event shall Synadia be liable for any damages arising from the use or inability to use the Library.
|
|
26
|
+
|
|
27
|
+
For questions or licensing inquiries, contact Synadia.
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ai-box-lib
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Python library for AWS Greengrass integration
|
|
5
|
+
Author-email: Your Name <your.email@example.com>
|
|
6
|
+
License: SOFTWARE LICENSE AGREEMENT
|
|
7
|
+
|
|
8
|
+
This Software Library ("Library") is licensed, not sold, to you by Synadia ("Licensor"). By using, copying, or distributing this Library, you agree to the following terms:
|
|
9
|
+
|
|
10
|
+
1. Authorized Use
|
|
11
|
+
- You may use this Library only if you have purchased a compatible device from Synadia.
|
|
12
|
+
- You may use the Library solely to develop software that runs on the purchased device.
|
|
13
|
+
- Any other use, including use on non-Synadia devices, is strictly prohibited.
|
|
14
|
+
|
|
15
|
+
2. Restrictions
|
|
16
|
+
- You may not distribute, sublicense, or otherwise make the Library available to any third party except as part of software running exclusively on the purchased device.
|
|
17
|
+
- You may not use the Library for any commercial purpose other than developing software for the purchased device.
|
|
18
|
+
- Reverse engineering, modification, or derivative works are permitted only for the purpose of developing software for the purchased device.
|
|
19
|
+
|
|
20
|
+
3. Ownership
|
|
21
|
+
- The Library remains the property of Synadia. No ownership rights are transferred.
|
|
22
|
+
|
|
23
|
+
4. Termination
|
|
24
|
+
- This license is automatically terminated if you breach any of its terms. Upon termination, you must destroy all copies of the Library.
|
|
25
|
+
|
|
26
|
+
5. Disclaimer
|
|
27
|
+
- The Library is provided "AS IS" without warranty of any kind. Synadia disclaims all warranties, express or implied, including but not limited to merchantability and fitness for a particular purpose.
|
|
28
|
+
|
|
29
|
+
6. Limitation of Liability
|
|
30
|
+
- In no event shall Synadia be liable for any damages arising from the use or inability to use the Library.
|
|
31
|
+
|
|
32
|
+
For questions or licensing inquiries, contact Synadia.
|
|
33
|
+
|
|
34
|
+
Requires-Python: >=3.11
|
|
35
|
+
Description-Content-Type: text/markdown
|
|
36
|
+
License-File: LICENSE
|
|
37
|
+
Requires-Dist: greengrasssdk
|
|
38
|
+
Requires-Dist: awsiotsdk
|
|
39
|
+
Dynamic: license-file
|
|
40
|
+
|
|
41
|
+
# AI Box Library
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
## Development
|
|
45
|
+
|
|
46
|
+
Use the `./src/ai-box-lib` to write all the library code which can be used by users.
|
|
47
|
+
|
|
48
|
+
In order to test the library locally you can run the following commands:
|
|
49
|
+
1.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = [
|
|
3
|
+
"setuptools>=61.0",
|
|
4
|
+
"wheel"
|
|
5
|
+
]
|
|
6
|
+
build-backend = "setuptools.build_meta"
|
|
7
|
+
|
|
8
|
+
[project]
|
|
9
|
+
name = "ai-box-lib"
|
|
10
|
+
version = "0.1.0"
|
|
11
|
+
description = "Python library for AWS Greengrass integration"
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "Your Name", email = "your.email@example.com" }
|
|
14
|
+
]
|
|
15
|
+
dependencies = [
|
|
16
|
+
"greengrasssdk",
|
|
17
|
+
"awsiotsdk",
|
|
18
|
+
]
|
|
19
|
+
readme = "README.md"
|
|
20
|
+
license = { file = "LICENSE" }
|
|
21
|
+
requires-python = ">=3.11"
|
|
File without changes
|
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Provides a client for inter-component communication within AWS Greengrass.
|
|
3
|
+
|
|
4
|
+
This module defines the `ComponentIO` class, a generic client that simplifies
|
|
5
|
+
publishing and subscribing to topics using the AWS IoT Greengrass Core IPC service.
|
|
6
|
+
It is designed to be used by Greengrass components to communicate with each other.
|
|
7
|
+
|
|
8
|
+
The client requires the `DEPLOYMENT_ID`, `COMPONENT_ID` and `COMPONENT_VERSION_ID` environment variables
|
|
9
|
+
to be set, which are typically provided by the Greengrass environment.
|
|
10
|
+
|
|
11
|
+
Example:
|
|
12
|
+
```python
|
|
13
|
+
|
|
14
|
+
# Define the type of data you expect to send/receive
|
|
15
|
+
io = ComponentIO[dict]()
|
|
16
|
+
io.connect()
|
|
17
|
+
|
|
18
|
+
def on_message(message: dict):
|
|
19
|
+
print(f"Received message: {message}")
|
|
20
|
+
|
|
21
|
+
async def main():
|
|
22
|
+
# Subscribe to a topic
|
|
23
|
+
unsubscribe = io._subscribe("my/topic", on_message)
|
|
24
|
+
|
|
25
|
+
# Publish a message
|
|
26
|
+
await io._publish("my/topic", {"key": "value"})
|
|
27
|
+
|
|
28
|
+
# Wait for a bit to receive the message
|
|
29
|
+
await asyncio.sleep(2)
|
|
30
|
+
|
|
31
|
+
# Clean up the subscription
|
|
32
|
+
unsubscribe()
|
|
33
|
+
|
|
34
|
+
if __name__ == "__main__":
|
|
35
|
+
asyncio.run(main())
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
"""
|
|
39
|
+
from dataclasses import dataclass
|
|
40
|
+
from typing import Any, Generic, Mapping, TypeVar, Optional, Callable
|
|
41
|
+
|
|
42
|
+
import os
|
|
43
|
+
import sys
|
|
44
|
+
import traceback
|
|
45
|
+
import json
|
|
46
|
+
import logging
|
|
47
|
+
from enum import Enum
|
|
48
|
+
from awsiot.greengrasscoreipc.clientv2 import GreengrassCoreIPCClientV2
|
|
49
|
+
from awsiot.greengrasscoreipc.model import (
|
|
50
|
+
BinaryMessage,
|
|
51
|
+
PublishMessage,
|
|
52
|
+
JsonMessage,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "ERROR"))
|
|
56
|
+
logger = logging.getLogger(__name__)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class DataType(Enum):
|
|
60
|
+
"""Enumeration of supported data types."""
|
|
61
|
+
|
|
62
|
+
FLOAT32 = "FLOAT32"
|
|
63
|
+
INT32 = "INT32"
|
|
64
|
+
STRING = "STRING"
|
|
65
|
+
BOOLEAN = "BOOLEAN"
|
|
66
|
+
BYTES = "BYTES"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass
|
|
70
|
+
class ContextDefinition:
|
|
71
|
+
"""Represents a data context with a name and index."""
|
|
72
|
+
|
|
73
|
+
name: str
|
|
74
|
+
index: int
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass
|
|
78
|
+
class ChannelDefinition:
|
|
79
|
+
"""Represents a data channel with a name and index."""
|
|
80
|
+
|
|
81
|
+
name: str
|
|
82
|
+
index: int
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@dataclass
|
|
86
|
+
class FeatureDefinition:
|
|
87
|
+
"""Represents a feature with a name and index."""
|
|
88
|
+
|
|
89
|
+
name: str
|
|
90
|
+
index: int
|
|
91
|
+
# pylint: disable=invalid-name
|
|
92
|
+
dataType: DataType
|
|
93
|
+
shape: list[int]
|
|
94
|
+
description: Optional[str] = None
|
|
95
|
+
|
|
96
|
+
S = TypeVar("S", bound=Mapping[str, str] | None)
|
|
97
|
+
P = TypeVar("P", bound=Mapping[str, Any] | None)
|
|
98
|
+
|
|
99
|
+
class ComponentIO(Generic[S, P]): # pylint: disable=too-few-public-methods
|
|
100
|
+
"""
|
|
101
|
+
A client for communication between Greengrass components via IPC.
|
|
102
|
+
|
|
103
|
+
This class wraps the AWS Greengrass Core IPC client to provide a simplified
|
|
104
|
+
interface for publishing and subscribing to topics. It automatically handles
|
|
105
|
+
message serialization (JSON) and deserialization.
|
|
106
|
+
|
|
107
|
+
The `DEPLOYMENT_ID`, `COMPONENT_ID` and `COMPONENT_VERSION_ID` environment variables must be set before
|
|
108
|
+
instantiating this class.
|
|
109
|
+
|
|
110
|
+
The `connect()` method must be called before any publish or subscribe
|
|
111
|
+
operations.
|
|
112
|
+
|
|
113
|
+
This is a generic class, and the type variable `T` represents the expected
|
|
114
|
+
type of the message payload.
|
|
115
|
+
|
|
116
|
+
This is a generic class with two type variables:
|
|
117
|
+
- S: Represents the expected type of incoming message payloads when subscribing to topics
|
|
118
|
+
- P: Represents the type of outgoing message payloads when publishing to topics
|
|
119
|
+
|
|
120
|
+
Attributes:
|
|
121
|
+
deployment_id (str): The ID of the deployment, read from the
|
|
122
|
+
`DEPLOYMENT_ID` environment variable.
|
|
123
|
+
component_id (str): The ID of the component, read from the
|
|
124
|
+
`COMPONENT_ID` environment variable.
|
|
125
|
+
component_version_id (str): The ID of the component, read from the
|
|
126
|
+
`COMPONENT_VERSION_ID` environment variable.
|
|
127
|
+
|
|
128
|
+
Initializes the ComponentIO client.
|
|
129
|
+
|
|
130
|
+
Reads deployment and component version IDs from environment variables.
|
|
131
|
+
|
|
132
|
+
Raises:
|
|
133
|
+
ValueError: If `DEPLOYMENT_ID`, `COMPONENT_ID`, and `COMPONENT_VERSION_ID` environment
|
|
134
|
+
variables are not set.
|
|
135
|
+
"""
|
|
136
|
+
|
|
137
|
+
_ipc: Optional[GreengrassCoreIPCClientV2]
|
|
138
|
+
deployment_id: str
|
|
139
|
+
component_id: str
|
|
140
|
+
component_version_id: str
|
|
141
|
+
|
|
142
|
+
def __init__(self) -> None:
|
|
143
|
+
"""Initializes the ComponentIO client."""
|
|
144
|
+
self.deployment_id = self._get_environment_variable("DEPLOYMENT_ID")
|
|
145
|
+
self.component_id = self._get_environment_variable("COMPONENT_ID")
|
|
146
|
+
self.component_version_id = self._get_environment_variable(
|
|
147
|
+
"COMPONENT_VERSION_ID"
|
|
148
|
+
)
|
|
149
|
+
self._ipc = None
|
|
150
|
+
|
|
151
|
+
def connect(self) -> None:
|
|
152
|
+
"""Connects to the Greengrass Core IPC service.
|
|
153
|
+
|
|
154
|
+
This method initializes the connection to the underlying message broker.
|
|
155
|
+
It must be called before using `_publish` or `_subscribe`. If already
|
|
156
|
+
connected, this method does nothing.
|
|
157
|
+
"""
|
|
158
|
+
if self._ipc is None:
|
|
159
|
+
self._ipc = GreengrassCoreIPCClientV2()
|
|
160
|
+
|
|
161
|
+
def _publish(self, topic: str, message: P) -> None:
|
|
162
|
+
"""Publishes a message to a specified topic.
|
|
163
|
+
|
|
164
|
+
The message is serialized based on its type. If it's `bytes` or
|
|
165
|
+
`bytearray`, it's sent as a binary message. Otherwise, it's serialized
|
|
166
|
+
to JSON.
|
|
167
|
+
|
|
168
|
+
topic: The topic to publish the message to.
|
|
169
|
+
message: The message payload to publish.
|
|
170
|
+
|
|
171
|
+
Raises:
|
|
172
|
+
RuntimeError: If the client is not connected. `connect()` must be
|
|
173
|
+
called first.
|
|
174
|
+
"""
|
|
175
|
+
if self._ipc is None:
|
|
176
|
+
raise RuntimeError(
|
|
177
|
+
"Client is not connected. Call connect() before publishing."
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
if isinstance(message, (bytes, bytearray)):
|
|
181
|
+
pm = PublishMessage(
|
|
182
|
+
binary_message=BinaryMessage(message=bytes(message)))
|
|
183
|
+
else:
|
|
184
|
+
# Let IPC SDK do JSON serialization of Python primitive/dict/list
|
|
185
|
+
# Convert Mapping to Dict if needed
|
|
186
|
+
dict_message = dict(message) if message is not None else None
|
|
187
|
+
pm = PublishMessage(
|
|
188
|
+
json_message=JsonMessage(message=dict_message)
|
|
189
|
+
) # type: ignore
|
|
190
|
+
self._ipc.publish_to_topic(topic=topic, publish_message=pm)
|
|
191
|
+
|
|
192
|
+
def _subscribe(
|
|
193
|
+
self, topic: str, handler: Callable[[S], None]
|
|
194
|
+
) -> Callable[[], None]:
|
|
195
|
+
"""Subscribes to a topic and registers a handler for incoming messages.
|
|
196
|
+
|
|
197
|
+
When a message is received on the topic, the `handler` function is
|
|
198
|
+
called with the message payload. The method automatically handles
|
|
199
|
+
deserialization. For binary messages, it attempts to decode them as
|
|
200
|
+
JSON first, falling back to raw bytes if decoding fails.
|
|
201
|
+
|
|
202
|
+
topic: The topic to subscribe to.
|
|
203
|
+
handler: A callback function that will be invoked with the
|
|
204
|
+
message payload when a message is received.
|
|
205
|
+
|
|
206
|
+
Raises:
|
|
207
|
+
RuntimeError: If the client is not connected. `connect()` must be
|
|
208
|
+
called first.
|
|
209
|
+
|
|
210
|
+
Returns:
|
|
211
|
+
A callable function that, when invoked, will close the subscription
|
|
212
|
+
and stop receiving messages.
|
|
213
|
+
"""
|
|
214
|
+
if self._ipc is None:
|
|
215
|
+
raise RuntimeError(
|
|
216
|
+
"Client is not connected. Call connect() before subscribing."
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
def _on_event(event) -> None:
|
|
220
|
+
try:
|
|
221
|
+
if not event:
|
|
222
|
+
return
|
|
223
|
+
if (
|
|
224
|
+
getattr(event, "json_message", None)
|
|
225
|
+
and event.json_message.message is not None
|
|
226
|
+
):
|
|
227
|
+
handler(event.json_message.message)
|
|
228
|
+
elif (
|
|
229
|
+
getattr(event, "binary_message", None)
|
|
230
|
+
and event.binary_message.message is not None
|
|
231
|
+
):
|
|
232
|
+
raw = event.binary_message.message
|
|
233
|
+
# Attempt JSON decode; fallback to raw bytes
|
|
234
|
+
try:
|
|
235
|
+
handler(json.loads(raw.decode("utf-8")))
|
|
236
|
+
except (json.JSONDecodeError, UnicodeDecodeError):
|
|
237
|
+
handler(raw)
|
|
238
|
+
except (TypeError, ValueError) as e:
|
|
239
|
+
print(
|
|
240
|
+
f"Failed to process subscription event: {e}", file=sys.stderr)
|
|
241
|
+
traceback.print_exc()
|
|
242
|
+
|
|
243
|
+
def _on_error(error: Exception) -> bool:
|
|
244
|
+
print(f"Error in subscription stream: {error}", file=sys.stderr)
|
|
245
|
+
return False
|
|
246
|
+
|
|
247
|
+
def _on_closed() -> None:
|
|
248
|
+
print("Subscription stream closed")
|
|
249
|
+
|
|
250
|
+
op = self._ipc.subscribe_to_topic(
|
|
251
|
+
topic=topic,
|
|
252
|
+
on_stream_event=_on_event,
|
|
253
|
+
on_stream_error=_on_error,
|
|
254
|
+
on_stream_closed=_on_closed,
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
def unsubscribe() -> None:
|
|
258
|
+
op[1].close()
|
|
259
|
+
|
|
260
|
+
return unsubscribe
|
|
261
|
+
|
|
262
|
+
def _get_environment_variable_context_list(
|
|
263
|
+
self, var_name: str | None
|
|
264
|
+
) -> list[ContextDefinition]:
|
|
265
|
+
value = self._get_environment_variable(var_name)
|
|
266
|
+
|
|
267
|
+
try:
|
|
268
|
+
return list(map(lambda c: ContextDefinition(**c), json.loads(value)))
|
|
269
|
+
except (json.JSONDecodeError, TypeError):
|
|
270
|
+
logger.error("Failed to parse: '%s'. Shutting down.", value)
|
|
271
|
+
sys.exit(3)
|
|
272
|
+
|
|
273
|
+
def _get_environment_variable_channel_list(
|
|
274
|
+
self, var_name: str | None
|
|
275
|
+
) -> list[ChannelDefinition]:
|
|
276
|
+
value = self._get_environment_variable(var_name)
|
|
277
|
+
|
|
278
|
+
try:
|
|
279
|
+
return list(map(lambda c: ChannelDefinition(**c), json.loads(value)))
|
|
280
|
+
except (json.JSONDecodeError, TypeError):
|
|
281
|
+
logger.error("Failed to parse: '%s'. Shutting down.", value)
|
|
282
|
+
sys.exit(3)
|
|
283
|
+
|
|
284
|
+
def _get_environment_variable_feature_list(
|
|
285
|
+
self, var_name: str | None
|
|
286
|
+
) -> list[FeatureDefinition]:
|
|
287
|
+
value = self._get_environment_variable(var_name)
|
|
288
|
+
|
|
289
|
+
try:
|
|
290
|
+
return list(map(lambda c: FeatureDefinition(**c), json.loads(value)))
|
|
291
|
+
except (json.JSONDecodeError, TypeError):
|
|
292
|
+
logger.error("Failed to parse: '%s'. Shutting down.", value)
|
|
293
|
+
sys.exit(3)
|
|
294
|
+
|
|
295
|
+
def _get_environment_variable(self, var_name: str | None) -> str:
|
|
296
|
+
if var_name is None:
|
|
297
|
+
logger.error(
|
|
298
|
+
"%s environment variable is not set. Shutting down.", var_name)
|
|
299
|
+
sys.exit(1)
|
|
300
|
+
|
|
301
|
+
value = os.getenv(var_name)
|
|
302
|
+
if value is None:
|
|
303
|
+
logger.error(
|
|
304
|
+
"%s environment variable is not set. Shutting down.", var_name)
|
|
305
|
+
sys.exit(3)
|
|
306
|
+
return value
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This module provides a client for context engine to publish data to a specified topic.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from typing import TypeVar
|
|
6
|
+
import os
|
|
7
|
+
import logging
|
|
8
|
+
from .component_io import ContextDefinition, ComponentIO
|
|
9
|
+
from .typing import ContextType
|
|
10
|
+
|
|
11
|
+
T = TypeVar("T", bound=ContextType)
|
|
12
|
+
|
|
13
|
+
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "ERROR"))
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ContextEngineClient(ComponentIO[T, T]):
|
|
18
|
+
"""Client for publishing data to a context engine service.
|
|
19
|
+
|
|
20
|
+
This component is designed to be part of a larger pipeline system. It
|
|
21
|
+
automatically constructs a unique NATS topic for publishing results based on the
|
|
22
|
+
pipeline and module IDs inherited from the `ComponentIO` base class.
|
|
23
|
+
|
|
24
|
+
This class is generic, allowing it to handle and publish any specified data type.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
Generic (T): The type of data that the client will publish.
|
|
28
|
+
|
|
29
|
+
Attributes:
|
|
30
|
+
publish_topic (str): The NATS topic where data will be published, formatted as
|
|
31
|
+
'context-engine/result'.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
context_definition: list[ContextDefinition]
|
|
35
|
+
|
|
36
|
+
def __init__(self) -> None:
|
|
37
|
+
"""Initializes the ContextEngineClient and sets the publish topic."""
|
|
38
|
+
super().__init__()
|
|
39
|
+
|
|
40
|
+
self.publish_topic = "context-engine/result"
|
|
41
|
+
self._state: T | None = None
|
|
42
|
+
self.context_definition: list[ContextDefinition] = []
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def context(self) -> T | None:
|
|
46
|
+
"""Read-only access to the latest received state."""
|
|
47
|
+
return self._state
|
|
48
|
+
|
|
49
|
+
def connect(self) -> None:
|
|
50
|
+
"""Connects to the message broker and subscribes to the publish topic."""
|
|
51
|
+
super().connect()
|
|
52
|
+
self._subscribe(self.publish_topic, self._handle_incoming_message)
|
|
53
|
+
|
|
54
|
+
def publish_data(self, data: T) -> None:
|
|
55
|
+
"""Asynchronously publishes data to the predefined topic.
|
|
56
|
+
|
|
57
|
+
This method is a convenience wrapper around the parent's `publish` method,
|
|
58
|
+
sending the data to the topic specified in `self.publish_topic`.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
data (T): The data payload to publish. The type is generic and
|
|
62
|
+
should be compatible with the configured message broker's serializer.
|
|
63
|
+
|
|
64
|
+
Raises:
|
|
65
|
+
ValueError: If any keys in the data do not correspond to existing channels.
|
|
66
|
+
"""
|
|
67
|
+
self._publish(self.publish_topic, data)
|
|
68
|
+
|
|
69
|
+
def _handle_incoming_message(self, message: T) -> None:
|
|
70
|
+
"""Handle incoming messages from the context engine and update state."""
|
|
71
|
+
logger.info("Received message %s", message)
|
|
72
|
+
self._state = message
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This module provides a client for data collectors to publish data to a specified topic.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from typing import TypeVar
|
|
6
|
+
import os
|
|
7
|
+
import logging
|
|
8
|
+
from .component_io import ChannelDefinition, ComponentIO
|
|
9
|
+
from .typing import ChannelType
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "ERROR"))
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
P = TypeVar("P", bound=ChannelType)
|
|
16
|
+
|
|
17
|
+
class DataCollectorClient(ComponentIO[None, P]):
|
|
18
|
+
"""Client for publishing data to a data collector service.
|
|
19
|
+
|
|
20
|
+
This component is designed to be part of a larger pipeline system. It
|
|
21
|
+
automatically constructs a unique NATS topic for publishing results based on the
|
|
22
|
+
pipeline and module IDs inherited from the `ComponentIO` base class.
|
|
23
|
+
|
|
24
|
+
This class is generic, allowing it to handle and publish any specified data type.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
Generic (P): The type of data that the client will publish.
|
|
28
|
+
|
|
29
|
+
Attributes:
|
|
30
|
+
publish_topic (str): The NATS topic where data will be published, formatted as
|
|
31
|
+
'data-collector/{component_id}/{component_version_id}/result'.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
channels: list[ChannelDefinition]
|
|
35
|
+
|
|
36
|
+
def __init__(self) -> None:
|
|
37
|
+
"""Initializes the DataCollectorClient and sets the publish topic.
|
|
38
|
+
|
|
39
|
+
Asynchronously publishes a data payload to the pre-configured topic.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
data (T): The data to be published. The type is determined by the
|
|
43
|
+
generic type `T` provided during the class instantiation.
|
|
44
|
+
"""
|
|
45
|
+
super().__init__()
|
|
46
|
+
|
|
47
|
+
self.channels = super()._get_environment_variable_channel_list("CHANNELS")
|
|
48
|
+
|
|
49
|
+
self.publish_topic = (
|
|
50
|
+
f"data-collector/{self.component_id}/{self.component_version_id}/result"
|
|
51
|
+
)
|
|
52
|
+
self.subscribe_context_engine_topic = (
|
|
53
|
+
"context-engine/result"
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
def publish_data(self, data: P) -> None:
|
|
57
|
+
"""Asynchronously publishes data to the predefined topic.
|
|
58
|
+
|
|
59
|
+
This method is a convenience wrapper around the parent's `publish` method,
|
|
60
|
+
sending the data to the topic specified in `self.publish_topic`.
|
|
61
|
+
|
|
62
|
+
Args:
|
|
63
|
+
data (T): The data payload to publish. The type is generic and
|
|
64
|
+
should be compatible with the configured message broker's serializer.
|
|
65
|
+
|
|
66
|
+
Raises:
|
|
67
|
+
ValueError: If any keys in the data do not correspond to existing channels.
|
|
68
|
+
"""
|
|
69
|
+
keys = list(data.keys())
|
|
70
|
+
channel_names = [channel.name for channel in self.channels]
|
|
71
|
+
missing_keys = set(channel_names) - set(keys)
|
|
72
|
+
if missing_keys:
|
|
73
|
+
logger.error("Missing channels for keys: %s", missing_keys)
|
|
74
|
+
raise ValueError(f"Missing channels for keys: {missing_keys}")
|
|
75
|
+
self._publish(self.publish_topic, data)
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""Defines the client for a pre-processor component in a data processing pipeline.
|
|
2
|
+
|
|
3
|
+
This module contains the `PreProcessorClient` class, which handles the communication
|
|
4
|
+
(publishing and subscribing) for a pre-processing stage. It is designed to
|
|
5
|
+
receive data from a data collector and publish the processed results for
|
|
6
|
+
subsequent components in the pipeline.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import logging
|
|
10
|
+
import os
|
|
11
|
+
from typing import Any, TypeVar, Callable
|
|
12
|
+
from .component_io import ComponentIO
|
|
13
|
+
from .typing import ChannelType, FeatureType
|
|
14
|
+
|
|
15
|
+
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "ERROR"))
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
S = TypeVar("S", bound=ChannelType)
|
|
20
|
+
P = TypeVar("P", bound=FeatureType)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class PreProcessorClient(ComponentIO[S, P]):
|
|
24
|
+
"""A client for a pre-processing component in a data pipeline.
|
|
25
|
+
|
|
26
|
+
This class facilitates communication for a pre-processor. It subscribes to data
|
|
27
|
+
from a 'data-collector' component and publishes the processed results. The specific
|
|
28
|
+
data type it handles is defined by the generic type `T`.
|
|
29
|
+
|
|
30
|
+
It inherits from `ComponentIO` to leverage common pipeline communication logic.
|
|
31
|
+
|
|
32
|
+
Attributes:
|
|
33
|
+
publish_topic (str): The NATS topic for publishing processed data.
|
|
34
|
+
subscribe_topic (str): The NATS topic for subscribing to raw data.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
def __init__(self) -> None:
|
|
38
|
+
"""Initializes the PreProcessorClient instance.
|
|
39
|
+
|
|
40
|
+
Sets up the NATS topics for publishing and subscribing based on the
|
|
41
|
+
pipeline and module IDs inherited from the `ComponentIO` base class.
|
|
42
|
+
"""
|
|
43
|
+
super().__init__()
|
|
44
|
+
|
|
45
|
+
self.channels = super()._get_environment_variable_channel_list("CHANNELS")
|
|
46
|
+
self.features = super()._get_environment_variable_feature_list("FEATURES")
|
|
47
|
+
self.pipeline_id = super()._get_environment_variable("PIPELINE_ID")
|
|
48
|
+
self.data_collector_component_id = super()._get_environment_variable(
|
|
49
|
+
"DATA_COLLECTOR_COMPONENT_ID"
|
|
50
|
+
)
|
|
51
|
+
self.data_collector_component_version_id = super()._get_environment_variable(
|
|
52
|
+
"DATA_COLLECTOR_COMPONENT_VERSION_ID"
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
self.publish_topic = (
|
|
56
|
+
f"pre-processor/{self.pipeline_id}/{self.component_id}/{self.component_version_id}/result"
|
|
57
|
+
)
|
|
58
|
+
self.subscribe_topic = (
|
|
59
|
+
f"data-collector/{self.data_collector_component_id}/{self.data_collector_component_version_id}/result"
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
def publish_data(self, data: P) -> None:
|
|
63
|
+
"""Asynchronously publishes processed data.
|
|
64
|
+
Args:
|
|
65
|
+
data (P): The processed data to be published to the pre-processor's
|
|
66
|
+
result topic.
|
|
67
|
+
"""
|
|
68
|
+
keys = list(data.keys())
|
|
69
|
+
feature_names = [feature.name for feature in self.features]
|
|
70
|
+
missing_keys = set(feature_names) - set(keys)
|
|
71
|
+
if missing_keys:
|
|
72
|
+
logger.error("Missing features for keys: %s", missing_keys)
|
|
73
|
+
raise ValueError(f"Missing features for keys: {missing_keys}")
|
|
74
|
+
|
|
75
|
+
for feature in self.features:
|
|
76
|
+
shape = self._shape_from_object(data[feature.name])
|
|
77
|
+
if list(shape) != feature.shape:
|
|
78
|
+
logger.error(
|
|
79
|
+
"Feature '%s' has incorrect shape. Expected %s, got %s",
|
|
80
|
+
feature.name,
|
|
81
|
+
feature.shape,
|
|
82
|
+
list(shape),
|
|
83
|
+
)
|
|
84
|
+
error_message = (
|
|
85
|
+
f"Feature '{feature.name}' has incorrect shape. "
|
|
86
|
+
f"Expected {feature.shape}, got {list(shape)}"
|
|
87
|
+
)
|
|
88
|
+
raise ValueError(error_message)
|
|
89
|
+
self._validate_list_type(data[feature.name], (float, int))
|
|
90
|
+
self._publish(self.publish_topic, data)
|
|
91
|
+
|
|
92
|
+
def subscribe(self, handler: Callable[[S], None]) -> Callable[[], None]:
|
|
93
|
+
"""Subscribes to raw data from a data collector.
|
|
94
|
+
|
|
95
|
+
Sets up a subscription to the corresponding data collector's result topic.
|
|
96
|
+
When a message is received, the provided handler is called with the data.
|
|
97
|
+
|
|
98
|
+
Args:
|
|
99
|
+
handler (Callable[[S], None]): A callback function to be invoked
|
|
100
|
+
with the received data.
|
|
101
|
+
|
|
102
|
+
Returns:
|
|
103
|
+
Callable[[], None]: A function that can be called to unsubscribe
|
|
104
|
+
from the topic.
|
|
105
|
+
"""
|
|
106
|
+
return self._subscribe(self.subscribe_topic, handler)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _shape_from_object(self, obj: list) -> tuple:
|
|
110
|
+
|
|
111
|
+
shape = []
|
|
112
|
+
def _shape_from_object_r(element: list, axis: int):
|
|
113
|
+
try:
|
|
114
|
+
for i, e in enumerate(element):
|
|
115
|
+
_shape_from_object_r(e, axis+1)
|
|
116
|
+
while len(shape) <= axis:
|
|
117
|
+
shape.append(0)
|
|
118
|
+
l = i + 1
|
|
119
|
+
s = shape[axis]
|
|
120
|
+
if l > s:
|
|
121
|
+
shape[axis] = l
|
|
122
|
+
except TypeError:
|
|
123
|
+
pass
|
|
124
|
+
|
|
125
|
+
_shape_from_object_r( obj, 0)
|
|
126
|
+
return tuple(shape)
|
|
127
|
+
|
|
128
|
+
def _validate_list_type(self, obj: Any, expected_type: tuple[type, ...]) -> None:
|
|
129
|
+
if not isinstance(obj, list):
|
|
130
|
+
raise TypeError(f"Expected list, got {type(obj)}")
|
|
131
|
+
for item in obj:
|
|
132
|
+
if isinstance(item, list):
|
|
133
|
+
self._validate_list_type(item, expected_type)
|
|
134
|
+
else:
|
|
135
|
+
if not isinstance(item, expected_type):
|
|
136
|
+
raise TypeError(f"Expected list of {expected_type}, got {type(item)}")
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ai-box-lib
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Python library for AWS Greengrass integration
|
|
5
|
+
Author-email: Your Name <your.email@example.com>
|
|
6
|
+
License: SOFTWARE LICENSE AGREEMENT
|
|
7
|
+
|
|
8
|
+
This Software Library ("Library") is licensed, not sold, to you by Synadia ("Licensor"). By using, copying, or distributing this Library, you agree to the following terms:
|
|
9
|
+
|
|
10
|
+
1. Authorized Use
|
|
11
|
+
- You may use this Library only if you have purchased a compatible device from Synadia.
|
|
12
|
+
- You may use the Library solely to develop software that runs on the purchased device.
|
|
13
|
+
- Any other use, including use on non-Synadia devices, is strictly prohibited.
|
|
14
|
+
|
|
15
|
+
2. Restrictions
|
|
16
|
+
- You may not distribute, sublicense, or otherwise make the Library available to any third party except as part of software running exclusively on the purchased device.
|
|
17
|
+
- You may not use the Library for any commercial purpose other than developing software for the purchased device.
|
|
18
|
+
- Reverse engineering, modification, or derivative works are permitted only for the purpose of developing software for the purchased device.
|
|
19
|
+
|
|
20
|
+
3. Ownership
|
|
21
|
+
- The Library remains the property of Synadia. No ownership rights are transferred.
|
|
22
|
+
|
|
23
|
+
4. Termination
|
|
24
|
+
- This license is automatically terminated if you breach any of its terms. Upon termination, you must destroy all copies of the Library.
|
|
25
|
+
|
|
26
|
+
5. Disclaimer
|
|
27
|
+
- The Library is provided "AS IS" without warranty of any kind. Synadia disclaims all warranties, express or implied, including but not limited to merchantability and fitness for a particular purpose.
|
|
28
|
+
|
|
29
|
+
6. Limitation of Liability
|
|
30
|
+
- In no event shall Synadia be liable for any damages arising from the use or inability to use the Library.
|
|
31
|
+
|
|
32
|
+
For questions or licensing inquiries, contact Synadia.
|
|
33
|
+
|
|
34
|
+
Requires-Python: >=3.11
|
|
35
|
+
Description-Content-Type: text/markdown
|
|
36
|
+
License-File: LICENSE
|
|
37
|
+
Requires-Dist: greengrasssdk
|
|
38
|
+
Requires-Dist: awsiotsdk
|
|
39
|
+
Dynamic: license-file
|
|
40
|
+
|
|
41
|
+
# AI Box Library
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
## Development
|
|
45
|
+
|
|
46
|
+
Use the `./src/ai-box-lib` to write all the library code which can be used by users.
|
|
47
|
+
|
|
48
|
+
In order to test the library locally you can run the following commands:
|
|
49
|
+
1.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/ai_box_lib/__init__.py
|
|
5
|
+
src/ai_box_lib/component_io.py
|
|
6
|
+
src/ai_box_lib/context_engine_client.py
|
|
7
|
+
src/ai_box_lib/data_collector_client.py
|
|
8
|
+
src/ai_box_lib/pre_processor_client.py
|
|
9
|
+
src/ai_box_lib/typing.py
|
|
10
|
+
src/ai_box_lib.egg-info/PKG-INFO
|
|
11
|
+
src/ai_box_lib.egg-info/SOURCES.txt
|
|
12
|
+
src/ai_box_lib.egg-info/dependency_links.txt
|
|
13
|
+
src/ai_box_lib.egg-info/requires.txt
|
|
14
|
+
src/ai_box_lib.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
ai_box_lib
|