cgse-common 0.15.0__py3-none-any.whl → 0.16.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cgse_common/cgse.py +1 -1
- {cgse_common-0.15.0.dist-info → cgse_common-0.16.0.dist-info}/METADATA +1 -1
- {cgse_common-0.15.0.dist-info → cgse_common-0.16.0.dist-info}/RECORD +10 -9
- egse/connect.py +369 -0
- egse/device.py +73 -0
- egse/env.py +1 -1
- egse/scpi.py +58 -9
- egse/system.py +51 -0
- {cgse_common-0.15.0.dist-info → cgse_common-0.16.0.dist-info}/WHEEL +0 -0
- {cgse_common-0.15.0.dist-info → cgse_common-0.16.0.dist-info}/entry_points.txt +0 -0
cgse_common/cgse.py
CHANGED
|
@@ -72,7 +72,7 @@ class SortedCommandGroup(TyperGroup):
|
|
|
72
72
|
commands = super().list_commands(ctx)
|
|
73
73
|
|
|
74
74
|
# Define priority commands in specific order
|
|
75
|
-
priority_commands = ["init", "version", "show", "top", "core", "
|
|
75
|
+
priority_commands = ["init", "version", "show", "top", "core", "reg", "not", "log", "cm", "sm", "pm"]
|
|
76
76
|
|
|
77
77
|
# Custom sort:
|
|
78
78
|
# First the priority commands in the given order (their index)
|
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
cgse_common/__init__.py,sha256=wTYOpVomEeDFFuqt4Ss9ROSAIa48UUnYCSafdEOx-CU,129
|
|
2
|
-
cgse_common/cgse.py,sha256=
|
|
2
|
+
cgse_common/cgse.py,sha256=4ZCm5UfaXld0RGI-Mjnm6dSAqfWCPX1qz1pew46J-1s,6590
|
|
3
3
|
cgse_common/settings.yaml,sha256=PS8HOoxbhwVoQ7zzDtZyhx25RZB6SGq3uXNuJTgtRIw,443
|
|
4
4
|
egse/bits.py,sha256=cg6diLN2IwNtnrlZyLgXPfDf9ttcHzS2A7Tjwlwc-lQ,12498
|
|
5
5
|
egse/calibration.py,sha256=a5JDaXTC6fMwQ1M-qrwNO31Ass-yYSXxDQUK_PPsZg4,8818
|
|
6
6
|
egse/config.py,sha256=qNW3uvwuEV9VSjiaoQfM_svBYfz4Kwxd1-jv1eTGqyw,9530
|
|
7
|
+
egse/connect.py,sha256=Kv6to6-4ihFcrAy9ErqAGdbhLHUAnqUgYfnBKMfgefU,15053
|
|
7
8
|
egse/counter.py,sha256=7UwBeTAu213xdNdGAOYpUWNQ4jD4yVM1bOG10Ax4UFs,5097
|
|
8
9
|
egse/decorators.py,sha256=B-zRa1WdLO71zqS5M27JBglcThYPho7seYfa4HOGj5c,27171
|
|
9
|
-
egse/device.py,sha256
|
|
10
|
+
egse/device.py,sha256=nn2HkN1KIHAmo37WZcqig-p2mQz1LgqpIfj1wPrUTLc,13240
|
|
10
11
|
egse/dicts.py,sha256=dUAq7PTPvs73OrZb2Fh3loxvYv4ifUiK6bBcgrFU77Y,3972
|
|
11
|
-
egse/env.py,sha256=
|
|
12
|
+
egse/env.py,sha256=mV5SHtMAr_FRyzrPWjIu0vbCJzJt1n_Tc6tIpwoEffA,28366
|
|
12
13
|
egse/exceptions.py,sha256=QB3MZRJizecWOj1cPbvG0UcIqFn7NRJ6rw1xtdNSFxw,1225
|
|
13
14
|
egse/heartbeat.py,sha256=xt5mePu9Zr9fLAhN1MLq1Z7aCOKtNIhRVCAmWhtNwP8,3039
|
|
14
15
|
egse/hk.py,sha256=AumSpB8SYXes75CB2iiKXfLkMK5IkVDHITFKrf8IT6g,32010
|
|
@@ -25,21 +26,21 @@ egse/ratelimit.py,sha256=JdJxD6UIi9LYngKEsG9zh8bTE9r_56D4EZCnp_fkrI0,9161
|
|
|
25
26
|
egse/reload.py,sha256=PzOE0m1tmcNcQPVFH8orMe_cMoQIIiH9Gw2anpQTC40,4717
|
|
26
27
|
egse/resource.py,sha256=kzNI6kJOE6Jd5QKJs2MkVAycUpwpOTLi1qydh3NSRng,15345
|
|
27
28
|
egse/response.py,sha256=F04uqOYv1ClpHgDLYZlKTuOCSldHs5TezI_4x6zf2Fw,2717
|
|
28
|
-
egse/scpi.py,sha256=
|
|
29
|
+
egse/scpi.py,sha256=WJ73EaLgRUV6ah1V41l0L7AXI-Dc6Jct7hPHlbbCIcg,15461
|
|
29
30
|
egse/settings.py,sha256=YrRsMUn_IpOVnhTqUGREQUjMw8-AQ6aUBulQiij9MwY,15486
|
|
30
31
|
egse/settings.yaml,sha256=mz9O2QqmiptezsMvxJRLhnC1ROwIHENX0nbnhMaXUpE,190
|
|
31
32
|
egse/setup.py,sha256=1k-5CjzY3_tZ6XCitNOIyZEWyAUC8LqGcnmdKS68vYw,33945
|
|
32
33
|
egse/signal.py,sha256=zW_36xm-RpzwuCu6x30dTvjkfnuyRebKKpl69Yk1QSc,7990
|
|
33
34
|
egse/socketdevice.py,sha256=R8XwYHTH3lFhFngfsGbi_L7bTnTLHxMTEKIF7gmm5rc,7465
|
|
34
35
|
egse/state.py,sha256=HdU2MFOlYRbawYRZmizV6Y8MgnZrUF0bx4fXaYU-M_s,3023
|
|
35
|
-
egse/system.py,sha256=
|
|
36
|
+
egse/system.py,sha256=akT5uTSzCwdWhJBXeBl4V8fes8iirbbYRN1G0-YQCmo,74784
|
|
36
37
|
egse/task.py,sha256=ODSLE05f31CgWsSVcVFFq1WYUZrJMb1LioPTx6VM824,2804
|
|
37
38
|
egse/version.py,sha256=e9GvelUZ9mfCDlRju4MWEJeMHJW9kUzK6SKzJpyj91s,6156
|
|
38
39
|
egse/zmq_ser.py,sha256=d2lETLkLUll_F1Phc1pI7MEeA41uX7YZ8lhSFmBQZVw,3022
|
|
39
40
|
egse/plugins/metrics/duckdb.py,sha256=E2eeNo3I7ajRuByodaYiPNvC0Zwyc7hsIlhr1W_eXdo,16148
|
|
40
41
|
egse/plugins/metrics/influxdb.py,sha256=ecxjA_csYwf8RW3sXjiQxZHREfyrfStH1HA_rAs1AA8,6690
|
|
41
42
|
egse/plugins/metrics/timescaledb.py,sha256=Ug0NWDV1Ky2VeFY6tDZL9xg6AFgnAEh2F_llVPnlRBA,21191
|
|
42
|
-
cgse_common-0.
|
|
43
|
-
cgse_common-0.
|
|
44
|
-
cgse_common-0.
|
|
45
|
-
cgse_common-0.
|
|
43
|
+
cgse_common-0.16.0.dist-info/METADATA,sha256=P4cA-5qH_HqxVpqDOcL5jsq9MEt3wChkYMeLQ9Zi8_0,3032
|
|
44
|
+
cgse_common-0.16.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
45
|
+
cgse_common-0.16.0.dist-info/entry_points.txt,sha256=erQovXd1bGzsngB0_sfY7IYRNwHIhwq3K8fmQvGS12o,198
|
|
46
|
+
cgse_common-0.16.0.dist-info/RECORD,,
|
egse/connect.py
ADDED
|
@@ -0,0 +1,369 @@
|
|
|
1
|
+
import random
|
|
2
|
+
import threading
|
|
3
|
+
import time
|
|
4
|
+
from enum import Enum
|
|
5
|
+
|
|
6
|
+
from egse.log import logging
|
|
7
|
+
|
|
8
|
+
logger = logging.getLogger("egse.connect")
|
|
9
|
+
|
|
10
|
+
# random.seed(time.monotonic()) # uncomment for testing only, main application should set a seed.
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ConnectionState(Enum):
|
|
14
|
+
DISCONNECTED = "disconnected"
|
|
15
|
+
CONNECTING = "connecting"
|
|
16
|
+
CONNECTED = "connected"
|
|
17
|
+
CIRCUIT_OPEN = "circuit_open" # Temporarily stopped trying
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class BackoffStrategy(Enum):
|
|
21
|
+
"""
|
|
22
|
+
Specifies the strategy for increasing the delay between retry attempts
|
|
23
|
+
in backoff algorithms to reduce load and avoid overwhelming services.
|
|
24
|
+
|
|
25
|
+
Strategies:
|
|
26
|
+
EXPONENTIAL:
|
|
27
|
+
The delay doubles with each retry attempt (e.g., 1s, 2s, 4s, 8s).
|
|
28
|
+
This is the most widely used approach because it quickly reduces load on struggling systems.
|
|
29
|
+
LINEAR:
|
|
30
|
+
The delay increases by a fixed amount each time (e.g., 1s, 2s, 3s, 4s).
|
|
31
|
+
This provides a more gradual reduction in request rate.
|
|
32
|
+
FIXED:
|
|
33
|
+
Uses the same delay between all retry attempts.
|
|
34
|
+
Simple but less adaptive to system conditions.
|
|
35
|
+
|
|
36
|
+
References:
|
|
37
|
+
- AWS Architecture Blog: Exponential Backoff And Jitter
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
EXPONENTIAL = "exponential"
|
|
41
|
+
"""The delay doubles with each retry attempt (e.g., 1s, 2s, 4s, 8s).
|
|
42
|
+
This is the most widely used approach because it quickly reduces load on struggling systems."""
|
|
43
|
+
LINEAR = "linear"
|
|
44
|
+
"""The delay increases by a fixed amount each time (e.g., 1s, 2s, 3s, 4s).
|
|
45
|
+
This provides a more gradual reduction in request rate."""
|
|
46
|
+
FIXED = "fixed"
|
|
47
|
+
"""Uses the same delay between all retry attempts. Simple but less adaptive to system conditions."""
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class JitterStrategy(Enum):
|
|
51
|
+
"""
|
|
52
|
+
Specifies the strategy for applying jitter (randomization) to retry intervals
|
|
53
|
+
in backoff algorithms to avoid synchronized retries and reduce load spikes.
|
|
54
|
+
|
|
55
|
+
Strategies:
|
|
56
|
+
NONE:
|
|
57
|
+
No jitter is applied. The retry interval is deterministic.
|
|
58
|
+
FULL:
|
|
59
|
+
Applies full jitter by selecting a random value uniformly between 0 and the calculated interval.
|
|
60
|
+
This maximizes randomness but can result in very short delays.
|
|
61
|
+
EQUAL:
|
|
62
|
+
Applies "equal jitter" as described in the AWS Architecture Blog.
|
|
63
|
+
The interval is randomized within [interval/2, interval], ensuring a minimum delay of half the interval.
|
|
64
|
+
Note: This is not the same as "a jitter of 50% around interval" (which would be [0.5 * interval, 1.5 * interval]).
|
|
65
|
+
PERCENT_10:
|
|
66
|
+
Applies a jitter of ±10% around the base interval, resulting in a random interval within [0.9 * interval, 1.1 * interval].
|
|
67
|
+
|
|
68
|
+
References:
|
|
69
|
+
- AWS Architecture Blog: Exponential Backoff And Jitter
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
NONE = "none"
|
|
73
|
+
"""No jitter is applied to the backoff."""
|
|
74
|
+
FULL = "full"
|
|
75
|
+
"""Maximum distribution but can be too random with very short intervals."""
|
|
76
|
+
EQUAL = "equal"
|
|
77
|
+
"""Best balance, maintains backoff properties while preventing synchronization."""
|
|
78
|
+
PERCENT_10 = "10%"
|
|
79
|
+
"""Add a jitter of 10% around the base interval."""
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def calculate_retry_interval(
|
|
83
|
+
attempt_number,
|
|
84
|
+
base_interval,
|
|
85
|
+
max_interval,
|
|
86
|
+
backoff_strategy: BackoffStrategy = BackoffStrategy.EXPONENTIAL,
|
|
87
|
+
jitter_strategy: JitterStrategy = JitterStrategy.EQUAL,
|
|
88
|
+
):
|
|
89
|
+
"""
|
|
90
|
+
Calculates the next retry interval based on the given backoff and jitter strategies.
|
|
91
|
+
|
|
92
|
+
Args:
|
|
93
|
+
attempt_number (int): The current retry attempt (starting from 0).
|
|
94
|
+
base_interval (float): The initial interval in seconds.
|
|
95
|
+
max_interval (float): The maximum allowed interval in seconds.
|
|
96
|
+
backoff_strategy (BackoffStrategy): Strategy for increasing the delay (exponential, linear, or fixed).
|
|
97
|
+
jitter_strategy (JitterStrategy): Strategy for randomizing the delay to avoid synchronization.
|
|
98
|
+
|
|
99
|
+
Returns:
|
|
100
|
+
float: The computed retry interval in seconds.
|
|
101
|
+
|
|
102
|
+
Notes:
|
|
103
|
+
- See the docstrings for BackoffStrategy and JitterStrategy for details on each strategy.
|
|
104
|
+
- Based on best practices from the AWS Architecture Blog: Exponential Backoff And Jitter.
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
if backoff_strategy == BackoffStrategy.EXPONENTIAL:
|
|
108
|
+
interval = min(base_interval * (2**attempt_number), max_interval)
|
|
109
|
+
elif backoff_strategy == BackoffStrategy.LINEAR:
|
|
110
|
+
interval = min(base_interval + attempt_number, max_interval)
|
|
111
|
+
else:
|
|
112
|
+
interval = base_interval
|
|
113
|
+
|
|
114
|
+
if jitter_strategy == JitterStrategy.NONE:
|
|
115
|
+
return interval
|
|
116
|
+
elif jitter_strategy == JitterStrategy.FULL:
|
|
117
|
+
return random.uniform(0, interval)
|
|
118
|
+
elif jitter_strategy == JitterStrategy.EQUAL:
|
|
119
|
+
return interval / 2 + random.uniform(0, interval / 2)
|
|
120
|
+
elif jitter_strategy == JitterStrategy.PERCENT_10:
|
|
121
|
+
jitter_amount = interval * 0.1
|
|
122
|
+
return interval + random.uniform(-jitter_amount, jitter_amount)
|
|
123
|
+
|
|
124
|
+
return interval
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
class AsyncServiceConnector:
|
|
128
|
+
"""
|
|
129
|
+
Asynchronous base class for robust service connection management with retry, backoff, and circuit breaker logic.
|
|
130
|
+
|
|
131
|
+
This class is intended to be subclassed for managing persistent connections to external services
|
|
132
|
+
(such as devices, databases, or remote APIs) that may be unreliable or temporarily unavailable.
|
|
133
|
+
|
|
134
|
+
Features:
|
|
135
|
+
- Automatic retry with configurable backoff and jitter strategies.
|
|
136
|
+
- Circuit breaker to prevent repeated connection attempts after multiple failures.
|
|
137
|
+
- Connection state tracking (disconnected, connecting, connected, circuit open).
|
|
138
|
+
|
|
139
|
+
Usage:
|
|
140
|
+
1. Subclass `AsyncServiceConnector` and override the `connect_to_service()` coroutine with your
|
|
141
|
+
actual connection logic. Optionally, override `health_check()` for custom health verification.
|
|
142
|
+
2. Store the actual connection object (e.g., socket, transport) as an instance attribute in your subclass.
|
|
143
|
+
3. Use `attempt_connection()` to initiate connection attempts; it will handle retries and backoff automatically.
|
|
144
|
+
4. Use `is_connected()` to check connection status.
|
|
145
|
+
|
|
146
|
+
Example:
|
|
147
|
+
class MyConnector(AsyncServiceConnector):
|
|
148
|
+
async def connect_to_service(self):
|
|
149
|
+
self.connection = await create_socket()
|
|
150
|
+
return self.connection is not None
|
|
151
|
+
|
|
152
|
+
def get_connection(self):
|
|
153
|
+
return self.connection
|
|
154
|
+
|
|
155
|
+
Note:
|
|
156
|
+
The base class does not manage or expose the underlying connection object.
|
|
157
|
+
Your subclass should provide a method or property to access it as needed.
|
|
158
|
+
"""
|
|
159
|
+
|
|
160
|
+
def __init__(self, service_name: str):
|
|
161
|
+
self.state = ConnectionState.DISCONNECTED
|
|
162
|
+
self.last_attempt = 0
|
|
163
|
+
self.base_interval = 1
|
|
164
|
+
self.retry_interval = 1 # Start with 1 second
|
|
165
|
+
self.max_retry_interval = 300 # Max 5 minutes
|
|
166
|
+
self.failure_count = 0
|
|
167
|
+
self.max_failures_before_circuit_break = 5
|
|
168
|
+
self.circuit_break_duration = 60 # 1 minute
|
|
169
|
+
self.circuit_opened_at = None
|
|
170
|
+
|
|
171
|
+
self.service_name = service_name
|
|
172
|
+
|
|
173
|
+
async def connect_to_service(self) -> bool:
|
|
174
|
+
logger.warning(
|
|
175
|
+
f"The connect_to_service() method is not implemented for {self.service_name}, connection will always fail."
|
|
176
|
+
)
|
|
177
|
+
return False
|
|
178
|
+
|
|
179
|
+
async def health_check(self) -> bool:
|
|
180
|
+
logger.warning(
|
|
181
|
+
f"The health_check() method is not implemented for {self.service_name}, check will always return false."
|
|
182
|
+
)
|
|
183
|
+
return False
|
|
184
|
+
|
|
185
|
+
def should_attempt_connection(self) -> bool:
|
|
186
|
+
"""Return True if we should attempt a new connection."""
|
|
187
|
+
now = time.monotonic()
|
|
188
|
+
|
|
189
|
+
# If circuit is open, check if we should close it
|
|
190
|
+
if self.state == ConnectionState.CIRCUIT_OPEN:
|
|
191
|
+
if now - self.circuit_opened_at > self.circuit_break_duration:
|
|
192
|
+
self.state = ConnectionState.DISCONNECTED
|
|
193
|
+
self.failure_count = 0
|
|
194
|
+
self.retry_interval = 1
|
|
195
|
+
return True
|
|
196
|
+
return False
|
|
197
|
+
|
|
198
|
+
# Regular backoff logic
|
|
199
|
+
return now - self.last_attempt >= self.retry_interval
|
|
200
|
+
|
|
201
|
+
async def attempt_connection(self):
|
|
202
|
+
"""Try to connect to the service.
|
|
203
|
+
|
|
204
|
+
This will execute the callable argument `connect_to_service` that was passed
|
|
205
|
+
into the constructor. That function shall return True when the connection
|
|
206
|
+
succeeded, False otherwise.
|
|
207
|
+
"""
|
|
208
|
+
if not self.should_attempt_connection():
|
|
209
|
+
return
|
|
210
|
+
|
|
211
|
+
self.state = ConnectionState.CONNECTING
|
|
212
|
+
self.last_attempt = time.monotonic()
|
|
213
|
+
|
|
214
|
+
try:
|
|
215
|
+
success = await self.connect_to_service()
|
|
216
|
+
|
|
217
|
+
if success:
|
|
218
|
+
self.state = ConnectionState.CONNECTED
|
|
219
|
+
self.failure_count = 0
|
|
220
|
+
self.retry_interval = 1 # Reset backoff
|
|
221
|
+
logger.info(f"Successfully connected to service {self.service_name}")
|
|
222
|
+
else:
|
|
223
|
+
# warning should have been logged by the connect_to_service() callable.
|
|
224
|
+
self.handle_connection_failure()
|
|
225
|
+
|
|
226
|
+
except Exception as exc:
|
|
227
|
+
logger.warning(f"Failed to connect to service {self.service_name}: {exc}")
|
|
228
|
+
self.handle_connection_failure()
|
|
229
|
+
|
|
230
|
+
def handle_connection_failure(self):
|
|
231
|
+
self.failure_count += 1
|
|
232
|
+
|
|
233
|
+
# Open circuit breaker if too many failures
|
|
234
|
+
if self.failure_count >= self.max_failures_before_circuit_break:
|
|
235
|
+
self.state = ConnectionState.CIRCUIT_OPEN
|
|
236
|
+
self.circuit_opened_at = time.time()
|
|
237
|
+
logger.warning(
|
|
238
|
+
f"Circuit breaker opened for service {self.service_name} after {self.failure_count} failures"
|
|
239
|
+
)
|
|
240
|
+
else:
|
|
241
|
+
self.state = ConnectionState.DISCONNECTED
|
|
242
|
+
self.retry_interval = calculate_retry_interval(
|
|
243
|
+
self.failure_count,
|
|
244
|
+
self.base_interval,
|
|
245
|
+
self.max_retry_interval,
|
|
246
|
+
BackoffStrategy.EXPONENTIAL,
|
|
247
|
+
JitterStrategy.EQUAL,
|
|
248
|
+
)
|
|
249
|
+
logger.debug(f"retry_interval={self.retry_interval}")
|
|
250
|
+
|
|
251
|
+
def is_connected(self) -> bool:
|
|
252
|
+
return self.state == ConnectionState.CONNECTED
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
class ServiceConnector:
|
|
256
|
+
"""
|
|
257
|
+
Synchronous base class for robust service connection management with retry, backoff, and circuit breaker logic.
|
|
258
|
+
|
|
259
|
+
This class is intended to be subclassed for managing persistent connections to external services
|
|
260
|
+
(such as devices, databases, or remote APIs) that may be unreliable or temporarily unavailable.
|
|
261
|
+
|
|
262
|
+
Features:
|
|
263
|
+
- Automatic retry with configurable backoff and jitter strategies.
|
|
264
|
+
- Circuit breaker to prevent repeated connection attempts after multiple failures.
|
|
265
|
+
- Connection state tracking (disconnected, connecting, connected, circuit open).
|
|
266
|
+
- Thread-safe operation using a lock for all state changes.
|
|
267
|
+
|
|
268
|
+
Usage:
|
|
269
|
+
1. Subclass `ServiceConnector` and override the `connect_to_service()` method with your
|
|
270
|
+
actual connection logic. Optionally, override `health_check()` for custom health verification.
|
|
271
|
+
2. Store the actual connection object (e.g., socket, transport) as an instance attribute in your subclass.
|
|
272
|
+
3. Use `attempt_connection()` to initiate connection attempts; it will handle retries and backoff automatically.
|
|
273
|
+
4. Use `is_connected()` to check connection status.
|
|
274
|
+
|
|
275
|
+
Example:
|
|
276
|
+
class MyConnector(ServiceConnector):
|
|
277
|
+
def connect_to_service(self):
|
|
278
|
+
self.connection = create_socket()
|
|
279
|
+
return self.connection is not None
|
|
280
|
+
|
|
281
|
+
def get_connection(self):
|
|
282
|
+
return self.connection
|
|
283
|
+
|
|
284
|
+
Note:
|
|
285
|
+
The base class does not manage or expose the underlying connection object.
|
|
286
|
+
Your subclass should provide a method or property to access it as needed.
|
|
287
|
+
"""
|
|
288
|
+
|
|
289
|
+
def __init__(self, service_name: str):
|
|
290
|
+
self.state = ConnectionState.DISCONNECTED
|
|
291
|
+
self.last_attempt = 0
|
|
292
|
+
self.base_interval = 1
|
|
293
|
+
self.retry_interval = 1
|
|
294
|
+
self.max_retry_interval = 300
|
|
295
|
+
self.failure_count = 0
|
|
296
|
+
self.max_failures_before_circuit_break = 5
|
|
297
|
+
self.circuit_break_duration = 60
|
|
298
|
+
self.circuit_opened_at = None
|
|
299
|
+
self.service_name = service_name
|
|
300
|
+
self._lock = threading.RLock()
|
|
301
|
+
|
|
302
|
+
def connect_to_service(self) -> bool:
|
|
303
|
+
logger.warning(
|
|
304
|
+
f"The connect_to_service() method is not implemented for {self.service_name}, connection will always fail."
|
|
305
|
+
)
|
|
306
|
+
return False
|
|
307
|
+
|
|
308
|
+
def health_check(self) -> bool:
|
|
309
|
+
logger.warning(
|
|
310
|
+
f"The health_check() method is not implemented for {self.service_name}, check will always return false."
|
|
311
|
+
)
|
|
312
|
+
return False
|
|
313
|
+
|
|
314
|
+
def should_attempt_connection(self) -> bool:
|
|
315
|
+
now = time.monotonic()
|
|
316
|
+
with self._lock:
|
|
317
|
+
if self.state == ConnectionState.CIRCUIT_OPEN:
|
|
318
|
+
if now - self.circuit_opened_at > self.circuit_break_duration:
|
|
319
|
+
self.state = ConnectionState.DISCONNECTED
|
|
320
|
+
self.failure_count = 0
|
|
321
|
+
self.retry_interval = 1
|
|
322
|
+
return True
|
|
323
|
+
return False
|
|
324
|
+
return now - self.last_attempt >= self.retry_interval
|
|
325
|
+
|
|
326
|
+
def attempt_connection(self):
|
|
327
|
+
with self._lock:
|
|
328
|
+
if not self.should_attempt_connection():
|
|
329
|
+
return
|
|
330
|
+
self.state = ConnectionState.CONNECTING
|
|
331
|
+
self.last_attempt = time.monotonic()
|
|
332
|
+
|
|
333
|
+
try:
|
|
334
|
+
success = self.connect_to_service()
|
|
335
|
+
with self._lock:
|
|
336
|
+
if success:
|
|
337
|
+
self.state = ConnectionState.CONNECTED
|
|
338
|
+
self.failure_count = 0
|
|
339
|
+
self.retry_interval = 1
|
|
340
|
+
logger.info(f"Successfully connected to service {self.service_name}")
|
|
341
|
+
else:
|
|
342
|
+
self.handle_connection_failure()
|
|
343
|
+
except Exception as exc:
|
|
344
|
+
logger.error(f"Failed to connect to service {self.service_name}: {exc}")
|
|
345
|
+
with self._lock:
|
|
346
|
+
self.handle_connection_failure()
|
|
347
|
+
|
|
348
|
+
def handle_connection_failure(self):
|
|
349
|
+
self.failure_count += 1
|
|
350
|
+
if self.failure_count >= self.max_failures_before_circuit_break:
|
|
351
|
+
self.state = ConnectionState.CIRCUIT_OPEN
|
|
352
|
+
self.circuit_opened_at = time.monotonic()
|
|
353
|
+
logger.warning(
|
|
354
|
+
f"Circuit breaker opened for service {self.service_name} after {self.failure_count} failures"
|
|
355
|
+
)
|
|
356
|
+
else:
|
|
357
|
+
self.state = ConnectionState.DISCONNECTED
|
|
358
|
+
self.retry_interval = calculate_retry_interval(
|
|
359
|
+
self.failure_count,
|
|
360
|
+
self.base_interval,
|
|
361
|
+
self.max_retry_interval,
|
|
362
|
+
BackoffStrategy.EXPONENTIAL,
|
|
363
|
+
JitterStrategy.EQUAL,
|
|
364
|
+
)
|
|
365
|
+
logger.debug(f"retry_interval={self.retry_interval}")
|
|
366
|
+
|
|
367
|
+
def is_connected(self) -> bool:
|
|
368
|
+
with self._lock:
|
|
369
|
+
return self.state == ConnectionState.CONNECTED
|
egse/device.py
CHANGED
|
@@ -236,6 +236,9 @@ class DeviceTransport:
|
|
|
236
236
|
|
|
237
237
|
raise NotImplementedError
|
|
238
238
|
|
|
239
|
+
def read_string(self, encoding="utf-8") -> str:
|
|
240
|
+
return self.read().decode(encoding).strip()
|
|
241
|
+
|
|
239
242
|
def trans(self, command: str) -> bytes:
|
|
240
243
|
"""
|
|
241
244
|
Send a single command to the device controller and block until a response from the
|
|
@@ -330,6 +333,76 @@ class AsyncDeviceTransport:
|
|
|
330
333
|
return await self.trans(command)
|
|
331
334
|
|
|
332
335
|
|
|
336
|
+
class AsyncDeviceConnectionInterface(DeviceConnectionObservable):
|
|
337
|
+
"""Generic connection interface for all Device classes and Controllers.
|
|
338
|
+
|
|
339
|
+
This interface shall be implemented in the Controllers that directly connect to the
|
|
340
|
+
hardware, but also in the simulators to guarantee an identical interface as the controllers.
|
|
341
|
+
|
|
342
|
+
This interface will be implemented in the Proxy classes through the
|
|
343
|
+
YAML definitions. Therefore, the YAML files shall define at least
|
|
344
|
+
the following commands: `connect`, `disconnect`, `reconnect`, `is_connected`.
|
|
345
|
+
"""
|
|
346
|
+
|
|
347
|
+
def __init__(self):
|
|
348
|
+
super().__init__()
|
|
349
|
+
|
|
350
|
+
def __enter__(self):
|
|
351
|
+
self.connect()
|
|
352
|
+
return self
|
|
353
|
+
|
|
354
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
355
|
+
self.disconnect()
|
|
356
|
+
|
|
357
|
+
async def connect(self) -> None:
|
|
358
|
+
"""Connect to the device controller.
|
|
359
|
+
|
|
360
|
+
Raises:
|
|
361
|
+
ConnectionError: when the connection can not be opened.
|
|
362
|
+
"""
|
|
363
|
+
|
|
364
|
+
raise NotImplementedError
|
|
365
|
+
|
|
366
|
+
async def disconnect(self) -> None:
|
|
367
|
+
"""Disconnect from the device controller.
|
|
368
|
+
|
|
369
|
+
Raises:
|
|
370
|
+
ConnectionError: when the connection can not be closed.
|
|
371
|
+
"""
|
|
372
|
+
raise NotImplementedError
|
|
373
|
+
|
|
374
|
+
async def reconnect(self):
|
|
375
|
+
"""Reconnect the device controller.
|
|
376
|
+
|
|
377
|
+
Raises:
|
|
378
|
+
ConnectionError: when the device can not be reconnected for some reason.
|
|
379
|
+
"""
|
|
380
|
+
raise NotImplementedError
|
|
381
|
+
|
|
382
|
+
async def is_connected(self) -> bool:
|
|
383
|
+
"""Check if the device is connected.
|
|
384
|
+
|
|
385
|
+
Returns:
|
|
386
|
+
True if the device is connected and responds to a command, False otherwise.
|
|
387
|
+
"""
|
|
388
|
+
raise NotImplementedError
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
class AsyncDeviceInterface(AsyncDeviceConnectionInterface):
|
|
392
|
+
"""Generic interface for all device classes."""
|
|
393
|
+
|
|
394
|
+
def is_simulator(self) -> bool:
|
|
395
|
+
"""Checks whether the device is a simulator rather than a real hardware controller.
|
|
396
|
+
|
|
397
|
+
This can be useful for testing purposes or when doing actual movement simulations.
|
|
398
|
+
|
|
399
|
+
Returns:
|
|
400
|
+
True if the Device is a Simulator; False if the Device is connected to real hardware.
|
|
401
|
+
"""
|
|
402
|
+
|
|
403
|
+
raise NotImplementedError
|
|
404
|
+
|
|
405
|
+
|
|
333
406
|
class DeviceFactoryInterface:
|
|
334
407
|
"""
|
|
335
408
|
Base class for creating a device factory class to access devices.
|
egse/env.py
CHANGED
|
@@ -451,7 +451,7 @@ def set_local_settings(path: str | Path | None):
|
|
|
451
451
|
_env.set("LOCAL_SETTINGS", path)
|
|
452
452
|
|
|
453
453
|
|
|
454
|
-
def get_local_settings_path() -> str
|
|
454
|
+
def get_local_settings_path() -> str | None:
|
|
455
455
|
"""
|
|
456
456
|
Returns the fully qualified filename of the local settings YAML file. When the local settings environment
|
|
457
457
|
variable is not defined or is an empty string, None is returned.
|
egse/scpi.py
CHANGED
|
@@ -4,13 +4,13 @@ from typing import Any
|
|
|
4
4
|
from typing import Dict
|
|
5
5
|
from typing import Optional
|
|
6
6
|
|
|
7
|
+
from egse.device import AsyncDeviceInterface
|
|
7
8
|
from egse.device import AsyncDeviceTransport
|
|
8
9
|
from egse.device import DeviceConnectionError
|
|
9
10
|
from egse.device import DeviceError
|
|
10
11
|
from egse.device import DeviceTimeoutError
|
|
11
12
|
from egse.log import logger
|
|
12
13
|
|
|
13
|
-
# Constants that can be overridden by specific device implementations
|
|
14
14
|
DEFAULT_READ_TIMEOUT = 1.0 # seconds
|
|
15
15
|
DEFAULT_CONNECT_TIMEOUT = 3.0 # seconds
|
|
16
16
|
IDENTIFICATION_QUERY = "*IDN?"
|
|
@@ -32,7 +32,7 @@ class SCPICommand:
|
|
|
32
32
|
raise NotImplementedError("Subclasses must implement get_cmd_string().")
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
class AsyncSCPIInterface(AsyncDeviceTransport):
|
|
35
|
+
class AsyncSCPIInterface(AsyncDeviceInterface, AsyncDeviceTransport):
|
|
36
36
|
"""Generic asynchronous interface for devices that use SCPI commands over Ethernet."""
|
|
37
37
|
|
|
38
38
|
def __init__(
|
|
@@ -56,6 +56,7 @@ class AsyncSCPIInterface(AsyncDeviceTransport):
|
|
|
56
56
|
read_timeout: Timeout for read operations in seconds
|
|
57
57
|
id_validation: String that should appear in the device's identification response
|
|
58
58
|
"""
|
|
59
|
+
super().__init__()
|
|
59
60
|
self.device_name = device_name
|
|
60
61
|
self.hostname = hostname
|
|
61
62
|
self.port = port
|
|
@@ -73,6 +74,53 @@ class AsyncSCPIInterface(AsyncDeviceTransport):
|
|
|
73
74
|
"""Prevents multiple coroutines from attempting to read, write or query from the same stream
|
|
74
75
|
at the same time."""
|
|
75
76
|
|
|
77
|
+
def is_simulator(self) -> bool:
|
|
78
|
+
return False
|
|
79
|
+
|
|
80
|
+
async def initialize(self, commands: list[tuple[str, bool]] = None, reset_device: bool = False):
|
|
81
|
+
"""Initialize the device with optional reset and command sequence.
|
|
82
|
+
|
|
83
|
+
Performs device initialization by optionally resetting the device and then
|
|
84
|
+
executing a sequence of commands. Each command can optionally expect a
|
|
85
|
+
response that will be logged for debugging purposes.
|
|
86
|
+
|
|
87
|
+
Args:
|
|
88
|
+
commands: List of tuples containing (command_string, expects_response).
|
|
89
|
+
Each tuple specifies a command to send and whether to wait for and
|
|
90
|
+
log the response. Defaults to None (no commands executed).
|
|
91
|
+
reset_device: Whether to send a reset command (*RST) before executing
|
|
92
|
+
the command sequence. Defaults to False.
|
|
93
|
+
|
|
94
|
+
Returns:
|
|
95
|
+
None
|
|
96
|
+
|
|
97
|
+
Raises:
|
|
98
|
+
Any exceptions raised by the underlying write() or trans() methods,
|
|
99
|
+
typically communication errors or device timeouts.
|
|
100
|
+
|
|
101
|
+
Example:
|
|
102
|
+
>>> await device.initialize([
|
|
103
|
+
... ("*IDN?", True), # Query device ID, expect response
|
|
104
|
+
... ("SYST:ERR?", True), # Check for errors, expect response
|
|
105
|
+
... ("OUTP ON", False) # Enable output, no response expected
|
|
106
|
+
... ], reset_device=True)
|
|
107
|
+
"""
|
|
108
|
+
|
|
109
|
+
commands = commands or []
|
|
110
|
+
|
|
111
|
+
if reset_device:
|
|
112
|
+
logger.info(f"Resetting the {self.device_name}...")
|
|
113
|
+
await self.write("*RST") # this also resets the user-defined buffer
|
|
114
|
+
|
|
115
|
+
for cmd, expects_response in commands:
|
|
116
|
+
if expects_response:
|
|
117
|
+
logger.debug(f"Sending {cmd}...")
|
|
118
|
+
response = (await self.trans(cmd)).decode().strip()
|
|
119
|
+
logger.debug(f"{response = }")
|
|
120
|
+
else:
|
|
121
|
+
logger.debug(f"Sending {cmd}...")
|
|
122
|
+
await self.write(cmd)
|
|
123
|
+
|
|
76
124
|
async def connect(self) -> None:
|
|
77
125
|
"""Connect to the device asynchronously.
|
|
78
126
|
|
|
@@ -146,11 +194,9 @@ class AsyncSCPIInterface(AsyncDeviceTransport):
|
|
|
146
194
|
|
|
147
195
|
async def reconnect(self) -> None:
|
|
148
196
|
"""Reconnect to the device asynchronously."""
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
await asyncio.sleep(0.1)
|
|
153
|
-
await self.connect()
|
|
197
|
+
await self.disconnect()
|
|
198
|
+
await asyncio.sleep(0.1)
|
|
199
|
+
await self.connect()
|
|
154
200
|
|
|
155
201
|
async def is_connected(self) -> bool:
|
|
156
202
|
"""Check if the device is connected and responds correctly to identification.
|
|
@@ -177,8 +223,7 @@ class AsyncSCPIInterface(AsyncDeviceTransport):
|
|
|
177
223
|
return True
|
|
178
224
|
|
|
179
225
|
except DeviceError as exc:
|
|
180
|
-
logger.
|
|
181
|
-
logger.error(f"{self.device_name}: Connection test failed")
|
|
226
|
+
logger.error(f"{self.device_name}: Connection test failed: {exc}", exc_info=True)
|
|
182
227
|
await self.disconnect()
|
|
183
228
|
return False
|
|
184
229
|
|
|
@@ -201,6 +246,7 @@ class AsyncSCPIInterface(AsyncDeviceTransport):
|
|
|
201
246
|
if not command.endswith("\n"):
|
|
202
247
|
command += "\n"
|
|
203
248
|
|
|
249
|
+
logger.info(f"-----> {command}")
|
|
204
250
|
self._writer.write(command.encode())
|
|
205
251
|
await self._writer.drain()
|
|
206
252
|
|
|
@@ -233,6 +279,7 @@ class AsyncSCPIInterface(AsyncDeviceTransport):
|
|
|
233
279
|
response = await asyncio.wait_for(
|
|
234
280
|
self._reader.readuntil(separator=b"\n"), timeout=self.read_timeout
|
|
235
281
|
)
|
|
282
|
+
logger.info(f"<----- {response}")
|
|
236
283
|
return response
|
|
237
284
|
|
|
238
285
|
except asyncio.IncompleteReadError as exc:
|
|
@@ -279,6 +326,7 @@ class AsyncSCPIInterface(AsyncDeviceTransport):
|
|
|
279
326
|
if not command.endswith("\n"):
|
|
280
327
|
command += "\n"
|
|
281
328
|
|
|
329
|
+
logger.info(f"-----> {command}")
|
|
282
330
|
self._writer.write(command.encode())
|
|
283
331
|
await self._writer.drain()
|
|
284
332
|
|
|
@@ -290,6 +338,7 @@ class AsyncSCPIInterface(AsyncDeviceTransport):
|
|
|
290
338
|
response = await asyncio.wait_for(
|
|
291
339
|
self._reader.readuntil(separator=b"\n"), timeout=self.read_timeout
|
|
292
340
|
)
|
|
341
|
+
logger.info(f"<----- {response}")
|
|
293
342
|
return response
|
|
294
343
|
|
|
295
344
|
except asyncio.IncompleteReadError as exc:
|
egse/system.py
CHANGED
|
@@ -29,6 +29,7 @@ import operator
|
|
|
29
29
|
import os
|
|
30
30
|
import platform # For getting the operating system name
|
|
31
31
|
import re
|
|
32
|
+
import shutil
|
|
32
33
|
import socket
|
|
33
34
|
import subprocess # For executing a shell command
|
|
34
35
|
import sys
|
|
@@ -2256,6 +2257,56 @@ def snake_to_title(snake_str: str) -> str:
|
|
|
2256
2257
|
return snake_str.replace("_", " ").title()
|
|
2257
2258
|
|
|
2258
2259
|
|
|
2260
|
+
def caffeinate(pid: int = None):
|
|
2261
|
+
"""Prevent your macOS system from entering idle sleep while a process is running.
|
|
2262
|
+
|
|
2263
|
+
This function uses the macOS 'caffeinate' utility to prevent the system from
|
|
2264
|
+
going to sleep due to inactivity. It's particularly useful for long-running
|
|
2265
|
+
background processes that may lose network connections or be interrupted
|
|
2266
|
+
when the system sleeps.
|
|
2267
|
+
|
|
2268
|
+
The function only operates on macOS systems and silently does nothing on
|
|
2269
|
+
other operating systems.
|
|
2270
|
+
|
|
2271
|
+
Args:
|
|
2272
|
+
pid (int, optional): Process ID to monitor. If provided, caffeinate will
|
|
2273
|
+
keep the system awake as long as the specified process is running.
|
|
2274
|
+
If None or 0, defaults to the current process ID (os.getpid()).
|
|
2275
|
+
|
|
2276
|
+
Returns:
|
|
2277
|
+
None
|
|
2278
|
+
|
|
2279
|
+
Raises:
|
|
2280
|
+
FileNotFoundError: If 'caffeinate' command is not found in PATH (shouldn't
|
|
2281
|
+
happen on standard macOS installations).
|
|
2282
|
+
OSError: If subprocess.Popen fails to start the caffeinate process.
|
|
2283
|
+
|
|
2284
|
+
Example:
|
|
2285
|
+
>>> # Keep system awake while current process runs
|
|
2286
|
+
>>> caffeinate()
|
|
2287
|
+
|
|
2288
|
+
>>> # Keep system awake while specific process runs
|
|
2289
|
+
>>> caffeinate(1234)
|
|
2290
|
+
|
|
2291
|
+
Note:
|
|
2292
|
+
- Uses 'caffeinate -i -w <pid>' which prevents idle sleep (-i) and monitors
|
|
2293
|
+
a specific process (-w)
|
|
2294
|
+
- The caffeinate process will automatically terminate when the monitored
|
|
2295
|
+
process exits
|
|
2296
|
+
- On non-macOS systems, this function does nothing
|
|
2297
|
+
- Logs a warning message when caffeinate is started
|
|
2298
|
+
|
|
2299
|
+
See Also:
|
|
2300
|
+
macOS caffeinate(8) man page for more details on the underlying utility.
|
|
2301
|
+
"""
|
|
2302
|
+
if not pid:
|
|
2303
|
+
pid = os.getpid()
|
|
2304
|
+
|
|
2305
|
+
if get_os_name() == "macos":
|
|
2306
|
+
logger.warning(f"Running 'caffeinate -i -w {pid}' on macOS to prevent the system from idle sleeping.")
|
|
2307
|
+
subprocess.Popen([shutil.which("caffeinate"), "-i", "-w", str(pid)])
|
|
2308
|
+
|
|
2309
|
+
|
|
2259
2310
|
ignore_m_warning("egse.system")
|
|
2260
2311
|
|
|
2261
2312
|
if __name__ == "__main__":
|
|
File without changes
|
|
File without changes
|