cgse-common 0.16.13__tar.gz → 0.17.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cgse_common-0.16.13 → cgse_common-0.17.0}/PKG-INFO +1 -1
- {cgse_common-0.16.13 → cgse_common-0.17.0}/pyproject.toml +2 -3
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/cgse_common/cgse.py +5 -1
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/decorators.py +2 -1
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/device.py +8 -2
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/env.py +115 -49
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/heartbeat.py +1 -1
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/log.py +9 -2
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/plugins/metrics/influxdb.py +34 -2
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/scpi.py +15 -7
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/settings.py +4 -5
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/setup.py +12 -6
- cgse_common-0.17.0/src/egse/socketdevice.py +377 -0
- cgse_common-0.16.13/src/egse/socketdevice.py +0 -205
- {cgse_common-0.16.13 → cgse_common-0.17.0}/.gitignore +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/README.md +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/justfile +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/noxfile.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/service_registry.db +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/cgse_common/__init__.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/cgse_common/settings.yaml +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/bits.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/calibration.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/config.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/counter.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/dicts.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/exceptions.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/hk.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/metrics.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/observer.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/obsid.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/persistence.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/plugin.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/plugins/metrics/duckdb.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/plugins/metrics/timescaledb.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/process.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/py.typed +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/randomwalk.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/ratelimit.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/reload.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/resource.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/response.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/settings.yaml +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/signal.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/state.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/system.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/task.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/version.py +0 -0
- {cgse_common-0.16.13 → cgse_common-0.17.0}/src/egse/zmq_ser.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "cgse-common"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.17.0"
|
|
4
4
|
description = "Software framework to support hardware testing"
|
|
5
5
|
authors = [
|
|
6
6
|
{name = "IvS KU Leuven"}
|
|
@@ -48,7 +48,6 @@ dependencies = [
|
|
|
48
48
|
|
|
49
49
|
[project.scripts]
|
|
50
50
|
cgse = 'cgse_common.cgse:app'
|
|
51
|
-
monitor = "egse.monitoring:app"
|
|
52
51
|
|
|
53
52
|
[project.entry-points."cgse.version"]
|
|
54
53
|
cgse-common = 'egse.version:get_version_installed'
|
|
@@ -61,7 +60,7 @@ pythonpath = "src"
|
|
|
61
60
|
testpaths = ["tests"]
|
|
62
61
|
addopts = "-ra --cov --cov-branch --cov-report html"
|
|
63
62
|
log_cli = true
|
|
64
|
-
log_cli_level = "
|
|
63
|
+
log_cli_level = "DEBUG"
|
|
65
64
|
filterwarnings = [
|
|
66
65
|
"ignore::DeprecationWarning"
|
|
67
66
|
]
|
|
@@ -72,7 +72,7 @@ class SortedCommandGroup(TyperGroup):
|
|
|
72
72
|
commands = super().list_commands(ctx)
|
|
73
73
|
|
|
74
74
|
# Define priority commands in specific order
|
|
75
|
-
priority_commands = ["init", "version", "show", "top", "core", "reg", "
|
|
75
|
+
priority_commands = ["init", "version", "show", "top", "core", "reg", "log", "nh", "cm", "sm", "pm"]
|
|
76
76
|
|
|
77
77
|
# Custom sort:
|
|
78
78
|
# First the priority commands in the given order (their index)
|
|
@@ -175,6 +175,10 @@ def main(ctx: typer.Context, verbose: bool = False):
|
|
|
175
175
|
- inspect, configure, monitor the core services and device control servers.
|
|
176
176
|
"""
|
|
177
177
|
|
|
178
|
+
from egse.env import setup_env
|
|
179
|
+
|
|
180
|
+
setup_env()
|
|
181
|
+
|
|
178
182
|
# This is more of a show-case for using application wide optional arguments and how to pass
|
|
179
183
|
# them into (sub-)commands.
|
|
180
184
|
|
|
@@ -15,7 +15,6 @@ from typing import Optional
|
|
|
15
15
|
|
|
16
16
|
import rich
|
|
17
17
|
|
|
18
|
-
from egse.settings import Settings
|
|
19
18
|
from egse.system import get_caller_info
|
|
20
19
|
from egse.log import logger
|
|
21
20
|
|
|
@@ -390,6 +389,8 @@ def profile(func):
|
|
|
390
389
|
if not hasattr(profile, "counter"):
|
|
391
390
|
profile.counter = 0
|
|
392
391
|
|
|
392
|
+
from egse.settings import Settings
|
|
393
|
+
|
|
393
394
|
@functools.wraps(func)
|
|
394
395
|
def wrapper_profile(*args, **kwargs):
|
|
395
396
|
if Settings.profiling():
|
|
@@ -199,7 +199,7 @@ class DeviceConnectionInterface(DeviceConnectionObservable):
|
|
|
199
199
|
|
|
200
200
|
|
|
201
201
|
class DeviceInterface(DeviceConnectionInterface):
|
|
202
|
-
"""
|
|
202
|
+
"""A generic interface for all device classes."""
|
|
203
203
|
|
|
204
204
|
@dynamic_interface
|
|
205
205
|
def is_simulator(self) -> bool:
|
|
@@ -237,6 +237,7 @@ class DeviceTransport:
|
|
|
237
237
|
raise NotImplementedError
|
|
238
238
|
|
|
239
239
|
def read_string(self, encoding="utf-8") -> str:
|
|
240
|
+
"""Reads a bytes object from the instrument and returns it converted into a stripped UTF-8 string."""
|
|
240
241
|
return self.read().decode(encoding).strip()
|
|
241
242
|
|
|
242
243
|
def trans(self, command: str) -> bytes:
|
|
@@ -297,6 +298,11 @@ class AsyncDeviceTransport:
|
|
|
297
298
|
|
|
298
299
|
raise NotImplementedError
|
|
299
300
|
|
|
301
|
+
async def read_string(self, encoding="utf-8") -> str:
|
|
302
|
+
"""Reads a bytes object from the instrument and returns it converted into a stripped UTF-8 string."""
|
|
303
|
+
b = await self.read()
|
|
304
|
+
return b.decode(encoding).strip()
|
|
305
|
+
|
|
300
306
|
async def trans(self, command: str) -> bytes:
|
|
301
307
|
"""
|
|
302
308
|
Send a single command to the device controller and block until a response from the
|
|
@@ -389,7 +395,7 @@ class AsyncDeviceConnectionInterface(DeviceConnectionObservable):
|
|
|
389
395
|
|
|
390
396
|
|
|
391
397
|
class AsyncDeviceInterface(AsyncDeviceConnectionInterface):
|
|
392
|
-
"""
|
|
398
|
+
"""A generic interface for all device classes."""
|
|
393
399
|
|
|
394
400
|
def is_simulator(self) -> bool:
|
|
395
401
|
"""Checks whether the device is a simulator rather than a real hardware controller.
|
|
@@ -43,6 +43,10 @@ from __future__ import annotations
|
|
|
43
43
|
|
|
44
44
|
__all__ = [
|
|
45
45
|
"bool_env",
|
|
46
|
+
"int_env",
|
|
47
|
+
"str_env",
|
|
48
|
+
"load_dotenv",
|
|
49
|
+
"setup_env",
|
|
46
50
|
"env_var",
|
|
47
51
|
"get_conf_data_location",
|
|
48
52
|
"get_conf_data_location_env_name",
|
|
@@ -68,14 +72,16 @@ import os
|
|
|
68
72
|
import warnings
|
|
69
73
|
from pathlib import Path
|
|
70
74
|
|
|
75
|
+
from dotenv import load_dotenv as _load_dotenv
|
|
76
|
+
from dotenv import find_dotenv
|
|
77
|
+
from rich.console import Console
|
|
78
|
+
|
|
79
|
+
from egse.decorators import static_vars
|
|
71
80
|
from egse.log import logger
|
|
72
81
|
from egse.system import all_logging_disabled
|
|
73
82
|
from egse.system import get_caller_info
|
|
74
83
|
from egse.system import ignore_m_warning
|
|
75
|
-
|
|
76
|
-
from rich.console import Console
|
|
77
|
-
|
|
78
|
-
console = Console(width=100)
|
|
84
|
+
from egse.system import type_name
|
|
79
85
|
|
|
80
86
|
# Every project shall have a PROJECT and a SITE_ID environment variable set. This variable will be used to
|
|
81
87
|
# create the other environment variables that are specific to the project.
|
|
@@ -99,7 +105,57 @@ KNOWN_PROJECT_ENVIRONMENT_VARIABLES = [
|
|
|
99
105
|
]
|
|
100
106
|
|
|
101
107
|
|
|
102
|
-
def
|
|
108
|
+
def int_env(var_name: str, default: int, *, minimum: int | None = 1) -> int:
|
|
109
|
+
"""Return an integer environment override, falling back to `default` on errors."""
|
|
110
|
+
raw = os.getenv(var_name)
|
|
111
|
+
if raw is None:
|
|
112
|
+
return default
|
|
113
|
+
try:
|
|
114
|
+
value = int(raw)
|
|
115
|
+
except ValueError:
|
|
116
|
+
logger.warning(f"Ignoring invalid integer for {var_name}: {raw!r}, returning {default=}")
|
|
117
|
+
return default
|
|
118
|
+
if minimum is not None and value < minimum:
|
|
119
|
+
logger.warning(f"Ignoring {var_name} because value {value} < {minimum}, returning {default=}")
|
|
120
|
+
return default
|
|
121
|
+
return value
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def bool_env(var_name: str, default: bool = False) -> bool:
|
|
125
|
+
"""Return True if the environment variable is set to 1, true, yes, or on. All case-insensitive."""
|
|
126
|
+
|
|
127
|
+
if value := os.getenv(var_name):
|
|
128
|
+
return value.strip().lower() in {"1", "true", "yes", "on"}
|
|
129
|
+
|
|
130
|
+
return default
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def str_env(var_name: str, default: str | None = None) -> str:
|
|
134
|
+
"""Return the value of the environment variable or default if variable is not defined."""
|
|
135
|
+
return os.getenv(var_name, default)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
VERBOSE_DEBUG = bool_env("VERBOSE_DEBUG")
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def load_dotenv():
|
|
142
|
+
"""Overwrites the load_dotenv function for CGSE specifics.
|
|
143
|
+
|
|
144
|
+
- set `CGSE_DOTENV_DISABLED=true` to disable loading the `.env` file
|
|
145
|
+
- the `.env` file is searched for relative to the current working directory,
|
|
146
|
+
unlike the default, which searches from the script location.
|
|
147
|
+
|
|
148
|
+
NOTE:
|
|
149
|
+
- don't use the `load_dotenv` function from the `dotenv` package
|
|
150
|
+
- don't use `load_dotenv` in any modules, only in entrypoints and apps.
|
|
151
|
+
"""
|
|
152
|
+
if not bool_env("CGSE_DOTENV_DISABLED") and (dotenv_location := find_dotenv(usecwd=True)):
|
|
153
|
+
logger.debug(f"Loading environment variables from {dotenv_location}.")
|
|
154
|
+
_load_dotenv(dotenv_path=dotenv_location)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
@static_vars(is_initialized=False)
|
|
158
|
+
def setup_env():
|
|
103
159
|
"""
|
|
104
160
|
Initialize the environment variables that are required for the CGSE to function properly.
|
|
105
161
|
This function will print a warning if any of the mandatory environment variables is not set.
|
|
@@ -110,6 +166,14 @@ def initialize():
|
|
|
110
166
|
|
|
111
167
|
global _env
|
|
112
168
|
|
|
169
|
+
if setup_env.is_initialized:
|
|
170
|
+
return
|
|
171
|
+
|
|
172
|
+
if VERBOSE_DEBUG:
|
|
173
|
+
logger.debug(f"Initialising the environment...")
|
|
174
|
+
|
|
175
|
+
load_dotenv()
|
|
176
|
+
|
|
113
177
|
for name in MANDATORY_ENVIRONMENT_VARIABLES:
|
|
114
178
|
try:
|
|
115
179
|
_env.set(name, os.environ[name])
|
|
@@ -123,8 +187,10 @@ def initialize():
|
|
|
123
187
|
project = _env.get("PROJECT")
|
|
124
188
|
|
|
125
189
|
for gen_var_name in KNOWN_PROJECT_ENVIRONMENT_VARIABLES:
|
|
126
|
-
|
|
127
|
-
_env.set(gen_var_name, os.environ.get(
|
|
190
|
+
_env_var = f"{project}_{gen_var_name}"
|
|
191
|
+
_env.set(gen_var_name, os.environ.get(_env_var, NoValue()))
|
|
192
|
+
|
|
193
|
+
setup_env.is_initialized = True
|
|
128
194
|
|
|
129
195
|
|
|
130
196
|
class _Env:
|
|
@@ -136,8 +202,12 @@ class _Env:
|
|
|
136
202
|
def set(self, key, value):
|
|
137
203
|
if value is None:
|
|
138
204
|
if key in self._env:
|
|
205
|
+
if VERBOSE_DEBUG:
|
|
206
|
+
logger.debug(f"Unsetting environment variable {key}")
|
|
139
207
|
del self._env[key]
|
|
140
208
|
else:
|
|
209
|
+
if VERBOSE_DEBUG:
|
|
210
|
+
logger.debug(f"Setting environment variable {key}={value}")
|
|
141
211
|
self._env[key] = value
|
|
142
212
|
|
|
143
213
|
def get(self, key) -> str:
|
|
@@ -166,11 +236,11 @@ class NoValue:
|
|
|
166
236
|
return False
|
|
167
237
|
|
|
168
238
|
def __repr__(self):
|
|
169
|
-
return f"{self
|
|
239
|
+
return f"{type_name(self)}"
|
|
170
240
|
|
|
171
241
|
|
|
172
242
|
# The module needs to be initialized before it can be used.
|
|
173
|
-
|
|
243
|
+
setup_env()
|
|
174
244
|
|
|
175
245
|
|
|
176
246
|
def _check_no_value(var_name, value):
|
|
@@ -244,7 +314,7 @@ def set_data_storage_location(location: str | Path | None):
|
|
|
244
314
|
_env.set("DATA_STORAGE_LOCATION", None)
|
|
245
315
|
return
|
|
246
316
|
|
|
247
|
-
if not Path(location).exists():
|
|
317
|
+
if not Path(location).expanduser().exists():
|
|
248
318
|
warnings.warn(f"The location you provided for the environment variable {env_name} doesn't exist: {location}.")
|
|
249
319
|
|
|
250
320
|
os.environ[env_name] = str(location)
|
|
@@ -312,7 +382,7 @@ def set_conf_data_location(location: str | Path | None):
|
|
|
312
382
|
_env.set("CONF_DATA_LOCATION", None)
|
|
313
383
|
return
|
|
314
384
|
|
|
315
|
-
if not Path(location).exists():
|
|
385
|
+
if not Path(location).expanduser().exists():
|
|
316
386
|
warnings.warn(f"The location you provided for the environment variable {env_name} doesn't exist: {location}.")
|
|
317
387
|
|
|
318
388
|
os.environ[env_name] = location
|
|
@@ -378,7 +448,7 @@ def set_log_file_location(location: str | Path | None):
|
|
|
378
448
|
_env.set("LOG_FILE_LOCATION", None)
|
|
379
449
|
return
|
|
380
450
|
|
|
381
|
-
if not Path(location).exists():
|
|
451
|
+
if not Path(location).expanduser().exists():
|
|
382
452
|
warnings.warn(f"The location you provided for the environment variable {env_name} doesn't exist: {location}.")
|
|
383
453
|
|
|
384
454
|
os.environ[env_name] = location
|
|
@@ -423,7 +493,7 @@ def get_log_file_location(site_id: str = None, check_exists: bool = False) -> st
|
|
|
423
493
|
log_data_root = f"{data_root}/log"
|
|
424
494
|
|
|
425
495
|
if check_exists:
|
|
426
|
-
if not Path(log_data_root).is_dir():
|
|
496
|
+
if not Path(log_data_root).expanduser().is_dir():
|
|
427
497
|
raise ValueError(f"The location that was constructed doesn't exist: {log_data_root}")
|
|
428
498
|
|
|
429
499
|
return log_data_root
|
|
@@ -453,8 +523,11 @@ def set_local_settings(path: str | Path | None):
|
|
|
453
523
|
_env.set("LOCAL_SETTINGS", None)
|
|
454
524
|
return
|
|
455
525
|
|
|
456
|
-
if not Path(path).
|
|
457
|
-
warnings.warn(
|
|
526
|
+
if not Path(path).expanduser().is_file():
|
|
527
|
+
warnings.warn(
|
|
528
|
+
f"The location you provided for the environment variable {env_name} doesn't exist or is not a "
|
|
529
|
+
f"regular file: {path}."
|
|
530
|
+
)
|
|
458
531
|
|
|
459
532
|
os.environ[env_name] = path
|
|
460
533
|
_env.set("LOCAL_SETTINGS", path)
|
|
@@ -481,9 +554,9 @@ def get_local_settings_path() -> str | None:
|
|
|
481
554
|
)
|
|
482
555
|
return None
|
|
483
556
|
|
|
484
|
-
if not Path(local_settings).
|
|
557
|
+
if not Path(local_settings).expanduser().is_file():
|
|
485
558
|
warnings.warn(
|
|
486
|
-
f"The local settings path '{local_settings}' doesn't exist. As a result, "
|
|
559
|
+
f"The local settings path '{local_settings}' doesn't exist or is not a regular file. As a result, "
|
|
487
560
|
f"the local settings for your project will not be loaded."
|
|
488
561
|
)
|
|
489
562
|
|
|
@@ -519,7 +592,7 @@ def get_conf_repo_location() -> str | None:
|
|
|
519
592
|
)
|
|
520
593
|
return None
|
|
521
594
|
|
|
522
|
-
if not Path(location).exists():
|
|
595
|
+
if not Path(location).expanduser().exists():
|
|
523
596
|
warnings.warn(f"The location of the configuration data repository doesn't exist: {location}.")
|
|
524
597
|
return None
|
|
525
598
|
|
|
@@ -545,7 +618,7 @@ def set_conf_repo_location(location: str | Path | None):
|
|
|
545
618
|
_env.set("CONF_REPO_LOCATION", None)
|
|
546
619
|
return
|
|
547
620
|
|
|
548
|
-
if not Path(location).exists():
|
|
621
|
+
if not Path(location).expanduser().exists():
|
|
549
622
|
warnings.warn(f"The location you provided for the environment variable {env_name} doesn't exist: {location}.")
|
|
550
623
|
|
|
551
624
|
os.environ[env_name] = location
|
|
@@ -574,20 +647,11 @@ def print_env():
|
|
|
574
647
|
console.print(f" {get_local_settings_env_name():{col_width}s}: {get_local_settings_path()}")
|
|
575
648
|
|
|
576
649
|
|
|
577
|
-
def bool_env(var_name: str) -> bool:
|
|
578
|
-
"""Return True if the environment variable is set to 1, true, yes, or on. All case-insensitive."""
|
|
579
|
-
|
|
580
|
-
if value := os.getenv(var_name):
|
|
581
|
-
return value.strip().lower() in {"1", "true", "yes", "on"}
|
|
582
|
-
|
|
583
|
-
return False
|
|
584
|
-
|
|
585
|
-
|
|
586
650
|
@contextlib.contextmanager
|
|
587
651
|
def env_var(**kwargs: str | int | float | bool | None):
|
|
588
652
|
"""
|
|
589
|
-
Context manager to run some code that need alternate settings for environment variables.
|
|
590
|
-
|
|
653
|
+
Context manager to run some code that need alternate settings for environment variables. This will automatically
|
|
654
|
+
initialize the CGSE environment upon entry and re-initialize when the context manager exits.
|
|
591
655
|
|
|
592
656
|
Note:
|
|
593
657
|
This context manager is different from the one in `egse.system` because of the CGSE environment changes.
|
|
@@ -614,7 +678,7 @@ def env_var(**kwargs: str | int | float | bool | None):
|
|
|
614
678
|
else:
|
|
615
679
|
os.environ[k] = v
|
|
616
680
|
|
|
617
|
-
|
|
681
|
+
setup_env()
|
|
618
682
|
|
|
619
683
|
yield
|
|
620
684
|
|
|
@@ -625,7 +689,7 @@ def env_var(**kwargs: str | int | float | bool | None):
|
|
|
625
689
|
else:
|
|
626
690
|
os.environ[k] = v
|
|
627
691
|
|
|
628
|
-
|
|
692
|
+
setup_env()
|
|
629
693
|
|
|
630
694
|
|
|
631
695
|
def main(args: list | None = None): # pragma: no cover
|
|
@@ -650,32 +714,34 @@ def main(args: list | None = None): # pragma: no cover
|
|
|
650
714
|
"--mkdir",
|
|
651
715
|
default=False,
|
|
652
716
|
action="store_true",
|
|
653
|
-
help="Create directory that doesn't exist.",
|
|
717
|
+
help="Create any directory that doesn't exist.",
|
|
654
718
|
)
|
|
655
719
|
|
|
656
720
|
args = parser.parse_args(args or [])
|
|
657
721
|
|
|
658
|
-
|
|
659
|
-
|
|
722
|
+
setup_env()
|
|
723
|
+
|
|
724
|
+
def check_env_dir(_env_var: str):
|
|
725
|
+
value = _env.get(_env_var)
|
|
660
726
|
|
|
661
727
|
if value == NoValue():
|
|
662
728
|
value = "[bold red]not set"
|
|
663
|
-
elif not value.
|
|
729
|
+
elif not Path(value).expanduser().is_absolute():
|
|
664
730
|
value = f"[default]{value} [bold orange3](this is a relative path!)"
|
|
665
|
-
elif not
|
|
731
|
+
elif not Path(value).expanduser().exists():
|
|
666
732
|
value = f"[default]{value} [bold red](location doesn't exist!)"
|
|
667
|
-
elif not
|
|
733
|
+
elif not Path(value).expanduser().is_dir():
|
|
668
734
|
value = f"[default]{value} [bold red](location is not a directory!)"
|
|
669
735
|
else:
|
|
670
736
|
value = f"[default]{value}"
|
|
671
737
|
return value
|
|
672
738
|
|
|
673
|
-
def check_env_file(
|
|
674
|
-
value = _env.get(
|
|
739
|
+
def check_env_file(_env_var: str):
|
|
740
|
+
value = _env.get(_env_var)
|
|
675
741
|
|
|
676
742
|
if not value:
|
|
677
743
|
value = "[bold red]not set"
|
|
678
|
-
elif not
|
|
744
|
+
elif not Path(value).expanduser().exists():
|
|
679
745
|
value = f"[default]{value} [bold red](location doesn't exist!)"
|
|
680
746
|
else:
|
|
681
747
|
value = f"[default]{value}"
|
|
@@ -701,7 +767,7 @@ def main(args: list | None = None): # pragma: no cover
|
|
|
701
767
|
try:
|
|
702
768
|
rich.print(f" {get_data_storage_location() = }", flush=True, end="")
|
|
703
769
|
location = get_data_storage_location()
|
|
704
|
-
if not Path(location).exists():
|
|
770
|
+
if not Path(location).expanduser().exists():
|
|
705
771
|
if args.mkdir:
|
|
706
772
|
rich.print(f" [green]⟶ Creating data storage location: {location} (+ daily + obs)[/]")
|
|
707
773
|
Path(location).mkdir(parents=True)
|
|
@@ -717,7 +783,7 @@ def main(args: list | None = None): # pragma: no cover
|
|
|
717
783
|
try:
|
|
718
784
|
rich.print(f" {get_conf_data_location() = }", flush=True, end="")
|
|
719
785
|
location = get_conf_data_location()
|
|
720
|
-
if not Path(location).exists():
|
|
786
|
+
if not Path(location).expanduser().exists():
|
|
721
787
|
if args.mkdir:
|
|
722
788
|
rich.print(f" [green]⟶ Creating configuration data location: {location}[/]")
|
|
723
789
|
Path(location).mkdir(parents=True)
|
|
@@ -731,7 +797,7 @@ def main(args: list | None = None): # pragma: no cover
|
|
|
731
797
|
try:
|
|
732
798
|
rich.print(f" {get_conf_repo_location() = }", flush=True, end="")
|
|
733
799
|
location = get_conf_repo_location()
|
|
734
|
-
if location is None or not Path(location).exists():
|
|
800
|
+
if location is None or not Path(location).expanduser().exists():
|
|
735
801
|
rich.print(" [red]⟶ ERROR: The configuration repository location doesn't exist![/]")
|
|
736
802
|
else:
|
|
737
803
|
rich.print()
|
|
@@ -741,7 +807,7 @@ def main(args: list | None = None): # pragma: no cover
|
|
|
741
807
|
try:
|
|
742
808
|
rich.print(f" {get_log_file_location() = }", flush=True, end="")
|
|
743
809
|
location = get_log_file_location()
|
|
744
|
-
if not Path(location).exists():
|
|
810
|
+
if not Path(location).expanduser().exists():
|
|
745
811
|
if args.mkdir:
|
|
746
812
|
rich.print(f" [green]⟶ Creating log files location: {location}[/]")
|
|
747
813
|
Path(location).mkdir(parents=True)
|
|
@@ -755,7 +821,7 @@ def main(args: list | None = None): # pragma: no cover
|
|
|
755
821
|
try:
|
|
756
822
|
rich.print(f" {get_local_settings_path() = }", flush=True, end="")
|
|
757
823
|
location = get_local_settings_path()
|
|
758
|
-
if location is None or not Path(location).exists():
|
|
824
|
+
if location is None or not Path(location).expanduser().exists():
|
|
759
825
|
rich.print(" [red]⟶ ERROR: The local settings file is not defined or doesn't exist![/]")
|
|
760
826
|
else:
|
|
761
827
|
rich.print()
|
|
@@ -782,8 +848,8 @@ def main(args: list | None = None): # pragma: no cover
|
|
|
782
848
|
of the name. By default, this directory is located in the overall data storage folder.
|
|
783
849
|
|
|
784
850
|
[bold]PROJECT_CONF_REPO_LOCATION[/bold]:
|
|
785
|
-
This variable is the root of the working copy of the
|
|
786
|
-
The value is usually set to `~/git/
|
|
851
|
+
This variable is the root of the working copy of the repo for configuration files.
|
|
852
|
+
The value is usually set to `~/git/{project}-conf`.
|
|
787
853
|
|
|
788
854
|
[bold]PROJECT_DATA_STORAGE_LOCATION[/bold]:
|
|
789
855
|
This directory contains all the data files from the control servers and other
|
|
@@ -805,7 +871,7 @@ def main(args: list | None = None): # pragma: no cover
|
|
|
805
871
|
[bold]PROJECT_LOCAL_SETTINGS[/bold]:
|
|
806
872
|
This file is used for local site-specific settings. When the environment
|
|
807
873
|
variable is not set, no local settings will be loaded. By default, this variable
|
|
808
|
-
|
|
874
|
+
assumes to be '/cgse/local_settings.yaml'.
|
|
809
875
|
"""
|
|
810
876
|
|
|
811
877
|
if args.doc:
|
|
@@ -80,7 +80,7 @@ if __name__ == "__main__":
|
|
|
80
80
|
broadcaster.start()
|
|
81
81
|
|
|
82
82
|
# You will see that the timestamp of this custom message is drifting because
|
|
83
|
-
# we do not take the execution time of the `set_message()` etc.
|
|
83
|
+
# we do not take the execution time of the `set_message()` etc. into account.
|
|
84
84
|
# In the actual HeartbeatBroadcaster above, we do take that into account.
|
|
85
85
|
|
|
86
86
|
try:
|
|
@@ -135,20 +135,27 @@ for handler in root_logger.handlers:
|
|
|
135
135
|
logger = egse_logger
|
|
136
136
|
|
|
137
137
|
if __name__ == "__main__":
|
|
138
|
+
from egse.env import str_env
|
|
139
|
+
|
|
138
140
|
root_logger = logging.getLogger()
|
|
139
141
|
|
|
140
|
-
|
|
142
|
+
print(
|
|
141
143
|
textwrap.dedent(
|
|
142
144
|
"""
|
|
145
|
+
Environment variables:
|
|
146
|
+
- LOG_LEVEL=debug|info|warning|critical
|
|
147
|
+
- LOG_FORMAT=full|clean
|
|
148
|
+
|
|
143
149
|
Example logging statements
|
|
144
150
|
- logging level set to INFO
|
|
151
|
+
- logging format set to full
|
|
145
152
|
- fields are separated by a colon ':'
|
|
146
153
|
- fields: date & time: process name : level : logger name : lineno : filename : message
|
|
147
154
|
"""
|
|
148
155
|
)
|
|
149
156
|
)
|
|
150
157
|
|
|
151
|
-
if
|
|
158
|
+
if str_env("LOG_FORMAT", "clean").strip().lower() == "full":
|
|
152
159
|
rich.print(
|
|
153
160
|
f"[b]{'Date & Time':^23s} : {'Process Name':20s} : {'Level':8s} : {'Logger Name':^25s} : {' Line '} : "
|
|
154
161
|
f"{'Filename':20s} : {'Message'}[/]"
|
|
@@ -37,11 +37,14 @@ import pandas
|
|
|
37
37
|
import pyarrow
|
|
38
38
|
from influxdb_client_3 import InfluxDBClient3
|
|
39
39
|
from influxdb_client_3 import Point
|
|
40
|
-
from influxdb_client_3 import
|
|
40
|
+
from influxdb_client_3 import WriteType
|
|
41
41
|
from influxdb_client_3 import write_client_options
|
|
42
42
|
from influxdb_client_3.exceptions import InfluxDB3ClientError
|
|
43
43
|
from influxdb_client_3.write_client.domain.write_precision import WritePrecision
|
|
44
44
|
|
|
45
|
+
from egse.env import bool_env
|
|
46
|
+
from egse.env import int_env
|
|
47
|
+
from egse.env import str_env
|
|
45
48
|
from egse.metrics import TimeSeriesRepository
|
|
46
49
|
from egse.system import type_name
|
|
47
50
|
|
|
@@ -63,9 +66,38 @@ class InfluxDBRepository(TimeSeriesRepository):
|
|
|
63
66
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
64
67
|
self.close()
|
|
65
68
|
|
|
69
|
+
def _load_client_options(self):
|
|
70
|
+
self._batch_size = int_env("CGSE_INFLUX_BATCH_SIZE", 1_000)
|
|
71
|
+
self._flush_interval = int_env("CGSE_INFLUX_FLUSH_MS", 1_000)
|
|
72
|
+
self._retry_interval = int_env("CGSE_INFLUX_RETRY_MS", 3_000)
|
|
73
|
+
self._max_retry_delay = int_env("CGSE_INFLUX_RETRY_MAX_DELAY_MS", 3_000)
|
|
74
|
+
self._max_retry_time = int_env("CGSE_INFLUX_RETRY_MAX_TIME_MS", 6_000, minimum=0)
|
|
75
|
+
self._max_retries = int_env("CGSE_INFLUX_MAX_RETRIES", 5, minimum=0)
|
|
76
|
+
self._no_sync = bool_env("CGSE_INFLUX_NO_SYNC", default=True)
|
|
77
|
+
self._write_type_env = str_env("CGSE_INFLUX_WRITE_TYPE")
|
|
78
|
+
self._write_type = WriteType.asynchronous
|
|
79
|
+
if self._write_type_env:
|
|
80
|
+
match self._write_type_env.strip().lower():
|
|
81
|
+
case "batch":
|
|
82
|
+
self._write_type = WriteType.batching
|
|
83
|
+
case "async":
|
|
84
|
+
self._write_type = WriteType.asynchronous
|
|
85
|
+
case "sync":
|
|
86
|
+
self._write_type = WriteType.synchronous
|
|
87
|
+
|
|
66
88
|
def connect(self):
|
|
89
|
+
self._load_client_options()
|
|
90
|
+
|
|
67
91
|
wco = write_client_options(
|
|
68
|
-
|
|
92
|
+
write_type=self._write_type,
|
|
93
|
+
batch_size=self._batch_size,
|
|
94
|
+
flush_interval=self._flush_interval,
|
|
95
|
+
retry_interval=self._retry_interval,
|
|
96
|
+
max_retries=self._max_retries,
|
|
97
|
+
max_retry_delay=self._max_retry_delay,
|
|
98
|
+
max_retry_time=self._max_retry_time,
|
|
99
|
+
no_sync=self._no_sync,
|
|
100
|
+
write_precision=self.metrics_time_precision,
|
|
69
101
|
)
|
|
70
102
|
self.client = InfluxDBClient3(host=self.host, database=self.database, token=self.token, write_options=wco)
|
|
71
103
|
|
|
@@ -77,7 +77,7 @@ class AsyncSCPIInterface(AsyncDeviceInterface, AsyncDeviceTransport):
|
|
|
77
77
|
def is_simulator(self) -> bool:
|
|
78
78
|
return False
|
|
79
79
|
|
|
80
|
-
async def initialize(self, commands: list[tuple[str, bool]] = None, reset_device: bool = False):
|
|
80
|
+
async def initialize(self, commands: list[tuple[str, bool]] = None, reset_device: bool = False) -> list[str | None]:
|
|
81
81
|
"""Initialize the device with optional reset and command sequence.
|
|
82
82
|
|
|
83
83
|
Performs device initialization by optionally resetting the device and then
|
|
@@ -92,21 +92,25 @@ class AsyncSCPIInterface(AsyncDeviceInterface, AsyncDeviceTransport):
|
|
|
92
92
|
the command sequence. Defaults to False.
|
|
93
93
|
|
|
94
94
|
Returns:
|
|
95
|
-
|
|
95
|
+
Response for each of the commands, or None when no response was expected.
|
|
96
96
|
|
|
97
97
|
Raises:
|
|
98
98
|
Any exceptions raised by the underlying write() or trans() methods,
|
|
99
99
|
typically communication errors or device timeouts.
|
|
100
100
|
|
|
101
101
|
Example:
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
102
|
+
await device.initialize(
|
|
103
|
+
[
|
|
104
|
+
("*IDN?", True), # Query device ID, expect response
|
|
105
|
+
("SYST:ERR?", True), # Check for errors, expect response
|
|
106
|
+
("OUTP ON", False), # Enable output, no response expected
|
|
107
|
+
],
|
|
108
|
+
reset_device=True
|
|
109
|
+
)
|
|
107
110
|
"""
|
|
108
111
|
|
|
109
112
|
commands = commands or []
|
|
113
|
+
responses = []
|
|
110
114
|
|
|
111
115
|
if reset_device:
|
|
112
116
|
logger.info(f"Resetting the {self.device_name}...")
|
|
@@ -116,10 +120,14 @@ class AsyncSCPIInterface(AsyncDeviceInterface, AsyncDeviceTransport):
|
|
|
116
120
|
if expects_response:
|
|
117
121
|
logger.debug(f"Sending {cmd}...")
|
|
118
122
|
response = (await self.trans(cmd)).decode().strip()
|
|
123
|
+
responses.append(response)
|
|
119
124
|
logger.debug(f"{response = }")
|
|
120
125
|
else:
|
|
121
126
|
logger.debug(f"Sending {cmd}...")
|
|
122
127
|
await self.write(cmd)
|
|
128
|
+
responses.append(None)
|
|
129
|
+
|
|
130
|
+
return responses
|
|
123
131
|
|
|
124
132
|
async def connect(self) -> None:
|
|
125
133
|
"""Connect to the device asynchronously.
|
|
@@ -66,9 +66,6 @@ The above code will read the YAML file from the given location and not from the
|
|
|
66
66
|
|
|
67
67
|
"""
|
|
68
68
|
|
|
69
|
-
from __future__ import annotations
|
|
70
|
-
|
|
71
|
-
import logging
|
|
72
69
|
import re
|
|
73
70
|
from pathlib import Path
|
|
74
71
|
from typing import Any
|
|
@@ -197,7 +194,7 @@ def load_local_settings(force: bool = False) -> attrdict:
|
|
|
197
194
|
local_settings_path = get_local_settings_path()
|
|
198
195
|
|
|
199
196
|
if local_settings_path:
|
|
200
|
-
path = Path(local_settings_path)
|
|
197
|
+
path = Path(local_settings_path).expanduser()
|
|
201
198
|
local_settings = load_settings_file(path.parent, path.name, force)
|
|
202
199
|
|
|
203
200
|
return local_settings
|
|
@@ -386,7 +383,9 @@ def main(args: list | None = None): # pragma: no cover
|
|
|
386
383
|
#
|
|
387
384
|
# Use the '--help' option to see what your choices are.
|
|
388
385
|
|
|
389
|
-
|
|
386
|
+
from egse.env import setup_env
|
|
387
|
+
|
|
388
|
+
setup_env()
|
|
390
389
|
|
|
391
390
|
import argparse
|
|
392
391
|
|