pvplot 0.6.7__tar.gz → 0.6.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pvplot-0.6.7 → pvplot-0.6.9}/PKG-INFO +2 -2
- {pvplot-0.6.7/pvplot → pvplot-0.6.9/apwork}/__main__.py +1 -1
- pvplot-0.6.9/apwork/apstrim/__init__.py +47 -0
- pvplot-0.6.9/apwork/apstrim/__main__.py +69 -0
- pvplot-0.6.9/apwork/apstrim/apstrim.py +425 -0
- pvplot-0.6.9/apwork/apstrim/liteCNS.py +32 -0
- pvplot-0.6.9/apwork/apstrim/pubEPICS.py +214 -0
- pvplot-0.6.9/apwork/apstrim/scan.py +336 -0
- pvplot-0.6.9/apwork/apstrim/view.py +237 -0
- pvplot-0.6.9/apwork/pvplot.py +875 -0
- {pvplot-0.6.7/ln → pvplot-0.6.9}/liteaccess/liteaccess.py +30 -21
- {pvplot-0.6.7 → pvplot-0.6.9}/ln/liteserver/device/i2c.py +257 -79
- {pvplot-0.6.7 → pvplot-0.6.9}/ln/liteserver/device/liteMCUFEC.py +64 -33
- {pvplot-0.6.7 → pvplot-0.6.9}/ln/liteserver/device/litePeakSimulator.py +11 -5
- {pvplot-0.6.7 → pvplot-0.6.9}/ln/liteserver/device/liteScaler.py +1 -2
- {pvplot-0.6.7 → pvplot-0.6.9}/ln/liteserver/device/senstation.py +52 -36
- pvplot-0.6.9/ln/liteserver/device/test/test_dht11.py +58 -0
- pvplot-0.6.9/ln/liteserver/liteserver.py +984 -0
- pvplot-0.6.7/ln/liteserver/liteserver.py → pvplot-0.6.9/ln/liteserver/liteserver_unlist.py +2 -1
- pvplot-0.6.9/pvplot/__main__.py +51 -0
- pvplot-0.6.9/pvplot/apstrimAccess.py +65 -0
- pvplot-0.6.9/pvplot/pvp.py +917 -0
- pvplot-0.6.9/pvplot/pvp1.py +935 -0
- pvplot-0.6.9/pvplot/pvplot.py +981 -0
- pvplot-0.6.7/pvplot/pvplot.py → pvplot-0.6.9/pvplot/pvpopld.py +5 -4
- {pvplot-0.6.7 → pvplot-0.6.9}/pyproject.toml +1 -1
- {pvplot-0.6.7 → pvplot-0.6.9}/LICENSE +0 -0
- {pvplot-0.6.7 → pvplot-0.6.9}/README.md +0 -0
- {pvplot-0.6.7 → pvplot-0.6.9}/cad_epics/__init__.py +0 -0
- {pvplot-0.6.7 → pvplot-0.6.9}/cad_epics/epics.py +0 -0
- {pvplot-0.6.7 → pvplot-0.6.9}/fallback/__main__.py +0 -0
- {pvplot-0.6.7/ln → pvplot-0.6.9}/liteaccess/.__init__.py.swp +0 -0
- {pvplot-0.6.7/ln → pvplot-0.6.9}/liteaccess/__init__.py +0 -0
- {pvplot-0.6.7/ln → pvplot-0.6.9}/liteaccess/liteCNS.py +0 -0
- {pvplot-0.6.7 → pvplot-0.6.9}/ln/liteserver/__init__.py +0 -0
- {pvplot-0.6.7 → pvplot-0.6.9}/ln/liteserver/device/__init__.py +0 -0
- {pvplot-0.6.7 → pvplot-0.6.9}/ln/liteserver/device/helpers.py +0 -0
- {pvplot-0.6.7 → pvplot-0.6.9}/ln/liteserver/device/i2c_312.py +0 -0
- {pvplot-0.6.7 → pvplot-0.6.9}/ln/liteserver/device/i2c_320.py +0 -0
- {pvplot-0.6.7 → pvplot-0.6.9}/ln/liteserver/device/i2c_322.py +0 -0
- {pvplot-0.6.7 → pvplot-0.6.9}/ln/liteserver/device/i2c_TLV493D.py +0 -0
- {pvplot-0.6.7 → pvplot-0.6.9}/ln/liteserver/device/liteCNSserver.py +0 -0
- {pvplot-0.6.7 → pvplot-0.6.9}/ln/liteserver/device/liteGQ.py +0 -0
- {pvplot-0.6.7 → pvplot-0.6.9}/ln/liteserver/device/liteLabjack.py +0 -0
- {pvplot-0.6.7 → pvplot-0.6.9}/ln/liteserver/device/liteUSBCam.py +0 -0
- {pvplot-0.6.7 → pvplot-0.6.9}/ln/liteserver/device/liteUvcCam.py +0 -0
- {pvplot-0.6.7 → pvplot-0.6.9}/ln/liteserver/device/liteVGM.py +0 -0
- {pvplot-0.6.7 → pvplot-0.6.9}/ln/liteserver/device/liteWLM.py +0 -0
- {pvplot-0.6.7 → pvplot-0.6.9}/pvplot/__init__.py +0 -0
- {pvplot-0.6.7 → pvplot-0.6.9}/tmp.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
2
|
Name: pvplot
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.9
|
|
4
4
|
Summary: Plotting tool for EPICS PVs, ADO and LITE parameters
|
|
5
5
|
Project-URL: Homepage, https://github.com/ASukhanov/pvplot
|
|
6
6
|
Project-URL: Bug Tracker, https://github.com/ASukhanov/pvplot/issues
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"""Dynamic plotting tool for control system frameworks.
|
|
3
3
|
Frameworks supported: EPICS, ADO, LiteServer.
|
|
4
4
|
The framework is distinguished by adding a prefix to Process Variable names:
|
|
5
|
-
'E:' - for EPICS, 'L:
|
|
5
|
+
'E:' - for EPICS, 'L:' - for LiteServer, 'A:' - apstrim file.
|
|
6
6
|
"""
|
|
7
7
|
import sys,os
|
|
8
8
|
from . import pvplot
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Copyright (c) 2021 Andrei Sukhanov. All rights reserved.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the MIT License, (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://github.com/ASukhanov/upstrim/blob/main/LICENSE
|
|
8
|
+
#
|
|
9
|
+
|
|
10
|
+
"""Logger of Control System parameters and data objects.
|
|
11
|
+
Features:
|
|
12
|
+
- Supported Control Systems infrastructures: ADO, EPICS, LITE.
|
|
13
|
+
- Very fast random access retrieval of objects for selected time interval.
|
|
14
|
+
- Nonhomogeneous and homogeneous data are processed equally fast.
|
|
15
|
+
- Self-describing data format, no schema required.
|
|
16
|
+
- Efficient binary serialization format.
|
|
17
|
+
- Like JSON. But it's faster and smaller.
|
|
18
|
+
- Numpy arrays supported.
|
|
19
|
+
- Optional online compression.
|
|
20
|
+
- Basic plotting of logged data.
|
|
21
|
+
- Data extraction from a file is allowed when the file is being written.
|
|
22
|
+
|
|
23
|
+
Example of command line usage:
|
|
24
|
+
|
|
25
|
+
# Serialization of EPICS PVs MeanValue_RBV and Waveform_RBV from simscope IOC
|
|
26
|
+
python -m apstrim -nEPICS --compress testAPD:scope1:MeanValue_RBV,Waveform_RBV
|
|
27
|
+
|
|
28
|
+
# Serialization of 'cycle' and 'y'-array from a liteServer, running at liteHost
|
|
29
|
+
python -m apstrim -nLITE --compress liteHost:dev1:cycle,y
|
|
30
|
+
|
|
31
|
+
# De-serialization and plotting of the logged data files
|
|
32
|
+
python -m apstrim.plot *.aps
|
|
33
|
+
|
|
34
|
+
Example of Python usage for EPICS infrastructure:
|
|
35
|
+
|
|
36
|
+
import apstrim
|
|
37
|
+
from apstrim.pubEPICS import Access as publisher
|
|
38
|
+
pvNames = ['testAPD:scope1:MeanValue_RBV','testAPD:scope1:Waveform_RBV']
|
|
39
|
+
aps = apstrim.apstrim(publisher, pvNames)
|
|
40
|
+
aps.start('myLogbook.aps')
|
|
41
|
+
...
|
|
42
|
+
aps.stop()
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
from .apstrim import apstrim
|
|
46
|
+
|
|
47
|
+
__version__ = '2.0.5'
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
Description = '''Serializer of Process Variables (from EPICS infrastructure)
|
|
2
|
+
or Data Objects from other infrastructures, e.g. ADO or LITE).'''
|
|
3
|
+
__version__ = '3.0.0 2023-09-24'# pubLITE removed, use_single_float removed
|
|
4
|
+
|
|
5
|
+
import sys, argparse
|
|
6
|
+
from .apstrim import apstrim, __version__
|
|
7
|
+
|
|
8
|
+
def main():
|
|
9
|
+
# parse common arguments
|
|
10
|
+
parser = argparse.ArgumentParser(description=Description
|
|
11
|
+
,formatter_class=argparse.ArgumentDefaultsHelpFormatter
|
|
12
|
+
,epilog=f'apstrim: {__version__}')
|
|
13
|
+
parser.add_argument('-c', '--compress', action='store_true', help=\
|
|
14
|
+
'Enable online compression')
|
|
15
|
+
#parser.add_argument('-D', '--dirSize', type=int, default=10240, help=\
|
|
16
|
+
#'Size of a directory section, set it to 0 to disable random access retrieval')
|
|
17
|
+
#parser.add_argument('-d', '--doublePrecision', action='store_true', help=\
|
|
18
|
+
#'Disable conversion of float64 to float32')
|
|
19
|
+
#parser.add_argument('-f', '--file', default=None, help=\
|
|
20
|
+
#'Configuration file')
|
|
21
|
+
parser.add_argument('-o', '--outfile', default='apstrim.aps', help=\
|
|
22
|
+
'Logbook file for storing PVs and data objects')
|
|
23
|
+
parser.add_argument('-n', '--namespace', default='EPICS', help=\
|
|
24
|
+
'Infrastructure namespace, e.g.: EPICS, ADO or LITE')
|
|
25
|
+
parser.add_argument('-t', '--sectionTime', type=float, default=60., help=\
|
|
26
|
+
'Time interval of writing of sections to logbook')
|
|
27
|
+
parser.add_argument('-T', '--acqTime', type=float, default=99e6, help=\
|
|
28
|
+
'How long (seconds) to take data.')
|
|
29
|
+
parser.add_argument('-q', '--quiet', action='store_true', help=\
|
|
30
|
+
'Quiet: dont print section progress')
|
|
31
|
+
parser.add_argument('-v', '--verbose', action='store_true', help=\
|
|
32
|
+
'Show more log messages')
|
|
33
|
+
parser.add_argument('pvNames', nargs='*', help=\
|
|
34
|
+
'Data Object names, one item per device parameters are comma-separated: dev1:par1,par2 dev2:par1,par2')
|
|
35
|
+
pargs = parser.parse_args()
|
|
36
|
+
#print(f'pargs:{pargs}')
|
|
37
|
+
|
|
38
|
+
s = pargs.namespace.upper()
|
|
39
|
+
if s == 'LITE':
|
|
40
|
+
from liteaccess import Access as publisher
|
|
41
|
+
elif s == 'EPICS':
|
|
42
|
+
from .pubEPICS import Access
|
|
43
|
+
publisher = Access()
|
|
44
|
+
else:
|
|
45
|
+
print(f'ERROR: Unsupported namespace {s}')
|
|
46
|
+
sys.exit(1)
|
|
47
|
+
|
|
48
|
+
pvNames = []
|
|
49
|
+
for pvn in pargs.pvNames:
|
|
50
|
+
tokens = pvn.split(',')
|
|
51
|
+
first = tokens[0]
|
|
52
|
+
pvNames.append(first)
|
|
53
|
+
if len(tokens) > 1:
|
|
54
|
+
dev = first.rsplit(':',1)[0]
|
|
55
|
+
for par in tokens[1:]:
|
|
56
|
+
pvNames.append(dev+':'+par)
|
|
57
|
+
#print(f'pvNames: {pvNames}')
|
|
58
|
+
|
|
59
|
+
apstrim.Verbosity = pargs.verbose
|
|
60
|
+
aps = apstrim(publisher, pvNames, pargs.sectionTime, compress=pargs.compress
|
|
61
|
+
#, quiet=pargs.quiet, use_single_float=not pargs.doublePrecision)
|
|
62
|
+
, quiet=pargs.quiet)
|
|
63
|
+
aps.start(pargs.outfile, howLong=pargs.acqTime)
|
|
64
|
+
|
|
65
|
+
txt = f'for {round(pargs.acqTime/60., 2)} m' if pargs.acqTime<99e6 else 'endlessly'
|
|
66
|
+
print(f'Streaming started {txt}, press Ctrl/C to stop.')
|
|
67
|
+
|
|
68
|
+
if __name__ == '__main__':
|
|
69
|
+
main()
|
|
@@ -0,0 +1,425 @@
|
|
|
1
|
+
# Copyright (c) 2021 Andrei Sukhanov. All rights reserved.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the MIT License, (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://github.com/ASukhanov/apstrim/blob/main/LICENSE
|
|
8
|
+
#
|
|
9
|
+
__version__ = '3.1.0 2024-01-31'
|
|
10
|
+
#TODO: Check if all PVs are alive before logging started
|
|
11
|
+
|
|
12
|
+
import sys, time, string, copy
|
|
13
|
+
import os, pathlib, datetime
|
|
14
|
+
import threading
|
|
15
|
+
import signal
|
|
16
|
+
from timeit import default_timer as timer
|
|
17
|
+
|
|
18
|
+
import numpy as np
|
|
19
|
+
|
|
20
|
+
#import msgpack as encoder
|
|
21
|
+
#if encoder.version < (1, 0, 2):
|
|
22
|
+
# print(f'MessagePack too old: {msgpack.version}')
|
|
23
|
+
# sys.exit()
|
|
24
|
+
#def encoderDump(buf):
|
|
25
|
+
# return encoder.packb(buf, use_single_float=True)
|
|
26
|
+
|
|
27
|
+
# Try CBOR encoding
|
|
28
|
+
import cbor2 as encoder
|
|
29
|
+
encoderDump = encoder.dumps
|
|
30
|
+
|
|
31
|
+
#````````````````````````````Globals``````````````````````````````````````````
|
|
32
|
+
Nano = 0.000000001
|
|
33
|
+
MinTimestamp = 1600000000 # Minimal possible timestamp
|
|
34
|
+
MAXU8 = 256 - 1
|
|
35
|
+
MAXI8 = int(256/2) - 1
|
|
36
|
+
MAXU16 = int(256**2) - 1
|
|
37
|
+
MAXI16 = int(256**2/2) - 1
|
|
38
|
+
MAXU32 = int(256**4) - 1
|
|
39
|
+
MAXI32 = int(256**4/2) - 1
|
|
40
|
+
SPTime = 0
|
|
41
|
+
SPVal = 1
|
|
42
|
+
#````````````````````````````Helper functions`````````````````````````````````
|
|
43
|
+
def _printTime(): return time.strftime("%m%d:%H%M%S")
|
|
44
|
+
def _printi(msg): print(f'INFO_AS@{_printTime()}: {msg}')
|
|
45
|
+
def _printw(msg): print(f'WARN_AS@{_printTime()}: {msg}')
|
|
46
|
+
def _printe(msg): print(f'ERROR_AS@{_printTime()}: {msg}')
|
|
47
|
+
def _printd(msg):
|
|
48
|
+
if apstrim.Verbosity > 0:
|
|
49
|
+
print(f'DBG_AS: {msg}')
|
|
50
|
+
|
|
51
|
+
def _croppedText(txt, limit=200):
|
|
52
|
+
if len(txt) > limit:
|
|
53
|
+
txt = txt[:limit]+'...'
|
|
54
|
+
return txt
|
|
55
|
+
|
|
56
|
+
def _packnp(data, use_single_float=True):
|
|
57
|
+
""" Pack data for fast extraction. If data can be converted to numpy
|
|
58
|
+
arrays, then it will be returned as
|
|
59
|
+
{'dtype':dtype, 'shape':shape, 'value':nparray.tobytes()},
|
|
60
|
+
if not, then they will be returned as is.
|
|
61
|
+
In msgpack the unpacking of bytes is ~100 times faster than unpacking of
|
|
62
|
+
integers"""
|
|
63
|
+
# 50% time is spent here
|
|
64
|
+
try:
|
|
65
|
+
l1 = len(data)
|
|
66
|
+
except:
|
|
67
|
+
#print('Single element, no need to pack')
|
|
68
|
+
return data
|
|
69
|
+
if l1 == 0:
|
|
70
|
+
#print('Empty list, no need to pack')
|
|
71
|
+
return None
|
|
72
|
+
atype = type(data[0])
|
|
73
|
+
#print( _croppedText(f'packing {l1} of {atype}: {data}'))
|
|
74
|
+
try:
|
|
75
|
+
npdata = np.array(data)
|
|
76
|
+
except Exception as e:
|
|
77
|
+
_printe(f'In _packnp: {e}')
|
|
78
|
+
sys.exit()
|
|
79
|
+
return data
|
|
80
|
+
#print(f'npdata shape: {npdata.shape} of {npdata.dtype}')
|
|
81
|
+
if npdata.shape == ():
|
|
82
|
+
return data
|
|
83
|
+
if npdata.dtype == 'int64':
|
|
84
|
+
mn = npdata.min()
|
|
85
|
+
mx = npdata.max()
|
|
86
|
+
if -MAXI32 < mn and mx < MAXI32:
|
|
87
|
+
if -MAXI16 < mn and mx < MAXI16:
|
|
88
|
+
if -MAXI8 < mn and mx < MAXI8:
|
|
89
|
+
npdata = npdata.astype('int8')
|
|
90
|
+
else:
|
|
91
|
+
npdata = npdata.astype('int16')
|
|
92
|
+
else:
|
|
93
|
+
npdata = npdata.astype('int32')
|
|
94
|
+
else:
|
|
95
|
+
if npdata.dtype == 'float64' and use_single_float:
|
|
96
|
+
npdata = npdata.astype('float32')
|
|
97
|
+
return {'dtype':str(npdata.dtype), 'shape':npdata.shape
|
|
98
|
+
,'bytes':npdata.tobytes()}
|
|
99
|
+
|
|
100
|
+
#````````````````````````````Serializer class`````````````````````````````````
|
|
101
|
+
class apstrim():
|
|
102
|
+
"""
|
|
103
|
+
Create an object streamer.
|
|
104
|
+
|
|
105
|
+
**publisher**: Is a class, providing a subscribe() method.
|
|
106
|
+
|
|
107
|
+
**devPar**: List of device:parameter strings.
|
|
108
|
+
|
|
109
|
+
**sectionInterval**: Data collection period. Data are collected
|
|
110
|
+
continously into a section, which is periodically dumped into the
|
|
111
|
+
logbook file with this time interval (in seconds).
|
|
112
|
+
|
|
113
|
+
**compression**: Enable Compression flag.
|
|
114
|
+
|
|
115
|
+
**quiet**: Do not print the section writing progress,
|
|
116
|
+
|
|
117
|
+
**use_single_float**: Use single precision float type for float. (default: True)
|
|
118
|
+
|
|
119
|
+
**dirSize**: Size of a Table Of Contents, which is written as a first
|
|
120
|
+
object in the logbook file. It contains file positions of sections and
|
|
121
|
+
used for random-access retrieval.
|
|
122
|
+
Default is 10KB, which is good for ~700 entries. If number of sections
|
|
123
|
+
becomes too big, then the table is downsampled twice to fit into this size.
|
|
124
|
+
"""
|
|
125
|
+
EventExit = threading.Event()
|
|
126
|
+
"""Calling the EventExit.set() will safely exit the application."""
|
|
127
|
+
Verbosity = 0
|
|
128
|
+
"""Show dedugging messages."""
|
|
129
|
+
_eventStop = threading.Event()
|
|
130
|
+
|
|
131
|
+
def __init__(self, publisher, devPars:list, sectionInterval=60.
|
|
132
|
+
# , compress=False, quiet=False, use_single_float=True, dirSize=10240):
|
|
133
|
+
, compress=False, quiet=False, dirSize=10240):
|
|
134
|
+
_printi(f'apstrim {__version__}, sectionInterval {sectionInterval}')
|
|
135
|
+
print(f'v: {self.Verbosity}')
|
|
136
|
+
signal.signal(signal.SIGINT, _safeExit)
|
|
137
|
+
signal.signal(signal.SIGTERM, _safeExit)
|
|
138
|
+
|
|
139
|
+
self.lock = threading.Lock()
|
|
140
|
+
self.publisher = publisher
|
|
141
|
+
self.devPars = devPars
|
|
142
|
+
self.sectionInterval = sectionInterval
|
|
143
|
+
self.quiet = quiet
|
|
144
|
+
#self.use_single_float = use_single_float
|
|
145
|
+
|
|
146
|
+
# table of contents - related variables
|
|
147
|
+
self.dirSize = dirSize
|
|
148
|
+
self.contents_downsampling_factor = 1# No downsampling
|
|
149
|
+
|
|
150
|
+
# create a section Abstract
|
|
151
|
+
self.abstractSection = {'abstract':{'apstrim':__version__,
|
|
152
|
+
'encoder':encoder.__name__, 'sectionInterval':sectionInterval}}
|
|
153
|
+
abstract = self.abstractSection['abstract']
|
|
154
|
+
|
|
155
|
+
if compress:
|
|
156
|
+
import lz4framed
|
|
157
|
+
self.compress = lz4framed.compress
|
|
158
|
+
abstract['compression'] = 'lz4framed'
|
|
159
|
+
else:
|
|
160
|
+
self.compress = None
|
|
161
|
+
abstract['compression'] = 'None'
|
|
162
|
+
_printi(f'Abstract section: {self.abstractSection}')
|
|
163
|
+
#self.par2Index = {p:i for i,p in enumerate(self.devPars)}
|
|
164
|
+
self.par2Index = [p for p in self.devPars]
|
|
165
|
+
if len(self.par2Index) == 0:
|
|
166
|
+
_printe(f'Could not build the list of parameters')
|
|
167
|
+
sys.exit()
|
|
168
|
+
_printi(f'parameters: {self.par2Index}')
|
|
169
|
+
|
|
170
|
+
# a section has to be created before subscription
|
|
171
|
+
self._create_logSection()
|
|
172
|
+
|
|
173
|
+
# subscribe to parameters
|
|
174
|
+
#for pname in self.par2Index.keys():
|
|
175
|
+
for pname in self.par2Index:
|
|
176
|
+
devPar = tuple(pname.rsplit(':',1))
|
|
177
|
+
_printd(f'Subscribing: {devPar}')
|
|
178
|
+
try:
|
|
179
|
+
self.publisher.subscribe(self._delivered, devPar)
|
|
180
|
+
except:# Exception as e:
|
|
181
|
+
_printe(f'Could not subscribe for {pname}')#: {e}')
|
|
182
|
+
continue
|
|
183
|
+
|
|
184
|
+
self.indexSection = encoderDump({'index':self.par2Index})
|
|
185
|
+
|
|
186
|
+
def start(self, fileName='apstrim.aps', howLong=99e6):
|
|
187
|
+
"""Start the streaming of the data objects to the logbook file.
|
|
188
|
+
If file is already exists then it will be renamed and
|
|
189
|
+
a new file will be open with the provided name.
|
|
190
|
+
|
|
191
|
+
**howLong**: Time interval (seconds) for data collection.
|
|
192
|
+
"""
|
|
193
|
+
self._eventStop.clear()
|
|
194
|
+
self.howLong = howLong
|
|
195
|
+
try:
|
|
196
|
+
modificationTime = pathlib.Path(fileName).stat().st_mtime
|
|
197
|
+
dt = datetime.datetime.fromtimestamp(modificationTime)
|
|
198
|
+
suffix = dt.strftime('_%Y%m%d_%H%M')
|
|
199
|
+
try: fname,ext = fileName.rsplit('.',1)
|
|
200
|
+
except: fname,ext = fileName,''
|
|
201
|
+
otherName = fname + suffix + '.' + ext
|
|
202
|
+
os.rename(fileName, otherName)
|
|
203
|
+
_printw(f'Existing file {fileName} have been renamed to {otherName}')
|
|
204
|
+
except Exception as e:
|
|
205
|
+
pass
|
|
206
|
+
|
|
207
|
+
self.logbook = open(fileName, 'wb')
|
|
208
|
+
|
|
209
|
+
# write a preliminary 'Table of contents' section
|
|
210
|
+
self.contentsSection = {'contents':{'size':self.dirSize}, 'data':{}}
|
|
211
|
+
self.dataContents = self.contentsSection['data']
|
|
212
|
+
self.logbook.write(encoderDump(self.contentsSection))
|
|
213
|
+
# skip the 'Table of contents' zone of the logbook
|
|
214
|
+
self.logbook.seek(self.dirSize)
|
|
215
|
+
|
|
216
|
+
# write the sections Abstract and Index
|
|
217
|
+
_printd(f'write Abstract@{self.logbook.tell()}')
|
|
218
|
+
self.logbook.write(encoderDump(self.abstractSection))
|
|
219
|
+
_printd(f'write Index@{self.logbook.tell()}')
|
|
220
|
+
self.logbook.write(self.indexSection)
|
|
221
|
+
savedPos = self.logbook.tell()
|
|
222
|
+
|
|
223
|
+
self._create_logSection()
|
|
224
|
+
|
|
225
|
+
#_printi('starting serialization thread')
|
|
226
|
+
myThread = threading.Thread(target=self._serialize_sections)
|
|
227
|
+
myThread.start()
|
|
228
|
+
|
|
229
|
+
_printi(f'Logbook file: {fileName} created')
|
|
230
|
+
|
|
231
|
+
def stop(self):
|
|
232
|
+
"""Stop the streaming."""
|
|
233
|
+
_printd('>stop()')
|
|
234
|
+
self._eventStop.set()
|
|
235
|
+
#self.logbook.close()
|
|
236
|
+
|
|
237
|
+
def _delivered(self, *args):
|
|
238
|
+
"""Callback, specified in the subscribe() request.
|
|
239
|
+
Called when the requested data have been changed.
|
|
240
|
+
args is a map of delivered objects."""
|
|
241
|
+
#_printd(f'delivered: {args}')
|
|
242
|
+
#self.timestampedMap = {}
|
|
243
|
+
with self.lock:
|
|
244
|
+
for devPar,props in args[0].items():
|
|
245
|
+
#print( _croppedText(f'devPar: {devPar,props}'))
|
|
246
|
+
try:
|
|
247
|
+
if isinstance(devPar, tuple):
|
|
248
|
+
# EPICS and ADO packing
|
|
249
|
+
dev,par = devPar
|
|
250
|
+
value = props['value']
|
|
251
|
+
timestamp = props.get('timestamp')# valid in EPICS and LITE
|
|
252
|
+
if timestamp == None:# decode ADO timestamp
|
|
253
|
+
timestamp = int(props['timestampSeconds']/Nano
|
|
254
|
+
+ props['timestampNanoSeconds'])
|
|
255
|
+
else:
|
|
256
|
+
timestamp = int(timestamp/Nano)
|
|
257
|
+
if timestamp < MinTimestamp:
|
|
258
|
+
# Timestamp is wrong, discard the parameter
|
|
259
|
+
_printd(f'timestamp is wrong {timestamp, MinTimestamp}')
|
|
260
|
+
continue
|
|
261
|
+
skey = self.par2Index.index(dev+':'+par)
|
|
262
|
+
self.timestamp = int(timestamp*Nano)
|
|
263
|
+
self.sectionPars[skey][SPTime].append(timestamp)
|
|
264
|
+
self.sectionPars[skey][SPVal].append(value)
|
|
265
|
+
elif devPar == 'ppmuser':# ADO has extra item, skip it.
|
|
266
|
+
continue
|
|
267
|
+
else:
|
|
268
|
+
#LITE packing:
|
|
269
|
+
dev = devPar
|
|
270
|
+
pars = props
|
|
271
|
+
##print( _croppedText(f'pars: {pars}'))
|
|
272
|
+
for par in pars:
|
|
273
|
+
try:
|
|
274
|
+
value = pars[par]['v']
|
|
275
|
+
timestamp = int(pars[par]['t'])
|
|
276
|
+
except: # try an old LITE packing
|
|
277
|
+
value = pars[par]['value']
|
|
278
|
+
timestamp = int(pars[par]['timestamp']/Nano)
|
|
279
|
+
if timestamp < MinTimestamp:
|
|
280
|
+
# Timestamp is wrong, discard the parameter
|
|
281
|
+
continue
|
|
282
|
+
skey = self.par2Index.index(dev+':'+par)
|
|
283
|
+
# add to parameter list
|
|
284
|
+
self.timestamp = int(timestamp*Nano)
|
|
285
|
+
self.sectionPars[skey][SPTime].append(timestamp)
|
|
286
|
+
#print( _croppedText(f'value: {value}'))
|
|
287
|
+
self.sectionPars[skey][SPVal].append(value)
|
|
288
|
+
#print(f'devPar {devPar}@{timestamp,skey}:{timestamp,value}')
|
|
289
|
+
except Exception as e:
|
|
290
|
+
_printw(f'exception in unpacking: {e}')
|
|
291
|
+
continue
|
|
292
|
+
#try: ts = self.timestamp
|
|
293
|
+
#except: ts = '?'
|
|
294
|
+
#print(f'served timestamp: {ts}')
|
|
295
|
+
#print( _croppedText(f'section: {self.section}'))
|
|
296
|
+
|
|
297
|
+
def _create_logSection(self):
|
|
298
|
+
with self.lock:
|
|
299
|
+
#print('create empty section')
|
|
300
|
+
try:
|
|
301
|
+
tstart = self.timestamp
|
|
302
|
+
except:
|
|
303
|
+
#tstart = int(time.time()/Nano)
|
|
304
|
+
tstart = int(time.time())
|
|
305
|
+
#self.sectionPars = {i:([],[]) for i in self.par2Index.values()}
|
|
306
|
+
self.sectionPars = {i:([],[]) for i in range(len(self.par2Index))}
|
|
307
|
+
self.section = {'tstart':tstart, 'tend':None
|
|
308
|
+
,'pars':self.sectionPars}
|
|
309
|
+
|
|
310
|
+
def _serialize_sections(self):
|
|
311
|
+
periodic_update = time.time()
|
|
312
|
+
statistics = [0, 0, 0, 0, 0.]#
|
|
313
|
+
NSections, NParLists, BytesRaw, BytesFinal, LogTime = 0,1,2,3,4
|
|
314
|
+
maxSections = self.howLong//self.sectionInterval
|
|
315
|
+
_printd(f'serialize_sections started.')
|
|
316
|
+
try:
|
|
317
|
+
while statistics[NSections] <= maxSections\
|
|
318
|
+
and not self._eventStop.is_set():
|
|
319
|
+
self._eventStop.wait(self.sectionInterval)
|
|
320
|
+
logTime = timer()
|
|
321
|
+
|
|
322
|
+
# register the section in the table of contents,
|
|
323
|
+
# this should be skipped when the contents downsampling is active.
|
|
324
|
+
rf = self.contents_downsampling_factor
|
|
325
|
+
if rf <=1 or (statistics[NSections]%rf) == 0:
|
|
326
|
+
self.dataContents[self.section['tstart']]\
|
|
327
|
+
= self.logbook.tell()
|
|
328
|
+
packed = encoderDump(self.contentsSection)
|
|
329
|
+
if len(packed) < self.dirSize:
|
|
330
|
+
self.packedContents = packed
|
|
331
|
+
else:
|
|
332
|
+
_printw((f'The contents size is too small for'
|
|
333
|
+
f' {len(packed)} bytes. Half of the entries will be'
|
|
334
|
+
' removed to allow for more entries.}'))
|
|
335
|
+
self.contents_downsampling_factor *= 2
|
|
336
|
+
downsampled_contents = dict(list(self.dataContents.items())\
|
|
337
|
+
[::self.contents_downsampling_factor])
|
|
338
|
+
self.contentsSection['data'] = downsampled_contents
|
|
339
|
+
_printd(f'downsampled contentsSection:{self.contentsSection}')
|
|
340
|
+
self.packedContents = encoderDump(self.contentsSection)
|
|
341
|
+
|
|
342
|
+
# Update Directory section on the file.
|
|
343
|
+
currentPos = self.logbook.tell()
|
|
344
|
+
self.logbook.seek(0)
|
|
345
|
+
self.logbook.write(self.packedContents)
|
|
346
|
+
self.logbook.seek(currentPos)
|
|
347
|
+
|
|
348
|
+
statistics[NSections] += 1
|
|
349
|
+
_printd(f'section{statistics[NSections]} {self.section["tstart"]} is ready for writing to logbook @ {self.logbook.tell()}')
|
|
350
|
+
try:
|
|
351
|
+
self.section['tend'] = self.timestamp
|
|
352
|
+
except:
|
|
353
|
+
_printw(f'No data recorded in section {statistics[NSections]}')
|
|
354
|
+
#sys.exit()
|
|
355
|
+
continue
|
|
356
|
+
|
|
357
|
+
# pack to numpy/bytes, they are very fast to unpack
|
|
358
|
+
npPacked = {}
|
|
359
|
+
with self.lock:
|
|
360
|
+
for key,val in self.sectionPars.items():
|
|
361
|
+
statistics[NParLists] += 1
|
|
362
|
+
#print( _croppedText(f'sectItem:{key,val}'))
|
|
363
|
+
sptimes = _packnp(val[SPTime])
|
|
364
|
+
if sptimes is None:
|
|
365
|
+
continue
|
|
366
|
+
spvals = _packnp(val[SPVal])
|
|
367
|
+
npPacked[key] = (sptimes, spvals)
|
|
368
|
+
if apstrim.Verbosity >= 1:
|
|
369
|
+
print( _croppedText(f"npPacked: {self.section['tstart'], npPacked.keys()}"))
|
|
370
|
+
for i,kValues in enumerate(npPacked.items()):
|
|
371
|
+
print( _croppedText(f'Index{i}: {kValues[0]}'))
|
|
372
|
+
for ii,value in enumerate(kValues[1]):
|
|
373
|
+
print( _croppedText(f'kValue{ii}[{len(value["bytes"])}]: {value}'))
|
|
374
|
+
|
|
375
|
+
# msgpack
|
|
376
|
+
toPack = {'tstart':self.section['tstart']
|
|
377
|
+
,'tstart':self.section['tstart'],'pars':npPacked}
|
|
378
|
+
packed = encoderDump(toPack)
|
|
379
|
+
statistics[BytesRaw] += len(packed)
|
|
380
|
+
|
|
381
|
+
# compress, takes almost no time.
|
|
382
|
+
if self.compress is not None:
|
|
383
|
+
compressed = self.compress(packed)
|
|
384
|
+
packed = encoderDump(compressed)
|
|
385
|
+
statistics[BytesFinal] += len(packed)
|
|
386
|
+
|
|
387
|
+
# write to file
|
|
388
|
+
self.logbook.write(packed)
|
|
389
|
+
self.logbook.flush()
|
|
390
|
+
|
|
391
|
+
# update statistics
|
|
392
|
+
self._create_logSection()
|
|
393
|
+
timestamp = time.time()
|
|
394
|
+
dt = timestamp - periodic_update
|
|
395
|
+
statistics[LogTime] += timer() - logTime
|
|
396
|
+
if dt > 10.:
|
|
397
|
+
periodic_update = timestamp
|
|
398
|
+
if not self.quiet:
|
|
399
|
+
dt = datetime.datetime.fromtimestamp(self.timestamp)
|
|
400
|
+
print(f'{dt.strftime("%y-%m-%d %H:%M:%S")} Logged'
|
|
401
|
+
f' {statistics[NSections]} sections,'
|
|
402
|
+
f' {statistics[NParLists]} parLists,'
|
|
403
|
+
f' {statistics[BytesFinal]/1000.} KBytes,'
|
|
404
|
+
f' {round(statistics[BytesRaw]/statistics[LogTime]/1e6,1)} MB/s')
|
|
405
|
+
except Exception as e:
|
|
406
|
+
print(f'ERROR: Exception in serialize_sections: {e}')
|
|
407
|
+
|
|
408
|
+
# logging is finished
|
|
409
|
+
# rewrite the contentsSection
|
|
410
|
+
self.logbook.seek(0)
|
|
411
|
+
self.logbook.write(self.packedContents)
|
|
412
|
+
|
|
413
|
+
# print status
|
|
414
|
+
msg = (f'Logging finished for {statistics[NSections]} sections,'
|
|
415
|
+
f' {statistics[NParLists]} parLists,'
|
|
416
|
+
f' {statistics[BytesFinal]/1000.} KB.')
|
|
417
|
+
if self.compress is not None:
|
|
418
|
+
msg += f' Compression ratio:{round(statistics[BytesRaw]/statistics[BytesFinal],2)}'
|
|
419
|
+
print(msg)
|
|
420
|
+
self.logbook.close()
|
|
421
|
+
|
|
422
|
+
def _safeExit(_signo, _stack_frame):
|
|
423
|
+
print('safeExit')
|
|
424
|
+
apstrim._eventStop.set()
|
|
425
|
+
apstrim.EventExit.set()
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""This script provide conversion of LDO name to (host:port,device).
|
|
3
|
+
Usage:
|
|
4
|
+
from liteserver import liteCNS
|
|
5
|
+
hostname = liteCNS.hostPort('pi108')
|
|
6
|
+
"""
|
|
7
|
+
# If CNSHostPort is defined, then the name will be resolved using a dedicated
|
|
8
|
+
__version__='v02 2023-03-17'# use python-based configuration
|
|
9
|
+
|
|
10
|
+
import sys
|
|
11
|
+
|
|
12
|
+
CNSHostPort = None#'hostName;9699'# host;port of the liteServer
|
|
13
|
+
|
|
14
|
+
#`````````````````````````````````````````````````````````````````````````````
|
|
15
|
+
def hostPort(cnsName='*', server=CNSHostPort):
|
|
16
|
+
if server is not None:
|
|
17
|
+
# CNSHostPort is defined, use liteCNSServer
|
|
18
|
+
raise NameError('ERROR:CNS, liteCNSServer is not supported yet')
|
|
19
|
+
|
|
20
|
+
#``````````read conguration file into the config dictionary```````````````
|
|
21
|
+
configDir = '/operations/app_store/liteServer'
|
|
22
|
+
moduleName = 'liteCNS_resolv'
|
|
23
|
+
sys.path.append(configDir)
|
|
24
|
+
from importlib import import_module#, reload
|
|
25
|
+
print(f'importing {moduleName}')
|
|
26
|
+
ConfigModule = import_module(moduleName)
|
|
27
|
+
print(f'Imported: {configDir}/{moduleName}')
|
|
28
|
+
if cnsName == '*':
|
|
29
|
+
r = ConfigModule.hosts
|
|
30
|
+
else:
|
|
31
|
+
r = ConfigModule.hosts[cnsName]
|
|
32
|
+
return r
|